From d2b2e6fb339da3a4eacb45ad8aa4d604989516bd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 22 Jul 2025 21:29:01 +0200 Subject: [PATCH 001/186] Remove workbench column from projects_table --- ..._remove_workbench_column_from_projects_.py | 37 +++++++++++++++++++ .../models/projects.py | 13 ++++--- 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py new file mode 100644 index 00000000000..20cc55b4445 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -0,0 +1,37 @@ +"""Remove workbench column from projects_table + +Revision ID: 201aa37f4d9a +Revises: 5679165336c8 +Create Date: 2025-07-22 19:25:42.125196+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "201aa37f4d9a" +down_revision = "5679165336c8" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("projects", "workbench") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "projects", + sa.Column( + "workbench", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + ) + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects.py b/packages/postgres-database/src/simcore_postgres_database/models/projects.py index 7af3c09fc65..0006055d83e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects.py @@ -96,12 +96,13 @@ class ProjectTemplateType(str, enum.Enum): default=None, ), # CHILDREN/CONTENT-------------------------- - sa.Column( - "workbench", - sa.JSON, - nullable=False, - doc="Pipeline with the project's workflow. Schema in models_library.projects.Workbench", - ), + # NOTE: commented out to check who still uses this + # sa.Column( + # "workbench", + # sa.JSON, + # nullable=False, + # doc="Pipeline with the project's workflow. Schema in models_library.projects.Workbench", + # ), # FRONT-END ---------------------------- sa.Column( "ui", From f6ae1d71a4b56605e76001cd675b3ba972c20628 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 25 Jul 2025 14:31:56 +0200 Subject: [PATCH 002/186] fix: remove workbench from faker factory --- .../pytest-simcore/src/pytest_simcore/helpers/faker_factories.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index 4b09b0ef06b..e85ef315d49 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -173,7 +173,6 @@ def random_project(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]: "prj_owner": fake.pyint(), "thumbnail": fake.image_url(width=120, height=120), "access_rights": {}, - "workbench": {}, "published": False, } From 20cef279fcfe1c9dc440de003fc416297aabb475 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 25 Jul 2025 21:28:00 +0200 Subject: [PATCH 003/186] fix: remove workbench references from catalog --- .../pytest_simcore/helpers/faker_factories.py | 31 ++++- .../repository/projects.py | 64 +++++++---- .../tests/unit/with_dbs/test_repositories.py | 106 +++++++++++++----- 3 files changed, 148 insertions(+), 53 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index e85ef315d49..c6d2c2a9da3 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -25,6 +25,15 @@ DEFAULT_FAKER: Final = Faker() +def random_service_key(fake: Faker = DEFAULT_FAKER, *, name: str | None = None) -> str: + """Generates a random service key""" + return f"simcore/services/{fake.random_element(['dynamic', 'computational'])}/{name or fake.name()}" + + +def random_service_version(fake: Faker = DEFAULT_FAKER) -> str: + return ".".join([str(fake.pyint()) for _ in range(3)]) + + def random_icon_url(fake: Faker): return fake.image_url(width=16, height=16) @@ -186,6 +195,26 @@ def random_project(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]: return data +def random_project_node(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]: + """Generates random fake data project nodes DATABASE table""" + from simcore_postgres_database.models.projects_nodes import projects_nodes + + _name = fake.name() + + data = { + "node_id": fake.uuid4(), + "project_uuid": fake.uuid4(), + "key": random_service_key(fake, name=_name), + "version": random_service_version(fake), + "label": _name, + } + + assert set(data.keys()).issubset({c.name for c in projects_nodes.columns}) + + data.update(overrides) + return data + + def random_group(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]: from simcore_postgres_database.models.groups import groups from simcore_postgres_database.webserver_models import GroupType @@ -481,7 +510,7 @@ def random_service_meta_data( ) -> dict[str, Any]: from simcore_postgres_database.models.services import services_meta_data - _version = ".".join([str(fake.pyint()) for _ in range(3)]) + _version = random_service_version(fake) _name = fake.name() data: dict[str, Any] = { diff --git a/services/catalog/src/simcore_service_catalog/repository/projects.py b/services/catalog/src/simcore_service_catalog/repository/projects.py index 48de3867a6c..4604aebc7ed 100644 --- a/services/catalog/src/simcore_service_catalog/repository/projects.py +++ b/services/catalog/src/simcore_service_catalog/repository/projects.py @@ -4,37 +4,53 @@ from models_library.services import ServiceKeyVersion from pydantic import ValidationError from simcore_postgres_database.models.projects import ProjectType, projects +from simcore_postgres_database.models.projects_nodes import projects_nodes from ._base import BaseRepository _logger = logging.getLogger(__name__) +_IGNORED_SERVICE_KEYS: set[str] = { + # NOTE: frontend only nodes + "simcore/services/frontend/file-picker", + "simcore/services/frontend/nodes-group", +} + + class ProjectsRepository(BaseRepository): async def list_services_from_published_templates(self) -> list[ServiceKeyVersion]: - list_of_published_services: list[ServiceKeyVersion] = [] async with self.db_engine.connect() as conn: - async for row in await conn.stream( - sa.select(projects).where( - (projects.c.type == ProjectType.TEMPLATE) - & (projects.c.published.is_(True)) + query = ( + sa.select(projects_nodes.c.key, projects_nodes.c.version) + .distinct() + .select_from( + projects_nodes.join( + projects, projects_nodes.c.project_uuid == projects.c.uuid + ) + ) + .where( + sa.and_( + projects.c.type == ProjectType.TEMPLATE, + projects.c.published.is_(True), + projects_nodes.c.key.notin_(_IGNORED_SERVICE_KEYS), + ) ) - ): - project_workbench = row.workbench - for node in project_workbench: - service = project_workbench[node] - try: - if ( - "file-picker" in service["key"] - or "nodes-group" in service["key"] - ): - # these 2 are not going to pass the validation tests, they are frontend only nodes. - continue - list_of_published_services.append(ServiceKeyVersion(**service)) - except ValidationError: - _logger.warning( - "service %s could not be validated", service, exc_info=True - ) - continue - - return list_of_published_services + ) + + services = [] + async for row in await conn.stream(query): + try: + service = ServiceKeyVersion.model_validate( + row, from_attributes=True + ) + services.append(service) + except ValidationError: + _logger.warning( + "service with key=%s and version=%s could not be validated", + row.key, + row.version, + exc_info=True, + ) + + return services diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index ec8fca12825..5e68dad4138 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -23,9 +23,10 @@ from packaging import version from pydantic import EmailStr, HttpUrl, TypeAdapter from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable -from pytest_simcore.helpers.faker_factories import random_project +from pytest_simcore.helpers.faker_factories import random_project, random_project_node from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan from simcore_postgres_database.models.projects import ProjectType, projects +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_service_catalog.models.services_db import ( ServiceAccessRightsDB, ServiceDBFilters, @@ -812,21 +813,41 @@ async def test_list_services_from_published_templates( type=ProjectType.TEMPLATE, published=True, prj_owner=user["id"], - workbench={ - "node-1": { - "key": "simcore/services/dynamic/jupyterlab", - "version": "1.0.0", - }, - "node-2": { - "key": "simcore/services/frontend/file-picker", - "version": "1.0.0", - }, - }, ), pk_col=projects.c.uuid, pk_value="template-1", ) ) + await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=random_project_node( + node_id="node-1.1", + project_uuid="template-1", + key="simcore/services/dynamic/jupyterlab", + version="1.0.0", + label="jupyterlab", + ), + pk_col=projects_nodes.c.node_id, + pk_value="node-1.1", + ) + ) + await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=random_project_node( + node_id="node-1.2", + project_uuid="template-1", + key="simcore/services/frontend/file-picker", + version="1.0.0", + label="file-picker", + ), + pk_col=projects_nodes.c.node_id, + pk_value="node-1.2", + ) + ) await stack.enter_async_context( insert_and_get_row_lifespan( sqlalchemy_async_engine, @@ -836,17 +857,26 @@ async def test_list_services_from_published_templates( type=ProjectType.TEMPLATE, published=False, prj_owner=user["id"], - workbench={ - "node-1": { - "key": "simcore/services/dynamic/some-service", - "version": "2.0.0", - }, - }, ), pk_col=projects.c.uuid, pk_value="template-2", ) ) + await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=random_project_node( + node_id="node-2.1", + project_uuid="template-2", + key="simcore/services/dynamic/some-service", + version="2.0.0", + label="some-service", + ), + pk_col=projects_nodes.c.node_id, + pk_value="node-2.1", + ) + ) # Act: Call the method services = await projects_repo.list_services_from_published_templates() @@ -874,21 +904,41 @@ async def test_list_services_from_published_templates_with_invalid_service( type=ProjectType.TEMPLATE, published=True, prj_owner=user["id"], - workbench={ - "node-1": { - "key": "simcore/services/frontend/file-picker", - "version": "1.0.0", - }, - "node-2": { - "key": "simcore/services/dynamic/invalid-service", - "version": "invalid", - }, - }, ), pk_col=projects.c.uuid, pk_value="template-1", ) ) + await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=random_project_node( + node_id="node-1.1", + project_uuid="template-1", + key="simcore/services/frontend/file-picker", + version="1.0.0", + label="file-picker", + ), + pk_col=projects_nodes.c.node_id, + pk_value="node-1.1", + ) + ) + await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=random_project_node( + node_id="node-1.2", + project_uuid="template-1", + key="simcore/services/dynamic/invalid-service", + version="invalid", # NOTE: invalid version + label="invalid-service", + ), + pk_col=projects_nodes.c.node_id, + pk_value="node-1.2", + ) + ) # Act: Call the method and capture logs with caplog.at_level(logging.WARNING): @@ -897,7 +947,7 @@ async def test_list_services_from_published_templates_with_invalid_service( # Assert: Validate the results assert len(services) == 0 # No valid services should be returned assert ( - "service {'key': 'simcore/services/dynamic/invalid-service', 'version': 'invalid'} could not be validated" + "service with key=simcore/services/dynamic/invalid-service and version=invalid could not be validated" in caplog.text ) From 77d26ecbfdafc165291b58b89e753768fb82bb5c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:17:41 +0200 Subject: [PATCH 004/186] fixes projecst repo --- .../modules/db/repositories/projects.py | 17 +++++++---- .../modules/db/tables.py | 28 ++++++------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 2935b6ec251..3287ef4cbc7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -4,9 +4,10 @@ from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo +from simcore_postgres_database.utils_repos import pass_or_acquire_connection from ....core.errors import ProjectNotFoundError -from ..tables import projects +from ..tables import projects, projects_nodes from ._base import BaseRepository logger = logging.getLogger(__name__) @@ -27,11 +28,15 @@ async def get_project(self, project_id: ProjectID) -> ProjectAtDB: async def is_node_present_in_workbench( self, project_id: ProjectID, node_uuid: NodeID ) -> bool: - try: - project = await self.get_project(project_id) - return f"{node_uuid}" in project.workbench - except ProjectNotFoundError: - return False + async with pass_or_acquire_connection(self.db_engine) as conn: + result = await conn.execute( + sa.select(projects_nodes.c.project_node_id).where( + projects_nodes.c.project_uuid == str(project_id), + projects_nodes.c.node_id == str(node_uuid), + ) + ) + project_node = result.one_or_none() + return project_node is not None async def get_project_id_from_node(self, node_id: NodeID) -> ProjectID: async with self.db_engine.connect() as conn: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py index f47250b651e..890e67ced3a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py @@ -1,26 +1,14 @@ -from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline -from simcore_postgres_database.models.comp_run_snapshot_tasks import ( - comp_run_snapshot_tasks, -) -from simcore_postgres_database.models.comp_runs import comp_runs -from simcore_postgres_database.models.comp_tasks import NodeClass, comp_tasks -from simcore_postgres_database.models.groups import user_to_groups -from simcore_postgres_database.models.groups_extra_properties import ( - groups_extra_properties, -) -from simcore_postgres_database.models.projects import ProjectType, projects -from simcore_postgres_database.models.projects_networks import projects_networks - -__all__ = [ +__all__: tuple[str, ...] = ( + "NodeClass", + "ProjectType", + "StateType", "comp_pipeline", + "comp_run_snapshot_tasks", "comp_runs", "comp_tasks", "groups_extra_properties", - "NodeClass", - "projects_networks", "projects", - "ProjectType", - "StateType", + "projects_networks", + "projects_nodes", "user_to_groups", - "comp_run_snapshot_tasks", -] +) From c209c6bcb484ff52064fda1c9fa227c77397fb26 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:26:55 +0200 Subject: [PATCH 005/186] fixes tests modules --- .../src/pytest_simcore/db_entries_mocks.py | 107 ++++++++++-------- .../modules/db/tables.py | 16 +++ 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 8a13ecae3a4..08c56d0660e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -4,6 +4,7 @@ # pylint:disable=no-value-for-parameter import contextlib +import logging from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any from uuid import uuid4 @@ -14,6 +15,7 @@ from models_library.products import ProductName from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID +from pytest_simcore.helpers.logging_tools import log_context from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.products import products @@ -30,6 +32,8 @@ from .helpers.postgres_tools import insert_and_get_row_lifespan from .helpers.postgres_users import sync_insert_and_get_user_and_secrets_lifespan +_logger = logging.getLogger(__name__) + @pytest.fixture() def create_registered_user( @@ -89,52 +93,65 @@ async def _( **project_overrides, ) -> ProjectAtDB: project_uuid = uuid4() - print(f"Created new project with uuid={project_uuid}") - project_config = { - "uuid": f"{project_uuid}", - "name": faker.name(), - "type": ProjectType.STANDARD.name, - "description": faker.text(), - "prj_owner": user["id"], - "access_rights": {"1": {"read": True, "write": True, "delete": True}}, - "thumbnail": "", - "workbench": {}, - } - project_config.update(**project_overrides) - async with sqlalchemy_async_engine.connect() as con, con.begin(): - result = await con.execute( - projects.insert() - .values(**project_config) - .returning(sa.literal_column("*")) - ) - - inserted_project = ProjectAtDB.model_validate(result.one()) - project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) - # NOTE: currently no resources is passed until it becomes necessary - default_node_config = { - "required_resources": {}, - "key": faker.pystr(), - "version": faker.pystr(), - "label": faker.pystr(), + with log_context( + logging.INFO, + "Creating new project with uuid=%s", + project_uuid, + logger=_logger, + ) as log_ctx: + + default_project_config = { + "uuid": f"{project_uuid}", + "name": faker.name(), + "type": ProjectType.STANDARD.name, + "description": faker.text(), + "prj_owner": user["id"], + "access_rights": {"1": {"read": True, "write": True, "delete": True}}, + "thumbnail": "", } - if project_nodes_overrides: - default_node_config.update(project_nodes_overrides) - await project_nodes_repo.add( - con, - nodes=[ - ProjectNodeCreate(node_id=NodeID(node_id), **default_node_config) - for node_id in inserted_project.workbench - ], - ) - await con.execute( - projects_to_products.insert().values( - project_uuid=f"{inserted_project.uuid}", - product_name=product_name, + default_project_config.update(**project_overrides) + project_workbench = default_project_config.pop("workbench", {}) + + async with sqlalchemy_async_engine.connect() as con, con.begin(): + result = await con.execute( + projects.insert() + .values(**default_project_config) + .returning(sa.literal_column("*")) ) - ) - print(f"--> created {inserted_project=}") - created_project_ids.append(f"{inserted_project.uuid}") - return inserted_project + + inserted_project = ProjectAtDB.model_validate( + {**dict(result.one()._asdict()), "workbench": project_workbench} + ) + + project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) + # NOTE: currently no resources is passed until it becomes necessary + default_node_config = { + "required_resources": {}, + "key": faker.pystr(), + "version": faker.pystr(), + "label": faker.pystr(), + } + if project_nodes_overrides: + default_node_config.update(project_nodes_overrides) + + await project_nodes_repo.add( + con, + nodes=[ + ProjectNodeCreate( + node_id=NodeID(node_id), **default_node_config + ) + for node_id in inserted_project.workbench + ], + ) + await con.execute( + projects_to_products.insert().values( + project_uuid=f"{inserted_project.uuid}", + product_name=product_name, + ) + ) + log_ctx.logger.info("Created project %s", inserted_project) + created_project_ids.append(f"{inserted_project.uuid}") + return inserted_project yield _ @@ -143,7 +160,7 @@ async def _( await con.execute( projects.delete().where(projects.c.uuid.in_(created_project_ids)) ) - print(f"<-- delete projects {created_project_ids=}") + _logger.info("<-- delete projects %s", created_project_ids) @pytest.fixture diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py index 890e67ced3a..6e11cf8b40c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py @@ -1,3 +1,17 @@ +from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline +from simcore_postgres_database.models.comp_run_snapshot_tasks import ( + comp_run_snapshot_tasks, +) +from simcore_postgres_database.models.comp_runs import comp_runs +from simcore_postgres_database.models.comp_tasks import NodeClass, comp_tasks +from simcore_postgres_database.models.groups import user_to_groups +from simcore_postgres_database.models.groups_extra_properties import ( + groups_extra_properties, +) +from simcore_postgres_database.models.projects import ProjectType, projects +from simcore_postgres_database.models.projects_networks import projects_networks +from simcore_postgres_database.models.projects_nodes import projects_nodes + __all__: tuple[str, ...] = ( "NodeClass", "ProjectType", @@ -12,3 +26,5 @@ "projects_nodes", "user_to_groups", ) + +# nopycln: file From 5b7935a81e3291c8d49dfd7a50771753bd741dba Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:22:21 +0200 Subject: [PATCH 006/186] faker --- .../models-library/tests/test_services_types.py | 17 +++++++++++++++-- .../pytest_simcore/helpers/faker_factories.py | 3 +-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/models-library/tests/test_services_types.py b/packages/models-library/tests/test_services_types.py index 206c531a78f..fd5e187685b 100644 --- a/packages/models-library/tests/test_services_types.py +++ b/packages/models-library/tests/test_services_types.py @@ -1,9 +1,13 @@ import pytest from models_library.projects import ProjectID from models_library.projects_nodes import NodeID -from models_library.services_types import ServiceRunID +from models_library.services_types import ServiceKey, ServiceRunID, ServiceVersion from models_library.users import UserID -from pydantic import PositiveInt +from pydantic import PositiveInt, TypeAdapter +from pytest_simcore.helpers.faker_factories import ( + random_service_key, + random_service_version, +) @pytest.mark.parametrize( @@ -38,3 +42,12 @@ def test_get_resource_tracking_run_id_for_dynamic(): assert isinstance( ServiceRunID.get_resource_tracking_run_id_for_dynamic(), ServiceRunID ) + + +def test_faker_factories_random_service_key_and_version_are_in_sync(): + + for _ in range(10): + key = random_service_key() + version = random_service_version() + TypeAdapter(ServiceKey).validate_python(key) + TypeAdapter(ServiceVersion).validate_python(version) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index c6d2c2a9da3..e94d56de6e6 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -26,8 +26,7 @@ def random_service_key(fake: Faker = DEFAULT_FAKER, *, name: str | None = None) -> str: - """Generates a random service key""" - return f"simcore/services/{fake.random_element(['dynamic', 'computational'])}/{name or fake.name()}" + return f"simcore/services/{fake.random_element(['dynamic', 'comp', 'frontend'])}/{name or fake.name().lower().replace(' ', '')}" def random_service_version(fake: Faker = DEFAULT_FAKER) -> str: From 019e04b9e2467702a0f959587480643dc621333d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:49:43 +0200 Subject: [PATCH 007/186] fixing nodes --- .../src/models_library/projects.py | 1 + .../src/pytest_simcore/db_entries_mocks.py | 50 ++++++++++--------- .../modules/db/repositories/projects.py | 32 ++++++++++-- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 2092d1fce93..2a17a61622f 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -102,6 +102,7 @@ class BaseProjectModel(BaseModel): last_change_date: datetime # Pipeline of nodes (SEE projects_nodes.py) + # FIXME: pedro removes this one workbench: Annotated[NodesDict, Field(description="Project's pipeline")] # validators diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 08c56d0660e..cfd42a62619 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -29,6 +29,7 @@ ) from sqlalchemy.ext.asyncio import AsyncEngine +from .helpers.faker_factories import random_service_key, random_service_version from .helpers.postgres_tools import insert_and_get_row_lifespan from .helpers.postgres_users import sync_insert_and_get_user_and_secrets_lifespan @@ -92,6 +93,7 @@ async def _( project_nodes_overrides: dict[str, Any] | None = None, **project_overrides, ) -> ProjectAtDB: + project_uuid = uuid4() with log_context( logging.INFO, @@ -100,7 +102,7 @@ async def _( logger=_logger, ) as log_ctx: - default_project_config = { + project_values = { "uuid": f"{project_uuid}", "name": faker.name(), "type": ProjectType.STANDARD.name, @@ -108,14 +110,14 @@ async def _( "prj_owner": user["id"], "access_rights": {"1": {"read": True, "write": True, "delete": True}}, "thumbnail": "", + **project_overrides, } - default_project_config.update(**project_overrides) - project_workbench = default_project_config.pop("workbench", {}) + project_workbench = project_values.pop("workbench", {}) async with sqlalchemy_async_engine.connect() as con, con.begin(): result = await con.execute( projects.insert() - .values(**default_project_config) + .values(**project_values) .returning(sa.literal_column("*")) ) @@ -124,25 +126,27 @@ async def _( ) project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) - # NOTE: currently no resources is passed until it becomes necessary - default_node_config = { - "required_resources": {}, - "key": faker.pystr(), - "version": faker.pystr(), - "label": faker.pystr(), - } - if project_nodes_overrides: - default_node_config.update(project_nodes_overrides) - - await project_nodes_repo.add( - con, - nodes=[ - ProjectNodeCreate( - node_id=NodeID(node_id), **default_node_config - ) - for node_id in inserted_project.workbench - ], - ) + + for node_id, node_data in project_workbench.items(): + # NOTE: currently no resources is passed until it becomes necessary + node_values = { + "required_resources": {}, + "key": random_service_key(fake=faker), + "version": random_service_version(fake=faker), + "label": faker.pystr(), + **node_data, + } + + if project_nodes_overrides: + node_values.update(project_nodes_overrides) + + await project_nodes_repo.add( + con, + nodes=[ + ProjectNodeCreate(node_id=NodeID(node_id), **node_values) + ], + ) + await con.execute( projects_to_products.insert().values( project_uuid=f"{inserted_project.uuid}", diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 3287ef4cbc7..8d743268b1f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -2,8 +2,9 @@ import sqlalchemy as sa from models_library.projects import ProjectAtDB, ProjectID +from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID -from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo +from simcore_postgres_database.utils_projects_nodes import ProjectNode, ProjectNodesRepo from simcore_postgres_database.utils_repos import pass_or_acquire_connection from ....core.errors import ProjectNotFoundError @@ -13,6 +14,21 @@ logger = logging.getLogger(__name__) +def _project_node_to_node(project_node: ProjectNode) -> Node: + """Converts a ProjectNode from the database to a Node model for the API. + + Handles field mapping and excludes database-specific fields that are not + part of the Node model. + """ + # Get all ProjectNode fields except those that don't belong in Node + exclude_fields = {"node_id", "required_resources", "created", "modified"} + node_data = project_node.model_dump( + exclude=exclude_fields, exclude_none=True, exclude_unset=True + ) + + return Node.model_validate(node_data) + + class ProjectsRepository(BaseRepository): async def get_project(self, project_id: ProjectID) -> ProjectAtDB: async with self.db_engine.connect() as conn: @@ -21,9 +37,17 @@ async def get_project(self, project_id: ProjectID) -> ProjectAtDB: sa.select(projects).where(projects.c.uuid == str(project_id)) ) ).one_or_none() - if not row: - raise ProjectNotFoundError(project_id=project_id) - return ProjectAtDB.model_validate(row) + if not row: + raise ProjectNotFoundError(project_id=project_id) + + repo = ProjectNodesRepo(project_uuid=project_id) + nodes = await repo.list(conn) + + project_workbench = { + f"{node.node_id}": _project_node_to_node(node) for node in nodes + } + data = {**row._asdict(), "workbench": project_workbench} + return ProjectAtDB.model_validate(data) async def is_node_present_in_workbench( self, project_id: ProjectID, node_uuid: NodeID From 9c0640e7661d6e6e23f6ac5e35209366beca8252 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 12:42:52 +0200 Subject: [PATCH 008/186] fixing fakes --- .../src/pytest_simcore/db_entries_mocks.py | 143 +++++++++--------- 1 file changed, 73 insertions(+), 70 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index cfd42a62619..08cfc183747 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -87,83 +87,86 @@ async def create_project( ) -> AsyncIterator[Callable[..., Awaitable[ProjectAtDB]]]: created_project_ids: list[str] = [] - async def _( - user: dict[str, Any], - *, - project_nodes_overrides: dict[str, Any] | None = None, - **project_overrides, - ) -> ProjectAtDB: - - project_uuid = uuid4() - with log_context( - logging.INFO, - "Creating new project with uuid=%s", - project_uuid, - logger=_logger, - ) as log_ctx: - - project_values = { - "uuid": f"{project_uuid}", - "name": faker.name(), - "type": ProjectType.STANDARD.name, - "description": faker.text(), - "prj_owner": user["id"], - "access_rights": {"1": {"read": True, "write": True, "delete": True}}, - "thumbnail": "", - **project_overrides, - } - project_workbench = project_values.pop("workbench", {}) - - async with sqlalchemy_async_engine.connect() as con, con.begin(): - result = await con.execute( - projects.insert() - .values(**project_values) - .returning(sa.literal_column("*")) + async with contextlib.AsyncExitStack() as stack: + + async def _( + user: dict[str, Any], + *, + project_nodes_overrides: dict[str, Any] | None = None, + **project_overrides, + ) -> ProjectAtDB: + + project_uuid = uuid4() + with log_context( + logging.INFO, + "Creating new project with uuid=%s", + project_uuid, + logger=_logger, + ) as log_ctx: + + project_values = { + "uuid": f"{project_uuid}", + "name": faker.name(), + "type": ProjectType.STANDARD.name, + "description": faker.text(), + "prj_owner": user["id"], + "access_rights": { + "1": {"read": True, "write": True, "delete": True} + }, + "thumbnail": "", + **project_overrides, + } + project_workbench = project_values.pop("workbench", {}) + + project_db_rows = await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects, + values=project_values, + pk_col=projects.c.uuid, + ) ) - inserted_project = ProjectAtDB.model_validate( - {**dict(result.one()._asdict()), "workbench": project_workbench} + {**project_db_rows, "workbench": project_workbench} ) - project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) - - for node_id, node_data in project_workbench.items(): - # NOTE: currently no resources is passed until it becomes necessary - node_values = { - "required_resources": {}, - "key": random_service_key(fake=faker), - "version": random_service_version(fake=faker), - "label": faker.pystr(), - **node_data, - } - - if project_nodes_overrides: - node_values.update(project_nodes_overrides) - - await project_nodes_repo.add( - con, - nodes=[ - ProjectNodeCreate(node_id=NodeID(node_id), **node_values) - ], - ) - - await con.execute( - projects_to_products.insert().values( - project_uuid=f"{inserted_project.uuid}", - product_name=product_name, + async with sqlalchemy_async_engine.connect() as con, con.begin(): + project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) + + for node_id, node_data in project_workbench.items(): + # NOTE: currently no resources is passed until it becomes necessary + node_values = { + "required_resources": {}, + "key": random_service_key(fake=faker), + "version": random_service_version(fake=faker), + "label": faker.pystr(), + **node_data, + } + + if project_nodes_overrides: + node_values.update(project_nodes_overrides) + + await project_nodes_repo.add( + con, + nodes=[ + ProjectNodeCreate( + node_id=NodeID(node_id), **node_values + ) + ], + ) + + await con.execute( + projects_to_products.insert().values( + project_uuid=f"{inserted_project.uuid}", + product_name=product_name, + ) ) - ) - log_ctx.logger.info("Created project %s", inserted_project) - created_project_ids.append(f"{inserted_project.uuid}") - return inserted_project + log_ctx.logger.info("Created project %s", inserted_project) + created_project_ids.append(f"{inserted_project.uuid}") + return inserted_project - yield _ + yield _ - # cleanup - async with sqlalchemy_async_engine.begin() as con: - await con.execute( - projects.delete().where(projects.c.uuid.in_(created_project_ids)) - ) _logger.info("<-- delete projects %s", created_project_ids) From f7cf17b1ebe8dc49de34b703475f4f6a54790509 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 12:45:53 +0200 Subject: [PATCH 009/186] fixing fakes --- .../models-library/tests/test_services_types.py | 16 ++++++++++------ .../pytest_simcore/helpers/faker_factories.py | 5 +++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/models-library/tests/test_services_types.py b/packages/models-library/tests/test_services_types.py index fd5e187685b..edb75301e74 100644 --- a/packages/models-library/tests/test_services_types.py +++ b/packages/models-library/tests/test_services_types.py @@ -44,10 +44,14 @@ def test_get_resource_tracking_run_id_for_dynamic(): ) -def test_faker_factories_random_service_key_and_version_are_in_sync(): +@pytest.mark.parametrize( + "service_key, service_version", + [(random_service_key(), random_service_version()) for _ in range(10)], +) +def test_service_key_and_version_are_in_sync( + service_key: ServiceKey, service_version: ServiceVersion +): + TypeAdapter(ServiceKey).validate_python(service_key) + TypeAdapter(ServiceVersion).validate_python(service_version) - for _ in range(10): - key = random_service_key() - version = random_service_version() - TypeAdapter(ServiceKey).validate_python(key) - TypeAdapter(ServiceVersion).validate_python(version) + assert service_key.startswith("simcore/services/") diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index e94d56de6e6..a5fbe36a640 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -26,11 +26,12 @@ def random_service_key(fake: Faker = DEFAULT_FAKER, *, name: str | None = None) -> str: - return f"simcore/services/{fake.random_element(['dynamic', 'comp', 'frontend'])}/{name or fake.name().lower().replace(' ', '')}" + suffix = fake.unique.word() if name is None else name + return f"simcore/services/{fake.random_element(['dynamic', 'comp', 'frontend'])}/{suffix}" def random_service_version(fake: Faker = DEFAULT_FAKER) -> str: - return ".".join([str(fake.pyint()) for _ in range(3)]) + return ".".join([str(fake.pyint(0, 100)) for _ in range(3)]) def random_icon_url(fake: Faker): From 17f60ded414c67b64de0a201428a4db940e47a34 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 14:11:21 +0200 Subject: [PATCH 010/186] fixes update --- .../director-v2/tests/integration/conftest.py | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 13a56f99e98..ae20f7ddac1 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -5,6 +5,7 @@ import asyncio import uuid from collections.abc import AsyncIterator, Awaitable, Callable +from typing import Any from unittest.mock import AsyncMock import httpx @@ -16,7 +17,7 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_postgres_database.models.comp_tasks import comp_tasks -from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_nodes import projects_nodes from starlette import status from tenacity import retry from tenacity.retry import retry_if_exception_type @@ -35,31 +36,40 @@ def mock_env(mock_env: EnvVarsDict, minio_s3_settings_envs: EnvVarsDict) -> EnvV def update_project_workbench_with_comp_tasks( postgres_db: sa.engine.Engine, ) -> Callable: - def updator(project_uuid: str): + def _updator(project_uuid: str): with postgres_db.connect() as con: + + # select all projects_nodes for this project result = con.execute( - projects.select().where(projects.c.uuid == project_uuid) + projects_nodes.select().where( + projects_nodes.c.project_uuid == project_uuid + ) ) - prj_row = result.first() - assert prj_row - prj_workbench = prj_row.workbench + project_nodes_map: dict[str, Any] = { + row.node_id: row._asdict() for row in result + } + # comp_tasks get and run_hash and outputs result = con.execute( - comp_tasks.select().where(comp_tasks.c.project_id == project_uuid) - ) - # let's get the results and run_hash - for task_row in result: - # pass these to the project workbench - prj_workbench[task_row.node_id]["outputs"] = task_row.outputs - prj_workbench[task_row.node_id]["runHash"] = task_row.run_hash - - con.execute( - projects.update() # pylint:disable=no-value-for-parameter - .values(workbench=prj_workbench) - .where(projects.c.uuid == project_uuid) + comp_tasks.select().where(comp_tasks.c.project_id == f"{project_uuid}") ) - - return updator + comp_tasks_rows = result.fetchall() + for task_row in comp_tasks_rows: + project_nodes_map[task_row.node_id]["outputs"] = task_row.outputs + project_nodes_map[task_row.node_id]["run_hash"] = task_row.run_hash + + # update projects_nodes with comp_tasks data + for node_id, node_data in project_nodes_map.items(): + con.execute( + projects_nodes.update() # pylint:disable=no-value-for-parameter + .values(**node_data) + .where( + (projects_nodes.c.node_id == node_id) + & (projects_nodes.c.project_uuid == project_uuid) + ) + ) + + return _updator @pytest.fixture(scope="session") From 1c9de9843e89f74d579582c0d5994eb755a416a3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 14:21:47 +0200 Subject: [PATCH 011/186] minor --- .../tests/integration/01/test_computation_api.py | 8 ++++---- services/director-v2/tests/integration/conftest.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index f5dce1567de..b880fe64f07 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -416,12 +416,12 @@ async def test_run_partial_computation( def _convert_to_pipeline_details( project: ProjectAtDB, - exp_pipeline_adj_list: dict[int, list[int]], - exp_node_states: dict[int, dict[str, Any]], + expected_pipeline_adj_list: dict[int, list[int]], + expected_node_states: dict[int, dict[str, Any]], ) -> PipelineDetails: workbench_node_uuids = list(project.workbench.keys()) converted_adj_list: dict[NodeID, list[NodeID]] = {} - for node_key, next_nodes in exp_pipeline_adj_list.items(): + for node_key, next_nodes in expected_pipeline_adj_list.items(): converted_adj_list[NodeID(workbench_node_uuids[node_key])] = [ NodeID(workbench_node_uuids[n]) for n in next_nodes ] @@ -434,7 +434,7 @@ def _convert_to_pipeline_details( currentStatus=s.get("currentStatus", RunningState.NOT_STARTED), progress=s.get("progress"), ) - for n, s in exp_node_states.items() + for n, s in expected_node_states.items() } pipeline_progress = 0 for node_id in converted_adj_list: diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index ae20f7ddac1..2212aa79eee 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -37,7 +37,7 @@ def update_project_workbench_with_comp_tasks( postgres_db: sa.engine.Engine, ) -> Callable: def _updator(project_uuid: str): - with postgres_db.connect() as con: + with postgres_db.connect() as con, con.begin(): # select all projects_nodes for this project result = con.execute( From 360dcb413ffc06d7828f4771e272167aa91c0d2e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 14:55:41 +0200 Subject: [PATCH 012/186] fixes migration tests --- .../tests/test_models_projects_to_jobs.py | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index e2e5cf0476e..a7db2a0623e 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -3,6 +3,7 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments +import json from collections.abc import Iterator import pytest @@ -14,7 +15,7 @@ from faker import Faker from pytest_simcore.helpers import postgres_tools from pytest_simcore.helpers.faker_factories import random_project, random_user -from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects import ProjectType, projects from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs @@ -97,7 +98,7 @@ def test_populate_projects_to_jobs_during_migration( "Study associated to solver job:" """{ "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a", + "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a", "created_at": "2025-01-27T13:12:58.676564Z" } @@ -120,8 +121,37 @@ def test_populate_projects_to_jobs_during_migration( prj_owner=user_id, ), ] + + default_column_values = { + # NOTE: not server_default values are not applied here! + "type": ProjectType.STANDARD.value, + "workbench": {}, + "access_rights": {}, + "published": False, + "hidden": False, + "workspace_id": None, + } + + # NOTE: cannot use `projects` table directly here because it changes + # throughout time for prj in projects_data: - conn.execute(sa.insert(projects).values(prj)) + for key, value in default_column_values.items(): + prj.setdefault(key, value) + + for key, value in prj.items(): + if isinstance(value, dict): + prj[key] = json.dumps(value) + + columns = list(prj.keys()) + values_clause = ", ".join(f":{col}" for col in columns) + columns_clause = ", ".join(columns) + stmt = sa.text( + f""" + INSERT INTO projects ({columns_clause}) + VALUES ({values_clause}) + """ # noqa: S608 + ).bindparams(**prj) + conn.execute(stmt) # MIGRATE UPGRADE: this should populate simcore_postgres_database.cli.upgrade.callback("head") From 86e02d61a91ae4c7790c2b2778fedbeebf5953b7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 14:57:43 +0200 Subject: [PATCH 013/186] fixes --- .../src/pytest_simcore/helpers/faker_factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index a5fbe36a640..131e45e8f9f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -27,7 +27,7 @@ def random_service_key(fake: Faker = DEFAULT_FAKER, *, name: str | None = None) -> str: suffix = fake.unique.word() if name is None else name - return f"simcore/services/{fake.random_element(['dynamic', 'comp', 'frontend'])}/{suffix}" + return f"simcore/services/{fake.random_element(['dynamic', 'comp', 'frontend'])}/{suffix.lower()}" def random_service_version(fake: Faker = DEFAULT_FAKER) -> str: From d1510e5daf77f9ddd10b7c06d17203bd22f1f922 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:20:52 +0200 Subject: [PATCH 014/186] simplified fixture --- .../tests/helpers/shared_comp_utils.py | 10 ++++---- .../integration/01/test_computation_api.py | 24 +++++++++---------- ...t_dynamic_sidecar_nodeports_integration.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/services/director-v2/tests/helpers/shared_comp_utils.py b/services/director-v2/tests/helpers/shared_comp_utils.py index f2ce2ff4283..583a2c71ea2 100644 --- a/services/director-v2/tests/helpers/shared_comp_utils.py +++ b/services/director-v2/tests/helpers/shared_comp_utils.py @@ -4,7 +4,7 @@ import httpx from models_library.api_schemas_directorv2.computations import ComputationGet -from models_library.projects import ProjectAtDB +from models_library.projects import ProjectID from models_library.projects_pipeline import PipelineDetails from models_library.projects_state import RunningState from models_library.users import UserID @@ -21,21 +21,21 @@ async def assert_computation_task_out_obj( task_out: ComputationGet, *, - project: ProjectAtDB, + project_uuid: ProjectID, exp_task_state: RunningState, exp_pipeline_details: PipelineDetails, iteration: PositiveInt | None, ) -> None: - assert task_out.id == project.uuid + assert task_out.id == project_uuid assert task_out.state == exp_task_state - assert task_out.url.path == f"/v2/computations/{project.uuid}" + assert task_out.url.path == f"/v2/computations/{project_uuid}" if exp_task_state in [ RunningState.PUBLISHED, RunningState.PENDING, RunningState.STARTED, ]: assert task_out.stop_url - assert task_out.stop_url.path == f"/v2/computations/{project.uuid}:stop" + assert task_out.stop_url.path == f"/v2/computations/{project_uuid}:stop" else: assert task_out.stop_url is None assert task_out.iteration == iteration diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index b880fe64f07..49d18a436e4 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -468,7 +468,7 @@ def _convert_to_pipeline_details( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=expected_pipeline_details, iteration=1, @@ -483,7 +483,7 @@ def _convert_to_pipeline_details( ) await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.SUCCESS, exp_pipeline_details=expected_pipeline_details_after_run, iteration=1, @@ -536,7 +536,7 @@ def _convert_to_pipeline_details( await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=expected_pipeline_details_forced, iteration=2, @@ -581,7 +581,7 @@ async def test_run_computation( # check the contents is correct: a pipeline that just started gets PUBLISHED await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, @@ -603,7 +603,7 @@ async def test_run_computation( await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.SUCCESS, exp_pipeline_details=fake_workbench_computational_pipeline_details_completed, iteration=1, @@ -651,7 +651,7 @@ async def test_run_computation( # check the contents is correct await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=expected_pipeline_details_forced, # NOTE: here the pipeline already ran so its states are different iteration=2, @@ -663,7 +663,7 @@ async def test_run_computation( ) await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.SUCCESS, exp_pipeline_details=fake_workbench_computational_pipeline_details_completed, iteration=2, @@ -704,7 +704,7 @@ async def test_abort_computation( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, @@ -781,7 +781,7 @@ async def test_update_and_delete_computation( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.NOT_STARTED, exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, @@ -800,7 +800,7 @@ async def test_update_and_delete_computation( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.NOT_STARTED, exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, @@ -819,7 +819,7 @@ async def test_update_and_delete_computation( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.NOT_STARTED, exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, @@ -837,7 +837,7 @@ async def test_update_and_delete_computation( # check the contents is correctb await assert_computation_task_out_obj( task_out, - project=sleepers_project, + project_uuid=sleepers_project.uuid, exp_task_state=RunningState.PUBLISHED, exp_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 52ee07ef378..4f6f9e6416f 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -1007,7 +1007,7 @@ async def test_nodeports_integration( await assert_computation_task_out_obj( task_out, - project=current_study, + project_uuid=current_study.uuid, exp_task_state=RunningState.SUCCESS, exp_pipeline_details=PipelineDetails.model_validate(fake_dy_success), iteration=1, From 791e8b38d7a4982339b198cc77cd5f99cda5cd32 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:42:00 +0200 Subject: [PATCH 015/186] comment --- .../201aa37f4d9a_remove_workbench_column_from_projects_.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py index 20cc55b4445..08d34f49da7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -19,6 +19,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + + # FIXME: verify all project nodes are actually moved from the projects.workbench to projects_nodes table! op.drop_column("projects", "workbench") # ### end Alembic commands ### From 00370e56da95f6119a657a18602ddfd9f063c71a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 28 Jul 2025 15:42:21 +0200 Subject: [PATCH 016/186] fix: create_node fixture --- .../simcore_storage_data_models.py | 24 ++++++--------- .../modules/db/projects.py | 29 ++++++++++++++----- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py index a41d4876612..4e65c238843 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py @@ -14,6 +14,7 @@ from models_library.users import UserID from pydantic import TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.storage_models import projects, users from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine @@ -219,26 +220,19 @@ async def _creator( project_id: ProjectID, node_id: NodeID | None = None, **kwargs ) -> NodeID: async with sqlalchemy_async_engine.begin() as conn: - result = await conn.execute( - sa.select(projects.c.workbench).where( - projects.c.uuid == f"{project_id}" - ) - ) - row = result.fetchone() - assert row - project_workbench: dict[str, Any] = row.workbench - new_node_id = node_id or NodeID(f"{faker.uuid4()}") - node_data = { + new_node_id = node_id or NodeID(faker.uuid4()) + node_values = { "key": "simcore/services/frontend/file-picker", "version": "1.0.0", "label": "pytest_fake_node", } - node_data.update(**kwargs) - project_workbench.update({f"{new_node_id}": node_data}) + node_values.update(**kwargs) await conn.execute( - projects.update() - .where(projects.c.uuid == f"{project_id}") - .values(workbench=project_workbench) + projects_nodes.insert().values( + node_id=f"{new_node_id}", + project_uuid=f"{project_id}", + **node_values, + ) ) return new_node_id diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 765430a6dd1..40183589e60 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -5,6 +5,7 @@ from models_library.projects import ProjectAtDB, ProjectID, ProjectIDStr from models_library.projects_nodes_io import NodeIDStr from pydantic import ValidationError +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.storage_models import projects from simcore_postgres_database.utils_repos import pass_or_acquire_connection from sqlalchemy.ext.asyncio import AsyncConnection @@ -54,16 +55,28 @@ async def get_project_id_and_node_id_to_names_map( connection: AsyncConnection | None = None, project_uuids: list[ProjectID], ) -> dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]]: - mapping = {} + names_map = {} async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( - sa.select(projects.c.uuid, projects.c.name, projects.c.workbench).where( - projects.c.uuid.in_(f"{pid}" for pid in project_uuids) + sa.select(projects.c.uuid, projects.c.name).where( + projects.c.uuid.in_( + [f"{project_uuid}" for project_uuid in project_uuids] + ) ) ): - mapping[ProjectID(f"{row.uuid}")] = {f"{row.uuid}": row.name} | { - f"{node_id}": node["label"] - for node_id, node in row.workbench.items() - } + names_map[ProjectID(row.uuid)] = {f"{row.uuid}": row.name} - return mapping + async for row in await conn.stream( + sa.select( + projects_nodes.c.node_id, + projects_nodes.c.project_uuid, + projects_nodes.c.label, + ).where( + projects_nodes.c.project_uuid.in_( + [f"{project_uuid}" for project_uuid in project_uuids] + ) + ) + ): + names_map[ProjectID(row.project_uuid)] |= {f"{row.node_id}": row.label} + + return names_map From 54028fb81156e59a995a478a475906c135059df9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 28 Jul 2025 15:56:28 +0200 Subject: [PATCH 017/186] fix: move project_exists --- .../utils_projects.py | 19 ++++++++++++++++++- .../modules/db/projects.py | 16 ---------------- .../simcore_service_storage/simcore_s3_dsm.py | 7 ++++--- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py index 577f9441004..ff551789f8b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from .models.projects import projects -from .utils_repos import transaction_context +from .utils_repos import pass_or_acquire_connection, transaction_context class DBBaseProjectError(OsparcErrorMixin, Exception): @@ -22,6 +22,23 @@ class ProjectsRepo: def __init__(self, engine): self.engine = engine + async def exists( + self, + project_uuid: uuid.UUID, + *, + connection: AsyncConnection | None = None, + ) -> bool: + async with pass_or_acquire_connection(self.engine, connection) as conn: + return ( + await conn.scalar( + sa.select(1) + .select_from(projects) + .where(projects.c.uuid == project_uuid) + .limit(1) + ) + is not None + ) + async def get_project_last_change_date( self, project_uuid: uuid.UUID, diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 40183589e60..3f6c64815dd 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -33,22 +33,6 @@ async def list_valid_projects_in( with suppress(ValidationError): yield ProjectAtDB.model_validate(row) - async def project_exists( - self, - *, - connection: AsyncConnection | None = None, - project_uuid: ProjectID, - ) -> bool: - async with pass_or_acquire_connection(self.db_engine, connection) as conn: - return bool( - await conn.scalar( - sa.select(sa.func.count()) - .select_from(projects) - .where(projects.c.uuid == f"{project_uuid}") - ) - == 1 - ) - async def get_project_id_and_node_id_to_names_map( self, *, diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index 7e2dbdc5baf..211532b9c6e 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -41,6 +41,7 @@ from servicelib.logging_utils import log_context from servicelib.progress_bar import ProgressBarData from servicelib.utils import ensure_ends_with, limited_gather +from simcore_postgres_database.utils_projects import ProjectsRepo from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy.ext.asyncio import AsyncEngine @@ -792,9 +793,9 @@ async def deep_copy_project_simcore_s3( task_progress.description = "Checking study access rights..." for prj_uuid in [src_project_uuid, dst_project_uuid]: - if not await ProjectRepository.instance( - get_db_engine(self.app) - ).project_exists(project_uuid=prj_uuid): + if not await ProjectsRepo(get_db_engine(self.app)).exists( + project_uuid=prj_uuid + ): raise ProjectNotFoundError(project_id=prj_uuid) source_access_rights = await AccessLayerRepository.instance( get_db_engine(self.app) From caa2799c2eef51577998481e30bbeb25cc33e3b0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:59:56 +0200 Subject: [PATCH 018/186] rename --- .../src/pytest_simcore/db_entries_mocks.py | 13 +++- .../tests/helpers/shared_comp_utils.py | 10 +-- .../integration/01/test_computation_api.py | 73 ++++++++++--------- ...t_dynamic_sidecar_nodeports_integration.py | 4 +- 4 files changed, 55 insertions(+), 45 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 08cfc183747..e638fdcb9e6 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -14,6 +14,7 @@ from faker import Faker from models_library.products import ProductName from models_library.projects import ProjectAtDB, ProjectID +from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID from pytest_simcore.helpers.logging_tools import log_context from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline @@ -134,23 +135,27 @@ async def _( project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) for node_id, node_data in project_workbench.items(): + # NOTE: workbench node have a lot of camecase fields. We validate with Node and + # export to ProjectNodeCreate with alias=False + node_model = Node.model_validate(node_data) + # NOTE: currently no resources is passed until it becomes necessary - node_values = { + project_workbench_node = { "required_resources": {}, "key": random_service_key(fake=faker), "version": random_service_version(fake=faker), "label": faker.pystr(), - **node_data, + **node_model.model_dump(mode="json", by_alias=False), } if project_nodes_overrides: - node_values.update(project_nodes_overrides) + project_workbench_node.update(project_nodes_overrides) await project_nodes_repo.add( con, nodes=[ ProjectNodeCreate( - node_id=NodeID(node_id), **node_values + node_id=NodeID(node_id), **project_workbench_node ) ], ) diff --git a/services/director-v2/tests/helpers/shared_comp_utils.py b/services/director-v2/tests/helpers/shared_comp_utils.py index 583a2c71ea2..7a664089d60 100644 --- a/services/director-v2/tests/helpers/shared_comp_utils.py +++ b/services/director-v2/tests/helpers/shared_comp_utils.py @@ -22,14 +22,14 @@ async def assert_computation_task_out_obj( task_out: ComputationGet, *, project_uuid: ProjectID, - exp_task_state: RunningState, - exp_pipeline_details: PipelineDetails, + expected_task_state: RunningState, + expected_pipeline_details: PipelineDetails, iteration: PositiveInt | None, ) -> None: assert task_out.id == project_uuid - assert task_out.state == exp_task_state + assert task_out.state == expected_task_state assert task_out.url.path == f"/v2/computations/{project_uuid}" - if exp_task_state in [ + if expected_task_state in [ RunningState.PUBLISHED, RunningState.PENDING, RunningState.STARTED, @@ -41,7 +41,7 @@ async def assert_computation_task_out_obj( assert task_out.iteration == iteration # check pipeline details contents received_task_out_pipeline = task_out.pipeline_details.model_dump() - expected_task_out_pipeline = exp_pipeline_details.model_dump() + expected_task_out_pipeline = expected_pipeline_details.model_dump() assert received_task_out_pipeline == expected_task_out_pipeline diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 49d18a436e4..2f5c46f86c8 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -415,26 +415,27 @@ async def test_run_partial_computation( ) def _convert_to_pipeline_details( - project: ProjectAtDB, + workbench_node_uuids: list[str], expected_pipeline_adj_list: dict[int, list[int]], expected_node_states: dict[int, dict[str, Any]], ) -> PipelineDetails: - workbench_node_uuids = list(project.workbench.keys()) + converted_adj_list: dict[NodeID, list[NodeID]] = {} for node_key, next_nodes in expected_pipeline_adj_list.items(): converted_adj_list[NodeID(workbench_node_uuids[node_key])] = [ NodeID(workbench_node_uuids[n]) for n in next_nodes ] converted_node_states: dict[NodeID, NodeState] = { - NodeID(workbench_node_uuids[n]): NodeState( - modified=s["modified"], + NodeID(workbench_node_uuids[node_index]): NodeState( + modified=node_state["modified"], dependencies={ - NodeID(workbench_node_uuids[dep_n]) for dep_n in s["dependencies"] + NodeID(workbench_node_uuids[dep_n]) + for dep_n in node_state["dependencies"] }, - currentStatus=s.get("currentStatus", RunningState.NOT_STARTED), - progress=s.get("progress"), + currentStatus=node_state.get("currentStatus", RunningState.NOT_STARTED), + progress=node_state.get("progress"), ) - for n, s in expected_node_states.items() + for node_index, node_state in expected_node_states.items() } pipeline_progress = 0 for node_id in converted_adj_list: @@ -448,7 +449,9 @@ def _convert_to_pipeline_details( # convert the ids to the node uuids from the project expected_pipeline_details = _convert_to_pipeline_details( - sleepers_project, params.exp_pipeline_adj_list, params.exp_node_states + workbench_node_uuids=list(sleepers_project.workbench.keys()), + expected_pipeline_adj_list=params.exp_pipeline_adj_list, + expected_node_states=params.exp_node_states, ) # send a valid project with sleepers @@ -469,8 +472,8 @@ def _convert_to_pipeline_details( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=expected_pipeline_details, + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=expected_pipeline_details, iteration=1, ) @@ -479,13 +482,15 @@ def _convert_to_pipeline_details( async_client, task_out.url, user["id"], sleepers_project.uuid ) expected_pipeline_details_after_run = _convert_to_pipeline_details( - sleepers_project, params.exp_pipeline_adj_list, params.exp_node_states_after_run + workbench_node_uuids=list(sleepers_project.workbench.keys()), + expected_pipeline_adj_list=params.exp_pipeline_adj_list, + expected_node_states=params.exp_node_states_after_run, ) await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.SUCCESS, - exp_pipeline_details=expected_pipeline_details_after_run, + expected_task_state=RunningState.SUCCESS, + expected_pipeline_details=expected_pipeline_details_after_run, iteration=1, ) @@ -537,8 +542,8 @@ def _convert_to_pipeline_details( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=expected_pipeline_details_forced, + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=expected_pipeline_details_forced, iteration=2, ) @@ -582,8 +587,8 @@ async def test_run_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=fake_workbench_computational_pipeline_details, + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, ) @@ -604,8 +609,8 @@ async def test_run_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.SUCCESS, - exp_pipeline_details=fake_workbench_computational_pipeline_details_completed, + expected_task_state=RunningState.SUCCESS, + expected_pipeline_details=fake_workbench_computational_pipeline_details_completed, iteration=1, ) @@ -652,8 +657,8 @@ async def test_run_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=expected_pipeline_details_forced, # NOTE: here the pipeline already ran so its states are different + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=expected_pipeline_details_forced, # NOTE: here the pipeline already ran so its states are different iteration=2, ) @@ -664,8 +669,8 @@ async def test_run_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.SUCCESS, - exp_pipeline_details=fake_workbench_computational_pipeline_details_completed, + expected_task_state=RunningState.SUCCESS, + expected_pipeline_details=fake_workbench_computational_pipeline_details_completed, iteration=2, ) @@ -705,8 +710,8 @@ async def test_abort_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=fake_workbench_computational_pipeline_details, + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, ) @@ -782,8 +787,8 @@ async def test_update_and_delete_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.NOT_STARTED, - exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, + expected_task_state=RunningState.NOT_STARTED, + expected_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, ) @@ -801,8 +806,8 @@ async def test_update_and_delete_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.NOT_STARTED, - exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, + expected_task_state=RunningState.NOT_STARTED, + expected_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, ) @@ -820,8 +825,8 @@ async def test_update_and_delete_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.NOT_STARTED, - exp_pipeline_details=fake_workbench_computational_pipeline_details_not_started, + expected_task_state=RunningState.NOT_STARTED, + expected_pipeline_details=fake_workbench_computational_pipeline_details_not_started, iteration=None, ) @@ -838,8 +843,8 @@ async def test_update_and_delete_computation( await assert_computation_task_out_obj( task_out, project_uuid=sleepers_project.uuid, - exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=fake_workbench_computational_pipeline_details, + expected_task_state=RunningState.PUBLISHED, + expected_pipeline_details=fake_workbench_computational_pipeline_details, iteration=1, ) diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 4f6f9e6416f..d16150111a2 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -1008,8 +1008,8 @@ async def test_nodeports_integration( await assert_computation_task_out_obj( task_out, project_uuid=current_study.uuid, - exp_task_state=RunningState.SUCCESS, - exp_pipeline_details=PipelineDetails.model_validate(fake_dy_success), + expected_task_state=RunningState.SUCCESS, + expected_pipeline_details=PipelineDetails.model_validate(fake_dy_success), iteration=1, ) update_project_workbench_with_comp_tasks(str(current_study.uuid)) From 6615e9bfaa26ed34b127eaa372a0223189a3db89 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 18:02:42 +0200 Subject: [PATCH 019/186] rm project dependency --- .../integration/01/test_computation_api.py | 36 +++++++++---------- ...t_dynamic_sidecar_nodeports_integration.py | 4 +-- .../director-v2/tests/integration/conftest.py | 6 ++-- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 2f5c46f86c8..a9b33b3f5d1 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -457,7 +457,7 @@ def _convert_to_pipeline_details( # send a valid project with sleepers task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -503,7 +503,7 @@ def _convert_to_pipeline_details( ): await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -525,7 +525,7 @@ def _convert_to_pipeline_details( ) task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -575,7 +575,7 @@ async def test_run_computation( # send a valid project with sleepers task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -622,7 +622,7 @@ async def test_run_computation( ): await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -646,7 +646,7 @@ async def test_run_computation( expected_pipeline_details_forced.progress = 0 task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -699,7 +699,7 @@ async def test_abort_computation( # send a valid project with sleepers task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -776,7 +776,7 @@ async def test_update_and_delete_computation( # send a valid project with sleepers task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -795,7 +795,7 @@ async def test_update_and_delete_computation( # update the pipeline task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -814,7 +814,7 @@ async def test_update_and_delete_computation( # update the pipeline task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -833,7 +833,7 @@ async def test_update_and_delete_computation( # start it now task_out = await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -864,7 +864,7 @@ async def test_update_and_delete_computation( with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_409_CONFLICT}"): await create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -917,7 +917,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp ): await create_pipeline( async_client, - project=project_with_dynamic_node, + project_uuid=project_with_dynamic_node.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, @@ -927,7 +927,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp # still this pipeline shall be createable if we do not want to start it await create_pipeline( async_client, - project=project_with_dynamic_node, + project_uuid=project_with_dynamic_node.uuid, user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -1124,7 +1124,7 @@ async def test_burst_create_computations( [ create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], product_name=osparc_product_name, product_api_base_url=osparc_product_api_base_url, @@ -1135,7 +1135,7 @@ async def test_burst_create_computations( + [ create_pipeline( async_client, - project=sleepers_project2, + project_uuid=sleepers_project2.uuid, user_id=user["id"], product_name=osparc_product_name, product_api_base_url=osparc_product_api_base_url, @@ -1153,7 +1153,7 @@ async def test_burst_create_computations( [ create_pipeline( async_client, - project=sleepers_project, + project_uuid=sleepers_project.uuid, user_id=user["id"], product_name=osparc_product_name, product_api_base_url=osparc_product_api_base_url, @@ -1164,7 +1164,7 @@ async def test_burst_create_computations( + [ create_pipeline( async_client, - project=sleepers_project2, + project_uuid=sleepers_project2.uuid, user_id=user["id"], product_name=osparc_product_name, product_api_base_url=osparc_product_api_base_url, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index d16150111a2..aea27017910 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -305,7 +305,7 @@ async def current_study( # create entries in comp_task table in order to pull output ports await create_pipeline( async_client, - project=project_at_db, + project_uuid=project_at_db.uuid, user_id=current_user["id"], start_pipeline=False, product_name=osparc_product_name, @@ -993,7 +993,7 @@ async def test_nodeports_integration( # STEP 2 task_out = await create_pipeline( async_client, - project=current_study, + project_uuid=current_study.uuid, user_id=current_user["id"], start_pipeline=True, product_name=osparc_product_name, diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 2212aa79eee..c94d96ae286 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -12,7 +12,7 @@ import pytest import sqlalchemy as sa from models_library.api_schemas_directorv2.computations import ComputationGet -from models_library.projects import ProjectAtDB +from models_library.projects import ProjectID from models_library.users import UserID from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -95,7 +95,7 @@ async def create_pipeline( async def _creator( client: httpx.AsyncClient, *, - project: ProjectAtDB, + project_uuid: ProjectID, user_id: UserID, product_name: str, product_api_base_url: str, @@ -106,7 +106,7 @@ async def _creator( COMPUTATION_URL, json={ "user_id": user_id, - "project_id": str(project.uuid), + "project_id": str(project_uuid), "start_pipeline": start_pipeline, "product_name": product_name, "product_api_base_url": product_api_base_url, From 8181580c666be7d6a03599651fbca8af9a3649a9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Jul 2025 18:39:32 +0200 Subject: [PATCH 020/186] Updates Node model and cleanup --- packages/models-library/src/models_library/projects_nodes.py | 3 ++- .../modules/db/repositories/comp_tasks/_utils.py | 3 +++ .../director-v2/src/simcore_service_director_v2/utils/dags.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index d7db50cf172..a75ee6d1895 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -383,7 +383,8 @@ def _update_json_schema_extra(schema: JsonDict) -> None: model_config = ConfigDict( extra="forbid", - populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, json_schema_extra=_update_json_schema_extra, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index eba9954771c..10103909a63 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -400,6 +400,9 @@ async def generate_tasks_list_from_project( raise WalletNotEnoughCreditsError( wallet_name=wallet_info.wallet_name, wallet_credit_amount=wallet_info.wallet_credit_amount, + user_id=user_id, + product_name=product_name, + project_id=project.uuid, ) assert rabbitmq_rpc_client # nosec diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dags.py b/services/director-v2/src/simcore_service_director_v2/utils/dags.py index a1ae4762278..37713ab76b5 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dags.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dags.py @@ -29,6 +29,7 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph: dag_graph: nx.DiGraph = nx.DiGraph() for node_id, node in workbench.items(): assert node.state # nosec + dag_graph.add_node( node_id, name=node.label, @@ -43,6 +44,9 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph: if node.input_nodes: for input_node_id in node.input_nodes: predecessor_node = workbench.get(NodeIDStr(input_node_id)) + assert ( + predecessor_node + ), f"Node {input_node_id} not found in workbench" # nosec if predecessor_node: dag_graph.add_edge(str(input_node_id), node_id) From e33be8624664aa73ea8f6d33f19a14c8cf10807e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 09:38:29 +0200 Subject: [PATCH 021/186] fix: use annotated functional validators --- .../models-library/src/models_library/projects.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 2a17a61622f..a9cea883d51 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -10,6 +10,7 @@ from common_library.basic_types import DEFAULT_FACTORY from pydantic import ( BaseModel, + BeforeValidator, ConfigDict, Field, HttpUrl, @@ -85,6 +86,7 @@ class BaseProjectModel(BaseModel): ] description: Annotated[ str, + BeforeValidator(none_to_empty_str_pre_validator), Field( description="longer one-line description about the project", examples=["Dabbling in temporal transitions ..."], @@ -92,6 +94,9 @@ class BaseProjectModel(BaseModel): ] thumbnail: Annotated[ HttpUrl | None, + BeforeValidator( + empty_str_to_none_pre_validator, + ), Field( description="url of the project thumbnail", examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"], @@ -105,15 +110,6 @@ class BaseProjectModel(BaseModel): # FIXME: pedro removes this one workbench: Annotated[NodesDict, Field(description="Project's pipeline")] - # validators - _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")( - empty_str_to_none_pre_validator - ) - - _none_description_is_empty = field_validator("description", mode="before")( - none_to_empty_str_pre_validator - ) - class ProjectAtDB(BaseProjectModel): # Model used to READ from database From 5bce45dbff6b839dddfd6c88b23251726c0365f2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 12:59:24 +0200 Subject: [PATCH 022/186] fix: fixtures still using workbench --- .../simcore_storage_data_models.py | 13 +++--- services/storage/tests/conftest.py | 32 +++++++++++---- .../storage/tests/unit/test_handlers_paths.py | 41 +++++++++++-------- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py index 4e65c238843..0b35c16b3d2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py @@ -215,10 +215,10 @@ async def _() -> None: @pytest.fixture async def create_project_node( user_id: UserID, sqlalchemy_async_engine: AsyncEngine, faker: Faker -) -> Callable[..., Awaitable[NodeID]]: +) -> Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]]: async def _creator( project_id: ProjectID, node_id: NodeID | None = None, **kwargs - ) -> NodeID: + ) -> tuple[NodeID, dict[str, Any]]: async with sqlalchemy_async_engine.begin() as conn: new_node_id = node_id or NodeID(faker.uuid4()) node_values = { @@ -227,13 +227,16 @@ async def _creator( "label": "pytest_fake_node", } node_values.update(**kwargs) - await conn.execute( - projects_nodes.insert().values( + result = await conn.execute( + projects_nodes.insert() + .values( node_id=f"{new_node_id}", project_uuid=f"{project_id}", **node_values, ) + .returning(sa.literal_column("*")) ) - return new_node_id + row = result.one() + return new_node_id, row._asdict() return _creator diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 32813640197..56d00c378c1 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -259,9 +259,13 @@ async def client( @pytest.fixture async def node_id( - project_id: ProjectID, create_project_node: Callable[[ProjectID], Awaitable[NodeID]] + project_id: ProjectID, + create_project_node: Callable[ + [ProjectID], Awaitable[tuple[NodeID, dict[str, Any]]] + ], ) -> NodeID: - return await create_project_node(project_id) + node_id, _ = await create_project_node(project_id) + return node_id @pytest.fixture @@ -784,7 +788,7 @@ async def _upload_folder_task( async def random_project_with_files( sqlalchemy_async_engine: AsyncEngine, create_project: Callable[..., Awaitable[dict[str, Any]]], - create_project_node: Callable[..., Awaitable[NodeID]], + create_project_node: Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]], create_simcore_file_id: Callable[ [ProjectID, NodeID, str, Path | None], SimcoreS3FileID ], @@ -798,17 +802,28 @@ async def random_project_with_files( upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]], ) -> Callable[ [ProjectWithFilesParams], - Awaitable[tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]], + Awaitable[ + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] + ], ]: async def _creator( project_params: ProjectWithFilesParams, - ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]: + ) -> tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ]: assert len(project_params.allowed_file_sizes) == len( project_params.allowed_file_checksums ) project = await create_project(name="random-project") node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]] = {} upload_tasks = [] + nodes: dict[NodeID, dict[str, Any]] = {} for _ in range(project_params.num_nodes): # Create a node with outputs (files and others) project_id = ProjectID(project["uuid"]) @@ -818,7 +833,7 @@ async def _creator( output3_file_id = create_simcore_file_id( project_id, node_id, output3_file_name, Path("outputs/output_3") ) - created_node_id = await create_project_node( + created_node_id, created_node = await create_project_node( ProjectID(project["uuid"]), node_id, outputs={ @@ -828,6 +843,7 @@ async def _creator( }, ) assert created_node_id == node_id + nodes[created_node_id] = created_node upload_tasks.append( _upload_one_file_task( @@ -878,7 +894,7 @@ async def _creator( node_to_files_mapping[node_id][file_id] = file_dict project = await get_updated_project(sqlalchemy_async_engine, project["uuid"]) - return project, node_to_files_mapping + return project, nodes, node_to_files_mapping return _creator @@ -933,7 +949,7 @@ async def output_file( yield file async with sqlalchemy_async_engine.begin() as conn: - result = await conn.execute( + await conn.execute( file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) ) diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index 6997bb5bf7d..a1a590fb21b 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -157,16 +157,16 @@ async def test_list_paths_pagination( user_id: UserID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], ): - project, list_of_files = with_random_project_with_files - num_nodes = len(list(project["workbench"])) + project, nodes, list_of_files = with_random_project_with_files # ls the nodes (DB-based) file_filter = Path(project["uuid"]) expected_paths = sorted( - ((file_filter / node_key, False) for node_key in project["workbench"]), + ((file_filter / f"{node_id}", False) for node_id in nodes), key=lambda x: x[0], ) await _assert_list_paths( @@ -176,12 +176,12 @@ async def test_list_paths_pagination( user_id, file_filter=file_filter, expected_paths=expected_paths, - limit=int(num_nodes / 2 + 0.5), + limit=int(len(nodes) / 2 + 0.5), ) # ls in the workspace (S3-based) # ls in the workspace - selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] ] @@ -240,11 +240,12 @@ async def test_list_paths_pagination_large_page( user_id: UserID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], ): - project, list_of_files = with_random_project_with_files - selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + project, nodes, list_of_files = with_random_project_with_files + selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] ] @@ -292,7 +293,11 @@ async def test_list_paths( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], project_params: ProjectWithFilesParams, @@ -305,7 +310,10 @@ async def test_list_paths( # ls root returns our projects expected_paths = sorted( - ((Path(f"{prj_db['uuid']}"), False) for prj_db, _ in project_to_files_mapping), + ( + (Path(f"{prj_db['uuid']}"), False) + for prj_db, _, _ in project_to_files_mapping + ), key=lambda x: x[0], ) await _assert_list_paths( @@ -318,9 +326,9 @@ async def test_list_paths( ) # ls with only some part of the path should return only the projects that match - selected_project, selected_project_files = random.choice( # noqa: S311 + selected_project, selected_nodes, selected_project_files = random.choice( project_to_files_mapping - ) + ) # noqa: S311 partial_file_filter = Path( selected_project["uuid"][: len(selected_project["uuid"]) // 2] ) @@ -340,7 +348,7 @@ async def test_list_paths( # now we ls inside one of the projects returns the nodes file_filter = Path(selected_project["uuid"]) expected_paths = sorted( - ((file_filter / node_key, False) for node_key in selected_project["workbench"]), + ((file_filter / f"{node_id}", False) for node_id in selected_nodes), key=lambda x: x[0], ) await _assert_list_paths( @@ -353,9 +361,7 @@ async def test_list_paths( ) # now we ls in one of the nodes - selected_node_id = NodeID( - random.choice(list(selected_project["workbench"])) # noqa: S311 - ) + selected_node_id = random.choice(list(selected_nodes)) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in selected_project_files[selected_node_id] ] @@ -623,6 +629,7 @@ async def test_path_compute_size( user_id: UserID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], project_params: ProjectWithFilesParams, @@ -630,7 +637,7 @@ async def test_path_compute_size( assert ( len(project_params.allowed_file_sizes) == 1 ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" - project, list_of_files = with_random_project_with_files + project, nodes, list_of_files = with_random_project_with_files total_num_files = sum( len(files_in_node) for files_in_node in list_of_files.values() @@ -649,7 +656,7 @@ async def test_path_compute_size( ) # get size of one of the nodes - selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 path = Path(project["uuid"]) / f"{selected_node_id}" selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] From a0d89e410c47e018c7ca328def3d67010a5d2de5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 13:21:14 +0200 Subject: [PATCH 023/186] fix: paths with problematic names --- .../storage/tests/unit/test_handlers_paths.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index a1a590fb21b..27ef9efdab0 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -31,6 +31,7 @@ from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy.ext.asyncio import AsyncEngine @@ -462,34 +463,46 @@ async def test_list_paths_with_display_name_containing_slashes( user_id: UserID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], sqlalchemy_async_engine: AsyncEngine, ): - project, list_of_files = with_random_project_with_files + project, nodes, list_of_files = with_random_project_with_files project_name_with_slashes = "soméà$èq¨thing with/ slas/h/es/" node_name_with_non_ascii = "my node / is not ascii: éàèù" - # adjust project to contain "difficult" characters + async with sqlalchemy_async_engine.begin() as conn: + # update project to contain "difficult" characters result = await conn.execute( sa.update(projects) .where(projects.c.uuid == project["uuid"]) .values(name=project_name_with_slashes) - .returning(sa.literal_column(f"{projects.c.name}, {projects.c.workbench}")) + .returning(projects.c.name) ) row = result.one() assert row.name == project_name_with_slashes - project_workbench = row.workbench - assert len(project_workbench) == 1 - node = next(iter(project_workbench.values())) - node["label"] = node_name_with_non_ascii - result = await conn.execute( - sa.update(projects) + + # update a node (first occurrence) to contain "difficult" characters + subquery = ( + sa.select(projects_nodes.c.node_id) + .select_from(projects_nodes.join(projects)) .where(projects.c.uuid == project["uuid"]) - .values(workbench=project_workbench) - .returning(sa.literal_column(f"{projects.c.name}, {projects.c.workbench}")) + .order_by(projects_nodes.c.node_id) + .limit(1) ) - row = result.one() + first_row = await conn.execute(subquery) + first_id = first_row.scalar_one_or_none() + + if first_id: + result = await conn.execute( + sa.update(projects_nodes) + .where(projects_nodes.c.node_id == first_id) + .values(label=node_name_with_non_ascii) + .returning(projects_nodes.c.label) + ) + row = result.one() + assert row.label == node_name_with_non_ascii # ls the root file_filter = None @@ -511,7 +524,7 @@ async def test_list_paths_with_display_name_containing_slashes( # ls the nodes to ensure / is still there between project and node file_filter = Path(project["uuid"]) expected_paths = sorted( - ((file_filter / node_key, False) for node_key in project["workbench"]), + ((file_filter / f"{node_id}", False) for node_id in nodes), key=lambda x: x[0], ) assert len(expected_paths) == 1, "test configuration problem" @@ -530,7 +543,7 @@ async def test_list_paths_with_display_name_containing_slashes( ), "display path parts should be url encoded" # ls in the node workspace - selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_id = random.choice(list(nodes)) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] ] From b58f26fe2ad8898e9a9c62f645e98ef6fbf4e285 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 13:29:03 +0200 Subject: [PATCH 024/186] fix: RPC tests --- .../tests/unit/test_rpc_handlers_paths.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_paths.py b/services/storage/tests/unit/test_rpc_handlers_paths.py index 04cbb47692c..1be3f405af9 100644 --- a/services/storage/tests/unit/test_rpc_handlers_paths.py +++ b/services/storage/tests/unit/test_rpc_handlers_paths.py @@ -23,7 +23,6 @@ from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE from models_library.products import ProductName from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID -from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams @@ -69,7 +68,7 @@ async def _assert_compute_path_size( path: Path, expected_total_size: int, ) -> ByteSize: - async_job, async_job_name = await compute_path_size( + async_job, _ = await compute_path_size( storage_rpc_client, product_name=product_name, user_id=user_id, @@ -79,7 +78,7 @@ async def _assert_compute_path_size( async for job_composed_result in wait_and_get_result( storage_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, - method_name=RPCMethodName(compute_path_size.__name__), + method_name=compute_path_size.__name__, job_id=async_job.job_id, job_filter=AsyncJobFilter( user_id=user_id, product_name=product_name, client_name="PYTEST_CLIENT_NAME" @@ -105,7 +104,7 @@ async def _assert_delete_paths( *, paths: set[Path], ) -> None: - async_job, async_job_name = await delete_paths( + async_job, _ = await delete_paths( storage_rpc_client, product_name=product_name, user_id=user_id, @@ -115,7 +114,7 @@ async def _assert_delete_paths( async for job_composed_result in wait_and_get_result( storage_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, - method_name=RPCMethodName(compute_path_size.__name__), + method_name=compute_path_size.__name__, job_id=async_job.job_id, job_filter=AsyncJobFilter( user_id=user_id, product_name=product_name, client_name="PYTEST_CLIENT_NAME" @@ -155,6 +154,7 @@ async def test_path_compute_size( location_id: LocationID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], project_params: ProjectWithFilesParams, @@ -163,7 +163,7 @@ async def test_path_compute_size( assert ( len(project_params.allowed_file_sizes) == 1 ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" - project, list_of_files = with_random_project_with_files + project, nodes, list_of_files = with_random_project_with_files total_num_files = sum( len(files_in_node) for files_in_node in list_of_files.values() @@ -182,7 +182,7 @@ async def test_path_compute_size( ) # get size of one of the nodes - selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_id = random.choice(list(nodes)) # noqa: S311 path = Path(project["uuid"]) / f"{selected_node_id}" selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] @@ -333,6 +333,7 @@ async def test_delete_paths( location_id: LocationID, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], ], project_params: ProjectWithFilesParams, @@ -342,7 +343,7 @@ async def test_delete_paths( assert ( len(project_params.allowed_file_sizes) == 1 ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" - project, list_of_files = with_random_project_with_files + project, nodes, list_of_files = with_random_project_with_files total_num_files = sum( len(files_in_node) for files_in_node in list_of_files.values() @@ -362,11 +363,7 @@ async def test_delete_paths( # now select multiple random files to delete selected_paths = random.sample( - list( - list_of_files[ - NodeID(random.choice(list(project["workbench"]))) # noqa: S311 - ] - ), + list(list_of_files[random.choice(list(nodes))]), # noqa: S311 round(project_params.workspace_files_count / 2), ) @@ -375,7 +372,7 @@ async def test_delete_paths( location_id, user_id, product_name, - paths=set({Path(_) for _ in selected_paths}), + paths={Path(_) for _ in selected_paths}, ) # the size is reduced by the amount of deleted files From 2700ee680ac7bf0886d88ba58be618122663081b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 13:37:01 +0200 Subject: [PATCH 025/186] fix: minor --- services/storage/tests/unit/test_handlers_paths.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index 27ef9efdab0..fff8f1af9bb 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -182,7 +182,7 @@ async def test_list_paths_pagination( # ls in the workspace (S3-based) # ls in the workspace - selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 + selected_node_id = random.choice(list(nodes)) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] ] @@ -246,7 +246,7 @@ async def test_list_paths_pagination_large_page( ], ): project, nodes, list_of_files = with_random_project_with_files - selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 + selected_node_id = random.choice(list(nodes)) # noqa: S311 selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] ] @@ -669,7 +669,7 @@ async def test_path_compute_size( ) # get size of one of the nodes - selected_node_id = random.choice(list(nodes.keys())) # noqa: S311 + selected_node_id = random.choice(list(nodes)) # noqa: S311 path = Path(project["uuid"]) / f"{selected_node_id}" selected_node_s3_keys = [ Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] From 14f50e6a4412eb2ce0b6127a77cbf9d10bc8d8a1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 14:12:39 +0200 Subject: [PATCH 026/186] fix: frontend data generator --- .../modules/db/projects.py | 8 ++--- .../simcore_service_storage/simcore_s3_dsm.py | 31 ++++++++++--------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 3f6c64815dd..de763297e1d 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -18,7 +18,7 @@ async def list_valid_projects_in( self, *, connection: AsyncConnection | None = None, - include_uuids: list[ProjectID], + project_uuids: list[ProjectID], ) -> AsyncIterator[ProjectAtDB]: """ @@ -27,7 +27,7 @@ async def list_valid_projects_in( async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( sa.select(projects).where( - projects.c.uuid.in_(f"{pid}" for pid in include_uuids) + projects.c.uuid.in_([f"{pid}" for pid in project_uuids]) ) ): with suppress(ValidationError): @@ -43,9 +43,7 @@ async def get_project_id_and_node_id_to_names_map( async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( sa.select(projects.c.uuid, projects.c.name).where( - projects.c.uuid.in_( - [f"{project_uuid}" for project_uuid in project_uuids] - ) + projects.c.uuid.in_([f"{pid}" for pid in project_uuids]) ) ): names_map[ProjectID(row.uuid)] = {f"{row.uuid}": row.name} diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index 211532b9c6e..aadcd75507b 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -117,25 +117,26 @@ async def _add_frontend_needed_data( # with information from the projects table! # NOTE: This part with the projects, should be done in the client code not here! - prj_names_mapping: dict[ProjectID | NodeID, str] = {} - - async for proj_data in ProjectRepository.instance(engine).list_valid_projects_in( - include_uuids=project_ids - ): - prj_names_mapping |= {proj_data.uuid: proj_data.name} | { - NodeID(node_id): node_data.label - for node_id, node_data in proj_data.workbench.items() - } + repo = ProjectRepository.instance(engine) + valid_project_uuids = [ + proj_data.uuid + async for proj_data in repo.list_valid_projects_in(project_uuids=project_ids) + ] + + prj_names_mapping = await repo.get_project_id_and_node_id_to_names_map( + project_uuids=valid_project_uuids + ) clean_data: list[FileMetaData] = [] for d in data: if d.project_id not in prj_names_mapping: continue assert d.project_id # nosec - d.project_name = prj_names_mapping[d.project_id] - if d.node_id in prj_names_mapping: + names_mapping = prj_names_mapping[d.project_id] + d.project_name = names_mapping[f"{d.project_id}"] + if d.node_id in names_mapping: assert d.node_id # nosec - d.node_name = prj_names_mapping[d.node_id] + d.node_name = names_mapping[f"{d.node_id}"] if d.node_name and d.project_name: clean_data.append(d) @@ -170,7 +171,7 @@ async def list_datasets(self, user_id: UserID) -> list[DatasetMetaData]: ) async for prj_data in ProjectRepository.instance( get_db_engine(self.app) - ).list_valid_projects_in(include_uuids=readable_projects_ids) + ).list_valid_projects_in(project_uuids=readable_projects_ids) ] async def list_files_in_dataset( @@ -782,8 +783,8 @@ async def deep_copy_project_simcore_s3( node_mapping: dict[NodeID, NodeID], task_progress: ProgressBarData, ) -> None: - src_project_uuid: ProjectID = ProjectID(src_project["uuid"]) - dst_project_uuid: ProjectID = ProjectID(dst_project["uuid"]) + src_project_uuid = ProjectID(src_project["uuid"]) + dst_project_uuid = ProjectID(dst_project["uuid"]) with log_context( _logger, logging.INFO, From 43630d56a5ebd70a4d762125ba037cd17a3a66a8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 14:53:21 +0200 Subject: [PATCH 027/186] fix: project_id type --- .../src/simcore_postgres_database/utils_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py index ff551789f8b..ee6a8a132e8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py @@ -33,7 +33,7 @@ async def exists( await conn.scalar( sa.select(1) .select_from(projects) - .where(projects.c.uuid == project_uuid) + .where(projects.c.uuid == f"{project_uuid}") .limit(1) ) is not None From 243c5e8ec8b9a13b5af246501533091b60b98297 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 15:02:37 +0200 Subject: [PATCH 028/186] fix: unpack created project/nodes --- .../tests/unit/test_handlers_datasets.py | 2 +- .../storage/tests/unit/test_handlers_files.py | 10 +++++--- .../unit/test_rpc_handlers_simcore_s3.py | 24 ++++++++++++++----- .../storage/tests/unit/test_simcore_s3_dsm.py | 8 +++++-- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 5808a63f1f1..47d83a98fb5 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -29,7 +29,7 @@ from servicelib.aiohttp import status from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = ["adminer"] diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index f07b63cdbe9..0ad9e81d47e 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -1518,14 +1518,18 @@ async def test_listing_with_project_id_filter( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], uuid_filter: bool, project_params: ProjectWithFilesParams, ): - src_project, src_projects_list = await random_project_with_files(project_params) - _, _ = await random_project_with_files(project_params) + src_project, _, src_projects_list = await random_project_with_files(project_params) + await random_project_with_files(project_params) assert len(src_projects_list.keys()) > 0 node_id = next(iter(src_projects_list.keys())) project_files_in_db = set(src_projects_list[node_id]) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index 2f76ea6135d..96773bbd956 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -220,13 +220,17 @@ async def test_copy_folders_from_valid_project_with_one_large_file( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], project_params: ProjectWithFilesParams, ): # 1. create a src project with 1 large file - src_project, src_projects_list = await random_project_with_files(project_params) + src_project, _, src_projects_list = await random_project_with_files(project_params) # 2. create a dst project without files dst_project, nodes_map = clone_project_data(src_project) dst_project = await create_project(**dst_project) @@ -313,13 +317,17 @@ async def test_copy_folders_from_valid_project( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], project_params: ProjectWithFilesParams, ): # 1. create a src project with some files - src_project, src_projects_list = await random_project_with_files(project_params) + src_project, _, src_projects_list = await random_project_with_files(project_params) # 2. create a dst project without files dst_project, nodes_map = clone_project_data(src_project) dst_project = await create_project(**dst_project) @@ -592,14 +600,18 @@ async def test_start_export_data( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], project_params: ProjectWithFilesParams, task_progress_spy: Mock, export_as: Literal["path", "download_link"], ): - _, src_projects_list = await random_project_with_files(project_params) + _, _, src_projects_list = await random_project_with_files(project_params) all_available_files: set[SimcoreS3FileID] = set() for x in src_projects_list.values(): diff --git a/services/storage/tests/unit/test_simcore_s3_dsm.py b/services/storage/tests/unit/test_simcore_s3_dsm.py index fdde44a8663..7aa3f69a191 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm.py @@ -173,11 +173,15 @@ async def paths_for_export( random_project_with_files: Callable[ [ProjectWithFilesParams], Awaitable[ - tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]] + tuple[ + dict[str, Any], + dict[NodeID, dict[str, Any]], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ] ], ], ) -> set[SimcoreS3FileID]: - _, file_mapping = await random_project_with_files( + _, _, file_mapping = await random_project_with_files( ProjectWithFilesParams( num_nodes=2, allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1KiB"),), From 427f99833358b1dca0276d187b7f4c42fc50fda1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 15:10:13 +0200 Subject: [PATCH 029/186] fix: unpack --- services/storage/tests/unit/test_rpc_handlers_simcore_s3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index 96773bbd956..ab0bfb4ad6d 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -493,13 +493,14 @@ async def test_create_and_delete_folders_from_project( product_name: ProductName, with_random_project_with_files: tuple[ dict[str, Any], + dict[NodeID, dict[str, Any]], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]], ], create_project: Callable[..., Awaitable[dict[str, Any]]], mock_datcore_download, num_concurrent_calls: int, ): - project_in_db, _ = with_random_project_with_files + project_in_db, _, _ = with_random_project_with_files # NOTE: here the point is to NOT have a limit on the number of calls!! await asyncio.gather( *[ From 841593dfebb7e87d6094227886b9dfb4be7c9ae6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 16:54:52 +0200 Subject: [PATCH 030/186] fix: clone test --- .../helpers/storage_utils_project.py | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_project.py b/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_project.py index ad4535c9d70..c9c680175ae 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_project.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_project.py @@ -2,12 +2,13 @@ from copy import deepcopy from typing import Any -from models_library.projects_nodes_io import NodeIDStr +from models_library.projects_nodes_io import NodeID def clone_project_data( - project: dict, -) -> tuple[dict[str, Any], dict[NodeIDStr, NodeIDStr]]: + project: dict[str, Any], + project_nodes: dict[NodeID, dict[str, Any]], +) -> tuple[dict[str, Any], dict[NodeID, dict[str, Any]], dict[NodeID, NodeID]]: project_copy = deepcopy(project) # Update project id @@ -17,28 +18,17 @@ def clone_project_data( project_copy.pop("id", None) project_copy["name"] = f"{project['name']}-copy" - # Workbench nodes shall be unique within the project context - def _create_new_node_uuid(old_uuid: NodeIDStr) -> NodeIDStr: - return NodeIDStr(uuidlib.uuid5(project_copy_uuid, old_uuid)) + # Nodes shall be unique within the project context + def _new_node_uuid(old: NodeID) -> NodeID: + return uuidlib.uuid5(project_copy_uuid, f"{old}") - nodes_map = {} - for node_uuid in project.get("workbench", {}): - nodes_map[node_uuid] = _create_new_node_uuid(node_uuid) + nodes_map = {node_uuid: _new_node_uuid(node_uuid) for node_uuid in project_nodes} + project_nodes_copy = { + nodes_map[old_node_id]: { + **deepcopy(data), + "node_id": nodes_map[old_node_id], # update the internal "node_id" field + } + for old_node_id, data in project_nodes.items() + } - def _replace_uuids(node): - if isinstance(node, str): - node = nodes_map.get(node, node) - elif isinstance(node, list): - node = [_replace_uuids(item) for item in node] - elif isinstance(node, dict): - _frozen_items = tuple(node.items()) - for key, value in _frozen_items: - if key in nodes_map: - new_key = nodes_map[key] - node[new_key] = node.pop(key) - key = new_key - node[key] = _replace_uuids(value) - return node - - project_copy["workbench"] = _replace_uuids(project_copy.get("workbench", {})) - return project_copy, nodes_map + return project_copy, project_nodes_copy, nodes_map From dbf42b61bb8320aff74926b92a7321589d635c74 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 29 Jul 2025 22:36:14 +0200 Subject: [PATCH 031/186] fix: project_nodes --- .../unit/test_rpc_handlers_simcore_s3.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index ab0bfb4ad6d..dd0e6842b84 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -230,9 +230,11 @@ async def test_copy_folders_from_valid_project_with_one_large_file( project_params: ProjectWithFilesParams, ): # 1. create a src project with 1 large file - src_project, _, src_projects_list = await random_project_with_files(project_params) + src_project, src_project_nodes, src_projects_list = await random_project_with_files( + project_params + ) # 2. create a dst project without files - dst_project, nodes_map = clone_project_data(src_project) + dst_project, _, nodes_map = clone_project_data(src_project, src_project_nodes) dst_project = await create_project(**dst_project) # copy the project files data = await _request_copy_folders( @@ -241,7 +243,7 @@ async def test_copy_folders_from_valid_project_with_one_large_file( product_name, src_project, dst_project, - nodes_map={NodeID(i): NodeID(j) for i, j in nodes_map.items()}, + nodes_map=nodes_map, ) assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) @@ -327,9 +329,11 @@ async def test_copy_folders_from_valid_project( project_params: ProjectWithFilesParams, ): # 1. create a src project with some files - src_project, _, src_projects_list = await random_project_with_files(project_params) + src_project, src_project_nodes, src_projects_list = await random_project_with_files( + project_params + ) # 2. create a dst project without files - dst_project, nodes_map = clone_project_data(src_project) + dst_project, _, nodes_map = clone_project_data(src_project, src_project_nodes) dst_project = await create_project(**dst_project) # copy the project files data = await _request_copy_folders( @@ -338,7 +342,7 @@ async def test_copy_folders_from_valid_project( product_name, src_project, dst_project, - nodes_map={NodeID(i): NodeID(j) for i, j in nodes_map.items()}, + nodes_map=nodes_map, ) assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) @@ -378,13 +382,14 @@ async def _create_and_delete_folders_from_project( user_id: UserID, product_name: ProductName, project: dict[str, Any], + project_nodes: dict[NodeID, dict[str, Any]], initialized_app: FastAPI, project_db_creator: Callable, check_list_files: bool, *, client_timeout: datetime.timedelta = datetime.timedelta(seconds=60), ) -> None: - destination_project, nodes_map = clone_project_data(project) + destination_project, _, nodes_map = clone_project_data(project, project_nodes) await project_db_creator(**destination_project) # creating a copy @@ -394,7 +399,7 @@ async def _create_and_delete_folders_from_project( product_name, project, destination_project, - nodes_map={NodeID(i): NodeID(j) for i, j in nodes_map.items()}, + nodes_map=nodes_map, client_timeout=client_timeout, ) @@ -500,7 +505,7 @@ async def test_create_and_delete_folders_from_project( mock_datcore_download, num_concurrent_calls: int, ): - project_in_db, _, _ = with_random_project_with_files + project_in_db, project_nodes_in_db, _ = with_random_project_with_files # NOTE: here the point is to NOT have a limit on the number of calls!! await asyncio.gather( *[ @@ -510,6 +515,7 @@ async def test_create_and_delete_folders_from_project( user_id, product_name, project_in_db, + project_nodes_in_db, initialized_app, create_project, check_list_files=False, From 1adee7e6d0e11c97f48a45e7da3bcf578bb9f4ed Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 00:14:48 +0200 Subject: [PATCH 032/186] fix: node names mapping --- packages/models-library/src/models_library/projects.py | 5 +++-- .../src/simcore_service_storage/modules/db/projects.py | 4 ++-- .../storage/src/simcore_service_storage/simcore_s3_dsm.py | 2 +- services/storage/tests/unit/test_handlers_files.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index a9cea883d51..8317bebf594 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -107,8 +107,9 @@ class BaseProjectModel(BaseModel): last_change_date: datetime # Pipeline of nodes (SEE projects_nodes.py) - # FIXME: pedro removes this one - workbench: Annotated[NodesDict, Field(description="Project's pipeline")] + # FIXME: pedro checks this one + # NOTE: GCG: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) + # workbench: Annotated[NodesDict, Field(description="Project's pipeline")] class ProjectAtDB(BaseProjectModel): diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index de763297e1d..9a4d25532c0 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -27,7 +27,7 @@ async def list_valid_projects_in( async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( sa.select(projects).where( - projects.c.uuid.in_([f"{pid}" for pid in project_uuids]) + projects.c.uuid.in_(f"{pid}" for pid in project_uuids) ) ): with suppress(ValidationError): @@ -43,7 +43,7 @@ async def get_project_id_and_node_id_to_names_map( async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( sa.select(projects.c.uuid, projects.c.name).where( - projects.c.uuid.in_([f"{pid}" for pid in project_uuids]) + projects.c.uuid.in_(f"{pid}" for pid in project_uuids) ) ): names_map[ProjectID(row.uuid)] = {f"{row.uuid}": row.name} diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index aadcd75507b..9d842472c2f 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -134,7 +134,7 @@ async def _add_frontend_needed_data( assert d.project_id # nosec names_mapping = prj_names_mapping[d.project_id] d.project_name = names_mapping[f"{d.project_id}"] - if d.node_id in names_mapping: + if f"{d.node_id}" in names_mapping: assert d.node_id # nosec d.node_name = names_mapping[f"{d.node_id}"] if d.node_name and d.project_name: diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 0ad9e81d47e..f4528233856 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -63,7 +63,7 @@ from types_aiobotocore_s3 import S3Client from yarl import URL -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = ["adminer"] From 2961b6f3a40ceda8124a7e61ac4e9b823b85b620 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 07:51:07 +0200 Subject: [PATCH 033/186] fix: nullable workbench --- packages/models-library/src/models_library/projects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 8317bebf594..b6f495e8544 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -109,7 +109,9 @@ class BaseProjectModel(BaseModel): # Pipeline of nodes (SEE projects_nodes.py) # FIXME: pedro checks this one # NOTE: GCG: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) - # workbench: Annotated[NodesDict, Field(description="Project's pipeline")] + workbench: Annotated[NodesDict, Field(description="Project's pipeline")] | None = ( + None + ) class ProjectAtDB(BaseProjectModel): From f5ae1c51ce97d2116ae8bb961541757ba374f367 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 08:02:42 +0200 Subject: [PATCH 034/186] fix: exclude workbench from validation --- packages/models-library/src/models_library/projects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index b6f495e8544..9f426c5527d 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -109,9 +109,9 @@ class BaseProjectModel(BaseModel): # Pipeline of nodes (SEE projects_nodes.py) # FIXME: pedro checks this one # NOTE: GCG: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) - workbench: Annotated[NodesDict, Field(description="Project's pipeline")] | None = ( - None - ) + workbench: Annotated[ + NodesDict, Field(description="Project's pipeline", exclude=True) + ] class ProjectAtDB(BaseProjectModel): From d9881a14d256cf0ae40c6260425b9e38c4c4f2f9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 08:58:28 +0200 Subject: [PATCH 035/186] fix: validate ignoring workbench --- .../src/models_library/projects.py | 18 ++++++++++++++---- .../modules/db/projects.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 9f426c5527d..ca3dde6bfe7 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -108,10 +108,20 @@ class BaseProjectModel(BaseModel): # Pipeline of nodes (SEE projects_nodes.py) # FIXME: pedro checks this one - # NOTE: GCG: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) - workbench: Annotated[ - NodesDict, Field(description="Project's pipeline", exclude=True) - ] + # NOTE: GCR: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) + workbench: Annotated[NodesDict, Field(description="Project's pipeline")] + + @classmethod + def model_validate_ignoring_workbench(cls, obj: Any): + if isinstance(obj, dict): + data = dict(obj) + data.pop("workbench", None) + else: + data = obj + model = cls.model_validate(data) + if isinstance(obj, dict) and "workbench" in obj: + model.workbench = obj["workbench"] + return model class ProjectAtDB(BaseProjectModel): diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 9a4d25532c0..7da1762eb7d 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -31,7 +31,7 @@ async def list_valid_projects_in( ) ): with suppress(ValidationError): - yield ProjectAtDB.model_validate(row) + yield ProjectAtDB.model_validate_ignoring_workbench(row._asdict()) async def get_project_id_and_node_id_to_names_map( self, From 1ceff4dbf5a3617dbfbebf31f0310b73a33a45f7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 09:25:26 +0200 Subject: [PATCH 036/186] fix: temp include workbench on a validation --- .../models-library/src/models_library/projects.py | 12 ------------ .../simcore_service_storage/modules/db/projects.py | 3 ++- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index ca3dde6bfe7..da7218e0774 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -111,18 +111,6 @@ class BaseProjectModel(BaseModel): # NOTE: GCR: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) workbench: Annotated[NodesDict, Field(description="Project's pipeline")] - @classmethod - def model_validate_ignoring_workbench(cls, obj: Any): - if isinstance(obj, dict): - data = dict(obj) - data.pop("workbench", None) - else: - data = obj - model = cls.model_validate(data) - if isinstance(obj, dict) and "workbench" in obj: - model.workbench = obj["workbench"] - return model - class ProjectAtDB(BaseProjectModel): # Model used to READ from database diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 7da1762eb7d..aa08f803b3f 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -31,7 +31,8 @@ async def list_valid_projects_in( ) ): with suppress(ValidationError): - yield ProjectAtDB.model_validate_ignoring_workbench(row._asdict()) + # FIXME: remove workbench once model is fixed + yield ProjectAtDB.model_validate(row._asdict() | {"workbench": {}}) async def get_project_id_and_node_id_to_names_map( self, From a47e4c80dfe7e286fce64d527602789f1b7bf361 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 11:01:31 +0200 Subject: [PATCH 037/186] fix: folder body --- .../unit/test_rpc_handlers_simcore_s3.py | 65 ++++++++++++++----- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index dd0e6842b84..645392d468d 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -35,7 +35,7 @@ from models_library.api_schemas_webserver.storage import PathToExport from models_library.basic_types import SHA256Str from models_library.products import ProductName -from models_library.projects_nodes_io import NodeID, NodeIDStr, SimcoreS3FileID +from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture @@ -72,22 +72,35 @@ async def _request_copy_folders( rpc_client: RabbitMQRPCClient, user_id: UserID, product_name: ProductName, - source_project: dict[str, Any], + src_project: dict[str, Any], + src_project_nodes: dict[NodeID, dict[str, Any]], dst_project: dict[str, Any], + dst_project_nodes: dict[NodeID, dict[str, Any]], nodes_map: dict[NodeID, NodeID], *, client_timeout: datetime.timedelta = datetime.timedelta(seconds=60), ) -> dict[str, Any]: with log_context( logging.INFO, - f"Copying folders from {source_project['uuid']} to {dst_project['uuid']}", + f"Copying folders from {src_project['uuid']} to {dst_project['uuid']}", ) as ctx: + source = src_project | { + "workbench": { + f"{node_id}": node for node_id, node in src_project_nodes.items() + } + } + destination = dst_project | { + "workbench": { + f"{node_id}": node for node_id, node in dst_project_nodes.items() + } + } + async_job_get, async_job_name = await copy_folders_from_project( rpc_client, user_id=user_id, product_name=product_name, body=FoldersBody( - source=source_project, destination=dst_project, nodes_map=nodes_map + source=source, destination=destination, nodes_map=nodes_map ), ) @@ -132,7 +145,9 @@ async def test_copy_folders_from_non_existing_project( user_id, product_name, incorrect_src_project, + {}, dst_project, + {}, nodes_map={}, ) @@ -144,7 +159,9 @@ async def test_copy_folders_from_non_existing_project( user_id, product_name, src_project, + {}, incorrect_dst_project, + {}, nodes_map={}, ) @@ -167,7 +184,9 @@ async def test_copy_folders_from_empty_project( user_id, product_name, src_project, + {}, dst_project, + {}, nodes_map={}, ) assert data == jsonable_encoder(dst_project) @@ -234,25 +253,28 @@ async def test_copy_folders_from_valid_project_with_one_large_file( project_params ) # 2. create a dst project without files - dst_project, _, nodes_map = clone_project_data(src_project, src_project_nodes) + dst_project, dst_project_nodes, nodes_map = clone_project_data( + src_project, src_project_nodes + ) dst_project = await create_project(**dst_project) - # copy the project files + data = await _request_copy_folders( storage_rabbitmq_rpc_client, user_id, product_name, src_project, + src_project_nodes, dst_project, + dst_project_nodes, nodes_map=nodes_map, ) + assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get( - TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") - ) + dst_node_id = nodes_map.get(src_node_id) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -333,7 +355,9 @@ async def test_copy_folders_from_valid_project( project_params ) # 2. create a dst project without files - dst_project, _, nodes_map = clone_project_data(src_project, src_project_nodes) + dst_project, dst_project_nodes, nodes_map = clone_project_data( + src_project, src_project_nodes + ) dst_project = await create_project(**dst_project) # copy the project files data = await _request_copy_folders( @@ -341,18 +365,19 @@ async def test_copy_folders_from_valid_project( user_id, product_name, src_project, + src_project_nodes, dst_project, + dst_project_nodes, nodes_map=nodes_map, ) + data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get( - TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") - ) + dst_node_id = nodes_map.get(src_node_id) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -389,8 +414,10 @@ async def _create_and_delete_folders_from_project( *, client_timeout: datetime.timedelta = datetime.timedelta(seconds=60), ) -> None: - destination_project, _, nodes_map = clone_project_data(project, project_nodes) - await project_db_creator(**destination_project) + dst_project, dst_project_nodes, nodes_map = clone_project_data( + project, project_nodes + ) + await project_db_creator(**dst_project) # creating a copy data = await _request_copy_folders( @@ -398,14 +425,18 @@ async def _create_and_delete_folders_from_project( user_id, product_name, project, - destination_project, + project_nodes, + dst_project, + dst_project_nodes, nodes_map=nodes_map, client_timeout=client_timeout, ) + data.pop("workbench", None) # remove workbench from the data + # data should be equal to the destination project, and all store entries should point to simcore.s3 # NOTE: data is jsonized where destination project is not! - assert jsonable_encoder(destination_project) == data + assert jsonable_encoder(dst_project) == data project_id = data["uuid"] From 45a07b8c8e8bcb523d82fdc6210a99e19bd4f98c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 11:04:17 +0200 Subject: [PATCH 038/186] fix: workbench --- services/storage/tests/unit/test_rpc_handlers_simcore_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index 645392d468d..95054b7a3b1 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -189,6 +189,7 @@ async def test_copy_folders_from_empty_project( {}, nodes_map={}, ) + data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder(dst_project) # check there is nothing in the dst project async with sqlalchemy_async_engine.connect() as conn: From 03acbd8afb7b29fc6ec2ff59a056282d3a83c086 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 11:07:15 +0200 Subject: [PATCH 039/186] fix: remove workbench --- services/storage/tests/unit/test_rpc_handlers_simcore_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index 95054b7a3b1..ebfdddab0d7 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -269,7 +269,7 @@ async def test_copy_folders_from_valid_project_with_one_large_file( dst_project_nodes, nodes_map=nodes_map, ) - + data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) From b16810cdb665d4bd4c72f4140e7b1bd1ab98e971 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 11:09:59 +0200 Subject: [PATCH 040/186] fix: workbench --- .../storage/tests/unit/test_rpc_handlers_simcore_s3.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index ebfdddab0d7..cbf59a87a88 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -116,6 +116,8 @@ async def _request_copy_folders( if async_job_result.done: result = await async_job_result.result() assert isinstance(result, AsyncJobResult) + assert isinstance(result.result, dict) + result.result.pop("workbench", None) # remove workbench from the data return result.result pytest.fail(reason="Copy folders failed!") @@ -189,7 +191,6 @@ async def test_copy_folders_from_empty_project( {}, nodes_map={}, ) - data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder(dst_project) # check there is nothing in the dst project async with sqlalchemy_async_engine.connect() as conn: @@ -269,7 +270,6 @@ async def test_copy_folders_from_valid_project_with_one_large_file( dst_project_nodes, nodes_map=nodes_map, ) - data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) @@ -371,7 +371,6 @@ async def test_copy_folders_from_valid_project( dst_project_nodes, nodes_map=nodes_map, ) - data.pop("workbench", None) # remove workbench from the data assert data == jsonable_encoder( await get_updated_project(sqlalchemy_async_engine, dst_project["uuid"]) ) @@ -432,9 +431,6 @@ async def _create_and_delete_folders_from_project( nodes_map=nodes_map, client_timeout=client_timeout, ) - - data.pop("workbench", None) # remove workbench from the data - # data should be equal to the destination project, and all store entries should point to simcore.s3 # NOTE: data is jsonized where destination project is not! assert jsonable_encoder(dst_project) == data From 6558b5a30ebdfd41feafbf7c101b6f91fb4d9ad3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 30 Jul 2025 23:07:48 +0200 Subject: [PATCH 041/186] fix: webserver --- .../helpers/webserver_projects.py | 99 +++++++++---------- .../director_v2/_computations_service.py | 23 +++-- .../projects/_projects_nodes_repository.py | 54 ++++++++-- .../projects/_projects_repository_legacy.py | 39 +------- .../_projects_repository_legacy_utils.py | 1 - .../server/tests/integration/02/conftest.py | 4 +- services/web/server/tests/unit/conftest.py | 4 +- .../server/tests/unit/with_dbs/02/conftest.py | 10 +- .../tests/unit/with_dbs/03/tags/conftest.py | 6 +- .../tests/unit/with_dbs/03/trash/conftest.py | 4 +- .../test_studies_dispatcher_studies_access.py | 6 +- .../unit/with_dbs/04/wallets/conftest.py | 6 +- .../server/tests/unit/with_dbs/conftest.py | 4 +- 13 files changed, 130 insertions(+), 130 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 917d70d24cc..541fac8dacb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -4,6 +4,8 @@ import json import uuid as uuidlib +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from pathlib import Path from typing import Any @@ -73,7 +75,9 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] - new_project = await db.insert_project( + workbench: dict[str, Any] = project_data.pop("workbench", {}) + + project_created = await db.insert_project( project_data, user_id, product_name=product_name, @@ -86,11 +90,9 @@ async def create_project( required_resources=ServiceResourcesDictHelpers.model_config[ "json_schema_extra" ]["examples"][0], - key=node_info.get("key"), - version=node_info.get("version"), - label=node_info.get("label"), + **node_info, ) - for node_id, node_info in project_data.get("workbench", {}).items() + for node_id, node_info in workbench.items() }, ) @@ -103,7 +105,7 @@ async def create_project( for group_id, permissions in _access_rights.items(): await update_or_insert_project_group( app, - project_id=new_project["uuid"], + project_id=project_created["uuid"], group_id=int(group_id), read=permissions["read"], write=permissions["write"], @@ -112,20 +114,22 @@ async def create_project( try: uuidlib.UUID(str(project_data["uuid"])) - assert new_project["uuid"] == project_data["uuid"] + assert project_created["uuid"] == project_data["uuid"] except (ValueError, AssertionError): # in that case the uuid gets replaced - assert new_project["uuid"] != project_data["uuid"] - project_data["uuid"] = new_project["uuid"] + assert project_created["uuid"] != project_data["uuid"] + project_data["uuid"] = project_created["uuid"] for key in DB_EXCLUSIVE_COLUMNS: project_data.pop(key, None) - new_project: ProjectDict = remap_keys( - new_project, + project_created: ProjectDict = remap_keys( + project_created, rename={"trashed": "trashedAt"}, ) - return new_project + + project_created["workbench"] = workbench # NOTE: restore workbench + return project_created async def delete_all_projects(app: web.Application): @@ -137,50 +141,35 @@ async def delete_all_projects(app: web.Application): await conn.execute(query) -class NewProject: - def __init__( - self, - params_override: dict | None = None, - app: web.Application | None = None, - *, - user_id: int, - product_name: str, - tests_data_dir: Path, - force_uuid: bool = False, - as_template: bool = False, - ): - assert app # nosec - - self.params_override = params_override - self.user_id = user_id - self.product_name = product_name - self.app = app - self.prj = {} - self.force_uuid = force_uuid - self.tests_data_dir = tests_data_dir - self.as_template = as_template - - assert tests_data_dir.exists() - assert tests_data_dir.is_dir() - - async def __aenter__(self) -> ProjectDict: - assert self.app # nosec - - self.prj = await create_project( - self.app, - self.params_override, - self.user_id, - product_name=self.product_name, - force_uuid=self.force_uuid, - default_project_json=self.tests_data_dir / "fake-project.json", - as_template=self.as_template, - ) - - return self.prj +@asynccontextmanager +async def new_project( + params_override: dict | None = None, + app: web.Application | None = None, + *, + user_id: int, + product_name: str, + tests_data_dir: Path, + force_uuid: bool = False, + as_template: bool = False, +) -> AsyncIterator[ProjectDict]: + assert app # nosec + assert tests_data_dir.exists() + assert tests_data_dir.is_dir() + + project = await create_project( + app, + params_override, + user_id, + product_name=product_name, + force_uuid=force_uuid, + default_project_json=tests_data_dir / "fake-project.json", + as_template=as_template, + ) - async def __aexit__(self, *args): - assert self.app # nosec - await delete_all_projects(self.app) + try: + yield project + finally: + await delete_all_projects(app) async def assert_get_same_project( diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py index 8611373d95d..129e3f75e23 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py @@ -1,3 +1,4 @@ +from collections.abc import Awaitable from decimal import Decimal from aiohttp import web @@ -14,6 +15,8 @@ ) from models_library.products import ProductName from models_library.projects import ProjectID +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID from models_library.rest_ordering import OrderBy from models_library.services_types import ServiceRunID from models_library.users import UserID @@ -29,6 +32,7 @@ CreditTransactionNotFoundError, ) from servicelib.utils import limited_gather +from simcore_service_webserver.projects._projects_nodes_repository import get_by_project from ..products.products_service import is_product_billable from ..projects.api import ( @@ -248,15 +252,23 @@ async def list_computations_latest_iteration_tasks( unique_project_uuids = {task.project_uuid for task in _tasks_get.items} # Fetch projects metadata concurrently # NOTE: MD: can be improved with a single batch call - project_dicts = await limited_gather( + + async def _wrap_with_id( + project_id: ProjectID, coro: Awaitable[list[tuple[NodeID, Node]]] + ) -> tuple[ProjectID, dict[NodeID, Node]]: + nodes = await coro + return project_id, dict(nodes) + + results = await limited_gather( *[ - get_project_dict_legacy(app, project_uuid=project_uuid) + _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid)) for project_uuid in unique_project_uuids ], limit=20, ) + # Build a dict: project_uuid -> workbench - project_uuid_to_workbench = {prj["uuid"]: prj["workbench"] for prj in project_dicts} + project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = dict(results) _service_run_ids = [item.service_run_id for item in _tasks_get.items] _is_product_billable = await is_product_billable(app, product_name=product_name) @@ -286,9 +298,8 @@ async def list_computations_latest_iteration_tasks( started_at=item.started_at, ended_at=item.ended_at, log_download_link=item.log_download_link, - node_name=project_uuid_to_workbench[f"{item.project_uuid}"][ - f"{item.node_id}" - ].get("label", ""), + node_name=project_uuid_to_workbench[item.project_uuid][item.node_id].label + or "", osparc_credits=credits_or_none, ) for item, credits_or_none in zip( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index cd6880732e0..7cfac797583 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -1,25 +1,30 @@ import logging import sqlalchemy as sa - from aiohttp import web from models_library.projects import ProjectID from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID -from simcore_postgres_database.utils_repos import transaction_context +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) from simcore_postgres_database.webserver_models import projects_nodes from sqlalchemy.ext.asyncio import AsyncConnection -from .exceptions import NodeNotFoundError from ..db.plugin import get_asyncpg_engine +from .exceptions import NodeNotFoundError _logger = logging.getLogger(__name__) _SELECTION_PROJECTS_NODES_DB_ARGS = [ + projects_nodes.c.node_id, projects_nodes.c.key, projects_nodes.c.version, projects_nodes.c.label, + projects_nodes.c.created, + projects_nodes.c.modified, projects_nodes.c.progress, projects_nodes.c.thumbnail, projects_nodes.c.input_access, @@ -43,27 +48,56 @@ async def get( project_id: ProjectID, node_id: NodeID, ) -> Node: - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - get_stmt = sa.select( - *_SELECTION_PROJECTS_NODES_DB_ARGS - ).where( + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + query = sa.select(*_SELECTION_PROJECTS_NODES_DB_ARGS).where( (projects_nodes.c.project_uuid == f"{project_id}") & (projects_nodes.c.node_id == f"{node_id}") ) - result = await conn.stream(get_stmt) + result = await conn.stream(query) assert result # nosec row = await result.first() if row is None: raise NodeNotFoundError( - project_uuid=f"{project_id}", - node_uuid=f"{node_id}" + project_uuid=f"{project_id}", node_uuid=f"{node_id}" ) assert row # nosec return Node.model_validate(row, from_attributes=True) +async def get_by_project( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, +) -> list[tuple[NodeID, Node]]: + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + query = sa.select(*_SELECTION_PROJECTS_NODES_DB_ARGS).where( + projects_nodes.c.project_uuid == f"{project_id}" + ) + + result = await conn.stream(query) + assert result # nosec + + from simcore_postgres_database.utils_projects_nodes import ProjectNode + + rows = await result.all() + return [ + ( + NodeID(row.node_id), + Node.model_validate( + ProjectNode.model_validate(row, from_attributes=True).model_dump( + exclude_none=True, + exclude_unset=True, + exclude={"node_id", "created", "modified"}, + ) + ), + ) + for row in rows + ] + + async def update( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 59f9ff2dc31..0fbeef889ec 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -234,41 +234,10 @@ def _reraise_if_not_unique_uuid_error(err: UniqueViolation): ) selected_values["tags"] = project_tag_ids - # NOTE: this will at some point completely replace workbench in the DB - if selected_values["workbench"]: - project_nodes_repo = ProjectNodesRepo( - project_uuid=project_uuid + if project_nodes: + await ProjectNodesRepo(project_uuid=project_uuid).add( + conn, nodes=list(project_nodes.values()) ) - if project_nodes is None: - project_nodes = { - NodeID(node_id): ProjectNodeCreate( - node_id=NodeID(node_id), - required_resources={}, - key=node_info.get("key"), - version=node_info.get("version"), - label=node_info.get("label"), - ) - for node_id, node_info in selected_values[ - "workbench" - ].items() - } - - nodes = [ - project_nodes.get( - NodeID(node_id), - ProjectNodeCreate( - node_id=NodeID(node_id), - required_resources={}, - key=node_info.get("key"), - version=node_info.get("version"), - label=node_info.get("label"), - ), - ) - for node_id, node_info in selected_values[ - "workbench" - ].items() - ] - await project_nodes_repo.add(conn, nodes=nodes) return selected_values async def insert_project( @@ -333,7 +302,6 @@ async def insert_project( # ensure we have the minimal amount of data here # All non-default in projects table insert_values.setdefault("name", "New Study") - insert_values.setdefault("workbench", {}) insert_values.setdefault("workspace_id", None) # must be valid uuid @@ -738,7 +706,6 @@ async def get_project_db(self, project_uuid: ProjectID) -> ProjectDBGet: result = await conn.execute( sa.select( *PROJECT_DB_COLS, - projects.c.workbench, ).where(projects.c.uuid == f"{project_uuid}") ) row = await result.fetchone() diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 5b482452625..348db7d4f74 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -224,7 +224,6 @@ async def _get_project( query = ( sa.select( *PROJECT_DB_COLS, - projects.c.workbench, users.c.primary_gid.label("trashed_by_primary_gid"), access_rights_subquery.c.access_rights, ) diff --git a/services/web/server/tests/integration/02/conftest.py b/services/web/server/tests/integration/02/conftest.py index 261e27bf067..c2b6407e166 100644 --- a/services/web/server/tests/integration/02/conftest.py +++ b/services/web/server/tests/integration/02/conftest.py @@ -9,7 +9,7 @@ import pytest from models_library.projects import ProjectID -from pytest_simcore.helpers.webserver_projects import NewProject +from pytest_simcore.helpers.webserver_projects import new_project @pytest.fixture(scope="session") @@ -45,7 +45,7 @@ async def user_project( fake_project["prjOwner"] = logged_user["name"] fake_project["uuid"] = f"{project_id}" - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/conftest.py b/services/web/server/tests/unit/conftest.py index 53b4892491e..c646ef6011e 100644 --- a/services/web/server/tests/unit/conftest.py +++ b/services/web/server/tests/unit/conftest.py @@ -16,7 +16,7 @@ from aiohttp.test_utils import TestClient from models_library.products import ProductName from pytest_mock import MockFixture, MockType -from pytest_simcore.helpers.webserver_projects import NewProject, empty_project_data +from pytest_simcore.helpers.webserver_projects import empty_project_data, new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.constants import FRONTEND_APP_DEFAULT @@ -106,7 +106,7 @@ async def user_project( tests_data_dir: Path, osparc_product_name: str, ) -> AsyncIterator[ProjectDict]: - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index 4b5fb923edb..3e2ce212407 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -27,7 +27,7 @@ from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status -from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects +from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from settings_library.catalog import CatalogSettings from simcore_service_webserver.application_settings import get_application_settings @@ -127,7 +127,7 @@ async def shared_project( }, }, ) - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], @@ -156,7 +156,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with NewProject( + async with new_project( project_data, client.app, user_id=user["id"], @@ -191,7 +191,7 @@ async def _creator(**prj_kwargs) -> ProjectDict: project_data |= prj_kwargs new_template_project = await created_projects_exit_stack.enter_async_context( - NewProject( + new_project( project_data, client.app, user_id=user["id"], @@ -289,7 +289,7 @@ async def _creator(num_dyn_services: int) -> ProjectDict: } } project = await stack.enter_async_context( - NewProject( + new_project( project_data, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index cdf12044e6c..665ffc1b94e 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -10,7 +10,7 @@ from aioresponses import aioresponses from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects +from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings @@ -110,7 +110,7 @@ async def shared_project( }, }, ) - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], @@ -139,7 +139,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with NewProject( + async with new_project( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/03/trash/conftest.py b/services/web/server/tests/unit/with_dbs/03/trash/conftest.py index f154713ec81..849840eca9f 100644 --- a/services/web/server/tests/unit/with_dbs/03/trash/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/trash/conftest.py @@ -20,7 +20,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import NewProject +from pytest_simcore.helpers.webserver_projects import new_project from pytest_simcore.helpers.webserver_users import NewUser, UserInfoDict from simcore_service_webserver.projects.models import ProjectDict @@ -59,7 +59,7 @@ async def other_user_project( tests_data_dir: Path, osparc_product_name: ProductName, ) -> AsyncIterable[ProjectDict]: - async with NewProject( + async with new_project( fake_project, client.app, user_id=other_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index f120ab0c23c..7fe7ee2f17e 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -30,7 +30,7 @@ from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects +from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE @@ -104,7 +104,7 @@ async def published_project( # everyone HAS read access "1": {"read": True, "write": False, "delete": False} } - async with NewProject( + async with new_project( project_data, client.app, user_id=user["id"], @@ -130,7 +130,7 @@ async def unpublished_project( project_data["uuid"] = "b134a337-a74f-40ff-a127-b36a1ccbede6" project_data["published"] = False # <-- - async with NewProject( + async with new_project( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py index f004044c34f..d7c62e5de64 100644 --- a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py @@ -13,7 +13,7 @@ from faker import Faker from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects +from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver.application_settings import ApplicationSettings @@ -72,7 +72,7 @@ async def shared_project( }, }, ) - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], @@ -101,7 +101,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with NewProject( + async with new_project( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 5705c4b95ca..71309bbe83e 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -50,7 +50,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import NewProject +from pytest_simcore.helpers.webserver_projects import new_project from pytest_simcore.helpers.webserver_users import UserInfoDict from redis import Redis from servicelib.aiohttp.application_keys import APP_AIOPG_ENGINE_KEY @@ -683,7 +683,7 @@ async def user_project( tests_data_dir: Path, osparc_product_name: ProductName, ) -> AsyncIterator[ProjectDict]: - async with NewProject( + async with new_project( fake_project, client.app, user_id=logged_user["id"], From 67c21838e95fd194186e07c421bef574e43291f6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 31 Jul 2025 09:09:47 +0200 Subject: [PATCH 042/186] fix: workbench --- .../director_v2/_computations_service.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py index 129e3f75e23..fb861982c1e 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py @@ -38,7 +38,6 @@ from ..projects.api import ( batch_get_project_name, check_user_project_permission, - get_project_dict_legacy, ) from ..projects.projects_metadata_service import ( get_project_custom_metadata_or_empty_dict, @@ -48,6 +47,13 @@ from ._comp_runs_collections_service import get_comp_run_collection_or_none_by_id +async def _wrap_with_id( + project_id: ProjectID, coro: Awaitable[list[tuple[NodeID, Node]]] +) -> tuple[ProjectID, dict[NodeID, Node]]: + nodes = await coro + return project_id, dict(nodes) + + async def _get_projects_metadata( app: web.Application, project_uuids: list[ProjectID], @@ -253,12 +259,6 @@ async def list_computations_latest_iteration_tasks( # Fetch projects metadata concurrently # NOTE: MD: can be improved with a single batch call - async def _wrap_with_id( - project_id: ProjectID, coro: Awaitable[list[tuple[NodeID, Node]]] - ) -> tuple[ProjectID, dict[NodeID, Node]]: - nodes = await coro - return project_id, dict(nodes) - results = await limited_gather( *[ _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid)) @@ -299,7 +299,7 @@ async def _wrap_with_id( ended_at=item.ended_at, log_download_link=item.log_download_link, node_name=project_uuid_to_workbench[item.project_uuid][item.node_id].label - or "", + or "Unknown", osparc_credits=credits_or_none, ) for item, credits_or_none in zip( @@ -421,15 +421,16 @@ async def list_computation_collection_run_tasks( # Get unique set of all project_uuids from comp_tasks unique_project_uuids = {task.project_uuid for task in _tasks_get.items} # NOTE: MD: can be improved with a single batch call - project_dicts = await limited_gather( + results = await limited_gather( *[ - get_project_dict_legacy(app, project_uuid=project_uuid) + _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid)) for project_uuid in unique_project_uuids ], limit=20, ) + # Build a dict: project_uuid -> workbench - project_uuid_to_workbench = {prj["uuid"]: prj["workbench"] for prj in project_dicts} + project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = dict(results) # Fetch projects metadata concurrently _projects_metadata = await _get_projects_metadata( @@ -466,9 +467,7 @@ async def list_computation_collection_run_tasks( log_download_link=item.log_download_link, name=( custom_metadata.get("job_name") - or project_uuid_to_workbench[f"{item.project_uuid}"][ - f"{item.node_id}" - ].get("label") + or project_uuid_to_workbench[item.project_uuid][item.node_id].label or "Unknown" ), osparc_credits=credits_or_none, From 4455d50b29df70baaa40a236367cae6cc6c11754 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 31 Jul 2025 09:36:36 +0200 Subject: [PATCH 043/186] fix: use batch call --- .../director_v2/_computations_service.py | 40 ++++++----------- .../projects/_projects_nodes_repository.py | 44 ++++++++++++++++++- 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py index fb861982c1e..9a8f4cb7005 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py @@ -1,4 +1,3 @@ -from collections.abc import Awaitable from decimal import Decimal from aiohttp import web @@ -32,9 +31,11 @@ CreditTransactionNotFoundError, ) from servicelib.utils import limited_gather -from simcore_service_webserver.projects._projects_nodes_repository import get_by_project from ..products.products_service import is_product_billable +from ..projects._projects_nodes_repository import ( + get_by_projects, +) from ..projects.api import ( batch_get_project_name, check_user_project_permission, @@ -47,13 +48,6 @@ from ._comp_runs_collections_service import get_comp_run_collection_or_none_by_id -async def _wrap_with_id( - project_id: ProjectID, coro: Awaitable[list[tuple[NodeID, Node]]] -) -> tuple[ProjectID, dict[NodeID, Node]]: - nodes = await coro - return project_id, dict(nodes) - - async def _get_projects_metadata( app: web.Application, project_uuids: list[ProjectID], @@ -257,18 +251,14 @@ async def list_computations_latest_iteration_tasks( # Get unique set of all project_uuids from comp_tasks unique_project_uuids = {task.project_uuid for task in _tasks_get.items} # Fetch projects metadata concurrently - # NOTE: MD: can be improved with a single batch call - - results = await limited_gather( - *[ - _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid)) - for project_uuid in unique_project_uuids - ], - limit=20, + _projects_nodes: dict[ProjectID, list[tuple[NodeID, Node]]] = await get_by_projects( + app, project_ids=unique_project_uuids ) # Build a dict: project_uuid -> workbench - project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = dict(results) + project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = { + project_uuid: dict(nodes) for project_uuid, nodes in _projects_nodes.items() + } _service_run_ids = [item.service_run_id for item in _tasks_get.items] _is_product_billable = await is_product_billable(app, product_name=product_name) @@ -420,17 +410,15 @@ async def list_computation_collection_run_tasks( # Get unique set of all project_uuids from comp_tasks unique_project_uuids = {task.project_uuid for task in _tasks_get.items} - # NOTE: MD: can be improved with a single batch call - results = await limited_gather( - *[ - _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid)) - for project_uuid in unique_project_uuids - ], - limit=20, + + _projects_nodes: dict[ProjectID, list[tuple[NodeID, Node]]] = await get_by_projects( + app, project_ids=unique_project_uuids ) # Build a dict: project_uuid -> workbench - project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = dict(results) + project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = { + project_uuid: dict(nodes) for project_uuid, nodes in _projects_nodes.items() + } # Fetch projects metadata concurrently _projects_metadata = await _get_projects_metadata( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 7cfac797583..2f7ebe79fc7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -5,6 +5,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID +from simcore_postgres_database.utils_projects_nodes import ProjectNode from simcore_postgres_database.utils_repos import ( pass_or_acquire_connection, transaction_context, @@ -20,6 +21,7 @@ _SELECTION_PROJECTS_NODES_DB_ARGS = [ projects_nodes.c.node_id, + projects_nodes.c.project_uuid, projects_nodes.c.key, projects_nodes.c.version, projects_nodes.c.label, @@ -80,8 +82,6 @@ async def get_by_project( result = await conn.stream(query) assert result # nosec - from simcore_postgres_database.utils_projects_nodes import ProjectNode - rows = await result.all() return [ ( @@ -98,6 +98,46 @@ async def get_by_project( ] +async def get_by_projects( + app: web.Application, + project_ids: set[ProjectID], + connection: AsyncConnection | None = None, +) -> dict[ProjectID, list[tuple[NodeID, Node]]]: + if not project_ids: + return {} + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + query = sa.select(*_SELECTION_PROJECTS_NODES_DB_ARGS).where( + projects_nodes.c.project_uuid.in_([f"{pid}" for pid in project_ids]) + ) + + result = await conn.stream(query) + assert result # nosec + + rows = await result.all() + + # Initialize dict with empty lists for all requested project_ids + projects_to_nodes: dict[ProjectID, list[tuple[NodeID, Node]]] = { + pid: [] for pid in project_ids + } + + # Fill in the actual data + for row in rows: + node = Node.model_validate( + ProjectNode.model_validate(row).model_dump( + exclude_none=True, + exclude_unset=True, + exclude={"node_id", "created", "modified"}, + ) + ) + + projects_to_nodes[ProjectID(row.project_uuid)].append( + (NodeID(row.node_id), node) + ) + + return projects_to_nodes + + async def update( app: web.Application, connection: AsyncConnection | None = None, From 0eec9383dc06df613836893f917d3d2ac3318c2b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:42:21 +0200 Subject: [PATCH 044/186] storage: returns minimal tuple --- .../modules/db/projects.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index aa08f803b3f..0beba4ebd45 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -1,10 +1,9 @@ from collections.abc import AsyncIterator -from contextlib import suppress +from typing import NamedTuple import sqlalchemy as sa -from models_library.projects import ProjectAtDB, ProjectID, ProjectIDStr +from models_library.projects import ProjectID, ProjectIDStr from models_library.projects_nodes_io import NodeIDStr -from pydantic import ValidationError from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.storage_models import projects from simcore_postgres_database.utils_repos import pass_or_acquire_connection @@ -13,26 +12,29 @@ from ._base import BaseRepository +class ProjectBasicTuple(NamedTuple): + uuid: ProjectID + name: str + + class ProjectRepository(BaseRepository): async def list_valid_projects_in( self, *, connection: AsyncConnection | None = None, project_uuids: list[ProjectID], - ) -> AsyncIterator[ProjectAtDB]: + ) -> AsyncIterator[ProjectBasicTuple]: """ NOTE that it lists ONLY validated projects in 'project_uuids' """ async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( - sa.select(projects).where( + sa.select(projects.c.uuid, projects.c.name).where( projects.c.uuid.in_(f"{pid}" for pid in project_uuids) ) ): - with suppress(ValidationError): - # FIXME: remove workbench once model is fixed - yield ProjectAtDB.model_validate(row._asdict() | {"workbench": {}}) + yield ProjectBasicTuple(uuid=ProjectID(row.uuid), name=row.name) async def get_project_id_and_node_id_to_names_map( self, From d52d3e76101f3cbfb42f075f3111994739731734 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:06:53 +0200 Subject: [PATCH 045/186] proper migration project.workbench --- .../tests/test_services_types.py | 2 +- ..._remove_workbench_column_from_projects_.py | 300 +++++++++++++++++- 2 files changed, 300 insertions(+), 2 deletions(-) diff --git a/packages/models-library/tests/test_services_types.py b/packages/models-library/tests/test_services_types.py index edb75301e74..0952baf7c4a 100644 --- a/packages/models-library/tests/test_services_types.py +++ b/packages/models-library/tests/test_services_types.py @@ -48,7 +48,7 @@ def test_get_resource_tracking_run_id_for_dynamic(): "service_key, service_version", [(random_service_key(), random_service_version()) for _ in range(10)], ) -def test_service_key_and_version_are_in_sync( +def test_faker_factory_service_key_and_version_are_in_sync( service_key: ServiceKey, service_version: ServiceVersion ): TypeAdapter(ServiceKey).validate_python(service_key) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py index 08d34f49da7..164651647de 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -6,6 +6,9 @@ """ +import json +from typing import Any + import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql @@ -17,10 +20,302 @@ depends_on = None +def _migrate_workbench_to_projects_nodes() -> None: + """Migrate nodes from projects.workbench to projects_nodes table.""" + + # Get database connection + connection = op.get_bind() + + # Fetch all projects with workbench data + projects_result = connection.execute( + sa.text("SELECT uuid, workbench FROM projects WHERE workbench IS NOT NULL") + ) + + errors: list[str] = [] + updated_nodes_count = 0 + inserted_nodes_count = 0 + + for project_uuid, workbench_json in projects_result: + if not workbench_json: + continue + + try: + workbench_data = ( + workbench_json + if isinstance(workbench_json, dict) + else json.loads(workbench_json) + ) + except (json.JSONDecodeError, TypeError) as e: + errors.append(f"Project {project_uuid}: Invalid workbench JSON - {e}") + continue + + if not isinstance(workbench_data, dict): + errors.append(f"Project {project_uuid}: Workbench is not a dictionary") + continue + + for node_id, node_data in workbench_data.items(): + if not isinstance(node_data, dict): + errors.append( + f"Project {project_uuid}, Node {node_id}: Node data is not a dictionary" + ) + continue + + # Validate required fields + missing_fields = [] + if not node_data.get("key"): + missing_fields.append("key") + if not node_data.get("version"): + missing_fields.append("version") + if not node_data.get("label"): + missing_fields.append("label") + + if missing_fields: + errors.append( + f"Project {project_uuid}, Node {node_id}: Missing required fields: {', '.join(missing_fields)}" + ) + continue + + # Check if node already exists + existing_node = connection.execute( + sa.text( + "SELECT project_node_id FROM projects_nodes WHERE project_uuid = :project_uuid AND node_id = :node_id" + ), + {"project_uuid": project_uuid, "node_id": node_id}, + ).fetchone() + + # Prepare node data for insertion/update + node_values = { + "project_uuid": project_uuid, + "node_id": node_id, + "key": node_data["key"], + "version": node_data["version"], + "label": node_data["label"], + "progress": node_data.get("progress"), + "thumbnail": node_data.get("thumbnail"), + "input_access": ( + json.dumps(node_data["input_access"]) + if node_data.get("input_access") + else None + ), + "input_nodes": ( + json.dumps(node_data["input_nodes"]) + if node_data.get("input_nodes") + else None + ), + "inputs": ( + json.dumps(node_data["inputs"]) if node_data.get("inputs") else None + ), + "inputs_required": ( + json.dumps(node_data["inputs_required"]) + if node_data.get("inputs_required") + else None + ), + "inputs_units": ( + json.dumps(node_data["inputs_units"]) + if node_data.get("inputs_units") + else None + ), + "output_nodes": ( + json.dumps(node_data["output_nodes"]) + if node_data.get("output_nodes") + else None + ), + "outputs": ( + json.dumps(node_data["outputs"]) + if node_data.get("outputs") + else None + ), + "run_hash": node_data.get( + "run_hash", node_data.get("runHash") + ), # Handle both camelCase and snake_case + "state": ( + json.dumps(node_data["state"]) if node_data.get("state") else None + ), + "parent": node_data.get("parent"), + "boot_options": ( + json.dumps(node_data["boot_options"]) + if node_data.get("boot_options", node_data.get("bootOptions")) + else None + ), + } + + if existing_node: + # Update existing node + update_sql = """ + UPDATE projects_nodes SET + key = :key, + version = :version, + label = :label, + progress = :progress, + thumbnail = :thumbnail, + input_access = :input_access::jsonb, + input_nodes = :input_nodes::jsonb, + inputs = :inputs::jsonb, + inputs_required = :inputs_required::jsonb, + inputs_units = :inputs_units::jsonb, + output_nodes = :output_nodes::jsonb, + outputs = :outputs::jsonb, + run_hash = :run_hash, + state = :state::jsonb, + parent = :parent, + boot_options = :boot_options::jsonb, + modified_datetime = NOW() + WHERE project_uuid = :project_uuid AND node_id = :node_id + """ + connection.execute(sa.text(update_sql), node_values) + updated_nodes_count += 1 + print(f"Updated existing node {node_id} in project {project_uuid}") + + else: + # Insert new node + insert_sql = """ + INSERT INTO projects_nodes ( + project_uuid, node_id, key, version, label, progress, thumbnail, + input_access, input_nodes, inputs, inputs_required, inputs_units, + output_nodes, outputs, run_hash, state, parent, boot_options, + required_resources, created_datetime, modified_datetime + ) VALUES ( + :project_uuid, :node_id, :key, :version, :label, :progress, :thumbnail, + :input_access::jsonb, :input_nodes::jsonb, :inputs::jsonb, + :inputs_required::jsonb, :inputs_units::jsonb, :output_nodes::jsonb, + :outputs::jsonb, :run_hash, :state::jsonb, :parent, :boot_options::jsonb, + '{}'::jsonb, NOW(), NOW() + ) + """ + connection.execute(sa.text(insert_sql), node_values) + inserted_nodes_count += 1 + + print( + f"Migration summary: {inserted_nodes_count} nodes inserted, {updated_nodes_count} nodes updated" + ) + + if errors: + error_message = f"Migration failed with {len(errors)} errors:\n" + "\n".join( + errors + ) + print(error_message) + raise RuntimeError(error_message) + + +def _restore_workbench_from_projects_nodes() -> None: + """Restore workbench data from projects_nodes table to projects.workbench column.""" + + # Get database connection + connection = op.get_bind() + + # Get all projects that have nodes in projects_nodes + projects_with_nodes = connection.execute( + sa.text( + """ + SELECT DISTINCT project_uuid + FROM projects_nodes + ORDER BY project_uuid + """ + ) + ) + + errors: list[str] = [] + restored_projects_count = 0 + + for (project_uuid,) in projects_with_nodes: + # Fetch all nodes for this project + nodes_result = connection.execute( + sa.text( + """ + SELECT node_id, key, version, label, progress, thumbnail, + input_access, input_nodes, inputs, inputs_required, inputs_units, + output_nodes, outputs, run_hash, state, parent, boot_options + FROM projects_nodes + WHERE project_uuid = :project_uuid + ORDER BY node_id + """ + ), + {"project_uuid": project_uuid}, + ) + + workbench_data: dict[str, Any] = {} + + for row in nodes_result: + node_id = row.node_id + + # Build node data dictionary + node_data: dict[str, Any] = { + "key": row.key, + "version": row.version, + "label": row.label, + } + + # Add optional fields if they exist + if row.progress is not None: + node_data["progress"] = float(row.progress) + if row.thumbnail: + node_data["thumbnail"] = row.thumbnail + if row.input_access: + node_data["inputAccess"] = row.input_access + if row.input_nodes: + node_data["inputNodes"] = row.input_nodes + if row.inputs: + node_data["inputs"] = row.inputs + if row.inputs_required: + node_data["inputsRequired"] = row.inputs_required + if row.inputs_units: + node_data["inputsUnits"] = row.inputs_units + if row.output_nodes: + node_data["outputNodes"] = row.output_nodes + if row.outputs: + node_data["outputs"] = row.outputs + if row.run_hash: + node_data["runHash"] = row.run_hash + if row.state: + node_data["state"] = row.state + if row.parent: + node_data["parent"] = row.parent + if row.boot_options: + node_data["bootOptions"] = row.boot_options + + workbench_data[node_id] = node_data + + if workbench_data: + try: + # Update the project with the restored workbench data + connection.execute( + sa.text( + """ + UPDATE projects + SET workbench = :workbench_data + WHERE uuid = :project_uuid + """ + ), + { + "project_uuid": project_uuid, + "workbench_data": json.dumps(workbench_data), + }, + ) + restored_projects_count += 1 + + except Exception as e: + errors.append( + f"Project {project_uuid}: Failed to restore workbench data - {e}" + ) + + print( + f"Downgrade summary: {restored_projects_count} projects restored with workbench data" + ) + + if errors: + error_message = f"Downgrade failed with {len(errors)} errors:\n" + "\n".join( + errors + ) + print(error_message) + raise RuntimeError(error_message) + + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - # FIXME: verify all project nodes are actually moved from the projects.workbench to projects_nodes table! + # Migrate workbench data to projects_nodes before dropping the column + _migrate_workbench_to_projects_nodes() + op.drop_column("projects", "workbench") # ### end Alembic commands ### @@ -36,4 +331,7 @@ def downgrade(): nullable=False, ), ) + + # Restore workbench data from projects_nodes table + _restore_workbench_from_projects_nodes() # ### end Alembic commands ### From 30b2b7f6b1ea981c64752e85479c3170b3b5faa0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:42:01 +0200 Subject: [PATCH 046/186] using insert_and_get_row --- .../simcore_storage_data_models.py | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py index 0b35c16b3d2..4dd4e31317d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py @@ -2,6 +2,7 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import contextlib from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from typing import Any @@ -20,6 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine from .helpers.faker_factories import DEFAULT_FAKER, random_project +from .helpers.postgres_tools import insert_and_get_row_lifespan from .helpers.postgres_users import insert_and_get_user_and_secrets_lifespan @@ -215,28 +217,38 @@ async def _() -> None: @pytest.fixture async def create_project_node( user_id: UserID, sqlalchemy_async_engine: AsyncEngine, faker: Faker -) -> Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]]: - async def _creator( - project_id: ProjectID, node_id: NodeID | None = None, **kwargs - ) -> tuple[NodeID, dict[str, Any]]: - async with sqlalchemy_async_engine.begin() as conn: +) -> AsyncIterator[Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]]]: + created_node_entries: list[tuple[NodeID, ProjectID]] = [] + + async with contextlib.AsyncExitStack() as stack: + + async def _creator( + project_id: ProjectID, node_id: NodeID | None = None, **kwargs + ) -> tuple[NodeID, dict[str, Any]]: new_node_id = node_id or NodeID(faker.uuid4()) node_values = { + "node_id": f"{new_node_id}", + "project_uuid": f"{project_id}", "key": "simcore/services/frontend/file-picker", "version": "1.0.0", "label": "pytest_fake_node", + **kwargs, } - node_values.update(**kwargs) - result = await conn.execute( - projects_nodes.insert() - .values( - node_id=f"{new_node_id}", - project_uuid=f"{project_id}", - **node_values, + + node_row = await stack.enter_async_context( + insert_and_get_row_lifespan( + sqlalchemy_async_engine, + table=projects_nodes, + values=node_values, + pk_col=projects_nodes.c.node_id, + pk_value=f"{new_node_id}", ) - .returning(sa.literal_column("*")) ) - row = result.one() - return new_node_id, row._asdict() - return _creator + created_node_entries.append((new_node_id, project_id)) + return new_node_id, node_row + + yield _creator + + # Cleanup is handled automatically by insert_and_get_row_lifespan + print("Deleting ", created_node_entries) From 38da3942211138283a19cb6d040835a01f3a25d9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:11:17 +0200 Subject: [PATCH 047/186] fixes logs --- .../src/simcore_service_director_v2/utils/dags.py | 4 ++-- .../tests/unit/with_dbs/comp_scheduler/test_manager.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dags.py b/services/director-v2/src/simcore_service_director_v2/utils/dags.py index 053ea946946..f0a55669c83 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dags.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dags.py @@ -44,9 +44,9 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph: if node.input_nodes: for input_node_id in node.input_nodes: predecessor_node = workbench.get(f"{input_node_id}") - assert ( + assert ( # nosec predecessor_node - ), f"Node {input_node_id} not found in workbench" # nosec + ), f"Node {input_node_id} not found in workbench" if predecessor_node: dag_graph.add_edge(str(input_node_id), node_id) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py index 134d03f05ec..ee53ef6b1ae 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py @@ -411,7 +411,12 @@ async def test_empty_pipeline_is_not_scheduled( use_on_demand_clusters=False, collection_run_id=fake_collection_run_id, ) - assert len(caplog.records) == 1 - assert "no computational dag defined" in caplog.records[0].message + + warning_log_regs = [ + log_rec for log_rec in caplog.records if log_rec.levelname == "WARNING" + ] + assert len(warning_log_regs) == 1 + assert "no computational dag defined" in warning_log_regs[0].message + await assert_comp_runs_empty(sqlalchemy_async_engine) _assert_scheduler_client_not_called(scheduler_rabbit_client_parser) From a0b71bbf80f4297b2febeff5bcde7182feabb849 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:23:04 +0200 Subject: [PATCH 048/186] doc --- .../modules/db/repositories/projects.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 8d743268b1f..0e21f2e1e2c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -23,7 +23,11 @@ def _project_node_to_node(project_node: ProjectNode) -> Node: # Get all ProjectNode fields except those that don't belong in Node exclude_fields = {"node_id", "required_resources", "created", "modified"} node_data = project_node.model_dump( - exclude=exclude_fields, exclude_none=True, exclude_unset=True + # NOTE: this setup ensures using the defaults provided in Node model when the db does not + # provide them, e.g. `state` + exclude=exclude_fields, + exclude_none=True, + exclude_unset=True, ) return Node.model_validate(node_data) From ab0a810892f1122f72074dfdcb495cc4d8288e6a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 11:11:50 +0200 Subject: [PATCH 049/186] fix: projects states handlers --- .../projects/_project_document_service.py | 37 ++++++++++--------- .../projects/_projects_repository.py | 4 +- .../projects/_projects_repository_legacy.py | 28 ++++++++++---- .../_projects_repository_legacy_utils.py | 21 ++++++++++- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py index dacc6833dc5..c57ce38ae9b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py @@ -30,7 +30,7 @@ from ..resource_manager.registry import get_registry from ..resource_manager.service import list_opened_project_ids from ..socketio._utils import get_socket_server -from . import _projects_repository +from . import _projects_nodes_repository, _projects_repository _logger = logging.getLogger(__name__) @@ -64,26 +64,29 @@ async def _create_project_document_and_increment_version() -> ( - the project document and its version must be kept in sync """ # Get the full project with workbench for document creation - project_with_workbench = await _projects_repository.get_project_with_workbench( + project = await _projects_repository.get_project( app=app, project_uuid=project_uuid ) + project_nodes = await _projects_nodes_repository.get_by_project( + app=app, project_id=project_uuid + ) + workbench = {f"{node_id}": node for node_id, node in project_nodes} + # Create project document project_document = ProjectDocument( - uuid=project_with_workbench.uuid, - workspace_id=project_with_workbench.workspace_id, - name=project_with_workbench.name, - description=project_with_workbench.description, - thumbnail=project_with_workbench.thumbnail, - last_change_date=project_with_workbench.last_change_date, - classifiers=project_with_workbench.classifiers, - dev=project_with_workbench.dev, - quality=project_with_workbench.quality, - workbench=project_with_workbench.workbench, - ui=project_with_workbench.ui, - type=cast(ProjectTypeAPI, project_with_workbench.type), - template_type=cast( - ProjectTemplateType, project_with_workbench.template_type - ), + uuid=project.uuid, + workspace_id=project.workspace_id, + name=project.name, + description=project.description, + thumbnail=project.thumbnail, + last_change_date=project.last_change_date, + classifiers=project.classifiers, + dev=project.dev, + quality=project.quality, + workbench=workbench, + ui=project.ui, + type=cast(ProjectTypeAPI, project.type), + template_type=cast(ProjectTemplateType, project.template_type), ) # Increment document version redis_client_sdk = get_redis_document_manager_client_sdk(app) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 83c357c3657..05d12a4af12 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -122,9 +122,7 @@ async def get_project_with_workbench( project_uuid: ProjectID, ) -> ProjectWithWorkbenchDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - query = sql.select(*PROJECT_DB_COLS, projects.c.workbench).where( - projects.c.uuid == f"{project_uuid}" - ) + query = sql.select(*PROJECT_DB_COLS).where(projects.c.uuid == f"{project_uuid}") result = await conn.execute(query) row = result.one_or_none() if row is None: diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 4b14d356ee2..3af20c038b8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -177,7 +177,7 @@ def _reraise_if_not_unique_uuid_error(err: UniqueViolation): with attempt: async with conn.begin(): project_index = None - project_uuid = ProjectID(f"{insert_values['uuid']}") + project_uuid = ProjectID(insert_values["uuid"]) try: result: ResultProxy = await conn.execute( @@ -296,12 +296,22 @@ async def insert_project( # must be valid uuid try: - ProjectID(str(insert_values.get("uuid"))) + ProjectID(f"{insert_values.get('uuid')}") except ValueError: if force_project_uuid: raise insert_values["uuid"] = f"{uuid1()}" + # extract workbench nodes + workbench: dict[str, Any] = insert_values.pop("workbench", {}) + project_nodes = project_nodes or {} + project_nodes |= { + NodeID(node_id): ProjectNodeCreate( + node_id=NodeID(node_id), **project_workbench_node + ) + for node_id, project_workbench_node in workbench.items() + } + inserted_project = await self._insert_project_in_db( insert_values, force_project_uuid=force_project_uuid, @@ -310,6 +320,8 @@ async def insert_project( project_nodes=project_nodes, ) + inserted_project["workbench"] = workbench + async with self.engine.acquire() as conn: # Returns created project with names as in the project schema user_email = await self._get_user_email(conn, user_id) @@ -365,7 +377,6 @@ def _create_private_workspace_query( private_workspace_query = ( sa.select( *PROJECT_DB_COLS, - projects.c.workbench, projects_to_products.c.product_name, projects_to_folders.c.folder_id, ) @@ -648,8 +659,11 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st # Therefore, if we use this model, it will return those default values, which is not backward-compatible # with the frontend. The frontend would need to check and adapt how it handles default values in # Workbench nodes, which are currently not returned if not set in the DB. - ProjectListAtDB.model_validate(row) - prjs_output.append(dict(row.items())) + prj_dict = dict(row.items()) | { + "workbench": await self._get_workbench(conn, row.uuid), + } + ProjectListAtDB.model_validate(prj_dict) + prjs_output.append(prj_dict) return ( prjs_output, @@ -847,7 +861,7 @@ async def update_project_node_data( extra=get_log_record_extra(user_id=user_id), ): partial_workbench_data: dict[NodeIDStr, Any] = { - NodeIDStr(f"{node_id}"): new_node_data, + f"{node_id}": new_node_data, } return await self._update_project_workbench_with_lock_and_notify( partial_workbench_data, @@ -1021,7 +1035,7 @@ async def add_project_node( ) -> None: # NOTE: permission check is done currently in update_project_workbench! partial_workbench_data: dict[NodeIDStr, Any] = { - NodeIDStr(f"{node.node_id}"): jsonable_encoder( + f"{node.node_id}": jsonable_encoder( old_struct_node, exclude_unset=True, ), diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 348db7d4f74..892388f723a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -21,6 +21,7 @@ from simcore_postgres_database.webserver_models import ( projects, ) +from simcore_service_webserver.projects._nodes_repository import ProjectNodesRepo from sqlalchemy.dialects.postgresql import insert as pg_insert from ..db.models import GroupType, groups, projects_tags, user_to_groups, users @@ -32,7 +33,7 @@ ProjectInvalidUsageError, ProjectNotFoundError, ) -from .models import ProjectDict +from .models import NodesDict, ProjectDict from .utils import find_changed_node_keys logger = logging.getLogger(__name__) @@ -187,6 +188,23 @@ async def _upsert_tags_in_project( .on_conflict_do_nothing() ) + async def _get_workbench( + self, + connection: SAConnection, + project_uuid: str, + ) -> NodesDict: + project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid)) + exclude_fields = {"node_id", "required_resources", "created", "modified"} + workbench: NodesDict = {} + + project_nodes = await project_nodes_repo.list(connection) + for project_node in project_nodes: + node_data = project_node.model_dump( + exclude=exclude_fields, exclude_none=True, exclude_unset=True + ) + workbench[f"{project_node.node_id}"] = Node.model_validate(node_data) + return workbench + async def _get_project( self, connection: SAConnection, @@ -263,6 +281,7 @@ async def _get_project( ) project: dict[str, Any] = dict(project_row.items()) + project["workbench"] = await self._get_workbench(connection, project_uuid) if "tags" not in exclude_foreign: tags = await self._get_tags_by_project( From e86fe272a04f83ac7a2c2e62be69b5f0d34791e9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 11:23:59 +0200 Subject: [PATCH 050/186] fix: typecheck --- .../projects/_projects_repository_legacy_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 892388f723a..517dfb97160 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -197,7 +197,7 @@ async def _get_workbench( exclude_fields = {"node_id", "required_resources", "created", "modified"} workbench: NodesDict = {} - project_nodes = await project_nodes_repo.list(connection) + project_nodes = await project_nodes_repo.list(connection) # type: ignore for project_node in project_nodes: node_data = project_node.model_dump( exclude=exclude_fields, exclude_none=True, exclude_unset=True From ae8016726c1cd2822b4e71ac0f4af857d6810ebe Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 11:28:21 +0200 Subject: [PATCH 051/186] fix: typecheck --- .../projects/_projects_repository_legacy_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 517dfb97160..2335d1b4150 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -8,7 +8,7 @@ import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy -from models_library.projects import ProjectID, ProjectType +from models_library.projects import NodesDict, ProjectID, ProjectType from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeIDStr from models_library.utils.change_case import camel_to_snake, snake_to_camel @@ -33,7 +33,7 @@ ProjectInvalidUsageError, ProjectNotFoundError, ) -from .models import NodesDict, ProjectDict +from .models import ProjectDict from .utils import find_changed_node_keys logger = logging.getLogger(__name__) From 5dde8797eff1bc71dc1a92fd98be3b9d07c1d568 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 11:29:03 +0200 Subject: [PATCH 052/186] fix: relative import --- .../projects/_projects_repository_legacy_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 2335d1b4150..3dd1a81d8c5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -21,10 +21,10 @@ from simcore_postgres_database.webserver_models import ( projects, ) -from simcore_service_webserver.projects._nodes_repository import ProjectNodesRepo from sqlalchemy.dialects.postgresql import insert as pg_insert from ..db.models import GroupType, groups, projects_tags, user_to_groups, users +from ..projects._nodes_repository import ProjectNodesRepo from ..users.exceptions import UserNotFoundError from ..utils import format_datetime from ._projects_repository import PROJECT_DB_COLS From a2a8a275c57f7024b3b410be1a343cdc71e1df55 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 11:32:35 +0200 Subject: [PATCH 053/186] fix: typecheck --- .../projects/_projects_repository_legacy_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 3dd1a81d8c5..ffe3d8382bb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -14,6 +14,7 @@ from models_library.utils.change_case import camel_to_snake, snake_to_camel from pydantic import ValidationError from simcore_postgres_database.models.project_to_groups import project_to_groups +from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo from simcore_postgres_database.webserver_models import ( ProjectTemplateType as ProjectTemplateTypeDB, ) @@ -24,7 +25,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from ..db.models import GroupType, groups, projects_tags, user_to_groups, users -from ..projects._nodes_repository import ProjectNodesRepo from ..users.exceptions import UserNotFoundError from ..utils import format_datetime from ._projects_repository import PROJECT_DB_COLS From f4fd95090e0f715b04f45fa21e34f88e7a9be133 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 12:00:04 +0200 Subject: [PATCH 054/186] fix: extract workbench --- .../projects/_jobs_repository.py | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index dca8d1c91cc..d73f14ec29f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -9,6 +9,7 @@ from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_metadata import projects_metadata +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_postgres_database.utils_repos import ( @@ -169,17 +170,68 @@ async def list_projects_marked_as_jobs( # Step 5: Query to get the total count total_query = sa.select(sa.func.count()).select_from(base_query) - # Step 6: Query to get the paginated list with full selection + # Step 6: Create subquery to aggregate project nodes into workbench structure + workbench_subquery = ( + sa.select( + projects_nodes.c.project_uuid, + sa.func.json_object_agg( + projects_nodes.c.node_id, + sa.func.json_build_object( + "key", + projects_nodes.c.key, + "version", + projects_nodes.c.version, + "label", + projects_nodes.c.label, + "progress", + projects_nodes.c.progress, + "thumbnail", + projects_nodes.c.thumbnail, + "inputAccess", + projects_nodes.c.input_access, + "inputNodes", + projects_nodes.c.input_nodes, + "inputs", + projects_nodes.c.inputs, + "inputsRequired", + projects_nodes.c.inputs_required, + "inputsUnits", + projects_nodes.c.inputs_units, + "outputNodes", + projects_nodes.c.output_nodes, + "outputs", + projects_nodes.c.outputs, + "runHash", + projects_nodes.c.run_hash, + "state", + projects_nodes.c.state, + "parent", + projects_nodes.c.parent, + "bootOptions", + projects_nodes.c.boot_options, + ), + ).label("workbench"), + ) + .group_by(projects_nodes.c.project_uuid) + .subquery() + ) + + # Step 7: Query to get the paginated list with full selection list_query = ( sa.select( *_PROJECT_DB_COLS, - projects.c.workbench, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), base_query.c.job_parent_resource_name, ) .select_from( base_query.join( projects, projects.c.uuid == base_query.c.project_uuid, + ).outerjoin( + workbench_subquery, + projects.c.uuid == workbench_subquery.c.project_uuid, ) ) .order_by( @@ -190,7 +242,7 @@ async def list_projects_marked_as_jobs( .offset(pagination_offset) ) - # Step 7: Execute queries + # Step 8: Execute queries async with pass_or_acquire_connection(self.engine, connection) as conn: total_count = await conn.scalar(total_query) assert isinstance(total_count, int) # nosec From 3a25b02ff564bd7f55ad03952d7d6f9cafd7c531 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 12:47:39 +0200 Subject: [PATCH 055/186] fix: use workbench subquery --- .../utils_projects_nodes.py | 50 ++++++++++++++++++- .../projects/_project_document_service.py | 10 ++-- .../projects/_projects_repository.py | 8 ++- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index f8248193952..23b6eb4b718 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -4,7 +4,7 @@ from typing import Annotated, Any import asyncpg.exceptions # type: ignore[import-untyped] -import sqlalchemy +import sqlalchemy as sa import sqlalchemy.exc from common_library.async_tools import maybe_await from common_library.basic_types import DEFAULT_FACTORY @@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field from simcore_postgres_database.utils_aiosqlalchemy import map_db_exception from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.sql.selectable import Subquery from ._protocols import DBConnection from .aiopg_errors import ForeignKeyViolation, UniqueViolation @@ -81,6 +82,53 @@ class ProjectNode(ProjectNodeCreate): model_config = ConfigDict(from_attributes=True) +def make_workbench_subquery() -> Subquery: + return ( + sa.select( + projects_nodes.c.project_uuid, + sa.func.json_object_agg( + projects_nodes.c.node_id, + sa.func.json_build_object( + "key", + projects_nodes.c.key, + "version", + projects_nodes.c.version, + "label", + projects_nodes.c.label, + "progress", + projects_nodes.c.progress, + "thumbnail", + projects_nodes.c.thumbnail, + "inputAccess", + projects_nodes.c.input_access, + "inputNodes", + projects_nodes.c.input_nodes, + "inputs", + projects_nodes.c.inputs, + "inputsRequired", + projects_nodes.c.inputs_required, + "inputsUnits", + projects_nodes.c.inputs_units, + "outputNodes", + projects_nodes.c.output_nodes, + "outputs", + projects_nodes.c.outputs, + "runHash", + projects_nodes.c.run_hash, + "state", + projects_nodes.c.state, + "parent", + projects_nodes.c.parent, + "bootOptions", + projects_nodes.c.boot_options, + ), + ).label("workbench"), + ) + .group_by(projects_nodes.c.project_uuid) + .subquery() + ) + + @dataclass(frozen=True, kw_only=True) class ProjectNodesRepo: project_uuid: uuid.UUID diff --git a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py index c57ce38ae9b..9ec79ebcda8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_project_document_service.py @@ -30,7 +30,7 @@ from ..resource_manager.registry import get_registry from ..resource_manager.service import list_opened_project_ids from ..socketio._utils import get_socket_server -from . import _projects_nodes_repository, _projects_repository +from . import _projects_repository _logger = logging.getLogger(__name__) @@ -64,13 +64,9 @@ async def _create_project_document_and_increment_version() -> ( - the project document and its version must be kept in sync """ # Get the full project with workbench for document creation - project = await _projects_repository.get_project( + project = await _projects_repository.get_project_with_workbench( app=app, project_uuid=project_uuid ) - project_nodes = await _projects_nodes_repository.get_by_project( - app=app, project_id=project_uuid - ) - workbench = {f"{node_id}": node for node_id, node in project_nodes} # Create project document project_document = ProjectDocument( @@ -83,7 +79,7 @@ async def _create_project_document_and_increment_version() -> ( classifiers=project.classifiers, dev=project.dev, quality=project.quality, - workbench=workbench, + workbench=project.workbench, ui=project.ui, type=cast(ProjectTypeAPI, project.type), template_type=cast(ProjectTemplateType, project.template_type), diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 05d12a4af12..4f0aa83becf 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -15,6 +15,7 @@ from pydantic import NonNegativeInt, PositiveInt from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users +from simcore_postgres_database.utils_projects_nodes import make_workbench_subquery from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -122,7 +123,12 @@ async def get_project_with_workbench( project_uuid: ProjectID, ) -> ProjectWithWorkbenchDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - query = sql.select(*PROJECT_DB_COLS).where(projects.c.uuid == f"{project_uuid}") + query = sql.select( + *PROJECT_DB_COLS, + sa.func.coalesce( + make_workbench_subquery().c.workbench, sa.text("'{}'::json") + ).label("workbench"), + ).where(projects.c.uuid == f"{project_uuid}") result = await conn.execute(query) row = result.one_or_none() if row is None: From a4cbb9a6a6a982027de2b8bc261df83350608dc7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 12:59:49 +0200 Subject: [PATCH 056/186] fix: use workbench subquery --- .../projects/_jobs_repository.py | 49 ++----------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index d73f14ec29f..83c1ff6a97b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -9,9 +9,9 @@ from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_metadata import projects_metadata -from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.models.projects_to_products import projects_to_products +from simcore_postgres_database.utils_projects_nodes import make_workbench_subquery from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -171,50 +171,7 @@ async def list_projects_marked_as_jobs( total_query = sa.select(sa.func.count()).select_from(base_query) # Step 6: Create subquery to aggregate project nodes into workbench structure - workbench_subquery = ( - sa.select( - projects_nodes.c.project_uuid, - sa.func.json_object_agg( - projects_nodes.c.node_id, - sa.func.json_build_object( - "key", - projects_nodes.c.key, - "version", - projects_nodes.c.version, - "label", - projects_nodes.c.label, - "progress", - projects_nodes.c.progress, - "thumbnail", - projects_nodes.c.thumbnail, - "inputAccess", - projects_nodes.c.input_access, - "inputNodes", - projects_nodes.c.input_nodes, - "inputs", - projects_nodes.c.inputs, - "inputsRequired", - projects_nodes.c.inputs_required, - "inputsUnits", - projects_nodes.c.inputs_units, - "outputNodes", - projects_nodes.c.output_nodes, - "outputs", - projects_nodes.c.outputs, - "runHash", - projects_nodes.c.run_hash, - "state", - projects_nodes.c.state, - "parent", - projects_nodes.c.parent, - "bootOptions", - projects_nodes.c.boot_options, - ), - ).label("workbench"), - ) - .group_by(projects_nodes.c.project_uuid) - .subquery() - ) + workbench_subquery = make_workbench_subquery() # Step 7: Query to get the paginated list with full selection list_query = ( @@ -242,7 +199,7 @@ async def list_projects_marked_as_jobs( .offset(pagination_offset) ) - # Step 8: Execute queries + # Step 7: Execute queries async with pass_or_acquire_connection(self.engine, connection) as conn: total_count = await conn.scalar(total_query) assert isinstance(total_count, int) # nosec From e5de7b1c4df4f363f9763c135eaea80308708135 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 13:25:50 +0200 Subject: [PATCH 057/186] fix: get project with workbench --- .../utils_projects_nodes.py | 6 +++++ .../projects/_projects_repository.py | 22 ++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 23b6eb4b718..ee492c41147 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -10,6 +10,7 @@ from common_library.basic_types import DEFAULT_FACTORY from common_library.errors_classes import OsparcErrorMixin from pydantic import BaseModel, ConfigDict, Field +from simcore_postgres_database.models.projects import projects from simcore_postgres_database.utils_aiosqlalchemy import map_db_exception from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql.selectable import Subquery @@ -124,6 +125,11 @@ def make_workbench_subquery() -> Subquery: ), ).label("workbench"), ) + .select_from( + projects_nodes.join( + projects, projects_nodes.c.project_uuid == projects.c.uuid + ) + ) .group_by(projects_nodes.c.project_uuid) .subquery() ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 4f0aa83becf..74762c2902f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -123,12 +123,22 @@ async def get_project_with_workbench( project_uuid: ProjectID, ) -> ProjectWithWorkbenchDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - query = sql.select( - *PROJECT_DB_COLS, - sa.func.coalesce( - make_workbench_subquery().c.workbench, sa.text("'{}'::json") - ).label("workbench"), - ).where(projects.c.uuid == f"{project_uuid}") + workbench_subquery = make_workbench_subquery() + query = ( + sql.select( + *PROJECT_DB_COLS, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), + ) + .select_from( + projects.outerjoin( + workbench_subquery, + projects.c.uuid == workbench_subquery.c.project_uuid, + ) + ) + .where(projects.c.uuid == f"{project_uuid}") + ) result = await conn.execute(query) row = result.one_or_none() if row is None: From 907708b75d906ae31474bec8fd5738c5be216778 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 13:29:03 +0200 Subject: [PATCH 058/186] fix: imports --- .../src/simcore_postgres_database/utils_projects_nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index ee492c41147..e3aa6fafdeb 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -10,15 +10,15 @@ from common_library.basic_types import DEFAULT_FACTORY from common_library.errors_classes import OsparcErrorMixin from pydantic import BaseModel, ConfigDict, Field -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.utils_aiosqlalchemy import map_db_exception from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql.selectable import Subquery from ._protocols import DBConnection from .aiopg_errors import ForeignKeyViolation, UniqueViolation +from .models.projects import projects from .models.projects_node_to_pricing_unit import projects_node_to_pricing_unit from .models.projects_nodes import projects_nodes +from .utils_aiosqlalchemy import map_db_exception # From 315653193f79e07f812548a9b4d9480ec7e54cf2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 13:41:03 +0200 Subject: [PATCH 059/186] fix: inputs required nullability --- packages/models-library/src/models_library/projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index a75ee6d1895..92afc4faf1c 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -192,7 +192,7 @@ class Node(BaseModel): ] = DEFAULT_FACTORY inputs_required: Annotated[ - list[InputID], + list[InputID] | None, Field( default_factory=list, description="Defines inputs that are required in order to run the service", From 7b07b66aee5eedac930a692d2ac76db826f7fddd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 13:47:11 +0200 Subject: [PATCH 060/186] fix: typecheck --- .../simcore_service_webserver/projects/_projects_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 96ea59ba24e..e98cc38c4d3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -681,8 +681,9 @@ def _check_required_input(required_input_key: KeyIDStr) -> None: if output_entry is None: unset_outputs_in_upstream.append((source_output_key, source_node.label)) - for required_input in node.inputs_required: - _check_required_input(required_input) + if node.inputs_required: + for required_input in node.inputs_required: + _check_required_input(required_input) node_with_required_inputs = node.label if unset_required_inputs: From 0a3d8d73a65ada0c9b792e1ae259a4db71092500 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 14:20:23 +0200 Subject: [PATCH 061/186] fix: project cancellation --- .../_projects_repository_legacy_utils.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index ffe3d8382bb..7056bee3e59 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -14,7 +14,10 @@ from models_library.utils.change_case import camel_to_snake, snake_to_camel from pydantic import ValidationError from simcore_postgres_database.models.project_to_groups import project_to_groups -from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo +from simcore_postgres_database.utils_projects_nodes import ( + ProjectNodesRepo, + make_workbench_subquery, +) from simcore_postgres_database.webserver_models import ( ProjectTemplateType as ProjectTemplateTypeDB, ) @@ -239,15 +242,23 @@ async def _get_project( .group_by(project_to_groups.c.project_uuid) ).subquery("access_rights_subquery") + workbench_subquery = make_workbench_subquery() + query = ( sa.select( *PROJECT_DB_COLS, users.c.primary_gid.label("trashed_by_primary_gid"), access_rights_subquery.c.access_rights, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), ) .select_from( - projects.join(access_rights_subquery, isouter=True).outerjoin( - users, projects.c.trashed_by == users.c.id + projects.join(access_rights_subquery, isouter=True) + .outerjoin(users, projects.c.trashed_by == users.c.id) + .outerjoin( + workbench_subquery, + projects.c.uuid == workbench_subquery.c.project_uuid, ) ) .where( @@ -281,7 +292,6 @@ async def _get_project( ) project: dict[str, Any] = dict(project_row.items()) - project["workbench"] = await self._get_workbench(connection, project_uuid) if "tags" not in exclude_foreign: tags = await self._get_tags_by_project( From e327c6bbbff539fa9c580aba8aed44ba3810a5f8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:21:31 +0200 Subject: [PATCH 062/186] model_dump_as_node --- .../src/models_library/projects_nodes_io.py | 3 +-- .../utils_projects_nodes.py | 16 ++++++++++++++++ .../modules/db/repositories/projects.py | 11 +---------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 90fdf141278..d4708582dea 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -30,10 +30,9 @@ UUID_RE, ) -NodeID = UUID - UUIDStr: TypeAlias = Annotated[str, StringConstraints(pattern=UUID_RE)] +NodeID: TypeAlias = UUID NodeIDStr: TypeAlias = UUIDStr LocationID: TypeAlias = int diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index e3aa6fafdeb..099606b8a21 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -82,6 +82,22 @@ class ProjectNode(ProjectNodeCreate): model_config = ConfigDict(from_attributes=True) + def model_dump_as_node(self) -> dict[str, Any]: + """Converts a ProjectNode from the database to a Node model for the API. + + Handles field mapping and excludes database-specific fields that are not + part of the Node model. + """ + # Get all ProjectNode fields except those that don't belong in Node + exclude_fields = {"node_id", "required_resources", "created", "modified"} + return self.model_dump( + # NOTE: this setup ensures using the defaults provided in Node model when the db does not + # provide them, e.g. `state` + exclude=exclude_fields, + exclude_none=True, + exclude_unset=True, + ) + def make_workbench_subquery() -> Subquery: return ( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 0e21f2e1e2c..4be5a9cf13e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -20,16 +20,7 @@ def _project_node_to_node(project_node: ProjectNode) -> Node: Handles field mapping and excludes database-specific fields that are not part of the Node model. """ - # Get all ProjectNode fields except those that don't belong in Node - exclude_fields = {"node_id", "required_resources", "created", "modified"} - node_data = project_node.model_dump( - # NOTE: this setup ensures using the defaults provided in Node model when the db does not - # provide them, e.g. `state` - exclude=exclude_fields, - exclude_none=True, - exclude_unset=True, - ) - + node_data = project_node.model_dump_as_node() return Node.model_validate(node_data) From ded3e8dc7715a35a2e74bdb89075309f73ba3c4d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:22:38 +0200 Subject: [PATCH 063/186] fixes fixture --- .../helpers/webserver_projects.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 541fac8dacb..2e90952f983 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -12,8 +12,10 @@ from aiohttp import web from aiohttp.test_utils import TestClient from common_library.dict_tools import remap_keys +from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID from models_library.services_resources import ServiceResourcesDictHelpers +from pydantic import TypeAdapter from simcore_postgres_database.utils_projects_nodes import ProjectNodeCreate from simcore_service_webserver.projects._groups_repository import ( update_or_insert_project_group, @@ -75,7 +77,12 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] - workbench: dict[str, Any] = project_data.pop("workbench", {}) + workbench = TypeAdapter(dict[NodeID, Node]).validate_python( + project_data.pop("workbench", {}) + ) + fake_required_resources: dict[str, Any] = ServiceResourcesDictHelpers.model_config[ + "json_schema_extra" + ]["examples"][0] project_created = await db.insert_project( project_data, @@ -85,14 +92,14 @@ async def create_project( force_as_template=as_template, # NOTE: fake initial resources until more is needed project_nodes={ - NodeID(node_id): ProjectNodeCreate( - node_id=NodeID(node_id), - required_resources=ServiceResourcesDictHelpers.model_config[ - "json_schema_extra" - ]["examples"][0], - **node_info, + node_id: ProjectNodeCreate( + node_id=node_id, + required_resources=fake_required_resources, + **node_model.model_dump( + by_alias=False, exclude_unset=True, mode="json" + ), ) - for node_id, node_info in workbench.items() + for node_id, node_model in workbench.items() }, ) From 9c23c06b0158d53937f919a1e6673788d024837f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:46:46 +0200 Subject: [PATCH 064/186] cleanup --- .../src/models_library/projects_nodes.py | 12 +++++++++--- .../unit/with_dbs/02/test_projects_ports_handlers.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 92afc4faf1c..488dbfa4a32 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -231,15 +231,20 @@ class Node(BaseModel): Field(default_factory=dict, description="values of output properties"), ] = DEFAULT_FACTORY - output_node: Annotated[bool | None, Field(deprecated=True, alias="outputNode")] = ( - None # <-- (DEPRECATED) Can be removed - ) + output_node: Annotated[ + bool | None, + Field( + deprecated=True, + alias="outputNode", + ), + ] = None # <-- (DEPRECATED) Can be removed output_nodes: Annotated[ # <-- (DEPRECATED) Can be removed list[NodeID] | None, Field( description="Used in group-nodes. Node IDs of those connected to the output", alias="outputNodes", + deprecated=True, ), ] = None @@ -247,6 +252,7 @@ class Node(BaseModel): NodeID | None, Field( description="Parent's (group-nodes') node ID s. Used to group", + deprecated=True, ), ] = None diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py index e8a1536c5e4..c3c3b8308d9 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py @@ -74,7 +74,7 @@ def mock_directorv2_service_api_responses( return aioresponses_mocker -@pytest.mark.acceptance_test() +@pytest.mark.acceptance_test @pytest.mark.parametrize( "user_role,expected", [ From 994cc0d7e03d694c5eca5f7cef266c73f8db7126 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:11:01 +0200 Subject: [PATCH 065/186] fixes workbench retrieval --- .../projects/_projects_repository_legacy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 3af20c038b8..c473bd74123 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -320,7 +320,10 @@ async def insert_project( project_nodes=project_nodes, ) - inserted_project["workbench"] = workbench + inserted_project["workbench"] = { + f"{node_id}": project_node.model_dump_as_node() + for node_id, project_node in project_nodes.items() + } async with self.engine.acquire() as conn: # Returns created project with names as in the project schema From 2ce1cc618e0dceef06e2c3d8e421474680dbff4c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:23:46 +0200 Subject: [PATCH 066/186] fixes insert batch --- .../utils_projects_nodes.py | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 099606b8a21..1cf8039d510 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -73,6 +73,22 @@ class ProjectNodeCreate(BaseModel): def get_field_names(cls, *, exclude: set[str]) -> set[str]: return cls.model_fields.keys() - exclude + def model_dump_as_node(self) -> dict[str, Any]: + """Converts a ProjectNode from the database to a Node model for the API. + + Handles field mapping and excludes database-specific fields that are not + part of the Node model. + """ + # Get all ProjectNode fields except those that don't belong in Node + exclude_fields = {"node_id", "required_resources"} + return self.model_dump( + # NOTE: this setup ensures using the defaults provided in Node model when the db does not + # provide them, e.g. `state` + exclude=exclude_fields, + exclude_none=True, + exclude_unset=True, + ) + model_config = ConfigDict(frozen=True) @@ -173,17 +189,18 @@ async def add( """ if not nodes: return [] + + values = [ + { + "project_uuid": f"{self.project_uuid}", + **node.model_dump(mode="json"), + } + for node in nodes + ] + insert_stmt = ( projects_nodes.insert() - .values( - [ - { - "project_uuid": f"{self.project_uuid}", - **node.model_dump(exclude_unset=True, mode="json"), - } - for node in nodes - ] - ) + .values(values) .returning( *[ c @@ -199,14 +216,17 @@ async def add( rows = await maybe_await(result.fetchall()) assert isinstance(rows, list) # nosec return [ProjectNode.model_validate(r) for r in rows] + except ForeignKeyViolation as exc: # this happens when the project does not exist, as we first check the node exists raise ProjectNodesProjectNotFoundError( project_uuid=self.project_uuid ) from exc + except UniqueViolation as exc: # this happens if the node already exists on creation raise ProjectNodesDuplicateNodeError from exc + except sqlalchemy.exc.IntegrityError as exc: raise map_db_exception( exc, From a9b80b20f325c7b957db71790501afc757f7e821 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:45:26 +0200 Subject: [PATCH 067/186] no need to restor workbench --- .../src/pytest_simcore/helpers/webserver_projects.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 2e90952f983..5ac046bb1eb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -135,7 +135,6 @@ async def create_project( rename={"trashed": "trashedAt"}, ) - project_created["workbench"] = workbench # NOTE: restore workbench return project_created From 087ec304394c653b696cad10e44e5493256e67ed Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Aug 2025 19:32:34 +0200 Subject: [PATCH 068/186] remove define `_projects_service.get_project_for_user` --- .../projects/_controller/ports_rest.py | 74 ++++++++++--------- .../projects/_nodes_repository.py | 29 ++++++-- .../projects/_nodes_service.py | 9 +++ 3 files changed, 72 insertions(+), 40 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py index f9336f6e7c3..2a50d0d379a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py @@ -8,10 +8,8 @@ ProjectOutputGet, ) from models_library.basic_types import KeyIDStr -from models_library.projects import ProjectID from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID -from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.services_io import JsonSchemaDict from pydantic import BaseModel, Field, TypeAdapter @@ -26,29 +24,14 @@ from ...models import ClientSessionHeaderParams from ...security.decorators import permission_required from ...utils_aiohttp import envelope_json_response -from .. import _ports_service, _projects_service -from .._access_rights_service import check_user_project_permission +from .. import _access_rights_service, _nodes_service, _ports_service from .._projects_repository_legacy import ProjectDBAPI -from ..models import ProjectDict from ._rest_exceptions import handle_plugin_requests_exceptions from ._rest_schemas import AuthenticatedRequestContext, ProjectPathParams log = logging.getLogger(__name__) -async def _get_validated_workbench_model( - app: web.Application, project_id: ProjectID, user_id: UserID -) -> dict[NodeID, Node]: - project: ProjectDict = await _projects_service.get_project_for_user( - app, - project_uuid=f"{project_id}", - user_id=user_id, - include_state=False, - ) - - return TypeAdapter(dict[NodeID, Node]).validate_python(project["workbench"]) - - routes = web.RouteTableDef() @@ -66,9 +49,17 @@ async def get_project_inputs(request: web.Request) -> web.Response: assert request.app # nosec - workbench = await _get_validated_workbench_model( - app=request.app, project_id=path_params.project_id, user_id=req_ctx.user_id + await _access_rights_service.check_user_project_permission( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + permission="read", + ) + workbench = await _nodes_service.get_project_nodes_map( + app=request.app, project_id=path_params.project_id ) + inputs: dict[NodeID, Any] = _ports_service.get_project_inputs(workbench) return envelope_json_response( @@ -94,8 +85,15 @@ async def update_project_inputs(request: web.Request) -> web.Response: assert request.app # nosec - workbench = await _get_validated_workbench_model( - app=request.app, project_id=path_params.project_id, user_id=req_ctx.user_id + await _access_rights_service.check_user_project_permission( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + permission="write", # because we are updating inputs later + ) + workbench = await _nodes_service.get_project_nodes_map( + app=request.app, project_id=path_params.project_id ) current_inputs: dict[NodeID, Any] = _ports_service.get_project_inputs(workbench) @@ -112,14 +110,6 @@ async def update_project_inputs(request: web.Request) -> web.Response: ) # patch workbench - await check_user_project_permission( - request.app, - project_id=path_params.project_id, - user_id=req_ctx.user_id, - product_name=req_ctx.product_name, - permission="write", - ) - assert db # nosec updated_project, _ = await db.update_project_multiple_node_data( user_id=req_ctx.user_id, @@ -159,9 +149,17 @@ async def get_project_outputs(request: web.Request) -> web.Response: assert request.app # nosec - workbench = await _get_validated_workbench_model( - app=request.app, project_id=path_params.project_id, user_id=req_ctx.user_id + await _access_rights_service.check_user_project_permission( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + permission="read", ) + workbench = await _nodes_service.get_project_nodes_map( + app=request.app, project_id=path_params.project_id + ) + outputs: dict[NodeID, Any] = await _ports_service.get_project_outputs( request.app, project_id=path_params.project_id, workbench=workbench ) @@ -206,10 +204,16 @@ async def list_project_metadata_ports(request: web.Request) -> web.Response: assert request.app # nosec - workbench = await _get_validated_workbench_model( - app=request.app, project_id=path_params.project_id, user_id=req_ctx.user_id + await _access_rights_service.check_user_project_permission( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + permission="read", + ) + workbench = await _nodes_service.get_project_nodes_map( + app=request.app, project_id=path_params.project_id ) - return envelope_json_response( [ ProjectMetadataPortGet( diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index f55d22fac82..2db16905f19 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -1,9 +1,12 @@ from aiohttp import web from models_library.projects import ProjectID +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID from models_library.services_types import ServiceKey, ServiceVersion +from pydantic import TypeAdapter from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo - -from ..db.plugin import get_database_engine_legacy +from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from simcore_service_webserver.db.plugin import get_asyncpg_engine async def get_project_nodes_services( @@ -11,8 +14,24 @@ async def get_project_nodes_services( ) -> list[tuple[ServiceKey, ServiceVersion]]: repo = ProjectNodesRepo(project_uuid=project_uuid) - async with get_database_engine_legacy(app).acquire() as conn: - nodes = await repo.list(conn) + async with pass_or_acquire_connection(get_asyncpg_engine(app)) as conn: + project_nodes = await repo.list(conn) # removes duplicates by preserving order - return list(dict.fromkeys((node.key, node.version) for node in nodes)) + return list(dict.fromkeys((node.key, node.version) for node in project_nodes)) + + +async def get_project_nodes_map( + app: web.Application, *, project_id: ProjectID +) -> dict[NodeID, Node]: + + repo = ProjectNodesRepo(project_uuid=project_id) + + async with pass_or_acquire_connection(get_asyncpg_engine(app)) as conn: + project_nodes = await repo.list(conn) + + workbench = { + project_node.node_id: project_node.model_dump_as_node() + for project_node in project_nodes + } + return TypeAdapter(dict[NodeID, Node]).validate_python(workbench) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py index 0206e1315cc..fad1f355d0a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py @@ -81,6 +81,15 @@ async def get_project_nodes_services( ) +async def get_project_nodes_map( + app: web.Application, *, project_id: ProjectID +) -> dict[NodeID, Node]: + """ + Returns a map of node_id to Node for the given project_id which used to be called the project's `workbench` + """ + return await _nodes_repository.get_project_nodes_map(app, project_id=project_id) + + # # PREVIEWS # From 7b2639c4a196914e2661bec32f55b45be318e557 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 22:57:31 +0200 Subject: [PATCH 069/186] fix: validate workbench against nodes --- .../projects/_projects_repository_legacy.py | 26 +++++++++++++++---- .../tests/unit/with_dbs/03/test_project_db.py | 1 - 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index c473bd74123..2185658136f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -305,9 +305,28 @@ async def insert_project( # extract workbench nodes workbench: dict[str, Any] = insert_values.pop("workbench", {}) project_nodes = project_nodes or {} + + # Get valid ProjectNodeCreate fields, excluding node_id since it's set separately + valid_fields = ProjectNodeCreate.get_field_names(exclude={"node_id"}) + + # Mapping from camelCase (workbench) to snake_case (ProjectNodeCreate) + field_mapping = { + "inputAccess": "input_access", + "inputNodes": "input_nodes", + "inputsUnits": "inputs_units", + "outputNodes": "output_nodes", + "runHash": "run_hash", + "bootOptions": "boot_options", + } + project_nodes |= { NodeID(node_id): ProjectNodeCreate( - node_id=NodeID(node_id), **project_workbench_node + node_id=NodeID(node_id), + **{ + str(field_mapping.get(field, field)): value + for field, value in project_workbench_node.items() + if field_mapping.get(field, field) in valid_fields + }, ) for node_id, project_workbench_node in workbench.items() } @@ -320,10 +339,7 @@ async def insert_project( project_nodes=project_nodes, ) - inserted_project["workbench"] = { - f"{node_id}": project_node.model_dump_as_node() - for node_id, project_node in project_nodes.items() - } + inserted_project["workbench"] = workbench async with self.engine.acquire() as conn: # Returns created project with names as in the project schema diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index e8731a32fb0..9b0d74109b7 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -168,7 +168,6 @@ def _assert_project_db_row( "description": project["description"], "thumbnail": project["thumbnail"], "prj_owner": None, - "workbench": project["workbench"], "published": False, "dev": project["dev"], "classifiers": project["classifiers"], From 1770502e65de5e4902e1a570e97896d0b928d470 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 5 Aug 2025 23:05:07 +0200 Subject: [PATCH 070/186] fix: return project with workbench when creating --- .../src/pytest_simcore/helpers/webserver_projects.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 5ac046bb1eb..ffe7070da5e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -77,9 +77,9 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] - workbench = TypeAdapter(dict[NodeID, Node]).validate_python( - project_data.pop("workbench", {}) - ) + raw_workbench = project_data.pop("workbench", {}) + + workbench = TypeAdapter(dict[NodeID, Node]).validate_python(raw_workbench) fake_required_resources: dict[str, Any] = ServiceResourcesDictHelpers.model_config[ "json_schema_extra" ]["examples"][0] @@ -134,7 +134,7 @@ async def create_project( project_created, rename={"trashed": "trashedAt"}, ) - + project_created["workbench"] = raw_workbench return project_created From 5b213c00598bdd1ef51898a78c0cf1199dfee243 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Aug 2025 00:13:06 +0200 Subject: [PATCH 071/186] fix: create project fixture --- .../helpers/webserver_projects.py | 47 +++++++++++++------ .../projects/_controller/projects_rest.py | 8 +++- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index ffe7070da5e..b8973f70096 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -12,10 +12,8 @@ from aiohttp import web from aiohttp.test_utils import TestClient from common_library.dict_tools import remap_keys -from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID from models_library.services_resources import ServiceResourcesDictHelpers -from pydantic import TypeAdapter from simcore_postgres_database.utils_projects_nodes import ProjectNodeCreate from simcore_service_webserver.projects._groups_repository import ( update_or_insert_project_group, @@ -77,30 +75,49 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] - raw_workbench = project_data.pop("workbench", {}) + raw_workbench: dict[str, Any] = project_data.pop("workbench", {}) + for raw_node in raw_workbench.values(): + if "position" in raw_node: + del raw_node["position"] + + # Get valid ProjectNodeCreate fields, excluding node_id since it's set separately + valid_fields = ProjectNodeCreate.get_field_names(exclude={"node_id"}) + + # Mapping from camelCase (workbench) to snake_case (ProjectNodeCreate) + field_mapping = { + "inputAccess": "input_access", + "inputNodes": "input_nodes", + "inputsUnits": "inputs_units", + "outputNodes": "output_nodes", + "runHash": "run_hash", + "bootOptions": "boot_options", + } - workbench = TypeAdapter(dict[NodeID, Node]).validate_python(raw_workbench) fake_required_resources: dict[str, Any] = ServiceResourcesDictHelpers.model_config[ "json_schema_extra" ]["examples"][0] + project_nodes = { + NodeID(node_id): ProjectNodeCreate( + node_id=NodeID(node_id), + # NOTE: fake initial resources until more is needed + required_resources=fake_required_resources, + **{ + str(field_mapping.get(field, field)): value + for field, value in raw_node.items() + if field_mapping.get(field, field) in valid_fields + }, + ) + for node_id, raw_node in raw_workbench.items() + } + project_created = await db.insert_project( project_data, user_id, product_name=product_name, force_project_uuid=force_uuid, force_as_template=as_template, - # NOTE: fake initial resources until more is needed - project_nodes={ - node_id: ProjectNodeCreate( - node_id=node_id, - required_resources=fake_required_resources, - **node_model.model_dump( - by_alias=False, exclude_unset=True, mode="json" - ), - ) - for node_id, node_model in workbench.items() - }, + project_nodes=project_nodes, ) if params_override and ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 538644fc2a2..6fa2c68a632 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -249,7 +249,9 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).data( + exclude_none=True, exclude_unset=True + ) return envelope_json_response(data) @@ -282,7 +284,9 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).data( + exclude_none=True, exclude_unset=True + ) return envelope_json_response(data) From ea6775167d8c0c2680c559aa612b9071f0e14e0d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Aug 2025 09:34:42 +0200 Subject: [PATCH 072/186] fix: workbench when share workpace --- .../projects/_projects_repository_legacy.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 2185658136f..08b7a52d42e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -61,6 +61,7 @@ ProjectNode, ProjectNodeCreate, ProjectNodesRepo, + make_workbench_subquery, ) from simcore_postgres_database.webserver_models import ( ProjectType, @@ -451,10 +452,14 @@ def _create_shared_workspace_query( .group_by(workspaces_access_rights.c.workspace_id) ).subquery("my_workspace_access_rights_subquery") + workbench_subquery = make_workbench_subquery() + shared_workspace_query = ( sa.select( *PROJECT_DB_COLS, - projects.c.workbench, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), projects_to_products.c.product_name, projects_to_folders.c.folder_id, ) @@ -473,6 +478,10 @@ def _create_shared_workspace_query( ), isouter=True, ) + .outerjoin( + workbench_subquery, + workbench_subquery.c.project_uuid == projects.c.uuid, + ) ) .where(projects_to_products.c.product_name == product_name) ) From 92956025e3ee74eb40a8af46ed5b375130b41745 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Aug 2025 10:20:09 +0200 Subject: [PATCH 073/186] fix: workbench when getting workspaces --- .../projects/_projects_repository_legacy.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 08b7a52d42e..07917a6d07f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -394,9 +394,14 @@ def _create_private_workspace_query( .group_by(project_to_groups.c.project_uuid) ).subquery("my_access_rights_subquery") + workbench_subquery = make_workbench_subquery() + private_workspace_query = ( sa.select( *PROJECT_DB_COLS, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), projects_to_products.c.product_name, projects_to_folders.c.folder_id, ) @@ -411,6 +416,10 @@ def _create_private_workspace_query( ), isouter=True, ) + .outerjoin( + workbench_subquery, + workbench_subquery.c.project_uuid == projects.c.uuid, + ) ) .where( (projects.c.workspace_id.is_(None)) # <-- Private workspace From 58549f7deb21e26dc43f78f9a9d5323c7bc64f72 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Aug 2025 10:42:55 +0200 Subject: [PATCH 074/186] fix: include None --- .../projects/_controller/projects_rest.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 6fa2c68a632..538644fc2a2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -249,9 +249,7 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data( - exclude_none=True, exclude_unset=True - ) + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return envelope_json_response(data) @@ -284,9 +282,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data( - exclude_none=True, exclude_unset=True - ) + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return envelope_json_response(data) From 41d4182fba5381c4ba56b7e1fe970e305d7deeca Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Aug 2025 12:36:44 +0200 Subject: [PATCH 075/186] fix: extract only not nulls --- .../utils_projects_nodes.py | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 1cf8039d510..dcc05075689 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -121,39 +121,41 @@ def make_workbench_subquery() -> Subquery: projects_nodes.c.project_uuid, sa.func.json_object_agg( projects_nodes.c.node_id, - sa.func.json_build_object( - "key", - projects_nodes.c.key, - "version", - projects_nodes.c.version, - "label", - projects_nodes.c.label, - "progress", - projects_nodes.c.progress, - "thumbnail", - projects_nodes.c.thumbnail, - "inputAccess", - projects_nodes.c.input_access, - "inputNodes", - projects_nodes.c.input_nodes, - "inputs", - projects_nodes.c.inputs, - "inputsRequired", - projects_nodes.c.inputs_required, - "inputsUnits", - projects_nodes.c.inputs_units, - "outputNodes", - projects_nodes.c.output_nodes, - "outputs", - projects_nodes.c.outputs, - "runHash", - projects_nodes.c.run_hash, - "state", - projects_nodes.c.state, - "parent", - projects_nodes.c.parent, - "bootOptions", - projects_nodes.c.boot_options, + sa.func.json_strip_nulls( + sa.func.json_build_object( + "key", + projects_nodes.c.key, + "version", + projects_nodes.c.version, + "label", + projects_nodes.c.label, + "progress", + projects_nodes.c.progress, + "thumbnail", + projects_nodes.c.thumbnail, + "inputAccess", + projects_nodes.c.input_access, + "inputNodes", + projects_nodes.c.input_nodes, + "inputs", + projects_nodes.c.inputs, + "inputsRequired", + projects_nodes.c.inputs_required, + "inputsUnits", + projects_nodes.c.inputs_units, + "outputNodes", + projects_nodes.c.output_nodes, + "outputs", + projects_nodes.c.outputs, + "runHash", + projects_nodes.c.run_hash, + "state", + projects_nodes.c.state, + "parent", + projects_nodes.c.parent, + "bootOptions", + projects_nodes.c.boot_options, + ), ), ).label("workbench"), ) From d4b90fff6dcc26bd8a3ce47397e53f0253ddc715 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 13:27:37 +0200 Subject: [PATCH 076/186] Add trigger/function when updating project nodes --- ...pdate_project_last_changed_date_column_.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py new file mode 100644 index 00000000000..47e49819e40 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py @@ -0,0 +1,63 @@ +"""Update project last_changed_date column when a node is created, updated or deleted. + +Revision ID: c9c165644731 +Revises: 201aa37f4d9a +Create Date: 2025-08-07 10:26:37.577990+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c9c165644731" +down_revision = "201aa37f4d9a" +branch_labels = None +depends_on = None + + +update_projects_last_changed_date = sa.DDL( + """ +CREATE OR REPLACE FUNCTION update_projects_last_changed_date() +RETURNS TRIGGER AS $$ +DECLARE + project_uuid VARCHAR; +BEGIN + IF TG_OP = 'DELETE' THEN + project_uuid := OLD.project_uuid; + ELSE + project_uuid := NEW.project_uuid; + END IF; + + UPDATE projects + SET last_changed_date = NOW() + WHERE uuid = project_uuid; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +""" +) + + +projects_nodes_changed = sa.DDL( + """ +DROP TRIGGER IF EXISTS projects_nodes_changed on projects_nodes; +CREATE TRIGGER projects_nodes_changed +AFTER INSERT OR UPDATE OR DELETE ON projects_nodes +FOR EACH ROW +EXECUTE FUNCTION update_projects_last_changed_date(); +""" +) + + +def upgrade(): + op.execute(update_projects_last_changed_date) + op.execute(projects_nodes_changed) + + +def downgrade(): + op.execute(sa.DDL("DROP FUNCTION IF EXISTS update_projects_last_changed_date();")) + op.execute( + sa.DDL("DROP TRIGGER IF EXISTS projects_nodes_changed ON projects_nodes;") + ) From d95331f92e4ac23a6cde05c164b037d496225b3c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 13:36:01 +0200 Subject: [PATCH 077/186] Fix script --- ...44731_update_project_last_changed_date_column_.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py index 47e49819e40..d98d135b1d6 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py @@ -16,9 +16,9 @@ depends_on = None -update_projects_last_changed_date = sa.DDL( +update_projects_last_change_date = sa.DDL( """ -CREATE OR REPLACE FUNCTION update_projects_last_changed_date() +CREATE OR REPLACE FUNCTION update_projects_last_change_date() RETURNS TRIGGER AS $$ DECLARE project_uuid VARCHAR; @@ -30,7 +30,7 @@ END IF; UPDATE projects - SET last_changed_date = NOW() + SET last_change_date = NOW() WHERE uuid = project_uuid; RETURN NULL; @@ -46,18 +46,18 @@ CREATE TRIGGER projects_nodes_changed AFTER INSERT OR UPDATE OR DELETE ON projects_nodes FOR EACH ROW -EXECUTE FUNCTION update_projects_last_changed_date(); +EXECUTE FUNCTION update_projects_last_change_date(); """ ) def upgrade(): - op.execute(update_projects_last_changed_date) + op.execute(update_projects_last_change_date) op.execute(projects_nodes_changed) def downgrade(): - op.execute(sa.DDL("DROP FUNCTION IF EXISTS update_projects_last_changed_date();")) + op.execute(sa.DDL("DROP FUNCTION IF EXISTS update_projects_last_change_date();")) op.execute( sa.DDL("DROP TRIGGER IF EXISTS projects_nodes_changed ON projects_nodes;") ) From a7972220cdc30f3f510b111dc55800d211683664 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 14:15:51 +0200 Subject: [PATCH 078/186] fix: delete project_node --- .../projects/_projects_nodes_repository.py | 16 ++++++++ .../projects/_projects_service.py | 40 +++++++++++++------ .../02/test_projects_nodes_handler.py | 11 +++-- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 2f7ebe79fc7..6d09ae6a163 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -43,6 +43,22 @@ ] +async def delete( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, + node_id: NodeID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + projects_nodes.delete().where( + (projects_nodes.c.project_uuid == f"{project_id}") + & (projects_nodes.c.node_id == f"{node_id}") + ) + ) + + async def get( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 8b05df85c19..f4abce96450 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -998,16 +998,14 @@ async def start_project_node( project_id: ProjectID, node_id: NodeID, ): - project = await get_project_for_user(request.app, f"{project_id}", user_id) - workbench = project.get("workbench", {}) - if not workbench.get(f"{node_id}"): - raise NodeNotFoundError(project_uuid=f"{project_id}", node_uuid=f"{node_id}") - node_details = Node.model_construct(**workbench[f"{node_id}"]) + node = await _projects_nodes_repository.get( + request.app, project_id=project_id, node_id=node_id + ) await _start_dynamic_service( request, - service_key=node_details.key, - service_version=node_details.version, + service_key=node.key, + service_version=node.version, product_name=product_name, product_api_base_url=product_api_base_url, user_id=user_id, @@ -1090,12 +1088,12 @@ async def delete_project_node( fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], ) - # remove the node from the db - db_legacy: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - assert db_legacy # nosec - await db_legacy.remove_project_node( - user_id, project_uuid, NodeID(node_uuid), client_session_id=client_session_id + await _projects_nodes_repository.delete( + request.app, + project_id=project_uuid, + node_id=NodeID(node_uuid), ) + # also ensure the project is updated by director-v2 since services product_name = products_web.get_product_name(request) await director_v2_service.create_or_update_pipeline( @@ -1105,6 +1103,24 @@ async def delete_project_node( request.app, project_id=project_uuid ) + ( + project_document, + document_version, + ) = await create_project_document_and_increment_version(request.app, project_uuid) + + user_primary_gid = await users_service.get_user_primary_group_id( + request.app, user_id + ) + + await notify_project_document_updated( + app=request.app, + project_id=project_uuid, + user_primary_gid=user_primary_gid, + client_session_id=client_session_id, + version=document_version, + document=project_document, + ) + async def update_project_linked_product( app: web.Application, project_id: ProjectID, product_name: str diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index ef3fd0c98bd..5de1a8217ea 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -49,6 +49,7 @@ from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from settings_library.redis import RedisSettings from simcore_postgres_database.models.projects import projects as projects_db_model +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects._controller.nodes_rest import ( _ProjectNodePreview, @@ -756,13 +757,11 @@ async def test_delete_node( # ensure the node is gone with postgres_db.connect() as conn: result = conn.execute( - sa.select(projects_db_model.c.workbench).where( - projects_db_model.c.uuid == user_project["uuid"] - ) + sa.select(sa.literal(1)) + .where(projects_nodes.c.node_id == node_id) + .limit(1) ) - assert result - workbench = result.one()[projects_db_model.c.workbench] - assert node_id not in workbench + assert result.scalar() is None @pytest.mark.parametrize(*standard_role_response(), ids=str) From bf033d26de1b89c04609e30e2cd4eccc36914458 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 14:19:11 +0200 Subject: [PATCH 079/186] remove unused --- .../projects/_projects_repository_legacy.py | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 07917a6d07f..3ac1656c6f2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -1089,29 +1089,7 @@ async def add_project_node( async with self.engine.acquire() as conn: await project_nodes_repo.add(conn, nodes=[node]) - async def remove_project_node( - self, - user_id: UserID, - project_id: ProjectID, - node_id: NodeID, - client_session_id: ClientSessionID | None, - ) -> None: - # NOTE: permission check is done currently in update_project_workbench! - partial_workbench_data: dict[NodeIDStr, Any] = { - NodeIDStr(f"{node_id}"): None, - } - await self._update_project_workbench_with_lock_and_notify( - partial_workbench_data, - user_id=user_id, - project_uuid=project_id, - allow_workbench_changes=True, - client_session_id=client_session_id, - ) - project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) - async with self.engine.acquire() as conn: - await project_nodes_repo.delete(conn, node_id=node_id) - - async def get_project_node( # NOTE: Not all Node data are here yet; they are in the workbench of a Project, waiting to be moved here. + async def get_project_node( self, project_id: ProjectID, node_id: NodeID ) -> ProjectNode: project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) From c9c6a1ea66bd787cc9bd894c3857dc56c96a65ee Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 15:02:13 +0200 Subject: [PATCH 080/186] fix: script downgrade order --- .../c9c165644731_update_project_last_changed_date_column_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py index d98d135b1d6..847c318d216 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/c9c165644731_update_project_last_changed_date_column_.py @@ -57,7 +57,7 @@ def upgrade(): def downgrade(): - op.execute(sa.DDL("DROP FUNCTION IF EXISTS update_projects_last_change_date();")) op.execute( sa.DDL("DROP TRIGGER IF EXISTS projects_nodes_changed ON projects_nodes;") ) + op.execute(sa.DDL("DROP FUNCTION IF EXISTS update_projects_last_change_date();")) From 73e8ca1bed3a47b287aad9e10a286f08911be7e3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 15:22:03 +0200 Subject: [PATCH 081/186] fix: test --- .../web/server/tests/unit/with_dbs/03/test_project_db.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index 9b0d74109b7..dfbdf91da41 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -178,9 +178,15 @@ def _assert_project_db_row( } expected_db_entries.update(kwargs) assert row + # Remove last_change_date from strict equality check project_entries_in_db = {k: row[k] for k in expected_db_entries} + project_last_change = project_entries_in_db.pop("last_change_date", None) + expected_db_entries.pop("last_change_date", None) assert project_entries_in_db == expected_db_entries - assert row["last_change_date"] >= row["creation_date"] + # last_change_date should be >= creation_date + assert project_last_change is not None + assert row["creation_date"] is not None + assert project_last_change >= row["creation_date"] @pytest.fixture From 9e0e95f2bd323088fc783c342d02f2c95137e02d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 15:39:27 +0200 Subject: [PATCH 082/186] fix: lastChangeDate assert --- .../02/test_projects_crud_handlers.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 4ec2ccf7eef..a16516a0551 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -207,6 +207,16 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") + got_last_change_date = got.pop("lastChangeDate", None) + template_project_last_change_date = template_project.pop("lastChangeDate", None) + if ( + got_last_change_date is not None + and template_project_last_change_date is not None + ): + assert to_datetime(got_last_change_date) >= to_datetime( + template_project_last_change_date + ) + assert got == {k: template_project[k] for k in got} assert not ProjectStateOutputSchema( @@ -220,6 +230,16 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") + got_last_change_date = got.pop("lastChangeDate", None) + user_project_last_change_date = user_project.pop("lastChangeDate", None) + if ( + got_last_change_date is not None + and user_project_last_change_date is not None + ): + assert to_datetime(got_last_change_date) >= to_datetime( + user_project_last_change_date + ) + assert got == {k: user_project[k] for k in got} assert ProjectStateOutputSchema(**project_state) @@ -237,6 +257,16 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") + got_last_change_date = got.pop("lastChangeDate", None) + user_project_last_change_date = user_project.pop("lastChangeDate", None) + if ( + got_last_change_date is not None + and user_project_last_change_date is not None + ): + assert to_datetime(got_last_change_date) >= to_datetime( + user_project_last_change_date + ) + assert got == {k: user_project[k] for k in got} assert not ProjectStateOutputSchema( **project_state @@ -255,6 +285,16 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") + got_last_change_date = got.pop("lastChangeDate", None) + template_project_last_change_date = template_project.pop("lastChangeDate", None) + if ( + got_last_change_date is not None + and template_project_last_change_date is not None + ): + assert to_datetime(got_last_change_date) >= to_datetime( + template_project_last_change_date + ) + assert got == {k: template_project[k] for k in got} assert not ProjectStateOutputSchema( **project_state From 790feafd15e8c5e16e8ac1b123043ce927f2b608 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 15:51:55 +0200 Subject: [PATCH 083/186] fix: test --- .../projects/_projects_service.py | 29 +++++++++++++++++-- .../02/test_projects_crud_handlers.py | 6 ++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index f4abce96450..345052dd775 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1249,6 +1249,22 @@ async def patch_project_node( partial_node=partial_node, ) + ( + project_document, + document_version, + ) = await create_project_document_and_increment_version(app, project_id) + + user_primary_gid = await users_service.get_user_primary_group_id(app, user_id) + + await notify_project_document_updated( + app=app, + project_id=project_id, + user_primary_gid=user_primary_gid, + client_session_id=client_session_id, + version=document_version, + document=project_document, + ) + # 4. Make calls to director-v2 to keep data in sync (ex. comp_* DB tables) await director_v2_service.create_or_update_pipeline( app, @@ -1262,10 +1278,17 @@ async def patch_project_node( app, project_id=project_id ) + updated_project = await _projects_repository.get_project_with_workbench( + app, project_uuid=project_id + ) + # 5. Updates project states for user, if inputs/outputs have been changed if {"inputs", "outputs"} & _node_patch_exclude_unset.keys(): updated_project = await add_project_states_for_user( - user_id=user_id, project=updated_project, is_template=False, app=app + user_id=user_id, + project=updated_project.model_dump(), + is_template=False, + app=app, ) for node_uuid in updated_project["workbench"]: await notify_project_node_update( @@ -1273,7 +1296,9 @@ async def patch_project_node( ) return - await notify_project_node_update(app, updated_project, node_id, errors=None) + await notify_project_node_update( + app, updated_project.model_dump(), node_id, errors=None + ) async def update_project_node_outputs( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index a16516a0551..f8e77d69cd9 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -168,6 +168,12 @@ async def _assert_get_same_project( project_permalink = data.pop("permalink", None) folder_id = data.pop("folderId", None) + got_last_change_date = data.pop("lastChangeDate", None) + project_last_change_date = project.pop("lastChangeDate", None) + if got_last_change_date is not None and project_last_change_date is not None: + assert to_datetime(got_last_change_date) >= to_datetime( + project_last_change_date + ) assert data == {k: project[k] for k in data} if project_state: From b81f5b297ac3ee81356e3631d3f6c5e4f92a3e4c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 16:18:42 +0200 Subject: [PATCH 084/186] fix: node creation --- .../src/models_library/projects_nodes.py | 4 + .../projects/_projects_nodes_repository.py | 18 ++++ .../projects/_projects_service.py | 94 +++++++++---------- .../02/test_projects_nodes_handler.py | 12 +-- 4 files changed, 74 insertions(+), 54 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 488dbfa4a32..61bc527e9d9 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -269,6 +269,10 @@ class Node(BaseModel): Field(default_factory=NodeState, description="The node's state object"), ] = DEFAULT_FACTORY + required_resources: Annotated[ + dict[str, Any] | None, Field(default_factory=dict) + ] = DEFAULT_FACTORY + boot_options: Annotated[ dict[EnvVarKey, str] | None, Field( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 6d09ae6a163..2c66a9e6231 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -43,6 +43,24 @@ ] +async def add( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, + node_id: NodeID, + node: Node, +) -> None: + values = node.model_dump(mode="json", exclude_none=True, exclude_unset=True) + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + projects_nodes.insert().values( + project_uuid=f"{project_id}", node_id=f"{node_id}", **values + ) + ) + + async def delete( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 345052dd775..e227c24f3f1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -92,7 +92,6 @@ from servicelib.utils import fire_and_forget_task, limited_gather, logged_gather from simcore_postgres_database.models.users import UserRole from simcore_postgres_database.utils_projects_nodes import ( - ProjectNodeCreate, ProjectNodesNodeNotFoundError, ) from simcore_postgres_database.webserver_models import ProjectType @@ -169,6 +168,30 @@ _logger = logging.getLogger(__name__) +async def _create_project_document_and_notify( + app, + *, + project_id: ProjectID, + user_id: UserID, + client_session_id: ClientSessionID | None, +): + ( + project_document, + document_version, + ) = await create_project_document_and_increment_version(app, project_id) + + user_primary_gid = await users_service.get_user_primary_group_id(app, user_id) + + await notify_project_document_updated( + app=app, + project_id=project_id, + user_primary_gid=user_primary_gid, + client_session_id=client_session_id, + version=document_version, + document=project_document, + ) + + async def patch_project_and_notify_users( app: web.Application, *, @@ -937,26 +960,23 @@ async def add_project_node( default_resources = await catalog_service.get_service_resources( request.app, user_id, service_key, service_version ) - db_legacy: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) - assert db_legacy # nosec - await db_legacy.add_project_node( - user_id, - ProjectID(project["uuid"]), - ProjectNodeCreate( - node_id=node_uuid, - required_resources=jsonable_encoder(default_resources), + + await _projects_nodes_repository.add( + request.app, + project_id=ProjectID(project["uuid"]), + node_id=node_uuid, + node=Node( key=service_key, version=service_version, label=service_key.split("/")[-1], + required_resources=jsonable_encoder(default_resources), ), - Node.model_validate( - { - "key": service_key, - "version": service_version, - "label": service_key.split("/")[-1], - } - ), - product_name, + ) + + await _create_project_document_and_notify( + request.app, + project_id=ProjectID(project["uuid"]), + user_id=user_id, client_session_id=client_session_id, ) @@ -1094,6 +1114,13 @@ async def delete_project_node( node_id=NodeID(node_uuid), ) + await _create_project_document_and_notify( + request.app, + project_id=project_uuid, + user_id=user_id, + client_session_id=client_session_id, + ) + # also ensure the project is updated by director-v2 since services product_name = products_web.get_product_name(request) await director_v2_service.create_or_update_pipeline( @@ -1103,24 +1130,6 @@ async def delete_project_node( request.app, project_id=project_uuid ) - ( - project_document, - document_version, - ) = await create_project_document_and_increment_version(request.app, project_uuid) - - user_primary_gid = await users_service.get_user_primary_group_id( - request.app, user_id - ) - - await notify_project_document_updated( - app=request.app, - project_id=project_uuid, - user_primary_gid=user_primary_gid, - client_session_id=client_session_id, - version=document_version, - document=project_document, - ) - async def update_project_linked_product( app: web.Application, project_id: ProjectID, product_name: str @@ -1249,20 +1258,11 @@ async def patch_project_node( partial_node=partial_node, ) - ( - project_document, - document_version, - ) = await create_project_document_and_increment_version(app, project_id) - - user_primary_gid = await users_service.get_user_primary_group_id(app, user_id) - - await notify_project_document_updated( - app=app, + await _create_project_document_and_notify( + app, project_id=project_id, - user_primary_gid=user_primary_gid, + user_id=user_id, client_session_id=client_session_id, - version=document_version, - document=project_document, ) # 4. Make calls to director-v2 to keep data in sync (ex. comp_* DB tables) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index 5de1a8217ea..d58f3c2ed54 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -371,16 +371,14 @@ async def test_create_node( # check database is updated assert "node_id" in data - create_node_id = data["node_id"] + node_id = data["node_id"] with postgres_db.connect() as conn: result = conn.execute( - sa.select(projects_db_model.c.workbench).where( - projects_db_model.c.uuid == user_project["uuid"] - ) + sa.select(sa.literal(1)) + .where(projects_nodes.c.node_id == node_id) + .limit(1) ) - assert result - workbench = result.one()[projects_db_model.c.workbench] - assert create_node_id in workbench + assert result.scalar() is not None else: assert error From f41304c7bdf02ffda1c5b337bfcb36f1776251d2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 16:23:21 +0200 Subject: [PATCH 085/186] typecheck --- .../simcore_service_webserver/projects/_projects_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index e227c24f3f1..ef842821e5c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1284,15 +1284,15 @@ async def patch_project_node( # 5. Updates project states for user, if inputs/outputs have been changed if {"inputs", "outputs"} & _node_patch_exclude_unset.keys(): - updated_project = await add_project_states_for_user( + updated_project_with_states = await add_project_states_for_user( user_id=user_id, project=updated_project.model_dump(), is_template=False, app=app, ) - for node_uuid in updated_project["workbench"]: + for node_uuid in updated_project_with_states["workbench"]: await notify_project_node_update( - app, updated_project, node_uuid, errors=None + app, updated_project_with_states, node_uuid, errors=None ) return From 86c5a75a595a5c7cdabf5ce14f39d2025da55de4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 16:28:42 +0200 Subject: [PATCH 086/186] typecheck --- .../projects/_projects_repository_legacy_utils.py | 2 +- .../projects/_projects_service.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 7056bee3e59..005de83d0ba 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -191,8 +191,8 @@ async def _upsert_tags_in_project( .on_conflict_do_nothing() ) + @staticmethod async def _get_workbench( - self, connection: SAConnection, project_uuid: str, ) -> NodesDict: diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index ef842821e5c..2bcba98ca0e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -161,7 +161,12 @@ ProjectTooManyUserSessionsError, ProjectTypeAndTemplateIncompatibilityError, ) -from .models import ProjectDBGet, ProjectDict, ProjectPatchInternalExtended +from .models import ( + ProjectDBGet, + ProjectDict, + ProjectPatchInternalExtended, + ProjectWithWorkbenchDBGet, +) from .settings import ProjectsSettings, get_plugin_settings from .utils import extract_dns_without_default_port @@ -1278,8 +1283,10 @@ async def patch_project_node( app, project_id=project_id ) - updated_project = await _projects_repository.get_project_with_workbench( - app, project_uuid=project_id + updated_project: ProjectWithWorkbenchDBGet = ( + await _projects_repository.get_project_with_workbench( + app, project_uuid=project_id + ) ) # 5. Updates project states for user, if inputs/outputs have been changed From b0b26670362ea2900991373e7f0a487d56d1ecf4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 16:32:12 +0200 Subject: [PATCH 087/186] remove old --- .../projects/_projects_service.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 2bcba98ca0e..6c6e2a82001 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1247,15 +1247,6 @@ async def patch_project_node( ) # 3. Patch the project node - updated_project, _ = await _projects_repository_legacy.update_project_node_data( - user_id=user_id, - project_uuid=project_id, - node_id=node_id, - product_name=product_name, - new_node_data=_node_patch_exclude_unset, - client_session_id=client_session_id, - ) - await _projects_nodes_repository.update( app, project_id=project_id, From 0a16f4082f03bf649aab8fb29fc8358328ad3c7d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 22:07:26 +0200 Subject: [PATCH 088/186] remove unused --- .../projects/_projects_repository_legacy.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 3ac1656c6f2..71a7e713a58 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -26,7 +26,6 @@ ProjectTemplateType, ) from models_library.projects_comments import CommentID, ProjectsCommentsDB -from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.resource_tracker import ( PricingPlanAndUnitIdsTuple, @@ -35,7 +34,6 @@ ) from models_library.rest_ordering import OrderBy, OrderDirection from models_library.users import UserID -from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.wallets import WalletDB, WalletID from models_library.workspaces import WorkspaceQuery, WorkspaceScope from pydantic import TypeAdapter @@ -1061,34 +1059,6 @@ async def _update_project_workbench( msg = "linter unhappy without this" raise RuntimeError(msg) - async def add_project_node( - self, - user_id: UserID, - project_id: ProjectID, - node: ProjectNodeCreate, - old_struct_node: Node, - product_name: str, - client_session_id: ClientSessionID | None, - ) -> None: - # NOTE: permission check is done currently in update_project_workbench! - partial_workbench_data: dict[NodeIDStr, Any] = { - f"{node.node_id}": jsonable_encoder( - old_struct_node, - exclude_unset=True, - ), - } - await self._update_project_workbench_with_lock_and_notify( - partial_workbench_data, - user_id=user_id, - project_uuid=project_id, - product_name=product_name, - allow_workbench_changes=True, - client_session_id=client_session_id, - ) - project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) - async with self.engine.acquire() as conn: - await project_nodes_repo.add(conn, nodes=[node]) - async def get_project_node( self, project_id: ProjectID, node_id: NodeID ) -> ProjectNode: From fba3af18f79f1634e2ba17d99319f9ab63203188 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 22:07:41 +0200 Subject: [PATCH 089/186] use new repo --- .../projects/_projects_service.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 6c6e2a82001..966be4e6846 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1171,17 +1171,6 @@ async def update_project_node_state( permission="write", # NOTE: MD: before only read was sufficient, double check this ) - # Delete this once workbench is removed from the projects table - # See: https://github.com/ITISFoundation/osparc-simcore/issues/7046 - await db_legacy.update_project_node_data( - user_id=user_id, - project_uuid=project_id, - node_id=node_id, - product_name=None, - new_node_data={"state": {"currentStatus": new_state}}, - client_session_id=client_session_id, - ) - await _projects_nodes_repository.update( app, project_id=project_id, @@ -1190,6 +1179,14 @@ async def update_project_node_state( state=NodeState(current_status=RunningState(new_state)) ), ) + + await _create_project_document_and_notify( + app, + project_id=project_id, + user_id=user_id, + client_session_id=client_session_id, + ) + return await get_project_for_user( app, user_id=user_id, project_uuid=f"{project_id}", include_state=True ) From 9a7c654991a250e47ac8f2adc2d46fb7bc6fb3e0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 7 Aug 2025 23:36:01 +0200 Subject: [PATCH 090/186] remove get project --- .../projects/_controller/nodes_rest.py | 8 +------- .../projects/_projects_service.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index f0637cc256e..f9be7add64d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -111,18 +111,12 @@ async def create_node(request: web.Request) -> web.Response: text=f"Service {body.service_key}:{body.service_version} is deprecated" ) - # ensure the project exists - project_data = await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_id}", - user_id=req_ctx.user_id, - ) data = { "node_id": await _projects_service.add_project_node( request, - project_data, req_ctx.user_id, req_ctx.product_name, + path_params.project_id, get_api_base_url(request), body.service_key, body.service_version, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 966be4e6846..6215d3c50f6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -935,9 +935,9 @@ async def _() -> None: async def add_project_node( request: web.Request, - project: dict[str, Any], user_id: UserID, product_name: str, + project_id: ProjectID, product_api_base_url: str, service_key: ServiceKey, service_version: ServiceVersion, @@ -948,14 +948,14 @@ async def add_project_node( "starting node %s:%s in project %s for user %s", service_key, service_version, - project["uuid"], + project_id, user_id, extra=get_log_record_extra(user_id=user_id), ) await check_user_project_permission( request.app, - project_id=project["uuid"], + project_id=project_id, user_id=user_id, product_name=product_name, permission="write", @@ -968,7 +968,7 @@ async def add_project_node( await _projects_nodes_repository.add( request.app, - project_id=ProjectID(project["uuid"]), + project_id=project_id, node_id=node_uuid, node=Node( key=service_key, @@ -980,7 +980,7 @@ async def add_project_node( await _create_project_document_and_notify( request.app, - project_id=ProjectID(project["uuid"]), + project_id=project_id, user_id=user_id, client_session_id=client_session_id, ) @@ -990,12 +990,12 @@ async def add_project_node( await director_v2_service.create_or_update_pipeline( request.app, user_id, - project["uuid"], + project_id, product_name, product_api_base_url, ) await dynamic_scheduler_service.update_projects_networks( - request.app, project_id=ProjectID(project["uuid"]) + request.app, project_id=project_id ) if _is_node_dynamic(service_key): @@ -1008,7 +1008,7 @@ async def add_project_node( product_name=product_name, product_api_base_url=product_api_base_url, user_id=user_id, - project_uuid=ProjectID(project["uuid"]), + project_uuid=project_id, node_uuid=node_uuid, ) From 725be27ac0600b459d3a205e5f90a85187bfede4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 13:01:35 +0200 Subject: [PATCH 091/186] update nodes --- .../projects/_nodes_repository.py | 30 +++++++++++++++++-- .../projects/_nodes_service.py | 13 +++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index 2db16905f19..2e9951e81cd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -1,11 +1,16 @@ +from typing import Any + from aiohttp import web from models_library.projects import ProjectID -from models_library.projects_nodes import Node +from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID from models_library.services_types import ServiceKey, ServiceVersion from pydantic import TypeAdapter from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo -from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) from simcore_service_webserver.db.plugin import get_asyncpg_engine @@ -35,3 +40,24 @@ async def get_project_nodes_map( for project_node in project_nodes } return TypeAdapter(dict[NodeID, Node]).validate_python(workbench) + + +async def update_project_nodes_map( + app: web.Application, + *, + project_id: ProjectID, + partial_nodes_map: dict[NodeID, PartialNode], +) -> dict[NodeID, Node]: + repo = ProjectNodesRepo(project_uuid=project_id) + + workbench: dict[NodeID, dict[str, Any]] = {} + async with transaction_context(get_asyncpg_engine(app)) as conn: + for node_id, node in partial_nodes_map.items(): + project_node = await repo.update( + conn, + node_id=node_id, + node=node, + ) + workbench[node_id] = project_node.model_dump_as_node() + + return TypeAdapter(dict[NodeID, Node]).validate_python(workbench) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py index fad1f355d0a..394a32fe210 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_service.py @@ -10,7 +10,7 @@ from models_library.api_schemas_storage.storage_schemas import FileMetaDataGet from models_library.basic_types import KeyIDStr from models_library.projects import ProjectID -from models_library.projects_nodes import Node +from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID, SimCoreFileLink from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID @@ -90,6 +90,17 @@ async def get_project_nodes_map( return await _nodes_repository.get_project_nodes_map(app, project_id=project_id) +async def update_project_nodes_map( + app: web.Application, + *, + project_id: ProjectID, + partial_nodes_map: dict[NodeID, PartialNode], +) -> dict[NodeID, Node]: + return await _nodes_repository.update_project_nodes_map( + app, project_id=project_id, partial_nodes_map=partial_nodes_map + ) + + # # PREVIEWS # From 5e14001062d3a2f18bac6b871f9b4f0147d735e7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 13:37:09 +0200 Subject: [PATCH 092/186] fix document creation --- .../projects/_controller/ports_rest.py | 42 ++++++++++--------- .../projects/_projects_service.py | 2 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py index 2a50d0d379a..eef9a270b9f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py @@ -8,9 +8,8 @@ ProjectOutputGet, ) from models_library.basic_types import KeyIDStr -from models_library.projects_nodes import Node +from models_library.projects_nodes import PartialNode from models_library.projects_nodes_io import NodeID -from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.services_io import JsonSchemaDict from pydantic import BaseModel, Field, TypeAdapter from servicelib.aiohttp.requests_validation import ( @@ -25,7 +24,7 @@ from ...security.decorators import permission_required from ...utils_aiohttp import envelope_json_response from .. import _access_rights_service, _nodes_service, _ports_service -from .._projects_repository_legacy import ProjectDBAPI +from .._projects_service import _create_project_document_and_notify from ._rest_exceptions import handle_plugin_requests_exceptions from ._rest_schemas import AuthenticatedRequestContext, ProjectPathParams @@ -77,7 +76,6 @@ async def get_project_inputs(request: web.Request) -> web.Response: @permission_required("project.update") @handle_plugin_requests_exceptions async def update_project_inputs(request: web.Request) -> web.Response: - db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) req_ctx = AuthenticatedRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) inputs_updates = await parse_request_body_as(list[ProjectInputUpdate], request) @@ -92,10 +90,12 @@ async def update_project_inputs(request: web.Request) -> web.Response: project_id=path_params.project_id, permission="write", # because we are updating inputs later ) - workbench = await _nodes_service.get_project_nodes_map( + current_workbench = await _nodes_service.get_project_nodes_map( app=request.app, project_id=path_params.project_id ) - current_inputs: dict[NodeID, Any] = _ports_service.get_project_inputs(workbench) + current_inputs: dict[NodeID, Any] = _ports_service.get_project_inputs( + current_workbench + ) # build workbench patch partial_workbench_data = {} @@ -104,30 +104,34 @@ async def update_project_inputs(request: web.Request) -> web.Response: if node_id not in current_inputs: raise web.HTTPBadRequest(text=f"Invalid input key [{node_id}]") - workbench[node_id].outputs = {KeyIDStr("out_1"): input_update.value} - partial_workbench_data[node_id] = workbench[node_id].model_dump( + current_workbench[node_id].outputs = {KeyIDStr("out_1"): input_update.value} + partial_workbench_data[node_id] = current_workbench[node_id].model_dump( include={"outputs"}, exclude_unset=True ) - # patch workbench - assert db # nosec - updated_project, _ = await db.update_project_multiple_node_data( + partial_nodes_map = TypeAdapter(dict[NodeID, PartialNode]).validate_python( + partial_workbench_data + ) + + updated_workbench = await _nodes_service.update_project_nodes_map( + request.app, + project_id=path_params.project_id, + partial_nodes_map=partial_nodes_map, + ) + + await _create_project_document_and_notify( + request.app, + project_id=path_params.project_id, user_id=req_ctx.user_id, - project_uuid=path_params.project_id, - product_name=req_ctx.product_name, - partial_workbench_data=jsonable_encoder(partial_workbench_data), client_session_id=header_params.client_session_id, ) - workbench = TypeAdapter(dict[NodeID, Node]).validate_python( - updated_project["workbench"] - ) - inputs: dict[NodeID, Any] = _ports_service.get_project_inputs(workbench) + inputs: dict[NodeID, Any] = _ports_service.get_project_inputs(updated_workbench) return envelope_json_response( { node_id: ProjectInputGet( - key=node_id, label=workbench[node_id].label, value=value + key=node_id, label=updated_workbench[node_id].label, value=value ) for node_id, value in inputs.items() } diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 6215d3c50f6..3cf0a4dd416 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1292,7 +1292,7 @@ async def patch_project_node( return await notify_project_node_update( - app, updated_project.model_dump(), node_id, errors=None + app, updated_project.model_dump(mode="json"), node_id, errors=None ) From 0cecabdbc21aa5c31c7cd76551931d471acc39ac Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 13:59:58 +0200 Subject: [PATCH 093/186] fix tests --- .../02/test_projects_nodes_handler.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index d58f3c2ed54..a95cf5ad43b 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -48,7 +48,6 @@ from servicelib.aiohttp import status from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from settings_library.redis import RedisSettings -from simcore_postgres_database.models.projects import projects as projects_db_model from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects._controller.nodes_rest import ( @@ -462,19 +461,20 @@ def inc_running_services(self, *args, **kwargs): # noqa: ARG002 # check that we do have NUM_DY_SERVICES nodes in the project with postgres_db.connect() as conn: result = conn.execute( - sa.select(projects_db_model.c.workbench).where( - projects_db_model.c.uuid == user_project["uuid"] + sa.select(projects_nodes.c.node_id).where( + projects_nodes.c.project_uuid == user_project["uuid"] ) ) assert result - workbench = result.one()[projects_db_model.c.workbench] - assert len(workbench) == NUM_DY_SERVICES + num_services_in_project + node_ids = result.scalars().all() + assert len(node_ids) == NUM_DY_SERVICES + num_services_in_project print(f"--> {NUM_DY_SERVICES} nodes were created concurrently") # # delete now # delete_node_tasks = [] - for node_id in workbench: + + for node_id in node_ids: delete_url = client.app.router["delete_node"].url_for( project_id=user_project["uuid"], node_id=node_id ) @@ -595,13 +595,13 @@ async def inc_running_services(self, *args, **kwargs): # noqa: ARG002 # check that we do have NUM_DY_SERVICES nodes in the project with postgres_db.connect() as conn: result = conn.execute( - sa.select(projects_db_model.c.workbench).where( - projects_db_model.c.uuid == project["uuid"] + sa.select(projects_nodes.c.node_id).where( + projects_nodes.c.project_uuid == project["uuid"] ) ) assert result - workbench = result.one()[projects_db_model.c.workbench] - assert len(workbench) == NUM_DY_SERVICES + node_ids = result.scalars().all() + assert len(node_ids) == NUM_DY_SERVICES @pytest.mark.parametrize(*standard_user_role()) From f8fa9e65d068708516fcbf853cf1b58adb4003f4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 14:23:53 +0200 Subject: [PATCH 094/186] fix test --- packages/models-library/src/models_library/projects_nodes.py | 2 +- .../src/pytest_simcore/helpers/webserver_projects.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 61bc527e9d9..2fd18746241 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -159,7 +159,7 @@ class Node(BaseModel): Field(description="The short name of the node", examples=["JupyterLab"]), ] progress: Annotated[ - float | None, + int | None, Field( ge=0, le=100, diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index b8973f70096..fc582dbcccf 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -210,6 +210,10 @@ async def assert_get_same_project( resp = await client.get(f"{url}") data, error = await assert_status(resp, expected) + # without our control + project.pop("lastChangeDate", None) + data.pop("lastChangeDate", None) + if not error: assert data == {k: project[k] for k in data} return data From 8ffa91b5e844302bd6bb774463351c850548da3a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 14:43:21 +0200 Subject: [PATCH 095/186] fix tests --- .../projects/_controller/ports_rest.py | 7 +- .../projects/_nodes_repository.py | 2 +- .../02/test_projects_ports_handlers.py | 94 ++++++++++--------- 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py index eef9a270b9f..1e2ea19a474 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/ports_rest.py @@ -113,12 +113,17 @@ async def update_project_inputs(request: web.Request) -> web.Response: partial_workbench_data ) - updated_workbench = await _nodes_service.update_project_nodes_map( + await _nodes_service.update_project_nodes_map( request.app, project_id=path_params.project_id, partial_nodes_map=partial_nodes_map, ) + # get updated workbench (including not updated nodes) + updated_workbench = await _nodes_service.get_project_nodes_map( + request.app, project_id=path_params.project_id + ) + await _create_project_document_and_notify( request.app, project_id=path_params.project_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index 2e9951e81cd..5dab5590224 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -56,7 +56,7 @@ async def update_project_nodes_map( project_node = await repo.update( conn, node_id=node_id, - node=node, + **node.model_dump(exclude_none=True, exclude_unset=True), ) workbench[node_id] = project_node.model_dump_as_node() diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py index c3c3b8308d9..bd2b14a4ad0 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py @@ -12,6 +12,7 @@ import pytest from aiohttp.test_utils import TestClient from aioresponses import aioresponses as AioResponsesMock # noqa: N812 +from deepdiff import DeepDiff from models_library.api_schemas_directorv2.computations import TasksOutputs from models_library.api_schemas_webserver.projects import ProjectGet from models_library.utils.fastapi_encoders import jsonable_encoder @@ -109,55 +110,62 @@ async def test_io_workflow( ports_meta, error = await assert_status(resp, expected_status_code=expected) if not error: - assert ports_meta == [ - { - "key": "38a0d401-af4b-4ea7-ab4c-5005c712a546", - "kind": "input", - "content_schema": { - "description": "Input integer value", - "title": "X", - "type": "integer", + diff = DeepDiff( + ports_meta, + [ + { + "key": "38a0d401-af4b-4ea7-ab4c-5005c712a546", + "kind": "input", + "content_schema": { + "description": "Input integer value", + "title": "X", + "type": "integer", + }, }, - }, - { - "key": "fc48252a-9dbb-4e07-bf9a-7af65a18f612", - "kind": "input", - "content_schema": { - "description": "Input integer value", - "title": "Z", - "type": "integer", + { + "key": "fc48252a-9dbb-4e07-bf9a-7af65a18f612", + "kind": "input", + "content_schema": { + "description": "Input integer value", + "title": "Z", + "type": "integer", + }, }, - }, - { - "key": "7bf0741f-bae4-410b-b662-fc34b47c27c9", - "kind": "input", - "content_schema": { - "description": "Input boolean value", - "title": "on", - "type": "boolean", + { + "key": "7bf0741f-bae4-410b-b662-fc34b47c27c9", + "kind": "input", + "content_schema": { + "description": "Input boolean value", + "title": "on", + "type": "boolean", + }, }, - }, - { - "key": "09fd512e-0768-44ca-81fa-0cecab74ec1a", - "kind": "output", - "content_schema": { - "description": "Output integer value", - "title": "Random sleep interval_2", - "type": "integer", + { + "key": "09fd512e-0768-44ca-81fa-0cecab74ec1a", + "kind": "output", + "content_schema": { + "description": "Output integer value", + "title": "Random sleep interval_2", + "type": "integer", + }, }, - }, - { - "key": "76f607b4-8761-4f96-824d-cab670bc45f5", - "kind": "output", - "content_schema": { - "description": "Output integer value", - "title": "Random sleep interval", - "type": "integer", + { + "key": "76f607b4-8761-4f96-824d-cab670bc45f5", + "kind": "output", + "content_schema": { + "description": "Output integer value", + "title": "Random sleep interval", + "type": "integer", + }, }, - }, - ] + ], + ignore_order=True, + ) - assert ports_meta == PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA + assert not diff + assert not DeepDiff( + ports_meta, PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA, ignore_order=True + ) # get_project_inputs expected_url = client.app.router["get_project_inputs"].url_for( From e5a9b5fcd4f3f73e1879096db89f99d66cf88930 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 14:50:51 +0200 Subject: [PATCH 096/186] fix --- .../src/pytest_simcore/helpers/webserver_projects.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index fc582dbcccf..a57ffd975e2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -12,6 +12,7 @@ from aiohttp import web from aiohttp.test_utils import TestClient from common_library.dict_tools import remap_keys +from deepdiff import DeepDiff from models_library.projects_nodes_io import NodeID from models_library.services_resources import ServiceResourcesDictHelpers from simcore_postgres_database.utils_projects_nodes import ProjectNodeCreate @@ -211,9 +212,9 @@ async def assert_get_same_project( data, error = await assert_status(resp, expected) # without our control - project.pop("lastChangeDate", None) - data.pop("lastChangeDate", None) if not error: - assert data == {k: project[k] for k in data} + assert not DeepDiff( + data, {k: project[k] for k in data}, exclude_paths="root['lastChangeDate']" + ) return data From 42ee9d1159e5cc2a6d915825b222e3225d25a3e2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 14:56:42 +0200 Subject: [PATCH 097/186] fix deepdiff --- .../02/test_projects_crud_handlers.py | 74 ++++++------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index f8e77d69cd9..d365f621c86 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -14,6 +14,7 @@ import sqlalchemy as sa from aiohttp.test_utils import TestClient from aioresponses import aioresponses +from deepdiff import DeepDiff from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import ( GetProjectInactivityResponse, @@ -168,13 +169,9 @@ async def _assert_get_same_project( project_permalink = data.pop("permalink", None) folder_id = data.pop("folderId", None) - got_last_change_date = data.pop("lastChangeDate", None) - project_last_change_date = project.pop("lastChangeDate", None) - if got_last_change_date is not None and project_last_change_date is not None: - assert to_datetime(got_last_change_date) >= to_datetime( - project_last_change_date - ) - assert data == {k: project[k] for k in data} + assert not DeepDiff( + data, {k: project[k] for k in data}, exclude_paths="root['lastChangeDate']" + ) if project_state: assert ProjectStateOutputSchema.model_validate(project_state) @@ -213,17 +210,11 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") - got_last_change_date = got.pop("lastChangeDate", None) - template_project_last_change_date = template_project.pop("lastChangeDate", None) - if ( - got_last_change_date is not None - and template_project_last_change_date is not None - ): - assert to_datetime(got_last_change_date) >= to_datetime( - template_project_last_change_date - ) - - assert got == {k: template_project[k] for k in got} + assert not DeepDiff( + got, + {k: template_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert not ProjectStateOutputSchema( **project_state @@ -236,17 +227,11 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") - got_last_change_date = got.pop("lastChangeDate", None) - user_project_last_change_date = user_project.pop("lastChangeDate", None) - if ( - got_last_change_date is not None - and user_project_last_change_date is not None - ): - assert to_datetime(got_last_change_date) >= to_datetime( - user_project_last_change_date - ) - - assert got == {k: user_project[k] for k in got} + assert not DeepDiff( + got, + {k: user_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert ProjectStateOutputSchema(**project_state) assert project_permalink is None @@ -263,17 +248,12 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") - got_last_change_date = got.pop("lastChangeDate", None) - user_project_last_change_date = user_project.pop("lastChangeDate", None) - if ( - got_last_change_date is not None - and user_project_last_change_date is not None - ): - assert to_datetime(got_last_change_date) >= to_datetime( - user_project_last_change_date - ) + assert not DeepDiff( + got, + {k: user_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) - assert got == {k: user_project[k] for k in got} assert not ProjectStateOutputSchema( **project_state ).share_state.locked, "Single user does not lock" @@ -291,17 +271,11 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") - got_last_change_date = got.pop("lastChangeDate", None) - template_project_last_change_date = template_project.pop("lastChangeDate", None) - if ( - got_last_change_date is not None - and template_project_last_change_date is not None - ): - assert to_datetime(got_last_change_date) >= to_datetime( - template_project_last_change_date - ) - - assert got == {k: template_project[k] for k in got} + assert not DeepDiff( + got, + {k: template_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert not ProjectStateOutputSchema( **project_state ).share_state.locked, "Templates are not locked" From 661b5a557c48ab8c07d12189a2d87410fc9e9d2e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 15:12:43 +0200 Subject: [PATCH 098/186] remove tests --- .../tests/unit/with_dbs/03/test_project_db.py | 268 +----------------- 1 file changed, 1 insertion(+), 267 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index dfbdf91da41..6a07ec722bc 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -19,7 +19,7 @@ from aiohttp.test_utils import TestClient from faker import Faker from models_library.projects import ProjectID, ProjectTemplateType -from models_library.projects_nodes_io import NodeID, NodeIDStr +from models_library.projects_nodes_io import NodeID from psycopg2.errors import UniqueViolation from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -45,9 +45,7 @@ ) from simcore_service_webserver.projects.api import has_user_project_access_rights from simcore_service_webserver.projects.exceptions import ( - NodeNotFoundError, ProjectNodeRequiredInputsNotSetError, - ProjectNotFoundError, ) from simcore_service_webserver.users.exceptions import UserNotFoundError from simcore_service_webserver.utils import to_datetime @@ -366,270 +364,6 @@ async def test_insert_project_to_db( await _assert_projects_nodes_db_rows(aiopg_engine, new_project) -@pytest.mark.parametrize( - "user_role", - [UserRole.USER], -) -async def test_patch_user_project_workbench_raises_if_project_does_not_exist( - fake_project: dict[str, Any], - logged_user: dict[str, Any], - db_api: ProjectDBAPI, - faker: Faker, -): - partial_workbench_data = { - faker.uuid4(): { - "key": "simcore/services/comp/sleepers", - "version": faker.numerify("%.#.#"), - "label": "I am a test node", - } - } - with pytest.raises(ProjectNotFoundError): - await db_api._update_project_workbench( # noqa: SLF001 - partial_workbench_data, - user_id=logged_user["id"], - project_uuid=fake_project["uuid"], - allow_workbench_changes=False, - ) - - -@pytest.mark.parametrize( - "user_role", - [UserRole.USER], -) -async def test_patch_user_project_workbench_creates_nodes( - fake_project: dict[str, Any], - logged_user: dict[str, Any], - db_api: ProjectDBAPI, - faker: Faker, - aiopg_engine: aiopg.sa.engine.Engine, - insert_project_in_db: Callable[..., Awaitable[dict[str, Any]]], -): - empty_fake_project = deepcopy(fake_project) - workbench = empty_fake_project.setdefault("workbench", {}) - assert isinstance(workbench, dict) - workbench.clear() - new_project = await insert_project_in_db( - empty_fake_project, user_id=logged_user["id"] - ) - await _assert_projects_nodes_db_rows(aiopg_engine, new_project) - partial_workbench_data = { - faker.uuid4(): { - "key": f"simcore/services/comp/{faker.pystr().lower()}", - "version": faker.numerify("%.#.#"), - "label": faker.text(), - } - for _ in range(faker.pyint(min_value=5, max_value=30)) - } - ( - patched_project, - changed_entries, - ) = await db_api._update_project_workbench( # noqa: SLF001 - partial_workbench_data, - user_id=logged_user["id"], - project_uuid=new_project["uuid"], - allow_workbench_changes=True, - ) - for node_id in partial_workbench_data: - assert node_id in patched_project["workbench"] - assert partial_workbench_data[node_id] == patched_project["workbench"][node_id] - assert node_id in changed_entries - assert changed_entries[node_id] == partial_workbench_data[node_id] - - -@pytest.mark.parametrize( - "user_role", - [UserRole.USER], -) -async def test_patch_user_project_workbench_creates_nodes_raises_if_invalid_node_is_passed( - fake_project: dict[str, Any], - logged_user: dict[str, Any], - db_api: ProjectDBAPI, - faker: Faker, - aiopg_engine: aiopg.sa.engine.Engine, - insert_project_in_db: Callable[..., Awaitable[dict[str, Any]]], -): - empty_fake_project = deepcopy(fake_project) - workbench = empty_fake_project.setdefault("workbench", {}) - assert isinstance(workbench, dict) - workbench.clear() - - new_project = await insert_project_in_db( - empty_fake_project, user_id=logged_user["id"] - ) - await _assert_projects_nodes_db_rows(aiopg_engine, new_project) - partial_workbench_data = { - faker.uuid4(): { - "version": faker.numerify("%.#.#"), - "label": faker.text(), - } - for _ in range(faker.pyint(min_value=5, max_value=30)) - } - with pytest.raises(NodeNotFoundError): - await db_api._update_project_workbench( # noqa: SLF001 - partial_workbench_data, - user_id=logged_user["id"], - project_uuid=new_project["uuid"], - allow_workbench_changes=True, - ) - - -@pytest.mark.parametrize( - "user_role", - [UserRole.USER], -) -@pytest.mark.parametrize("number_of_nodes", [1, randint(250, 300)]) # noqa: S311 -async def test_patch_user_project_workbench_concurrently( - fake_project: dict[str, Any], - postgres_db: sa.engine.Engine, - logged_user: dict[str, Any], - primary_group: dict[str, str], - db_api: ProjectDBAPI, - number_of_nodes: int, - aiopg_engine: aiopg.sa.engine.Engine, - insert_project_in_db: Callable[..., Awaitable[dict[str, Any]]], -): - _NUMBER_OF_NODES = number_of_nodes - BASE_UUID = UUID("ccc0839f-93b8-4387-ab16-197281060927") - node_uuids = [str(uuid5(BASE_UUID, f"{n}")) for n in range(_NUMBER_OF_NODES)] - - # create a project with a lot of nodes - fake_project["workbench"] = { - node_uuids[n]: { - "key": "simcore/services/comp/sleepers", - "version": "1.43.5", - "label": f"I am node {n}", - } - for n in range(_NUMBER_OF_NODES) - } - expected_project = deepcopy(fake_project) - - # add the project - original_project = deepcopy(fake_project) - new_project = await insert_project_in_db(fake_project, user_id=logged_user["id"]) - - _assert_added_project( - original_project, - new_project, - exp_overrides={ - "prjOwner": logged_user["email"], - }, - ) - _assert_project_db_row( - postgres_db, - new_project, - prj_owner=logged_user["id"], - ) - await _assert_projects_nodes_db_rows(aiopg_engine, new_project) - - # patch all the nodes concurrently - randomly_created_outputs = [ - { - "outputs": {f"out_{k}": f"{k}"} # noqa: B035 - for k in range(randint(1, 10)) # noqa: S311 - } - for n in range(_NUMBER_OF_NODES) - ] - for n in range(_NUMBER_OF_NODES): - expected_project["workbench"][node_uuids[n]].update(randomly_created_outputs[n]) - - patched_projects: list[tuple[dict[str, Any], dict[str, Any]]] = ( - await asyncio.gather( - *[ - db_api._update_project_workbench( # noqa: SLF001 - {NodeIDStr(node_uuids[n]): randomly_created_outputs[n]}, - user_id=logged_user["id"], - project_uuid=new_project["uuid"], - allow_workbench_changes=False, - ) - for n in range(_NUMBER_OF_NODES) - ] - ) - ) - # NOTE: each returned project contains the project with some updated workbenches - # the ordering is uncontrolled. - # The important thing is that the final result shall contain ALL the changes - - for (prj, changed_entries), node_uuid, exp_outputs in zip( - patched_projects, node_uuids, randomly_created_outputs, strict=True - ): - assert prj["workbench"][node_uuid]["outputs"] == exp_outputs["outputs"] - assert changed_entries == {node_uuid: {"outputs": exp_outputs["outputs"]}} - - # get the latest change date - latest_change_date = max( - to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects - ) - - # check the nodes are completely patched as expected - _assert_project_db_row( - postgres_db, - expected_project, - prj_owner=logged_user["id"], - creation_date=to_datetime(new_project["creationDate"]), - last_change_date=latest_change_date, - ) - - # now concurrently remove the outputs - for n in range(_NUMBER_OF_NODES): - expected_project["workbench"][node_uuids[n]]["outputs"] = {} - - patched_projects = await asyncio.gather( - *[ - db_api._update_project_workbench( # noqa: SLF001 - {NodeIDStr(node_uuids[n]): {"outputs": {}}}, - user_id=logged_user["id"], - project_uuid=new_project["uuid"], - allow_workbench_changes=False, - ) - for n in range(_NUMBER_OF_NODES) - ] - ) - - # get the latest change date - latest_change_date = max( - to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects - ) - - # check the nodes are completely patched as expected - _assert_project_db_row( - postgres_db, - expected_project, - prj_owner=logged_user["id"], - creation_date=to_datetime(new_project["creationDate"]), - last_change_date=latest_change_date, - ) - - # now concurrently remove the outputs - for n in range(_NUMBER_OF_NODES): - expected_project["workbench"][node_uuids[n]]["outputs"] = {} - - patched_projects = await asyncio.gather( - *[ - db_api._update_project_workbench( # noqa: SLF001 - {NodeIDStr(node_uuids[n]): {"outputs": {}}}, - user_id=logged_user["id"], - project_uuid=new_project["uuid"], - allow_workbench_changes=False, - ) - for n in range(_NUMBER_OF_NODES) - ] - ) - - # get the latest change date - latest_change_date = max( - to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects - ) - - # check the nodes are completely patched as expected - _assert_project_db_row( - postgres_db, - expected_project, - prj_owner=logged_user["id"], - creation_date=to_datetime(new_project["creationDate"]), - last_change_date=latest_change_date, - ) - - @pytest.fixture() async def some_projects_and_nodes( logged_user: dict[str, Any], From 325608c37f380be8082ca4b14fcd6f7ab639fa6b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 15:26:53 +0200 Subject: [PATCH 099/186] fix model_dump --- .../src/simcore_service_webserver/projects/_projects_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 3cf0a4dd416..7af7c6e384a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1281,7 +1281,7 @@ async def patch_project_node( if {"inputs", "outputs"} & _node_patch_exclude_unset.keys(): updated_project_with_states = await add_project_states_for_user( user_id=user_id, - project=updated_project.model_dump(), + project=updated_project.model_dump(mode="json"), is_template=False, app=app, ) From 127e90a75c3f6014f7dbfe8e501822841ec799fc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 16:04:04 +0200 Subject: [PATCH 100/186] fix --- .../unit/with_dbs/02/test_projects_states_handlers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 67845662e0b..a78cd0e1627 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -70,6 +70,7 @@ from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.licenses._licensed_resources_service import DeepDiff from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.socketio.messages import SOCKET_IO_PROJECT_UPDATED_EVENT from simcore_service_webserver.utils import to_datetime @@ -1929,7 +1930,11 @@ async def test_open_shared_project_at_same_time( elif data: project_status = ProjectStateOutputSchema(**data.pop("state")) data.pop("folderId") - assert data == {k: shared_project[k] for k in data} + assert not DeepDiff( + data, + {k: shared_project[k] for k in data}, + exclude_paths=["root['lastChangeDate']"], + ) assert project_status.share_state.locked assert project_status.share_state.current_user_groupids assert len(project_status.share_state.current_user_groupids) == 1 From df2e20461a3ecd1c11d58e1962a1d4ee8c6fa6f7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 16:19:42 +0200 Subject: [PATCH 101/186] fix test --- .../notifications/test_rabbitmq_consumers.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/services/web/server/tests/integration/01/notifications/test_rabbitmq_consumers.py b/services/web/server/tests/integration/01/notifications/test_rabbitmq_consumers.py index 9f01181b6f7..9952f00a603 100644 --- a/services/web/server/tests/integration/01/notifications/test_rabbitmq_consumers.py +++ b/services/web/server/tests/integration/01/notifications/test_rabbitmq_consumers.py @@ -40,7 +40,7 @@ ) from servicelib.rabbitmq import RabbitMQClient from settings_library.rabbit import RabbitSettings -from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.models.users import UserRole from simcore_service_webserver.application_settings import setup_settings from simcore_service_webserver.db.plugin import setup_db @@ -437,20 +437,18 @@ async def test_progress_computational_workflow( # check the database. doing it after the waiting calls above is safe async with aiopg_engine.acquire() as conn: - assert projects is not None + assert projects_nodes is not None result = await conn.execute( - sa.select(projects.c.workbench).where( - projects.c.uuid == str(user_project_id) + sa.select(projects_nodes).where( + projects_nodes.c.project_uuid == f"{user_project_id}" ) ) - row = await result.fetchone() - assert row - project_workbench = dict(row[projects.c.workbench]) + rows = await result.fetchall() + assert rows + node_dict = {row["node_id"]: dict(row) for row in rows} + # NOTE: the progress might still be present but is not used anymore - assert ( - project_workbench[f"{random_node_id_in_user_project}"].get("progress", 0) - == 0 - ) + assert node_dict[f"{random_node_id_in_user_project}"].get("progress", 0) == 0 @pytest.mark.parametrize("user_role", [UserRole.GUEST], ids=str) From 824200ace94bf340d5238a0fada8bb3fee54b802 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 8 Aug 2025 16:34:32 +0200 Subject: [PATCH 102/186] fix int test --- .../director-v2/tests/integration/01/test_computation_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index a9b33b3f5d1..4b826ec69d0 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -206,7 +206,7 @@ async def test_start_empty_computation_is_refused( ): await create_pipeline( async_client, - project=empty_project, + project_uuid=empty_project.uuid, user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, From c69df7984a2638a1628d75e4eb4c9c29473a2b23 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 11 Aug 2025 09:41:54 +0200 Subject: [PATCH 103/186] tests: ignore order --- .../02/test_projects_nodes_handlers__patch.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index 9c8614ac3bf..d00ca810b5a 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -11,6 +11,7 @@ import pytest from aiohttp.test_utils import TestClient +from deepdiff import DeepDiff from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_users import UserInfoDict @@ -259,10 +260,14 @@ async def test_patch_project_node_inputs_notifies( await assert_status(resp, expected) assert mocked_notify_project_node_update.call_count > 1 # 1 message per node updated - assert [ - call_args[0][2] - for call_args in mocked_notify_project_node_update.await_args_list - ] == list(user_project["workbench"].keys()) + assert not DeepDiff( + [ + call_args[0][2] + for call_args in mocked_notify_project_node_update.await_args_list + ], + list(user_project["workbench"].keys()), + ignore_order=True, + ) @pytest.mark.parametrize( From 6c0431f6bb17ece72d867faa43153e39a70466a7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 11 Aug 2025 11:05:03 +0200 Subject: [PATCH 104/186] fix: inputs required --- .../src/simcore_postgres_database/utils_projects_nodes.py | 1 + .../projects/_projects_repository_legacy.py | 1 + .../projects/_projects_service.py | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index dcc05075689..3678043eb77 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -61,6 +61,7 @@ class ProjectNodeCreate(BaseModel): input_access: dict[str, Any] | None = None input_nodes: list[str] | None = None inputs: dict[str, Any] | None = None + inputs_required: list[str] | None = None inputs_units: dict[str, Any] | None = None output_nodes: list[str] | None = None outputs: dict[str, Any] | None = None diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 71a7e713a58..040f542be25 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -312,6 +312,7 @@ async def insert_project( field_mapping = { "inputAccess": "input_access", "inputNodes": "input_nodes", + "inputsRequired": "inputs_required", "inputsUnits": "inputs_units", "outputNodes": "output_nodes", "runHash": "run_hash", diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 7af7c6e384a..ff90f9b24ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -679,11 +679,11 @@ async def _check_project_node_has_all_required_inputs( permission="read", ) - project_dict, _ = await db.get_project_dict_and_type(f"{project_uuid}") + nodes = await _projects_nodes_repository.get_by_project( + app, project_id=project_uuid + ) - nodes_map: dict[NodeID, Node] = { - NodeID(k): Node(**v) for k, v in project_dict["workbench"].items() - } + nodes_map = dict(nodes) node = nodes_map[node_id] unset_required_inputs: list[str] = [] From ee01c016bb5c5b1f501ace32385552702e521bab Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 11 Aug 2025 12:58:34 +0200 Subject: [PATCH 105/186] fix: node outputs --- .../projects/_projects_nodes_repository.py | 6 ++- .../projects/_projects_service.py | 39 +++++++++---------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 2c66a9e6231..1823cb12036 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -179,15 +179,17 @@ async def update( project_id: ProjectID, node_id: NodeID, partial_node: PartialNode, -) -> None: +) -> Node: values = partial_node.model_dump(mode="json", exclude_unset=True) async with transaction_context(get_asyncpg_engine(app), connection) as conn: - await conn.stream( + result = await conn.stream( projects_nodes.update() .values(**values) .where( (projects_nodes.c.project_uuid == f"{project_id}") & (projects_nodes.c.node_id == f"{node_id}") ) + .returning(*_SELECTION_PROJECTS_NODES_DB_ARGS) ) + return Node.model_validate(await result.first(), from_attributes=True) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index ff90f9b24ee..2e86bf1f4c2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1329,16 +1329,7 @@ async def update_project_node_outputs( permission="write", # NOTE: MD: before only read was sufficient, double check this ) - updated_project, changed_entries = await db_legacy.update_project_node_data( - user_id=user_id, - project_uuid=project_id, - node_id=node_id, - product_name=None, - new_node_data={"outputs": new_outputs, "runHash": new_run_hash}, - client_session_id=client_session_id, - ) - - await _projects_nodes_repository.update( + updated_node = await _projects_nodes_repository.update( app, project_id=project_id, node_id=node_id, @@ -1347,23 +1338,31 @@ async def update_project_node_outputs( ), ) + await _create_project_document_and_notify( + app, + project_id=project_id, + user_id=user_id, + client_session_id=client_session_id, + ) + _logger.debug( "patched project %s, following entries changed: %s", project_id, - pformat(changed_entries), + pformat(updated_node), ) - updated_project = await add_project_states_for_user( - user_id=user_id, project=updated_project, is_template=False, app=app + + updated_project = await _projects_repository.get_project_with_workbench( + app, project_uuid=project_id ) - # changed entries come in the form of {node_uuid: {outputs: {changed_key1: value1, changed_key2: value2}}} - # we do want only the key names - changed_keys = ( - changed_entries.get(TypeAdapter(NodeIDStr).validate_python(f"{node_id}"), {}) - .get("outputs", {}) - .keys() + updated_project_with_states = await add_project_states_for_user( + user_id=user_id, + project=updated_project.model_dump(mode="json"), + is_template=False, + app=app, ) - return updated_project, changed_keys + + return updated_project_with_states, list(new_outputs.keys()) async def list_node_ids_in_project( From 6a51b9a704ae4545271c0a2a9e531951e6ac5f45 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 11 Aug 2025 13:28:04 +0200 Subject: [PATCH 106/186] fix: more efficient get product --- .../projects/_projects_repository.py | 26 ++++++++++++++++--- .../projects/_projects_service.py | 10 ++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 74762c2902f..ac18d30b225 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -8,12 +8,14 @@ from common_library.exclude import Unset, is_set from models_library.basic_types import IDStr from models_library.groups import GroupID +from models_library.products import ProductName from models_library.projects import ProjectID from models_library.rest_ordering import OrderBy, OrderDirection from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE from models_library.workspaces import WorkspaceID -from pydantic import NonNegativeInt, PositiveInt +from pydantic import NonNegativeInt, PositiveInt, TypeAdapter from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_postgres_database.models.users import users from simcore_postgres_database.utils_projects_nodes import make_workbench_subquery from simcore_postgres_database.utils_repos import ( @@ -108,14 +110,32 @@ async def get_project( project_uuid: ProjectID, ) -> ProjectDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - query = sql.select(*PROJECT_DB_COLS).where(projects.c.uuid == f"{project_uuid}") - result = await conn.execute(query) + result = await conn.execute( + sa.select(*PROJECT_DB_COLS).where(projects.c.uuid == f"{project_uuid}") + ) row = result.one_or_none() if row is None: raise ProjectNotFoundError(project_uuid=project_uuid) return ProjectDBGet.model_validate(row) +async def get_project_product( + app, + connection: AsyncConnection | None = None, + *, + project_uuid: ProjectID, +) -> ProductName: + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.scalar( + sa.select(projects_to_products.c.product_name).where( + projects_to_products.c.project_uuid == f"{project_uuid}" + ) + ) + if result is None: + raise ProjectNotFoundError(project_uuid=project_uuid) + return TypeAdapter(ProductName).validate_python(result) + + async def get_project_with_workbench( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 2e86bf1f4c2..1efd8969908 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1161,8 +1161,9 @@ async def update_project_node_state( user_id, ) - db_legacy: ProjectDBAPI = app[APP_PROJECT_DBAPI] - product_name = await db_legacy.get_project_product(project_id) + product_name = await _projects_repository.get_project_product( + app, project_uuid=project_id + ) await check_user_project_permission( app, project_id=project_id, @@ -1319,8 +1320,9 @@ async def update_project_node_outputs( ) new_outputs = new_outputs or {} - db_legacy: ProjectDBAPI = app[APP_PROJECT_DBAPI] - product_name = await db_legacy.get_project_product(project_id) + product_name = await _projects_repository.get_project_product( + app, project_uuid=project_id + ) await check_user_project_permission( app, project_id=project_id, From 77762b37bd3e029ab6fda5cca2a3ddaec8dc562e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 12 Aug 2025 12:51:54 +0200 Subject: [PATCH 107/186] refactor: remove access to workbench --- .../projects/_projects_service.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 1efd8969908..7cef3a8d2b4 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1213,8 +1213,6 @@ async def patch_project_node( mode="json", exclude_unset=True, by_alias=True ) - _projects_repository_legacy = ProjectDBAPI.get_from_app_context(app) - # 1. Check user permissions await check_user_project_permission( app, @@ -1226,14 +1224,15 @@ async def patch_project_node( # 2. If patching service key or version make sure it's valid if _node_patch_exclude_unset.get("key") or _node_patch_exclude_unset.get("version"): - _project, _ = await _projects_repository_legacy.get_project_dict_and_type( - project_uuid=f"{project_id}" + _project_node = await _projects_nodes_repository.get( + app, + project_id=project_id, + node_id=node_id, ) - _project_node_data = _project["workbench"][f"{node_id}"] - _service_key = _node_patch_exclude_unset.get("key", _project_node_data["key"]) + _service_key = _node_patch_exclude_unset.get("key", _project_node.key) _service_version = _node_patch_exclude_unset.get( - "version", _project_node_data["version"] + "version", _project_node.version ) rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) await catalog_rpc.check_for_service( From 289544c71b39de149aca0ebdef896cdb7bd57deb Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 12 Aug 2025 13:48:34 +0200 Subject: [PATCH 108/186] Fix bug Refactors group notification to use a dedicated API for fetching project group permissions, enhancing consistency and maintainability. Removes reliance on direct access to project permission structures. --- .../projects/_groups_service.py | 17 +++++++++++++++++ .../projects/_projects_service.py | 18 ++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_service.py b/services/web/server/src/simcore_service_webserver/projects/_groups_service.py index 37207fd2c6d..ee663ba7a52 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_service.py @@ -252,3 +252,20 @@ async def create_project_group_without_checking_permissions( write=write, delete=delete, ) + + +async def list_project_groups_by_project_without_checking_permissions( + app: web.Application, + *, + project_id: ProjectID, +) -> list[ProjectGroupGet]: + project_groups_db: list[ProjectGroupGetDB] = ( + await _groups_repository.list_project_groups(app=app, project_id=project_id) + ) + + project_groups_api: list[ProjectGroupGet] = [ + ProjectGroupGet.model_validate(group.model_dump()) + for group in project_groups_db + ] + + return project_groups_api diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index f8527dea233..55426aa4e72 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -12,7 +12,7 @@ import datetime import logging from collections import defaultdict -from collections.abc import Generator, Iterable +from collections.abc import Iterable from contextlib import suppress from decimal import Decimal from pprint import pformat @@ -130,6 +130,7 @@ from ..workspaces import _workspaces_repository as workspaces_workspaces_repository from . import ( _crud_api_delete, + _groups_service, _nodes_service, _projects_nodes_repository, _projects_repository, @@ -2107,9 +2108,13 @@ async def notify_project_state_update( message=message, ) else: - rooms_to_notify: Generator[GroupID, None, None] = ( - gid for gid, rights in project["accessRights"].items() if rights["read"] + project_group_get_list = await _groups_service.list_project_groups_by_project_without_checking_permissions( + app, project_id=project["uuid"] ) + + rooms_to_notify = [ + item.gid for item in project_group_get_list if item.read is True + ] for room in rooms_to_notify: await send_message_to_standard_group(app, group_id=room, message=message) @@ -2123,9 +2128,10 @@ async def notify_project_node_update( if await is_project_hidden(app, ProjectID(project["uuid"])): return - rooms_to_notify: list[GroupID] = [ - gid for gid, rights in project["accessRights"].items() if rights["read"] - ] + project_group_get_list = await _groups_service.list_project_groups_by_project_without_checking_permissions( + app, project_id=project["uuid"] + ) + rooms_to_notify = [item.gid for item in project_group_get_list if item.read is True] message = SocketMessageDict( event_type=SOCKET_IO_NODE_UPDATED_EVENT, From 7db6618c1cfab74c7dfea1e617405a4e7510a165 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 12 Aug 2025 16:27:27 +0200 Subject: [PATCH 109/186] fix: project node states --- .../simcore_service_webserver/projects/_projects_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 6dd2a4363cd..3ce7adda553 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1865,9 +1865,9 @@ async def add_project_states_for_user( node_state_dict = json_loads( node_state.model_dump_json(by_alias=True, exclude_unset=True) ) - prj_node.setdefault("state", {}).update(node_state_dict) + prj_node.state = NodeState.model_validate(node_state_dict) prj_node_progress = node_state_dict.get("progress", None) or 0 - prj_node.update({"progress": round(prj_node_progress * 100.0)}) + prj_node.progress = round(prj_node_progress * 100.0) project["state"] = ProjectState( share_state=share_state, state=ProjectRunningState(value=running_state) From 9fb1220a5bdf0800f94c8d00dc49b18a31b519c6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 08:24:48 +0200 Subject: [PATCH 110/186] fix: project states --- .../simcore_service_webserver/projects/_projects_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 3ce7adda553..b63f8f62462 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1865,9 +1865,9 @@ async def add_project_states_for_user( node_state_dict = json_loads( node_state.model_dump_json(by_alias=True, exclude_unset=True) ) - prj_node.state = NodeState.model_validate(node_state_dict) + prj_node["state"] = NodeState.model_validate(node_state_dict) prj_node_progress = node_state_dict.get("progress", None) or 0 - prj_node.progress = round(prj_node_progress * 100.0) + prj_node["progress"] = round(prj_node_progress * 100.0) project["state"] = ProjectState( share_state=share_state, state=ProjectRunningState(value=running_state) From 5757eb8829c0cc55329f8c7783563e1b0ee1e657 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 09:54:00 +0200 Subject: [PATCH 111/186] fix: validator --- .../src/models_library/api_schemas_webserver/projects.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 39bdfe4be26..88259d8c6a6 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -126,7 +126,7 @@ class ProjectGet(OutputSchema): # display name: str - description: str + description: Annotated[str, BeforeValidator(none_to_empty_str_pre_validator)] thumbnail: HttpUrl | Literal[""] type: ProjectType @@ -166,10 +166,6 @@ class ProjectGet(OutputSchema): workspace_id: WorkspaceID | None folder_id: FolderID | None - _empty_description = field_validator("description", mode="before")( - none_to_empty_str_pre_validator - ) - @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: schema.update( From 19834d209ce2f8906a027dd045897a99e9731b88 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 09:55:13 +0200 Subject: [PATCH 112/186] fix: get project with None --- .../projects/_controller/projects_rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 538644fc2a2..bb1624ead0b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -282,7 +282,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).data() return envelope_json_response(data) From bbc6560ee299cd9fe2a85de84208936abfa0335e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 10:03:35 +0200 Subject: [PATCH 113/186] fix: get project --- .../unit/with_dbs/02/test_projects_nodes_handlers__patch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index d00ca810b5a..16937bd6fc1 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -166,7 +166,9 @@ async def test_patch_project_node( "output_1": { "store": 0, "path": "9934cba6-4b51-11ef-968a-02420a00f1c1/571ffc8d-fa6e-411f-afc8-9c62d08dd2fa/matus.txt", + "label": "matus.txt", "eTag": "d41d8cd98f00b204e9800998ecf8427e", + "dataset": None, } } } From 393e7dbe4b4004016d8d569f3795c462210bc244 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 11:48:15 +0200 Subject: [PATCH 114/186] fix: tags --- .../projects/_controller/projects_rest.py | 3 +-- .../src/simcore_service_webserver/projects/_tags_service.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index bb1624ead0b..6837c493416 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -281,8 +281,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - - data = ProjectGet.from_domain_model(project).data() + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return envelope_json_response(data) diff --git a/services/web/server/src/simcore_service_webserver/projects/_tags_service.py b/services/web/server/src/simcore_service_webserver/projects/_tags_service.py index d7f1af590a2..e733747e67e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_tags_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_tags_service.py @@ -8,6 +8,7 @@ from models_library.workspaces import UserWorkspaceWithAccessRights from ..workspaces import _workspaces_repository as workspaces_workspaces_repository +from . import _projects_repository from ._access_rights_service import check_user_project_permission from ._projects_repository_legacy import ProjectDBAPI from .models import ProjectDict @@ -20,7 +21,9 @@ async def add_tag( ) -> ProjectDict: db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) - product_name = await db.get_project_product(project_uuid) + product_name = await _projects_repository.get_project_product( + app, project_uuid=project_uuid + ) await check_user_project_permission( app, project_id=project_uuid, From 546003f298e792c8ac8611723ff62c2fb276c658 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 14:16:37 +0200 Subject: [PATCH 115/186] fix: workbench --- .../utils_projects_nodes.py | 68 +++++++++---------- .../projects/_projects_service.py | 5 +- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 3678043eb77..d397d96b42a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -122,41 +122,39 @@ def make_workbench_subquery() -> Subquery: projects_nodes.c.project_uuid, sa.func.json_object_agg( projects_nodes.c.node_id, - sa.func.json_strip_nulls( - sa.func.json_build_object( - "key", - projects_nodes.c.key, - "version", - projects_nodes.c.version, - "label", - projects_nodes.c.label, - "progress", - projects_nodes.c.progress, - "thumbnail", - projects_nodes.c.thumbnail, - "inputAccess", - projects_nodes.c.input_access, - "inputNodes", - projects_nodes.c.input_nodes, - "inputs", - projects_nodes.c.inputs, - "inputsRequired", - projects_nodes.c.inputs_required, - "inputsUnits", - projects_nodes.c.inputs_units, - "outputNodes", - projects_nodes.c.output_nodes, - "outputs", - projects_nodes.c.outputs, - "runHash", - projects_nodes.c.run_hash, - "state", - projects_nodes.c.state, - "parent", - projects_nodes.c.parent, - "bootOptions", - projects_nodes.c.boot_options, - ), + sa.func.json_build_object( + "key", + projects_nodes.c.key, + "version", + projects_nodes.c.version, + "label", + projects_nodes.c.label, + "progress", + projects_nodes.c.progress, + "thumbnail", + projects_nodes.c.thumbnail, + "inputAccess", + projects_nodes.c.input_access, + "inputNodes", + projects_nodes.c.input_nodes, + "inputs", + projects_nodes.c.inputs, + "inputsRequired", + projects_nodes.c.inputs_required, + "inputsUnits", + projects_nodes.c.inputs_units, + "outputNodes", + projects_nodes.c.output_nodes, + "outputs", + projects_nodes.c.outputs, + "runHash", + projects_nodes.c.run_hash, + "state", + projects_nodes.c.state, + "parent", + projects_nodes.c.parent, + "bootOptions", + projects_nodes.c.boot_options, ), ).label("workbench"), ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index b63f8f62462..7971afc8449 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -267,7 +267,10 @@ async def get_project_for_user( """ db = ProjectDBAPI.get_from_app_context(app) - product_name = await db.get_project_product(ProjectID(project_uuid)) + product_name = await _projects_repository.get_project_product( + app, project_uuid=ProjectID(project_uuid) + ) + user_project_access = await check_user_project_permission( app, project_id=ProjectID(project_uuid), From e2df3dd0733485da080c20ac220ef2d172c45e13 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 15:22:46 +0200 Subject: [PATCH 116/186] fix: get project response --- .../projects/_controller/projects_rest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 6837c493416..e4a9a8a2c42 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -25,6 +25,7 @@ X_SIMCORE_USER_AGENT, ) from servicelib.redis import get_project_locked_state +from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..._meta import API_VTAG as VTAG from ...login.decorators import login_required @@ -281,7 +282,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).model_dump(**RESPONSE_MODEL_POLICY) return envelope_json_response(data) From 20f70f6b5608859be7b81e59193ac84198028939 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 16:19:29 +0200 Subject: [PATCH 117/186] fix: workbench --- .../projects/_projects_repository_legacy_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 005de83d0ba..f16fe98209b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -8,7 +8,7 @@ import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy -from models_library.projects import NodesDict, ProjectID, ProjectType +from models_library.projects import ProjectID, ProjectType from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeIDStr from models_library.utils.change_case import camel_to_snake, snake_to_camel @@ -195,17 +195,17 @@ async def _upsert_tags_in_project( async def _get_workbench( connection: SAConnection, project_uuid: str, - ) -> NodesDict: + ) -> dict[str, Any]: project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid)) exclude_fields = {"node_id", "required_resources", "created", "modified"} - workbench: NodesDict = {} + workbench: dict[str, Any] = {} project_nodes = await project_nodes_repo.list(connection) # type: ignore for project_node in project_nodes: node_data = project_node.model_dump( exclude=exclude_fields, exclude_none=True, exclude_unset=True ) - workbench[f"{project_node.node_id}"] = Node.model_validate(node_data) + workbench[f"{project_node.node_id}"] = node_data return workbench async def _get_project( From 131902091ad43f70f6a59686668bcf22f0fcf030 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 16:41:14 +0200 Subject: [PATCH 118/186] fix: node state --- .../director-v2/tests/integration/01/test_computation_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 4b826ec69d0..c55ef064e9b 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -432,7 +432,9 @@ def _convert_to_pipeline_details( NodeID(workbench_node_uuids[dep_n]) for dep_n in node_state["dependencies"] }, - currentStatus=node_state.get("currentStatus", RunningState.NOT_STARTED), + current_status=node_state.get( + "currentStatus", RunningState.NOT_STARTED + ), progress=node_state.get("progress"), ) for node_index, node_state in expected_node_states.items() From bfa1685272a9fbeeda5f8536e373cd6bee785311 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 13 Aug 2025 22:50:19 +0200 Subject: [PATCH 119/186] remove workbench --- .../tests/integration/01/test_computation.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/services/web/server/tests/integration/01/test_computation.py b/services/web/server/tests/integration/01/test_computation.py index 0fd534ea1b8..fdd3026e482 100644 --- a/services/web/server/tests/integration/01/test_computation.py +++ b/services/web/server/tests/integration/01/test_computation.py @@ -25,8 +25,8 @@ from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_postgres_database.models.comp_runs_collections import comp_runs_collections -from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_metadata import projects_metadata +from simcore_postgres_database.models.projects_nodes import projects_nodes from simcore_postgres_database.models.users import UserRole from simcore_postgres_database.webserver_models import ( NodeClass, @@ -245,17 +245,17 @@ def _get_project_workbench_from_db( # this check is only there to check the comp_pipeline is there print(f"--> looking for project {project_id=} in projects table...") with postgres_db.connect() as conn: - project_in_db = conn.execute( - sa.select(projects).where(projects.c.uuid == project_id) - ).fetchone() + rows = ( + conn.execute( + sa.select(projects_nodes).where( + projects_nodes.c.project_uuid == project_id + ) + ) + .mappings() + .all() + ) # list[dict] - assert ( - project_in_db - ), f"missing pipeline in the database under comp_pipeline {project_id}" - print( - f"<-- found following workbench: {json_dumps(project_in_db.workbench, indent=2)}" - ) - return project_in_db.workbench + return {row["node_id"]: dict(row) for row in rows} async def _assert_and_wait_for_pipeline_state( From 044edc59550da32cf73b5c9351d1899725a9a713 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 10:31:03 +0200 Subject: [PATCH 120/186] fix: repo --- .../modules/db/repositories/projects.py | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 4be5a9cf13e..1bd071a00ed 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -2,9 +2,11 @@ import sqlalchemy as sa from models_library.projects import ProjectAtDB, ProjectID -from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID -from simcore_postgres_database.utils_projects_nodes import ProjectNode, ProjectNodesRepo +from simcore_postgres_database.utils_projects_nodes import ( + ProjectNodesRepo, + make_workbench_subquery, +) from simcore_postgres_database.utils_repos import pass_or_acquire_connection from ....core.errors import ProjectNotFoundError @@ -14,48 +16,47 @@ logger = logging.getLogger(__name__) -def _project_node_to_node(project_node: ProjectNode) -> Node: - """Converts a ProjectNode from the database to a Node model for the API. - - Handles field mapping and excludes database-specific fields that are not - part of the Node model. - """ - node_data = project_node.model_dump_as_node() - return Node.model_validate(node_data) - - class ProjectsRepository(BaseRepository): async def get_project(self, project_id: ProjectID) -> ProjectAtDB: + workbench_subquery = make_workbench_subquery() + async with self.db_engine.connect() as conn: - row = ( - await conn.execute( - sa.select(projects).where(projects.c.uuid == str(project_id)) + query = ( + sa.select( + projects, + sa.func.coalesce( + workbench_subquery.c.workbench, sa.text("'{}'::json") + ).label("workbench"), + ) + .select_from( + projects.outerjoin( + workbench_subquery, + projects.c.uuid == workbench_subquery.c.project_uuid, + ) ) - ).one_or_none() + .where(projects.c.uuid == str(project_id)) + ) + result = await conn.execute(query) + row = result.one_or_none() if not row: raise ProjectNotFoundError(project_id=project_id) - - repo = ProjectNodesRepo(project_uuid=project_id) - nodes = await repo.list(conn) - - project_workbench = { - f"{node.node_id}": _project_node_to_node(node) for node in nodes - } - data = {**row._asdict(), "workbench": project_workbench} - return ProjectAtDB.model_validate(data) + return ProjectAtDB.model_validate(row) async def is_node_present_in_workbench( self, project_id: ProjectID, node_uuid: NodeID ) -> bool: async with pass_or_acquire_connection(self.db_engine) as conn: - result = await conn.execute( - sa.select(projects_nodes.c.project_node_id).where( + stmt = ( + sa.select(sa.literal(1)) + .where( projects_nodes.c.project_uuid == str(project_id), projects_nodes.c.node_id == str(node_uuid), ) + .limit(1) ) - project_node = result.one_or_none() - return project_node is not None + + result = await conn.execute(stmt) + return result.scalar_one_or_none() is not None async def get_project_id_from_node(self, node_id: NodeID) -> ProjectID: async with self.db_engine.connect() as conn: From de831c42ba3fd449c0cf535f1b478a37ea6b03bc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 11:17:45 +0200 Subject: [PATCH 121/186] fix: new syntax --- .../src/simcore_sdk/node_ports_v2/port.py | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 014aff56529..4821feb3606 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -114,10 +114,9 @@ def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue: and not isinstance(v, PortLink) ): if port_utils.is_file_type(property_type): - if not isinstance(v, (FileLink, DownloadLink)): - raise ValueError( - f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas" - ) + if not isinstance(v, FileLink | DownloadLink): + msg = f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas" + raise ValueError(msg) elif property_type == "ref_contentSchema": v, _ = validate_port_content( port_key=info.data.get("key"), @@ -125,10 +124,11 @@ def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue: unit=None, content_schema=info.data.get("content_schema", {}), ) - elif isinstance(v, (list, dict)): - raise TypeError( + elif isinstance(v, list | dict): + msg = ( f"Containers as {v} currently only supported within content_schema." ) + raise TypeError(msg) return v @field_validator("value_item", "value_concrete", mode="before") @@ -196,26 +196,26 @@ async def get_value( async def _evaluate() -> ItemValue | None: if isinstance(self.value, PortLink): # this is a link to another node's port - other_port_itemvalue: None | ( - ItemValue - ) = await port_utils.get_value_link_from_port_link( - self.value, - # pylint: disable=protected-access - self._node_ports._node_ports_creator_cb, - file_link_type=file_link_type, + other_port_itemvalue: None | (ItemValue) = ( + await port_utils.get_value_link_from_port_link( + self.value, + # pylint: disable=protected-access + self._node_ports._node_ports_creator_cb, + file_link_type=file_link_type, + ) ) return other_port_itemvalue if isinstance(self.value, FileLink): # let's get the download/upload link from storage - url_itemvalue: None | ( - AnyUrl - ) = await port_utils.get_download_link_from_storage( - # pylint: disable=protected-access - user_id=self._node_ports.user_id, - value=self.value, - link_type=file_link_type, + url_itemvalue: None | (AnyUrl) = ( + await port_utils.get_download_link_from_storage( + # pylint: disable=protected-access + user_id=self._node_ports.user_id, + value=self.value, + link_type=file_link_type, + ) ) return url_itemvalue @@ -256,15 +256,15 @@ async def _evaluate() -> ItemConcreteValue | None: if isinstance(self.value, PortLink): # this is a link to another node - other_port_concretevalue: None | ( - ItemConcreteValue - ) = await port_utils.get_value_from_link( - # pylint: disable=protected-access - key=self.key, - value=self.value, - file_to_key_map=self.file_to_key_map, - node_port_creator=self._node_ports._node_ports_creator_cb, # noqa: SLF001 - progress_bar=progress_bar, + other_port_concretevalue: None | (ItemConcreteValue) = ( + await port_utils.get_value_from_link( + # pylint: disable=protected-access + key=self.key, + value=self.value, + file_to_key_map=self.file_to_key_map, + node_port_creator=self._node_ports._node_ports_creator_cb, # noqa: SLF001 + progress_bar=progress_bar, + ) ) value = other_port_concretevalue From 9dada931e9741172548ef406d6cc473d2b43ade2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 12:55:32 +0200 Subject: [PATCH 122/186] fix: include unset --- .../projects/_projects_nodes_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 1823cb12036..e2205b8bd10 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -51,7 +51,7 @@ async def add( node_id: NodeID, node: Node, ) -> None: - values = node.model_dump(mode="json", exclude_none=True, exclude_unset=True) + values = node.model_dump(mode="json", exclude_none=True) async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( From d453f768fb68d454c5a56f22e92c2c25e4295e80 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 13:20:22 +0200 Subject: [PATCH 123/186] tests: prettify deepdiff --- .../src/pytest_simcore/helpers/webserver_projects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index a57ffd975e2..040a5f5914b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -214,7 +214,8 @@ async def assert_get_same_project( # without our control if not error: - assert not DeepDiff( + diff = DeepDiff( data, {k: project[k] for k in data}, exclude_paths="root['lastChangeDate']" ) + assert not diff, diff.pretty() return data From f29cf8c30dd50449b45ddcfcd9afde4412128204 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 13:27:07 +0200 Subject: [PATCH 124/186] fix: test --- .../tests/integration/01/test_computation_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index c55ef064e9b..5e58cbf66ec 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -521,9 +521,9 @@ def _convert_to_pipeline_details( # force run it this time. # the task are up-to-date but we force run them expected_pipeline_details_forced = _convert_to_pipeline_details( - sleepers_project, - params.exp_pipeline_adj_list_after_force_run, - params.exp_node_states_after_force_run, + workbench_node_uuids=list(sleepers_project.workbench.keys()), + expected_pipeline_adj_list=params.exp_pipeline_adj_list_after_force_run, + expected_node_states=params.exp_node_states_after_force_run, ) task_out = await create_pipeline( async_client, From 7928dfb4ed5a491605df808741ec061457fdffb8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 16:33:10 +0200 Subject: [PATCH 125/186] fix: model dump without aliases --- .../pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index e638fdcb9e6..0d1de1d3ee7 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -142,10 +142,7 @@ async def _( # NOTE: currently no resources is passed until it becomes necessary project_workbench_node = { "required_resources": {}, - "key": random_service_key(fake=faker), - "version": random_service_version(fake=faker), - "label": faker.pystr(), - **node_model.model_dump(mode="json", by_alias=False), + **node_model.model_dump(mode="json"), } if project_nodes_overrides: From 88c31597edce18b61c14b1531ded4677838a792f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 16:39:29 +0200 Subject: [PATCH 126/186] typecheck --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 0d1de1d3ee7..0ac157a1d26 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -30,7 +30,6 @@ ) from sqlalchemy.ext.asyncio import AsyncEngine -from .helpers.faker_factories import random_service_key, random_service_version from .helpers.postgres_tools import insert_and_get_row_lifespan from .helpers.postgres_users import sync_insert_and_get_user_and_secrets_lifespan From 5dd1c82883940a703527c0d047a7be2cf936b1ed Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 21:24:17 +0200 Subject: [PATCH 127/186] fix: node creation --- .../pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 0ac157a1d26..1ddee94816a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -136,7 +136,7 @@ async def _( for node_id, node_data in project_workbench.items(): # NOTE: workbench node have a lot of camecase fields. We validate with Node and # export to ProjectNodeCreate with alias=False - node_model = Node.model_validate(node_data) + node_model = ProjectNodeCreate.model_validate({"node_id": node_id, **node_data}, from_attributes=True) # NOTE: currently no resources is passed until it becomes necessary project_workbench_node = { @@ -150,9 +150,7 @@ async def _( await project_nodes_repo.add( con, nodes=[ - ProjectNodeCreate( - node_id=NodeID(node_id), **project_workbench_node - ) + node_model ], ) From c1826f803408c2c7e000ba1e74bbbd8be0b61c3f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 21:31:41 +0200 Subject: [PATCH 128/186] fix: typecheck --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 1ddee94816a..e3f84d6c08a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -14,8 +14,6 @@ from faker import Faker from models_library.products import ProductName from models_library.projects import ProjectAtDB, ProjectID -from models_library.projects_nodes import Node -from models_library.projects_nodes_io import NodeID from pytest_simcore.helpers.logging_tools import log_context from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline from simcore_postgres_database.models.comp_tasks import comp_tasks From ae75ac182d1a04bdb905f86639a788e094381d06 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 23:32:57 +0200 Subject: [PATCH 129/186] fix: node creation --- .../src/pytest_simcore/db_entries_mocks.py | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index e3f84d6c08a..df6f6b843cb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -4,16 +4,20 @@ # pylint:disable=no-value-for-parameter import contextlib +from functools import partial import logging from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any from uuid import uuid4 +import uuid import pytest import sqlalchemy as sa from faker import Faker +from common_library.dict_tools import remap_keys from models_library.products import ProductName from models_library.projects import ProjectAtDB, ProjectID +from models_library.projects_nodes import Node from pytest_simcore.helpers.logging_tools import log_context from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline from simcore_postgres_database.models.comp_tasks import comp_tasks @@ -78,6 +82,9 @@ async def with_product( ) as created_product: yield created_product +class _NodeWithId(Node): + node_id: uuid.UUID + @pytest.fixture async def create_project( @@ -128,29 +135,43 @@ async def _( {**project_db_rows, "workbench": project_workbench} ) + nodes = [] async with sqlalchemy_async_engine.connect() as con, con.begin(): project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) for node_id, node_data in project_workbench.items(): # NOTE: workbench node have a lot of camecase fields. We validate with Node and # export to ProjectNodeCreate with alias=False - node_model = ProjectNodeCreate.model_validate({"node_id": node_id, **node_data}, from_attributes=True) + + node_model = Node.model_validate(node_data).model_dump(mode="json", by_alias=True) + + field_mapping = { + "inputAccess": "input_access", + "inputNodes": "input_nodes", + "inputsRequired": "inputs_required", + "inputsUnits": "inputs_units", + "outputNodes": "output_nodes", + "runHash": "run_hash", + "bootOptions": "boot_options", + } + + node = remap_keys(node_model, field_mapping) # NOTE: currently no resources is passed until it becomes necessary project_workbench_node = { "required_resources": {}, - **node_model.model_dump(mode="json"), + **node, } if project_nodes_overrides: project_workbench_node.update(project_nodes_overrides) - await project_nodes_repo.add( - con, - nodes=[ - node_model - ], - ) + nodes.append(ProjectNodeCreate(node_id=node_id, **project_workbench_node)) + + await project_nodes_repo.add( + con, + nodes=nodes, + ) await con.execute( projects_to_products.insert().values( From 3547a2128a953f583ed3fc4d8e25a76e09da2b7e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Aug 2025 23:38:32 +0200 Subject: [PATCH 130/186] typecheck --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index df6f6b843cb..23330a6d062 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -4,7 +4,6 @@ # pylint:disable=no-value-for-parameter import contextlib -from functools import partial import logging from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any From 6104f45026811cc8e5c595486a1c0534192fdc6b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 09:42:53 +0200 Subject: [PATCH 131/186] fix: response --- .../projects/_controller/projects_rest.py | 5 ++--- .../tests/unit/with_dbs/02/test_projects_states_handlers.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index e4a9a8a2c42..3fcf6784c67 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -25,7 +25,6 @@ X_SIMCORE_USER_AGENT, ) from servicelib.redis import get_project_locked_state -from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..._meta import API_VTAG as VTAG from ...login.decorators import login_required @@ -250,7 +249,7 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).model_dump(by_alias=True, exclude_unset=True, exclude_none=True) return envelope_json_response(data) @@ -282,7 +281,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).model_dump(**RESPONSE_MODEL_POLICY) + data = ProjectGet.from_domain_model(project).model_dump(by_alias=True, exclude_unset=True, exclude_none=True) return envelope_json_response(data) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 806d5b1b4d6..ad80870ff6d 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -1177,7 +1177,7 @@ async def test_get_active_project( ) assert not error assert ProjectStateOutputSchema(**data.pop("state")).share_state.locked - data.pop("folderId") + data.pop("folderId", None) user_project_last_change_date = user_project.pop("lastChangeDate") data_last_change_date = data.pop("lastChangeDate") From 994adda3c5a473cd49919749a250ce76eb98cc59 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 10:03:35 +0200 Subject: [PATCH 132/186] fix: match --- .../src/common_library/dict_tools.py | 12 ++++++++ .../common-library/tests/test_dict_tools.py | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/packages/common-library/src/common_library/dict_tools.py b/packages/common-library/src/common_library/dict_tools.py index 43ef7166308..164218f9885 100644 --- a/packages/common-library/src/common_library/dict_tools.py +++ b/packages/common-library/src/common_library/dict_tools.py @@ -58,3 +58,15 @@ def update_dict(obj: dict, **updates): {key: update_value(obj[key]) if callable(update_value) else update_value} ) return obj + + +def assert_equal_ignoring_none(expected: dict, actual: dict): + for key, exp_value in expected.items(): + if exp_value is None: + continue + assert key in actual, f"Missing key {key}" + act_value = actual[key] + if isinstance(exp_value, dict) and isinstance(act_value, dict): + assert_equal_ignoring_none(exp_value, act_value) + else: + assert act_value == exp_value, f"Mismatch in {key}: {act_value} != {exp_value}" diff --git a/packages/common-library/tests/test_dict_tools.py b/packages/common-library/tests/test_dict_tools.py index fb374ff1791..f3cacdbed3b 100644 --- a/packages/common-library/tests/test_dict_tools.py +++ b/packages/common-library/tests/test_dict_tools.py @@ -6,6 +6,7 @@ from typing import Any import pytest +from common_library.dict_tools import assert_equal_ignoring_none from common_library.dict_tools import ( copy_from_dict, get_from_dict, @@ -160,3 +161,31 @@ def test_copy_from_dict(data: dict[str, Any]): assert selected_data["Status"]["State"] == data["Status"]["State"] assert "Message" not in selected_data["Status"]["State"] assert "running" in data["Status"]["State"] + + +@pytest.mark.parametrize( + "expected, actual", + [ + ({"a": 1, "b": 2}, {"a": 1, "b": 2, "c": 3}), + ({"a": 1, "b": None}, {"a": 1, "b": 42}), + ({"a": {"x": 10, "y": None}}, {"a": {"x": 10, "y": 99}}), + ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 20, "z": 30}}), + ({}, {"foo": "bar"}), + ], +) +def test_assert_equal_ignoring_none_passes(expected, actual): + assert_equal_ignoring_none(expected, actual) + +@pytest.mark.parametrize( + "expected, actual, error_msg", + [ + ({"a": 1, "b": 2}, {"a": 1}, "Missing key b"), + ({"a": 1, "b": 2}, {"a": 1, "b": 3}, "Mismatch in b: 3 != 2"), + ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 99}}, "Mismatch in y: 99 != 20"), + ({"a": {"x": 10}}, {"a": {}}, "Missing key x"), + ], +) +def test_assert_equal_ignoring_none_fails(expected, actual, error_msg): + with pytest.raises(AssertionError) as exc_info: + assert_equal_ignoring_none(expected, actual) + assert error_msg in str(exc_info.value) From f6ba587adf822eb8beb3870b2c730200b9d4b4c7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 10:07:43 +0200 Subject: [PATCH 133/186] fix: nullability --- .../models_library/api_schemas_webserver/projects.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 88259d8c6a6..d5eacc68368 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -130,7 +130,7 @@ class ProjectGet(OutputSchema): thumbnail: HttpUrl | Literal[""] type: ProjectType - template_type: ProjectTemplateType | None + template_type: ProjectTemplateType | None = None workbench: NodesDict @@ -141,10 +141,10 @@ class ProjectGet(OutputSchema): creation_date: DateTimeStr last_change_date: DateTimeStr state: ProjectStateOutputSchema | None = None - trashed_at: datetime | None + trashed_at: datetime | None = None trashed_by: Annotated[ GroupID | None, Field(description="The primary gid of the user who trashed") - ] + ] = None # labeling tags: list[int] @@ -163,8 +163,8 @@ class ProjectGet(OutputSchema): permalink: ProjectPermalink | None = None - workspace_id: WorkspaceID | None - folder_id: FolderID | None + workspace_id: WorkspaceID | None = None + folder_id: FolderID | None = None @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: From da639868a0ee9d58164c5696db3c75da100aab6b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 10:15:46 +0200 Subject: [PATCH 134/186] fix test --- .../tests/unit/with_dbs/02/test_projects_crud_handlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index d365f621c86..209101cb681 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -14,6 +14,7 @@ import sqlalchemy as sa from aiohttp.test_utils import TestClient from aioresponses import aioresponses +from common_library.dict_tools import assert_equal_ignoring_none from deepdiff import DeepDiff from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import ( @@ -413,6 +414,7 @@ async def test_list_projects_with_innaccessible_services( (UserRole.TESTER, status.HTTP_200_OK), ], ) +@pytest.mark.testit async def test_get_project( client: TestClient, logged_user: UserInfoDict, @@ -638,7 +640,7 @@ async def test_new_template_from_project( ) assert len(templates) == 1 - assert templates[0] == template_project + assert_equal_ignoring_none(template_project, templates[0], user_project) assert template_project["name"] == user_project["name"] assert template_project["description"] == user_project["description"] From 0df85db162eb0261976261a866a206a25a8b1689 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:03:49 +0200 Subject: [PATCH 135/186] remove workbench --- tests/e2e/tutorials/sleepers_project_template_sql.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/tutorials/sleepers_project_template_sql.csv b/tests/e2e/tutorials/sleepers_project_template_sql.csv index 7e96b580dcf..1cd53e34516 100644 --- a/tests/e2e/tutorials/sleepers_project_template_sql.csv +++ b/tests/e2e/tutorials/sleepers_project_template_sql.csv @@ -1,2 +1,2 @@ -id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,dev,classifiers,ui,quality,hidden,workspace_id,trashed,trashed_explicitly,trashed_by,template_type -10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,"{""027e3ff9-3119-45dd-b8a2-2e31661a7385"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 0"", ""inputs"": {""in_2"": 2}, ""inputAccess"": {""in_1"": ""Invisible"", ""in_2"": ""ReadOnly""}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 300}}, ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 1"", ""inputs"": {""in_1"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_1""}, ""in_2"": 2}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 200}}, ""bf405067-d168-44ba-b6dc-bb3e08542f92"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 2"", ""inputs"": {""in_1"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_2""}}, ""inputNodes"": [""562aaea9-95ff-46f3-8e84-db8f3c9e3a39""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 550, ""y"": 200}}, ""de2578c5-431e-5065-a079-a5a0476e3c10"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 3"", ""inputs"": {""in_2"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_2""}}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 420, ""y"": 400}}, ""de2578c5-431e-559d-aa19-dc9293e10e4c"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 4"", ""inputs"": {""in_1"": {""nodeUuid"": ""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""de2578c5-431e-5065-a079-a5a0476e3c10"", ""output"": ""out_2""}}, ""inputNodes"": [""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""de2578c5-431e-5065-a079-a5a0476e3c10""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 800, ""y"": 300}}}",t,"{""1"": {""read"": true, ""write"": false, ""delete"": false}}",{},{},{},{},f,,,f,,TEMPLATE +id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,published,access_rights,dev,classifiers,ui,quality,hidden,workspace_id,trashed,trashed_explicitly,trashed_by,template_type +10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,t,"{""1"": {""read"": true, ""write"": false, ""delete"": false}}",{},{},{},{},f,,,f,,TEMPLATE From e9793e1d52e3fb72916021762310f8b845bed29b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:19:54 +0200 Subject: [PATCH 136/186] fix: project creation with nodes --- .../projects/_projects_repository_legacy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 9eab4f4eb48..4fc772ab25c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -26,6 +26,7 @@ ProjectTemplateType, ) from models_library.projects_comments import CommentID, ProjectsCommentsDB +from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.resource_tracker import ( PricingPlanAndUnitIdsTuple, @@ -325,7 +326,7 @@ async def insert_project( node_id=NodeID(node_id), **{ str(field_mapping.get(field, field)): value - for field, value in project_workbench_node.items() + for field, value in Node.model_validate(project_workbench_node).model_dump(mode="json", by_alias=True).items() if field_mapping.get(field, field) in valid_fields }, ) From 198bb4b2eee11e905cedd90795170cc703d90a60 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:21:09 +0200 Subject: [PATCH 137/186] clean --- .../server/tests/unit/with_dbs/02/test_projects_crud_handlers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 209101cb681..92a1efc8cf5 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -414,7 +414,6 @@ async def test_list_projects_with_innaccessible_services( (UserRole.TESTER, status.HTTP_200_OK), ], ) -@pytest.mark.testit async def test_get_project( client: TestClient, logged_user: UserInfoDict, From 2748c5a8fc4512eb753ee5727b5eba7d1848eb96 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:26:15 +0200 Subject: [PATCH 138/186] remove unused --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 23330a6d062..a6054142aa4 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -81,9 +81,6 @@ async def with_product( ) as created_product: yield created_product -class _NodeWithId(Node): - node_id: uuid.UUID - @pytest.fixture async def create_project( From f4655a97184f0212c11a81b8643e490d4f538de5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:27:06 +0200 Subject: [PATCH 139/186] remove fixme --- packages/models-library/src/models_library/projects.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index da7218e0774..e66c5a54543 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -107,8 +107,6 @@ class BaseProjectModel(BaseModel): last_change_date: datetime # Pipeline of nodes (SEE projects_nodes.py) - # FIXME: pedro checks this one - # NOTE: GCR: a validation failed (See: services/storage/src/simcore_service_storage/modules/db/projects.py) workbench: Annotated[NodesDict, Field(description="Project's pipeline")] From b8c966ef70765047ca658e502308421398c3434c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:36:20 +0200 Subject: [PATCH 140/186] fix: streams --- .../projects/_projects_nodes_repository.py | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index e2205b8bd10..f79e3f409ea 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -113,23 +113,23 @@ async def get_by_project( projects_nodes.c.project_uuid == f"{project_id}" ) - result = await conn.stream(query) - assert result # nosec + stream = await conn.stream(query) + assert stream # nosec - rows = await result.all() - return [ - ( - NodeID(row.node_id), - Node.model_validate( - ProjectNode.model_validate(row, from_attributes=True).model_dump( - exclude_none=True, - exclude_unset=True, - exclude={"node_id", "created", "modified"}, - ) - ), + result: list[tuple[NodeID, Node]] = [] + async for row in stream: + # build Model only once on top of row + pn = ProjectNode.model_validate(row, from_attributes=True) + node = Node.model_validate( + pn.model_dump( + exclude_none=True, + exclude_unset=True, + exclude={"node_id", "created", "modified"}, + ) ) - for row in rows - ] + result.append((NodeID(row.node_id), node)) + + return result async def get_by_projects( @@ -145,10 +145,8 @@ async def get_by_projects( projects_nodes.c.project_uuid.in_([f"{pid}" for pid in project_ids]) ) - result = await conn.stream(query) - assert result # nosec - - rows = await result.all() + stream = await conn.stream(query) + assert stream # nosec # Initialize dict with empty lists for all requested project_ids projects_to_nodes: dict[ProjectID, list[tuple[NodeID, Node]]] = { @@ -156,7 +154,7 @@ async def get_by_projects( } # Fill in the actual data - for row in rows: + async for row in stream: node = Node.model_validate( ProjectNode.model_validate(row).model_dump( exclude_none=True, From 8eabd509b75006a9481253ac2f8be7b19df695a1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:37:03 +0200 Subject: [PATCH 141/186] fix relative --- .../src/simcore_service_webserver/projects/_nodes_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index 5dab5590224..fc438c09c24 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -11,7 +11,7 @@ pass_or_acquire_connection, transaction_context, ) -from simcore_service_webserver.db.plugin import get_asyncpg_engine +from ..db.plugin import get_asyncpg_engine async def get_project_nodes_services( From a415e875a4cc6d2b86e736a8b6a0be80b2a1d64e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:40:07 +0200 Subject: [PATCH 142/186] fix: revert casing --- .../src/pytest_simcore/helpers/webserver_projects.py | 2 +- services/web/server/tests/integration/01/conftest.py | 4 ++-- services/web/server/tests/unit/conftest.py | 4 ++-- services/web/server/tests/unit/with_dbs/02/conftest.py | 10 +++++----- .../web/server/tests/unit/with_dbs/03/tags/conftest.py | 6 +++--- .../server/tests/unit/with_dbs/03/trash/conftest.py | 4 ++-- .../test_studies_dispatcher_studies_access.py | 6 +++--- .../server/tests/unit/with_dbs/04/wallets/conftest.py | 6 +++--- services/web/server/tests/unit/with_dbs/conftest.py | 4 ++-- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 040a5f5914b..ed54246ef6a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -166,7 +166,7 @@ async def delete_all_projects(app: web.Application): @asynccontextmanager -async def new_project( +async def NewProject( params_override: dict | None = None, app: web.Application | None = None, *, diff --git a/services/web/server/tests/integration/01/conftest.py b/services/web/server/tests/integration/01/conftest.py index c2b6407e166..261e27bf067 100644 --- a/services/web/server/tests/integration/01/conftest.py +++ b/services/web/server/tests/integration/01/conftest.py @@ -9,7 +9,7 @@ import pytest from models_library.projects import ProjectID -from pytest_simcore.helpers.webserver_projects import new_project +from pytest_simcore.helpers.webserver_projects import NewProject @pytest.fixture(scope="session") @@ -45,7 +45,7 @@ async def user_project( fake_project["prjOwner"] = logged_user["name"] fake_project["uuid"] = f"{project_id}" - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/conftest.py b/services/web/server/tests/unit/conftest.py index c646ef6011e..0755d30b3f9 100644 --- a/services/web/server/tests/unit/conftest.py +++ b/services/web/server/tests/unit/conftest.py @@ -16,7 +16,7 @@ from aiohttp.test_utils import TestClient from models_library.products import ProductName from pytest_mock import MockFixture, MockType -from pytest_simcore.helpers.webserver_projects import empty_project_data, new_project +from pytest_simcore.helpers.webserver_projects import empty_project_data, NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.constants import FRONTEND_APP_DEFAULT @@ -106,7 +106,7 @@ async def user_project( tests_data_dir: Path, osparc_product_name: str, ) -> AsyncIterator[ProjectDict]: - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index 87fb2f377ae..8aa6c60832b 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -30,7 +30,7 @@ from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project +from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from settings_library.catalog import CatalogSettings from simcore_service_webserver.application_settings import get_application_settings @@ -130,7 +130,7 @@ async def shared_project( }, }, ) - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], @@ -159,7 +159,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with new_project( + async with NewProject( project_data, client.app, user_id=user["id"], @@ -194,7 +194,7 @@ async def _creator(**prj_kwargs) -> ProjectDict: project_data |= prj_kwargs new_template_project = await created_projects_exit_stack.enter_async_context( - new_project( + NewProject( project_data, client.app, user_id=user["id"], @@ -292,7 +292,7 @@ async def _creator(num_dyn_services: int) -> ProjectDict: } } project = await stack.enter_async_context( - new_project( + NewProject( project_data, client.app, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index 665ffc1b94e..c5449a66a28 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -10,7 +10,7 @@ from aioresponses import aioresponses from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project +from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings @@ -110,7 +110,7 @@ async def shared_project( }, }, ) - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], @@ -139,7 +139,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with new_project( + async with NewProject( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/03/trash/conftest.py b/services/web/server/tests/unit/with_dbs/03/trash/conftest.py index 849840eca9f..f154713ec81 100644 --- a/services/web/server/tests/unit/with_dbs/03/trash/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/trash/conftest.py @@ -20,7 +20,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import new_project +from pytest_simcore.helpers.webserver_projects import NewProject from pytest_simcore.helpers.webserver_users import NewUser, UserInfoDict from simcore_service_webserver.projects.models import ProjectDict @@ -59,7 +59,7 @@ async def other_user_project( tests_data_dir: Path, osparc_product_name: ProductName, ) -> AsyncIterable[ProjectDict]: - async with new_project( + async with NewProject( fake_project, client.app, user_id=other_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 7fe7ee2f17e..a743333a0e0 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -30,7 +30,7 @@ from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project +from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE @@ -104,7 +104,7 @@ async def published_project( # everyone HAS read access "1": {"read": True, "write": False, "delete": False} } - async with new_project( + async with NewProject( project_data, client.app, user_id=user["id"], @@ -130,7 +130,7 @@ async def unpublished_project( project_data["uuid"] = "b134a337-a74f-40ff-a127-b36a1ccbede6" project_data["published"] = False # <-- - async with new_project( + async with NewProject( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py index d7c62e5de64..70784ebca61 100644 --- a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py @@ -13,7 +13,7 @@ from faker import Faker from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import delete_all_projects, new_project +from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver.application_settings import ApplicationSettings @@ -72,7 +72,7 @@ async def shared_project( }, }, ) - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], @@ -101,7 +101,7 @@ async def template_project( str(all_group["gid"]): {"read": True, "write": False, "delete": False} } - async with new_project( + async with NewProject( project_data, client.app, user_id=user["id"], diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 71309bbe83e..5705c4b95ca 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -50,7 +50,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem -from pytest_simcore.helpers.webserver_projects import new_project +from pytest_simcore.helpers.webserver_projects import NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from redis import Redis from servicelib.aiohttp.application_keys import APP_AIOPG_ENGINE_KEY @@ -683,7 +683,7 @@ async def user_project( tests_data_dir: Path, osparc_product_name: ProductName, ) -> AsyncIterator[ProjectDict]: - async with new_project( + async with NewProject( fake_project, client.app, user_id=logged_user["id"], From 13bbe347101a60922d9519e31fc00b36af74b203 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:42:14 +0200 Subject: [PATCH 143/186] fix random --- services/storage/tests/unit/test_handlers_paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index fff8f1af9bb..7f61991494b 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -10,6 +10,7 @@ import random from collections.abc import Awaitable, Callable from pathlib import Path +import secrets from typing import Any, TypeAlias from urllib.parse import quote @@ -327,9 +328,9 @@ async def test_list_paths( ) # ls with only some part of the path should return only the projects that match - selected_project, selected_nodes, selected_project_files = random.choice( + selected_project, selected_nodes, selected_project_files = secrets.choice( project_to_files_mapping - ) # noqa: S311 + ) partial_file_filter = Path( selected_project["uuid"][: len(selected_project["uuid"]) // 2] ) From 99c2eb728060713a8f5195a292615bf49e60bdf2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:45:04 +0200 Subject: [PATCH 144/186] rename --- .../src/simcore_postgres_database/utils_projects_nodes.py | 2 +- .../modules/db/repositories/projects.py | 4 ++-- .../simcore_service_webserver/projects/_jobs_repository.py | 4 ++-- .../projects/_projects_repository.py | 4 ++-- .../projects/_projects_repository_legacy.py | 6 +++--- .../projects/_projects_repository_legacy_utils.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index d397d96b42a..e55692702fe 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -116,7 +116,7 @@ def model_dump_as_node(self) -> dict[str, Any]: ) -def make_workbench_subquery() -> Subquery: +def create_workbench_subquery() -> Subquery: return ( sa.select( projects_nodes.c.project_uuid, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 1bd071a00ed..965f9b6dd2b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -5,7 +5,7 @@ from models_library.projects_nodes_io import NodeID from simcore_postgres_database.utils_projects_nodes import ( ProjectNodesRepo, - make_workbench_subquery, + create_workbench_subquery, ) from simcore_postgres_database.utils_repos import pass_or_acquire_connection @@ -18,7 +18,7 @@ class ProjectsRepository(BaseRepository): async def get_project(self, project_id: ProjectID) -> ProjectAtDB: - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() async with self.db_engine.connect() as conn: query = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 83c1ff6a97b..b398665644d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -11,7 +11,7 @@ from simcore_postgres_database.models.projects_metadata import projects_metadata from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.models.projects_to_products import projects_to_products -from simcore_postgres_database.utils_projects_nodes import make_workbench_subquery +from simcore_postgres_database.utils_projects_nodes import create_workbench_subquery from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -171,7 +171,7 @@ async def list_projects_marked_as_jobs( total_query = sa.select(sa.func.count()).select_from(base_query) # Step 6: Create subquery to aggregate project nodes into workbench structure - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() # Step 7: Query to get the paginated list with full selection list_query = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index ac18d30b225..005f267317f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -17,7 +17,7 @@ from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_postgres_database.models.users import users -from simcore_postgres_database.utils_projects_nodes import make_workbench_subquery +from simcore_postgres_database.utils_projects_nodes import create_workbench_subquery from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -143,7 +143,7 @@ async def get_project_with_workbench( project_uuid: ProjectID, ) -> ProjectWithWorkbenchDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() query = ( sql.select( *PROJECT_DB_COLS, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 4fc772ab25c..8956bd3e224 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -60,7 +60,7 @@ ProjectNode, ProjectNodeCreate, ProjectNodesRepo, - make_workbench_subquery, + create_workbench_subquery, ) from simcore_postgres_database.webserver_models import ( ProjectType, @@ -395,7 +395,7 @@ def _create_private_workspace_query( .group_by(project_to_groups.c.project_uuid) ).subquery("my_access_rights_subquery") - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() private_workspace_query = ( sa.select( @@ -462,7 +462,7 @@ def _create_shared_workspace_query( .group_by(workspaces_access_rights.c.workspace_id) ).subquery("my_workspace_access_rights_subquery") - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() shared_workspace_query = ( sa.select( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index f16fe98209b..62d3feb981a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -16,7 +16,7 @@ from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.utils_projects_nodes import ( ProjectNodesRepo, - make_workbench_subquery, + create_workbench_subquery, ) from simcore_postgres_database.webserver_models import ( ProjectTemplateType as ProjectTemplateTypeDB, @@ -242,7 +242,7 @@ async def _get_project( .group_by(project_to_groups.c.project_uuid) ).subquery("access_rights_subquery") - workbench_subquery = make_workbench_subquery() + workbench_subquery = create_workbench_subquery() query = ( sa.select( From 7e3bbe972e4ab6e5ed2a2daad89709cccc3ed22c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:50:24 +0200 Subject: [PATCH 145/186] fix helper --- .../src/common_library/dict_tools.py | 12 -------- .../common-library/tests/test_dict_tools.py | 28 ----------------- .../pytest_simcore/helpers/assert_checks.py | 12 ++++++++ .../tests/test_helpers_asserts_checks.py | 30 +++++++++++++++++++ .../02/test_projects_crud_handlers.py | 2 +- 5 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 packages/pytest-simcore/tests/test_helpers_asserts_checks.py diff --git a/packages/common-library/src/common_library/dict_tools.py b/packages/common-library/src/common_library/dict_tools.py index 164218f9885..43ef7166308 100644 --- a/packages/common-library/src/common_library/dict_tools.py +++ b/packages/common-library/src/common_library/dict_tools.py @@ -58,15 +58,3 @@ def update_dict(obj: dict, **updates): {key: update_value(obj[key]) if callable(update_value) else update_value} ) return obj - - -def assert_equal_ignoring_none(expected: dict, actual: dict): - for key, exp_value in expected.items(): - if exp_value is None: - continue - assert key in actual, f"Missing key {key}" - act_value = actual[key] - if isinstance(exp_value, dict) and isinstance(act_value, dict): - assert_equal_ignoring_none(exp_value, act_value) - else: - assert act_value == exp_value, f"Mismatch in {key}: {act_value} != {exp_value}" diff --git a/packages/common-library/tests/test_dict_tools.py b/packages/common-library/tests/test_dict_tools.py index f3cacdbed3b..9c23d3ef7b3 100644 --- a/packages/common-library/tests/test_dict_tools.py +++ b/packages/common-library/tests/test_dict_tools.py @@ -161,31 +161,3 @@ def test_copy_from_dict(data: dict[str, Any]): assert selected_data["Status"]["State"] == data["Status"]["State"] assert "Message" not in selected_data["Status"]["State"] assert "running" in data["Status"]["State"] - - -@pytest.mark.parametrize( - "expected, actual", - [ - ({"a": 1, "b": 2}, {"a": 1, "b": 2, "c": 3}), - ({"a": 1, "b": None}, {"a": 1, "b": 42}), - ({"a": {"x": 10, "y": None}}, {"a": {"x": 10, "y": 99}}), - ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 20, "z": 30}}), - ({}, {"foo": "bar"}), - ], -) -def test_assert_equal_ignoring_none_passes(expected, actual): - assert_equal_ignoring_none(expected, actual) - -@pytest.mark.parametrize( - "expected, actual, error_msg", - [ - ({"a": 1, "b": 2}, {"a": 1}, "Missing key b"), - ({"a": 1, "b": 2}, {"a": 1, "b": 3}, "Mismatch in b: 3 != 2"), - ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 99}}, "Mismatch in y: 99 != 20"), - ({"a": {"x": 10}}, {"a": {}}, "Missing key x"), - ], -) -def test_assert_equal_ignoring_none_fails(expected, actual, error_msg): - with pytest.raises(AssertionError) as exc_info: - assert_equal_ignoring_none(expected, actual) - assert error_msg in str(exc_info.value) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/assert_checks.py b/packages/pytest-simcore/src/pytest_simcore/helpers/assert_checks.py index fc931cbebd5..80205893bc7 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/assert_checks.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/assert_checks.py @@ -97,3 +97,15 @@ def _do_assert_error( assert expected_error_code in codes return data, error + + +def assert_equal_ignoring_none(expected: dict, actual: dict): + for key, exp_value in expected.items(): + if exp_value is None: + continue + assert key in actual, f"Missing key {key}" + act_value = actual[key] + if isinstance(exp_value, dict) and isinstance(act_value, dict): + assert_equal_ignoring_none(exp_value, act_value) + else: + assert act_value == exp_value, f"Mismatch in {key}: {act_value} != {exp_value}" diff --git a/packages/pytest-simcore/tests/test_helpers_asserts_checks.py b/packages/pytest-simcore/tests/test_helpers_asserts_checks.py new file mode 100644 index 00000000000..9f7533e7f8e --- /dev/null +++ b/packages/pytest-simcore/tests/test_helpers_asserts_checks.py @@ -0,0 +1,30 @@ +import pytest +from pytest_simcore.helpers.assert_checks import assert_equal_ignoring_none + + +@pytest.mark.parametrize( + "expected, actual", + [ + ({"a": 1, "b": 2}, {"a": 1, "b": 2, "c": 3}), + ({"a": 1, "b": None}, {"a": 1, "b": 42}), + ({"a": {"x": 10, "y": None}}, {"a": {"x": 10, "y": 99}}), + ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 20, "z": 30}}), + ({}, {"foo": "bar"}), + ], +) +def test_assert_equal_ignoring_none_passes(expected, actual): + assert_equal_ignoring_none(expected, actual) + +@pytest.mark.parametrize( + "expected, actual, error_msg", + [ + ({"a": 1, "b": 2}, {"a": 1}, "Missing key b"), + ({"a": 1, "b": 2}, {"a": 1, "b": 3}, "Mismatch in b: 3 != 2"), + ({"a": {"x": 10, "y": 20}}, {"a": {"x": 10, "y": 99}}, "Mismatch in y: 99 != 20"), + ({"a": {"x": 10}}, {"a": {}}, "Missing key x"), + ], +) +def test_assert_equal_ignoring_none_fails(expected, actual, error_msg): + with pytest.raises(AssertionError) as exc_info: + assert_equal_ignoring_none(expected, actual) + assert error_msg in str(exc_info.value) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 92a1efc8cf5..c211023c29e 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -639,7 +639,7 @@ async def test_new_template_from_project( ) assert len(templates) == 1 - assert_equal_ignoring_none(template_project, templates[0], user_project) + assert_equal_ignoring_none(template_project, templates[0]) assert template_project["name"] == user_project["name"] assert template_project["description"] == user_project["description"] From 7d71e9f6191066de90d2ea99ccb9064ed93774ae Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:51:47 +0200 Subject: [PATCH 146/186] fix --- packages/models-library/tests/test_services_types.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/models-library/tests/test_services_types.py b/packages/models-library/tests/test_services_types.py index 0952baf7c4a..e3f0c9d472b 100644 --- a/packages/models-library/tests/test_services_types.py +++ b/packages/models-library/tests/test_services_types.py @@ -53,5 +53,3 @@ def test_faker_factory_service_key_and_version_are_in_sync( ): TypeAdapter(ServiceKey).validate_python(service_key) TypeAdapter(ServiceVersion).validate_python(service_version) - - assert service_key.startswith("simcore/services/") From c946376be2a6b0c1d8ac30bbb8823414e4193ecc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:52:36 +0200 Subject: [PATCH 147/186] fix --- .../catalog/src/simcore_service_catalog/repository/projects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/catalog/src/simcore_service_catalog/repository/projects.py b/services/catalog/src/simcore_service_catalog/repository/projects.py index 4604aebc7ed..084395baac1 100644 --- a/services/catalog/src/simcore_service_catalog/repository/projects.py +++ b/services/catalog/src/simcore_service_catalog/repository/projects.py @@ -1,4 +1,5 @@ import logging +from typing import Final import sqlalchemy as sa from models_library.services import ServiceKeyVersion @@ -11,7 +12,7 @@ _logger = logging.getLogger(__name__) -_IGNORED_SERVICE_KEYS: set[str] = { +_IGNORED_SERVICE_KEYS: Final[set[str]] = { # NOTE: frontend only nodes "simcore/services/frontend/file-picker", "simcore/services/frontend/nodes-group", From 6972a845c14dd7b472d24adaef605f58f0fb96e3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:53:01 +0200 Subject: [PATCH 148/186] fix --- .../storage/src/simcore_service_storage/modules/db/projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/db/projects.py b/services/storage/src/simcore_service_storage/modules/db/projects.py index 0beba4ebd45..71f2c3f6e9f 100644 --- a/services/storage/src/simcore_service_storage/modules/db/projects.py +++ b/services/storage/src/simcore_service_storage/modules/db/projects.py @@ -42,7 +42,7 @@ async def get_project_id_and_node_id_to_names_map( connection: AsyncConnection | None = None, project_uuids: list[ProjectID], ) -> dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]]: - names_map = {} + names_map: dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]] = {} async with pass_or_acquire_connection(self.db_engine, connection) as conn: async for row in await conn.stream( sa.select(projects.c.uuid, projects.c.name).where( From 3bf039924a1a4c4957b6cc5a0a970223b89730e6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:53:44 +0200 Subject: [PATCH 149/186] fix --- .../director-v2/tests/integration/01/test_computation_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 5e58cbf66ec..2cb0ecca030 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -25,7 +25,7 @@ from models_library.clusters import ClusterAuthentication from models_library.projects import ProjectAtDB from models_library.projects_nodes import NodeState -from models_library.projects_nodes_io import NodeID +from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.projects_pipeline import PipelineDetails from models_library.projects_state import RunningState from models_library.users import UserID @@ -415,7 +415,7 @@ async def test_run_partial_computation( ) def _convert_to_pipeline_details( - workbench_node_uuids: list[str], + workbench_node_uuids: list[NodeIDStr], expected_pipeline_adj_list: dict[int, list[int]], expected_node_states: dict[int, dict[str, Any]], ) -> PipelineDetails: From 6f2dbf3376ab281569327bc73816cf2a3472242f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:56:34 +0200 Subject: [PATCH 150/186] fix warnings logs --- .../tests/unit/with_dbs/comp_scheduler/test_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py index ee53ef6b1ae..167430ad82b 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py @@ -412,9 +412,7 @@ async def test_empty_pipeline_is_not_scheduled( collection_run_id=fake_collection_run_id, ) - warning_log_regs = [ - log_rec for log_rec in caplog.records if log_rec.levelname == "WARNING" - ] + warning_log_regs = list(caplog.records) assert len(warning_log_regs) == 1 assert "no computational dag defined" in warning_log_regs[0].message From f32533958905981b996ded610754342f8b6ee392 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 11:57:44 +0200 Subject: [PATCH 151/186] remove position --- .../src/pytest_simcore/helpers/webserver_projects.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index ed54246ef6a..73a6fc05004 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -77,9 +77,6 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] raw_workbench: dict[str, Any] = project_data.pop("workbench", {}) - for raw_node in raw_workbench.values(): - if "position" in raw_node: - del raw_node["position"] # Get valid ProjectNodeCreate fields, excluding node_id since it's set separately valid_fields = ProjectNodeCreate.get_field_names(exclude={"node_id"}) From bb16e26942c1929c46ace9dbf37bc8da76a396ff Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:27:44 +0200 Subject: [PATCH 152/186] fix --- services/web/server/tests/integration/01/test_computation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/integration/01/test_computation.py b/services/web/server/tests/integration/01/test_computation.py index fdd3026e482..02b494e0ee1 100644 --- a/services/web/server/tests/integration/01/test_computation.py +++ b/services/web/server/tests/integration/01/test_computation.py @@ -253,7 +253,7 @@ def _get_project_workbench_from_db( ) .mappings() .all() - ) # list[dict] + ) return {row["node_id"]: dict(row) for row in rows} From df073b1eb4484b963808fe60b21f869902170581 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:28:21 +0200 Subject: [PATCH 153/186] fix tests --- packages/common-library/tests/test_dict_tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/common-library/tests/test_dict_tools.py b/packages/common-library/tests/test_dict_tools.py index 9c23d3ef7b3..fb374ff1791 100644 --- a/packages/common-library/tests/test_dict_tools.py +++ b/packages/common-library/tests/test_dict_tools.py @@ -6,7 +6,6 @@ from typing import Any import pytest -from common_library.dict_tools import assert_equal_ignoring_none from common_library.dict_tools import ( copy_from_dict, get_from_dict, From 89404b95c73330902fd18f697c1017190fb2b61b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:32:34 +0200 Subject: [PATCH 154/186] fix --- .../src/pytest_simcore/helpers/webserver_projects.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 73a6fc05004..c8c71bd2754 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -77,6 +77,9 @@ async def create_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] raw_workbench: dict[str, Any] = project_data.pop("workbench", {}) + for raw_node in raw_workbench.values(): # back-compatibility with old format + if "position" in raw_node: + del raw_node["position"] # Get valid ProjectNodeCreate fields, excluding node_id since it's set separately valid_fields = ProjectNodeCreate.get_field_names(exclude={"node_id"}) From 669b4a8a5e5b2dc44657d4e8f4dcaed35f8b2e96 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:33:31 +0200 Subject: [PATCH 155/186] fix --- .../tests/unit/with_dbs/02/test_projects_crud_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index c211023c29e..fe07cfa6108 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -14,7 +14,7 @@ import sqlalchemy as sa from aiohttp.test_utils import TestClient from aioresponses import aioresponses -from common_library.dict_tools import assert_equal_ignoring_none +from pytest_simcore.helpers.assert_checks import assert_equal_ignoring_none from deepdiff import DeepDiff from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import ( From 6105bda45f946d364c8660574218c986e4340db2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:38:12 +0200 Subject: [PATCH 156/186] fix logs --- .../tests/unit/with_dbs/comp_scheduler/test_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py index 167430ad82b..ee53ef6b1ae 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py @@ -412,7 +412,9 @@ async def test_empty_pipeline_is_not_scheduled( collection_run_id=fake_collection_run_id, ) - warning_log_regs = list(caplog.records) + warning_log_regs = [ + log_rec for log_rec in caplog.records if log_rec.levelname == "WARNING" + ] assert len(warning_log_regs) == 1 assert "no computational dag defined" in warning_log_regs[0].message From 22d39eda26eea85345fbfee6c291db284956d5d6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:44:48 +0200 Subject: [PATCH 157/186] fix --- .../tests/unit/with_dbs/02/test_projects_cancellations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py index fdd9ec0549b..66d07b7e17f 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py @@ -236,7 +236,7 @@ async def test_creating_new_project_from_template_without_copying_data_creates_s EXPECTED_DELETED_FIELDS = ["outputs", "progress", "runHash"] for node_data in project_workbench.values(): for field in EXPECTED_DELETED_FIELDS: - assert field not in node_data + assert field not in node_data or not node_data[field] @pytest.mark.parametrize(*_standard_user_role_response()) From 1b53a30a8eec8455d112895f877e1bea63144cad Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:49:22 +0200 Subject: [PATCH 158/186] fix --- .../tests/unit/with_dbs/02/test_projects_cancellations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py index 66d07b7e17f..541ae22e1b3 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py @@ -286,7 +286,7 @@ async def test_creating_new_project_as_template_without_copying_data_creates_ske EXPECTED_DELETED_FIELDS = ["outputs", "progress", "runHash"] for node_data in project_workbench.values(): for field in EXPECTED_DELETED_FIELDS: - assert field not in node_data + assert field not in node_data or not node_data[field] @pytest.mark.parametrize(*_standard_user_role_response()) From 75d7e55407c226fc1eb401665d8b235b276e5f8a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 12:56:38 +0200 Subject: [PATCH 159/186] fix --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index a6054142aa4..ed77230d451 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -8,7 +8,6 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any from uuid import uuid4 -import uuid import pytest import sqlalchemy as sa From b82302f6e8864d1d8e3fa737905c11e1ae0e4975 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 13:19:42 +0200 Subject: [PATCH 160/186] fix --- .../workspaces/test_workspaces__folders_and_projects_crud.py | 2 +- .../test_workspaces__moving_projects_between_workspaces.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index ef2d49bdf37..1d1099861ca 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -75,7 +75,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["uuid"] == project["uuid"] assert data["workspaceId"] == added_workspace.workspace_id - assert data["folderId"] is None + assert data.get("folderId") is None # Create folder in workspace url = client.app.router["create_folder"].url_for() diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index b19e076cdc3..9ae3282474e 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -91,7 +91,7 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert data["workspaceId"] is None # <-- Workspace ID is None + assert data.get("workspaceId") is None # <-- Workspace ID is None # Move project from your private workspace to shared workspace base_url = client.app.router["move_project_to_workspace"].url_for( @@ -248,7 +248,7 @@ async def test_moving_between_workspaces_check_removed_from_folder( base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert data["workspaceId"] is None # <-- Workspace ID is None + assert data.get("workspaceId") is None # <-- Workspace ID is None # Check project_to_folders DB is empty with postgres_db.connect() as con: From e12c40bf8f4bcebeafdfd2a870caa5796ca6d7b3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 13:23:47 +0200 Subject: [PATCH 161/186] fix --- .../unit/with_dbs/02/test_projects_states_handlers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index ad80870ff6d..8fba443e773 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -90,10 +90,15 @@ def assert_replaced(current_project, update_data): def _extract(dikt, keys): return {k: dikt[k] for k in keys} - modified = [ + skip = [ "lastChangeDate", + "templateType", + "trashedAt", + "trashedBy", + "workspaceId", + "folderId", ] - keep = [k for k in update_data if k not in modified] + keep = [k for k in update_data if k not in skip] assert _extract(current_project, keep) == _extract(update_data, keep) From 9dfe2dfeb8dfcc3e7f849071a18830495b9e1b10 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Aug 2025 15:19:57 +0200 Subject: [PATCH 162/186] revert models --- .../src/models_library/api_schemas_webserver/projects.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index d5eacc68368..e9f5be2054f 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -130,7 +130,7 @@ class ProjectGet(OutputSchema): thumbnail: HttpUrl | Literal[""] type: ProjectType - template_type: ProjectTemplateType | None = None + template_type: ProjectTemplateType | None workbench: NodesDict @@ -141,7 +141,7 @@ class ProjectGet(OutputSchema): creation_date: DateTimeStr last_change_date: DateTimeStr state: ProjectStateOutputSchema | None = None - trashed_at: datetime | None = None + trashed_at: datetime | None trashed_by: Annotated[ GroupID | None, Field(description="The primary gid of the user who trashed") ] = None @@ -163,8 +163,8 @@ class ProjectGet(OutputSchema): permalink: ProjectPermalink | None = None - workspace_id: WorkspaceID | None = None - folder_id: FolderID | None = None + workspace_id: WorkspaceID | None + folder_id: FolderID | None @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: From 88f91f59a4d3e17478c5cc36f1b32a000da0c136 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Aug 2025 09:59:38 +0200 Subject: [PATCH 163/186] fix: improve listing --- .../utils_projects_nodes.py | 3 +- .../modules/db/repositories/projects.py | 2 +- .../projects/_jobs_repository.py | 23 ++++------- .../projects/_projects_repository.py | 2 +- .../projects/_projects_repository_legacy.py | 22 +---------- .../_projects_repository_legacy_utils.py | 38 +++++++++---------- 6 files changed, 32 insertions(+), 58 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index e55692702fe..4c5823ed134 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -116,7 +116,7 @@ def model_dump_as_node(self) -> dict[str, Any]: ) -def create_workbench_subquery() -> Subquery: +def create_workbench_subquery(project_id: str) -> Subquery: return ( sa.select( projects_nodes.c.project_uuid, @@ -163,6 +163,7 @@ def create_workbench_subquery() -> Subquery: projects, projects_nodes.c.project_uuid == projects.c.uuid ) ) + .where(projects.c.uuid == project_id) .group_by(projects_nodes.c.project_uuid) .subquery() ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 965f9b6dd2b..8e3c4eb6e72 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -18,7 +18,7 @@ class ProjectsRepository(BaseRepository): async def get_project(self, project_id: ProjectID) -> ProjectAtDB: - workbench_subquery = create_workbench_subquery() + workbench_subquery = create_workbench_subquery(f"{project_id}") async with self.db_engine.connect() as conn: query = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index b398665644d..9cc7d30e4ec 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -4,14 +4,12 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import TypeAdapter from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_metadata import projects_metadata from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.models.projects_to_products import projects_to_products -from simcore_postgres_database.utils_projects_nodes import create_workbench_subquery from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -20,6 +18,8 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncConnection +from simcore_service_webserver.projects._projects_repository_legacy_utils import get_project_workbench + from ..db.base_repository import BaseRepository from .models import ProjectDBGet, ProjectJobDBGet @@ -170,25 +170,16 @@ async def list_projects_marked_as_jobs( # Step 5: Query to get the total count total_query = sa.select(sa.func.count()).select_from(base_query) - # Step 6: Create subquery to aggregate project nodes into workbench structure - workbench_subquery = create_workbench_subquery() - - # Step 7: Query to get the paginated list with full selection + # Step 6: Query to get the paginated list with full selection list_query = ( sa.select( *_PROJECT_DB_COLS, - sa.func.coalesce( - workbench_subquery.c.workbench, sa.text("'{}'::json") - ).label("workbench"), base_query.c.job_parent_resource_name, ) .select_from( base_query.join( projects, projects.c.uuid == base_query.c.project_uuid, - ).outerjoin( - workbench_subquery, - projects.c.uuid == workbench_subquery.c.project_uuid, ) ) .order_by( @@ -204,9 +195,9 @@ async def list_projects_marked_as_jobs( total_count = await conn.scalar(total_query) assert isinstance(total_count, int) # nosec - result = await conn.execute(list_query) - projects_list = TypeAdapter(list[ProjectJobDBGet]).validate_python( - result.fetchall() - ) + projects_list = [] + async for project_row in await conn.stream(list_query): + workbench = await get_project_workbench(conn, project_row.uuid) + projects_list.append(ProjectJobDBGet.model_validate({**project_row, "workbench": workbench})) return total_count, projects_list diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 005f267317f..8c3099d1a28 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -143,7 +143,7 @@ async def get_project_with_workbench( project_uuid: ProjectID, ) -> ProjectWithWorkbenchDBGet: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - workbench_subquery = create_workbench_subquery() + workbench_subquery = create_workbench_subquery(f"{project_uuid}") query = ( sql.select( *PROJECT_DB_COLS, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 8956bd3e224..fca69cf286e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -60,7 +60,6 @@ ProjectNode, ProjectNodeCreate, ProjectNodesRepo, - create_workbench_subquery, ) from simcore_postgres_database.webserver_models import ( ProjectType, @@ -96,6 +95,7 @@ convert_to_schema_names, create_project_access_rights, patch_workbench, + get_project_workbench ) from ._socketio_service import notify_project_document_updated from .exceptions import ( @@ -395,14 +395,9 @@ def _create_private_workspace_query( .group_by(project_to_groups.c.project_uuid) ).subquery("my_access_rights_subquery") - workbench_subquery = create_workbench_subquery() - private_workspace_query = ( sa.select( *PROJECT_DB_COLS, - sa.func.coalesce( - workbench_subquery.c.workbench, sa.text("'{}'::json") - ).label("workbench"), projects_to_products.c.product_name, projects_to_folders.c.folder_id, ) @@ -417,10 +412,6 @@ def _create_private_workspace_query( ), isouter=True, ) - .outerjoin( - workbench_subquery, - workbench_subquery.c.project_uuid == projects.c.uuid, - ) ) .where( (projects.c.workspace_id.is_(None)) # <-- Private workspace @@ -462,14 +453,9 @@ def _create_shared_workspace_query( .group_by(workspaces_access_rights.c.workspace_id) ).subquery("my_workspace_access_rights_subquery") - workbench_subquery = create_workbench_subquery() - shared_workspace_query = ( sa.select( *PROJECT_DB_COLS, - sa.func.coalesce( - workbench_subquery.c.workbench, sa.text("'{}'::json") - ).label("workbench"), projects_to_products.c.product_name, projects_to_folders.c.folder_id, ) @@ -488,10 +474,6 @@ def _create_shared_workspace_query( ), isouter=True, ) - .outerjoin( - workbench_subquery, - workbench_subquery.c.project_uuid == projects.c.uuid, - ) ) .where(projects_to_products.c.product_name == product_name) ) @@ -698,7 +680,7 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st # with the frontend. The frontend would need to check and adapt how it handles default values in # Workbench nodes, which are currently not returned if not set in the DB. prj_dict = dict(row.items()) | { - "workbench": await self._get_workbench(conn, row.uuid), + "workbench": await get_project_workbench(conn, row.uuid), } ProjectListAtDB.model_validate(prj_dict) prjs_output.append(prj_dict) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 62d3feb981a..93b7adc39ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -191,23 +191,6 @@ async def _upsert_tags_in_project( .on_conflict_do_nothing() ) - @staticmethod - async def _get_workbench( - connection: SAConnection, - project_uuid: str, - ) -> dict[str, Any]: - project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid)) - exclude_fields = {"node_id", "required_resources", "created", "modified"} - workbench: dict[str, Any] = {} - - project_nodes = await project_nodes_repo.list(connection) # type: ignore - for project_node in project_nodes: - node_data = project_node.model_dump( - exclude=exclude_fields, exclude_none=True, exclude_unset=True - ) - workbench[f"{project_node.node_id}"] = node_data - return workbench - async def _get_project( self, connection: SAConnection, @@ -238,11 +221,11 @@ async def _get_project( ), ).label("access_rights"), ) - .where(project_to_groups.c.project_uuid == f"{project_uuid}") + .where(project_to_groups.c.project_uuid == project_uuid) .group_by(project_to_groups.c.project_uuid) ).subquery("access_rights_subquery") - workbench_subquery = create_workbench_subquery() + workbench_subquery = create_workbench_subquery(project_uuid) query = ( sa.select( @@ -393,3 +376,20 @@ def patch_workbench( # patch current_node_data.update(new_node_data) return (patched_project, changed_entries) + + +async def get_project_workbench( + connection: SAConnection, + project_uuid: str, +) -> dict[str, Any]: + project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid)) + exclude_fields = {"node_id", "required_resources", "created", "modified"} + workbench: dict[str, Any] = {} + + project_nodes = await project_nodes_repo.list(connection) # type: ignore + for project_node in project_nodes: + node_data = project_node.model_dump( + exclude=exclude_fields, exclude_none=True, exclude_unset=True + ) + workbench[f"{project_node.node_id}"] = node_data + return workbench From db156e44c5cd58915e3f59326931e50cef9dd381 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Aug 2025 09:59:47 +0200 Subject: [PATCH 164/186] fix: script --- .../201aa37f4d9a_remove_workbench_column_from_projects_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py index 164651647de..bdc35b7bed9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -1,7 +1,7 @@ """Remove workbench column from projects_table Revision ID: 201aa37f4d9a -Revises: 5679165336c8 +Revises: 5b998370916a Create Date: 2025-07-22 19:25:42.125196+00:00 """ @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "201aa37f4d9a" -down_revision = "5679165336c8" +down_revision = "5b998370916a" branch_labels = None depends_on = None From 6e17b8009a219bc8016b86256ffbc15694cec760 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Aug 2025 10:26:13 +0200 Subject: [PATCH 165/186] fix: connection --- .../projects/_projects_repository_legacy_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 93b7adc39ee..76e5f8102c4 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -6,6 +6,7 @@ from typing import Any, Literal, cast import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncConnection from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy from models_library.projects import ProjectID, ProjectType @@ -379,7 +380,7 @@ def patch_workbench( async def get_project_workbench( - connection: SAConnection, + connection: AsyncConnection, project_uuid: str, ) -> dict[str, Any]: project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid)) From b7bb03632c5ebfeaf6d993ecd03e216a61176f61 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Aug 2025 10:43:07 +0200 Subject: [PATCH 166/186] fix --- .../simcore_service_webserver/projects/_jobs_repository.py | 5 +++-- .../projects/_projects_repository_legacy_utils.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 6c4fd82ecd3..008de54506b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -221,7 +221,6 @@ async def get_project_marked_as_job( query = ( sa.select( *_PROJECT_DB_COLS, - projects.c.workbench, projects_to_jobs.c.job_parent_resource_name, projects_to_jobs.c.storage_assets_deleted, ) @@ -244,4 +243,6 @@ async def get_project_marked_as_job( row = result.first() if row is None: return None - return TypeAdapter(ProjectJobDBGet).validate_python(row) + + workbench = await get_project_workbench(conn, row.uuid) + return ProjectJobDBGet.model_validate({**row, "workbench": workbench}) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 76e5f8102c4..8c088dfad06 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -387,7 +387,7 @@ async def get_project_workbench( exclude_fields = {"node_id", "required_resources", "created", "modified"} workbench: dict[str, Any] = {} - project_nodes = await project_nodes_repo.list(connection) # type: ignore + project_nodes = await project_nodes_repo.list(connection) for project_node in project_nodes: node_data = project_node.model_dump( exclude=exclude_fields, exclude_none=True, exclude_unset=True From ec1d762c12b598309f1051b842a30ce31955d45a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Aug 2025 15:30:36 +0200 Subject: [PATCH 167/186] fix: model dump --- .../projects/_controller/projects_rest.py | 8 ++++++-- .../with_dbs/02/test_projects_nodes_handlers__patch.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 3fcf6784c67..2c487300915 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -249,7 +249,9 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).model_dump(by_alias=True, exclude_unset=True, exclude_none=True) + data = ProjectGet.from_domain_model(project).model_dump( + by_alias=True, exclude_unset=True, exclude_none=True + ) return envelope_json_response(data) @@ -281,7 +283,9 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain_model(project).model_dump(by_alias=True, exclude_unset=True, exclude_none=True) + data = ProjectGet.from_domain_model(project).model_dump( + by_alias=True, exclude_unset=True + ) return envelope_json_response(data) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index 16937bd6fc1..1295cae6900 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -185,7 +185,6 @@ async def test_patch_project_node( _tested_node = data["workbench"][node_id] assert _tested_node["label"] == "testing-string" - assert _tested_node["progress"] is None assert _tested_node["key"] == _patch_key["key"] assert _tested_node["version"] == _patch_version["version"] assert _tested_node["inputs"] == _patch_inputs["inputs"] From 5718e92012db9dd7ef091649eafcdd6679d40db4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Aug 2025 15:56:44 +0200 Subject: [PATCH 168/186] try to fix --- packages/models-library/src/models_library/projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 2fd18746241..076250a7455 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -295,7 +295,7 @@ def _convert_empty_str_to_none(cls, v): @field_validator("state", mode="before") @classmethod def _convert_from_enum(cls, v): - if isinstance(v, str): + if isinstance(v, str | None): # the old version of state was a enum of RunningState running_state_value = _convert_old_enum_name(v) From 260c4aba6a440c7b8709ecf7c5421033fc2ba564 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Aug 2025 15:59:00 +0200 Subject: [PATCH 169/186] try to fix 2.0 --- packages/models-library/src/models_library/projects_nodes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 076250a7455..94ca4e7f9bf 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -295,11 +295,13 @@ def _convert_empty_str_to_none(cls, v): @field_validator("state", mode="before") @classmethod def _convert_from_enum(cls, v): - if isinstance(v, str | None): + if isinstance(v, str): # the old version of state was a enum of RunningState running_state_value = _convert_old_enum_name(v) return NodeState(current_status=running_state_value) + if v is None: + return NodeState(current_status=RunningState.NOT_STARTED) return v @staticmethod From cc86d5367e5fb1124b7bb39c97184313d5bce0b8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Aug 2025 16:40:11 +0200 Subject: [PATCH 170/186] fix: workbench nulls --- .../src/models_library/projects_nodes.py | 2 - .../utils_projects_nodes.py | 71 ++++++++++--------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 94ca4e7f9bf..2fd18746241 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -300,8 +300,6 @@ def _convert_from_enum(cls, v): # the old version of state was a enum of RunningState running_state_value = _convert_old_enum_name(v) return NodeState(current_status=running_state_value) - if v is None: - return NodeState(current_status=RunningState.NOT_STARTED) return v @staticmethod diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 4c5823ed134..397b8656a7f 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -117,45 +117,46 @@ def model_dump_as_node(self) -> dict[str, Any]: def create_workbench_subquery(project_id: str) -> Subquery: + workbench_obj = sa.func.json_build_object( + "key", + projects_nodes.c.key, + "version", + projects_nodes.c.version, + "label", + projects_nodes.c.label, + "progress", + projects_nodes.c.progress, + "thumbnail", + projects_nodes.c.thumbnail, + "inputAccess", + projects_nodes.c.input_access, + "inputNodes", + projects_nodes.c.input_nodes, + "inputs", + projects_nodes.c.inputs, + "inputsRequired", + projects_nodes.c.inputs_required, + "inputsUnits", + projects_nodes.c.inputs_units, + "outputNodes", + projects_nodes.c.output_nodes, + "outputs", + projects_nodes.c.outputs, + "runHash", + projects_nodes.c.run_hash, + "state", + projects_nodes.c.state, + "parent", + projects_nodes.c.parent, + "bootOptions", + projects_nodes.c.boot_options, + ) + return ( sa.select( projects_nodes.c.project_uuid, sa.func.json_object_agg( - projects_nodes.c.node_id, - sa.func.json_build_object( - "key", - projects_nodes.c.key, - "version", - projects_nodes.c.version, - "label", - projects_nodes.c.label, - "progress", - projects_nodes.c.progress, - "thumbnail", - projects_nodes.c.thumbnail, - "inputAccess", - projects_nodes.c.input_access, - "inputNodes", - projects_nodes.c.input_nodes, - "inputs", - projects_nodes.c.inputs, - "inputsRequired", - projects_nodes.c.inputs_required, - "inputsUnits", - projects_nodes.c.inputs_units, - "outputNodes", - projects_nodes.c.output_nodes, - "outputs", - projects_nodes.c.outputs, - "runHash", - projects_nodes.c.run_hash, - "state", - projects_nodes.c.state, - "parent", - projects_nodes.c.parent, - "bootOptions", - projects_nodes.c.boot_options, - ), + projects_nodes.c.node_id, sa.func.json_strip_nulls(workbench_obj) ).label("workbench"), ) .select_from( From 413f4dcb94ea240aafda72314c31b6e67f1915b1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Aug 2025 16:55:59 +0200 Subject: [PATCH 171/186] fix: test --- .../web/server/tests/integration/01/test_computation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/integration/01/test_computation.py b/services/web/server/tests/integration/01/test_computation.py index 02b494e0ee1..91530aa31c8 100644 --- a/services/web/server/tests/integration/01/test_computation.py +++ b/services/web/server/tests/integration/01/test_computation.py @@ -321,16 +321,16 @@ async def _assert_and_wait_for_comp_task_states_to_be_transmitted_in_projects( # if this one is in, the other should also be but let's check it carefully assert node_values.run_hash - assert "runHash" in node_in_project_table - assert node_values.run_hash == node_in_project_table["runHash"] + assert "run_hash" in node_in_project_table + assert node_values.run_hash == node_in_project_table["run_hash"] assert node_values.state assert "state" in node_in_project_table - assert "currentStatus" in node_in_project_table["state"] + assert "current_status" in node_in_project_table["state"] # NOTE: beware that the comp_tasks has StateType and Workbench has RunningState (sic) assert ( DB_TO_RUNNING_STATE[node_values.state].value - == node_in_project_table["state"]["currentStatus"] + == node_in_project_table["state"]["current_status"] ) print( "--> tasks were properly transferred! " From fb3179e80efcce8a97799f6c3a30fffa7abaf754 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 22 Aug 2025 15:37:35 +0200 Subject: [PATCH 172/186] fix: alembic --- .../201aa37f4d9a_remove_workbench_column_from_projects_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py index bdc35b7bed9..43e125418e7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -1,7 +1,7 @@ """Remove workbench column from projects_table Revision ID: 201aa37f4d9a -Revises: 5b998370916a +Revises: b566f1b29012 Create Date: 2025-07-22 19:25:42.125196+00:00 """ @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "201aa37f4d9a" -down_revision = "5b998370916a" +down_revision = "b566f1b29012" branch_labels = None depends_on = None From ed060af11894dc9353f79a64a0b4cde850a1104d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 22 Aug 2025 15:41:28 +0200 Subject: [PATCH 173/186] fix: typecheck --- .../projects/_projects_repository_legacy.py | 1 + .../src/simcore_service_webserver/projects/_projects_service.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index 74615e8eda3..fe34803645a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -10,6 +10,7 @@ from typing import Any, Self, cast from uuid import uuid1 +from models_library.utils._original_fastapi_encoders import jsonable_encoder import sqlalchemy as sa from aiohttp import web from aiopg.sa import Engine diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index d77206d1b32..810f0f08295 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1376,7 +1376,6 @@ async def update_project_node_outputs( updated_project_with_states = await add_project_states_for_user( user_id=user_id, project=updated_project.model_dump(mode="json"), - is_template=False, app=app, ) From ac12cba8a557b3127d4c490a07bda016421658e4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 22 Aug 2025 15:57:10 +0200 Subject: [PATCH 174/186] fix: duplicate by_alias --- .../projects/_controller/projects_rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index ae1e8047eab..0c411dacc6b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -292,7 +292,7 @@ async def get_project(request: web.Request): project=project, ) - data = ProjectGet.from_domain_model(project).data(by_alias=True, exclude_unset=True) + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return envelope_json_response(data) From eafc2d3237793ee3e93b950d5622687fbd78d0d7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:36:28 +0200 Subject: [PATCH 175/186] updates webserver OAS --- .../api/v0/openapi.yaml | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 2e40aa33250..20e1361b942 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13888,9 +13888,9 @@ components: description: The short name of the node progress: anyOf: - - type: number - maximum: 100.0 - minimum: 0.0 + - type: integer + maximum: 100 + minimum: 0 - type: 'null' title: Progress description: the node progress value (deprecated in DB, still used for API @@ -13971,6 +13971,7 @@ components: - type: 'null' title: Outputnodes description: Used in group-nodes. Node IDs of those connected to the output + deprecated: true parent: anyOf: - type: string @@ -13978,6 +13979,7 @@ components: - type: 'null' title: Parent description: Parent's (group-nodes') node ID s. Used to group + deprecated: true position: anyOf: - $ref: '#/components/schemas/Position' @@ -13989,6 +13991,12 @@ components: - $ref: '#/components/schemas/NodeState-Input' - type: 'null' description: The node's state object + required_resources: + anyOf: + - additionalProperties: true + type: object + - type: 'null' + title: Required Resources bootOptions: anyOf: - type: object @@ -14023,9 +14031,9 @@ components: description: The short name of the node progress: anyOf: - - type: number - maximum: 100.0 - minimum: 0.0 + - type: integer + maximum: 100 + minimum: 0 - type: 'null' title: Progress description: the node progress value (deprecated in DB, still used for API @@ -14106,6 +14114,7 @@ components: - type: 'null' title: Outputnodes description: Used in group-nodes. Node IDs of those connected to the output + deprecated: true parent: anyOf: - type: string @@ -14113,6 +14122,7 @@ components: - type: 'null' title: Parent description: Parent's (group-nodes') node ID s. Used to group + deprecated: true position: anyOf: - $ref: '#/components/schemas/Position' @@ -14124,6 +14134,12 @@ components: - $ref: '#/components/schemas/NodeState-Output' - type: 'null' description: The node's state object + required_resources: + anyOf: + - additionalProperties: true + type: object + - type: 'null' + title: Required Resources bootOptions: anyOf: - type: object @@ -15863,7 +15879,6 @@ components: - creationDate - lastChangeDate - trashedAt - - trashedBy - tags - dev - workspaceId @@ -16066,7 +16081,6 @@ components: - creationDate - lastChangeDate - trashedAt - - trashedBy - tags - dev - workspaceId From f9483cba40683dfea790af37ad8279817ea1040c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:12:38 +0200 Subject: [PATCH 176/186] fixes update node's states --- .../src/models_library/projects_nodes.py | 10 +++++++++- .../projects/_projects_service.py | 17 +++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 9e37c044615..ded2249c1aa 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -177,14 +177,22 @@ class NodeState(BaseModel): model_config = ConfigDict( extra="forbid", - populate_by_name=True, + validate_by_alias=True, + validate_by_name=True, json_schema_extra={ "examples": [ + # example with alias name { "modified": True, "dependencies": [], "currentStatus": "NOT_STARTED", }, + # example with field name + { + "modified": True, + "dependencies": [], + "current_status": "NOT_STARTED", + }, { "modified": True, "dependencies": ["42838344-03de-4ce2-8d93-589a5dcdfd05"], diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 810f0f08295..03d0ed52e63 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1962,15 +1962,15 @@ async def add_project_states_for_user( node_id=NodeID(node_uuid), ) if NodeID(node_uuid) in computational_node_states: - node_state = computational_node_states[NodeID(node_uuid)].model_copy( - update={"lock_state": node_lock_state} - ) + computed_node_state = computational_node_states[ + NodeID(node_uuid) + ].model_copy(update={"lock_state": node_lock_state}) else: # if the node is not in the computational state, we create a new one service_is_running = node_lock_state and ( node_lock_state.status is NodeShareStatus.OPENED ) - node_state = NodeState( + computed_node_state = NodeState( current_status=( RunningState.STARTED if service_is_running @@ -1980,9 +1980,14 @@ async def add_project_states_for_user( ) # upgrade the project - node.setdefault("state", {}).update( - node_state.model_dump(mode="json", by_alias=True, exclude_unset=True) + # NOTE: copy&dump step avoids both alias and field-names to be keys in the dict + # e.g. "current_status" and "currentStatus" + current_node_state = NodeState.model_validate(node.get("state", {})) + updated_node_state = current_node_state.model_copy( + update=computed_node_state.model_dump(mode="json", exclude_unset=True) ) + node["state"] = updated_node_state.model_dump(by_alias=True, exclude_unset=True) + if "progress" in node["state"] and node["state"]["progress"] is not None: # ensure progress is a percentage node["progress"] = round(node["state"]["progress"] * 100.0) From 6b0048fb41385e97750de1eb2bfc94d4d08fa781 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:15:02 +0200 Subject: [PATCH 177/186] fixes mypy --- .../models-library/src/models_library/projects_nodes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index ded2249c1aa..4f9e055605d 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -478,6 +478,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class PartialNode(Node): - key: ServiceKey | None = None - version: ServiceVersion | None = None - label: str | None = None + # NOTE: `type: ignore[assignment]` is needed because mypy gets confused when overriding the types by adding the Union with None + key: ServiceKey | None = None # type: ignore[assignment] + version: ServiceVersion | None = None # type: ignore[assignment] + label: str | None = None # type: ignore[assignment] From c93168ace0117b32b0ec9a3322d5748243ce5e75 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:47:31 +0200 Subject: [PATCH 178/186] fixes test listerner --- .../projects/_projects_service.py | 9 ++++++--- .../test_notifications__db_comp_tasks_listening_task.py | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 03d0ed52e63..0b8519032c7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1471,6 +1471,7 @@ async def _trigger_connected_service_retrieve( # find the nodes that need to retrieve data for node_uuid, node in workbench.items(): + # check this node is dynamic if not _is_node_dynamic(node["key"]): continue @@ -1482,9 +1483,11 @@ async def _trigger_connected_service_retrieve( if not isinstance(port_value, dict): continue - input_node_uuid = port_value.get("nodeUuid") + # FIXME: hack to support both field and alias names because cannot guarantee which one is stored in workbench + input_node_uuid = port_value.get("nodeUuid", port_value.get("node_uuid")) if input_node_uuid != updated_node_uuid: continue + # so this node is linked to the updated one, now check if the port was changed? linked_input_port = port_value.get("output") if linked_input_port in changed_keys: @@ -1492,8 +1495,8 @@ async def _trigger_connected_service_retrieve( # call /retrieve on the nodes update_tasks = [ - dynamic_scheduler_service.retrieve_inputs(app, NodeID(node), keys) - for node, keys in nodes_keys_to_update.items() + dynamic_scheduler_service.retrieve_inputs(app, NodeID(node_id), keys) + for node_id, keys in nodes_keys_to_update.items() ] await logged_gather(*update_tasks, reraise=False) diff --git a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py index cb63e262518..9c360f74ae7 100644 --- a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py +++ b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py @@ -232,8 +232,11 @@ async def mock_dynamic_service_rpc( """ Mocks the dynamic service RPC calls to avoid actual service calls during tests. """ - return mocker.patch( - "servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services.retrieve_inputs", + import servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services + + return mocker.patch.object( + servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services, + "retrieve_inputs", autospec=True, ) From c66590dabfabcc7020f848290a234dcbea6983a3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:52:58 +0200 Subject: [PATCH 179/186] fixes test helper --- .../server/tests/integration/01/test_computation.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/services/web/server/tests/integration/01/test_computation.py b/services/web/server/tests/integration/01/test_computation.py index f9f1dab0a46..07318a99406 100644 --- a/services/web/server/tests/integration/01/test_computation.py +++ b/services/web/server/tests/integration/01/test_computation.py @@ -254,15 +254,10 @@ async def _get_project_workbench_from_db( # this check is only there to check the comp_pipeline is there print(f"--> looking for project {project_id=} in projects table...") async with sqlalchemy_async_engine.connect() as conn: - rows = ( - conn.execute( - sa.select(projects_nodes).where( - projects_nodes.c.project_uuid == project_id - ) - ) - .mappings() - .all() + result = await conn.execute( + sa.select(projects_nodes).where(projects_nodes.c.project_uuid == project_id) ) + rows = result.mappings().all() return {row["node_id"]: dict(row) for row in rows} From f3512b317121bf3e8e2c76506f11aa2f1390afa1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:08:59 +0200 Subject: [PATCH 180/186] db returns None. sa.func.json_strip_nulls -> sa.func.json --- .../src/simcore_postgres_database/utils_projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 397b8656a7f..452babedcc1 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -156,7 +156,7 @@ def create_workbench_subquery(project_id: str) -> Subquery: sa.select( projects_nodes.c.project_uuid, sa.func.json_object_agg( - projects_nodes.c.node_id, sa.func.json_strip_nulls(workbench_obj) + projects_nodes.c.node_id, sa.func.json(workbench_obj) ).label("workbench"), ) .select_from( From fb73d03b38f659675473d7b5753f008e27229c6a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:25:56 +0200 Subject: [PATCH 181/186] type --- packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 4821feb3606..2eb80a6a013 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -194,9 +194,10 @@ async def get_value( ) async def _evaluate() -> ItemValue | None: + # NOTE: review types returned by this function !!! if isinstance(self.value, PortLink): # this is a link to another node's port - other_port_itemvalue: None | (ItemValue) = ( + other_port_itemvalue: ItemValue | None = ( await port_utils.get_value_link_from_port_link( self.value, # pylint: disable=protected-access @@ -209,7 +210,7 @@ async def _evaluate() -> ItemValue | None: if isinstance(self.value, FileLink): # let's get the download/upload link from storage - url_itemvalue: None | (AnyUrl) = ( + url_itemvalue: AnyUrl | None = ( await port_utils.get_download_link_from_storage( # pylint: disable=protected-access user_id=self._node_ports.user_id, From 7c6197d1bc0b7fd13b4cbd033c1f114801b89156 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:09:34 +0200 Subject: [PATCH 182/186] Adapts revisitons --- .../201aa37f4d9a_remove_workbench_column_from_projects_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py index 43e125418e7..a97c7b2fb8b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/201aa37f4d9a_remove_workbench_column_from_projects_.py @@ -1,7 +1,7 @@ """Remove workbench column from projects_table Revision ID: 201aa37f4d9a -Revises: b566f1b29012 +Revises: 06eafd25d004 Create Date: 2025-07-22 19:25:42.125196+00:00 """ @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "201aa37f4d9a" -down_revision = "b566f1b29012" +down_revision = "06eafd25d004" branch_labels = None depends_on = None From 4b708688d7c45cb1456f3e1fd72dd6912d751f33 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:36:40 +0200 Subject: [PATCH 183/186] fix --- .../simcore_service_webserver/projects/_projects_service.py | 5 ++++- .../tests/unit/with_dbs/02/test_projects_nodes_handler.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index df07fc734c0..eafc4a95709 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -1991,7 +1991,10 @@ async def add_project_states_for_user( # upgrade the project # NOTE: copy&dump step avoids both alias and field-names to be keys in the dict # e.g. "current_status" and "currentStatus" - current_node_state = NodeState.model_validate(node.get("state", {})) + current_node_state = NodeState.model_validate( + node.get("state") + or {} # NOTE: that node.get("state") can exists and be None! + ) updated_node_state = current_node_state.model_copy( update=computed_node_state.model_dump(mode="json", exclude_unset=True) ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index b7996e38281..0e4838e2b5c 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -483,7 +483,7 @@ def inc_running_services(self, *args, **kwargs): # noqa: ARG002 assert result node_ids = result.scalars().all() assert len(node_ids) == NUM_DY_SERVICES + num_services_in_project - assert set(running_services.running_services_uuids).issubset(node_ids) + assert {f"{i}" for i in running_services.running_services_uuids}.issubset(node_ids) print(f"--> {NUM_DY_SERVICES} nodes were created concurrently") # From 55da438feb50ae8a9cb45305d8af61269d1d8223 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 12 Sep 2025 21:28:21 +0200 Subject: [PATCH 184/186] uses adapters --- .../utils_projects_nodes.py | 33 +------------------ .../projects/_nodes_repository.py | 13 +++++--- 2 files changed, 9 insertions(+), 37 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 69cd2d37eae..c69e1bbd5d8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -4,6 +4,7 @@ from typing import Annotated, Any import asyncpg.exceptions # type: ignore[import-untyped] +import sqlalchemy as sa import sqlalchemy.exc from common_library.async_tools import maybe_await from common_library.basic_types import DEFAULT_FACTORY @@ -73,22 +74,6 @@ class ProjectNodeCreate(BaseModel): def get_field_names(cls, *, exclude: set[str]) -> set[str]: return cls.model_fields.keys() - exclude - def model_dump_as_node(self) -> dict[str, Any]: - """Converts a ProjectNode from the database to a Node model for the API. - - Handles field mapping and excludes database-specific fields that are not - part of the Node model. - """ - # Get all ProjectNode fields except those that don't belong in Node - exclude_fields = {"node_id", "required_resources"} - return self.model_dump( - # NOTE: this setup ensures using the defaults provided in Node model when the db does not - # provide them, e.g. `state` - exclude=exclude_fields, - exclude_none=True, - exclude_unset=True, - ) - model_config = ConfigDict(frozen=True) @@ -98,22 +83,6 @@ class ProjectNode(ProjectNodeCreate): model_config = ConfigDict(from_attributes=True) - def model_dump_as_node(self) -> dict[str, Any]: - """Converts a ProjectNode from the database to a Node model for the API. - - Handles field mapping and excludes database-specific fields that are not - part of the Node model. - """ - # Get all ProjectNode fields except those that don't belong in Node - exclude_fields = {"node_id", "required_resources", "created", "modified"} - return self.model_dump( - # NOTE: this setup ensures using the defaults provided in Node model when the db does not - # provide them, e.g. `state` - exclude=exclude_fields, - exclude_none=True, - exclude_unset=True, - ) - def create_workbench_subquery(project_id: str) -> Subquery: workbench_obj = sa.func.json_build_object( diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index 2a1e1ce754a..8a01f5b5f87 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -1,5 +1,3 @@ -from typing import Any - from aiohttp import web from models_library.projects import ProjectID from models_library.projects_nodes import Node, PartialNode @@ -13,6 +11,7 @@ ) from ..db.plugin import get_asyncpg_engine +from . import _nodes_models_adapters async def get_project_nodes_services( @@ -37,7 +36,9 @@ async def get_project_nodes_map( project_nodes = await repo.list(conn) workbench = { - project_node.node_id: project_node.model_dump_as_node() + project_node.node_id: _nodes_models_adapters.node_from_project_node( + project_node + ) for project_node in project_nodes } return TypeAdapter(dict[NodeID, Node]).validate_python(workbench) @@ -51,7 +52,7 @@ async def update_project_nodes_map( ) -> dict[NodeID, Node]: repo = ProjectNodesRepo(project_uuid=project_id) - workbench: dict[NodeID, dict[str, Any]] = {} + workbench: dict[NodeID, Node] = {} async with transaction_context(get_asyncpg_engine(app)) as conn: for node_id, node in partial_nodes_map.items(): project_node = await repo.update( @@ -59,7 +60,9 @@ async def update_project_nodes_map( node_id=node_id, **node.model_dump(exclude_none=True, exclude_unset=True), ) - workbench[node_id] = project_node.model_dump_as_node() + workbench[node_id] = _nodes_models_adapters.node_from_project_node( + project_node + ) return TypeAdapter(dict[NodeID, Node]).validate_python(workbench) From 1e52c3c3a456bb486c9bf38293aa24a4984ceae2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 12 Sep 2025 21:33:30 +0200 Subject: [PATCH 185/186] minor --- packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 2eb80a6a013..a7f9ec22fd0 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -257,7 +257,7 @@ async def _evaluate() -> ItemConcreteValue | None: if isinstance(self.value, PortLink): # this is a link to another node - other_port_concretevalue: None | (ItemConcreteValue) = ( + other_port_concretevalue: None | ItemConcreteValue = ( await port_utils.get_value_from_link( # pylint: disable=protected-access key=self.key, From 2ec112431f2a17588525bbb3ff0b3076236e8d88 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 12 Sep 2025 21:55:22 +0200 Subject: [PATCH 186/186] bad sort --- services/web/server/tests/unit/conftest.py | 2 +- .../tests/unit/with_dbs/02/test_projects_states_handlers.py | 2 +- services/web/server/tests/unit/with_dbs/03/tags/conftest.py | 2 +- services/web/server/tests/unit/with_dbs/04/wallets/conftest.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/conftest.py b/services/web/server/tests/unit/conftest.py index 0755d30b3f9..53b4892491e 100644 --- a/services/web/server/tests/unit/conftest.py +++ b/services/web/server/tests/unit/conftest.py @@ -16,7 +16,7 @@ from aiohttp.test_utils import TestClient from models_library.products import ProductName from pytest_mock import MockFixture, MockType -from pytest_simcore.helpers.webserver_projects import empty_project_data, NewProject +from pytest_simcore.helpers.webserver_projects import NewProject, empty_project_data from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.constants import FRONTEND_APP_DEFAULT diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index b818165deab..cfa61a8e5df 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -22,6 +22,7 @@ import sqlalchemy as sa from aiohttp import ClientResponse from aiohttp.test_utils import TestClient, TestServer +from deepdiff import DeepDiff # type: ignore[attr-defined] from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( @@ -68,7 +69,6 @@ from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.licenses._licensed_resources_service import DeepDiff from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.socketio.messages import SOCKET_IO_PROJECT_UPDATED_EVENT from simcore_service_webserver.utils import to_datetime diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index c5449a66a28..cdf12044e6c 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -10,7 +10,7 @@ from aioresponses import aioresponses from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject +from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings diff --git a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py index 70784ebca61..f004044c34f 100644 --- a/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/wallets/conftest.py @@ -13,7 +13,7 @@ from faker import Faker from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.webserver_projects import delete_all_projects, NewProject +from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from pytest_simcore.helpers.webserver_users import UserInfoDict from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver.application_settings import ApplicationSettings