♻️ Refactor Models, Schema Validation, and DB Utilities (Spin-off 1 from PR #8141) (#8360)

Author: pcrespov

packages/aws-library/src/aws_library/s3/__init__.py

@@ -22,10 +22,10 @@
 )
 
 __all__: tuple[str, ...] = (
-    "CopiedBytesTransferredCallback",
-    "MultiPartUploadLinks",
     "PRESIGNED_LINK_MAX_SIZE",
     "S3_MAX_FILE_SIZE",
+    "CopiedBytesTransferredCallback",
+    "MultiPartUploadLinks",
     "S3AccessError",
     "S3BucketInvalidError",
     "S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
     "S3RuntimeError",
     "S3UploadNotFoundError",
     "SimcoreS3API",
-    "UploadedBytesTransferredCallback",
     "UploadID",
+    "UploadedBytesTransferredCallback",
 )
 
 # nopycln: file

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -126,7 +126,7 @@ class ProjectGet(OutputSchema):
 
     # display
     name: str
-    description: str
+    description: Annotated[str, BeforeValidator(none_to_empty_str_pre_validator)]
     thumbnail: HttpUrl | Literal[""]
 
     type: ProjectType
@@ -144,7 +144,7 @@ class ProjectGet(OutputSchema):
     trashed_at: datetime | None
     trashed_by: Annotated[
         GroupID | None, Field(description="The primary gid of the user who trashed")
-    ]
+    ] = None
 
     # labeling
     tags: list[int]
@@ -166,10 +166,6 @@ class ProjectGet(OutputSchema):
     workspace_id: WorkspaceID | None
     folder_id: FolderID | None
 
-    _empty_description = field_validator("description", mode="before")(
-        none_to_empty_str_pre_validator
-    )
-
     @staticmethod
     def _update_json_schema_extra(schema: JsonDict) -> None:
         schema.update(

packages/models-library/src/models_library/projects.py

@@ -10,6 +10,7 @@
 from common_library.basic_types import DEFAULT_FACTORY
 from pydantic import (
     BaseModel,
+    BeforeValidator,
     ConfigDict,
     Field,
     HttpUrl,
@@ -85,13 +86,17 @@ class BaseProjectModel(BaseModel):
     ]
     description: Annotated[
         str,
+        BeforeValidator(none_to_empty_str_pre_validator),
         Field(
             description="longer one-line description about the project",
             examples=["Dabbling in temporal transitions ..."],
         ),
     ]
     thumbnail: Annotated[
         HttpUrl | None,
+        BeforeValidator(
+            empty_str_to_none_pre_validator,
+        ),
         Field(
             description="url of the project thumbnail",
             examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
@@ -104,15 +109,6 @@ class BaseProjectModel(BaseModel):
     # Pipeline of nodes (SEE projects_nodes.py)
     workbench: Annotated[NodesDict, Field(description="Project's pipeline")]
 
-    # validators
-    _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")(
-        empty_str_to_none_pre_validator
-    )
-
-    _none_description_is_empty = field_validator("description", mode="before")(
-        none_to_empty_str_pre_validator
-    )
-
 
 class ProjectAtDB(BaseProjectModel):
     # Model used to READ from database

packages/models-library/src/models_library/projects_nodes.py

@@ -177,14 +177,22 @@ class NodeState(BaseModel):
 
     model_config = ConfigDict(
         extra="forbid",
-        populate_by_name=True,
+        validate_by_alias=True,
+        validate_by_name=True,
         json_schema_extra={
             "examples": [
+                # example with alias name
                 {
                     "modified": True,
                     "dependencies": [],
                     "currentStatus": "NOT_STARTED",
                 },
+                # example with field name
+                {
+                    "modified": True,
+                    "dependencies": [],
+                    "current_status": "NOT_STARTED",
+                },
                 {
                     "modified": True,
                     "dependencies": ["42838344-03de-4ce2-8d93-589a5dcdfd05"],
@@ -234,16 +242,19 @@ class Node(BaseModel):
         Field(
             ge=0,
             le=100,
-            description="the node progress value (deprecated in DB, still used for API only)",
-            deprecated=True,  # <-- Think this is not true, it is still used by the File Picker (frontend nodes)
+            description="the node progress value",
+            deprecated=True,  # NOTE: still used in the File Picker (frontend nodes) and must be removed first from there before retiring it here
+            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/8365
         ),
     ] = None
 
-    thumbnail: Annotated[  # <-- (DEPRECATED) Can be removed
+    thumbnail: Annotated[
         str | HttpUrl | None,
         Field(
             description="url of the latest screenshot of the node",
             examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
+            deprecated=True,
+            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/8365
         ),
     ] = None
 
@@ -302,22 +313,31 @@ class Node(BaseModel):
         Field(default_factory=dict, description="values of output properties"),
     ] = DEFAULT_FACTORY
 
-    output_node: Annotated[bool | None, Field(deprecated=True, alias="outputNode")] = (
-        None  # <-- (DEPRECATED) Can be removed
-    )
+    output_node: Annotated[
+        bool | None,
+        Field(
+            deprecated=True,
+            alias="outputNode",
+            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/8365
+        ),
+    ] = None
 
     output_nodes: Annotated[  # <-- (DEPRECATED) Can be removed
         list[NodeID] | None,
         Field(
             description="Used in group-nodes. Node IDs of those connected to the output",
             alias="outputNodes",
+            deprecated=True,
+            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/8365
         ),
     ] = None
 
-    parent: Annotated[  # <-- (DEPRECATED) Can be removed
+    parent: Annotated[
         NodeID | None,
         Field(
             description="Parent's (group-nodes') node ID s. Used to group",
+            deprecated=True,
+            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/8365
         ),
     ] = None
 
@@ -334,6 +354,9 @@ class Node(BaseModel):
         Field(default_factory=NodeState, description="The node's state object"),
     ] = DEFAULT_FACTORY
 
+    # NOTE: requested_resources should be here! WARNING: this model is used both in database and rest api!
+    # Model for project_nodes table should NOT be Node but a different one !
+
     boot_options: Annotated[
         dict[EnvVarKey, str] | None,
         Field(
@@ -453,12 +476,14 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
 
     model_config = ConfigDict(
         extra="forbid",
-        populate_by_name=True,
+        validate_by_name=True,
+        validate_by_alias=True,
         json_schema_extra=_update_json_schema_extra,
     )
 
 
 class PartialNode(Node):
-    key: Annotated[ServiceKey, Field(default=None)]
-    version: Annotated[ServiceVersion, Field(default=None)]
-    label: Annotated[str, Field(default=None)]
+    # NOTE: `type: ignore[assignment]` is needed because mypy gets confused when overriding the types by adding the Union with None
+    key: ServiceKey | None = None  # type: ignore[assignment]
+    version: ServiceVersion | None = None  # type: ignore[assignment]
+    label: str | None = None  # type: ignore[assignment]

packages/models-library/src/models_library/projects_nodes_io.py

@@ -30,10 +30,9 @@
     UUID_RE,
 )
 
-NodeID = UUID
-
 UUIDStr: TypeAlias = Annotated[str, StringConstraints(pattern=UUID_RE)]
 
+NodeID: TypeAlias = UUID
 NodeIDStr: TypeAlias = UUIDStr
 
 LocationID: TypeAlias = int

packages/models-library/tests/test_services_types.py

@@ -1,9 +1,13 @@
 import pytest
 from models_library.projects import ProjectID
 from models_library.projects_nodes import NodeID
-from models_library.services_types import ServiceRunID
+from models_library.services_types import ServiceKey, ServiceRunID, ServiceVersion
 from models_library.users import UserID
-from pydantic import PositiveInt
+from pydantic import PositiveInt, TypeAdapter
+from pytest_simcore.helpers.faker_factories import (
+    random_service_key,
+    random_service_version,
+)
 
 
 @pytest.mark.parametrize(
@@ -38,3 +42,14 @@ def test_get_resource_tracking_run_id_for_dynamic():
     assert isinstance(
         ServiceRunID.get_resource_tracking_run_id_for_dynamic(), ServiceRunID
     )
+
+
+@pytest.mark.parametrize(
+    "service_key, service_version",
+    [(random_service_key(), random_service_version()) for _ in range(10)],
+)
+def test_faker_factory_service_key_and_version_are_in_sync(
+    service_key: ServiceKey, service_version: ServiceVersion
+):
+    TypeAdapter(ServiceKey).validate_python(service_key)
+    TypeAdapter(ServiceVersion).validate_python(service_version)

packages/postgres-database/src/simcore_postgres_database/utils_projects.py

@@ -7,7 +7,7 @@
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from .models.projects import projects
-from .utils_repos import transaction_context
+from .utils_repos import pass_or_acquire_connection, transaction_context
 
 
 class DBBaseProjectError(OsparcErrorMixin, Exception):
@@ -22,6 +22,23 @@ class ProjectsRepo:
     def __init__(self, engine):
         self.engine = engine
 
+    async def exists(
+        self,
+        project_uuid: uuid.UUID,
+        *,
+        connection: AsyncConnection | None = None,
+    ) -> bool:
+        async with pass_or_acquire_connection(self.engine, connection) as conn:
+            return (
+                await conn.scalar(
+                    sa.select(1)
+                    .select_from(projects)
+                    .where(projects.c.uuid == f"{project_uuid}")
+                    .limit(1)
+                )
+                is not None
+            )
+
     async def get_project_last_change_date(
         self,
         project_uuid: uuid.UUID,

packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py

@@ -4,19 +4,18 @@
 from typing import Annotated, Any
 
 import asyncpg.exceptions  # type: ignore[import-untyped]
-import sqlalchemy
 import sqlalchemy.exc
 from common_library.async_tools import maybe_await
 from common_library.basic_types import DEFAULT_FACTORY
 from common_library.errors_classes import OsparcErrorMixin
 from pydantic import BaseModel, ConfigDict, Field
-from simcore_postgres_database.utils_aiosqlalchemy import map_db_exception
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 from ._protocols import DBConnection
 from .aiopg_errors import ForeignKeyViolation, UniqueViolation
 from .models.projects_node_to_pricing_unit import projects_node_to_pricing_unit
 from .models.projects_nodes import projects_nodes
+from .utils_aiosqlalchemy import map_db_exception
 
 
 #
@@ -59,6 +58,7 @@ class ProjectNodeCreate(BaseModel):
     input_access: dict[str, Any] | None = None
     input_nodes: list[str] | None = None
     inputs: dict[str, Any] | None = None
+    inputs_required: list[str] | None = None
     inputs_units: dict[str, Any] | None = None
     output_nodes: list[str] | None = None
     outputs: dict[str, Any] | None = None
@@ -103,17 +103,18 @@ async def add(
         """
         if not nodes:
             return []
+
+        values = [
+            {
+                "project_uuid": f"{self.project_uuid}",
+                **node.model_dump(mode="json"),
+            }
+            for node in nodes
+        ]
+
         insert_stmt = (
             projects_nodes.insert()
-            .values(
-                [
-                    {
-                        "project_uuid": f"{self.project_uuid}",
-                        **node.model_dump(exclude_unset=True, mode="json"),
-                    }
-                    for node in nodes
-                ]
-            )
+            .values(values)
             .returning(
                 *[
                     c
@@ -129,14 +130,17 @@ async def add(
             rows = await maybe_await(result.fetchall())
             assert isinstance(rows, list)  # nosec
             return [ProjectNode.model_validate(r) for r in rows]
+
         except ForeignKeyViolation as exc:
             # this happens when the project does not exist, as we first check the node exists
             raise ProjectNodesProjectNotFoundError(
                 project_uuid=self.project_uuid
             ) from exc
+
         except UniqueViolation as exc:
             # this happens if the node already exists on creation
             raise ProjectNodesDuplicateNodeError from exc
+
         except sqlalchemy.exc.IntegrityError as exc:
             raise map_db_exception(
                 exc,