♻️ [Maintenance] Refactor Tests Utilities and Typing (Spin-off 2 from PR #8141) (#8368)

Author: pcrespov

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py

@@ -114,21 +114,21 @@ def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue:
             and not isinstance(v, PortLink)
         ):
             if port_utils.is_file_type(property_type):
-                if not isinstance(v, (FileLink, DownloadLink)):
-                    raise ValueError(
-                        f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas"
-                    )
+                if not isinstance(v, FileLink | DownloadLink):
+                    msg = f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas"
+                    raise ValueError(msg)
             elif property_type == "ref_contentSchema":
                 v, _ = validate_port_content(
                     port_key=info.data.get("key"),
                     value=v,
                     unit=None,
                     content_schema=info.data.get("content_schema", {}),
                 )
-            elif isinstance(v, (list, dict)):
-                raise TypeError(
+            elif isinstance(v, list | dict):
+                msg = (
                     f"Containers as {v} currently only supported within content_schema."
                 )
+                raise TypeError(msg)
         return v
 
     @field_validator("value_item", "value_concrete", mode="before")
@@ -194,28 +194,29 @@ async def get_value(
         )
 
         async def _evaluate() -> ItemValue | None:
+            # NOTE: review types returned by this function !!!
             if isinstance(self.value, PortLink):
                 # this is a link to another node's port
-                other_port_itemvalue: None | (
-                    ItemValue
-                ) = await port_utils.get_value_link_from_port_link(
-                    self.value,
-                    # pylint: disable=protected-access
-                    self._node_ports._node_ports_creator_cb,
-                    file_link_type=file_link_type,
+                other_port_itemvalue: ItemValue | None = (
+                    await port_utils.get_value_link_from_port_link(
+                        self.value,
+                        # pylint: disable=protected-access
+                        self._node_ports._node_ports_creator_cb,
+                        file_link_type=file_link_type,
+                    )
                 )
 
                 return other_port_itemvalue
 
             if isinstance(self.value, FileLink):
                 # let's get the download/upload link from storage
-                url_itemvalue: None | (
-                    AnyUrl
-                ) = await port_utils.get_download_link_from_storage(
-                    # pylint: disable=protected-access
-                    user_id=self._node_ports.user_id,
-                    value=self.value,
-                    link_type=file_link_type,
+                url_itemvalue: AnyUrl | None = (
+                    await port_utils.get_download_link_from_storage(
+                        # pylint: disable=protected-access
+                        user_id=self._node_ports.user_id,
+                        value=self.value,
+                        link_type=file_link_type,
+                    )
                 )
                 return url_itemvalue
 
@@ -256,15 +257,15 @@ async def _evaluate() -> ItemConcreteValue | None:
 
             if isinstance(self.value, PortLink):
                 # this is a link to another node
-                other_port_concretevalue: None | (
-                    ItemConcreteValue
-                ) = await port_utils.get_value_from_link(
-                    # pylint: disable=protected-access
-                    key=self.key,
-                    value=self.value,
-                    file_to_key_map=self.file_to_key_map,
-                    node_port_creator=self._node_ports._node_ports_creator_cb,  # noqa: SLF001
-                    progress_bar=progress_bar,
+                other_port_concretevalue: None | ItemConcreteValue = (
+                    await port_utils.get_value_from_link(
+                        # pylint: disable=protected-access
+                        key=self.key,
+                        value=self.value,
+                        file_to_key_map=self.file_to_key_map,
+                        node_port_creator=self._node_ports._node_ports_creator_cb,  # noqa: SLF001
+                        progress_bar=progress_bar,
+                    )
                 )
                 value = other_port_concretevalue
 

services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py

@@ -400,6 +400,9 @@ async def generate_tasks_list_from_project(
             raise WalletNotEnoughCreditsError(
                 wallet_name=wallet_info.wallet_name,
                 wallet_credit_amount=wallet_info.wallet_credit_amount,
+                user_id=user_id,
+                product_name=product_name,
+                project_id=project.uuid,
             )
 
         assert rabbitmq_rpc_client  # nosec

services/director-v2/src/simcore_service_director_v2/modules/db/tables.py

@@ -10,17 +10,21 @@
 )
 from simcore_postgres_database.models.projects import ProjectType, projects
 from simcore_postgres_database.models.projects_networks import projects_networks
+from simcore_postgres_database.models.projects_nodes import projects_nodes
 
-__all__ = [
+__all__: tuple[str, ...] = (
+    "NodeClass",
+    "ProjectType",
+    "StateType",
     "comp_pipeline",
+    "comp_run_snapshot_tasks",
     "comp_runs",
     "comp_tasks",
     "groups_extra_properties",
-    "NodeClass",
-    "projects_networks",
     "projects",
-    "ProjectType",
-    "StateType",
+    "projects_networks",
+    "projects_nodes",
     "user_to_groups",
-    "comp_run_snapshot_tasks",
-]
+)
+
+# nopycln: file

services/director-v2/src/simcore_service_director_v2/utils/dags.py

@@ -29,6 +29,7 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph:
     dag_graph: nx.DiGraph = nx.DiGraph()
     for node_id, node in workbench.items():
         assert node.state  # nosec
+
         dag_graph.add_node(
             node_id,
             name=node.label,
@@ -43,6 +44,9 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph:
         if node.input_nodes:
             for input_node_id in node.input_nodes:
                 predecessor_node = workbench.get(f"{input_node_id}")
+                assert (  # nosec
+                    predecessor_node
+                ), f"Node {input_node_id} not found in workbench"
                 if predecessor_node:
                     dag_graph.add_edge(str(input_node_id), node_id)
 

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -14109,12 +14109,6 @@ components:
           - $ref: '#/components/schemas/NodeState-Input'
           - type: 'null'
           description: The node's state object
-        required_resources:
-          anyOf:
-          - additionalProperties: true
-            type: object
-          - type: 'null'
-          title: Required Resources
         bootOptions:
           anyOf:
           - type: object
@@ -14252,12 +14246,6 @@ components:
           - $ref: '#/components/schemas/NodeState-Output'
           - type: 'null'
           description: The node's state object
-        required_resources:
-          anyOf:
-          - additionalProperties: true
-            type: object
-          - type: 'null'
-          title: Required Resources
         bootOptions:
           anyOf:
           - type: object

services/web/server/tests/data/fake-project.json

@@ -77,7 +77,7 @@
         "x": 1073,
         "y": 307
       },
-      "progress": 100,
+      "progress": 100.0,
       "state": {
         "currentStatus": "NOT_STARTED",
         "lock_state": {

services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py

@@ -15,6 +15,7 @@
 import sqlalchemy as sa
 from aiohttp.test_utils import TestClient
 from aioresponses import aioresponses
+from deepdiff import DeepDiff
 from faker import Faker
 from models_library.api_schemas_directorv2.dynamic_services import (
     GetProjectInactivityResponse,
@@ -23,7 +24,10 @@
 from models_library.products import ProductName
 from pydantic import TypeAdapter
 from pytest_mock import MockerFixture
-from pytest_simcore.helpers.assert_checks import assert_status
+from pytest_simcore.helpers.assert_checks import (
+    assert_equal_ignoring_none,
+    assert_status,
+)
 from pytest_simcore.helpers.webserver_parametrizations import (
     ExpectedResponse,
     MockedStorageSubsystem,
@@ -174,7 +178,9 @@ async def _assert_get_same_project(
         project_permalink = data.pop("permalink", None)
         folder_id = data.pop("folderId", None)
 
-        assert data == {k: project[k] for k in data}
+        assert not DeepDiff(
+            data, {k: project[k] for k in data}, exclude_paths="root['lastChangeDate']"
+        )
 
         if project_state:
             assert ProjectStateOutputSchema.model_validate(project_state)
@@ -215,7 +221,11 @@ async def test_list_projects(
         project_permalink = got.pop("permalink")
         folder_id = got.pop("folderId")
 
-        assert got == {k: template_project[k] for k in got}
+        assert not DeepDiff(
+            got,
+            {k: template_project[k] for k in got},
+            exclude_paths="root['lastChangeDate']",
+        )
 
         assert not ProjectStateOutputSchema(
             **project_state
@@ -228,7 +238,11 @@ async def test_list_projects(
         project_permalink = got.pop("permalink", None)
         folder_id = got.pop("folderId")
 
-        assert got == {k: user_project[k] for k in got}
+        assert not DeepDiff(
+            got,
+            {k: user_project[k] for k in got},
+            exclude_paths="root['lastChangeDate']",
+        )
 
         assert ProjectStateOutputSchema(**project_state)
         assert project_permalink is None
@@ -245,7 +259,12 @@ async def test_list_projects(
         project_permalink = got.pop("permalink", None)
         folder_id = got.pop("folderId")
 
-        assert got == {k: user_project[k] for k in got}
+        assert not DeepDiff(
+            got,
+            {k: user_project[k] for k in got},
+            exclude_paths="root['lastChangeDate']",
+        )
+
         assert not ProjectStateOutputSchema(
             **project_state
         ).share_state.locked, "Single user does not lock"
@@ -263,7 +282,11 @@ async def test_list_projects(
         project_permalink = got.pop("permalink")
         folder_id = got.pop("folderId")
 
-        assert got == {k: template_project[k] for k in got}
+        assert not DeepDiff(
+            got,
+            {k: template_project[k] for k in got},
+            exclude_paths="root['lastChangeDate']",
+        )
         assert not ProjectStateOutputSchema(
             **project_state
         ).share_state.locked, "Templates are not locked"
@@ -632,7 +655,7 @@ async def test_new_template_from_project(
         )
 
         assert len(templates) == 1
-        assert templates[0] == template_project
+        assert_equal_ignoring_none(template_project, templates[0])
 
         assert template_project["name"] == user_project["name"]
         assert template_project["description"] == user_project["description"]

services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py

@@ -12,6 +12,7 @@
 
 import pytest
 from aiohttp.test_utils import TestClient
+from deepdiff import DeepDiff
 from pytest_mock.plugin import MockerFixture
 from pytest_simcore.helpers.assert_checks import assert_status
 from pytest_simcore.helpers.webserver_users import UserInfoDict
@@ -168,7 +169,9 @@ async def test_patch_project_node(
             "output_1": {
                 "store": 0,
                 "path": "9934cba6-4b51-11ef-968a-02420a00f1c1/571ffc8d-fa6e-411f-afc8-9c62d08dd2fa/matus.txt",
+                "label": "matus.txt",
                 "eTag": "d41d8cd98f00b204e9800998ecf8427e",
+                "dataset": None,
             }
         }
     }
@@ -185,7 +188,6 @@ async def test_patch_project_node(
     _tested_node = data["workbench"][node_id]
 
     assert _tested_node["label"] == "testing-string"
-    assert _tested_node["progress"] is None
     assert _tested_node["key"] == _patch_key["key"]
     assert _tested_node["version"] == _patch_version["version"]
     assert _tested_node["inputs"] == _patch_inputs["inputs"]
@@ -262,10 +264,14 @@ async def test_patch_project_node_inputs_notifies(
     await assert_status(resp, expected)
     assert mocked_notify_project_node_update.call_count > 1
     # 1 message per node updated
-    assert [
-        call_args[0][2]
-        for call_args in mocked_notify_project_node_update.await_args_list
-    ] == list(user_project["workbench"].keys())
+    assert not DeepDiff(
+        [
+            call_args[0][2]
+            for call_args in mocked_notify_project_node_update.await_args_list
+        ],
+        list(user_project["workbench"].keys()),
+        ignore_order=True,
+    )
 
 
 @pytest.mark.parametrize(