MRG: Merge pull request #525 from octue/fix/fix-serialisation-of-datasets-instantiated-from-files

Fix serialisation of datasets instantiated from files

Author: cortadocodes

docs/source/inter_service_compatibility.rst

@@ -50,3 +50,4 @@
 {"parent_sdk_version": "0.17.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmp5az2xpjx\\\"\\n    },\\n    \\\"id\\\": \\\"9c7cde00-f78f-4c01-b46f-80dd738ab95f\\\",\\n    \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "b9a02ab8-c5e2-4182-9979-f65923beef8e", "forward_logs": "1"}}}
 {"parent_sdk_version": "0.16.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": {\\n            \\\"files\\\": [\\n                {\\n                    \\\"_cloud_metadata\\\": {},\\n                    \\\"cloud_path\\\": null,\\n                    \\\"id\\\": \\\"d04f8c48-c0b4-4ed6-a5ce-308d69ab0d3f\\\",\\n                    \\\"labels\\\": [],\\n                    \\\"name\\\": \\\"a_test_file.csv\\\",\\n                    \\\"path\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphspb3xz1/path-within-dataset/a_test_file.csv\\\",\\n                    \\\"tags\\\": {},\\n                    \\\"timestamp\\\": null\\n                },\\n                {\\n                    \\\"_cloud_metadata\\\": {},\\n                    \\\"cloud_path\\\": null,\\n                    \\\"id\\\": \\\"b3492362-59b3-4d17-b08f-b88ebd1bd2cd\\\",\\n                    \\\"labels\\\": [],\\n                    \\\"name\\\": \\\"another_test_file.csv\\\",\\n                    \\\"path\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphspb3xz1/path-within-dataset/another_test_file.csv\\\",\\n                    \\\"tags\\\": {},\\n                    \\\"timestamp\\\": null\\n                }\\n            ],\\n            \\\"id\\\": \\\"59eb7ee8-3908-4a23-8c08-568b0f6d24dd\\\",\\n            \\\"labels\\\": [],\\n            \\\"name\\\": \\\"tmphspb3xz1\\\",\\n            \\\"path\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmphspb3xz1\\\",\\n            \\\"tags\\\": {}\\n        }\\n    },\\n    \\\"id\\\": \\\"6a9c77d9-3f25-4fa6-8657-e3c44761c624\\\",\\n    \\\"keys\\\": null,\\n    \\\"name\\\": null,\\n    \\\"path\\\": \\\".\\\"\\n}\"}", "attributes": {"question_uuid": "694169c2-16a6-488e-bc23-bef05c1cfdac", "forward_logs": "1"}}}
 {"parent_sdk_version": "0.38.0", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpc_nmpiz2\\\"\\n    },\\n    \\\"id\\\": \\\"739b1ae5-99f8-4bd0-b344-38f35d74b8a8\\\",\\n    \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "7bb5e29f-c8de-4fb4-923b-b5c854a24f49", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.0"}}}
+{"parent_sdk_version": "0.38.1", "question": {"data": "{\"input_values\": {\"height\": 4, \"width\": 72}, \"input_manifest\": \"{\\n    \\\"datasets\\\": {\\n        \\\"my_dataset\\\": \\\"/var/folders/sk/hf5fbp616c77tsys9lz55qn40000gp/T/tmpt1uu7685\\\"\\n    },\\n    \\\"id\\\": \\\"8b3c39f9-5232-4898-998d-934defdc5626\\\",\\n    \\\"name\\\": null\\n}\"}", "attributes": {"question_uuid": "a96e6ec5-3798-4160-b45f-6f248834f740", "forward_logs": "1", "allow_save_diagnostics_data_on_crash": "1", "octue_sdk_version": "0.38.1"}}}

octue/metadata/recorded_questions.jsonl

@@ -131,7 +131,7 @@ def finalise(self, upload_output_datasets_to=None):
             serialised_strands["output_values"] = json.dumps(self.output_values, cls=OctueJSONEncoder)
 
         if self.output_manifest:
-            serialised_strands["output_manifest"] = self.output_manifest.to_primitive()
+            serialised_strands["output_manifest"] = self.output_manifest.serialise()
 
         self.twine.validate(**serialised_strands)
         self._finalised = True

octue/metadata/version_compatibilities.json

@@ -73,6 +73,7 @@ def __init__(
         self._recursive = recursive
         self._ignore_stored_metadata = ignore_stored_metadata
         self._cloud_metadata = {}
+        self._instantiated_from_files_argument = False
 
         if files:
             if not any((isinstance(files, list), isinstance(files, set), isinstance(files, tuple))):
@@ -82,6 +83,7 @@ def __init__(
                 )
 
             self.files = self._instantiate_datafiles(files)
+            self._instantiated_from_files_argument = True
             return
 
         if storage.path.is_cloud_path(self.path):
@@ -377,13 +379,18 @@ def to_primitive(self, include_files=True):
         """
         serialised_dataset = super().to_primitive()
 
-        if self.exists_in_cloud:
-            path_type = "cloud_path"
-        else:
-            path_type = "local_path"
-
         if include_files:
-            serialised_dataset["files"] = sorted(getattr(datafile, path_type) for datafile in self.files)
+            serialised_dataset["files"] = []
+
+            for datafile in self.files:
+                if datafile.exists_in_cloud:
+                    datafile_path_type = "cloud_path"
+                else:
+                    datafile_path_type = "local_path"
+
+                serialised_dataset["files"].append(getattr(datafile, datafile_path_type))
+
+            serialised_dataset["files"] = sorted(serialised_dataset["files"])
 
         return serialised_dataset
 

octue/resources/analysis.py

@@ -114,7 +114,14 @@ def to_primitive(self):
         :return dict:
         """
         self_as_primitive = super().to_primitive()
-        self_as_primitive["datasets"] = {name: dataset.path for name, dataset in self.datasets.items()}
+        self_as_primitive["datasets"] = {}
+
+        for name, dataset in self.datasets.items():
+            if dataset._instantiated_from_files_argument:
+                self_as_primitive["datasets"][name] = dataset.to_primitive()
+            else:
+                self_as_primitive["datasets"][name] = dataset.path
+
         return self_as_primitive
 
     def _instantiate_datasets(self, datasets):
@@ -150,9 +157,9 @@ def _instantiate_dataset(self, key_and_dataset):
         if isinstance(dataset, str):
             return (key, Dataset(path=dataset, recursive=True))
 
-        # If `dataset` is a dictionary including a "path" key:
-        if storage.path.is_cloud_path(dataset["path"]):
-            return (key, Dataset(path=dataset["path"], recursive=True))
+        # If `dataset` is a cloud dataset and is represented as a dictionary including a "path" key:
+        if storage.path.is_cloud_path(dataset.get("path", "")):
+            return (key, Dataset(path=dataset["path"], files=dataset.get("files"), recursive=True))
 
         return (key, Dataset(**dataset))