MRG: Merge pull request #8 from octue/enhancement/add-output-manifest

Add output manifest and output datasets upload

Author: cortadocodes

app.py

@@ -1,6 +1,10 @@
 import logging
+import os
+import tempfile
 import time
 
+from octue.resources import Datafile, Dataset
+
 from example_package.submodule import do_something
 
 
@@ -10,6 +14,15 @@
 def run(analysis):
     logger.info("Started example analysis.")
     do_something()
-    time.sleep(5)
+    time.sleep(2)
     analysis.output_values = [1, 2, 3, 4, 5]
+
+    with tempfile.TemporaryDirectory() as temporary_directory:
+
+        with Datafile(os.path.join(temporary_directory, "output.dat"), mode="w") as (datafile, f):
+            f.write("This is some example service output.")
+
+        analysis.output_manifest.datasets["example_dataset"] = Dataset(path=temporary_directory, files={datafile})
+        analysis.finalise(upload_output_datasets_to="gs://octue-test-bucket/example_output_datasets")
+
     logger.info("Finished example analysis.")

requirements-dev.txt

@@ -0,0 +1,3 @@
+gcp-storage-emulator==2022.4.13
+
+-e .

setup.py

@@ -2,9 +2,9 @@
 
 setup(
     name="example-service",
-    version="0.1.6",
+    version="0.1.7",
     install_requires=[
-        "octue==0.15.7",
+        "octue==0.21.0",
     ],
     url="https://www.github.com/octue/example-service-cloud-run",
     author="cortadocodes",

tests/__init__.py

@@ -0,0 +1,39 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from octue import Runner
+from octue.cloud.emulators import GoogleCloudStorageEmulatorTestResultModifier, mock_generate_signed_url
+from octue.resources import Manifest
+
+
+REPOSITORY_ROOT = os.path.dirname(os.path.dirname(__file__))
+TWINE_PATH = os.path.join(REPOSITORY_ROOT, "twine.json")
+TEST_BUCKET_NAME = "octue-test-bucket"
+
+
+class TestApp(unittest.TestCase):
+    test_result_modifier = GoogleCloudStorageEmulatorTestResultModifier(default_bucket_name=TEST_BUCKET_NAME)
+    setattr(unittest.TestResult, "startTestRun", test_result_modifier.startTestRun)
+    setattr(unittest.TestResult, "stopTestRun", test_result_modifier.stopTestRun)
+
+    def test_app(self):
+        """Test that the app takes in input in the correct format and returns an analysis with the correct output
+        values.
+        """
+        runner = Runner(app_src=REPOSITORY_ROOT, twine=TWINE_PATH)
+
+        with patch("octue.resources.dataset.Dataset._save_local_metadata"):
+            with patch("google.cloud.storage.blob.Blob.generate_signed_url", mock_generate_signed_url):
+                analysis = runner.run(input_values={"n_iterations": 3})
+
+        # Check the output values.
+        self.assertEqual(analysis.output_values, [1, 2, 3, 4, 5])
+
+        # Test that the signed URLs for the dataset and its files work and can be used to reinstantiate the output
+        # manifest after serialisation.
+        downloaded_output_manifest = Manifest.deserialise(analysis.output_manifest.to_primitive())
+
+        # Check that the output dataset and its files can be accessed.
+        with downloaded_output_manifest.datasets["example_dataset"].files.one() as (datafile, f):
+            self.assertEqual(f.read(), "This is some example service output.")

tests/test_app.py

@@ -0,0 +1,10 @@
+[tox]
+envlist = {py38}
+
+[testenv]
+passenv = GOOGLE_APPLICATION_CREDENTIALS GOOGLE_CLOUD_PROJECT TEST_PROJECT_NAME
+setenv =
+    PYTHONPATH = {toxinidir}:{toxinidir}/octue
+commands =
+    python -m unittest discover
+deps = -r requirements-dev.txt

tox.ini

@@ -15,5 +15,10 @@
 		"items": {
 			"type": "number"
 		}
-	}
+	},
+    "output_manifest": {
+        "datasets": {
+            "example_dataset": {}
+        }
+    }
 }