Correctly handle file and directory literals (#218)

* Correctly handle file and directory literals produced by expression tool in
output.

* Fix sandboxjs to use "docker images" before "docker pull" so it doesn't rely on
being online to use node:slim for JS evaluation.

Author: tetron

cwltool/draft2tool.py

@@ -16,7 +16,7 @@
 import shellescape
 from typing import Any, Callable, cast, Generator, Text, Union
 
-from .process import Process, shortname, uniquename, getListing, normalizeFilesDirs
+from .process import Process, shortname, uniquename, getListing, normalizeFilesDirs, compute_checksums
 from .errors import WorkflowException
 from .utils import aslist
 from . import expression
@@ -146,19 +146,6 @@ def check_adjust(builder, f):
         raise WorkflowException("Invalid filename: '%s' contains illegal characters" % (f["basename"]))
     return f
 
-def compute_checksums(fs_access, fileobj):
-    if "checksum" not in fileobj:
-        checksum = hashlib.sha1()
-        with fs_access.open(fileobj["location"], "rb") as f:
-            contents = f.read(1024*1024)
-            while contents != "":
-                checksum.update(contents)
-                contents = f.read(1024*1024)
-            f.seek(0, 2)
-            filesize = f.tell()
-        fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()
-        fileobj["size"] = filesize
-
 class CommandLineTool(Process):
     def __init__(self, toolpath_object, **kwargs):
         # type: (Dict[Text, Any], **Any) -> None

cwltool/load_tool.py

@@ -105,6 +105,8 @@ def validate_document(document_loader, workflowobj, uri,
     fileuri = urlparse.urldefrag(uri)[0]
 
     if "cwlVersion" in workflowobj:
+        if not isinstance(workflowobj["cwlVersion"], (str, Text)):
+            raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
         workflowobj["cwlVersion"] = re.sub(
             r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
             workflowobj["cwlVersion"])

cwltool/pathmapper.py

@@ -145,21 +145,21 @@ def visitlisting(self, listing, stagedir, basedir):
         for ld in listing:
             tgt = os.path.join(stagedir, ld["basename"])
             if ld["class"] == "Directory":
-                self.visit(ld, tgt, basedir, copy=ld.get("writable", False))
+                self.visit(ld, stagedir, basedir, copy=ld.get("writable", False))
             else:
                 self.visit(ld, stagedir, basedir, copy=ld.get("writable", False))
 
     def visit(self, obj, stagedir, basedir, copy=False):
         # type: (Dict[Text, Any], Text, Text, bool) -> None
+        tgt = os.path.join(stagedir, obj["basename"])
         if obj["class"] == "Directory":
-            self._pathmap[obj["location"]] = MapperEnt(obj["location"], stagedir, "Directory")
-            self.visitlisting(obj.get("listing", []), stagedir, basedir)
+            self._pathmap[obj["location"]] = MapperEnt(obj["location"], tgt, "Directory")
+            self.visitlisting(obj.get("listing", []), tgt, basedir)
         elif obj["class"] == "File":
             path = obj["location"]
             if path in self._pathmap:
                 return
             ab = abspath(path, basedir)
-            tgt = os.path.join(stagedir, obj["basename"])
             if "contents" in obj and obj["location"].startswith("_:"):
                 self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile")
             else:

cwltool/process.py

@@ -12,6 +12,7 @@
 import errno
 import shutil
 import uuid
+import hashlib
 
 import abc
 import schema_salad.validate as validate
@@ -200,14 +201,14 @@ def relocateOutputs(outputObj, outdir, output_dirs, action):
         return outputObj
 
     def moveIt(src, dst):
-        for a in output_dirs:
-            if src.startswith(a):
-                if action == "move":
+        if action == "move":
+            for a in output_dirs:
+                if src.startswith(a):
                     _logger.debug("Moving %s to %s", src, dst)
                     shutil.move(src, dst)
-                elif action == "copy":
-                    _logger.debug("Copying %s to %s", src, dst)
-                    shutil.copy(src, dst)
+                    return
+        _logger.debug("Copying %s to %s", src, dst)
+        shutil.copy(src, dst)
 
     outfiles = []  # type: List[Dict[Text, Any]]
     collectFilesAndDirs(outputObj, outfiles)
@@ -216,6 +217,10 @@ def moveIt(src, dst):
 
     def _check_adjust(f):
         f["location"] = "file://" + pm.mapper(f["location"])[1]
+        if "contents" in f:
+            del f["contents"]
+        if f["class"] == "File":
+            compute_checksums(StdFsAccess(""), f)
         return f
 
     adjustFileObjs(outputObj, _check_adjust)
@@ -700,3 +705,16 @@ def scandeps(base, doc, reffields, urlfields, loadref):
         normalizeFilesDirs(r)
         r = mergedirs(r)
     return r
+
+def compute_checksums(fs_access, fileobj):
+    if "checksum" not in fileobj:
+        checksum = hashlib.sha1()
+        with fs_access.open(fileobj["location"], "rb") as f:
+            contents = f.read(1024*1024)
+            while contents != "":
+                checksum.update(contents)
+                contents = f.read(1024*1024)
+            f.seek(0, 2)
+            filesize = f.tell()
+        fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()
+        fileobj["size"] = filesize

cwltool/sandboxjs.py

@@ -45,8 +45,10 @@ def new_js_proc():
             nodeimg = "node:slim"
             global have_node_slim
             if not have_node_slim:
-                nodejsimg = subprocess.check_output(["docker", "pull", nodeimg])
-                _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg)
+                dockerimgs = subprocess.check_output(["docker", "images", nodeimg])
+                if len(dockerimgs.split("\n")) <= 1:
+                    nodejsimg = subprocess.check_output(["docker", "pull", nodeimg])
+                    _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg)
                 have_node_slim = True
             nodejs = subprocess.Popen(["docker", "run",
                                        "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR",