fix the hardcoded dir flags and streamline explicit copying for cwltool (PolusAI#344)

Author: vjaganat90

src/sophios/api/pythonapi.py

@@ -821,9 +821,4 @@ def run(self, compile_args_dict: Dict[str, str] = {}, run_args_dict: Dict[str, s
         else:
             run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)
 
-        # Finally, since there is an output file copying bug in cwltool,
-        # we need to copy the output files manually. See comment above.
-        if args.cwl_runner == 'cwltool' and args.copy_output_files:
-            run_local_module.copy_output_files(self.process_name)
-
 # Process = Union[Step, Workflow]

src/sophios/main.py

@@ -199,10 +199,6 @@ def main() -> None:
         else:
             run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False)
 
-        # Finally, since there is an output file copying bug in cwltool,
-        # we need to copy the output files manually. See comment above.
-        if args.cwl_runner == 'cwltool' and args.copy_output_files:
-            run_local.copy_output_files(yaml_stem)
     elif args.generate_cwl_workflow:
         io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
     else:

src/sophios/run_local.py

@@ -165,7 +165,7 @@ def build_cmd(workflow_name: str, basepath: str, cwl_runner: str,
     cmd = [script] + container_pull + quiet + provenance + \
         container_cmd_ + write_summary + skip_schemas + path_check
     if cwl_runner == 'cwltool':
-        cmd += ['--leave-outputs', '--enable-ext',
+        cmd += ['--move-outputs', '--enable-ext',
                 f'{basepath}/{workflow_name}.cwl', f'{basepath}/{workflow_name}_inputs.yml']
     elif cwl_runner == 'toil-cwl-runner':
         container_pull = []
@@ -176,8 +176,7 @@ def build_cmd(workflow_name: str, basepath: str, cwl_runner: str,
                 '--jobStore', f'file:{basepath}/jobStore_{workflow_name}',  # NOTE: This is the equivalent of --cachedir
                 '--clean', 'always',  # This effectively disables caching, but is reproducible
                 '--disableProgress',  # disable the progress bar in the terminal, saves UI cycle
-                '--workDir', '/data1',
-                '--coordinationDir', '/data1',
+                '--enable-ext',
                 '--logLevel', 'INFO',
                 f'{basepath}/{workflow_name}.cwl', f'{basepath}/{workflow_name}_inputs.yml']
         cmd += passthrough_args
@@ -232,6 +231,8 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[
                 print('via cwltool.main.main python API')
                 retval = cwltool.main.main(cmd[1:])
                 print(f'Final output json metadata blob is in output_{yaml_stem}.json')
+                if args.copy_output_files:
+                    copy_output_files(yaml_stem)
             elif cwl_runner == 'toil-cwl-runner':
                 print('via toil.cwl.cwltoil.main python API')
                 retval = toil.cwl.cwltoil.main(cmd[1:])

tests/test_examples.py

@@ -228,11 +228,6 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp
     retval = sophios.run_local.run_local(args, rose_tree, None, cwl_runner, True)
     assert retval == 0
 
-    # Finally, since there is an output file copying bug in cwltool,
-    # we need to copy the output files manually. See comment above.
-    if args.cwl_runner == 'cwltool' and args.copy_output_files:
-        sophios.run_local.copy_output_files(yaml_stem)
-
 
 @pytest.mark.fast
 @pytest.mark.serial