Release and updates to v1.2.0 beta.3 (#171)

* add FQCN to action plugins (#167)

* add FQCN to action plugins

* fix pep8

* Doc generation updates to correct generation

* Update Makefile to remove inclusive auto gen calls and fixed doc links

* update comment in conf

* correct bad whitespace

Co-authored-by: Blake Becker <[email protected]>

Author: ddimatos

docs/source/conf.py

@@ -150,7 +150,8 @@
 # White list which Git tags documentation will be generated and linked into the
 # version selection box. This is currently a manual selection, until more
 # versions are released, there are no regular expressions used.
-scv_whitelist_tags = ('v1.0.0', 'v1.1.0', 'v1.2.0-beta.1', 'v1.2.0-beta.2')
+scv_whitelist_tags = ('v1.0.0', 'v1.1.0', 'v1.2.0-beta.1', 'v1.2.0-beta.2',
+                      'v1.2.0-beta.3')
 
 # Sort versions by one or more values. Valid values are semver, alpha, and time.
 # Semantic is referred to as 'semver', this would ensure our latest VRM is

plugins/action/zos_fetch.py

@@ -1,7 +1,7 @@
 # Copyright (c) IBM Corporation 2019, 2020
 # Apache License, Version 2.0 (see https://opensource.org/licenses/Apache-2.0)
 
-from __future__ import (absolute_import, division, print_function)
+from __future__ import absolute_import, division, print_function
 
 __metaclass__ = type
 
@@ -23,21 +23,23 @@
 def _update_result(result, src, dest, ds_type="USS", is_binary=False):
     """ Helper function to update output result with the provided values """
     data_set_types = {
-        'PS': "Sequential",
-        'PO': "Partitioned",
-        'PDSE': "Partitioned Extended",
-        'PE': "Partitioned Extended",
-        'VSAM': "VSAM",
-        'USS': "USS"
+        "PS": "Sequential",
+        "PO": "Partitioned",
+        "PDSE": "Partitioned Extended",
+        "PE": "Partitioned Extended",
+        "VSAM": "VSAM",
+        "USS": "USS",
     }
-    file_or_ds = "file" if ds_type == 'USS' else "data set"
+    file_or_ds = "file" if ds_type == "USS" else "data set"
     updated_result = dict((k, v) for k, v in result.items())
-    updated_result.update({
-        'file': src,
-        'dest': dest,
-        'data_set_type': data_set_types[ds_type],
-        'is_binary': is_binary
-    })
+    updated_result.update(
+        {
+            "file": src,
+            "dest": dest,
+            "data_set_type": data_set_types[ds_type],
+            "is_binary": is_binary,
+        }
+    )
     return updated_result
 
 
@@ -59,7 +61,7 @@ def _get_file_checksum(src):
     blksize = 64 * 1024
     hash_digest = sha256()
     try:
-        with open(to_bytes(src, errors='surrogate_or_strict'), 'rb') as infile:
+        with open(to_bytes(src, errors="surrogate_or_strict"), "rb") as infile:
             block = infile.read(blksize)
             while block:
                 hash_digest.update(block)
@@ -101,7 +103,7 @@ def run(self, tmp=None, task_vars=None):
             self._task.args.get('ignore_sftp_stderr'), default=False
         )
         validate_checksum = _process_boolean(
-            self._task.args.get('validate_checksum'), default=True
+            self._task.args.get("validate_checksum"), default=True
         )
 
         # ********************************************************** #
@@ -112,14 +114,10 @@ def run(self, tmp=None, task_vars=None):
         if src is None or dest is None:
             msg = "Source and destination are required"
         elif not isinstance(src, string_types):
-            msg = (
-                "Invalid type supplied for 'source' option, "
-                "it must be a string"
-            )
+            msg = "Invalid type supplied for 'source' option, " "it must be a string"
         elif not isinstance(dest, string_types):
             msg = (
-                "Invalid type supplied for 'destination' option, "
-                "it must be a string"
+                "Invalid type supplied for 'destination' option, " "it must be a string"
             )
         elif len(src) < 1 or len(dest) < 1:
             msg = "Source and destination parameters must not be empty"
@@ -128,12 +126,12 @@ def run(self, tmp=None, task_vars=None):
             msg = "Invalid port provided for SFTP. Expected an integer between 0 to 65535."
 
         if msg:
-            result['msg'] = msg
-            result['failed'] = True
+            result["msg"] = msg
+            result["failed"] = True
             return result
 
         ds_type = None
-        fetch_member = '(' in src and src.endswith(')')
+        fetch_member = "(" in src and src.endswith(")")
         if fetch_member:
             member_name = src[src.find('(') + 1:src.find(')')]
         src = self._connection._shell.join_path(src)
@@ -155,24 +153,23 @@ def run(self, tmp=None, task_vars=None):
         #  and dest is: /tmp/, then updated dest would be /tmp/DATA  #
         # ********************************************************** #
 
-        if os.path.sep not in self._connection._shell.join_path('a', ''):
+        if os.path.sep not in self._connection._shell.join_path("a", ""):
             src = self._connection._shell._unquote(src)
-            source_local = src.replace('\\', '/')
+            source_local = src.replace("\\", "/")
         else:
             source_local = src
 
         dest = os.path.expanduser(dest)
         if flat:
-            if (
-                os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and
-                not dest.endswith(os.sep)
-            ):
-                result['msg'] = (
+            if os.path.isdir(
+                to_bytes(dest, errors="surrogate_or_strict")
+            ) and not dest.endswith(os.sep):
+                result["msg"] = (
                     "dest is an existing directory, append a forward "
                     "slash to the dest if you want to fetch src into "
                     "that directory"
                 )
-                result['failed'] = True
+                result["failed"] = True
                 return result
             if dest.endswith(os.sep):
                 if fetch_member:
@@ -184,22 +181,22 @@ def run(self, tmp=None, task_vars=None):
             if not dest.startswith("/"):
                 dest = self._loader.path_dwim(dest)
         else:
-            if 'inventory_hostname' in task_vars:
-                target_name = task_vars['inventory_hostname']
+            if "inventory_hostname" in task_vars:
+                target_name = task_vars["inventory_hostname"]
             else:
                 target_name = self._play_context.remote_addr
             suffix = member_name if fetch_member else source_local
             dest = "{0}/{1}/{2}".format(
-                self._loader.path_dwim(dest),
-                target_name,
-                suffix
+                self._loader.path_dwim(dest), target_name, suffix
             )
             try:
                 dirname = os.path.dirname(dest).replace("//", "/")
                 if not os.path.exists(dirname):
                     os.makedirs(dirname)
             except OSError as err:
-                result["msg"] = "Unable to create destination directory {0}".format(dirname)
+                result["msg"] = "Unable to create destination directory {0}".format(
+                    dirname
+                )
                 result["stderr"] = str(err)
                 result["stderr_lines"] = str(err).splitlines()
                 result["failed"] = True
@@ -216,57 +213,63 @@ def run(self, tmp=None, task_vars=None):
             fetch_res = self._execute_module(
                 module_name='zos_fetch',
                 module_args=self._task.args,
-                task_vars=task_vars
+                task_vars=task_vars,
             )
-            ds_type = fetch_res.get('ds_type')
-            src = fetch_res.get('file')
-            remote_path = fetch_res.get('remote_path')
-
-            if fetch_res.get('msg'):
-                result['msg'] = fetch_res.get('msg')
-                result['stdout'] = fetch_res.get('stdout') or fetch_res.get("module_stdout")
-                result['stderr'] = fetch_res.get('stderr') or fetch_res.get("module_stderr")
-                result['stdout_lines'] = fetch_res.get('stdout_lines')
-                result['stderr_lines'] = fetch_res.get('stderr_lines')
+            ds_type = fetch_res.get("ds_type")
+            src = fetch_res.get("file")
+            remote_path = fetch_res.get("remote_path")
+
+            if fetch_res.get("msg"):
+                result["msg"] = fetch_res.get("msg")
+                result["stdout"] = fetch_res.get("stdout") or fetch_res.get(
+                    "module_stdout"
+                )
+                result["stderr"] = fetch_res.get("stderr") or fetch_res.get(
+                    "module_stderr"
+                )
+                result["stdout_lines"] = fetch_res.get("stdout_lines")
+                result["stderr_lines"] = fetch_res.get("stderr_lines")
                 result["rc"] = fetch_res.get("rc")
-                result['failed'] = True
+                result["failed"] = True
                 return result
 
-            elif fetch_res.get('note'):
-                result['note'] = fetch_res.get('note')
+            elif fetch_res.get("note"):
+                result["note"] = fetch_res.get("note")
                 return result
 
             if ds_type in SUPPORTED_DS_TYPES:
                 if ds_type == "PO" and os.path.isfile(dest) and not fetch_member:
-                    result["msg"] = "Destination must be a directory to fetch a partitioned data set"
+                    result[
+                        "msg"
+                    ] = "Destination must be a directory to fetch a partitioned data set"
                     result["failed"] = True
                     return result
 
                 fetch_content = self._transfer_remote_content(
                     dest, remote_path, ds_type, sftp_port, ignore_stderr=ignore_sftp_stderr
                 )
-                if fetch_content.get('msg'):
+                if fetch_content.get("msg"):
                     return fetch_content
 
                 if validate_checksum and ds_type != "PO" and not is_binary:
                     new_checksum = _get_file_checksum(dest)
-                    result['changed'] = local_checksum != new_checksum
-                    result['checksum'] = new_checksum
+                    result["changed"] = local_checksum != new_checksum
+                    result["checksum"] = new_checksum
                 else:
-                    result['changed'] = True
+                    result["changed"] = True
 
             else:
-                result['msg'] = (
+                result["msg"] = (
                     "The data set type '{0}' is not"
                     " currently supported".format(ds_type)
                 )
-                result['failed'] = True
+                result["failed"] = True
                 return result
         except Exception as err:
-            result['msg'] = "Failure during module execution"
-            result['stderr'] = str(err)
-            result['stderr_lines'] = str(err).splitlines()
-            result['failed'] = True
+            result["msg"] = "Failure during module execution"
+            result["stderr"] = str(err)
+            result["stderr_lines"] = str(err).splitlines()
+            result["failed"] = True
             return result
 
         # ********************************************************** #
@@ -292,10 +295,7 @@ def _transfer_remote_content(self, dest, remote_path, src_type, port, ignore_std
             stdin = stdin.replace(" -r", "")
 
         transfer_pds = subprocess.Popen(
-            cmd,
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
+            cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )
         out, err = transfer_pds.communicate(to_bytes(stdin))
         err = _detect_sftp_errors(err)
@@ -305,8 +305,8 @@ def _transfer_remote_content(self, dest, remote_path, src_type, port, ignore_std
             result['msg'] = "Error transferring remote data from z/OS system"
             result['rc'] = transfer_pds.returncode
         if result.get("msg"):
-            result['stderr'] = err
-            result['failed'] = True
+            result["stderr"] = err
+            result["failed"] = True
         return result
 
     def _remote_cleanup(self, remote_path, src_type, encoding):