Implement explicit use of a hash for git repositories.

Increment the cfg schema to 1.1, and add explicit support for git checkouts via hashes from the config file.

Adds new unit tests for some hash related functionality. Adds checkout of
hashes to some basic systems tests.

Testing:
  make test - python2/3 - all tests pass

Author: bjandre

README.md

@@ -182,14 +182,15 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
 
   * tag (string) : tag to checkout
 
-    This can also be a git SHA-1
+  * hash (string) : the git hash to checkout. Only applies to git
+    repositories.
 
   * branch (string) : branch to checkout from the specified
     repository. Specifying a branch on a remote repository means that
     checkout_externals will checkout the version of the branch in the remote,
     not the the version in the local repository (if it exists).
 
-    Note: either tag or branch must be supplied, but not both.
+    Note: one and only one of tag, branch hash must be supplied.
 
   * externals (string) : used to make manage_externals aware of
     sub-externals required by an external. This is a relative path to

manic/checkout.py

@@ -207,14 +207,15 @@ def commandline_arguments(args=None):
 
   * tag (string) : tag to checkout
 
-    This can also be a git SHA-1
+  * hash (string) : the git hash to checkout. Only applies to git
+    repositories.
 
   * branch (string) : branch to checkout from the specified
     repository. Specifying a branch on a remote repository means that
     %(prog)s will checkout the version of the branch in the remote,
     not the the version in the local repository (if it exists).
 
-    Note: either tag or branch must be supplied, but not both.
+    Note: one and only one of tag, branch hash must be supplied.
 
   * externals (string) : used to make manage_externals aware of
     sub-externals required by an external. This is a relative path to

manic/externals_description.py

@@ -176,6 +176,7 @@ class ExternalsDescription(dict):
     PATH = 'local_path'
     PROTOCOL = 'protocol'
     REPO_URL = 'repo_url'
+    HASH = 'hash'
     NAME = 'name'
 
     PROTOCOL_EXTERNALS_ONLY = 'externals_only'
@@ -197,6 +198,7 @@ class ExternalsDescription(dict):
                              REPO_URL: 'string',
                              TAG: 'string',
                              BRANCH: 'string',
+                             HASH: 'string',
                              }
                       }
 
@@ -269,18 +271,33 @@ def _check_data(self):
                     self[ext_name][self.REPO][self.PROTOCOL], ext_name)
                 fatal_error(msg)
 
+            if (self[ext_name][self.REPO][self.PROTOCOL] ==
+                    self.PROTOCOL_SVN):
+                if self.HASH in self[ext_name][self.REPO]:
+                    msg = ('In repo description for "{0}". svn repositories '
+                           'may not include the "hash" keyword.'.format(
+                               ext_name))
+                    fatal_error(msg)
+
             if (self[ext_name][self.REPO][self.PROTOCOL] !=
                     self.PROTOCOL_EXTERNALS_ONLY):
                 ref_count = 0
                 found_refs = ''
                 if self.TAG in self[ext_name][self.REPO]:
                     ref_count += 1
-                    found_refs = '"{0}", {1}'.format(
-                        self[ext_name][self.REPO][self.TAG], found_refs)
+                    found_refs = '"{0} = {1}", {2}'.format(
+                        self.TAG, self[ext_name][self.REPO][self.TAG],
+                        found_refs)
                 if self.BRANCH in self[ext_name][self.REPO]:
                     ref_count += 1
-                    found_refs = '"{0}", {1}'.format(
-                        self[ext_name][self.REPO][self.BRANCH], found_refs)
+                    found_refs = '"{0} = {1}", {2}'.format(
+                        self.BRANCH, self[ext_name][self.REPO][self.BRANCH],
+                        found_refs)
+                if self.HASH in self[ext_name][self.REPO]:
+                    ref_count += 1
+                    found_refs = '"{0} = {1}", {2}'.format(
+                        self.HASH, self[ext_name][self.REPO][self.HASH],
+                        found_refs)
 
                 if ref_count > 1:
                     msg = ('Model description is over specified! Only one of '
@@ -322,6 +339,8 @@ def _check_optional(self):
                 self[field][self.REPO][self.TAG] = EMPTY_STR
             if self.BRANCH not in self[field][self.REPO]:
                 self[field][self.REPO][self.BRANCH] = EMPTY_STR
+            if self.HASH not in self[field][self.REPO]:
+                self[field][self.REPO][self.HASH] = EMPTY_STR
             if self.REPO_URL not in self[field][self.REPO]:
                 self[field][self.REPO][self.REPO_URL] = EMPTY_STR
 
@@ -428,7 +447,7 @@ def __init__(self, model_data):
         """
         ExternalsDescription.__init__(self)
         self._schema_major = 1
-        self._schema_minor = 0
+        self._schema_minor = 1
         self._schema_patch = 0
         self._input_major, self._input_minor, self._input_patch = \
             get_cfg_schema_version(model_data)

manic/repository.py

@@ -19,16 +19,26 @@ def __init__(self, component_name, repo):
         self._protocol = repo[ExternalsDescription.PROTOCOL]
         self._tag = repo[ExternalsDescription.TAG]
         self._branch = repo[ExternalsDescription.BRANCH]
+        self._hash = repo[ExternalsDescription.HASH]
         self._url = repo[ExternalsDescription.REPO_URL]
 
         if self._url is EMPTY_STR:
             fatal_error('repo must have a URL')
 
-        if self._tag is EMPTY_STR and self._branch is EMPTY_STR:
-            fatal_error('repo must have either a branch or a tag element')
+        if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and
+                (self._hash is EMPTY_STR)):
+            fatal_error('{0} repo must have a branch, tag or hash element')
 
-        if self._tag is not EMPTY_STR and self._branch is not EMPTY_STR:
-            fatal_error('repo cannot have both a tag and a branch element')
+        ref_count = 0
+        if self._tag is not EMPTY_STR:
+            ref_count += 1
+        if self._branch is not EMPTY_STR:
+            ref_count += 1
+        if self._hash is not EMPTY_STR:
+            ref_count += 1
+        if ref_count != 1:
+            fatal_error('repo {0} must have exactly one of '
+                        'tag, branch or hash.'.format(self._name))
 
     def checkout(self, base_dir_path, repo_dir_name, verbosity):  # pylint: disable=unused-argument
         """
@@ -63,3 +73,8 @@ def branch(self):
         """Public access of repo branch.
         """
         return self._branch
+
+    def hash(self):
+        """Public access of repo hash.
+        """
+        return self._hash

manic/repository_git.py

@@ -229,6 +229,14 @@ def compare_refs(current_ref, expected_ref):
                     expected_ref = "unknown_remote/{0}".format(self._branch)
                 else:
                     expected_ref = "{0}/{1}".format(remote_name, self._branch)
+        elif self._hash:
+            # NOTE(bja, 2018-03) For comparison purposes, we could
+            # determine which is longer and check that the short ref
+            # is a substring of the long ref. But it is simpler to
+            # just expand both to the full sha and do an exact
+            # comparison.
+            _, expected_ref = self._git_revparse_commit(self._hash)
+            _, current_ref = self._git_revparse_commit(current_ref)
         else:
             expected_ref = self._tag
 
@@ -331,48 +339,64 @@ def _checkout_local_ref(self, verbosity):
         """
         if self._tag:
             ref = self._tag
-        else:
+        elif self._branch:
             ref = self._branch
+        else:
+            ref = self._hash
+
         self._check_for_valid_ref(ref)
         self._git_checkout_ref(ref, verbosity)
 
     def _checkout_external_ref(self, verbosity):
         """Checkout the reference from a remote repository
         """
+        if self._tag:
+            ref = self._tag
+        elif self._branch:
+            ref = self._branch
+        else:
+            ref = self._hash
+
         remote_name = self._determine_remote_name()
         if not remote_name:
             remote_name = self._create_remote_name()
             self._git_remote_add(remote_name, self._url)
         self._git_fetch(remote_name)
-        if self._tag:
-            is_unique_tag, check_msg = self._is_unique_tag(self._tag,
-                                                           remote_name)
-            if not is_unique_tag:
-                msg = ('In repo "{0}": tag "{1}" {2}'.format(
-                    self._name, self._tag, check_msg))
-                fatal_error(msg)
-            ref = self._tag
-        else:
-            ref = '{0}/{1}'.format(remote_name, self._branch)
+
+        # NOTE(bja, 2018-03) we need to send seperate ref and remote
+        # name to check_for_vaild_ref, but the combined name to
+        # checkout_ref!
+        self._check_for_valid_ref(ref, remote_name)
+
+        if self._branch:
+            ref = '{0}/{1}'.format(remote_name, ref)
         self._git_checkout_ref(ref, verbosity)
 
-    def _check_for_valid_ref(self, ref):
+    def _check_for_valid_ref(self, ref, remote_name=None):
         """Try some basic sanity checks on the user supplied reference so we
         can provide a more useful error message than calledprocess
         error...
 
         """
         is_tag = self._ref_is_tag(ref)
-        is_branch = self._ref_is_branch(ref)
-        is_commit = self._ref_is_commit(ref)
+        is_branch = self._ref_is_branch(ref, remote_name)
+        is_hash = self._ref_is_hash(ref)
 
-        is_valid = is_tag or is_branch or is_commit
+        is_valid = is_tag or is_branch or is_hash
         if not is_valid:
             msg = ('In repo "{0}": reference "{1}" does not appear to be a '
-                   'valid tag, branch or commit! Please verify the reference '
+                   'valid tag, branch or hash! Please verify the reference '
                    'name (e.g. spelling), is available from: {2} '.format(
                        self._name, ref, self._url))
             fatal_error(msg)
+
+        if is_tag:
+            is_unique_tag, msg = self._is_unique_tag(ref, remote_name)
+            if not is_unique_tag:
+                msg = ('In repo "{0}": tag "{1}" {2}'.format(
+                    self._name, self._tag, msg))
+                fatal_error(msg)
+
         return is_valid
 
     def _is_unique_tag(self, ref, remote_name):
@@ -388,7 +412,7 @@ def _is_unique_tag(self, ref, remote_name):
         """
         is_tag = self._ref_is_tag(ref)
         is_branch = self._ref_is_branch(ref, remote_name)
-        is_commit = self._ref_is_commit(ref)
+        is_hash = self._ref_is_hash(ref)
 
         msg = ''
         is_unique_tag = False
@@ -407,13 +431,13 @@ def _is_unique_tag(self, ref, remote_name):
                    'exist. Please check the name.')
             is_unique_tag = False
         else:  # not is_tag and not is_branch:
-            if is_commit:
+            if is_hash:
                 # probably a sha1 or HEAD, etc, we call it a tag
                 msg = 'is ok'
                 is_unique_tag = True
             else:
                 # undetermined state.
-                msg = ('does not appear to be a valid tag, branch or commit! '
+                msg = ('does not appear to be a valid tag, branch or hash! '
                        'Please check the name and repository.')
                 is_unique_tag = False
 
@@ -494,11 +518,32 @@ def _ref_is_commit(self, ref):
         error!
         """
         is_commit = False
-        value = self._git_revparse_commit(ref)
+        value, _ = self._git_revparse_commit(ref)
         if value == 0:
             is_commit = True
         return is_commit
 
+    def _ref_is_hash(self, ref):
+        """Verify that a reference is a valid hash according to git.
+
+        Git doesn't seem to provide an exact way to determine if user
+        supplied reference is an actual hash. So we verify that the
+        ref is a valid commit and return the underlying commit
+        hash. Then check that the commit hash begins with the user
+        supplied string.
+
+        Note: values returned by git_showref_* and git_revparse are
+        shell return codes, which are zero for success, non-zero for
+        error!
+
+        """
+        is_hash = False
+        status, git_output = self._git_revparse_commit(ref)
+        if status == 0:
+            if git_output.strip().startswith(ref):
+                is_hash = True
+        return is_hash
+
     def _status_summary(self, stat, repo_dir_path):
         """Determine the clean/dirty status of a git repository
 
@@ -600,8 +645,9 @@ def _git_revparse_commit(ref):
         """
         cmd = ['git', 'rev-parse', '--quiet', '--verify',
                '{0}^{1}'.format(ref, '{commit}'), ]
-        status = execute_subprocess(cmd, status_to_caller=True)
-        return status
+        status, git_output = execute_subprocess(cmd, status_to_caller=True,
+                                                output_to_caller=True)
+        return status, git_output
 
     @staticmethod
     def _git_status_porcelain_v1z():

manic/utils.py

@@ -288,16 +288,6 @@ def execute_subprocess(commands, status_to_caller=False,
     return code, otherwise execute_subprocess treats non-zero return
     status as an error and raises an exception.
 
-    NOTE(bja, 2018-03) if the user doesn't have credentials setup
-    correctly, then svn and git will prompt for a username/password or
-    accepting the domain as trusted. We need to detect this and print
-    enough info for the user to determine what happened and enter the
-    appropriate information. When we detect some pre-determined
-    conditions, we turn on screen output so the user can see what is
-    needed. There doesn't appear to be a way to detect if the user
-    entered any information in the terminal. So there is no way to
-    turn off output.
-
     NOTE(bja, 2018-03) we are polling the running process to avoid
     having it hang indefinitely if there is input that we don't
     detect. Some large checkouts are multiple minutes long. For now we
@@ -308,7 +298,6 @@ def execute_subprocess(commands, status_to_caller=False,
         os.getcwd())
     logging.info(msg)
     logging.info(commands)
-    return_to_caller = status_to_caller or output_to_caller
     try:
         ret_value = _poll_subprocess(
             commands, status_to_caller, output_to_caller)
@@ -330,15 +319,20 @@ def execute_subprocess(commands, status_to_caller=False,
         # simple status check. If returning, it is the callers
         # responsibility determine if an error occurred and handle it
         # appropriately.
-        if not return_to_caller:
+        if status_to_caller and output_to_caller:
+            ret_value = (error.returncode, error.output)
+        elif status_to_caller:
+            ret_value = error.returncode
+        elif output_to_caller:
+            ret_value = error.output
+        else:
             msg_context = ('Process did not run successfully; '
                            'returned status {0}'.format(error.returncode))
             msg = failed_command_msg(msg_context, commands,
                                      output=error.output)
             logging.error(error)
             log_process_output(error.output)
             fatal_error(msg)
-        ret_value = error.returncode
 
     return ret_value
 

test/repos/simple-ext.git/objects/09/0e1034746b2c865f7b0280813dbf4061a700e8

@@ -0,0 +1,2 @@
+x
��Q� ��{
XXdc�7Y`�ۚ�o=�/3��u���o�P�w��6�YB�9�MĜc��&iښy˦K�K9(�)
+�Raq�$)�+�|�ȧ��n����Mᜟ��i��k(�|
G�
����Fk�N�{]��X+��,��xoC#

test/repos/simple-ext.git/objects/31/dbcd6de441e671a467ef317146539b7ffabb11

@@ -0,0 +1 @@
+x
5��	B1=��W���b�@bf!7dW���E��0L�V�m�ý�c�᲏N=09�%l~�hP?����rPkևЏ)]��5yB����.��mg4��n�s��$*�