Added support for inheritance in importing repository files.

Signed-off-by: Leander Stephen D'Souza <[email protected]>

Author: leander-dsouza

README.rst

@@ -105,6 +105,30 @@ Only for this command vcs2l supports the pseudo clients ``tar`` and ``zip`` whic
 For those two types the ``version`` key is optional.
 If specified only entries from the archive which are in the subfolder specified by the version value are being extracted.
 
+Import with extends functionality
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``vcs import`` command supports an ``extends`` key at the top level of the YAML file. The value of that key is a path or URL to another YAML file which is imported first.
+This parent file can itself also contain the key to chain multiple files. The child is given precedence over the parent in case of duplicate repository entries.
+In order to avoid infinite loops in case of circular imports the tool detects already imported files and raises an error if such a file is encountered again.
+
+.. code-block:: yaml
+
+  # parent_repos.yaml
+  repositories:
+    vcs2l:
+      type: git
+      url: https://github.com/ros-infrastructure/vcs2l.git
+      version: main
+
+  # child_repos.yaml
+  extends: parent_repos.yaml
+  repositories:
+    vcs2l:
+      type: git
+      url: https://github.com/ros-infrastructure/vcs2l.git
+      version: 1.1.3
+
 
 Validate repositories file
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

test/import_extends.txt

@@ -0,0 +1,67 @@
+......
+=== ./immutable/hash (git) ===
+Cloning into '.'...
+Note: switching to 'e700793cb2b8d25ce83a611561bd167293fd66eb'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at e700793 1.1.3
+=== ./immutable/hash_tar (tar) ===
+Downloaded tarball from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it
+=== ./immutable/hash_zip (zip) ===
+Downloaded zipfile from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it
+=== ./immutable/tag (git) ===
+Cloning into '.'...
+Note: switching to 'tags/1.1.3'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at e700793 1.1.3
+=== ./vcs2l (git) ===
+Cloning into '.'...
+Note: switching to '1.1.3'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at e700793 1.1.3
+=== ./without_version (git) ===
+Cloning into '.'...

test/list_extends.repos

@@ -0,0 +1,16 @@
+# Check for hash, tag, and branch imports with extends functionality
+---
+extends: list.repos
+repositories:
+  immutable/hash:
+    type: git
+    url: https://github.com/ros-infrastructure/vcs2l.git
+    version: e700793cb2b8d25ce83a611561bd167293fd66eb  # 1.1.3
+  immutable/tag:
+    type: git
+    url: https://github.com/ros-infrastructure/vcs2l.git
+    version: tags/1.1.3
+  vcs2l:
+    type: git
+    url: https://github.com/ros-infrastructure/vcs2l.git
+    version: 1.1.3

test/list_extends_loop_child.repos

@@ -0,0 +1,8 @@
+# Child file that creates an extends a circular import on its parent.
+---
+extends: list_extends_loop_parent.repos
+repositories:
+  vcs2l:
+    type: git
+    url: https://github.com/ros-infrastructure/vcs2l.git
+    version: 1.1.3

test/list_extends_loop_parent.repos

@@ -0,0 +1,8 @@
+# Parent file that creates an extends a circular import on its child.
+---
+extends: list_extends_loop_child.repos
+repositories:
+  vcs2l:
+    type: git
+    url: https://github.com/ros-infrastructure/vcs2l.git
+    version: main

test/test_commands.py

@@ -14,6 +14,10 @@
 REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list.repos')
 REPOS_FILE_URL = file_uri_scheme + REPOS_FILE
 REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos')
+REPOS_EXTENDS_FILE = os.path.join(os.path.dirname(__file__), 'list_extends.repos')
+REPOS_EXTENDS_LOOP_FILE = os.path.join(
+    os.path.dirname(__file__), 'list_extends_loop_child.repos'
+)
 BAD_REPOS_FILE = os.path.join(os.path.dirname(__file__), 'bad.repos')
 TEST_WORKSPACE = os.path.join(
     os.path.dirname(os.path.dirname(__file__)), 'test_workspace'
@@ -312,23 +316,43 @@ def test_import_shallow(self):
         finally:
             rmtree(workdir)
 
-    def test_import_url(self):
-        workdir = os.path.join(TEST_WORKSPACE, 'import-url')
+    def import_common(self, import_file, repos_file):
+        """Common test function for import operations
+
+        Args:
+            import_file: Assertion expected output file name (without .txt)
+            repos_file: path to the .repos file to use
+        """
+        workdir = os.path.join(TEST_WORKSPACE, import_file)
         os.makedirs(workdir)
         try:
             output = run_command(
-                'import', ['--input', REPOS_FILE_URL, '.'], subfolder='import-url'
+                'import', ['--input', repos_file, '.'], subfolder=import_file
             )
             # the actual output contains absolute paths
             output = output.replace(
                 b'repository in ' + workdir.encode() + b'/', b'repository in ./'
             )
-            expected = get_expected_output('import')
+            expected = get_expected_output(import_file)
             # newer git versions don't append ... after the commit hash
             assert output == expected or output == expected.replace(b'... ', b' ')
         finally:
             rmtree(workdir)
 
+    def test_import_file_url(self):
+        """Test import from file URL."""
+        self.import_common('import', REPOS_FILE_URL)
+
+    def test_import_extends(self):
+        """Test import with extends functionality."""
+        self.import_common('import_extends', REPOS_EXTENDS_FILE)
+
+    def test_import_extends_loop(self):
+        """Test import with extends functionality that creates a circular import."""
+        with self.assertRaises(subprocess.CalledProcessError) as e:
+            run_command('import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.'])
+        self.assertIn(b'Circular import detected:', e.exception.output)
+
     def test_validate(self):
         output = run_command('validate', ['--input', REPOS_FILE])
         expected = get_expected_output('validate')

vcs2l/commands/import_.py

@@ -86,23 +86,84 @@ def file_or_url_type(value):
     return request.Request(value, headers={'User-Agent': 'vcs2l/' + vcs2l_version})
 
 
-def get_repositories(yaml_file):
+def load_yaml_file(yaml_file):
+    """Load and parse a YAML file."""
     try:
-        root = yaml.safe_load(yaml_file)
+        return yaml.safe_load(yaml_file)
     except yaml.YAMLError as e:
-        raise RuntimeError('Input data is not valid yaml format: %s' % e)
+        raise RuntimeError('Input data is not valid yaml format: %s' % e) from e
 
+
+def get_repositories_from_root(root):
+    """Extract repositories from the parsed YAML root object."""
     try:
         repositories = root['repositories']
         return get_repos_in_vcs2l_format(repositories)
     except KeyError as e:
-        raise RuntimeError('Input data is not valid format: %s' % e)
+        raise RuntimeError('Input data is not valid format: %s' % e) from e
     except TypeError as e:
         # try rosinstall file format
         try:
             return get_repos_in_rosinstall_format(root)
         except Exception:
-            raise RuntimeError('Input data is not valid format: %s' % e)
+            raise RuntimeError('Input data is not valid format: %s' % e) from e
+
+
+def get_repositories(yaml_file, visited_files=None):
+    """Recursively get repositories from a YAML file, handling inheritance."""
+    if visited_files is None:
+        visited_files = set()
+
+    # Get absolute path to handle relative paths consistently
+    current_file_path = os.path.abspath(yaml_file.name)
+
+    if current_file_path in visited_files:
+        raise RuntimeError('Circular import detected: %s' % current_file_path)
+
+    visited_files.add(current_file_path)
+
+    try:
+        root = load_yaml_file(yaml_file)
+
+        combined_repos = {}
+
+        if 'extends' in root:
+            parent_file = root['extends']
+
+            # If absolute path is not valid, try relative to current file
+            if not os.path.isabs(parent_file):
+                current_dir = os.path.dirname(current_file_path)
+                parent_file = os.path.join(current_dir, parent_file)
+
+            if not os.path.exists(parent_file):
+                raise RuntimeError('Parent file not found: %s' % parent_file)
+
+            try:
+                # Recursively get repositories from parent file
+                with open(parent_file, 'r', encoding='utf-8') as parent_f:
+                    parent_repos = get_repositories(parent_f, visited_files.copy())
+                    combined_repos.update(parent_repos)
+            except Exception as e:
+                if str(e).startswith('Circular import detected:'):
+                    raise
+                raise RuntimeError(
+                    'Error reading parent file %s: \n%s' % (parent_file, str(e))
+                ) from e
+
+        current_repos = get_repositories_from_root(root)
+        combined_repos.update(current_repos)
+
+        return combined_repos
+
+    except FileNotFoundError as e:
+        raise RuntimeError('File not found: %s' % yaml_file) from e
+    except yaml.YAMLError as e:
+        raise RuntimeError(
+            'Error parsing YAML file %s: %s' % (yaml_file, str(e))
+        ) from e
+    finally:
+        # Remove current file from visited set when leaving this call
+        visited_files.discard(current_file_path)
 
 
 def get_repos_in_vcs2l_format(repositories):