From b186be51fe14d0a42755824861d470f0bd6642e1 Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Mon, 25 Aug 2025 00:14:15 +0100 Subject: [PATCH 1/6] Added support for inheritance in importing repository files. Signed-off-by: Leander Stephen D'Souza --- README.md | 25 ++++++++++ test/import_extends.txt | 67 +++++++++++++++++++++++++++ test/list_extends.repos | 16 +++++++ test/list_extends_loop_child.repos | 8 ++++ test/list_extends_loop_parent.repos | 8 ++++ test/test_commands.py | 32 +++++++++++-- vcs2l/commands/import_.py | 71 +++++++++++++++++++++++++++-- 7 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 test/import_extends.txt create mode 100644 test/list_extends.repos create mode 100644 test/list_extends_loop_child.repos create mode 100644 test/list_extends_loop_parent.repos diff --git a/README.md b/README.md index 8f2546e..c297ad2 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,31 @@ The `import` command also supports input in the [rosinstall file format](http:// Only for this command vcs2l supports the pseudo clients `tar` and `zip` which fetch a tarball / zipfile from a URL and unpack its content. For those two types the `version` key is optional. If specified only entries from the archive which are in the subfolder specified by the version value are being extracted. +### Import with extends functionality + +The `vcs import` command supports an `extends` key at the top level of the YAML file. +The value of that key is a path or URL to another YAML file which is imported first. +This parent file can itself also contain the key to chain multiple files. +The child is given precedence over the parent in case of duplicate repository entries. +In order to avoid infinite loops in case of circular imports the tool detects already imported files and raises an error if such a file is encountered again. + +```yaml +# parent.repos +repositories: + vcs2l: + type: git + url: git@github.com:ros-infrastructure/vcs2l.git + version: main + +# child.repos +extends: parent.repos +repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.5 +``` + ### Delete set of repositories The `vcs delete` command removes all directories of repositories which are passed in via `stdin` in YAML format. diff --git a/test/import_extends.txt b/test/import_extends.txt new file mode 100644 index 0000000..cd9f1f7 --- /dev/null +++ b/test/import_extends.txt @@ -0,0 +1,67 @@ +...... +=== ./immutable/hash (git) === +Cloning into '.'... +Note: switching to 'e700793cb2b8d25ce83a611561bd167293fd66eb'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at e700793 1.1.3 +=== ./immutable/hash_tar (tar) === +Downloaded tarball from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it +=== ./immutable/hash_zip (zip) === +Downloaded zipfile from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it +=== ./immutable/tag (git) === +Cloning into '.'... +Note: switching to 'tags/1.1.3'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at e700793 1.1.3 +=== ./vcs2l (git) === +Cloning into '.'... +Note: switching to '1.1.3'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at e700793 1.1.3 +=== ./without_version (git) === +Cloning into '.'... diff --git a/test/list_extends.repos b/test/list_extends.repos new file mode 100644 index 0000000..4d11803 --- /dev/null +++ b/test/list_extends.repos @@ -0,0 +1,16 @@ +# Check for hash, tag, and branch imports with extends functionality +--- +extends: list.repos +repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: e700793cb2b8d25ce83a611561bd167293fd66eb # 1.1.3 + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: tags/1.1.3 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.3 diff --git a/test/list_extends_loop_child.repos b/test/list_extends_loop_child.repos new file mode 100644 index 0000000..4df726f --- /dev/null +++ b/test/list_extends_loop_child.repos @@ -0,0 +1,8 @@ +# Child file that creates an extends a circular import on its parent. +--- +extends: list_extends_loop_parent.repos +repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.3 diff --git a/test/list_extends_loop_parent.repos b/test/list_extends_loop_parent.repos new file mode 100644 index 0000000..618ae7f --- /dev/null +++ b/test/list_extends_loop_parent.repos @@ -0,0 +1,8 @@ +# Parent file that creates an extends a circular import on its child. +--- +extends: list_extends_loop_child.repos +repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: main diff --git a/test/test_commands.py b/test/test_commands.py index 39c75a6..4b94d8d 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -14,6 +14,10 @@ REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list.repos') REPOS_FILE_URL = file_uri_scheme + REPOS_FILE REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos') +REPOS_EXTENDS_FILE = os.path.join(os.path.dirname(__file__), 'list_extends.repos') +REPOS_EXTENDS_LOOP_FILE = os.path.join( + os.path.dirname(__file__), 'list_extends_loop_child.repos' +) BAD_REPOS_FILE = os.path.join(os.path.dirname(__file__), 'bad.repos') TEST_WORKSPACE = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'test_workspace' @@ -277,18 +281,24 @@ def test_import_shallow(self): finally: rmtree(workdir) - def test_import_url(self): - workdir = os.path.join(TEST_WORKSPACE, 'import-url') + def import_common(self, import_file, repos_file): + """Common test function for import operations + + Args: + import_file: Assertion expected output file name (without .txt) + repos_file: path to the .repos file to use + """ + workdir = os.path.join(TEST_WORKSPACE, import_file) os.makedirs(workdir) try: output = run_command( - 'import', ['--input', REPOS_FILE_URL, '.'], subfolder='import-url' + 'import', ['--input', repos_file, '.'], subfolder=import_file ) # the actual output contains absolute paths output = output.replace( b'repository in ' + workdir.encode() + b'/', b'repository in ./' ) - expected = get_expected_output('import') + expected = get_expected_output(import_file) # newer git versions don't append ... after the commit hash assert output == expected or output == expected.replace(b'... ', b' ') finally: @@ -327,6 +337,20 @@ def test_deletion(self): finally: rmtree(workdir) + def test_import_file_url(self): + """Test import from file URL.""" + self.import_common('import', REPOS_FILE_URL) + + def test_import_extends(self): + """Test import with extends functionality.""" + self.import_common('import_extends', REPOS_EXTENDS_FILE) + + def test_import_extends_loop(self): + """Test import with extends functionality that creates a circular import.""" + with self.assertRaises(subprocess.CalledProcessError) as e: + run_command('import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.']) + self.assertIn(b'Circular import detected:', e.exception.output) + def test_validate(self): output = run_command('validate', ['--input', REPOS_FILE]) expected = get_expected_output('validate') diff --git a/vcs2l/commands/import_.py b/vcs2l/commands/import_.py index b1f0c8b..fe3fc17 100644 --- a/vcs2l/commands/import_.py +++ b/vcs2l/commands/import_.py @@ -86,23 +86,84 @@ def file_or_url_type(value): return request.Request(value, headers={'User-Agent': 'vcs2l/' + vcs2l_version}) -def get_repositories(yaml_file): +def load_yaml_file(yaml_file): + """Load and parse a YAML file.""" try: - root = yaml.safe_load(yaml_file) + return yaml.safe_load(yaml_file) except yaml.YAMLError as e: - raise RuntimeError('Input data is not valid yaml format: %s' % e) + raise RuntimeError('Input data is not valid yaml format: %s' % e) from e + +def get_repositories_from_root(root): + """Extract repositories from the parsed YAML root object.""" try: repositories = root['repositories'] return get_repos_in_vcs2l_format(repositories) except KeyError as e: - raise RuntimeError('Input data is not valid format: %s' % e) + raise RuntimeError('Input data is not valid format: %s' % e) from e except TypeError as e: # try rosinstall file format try: return get_repos_in_rosinstall_format(root) except Exception: - raise RuntimeError('Input data is not valid format: %s' % e) + raise RuntimeError('Input data is not valid format: %s' % e) from e + + +def get_repositories(yaml_file, visited_files=None): + """Recursively get repositories from a YAML file, handling inheritance.""" + if visited_files is None: + visited_files = set() + + # Get absolute path to handle relative paths consistently + current_file_path = os.path.abspath(yaml_file.name) + + if current_file_path in visited_files: + raise RuntimeError(f'Circular import detected: {current_file_path}') + + visited_files.add(current_file_path) + + try: + root = load_yaml_file(yaml_file) + + combined_repos = {} + + if 'extends' in root: + parent_file = root['extends'] + + # If absolute path is not valid, try relative to current file + if not os.path.isabs(parent_file): + current_dir = os.path.dirname(current_file_path) + parent_file = os.path.join(current_dir, parent_file) + + if not os.path.exists(parent_file): + raise RuntimeError(f'Parent file not found: {parent_file}') + + try: + # Recursively get repositories from parent file + with open(parent_file, 'r', encoding='utf-8') as parent_f: + parent_repos = get_repositories(parent_f, visited_files.copy()) + combined_repos.update(parent_repos) + except Exception as e: + if str(e).startswith('Circular import detected:'): + raise + raise RuntimeError( + f'Error reading parent file {parent_file}: \n{str(e)}' + ) from e + + current_repos = get_repositories_from_root(root) + combined_repos.update(current_repos) + + return combined_repos + + except FileNotFoundError as e: + raise RuntimeError(f'File not found: {yaml_file}') from e + except yaml.YAMLError as e: + raise RuntimeError( + f'Error parsing YAML file {yaml_file}: {str(e)}' + ) from e + finally: + # Remove current file from visited set when leaving this call + visited_files.discard(current_file_path) def get_repos_in_vcs2l_format(repositories): From 62bf9131f343a66c1ba5fff99e223621a9c77e7a Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Mon, 15 Sep 2025 13:52:15 +0100 Subject: [PATCH 2/6] Update documentation and file names to reflect PR feedback. Signed-off-by: Leander Stephen D'Souza --- README.md | 115 +++++++++++++++--- test/list_extends_loop_parent.repos | 8 -- ...ist_extends.repos => list_extension.repos} | 0 test/loop_base.repos | 12 ++ ..._loop_child.repos => loop_extension.repos} | 4 +- test/test_commands.py | 4 +- 6 files changed, 113 insertions(+), 30 deletions(-) delete mode 100644 test/list_extends_loop_parent.repos rename test/{list_extends.repos => list_extension.repos} (100%) create mode 100644 test/loop_base.repos rename test/{list_extends_loop_child.repos => loop_extension.repos} (51%) diff --git a/README.md b/README.md index c297ad2..870ea1d 100644 --- a/README.md +++ b/README.md @@ -92,29 +92,108 @@ Only for this command vcs2l supports the pseudo clients `tar` and `zip` which fe ### Import with extends functionality -The `vcs import` command supports an `extends` key at the top level of the YAML file. -The value of that key is a path or URL to another YAML file which is imported first. -This parent file can itself also contain the key to chain multiple files. -The child is given precedence over the parent in case of duplicate repository entries. +The `vcs import` command supports an `extends` key at the top level of the YAML file. The value of that key is a path or URL to another YAML file which is imported first. +This base file can itself also contain the key to chain multiple files. The extension to this base file is given precedence over the parent in case of duplicate repository entries. + +#### Normal Extension + +For instance, consider the following two files: + +- **`base.repos`**: contains three repositories `vcs2l`, `immutable/hash` and `immutable/tag`, checked out at specific versions. + + ```yaml + --- + repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: main + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 377d5b3d03c212f015cc832fdb368f4534d0d583 + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.3 + ``` + +- **`base_extension.repos`**: extends the base file and overrides the version of `immutable/hash` and `immutable/tag` repositories. + + ```yaml + --- + extends: base.repos + repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 25e4ae2f1dd28b0efcd656f4b1c9679d8a7d6c22 + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.5 + ``` +The resulting extension import would import vcs2l at version `main`, `immutable/hash` at version `25e4ae2` and `immutable/tag` at version `1.1.5`. + +#### Circular Loop Protection + In order to avoid infinite loops in case of circular imports the tool detects already imported files and raises an error if such a file is encountered again. -```yaml -# parent.repos -repositories: - vcs2l: - type: git - url: git@github.com:ros-infrastructure/vcs2l.git - version: main +For instance, consider the following two files: + +- **`loop_base.repos`**: extends the `loop_extension.repos` file, and contains two repositories `vcs2l` and `immutable/tag`. + + ```yaml + --- + extends: loop_extension.repos + repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: main + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.3 + ``` + +- **`loop_extension.repos`**: extends the `loop_base.repos` file, and modifies the version of `immutable/tag` with `1.1.5`. + + ```yaml + --- + extends: loop_base.repos + repositories: + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.5 + ``` +The resulting extension import would prevent the download and raise the following error: -# child.repos -extends: parent.repos -repositories: - vcs2l: - type: git - url: https://github.com/ros-infrastructure/vcs2l.git - version: 1.1.5 +```bash +Circular import detected: /loop_extension.repos ``` +#### File path behaviour + +Currently there are two ways to specify the path to the repository file passed to `vcs import`: + +1. **Recommended**: Using `--input`. + + * For instance: `vcs import --input my.repos ` + + * The extended files are searched relative to the location of `my.repos`. + + * You do not require to be in the same directory as `my.repos` to run the command. + +2. Using the input redirection operator `<` to pass a local file path via `stdin`. + + * For instance: `vcs import < my.repos ` + + * The extended files are searched relative to the current working directory. + + * Therefore, you have to be in the **same** directory as `my.repos` to run the command. In addition, all the extended files must also be relative to the current working directory. + ### Delete set of repositories The `vcs delete` command removes all directories of repositories which are passed in via `stdin` in YAML format. diff --git a/test/list_extends_loop_parent.repos b/test/list_extends_loop_parent.repos deleted file mode 100644 index 618ae7f..0000000 --- a/test/list_extends_loop_parent.repos +++ /dev/null @@ -1,8 +0,0 @@ -# Parent file that creates an extends a circular import on its child. ---- -extends: list_extends_loop_child.repos -repositories: - vcs2l: - type: git - url: https://github.com/ros-infrastructure/vcs2l.git - version: main diff --git a/test/list_extends.repos b/test/list_extension.repos similarity index 100% rename from test/list_extends.repos rename to test/list_extension.repos diff --git a/test/loop_base.repos b/test/loop_base.repos new file mode 100644 index 0000000..d2c50ec --- /dev/null +++ b/test/loop_base.repos @@ -0,0 +1,12 @@ +# Base repositories file that creates an extends a circular import on its extension implementation. +--- +extends: loop_extension.repos +repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: main + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: tags/1.1.3 diff --git a/test/list_extends_loop_child.repos b/test/loop_extension.repos similarity index 51% rename from test/list_extends_loop_child.repos rename to test/loop_extension.repos index 4df726f..7debf28 100644 --- a/test/list_extends_loop_child.repos +++ b/test/loop_extension.repos @@ -1,6 +1,6 @@ -# Child file that creates an extends a circular import on its parent. +# Repositories extension file that creates an extends a circular import on its base. --- -extends: list_extends_loop_parent.repos +extends: loop_base.repos repositories: vcs2l: type: git diff --git a/test/test_commands.py b/test/test_commands.py index 4b94d8d..f34e6c7 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -14,9 +14,9 @@ REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list.repos') REPOS_FILE_URL = file_uri_scheme + REPOS_FILE REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos') -REPOS_EXTENDS_FILE = os.path.join(os.path.dirname(__file__), 'list_extends.repos') +REPOS_EXTENDS_FILE = os.path.join(os.path.dirname(__file__), 'list_extension.repos') REPOS_EXTENDS_LOOP_FILE = os.path.join( - os.path.dirname(__file__), 'list_extends_loop_child.repos' + os.path.dirname(__file__), 'loop_extension.repos' ) BAD_REPOS_FILE = os.path.join(os.path.dirname(__file__), 'bad.repos') TEST_WORKSPACE = os.path.join( From 8c7697cc52a4bdeb121ecdf10129696676a4086b Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Mon, 15 Sep 2025 14:49:52 +0100 Subject: [PATCH 3/6] Added multi-level inheritance support for import. Signed-off-by: Leander Stephen D'Souza --- README.md | 58 ++++++++++++++++++++++++++++++ test/import_multiple_extends.txt | 49 +++++++++++++++++++++++++ test/list_extension.repos | 2 +- test/list_extension_2.repos | 16 +++++++++ test/list_multiple_extension.repos | 14 ++++++++ test/test_commands.py | 7 ++++ vcs2l/commands/import_.py | 54 ++++++++++++++++------------ 7 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 test/import_multiple_extends.txt create mode 100644 test/list_extension_2.repos create mode 100644 test/list_multiple_extension.repos diff --git a/README.md b/README.md index 870ea1d..ba74e89 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,64 @@ For instance, consider the following two files: ``` The resulting extension import would import vcs2l at version `main`, `immutable/hash` at version `25e4ae2` and `immutable/tag` at version `1.1.5`. +#### Multiple Extensions + +The `extends` key also supports a list of files to extend from. The files are imported in the order they are specified and the precedence is given to the last file in case of duplicate repository entries. + +For instance, consider the following three files: + +- **`base_1.repos`**: contains two repositories `vcs2l` and `immutable/hash`, checked out at `1.1.3`. + + ```yaml + --- + repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: e700793cb2b8d25ce83a611561bd167293fd66eb # 1.1.3 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.3 + ``` + +- **`base_2.repos`**: contains two repositories `vcs2l` and `immutable/hash`, checked out at `1.1.4`. + + ```yaml + --- + repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 2c7ff89d12d8a77c36b60d1f7ba3039cdd3f742b # 1.1.4 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.4 + ``` + +- **`multiple_extension.repos`**: extends both base files and overrides the version of `vcs2l` repository. + + ```yaml + --- + extends: + - base_1.repos # Lower priority + - base_2.repos # Higher priority + repositories: + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.5 + ``` + +The resulting extension import would import `immutable/hash` at version `1.1.4` (from `base_2.repos`) and `vcs2l` at version `1.1.5`. + +Duplicate file names in the `extends` list are not allowed and would raise the following error: + +```bash +Duplicate entries found in extends in file: /multiple_extension.repos +``` + #### Circular Loop Protection In order to avoid infinite loops in case of circular imports the tool detects already imported files and raises an error if such a file is encountered again. diff --git a/test/import_multiple_extends.txt b/test/import_multiple_extends.txt new file mode 100644 index 0000000..8dbc6df --- /dev/null +++ b/test/import_multiple_extends.txt @@ -0,0 +1,49 @@ +...... +=== ./immutable/hash (git) === +Cloning into '.'... +Note: switching to '2c7ff89d12d8a77c36b60d1f7ba3039cdd3f742b'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 2c7ff89 1.1.4 +=== ./immutable/hash_tar (tar) === +Downloaded tarball from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it +=== ./immutable/hash_zip (zip) === +Downloaded zipfile from 'https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it +=== ./immutable/tag (git) === +Cloning into '.'... +Note: switching to 'tags/1.1.5'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 25e4ae2 1.1.5 +=== ./vcs2l (git) === +Cloning into '.'... +=== ./without_version (git) === +Cloning into '.'... diff --git a/test/list_extension.repos b/test/list_extension.repos index 4d11803..87d760a 100644 --- a/test/list_extension.repos +++ b/test/list_extension.repos @@ -1,4 +1,4 @@ -# Check for hash, tag, and branch imports with extends functionality +# Check for hash, tag, and branch imports with extends functionality with predominant tag 1.1.3 --- extends: list.repos repositories: diff --git a/test/list_extension_2.repos b/test/list_extension_2.repos new file mode 100644 index 0000000..053e1a2 --- /dev/null +++ b/test/list_extension_2.repos @@ -0,0 +1,16 @@ +# Check for hash, tag, and branch imports with extends functionality with predominant tag 1.1.4 +--- +extends: list.repos +repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 2c7ff89d12d8a77c36b60d1f7ba3039cdd3f742b # 1.1.4 + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: tags/1.1.4 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 1.1.4 diff --git a/test/list_multiple_extension.repos b/test/list_multiple_extension.repos new file mode 100644 index 0000000..3f6282a --- /dev/null +++ b/test/list_multiple_extension.repos @@ -0,0 +1,14 @@ +# Check for multiple levels of extension in repositories files. +--- +extends: + - list_extension.repos # predominant tag is 1.1.3 # Lower priority + - list_extension_2.repos # predominant tag is 1.1.4 # Higher priority +repositories: + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: tags/1.1.5 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: main diff --git a/test/test_commands.py b/test/test_commands.py index f34e6c7..32338d5 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -15,6 +15,9 @@ REPOS_FILE_URL = file_uri_scheme + REPOS_FILE REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos') REPOS_EXTENDS_FILE = os.path.join(os.path.dirname(__file__), 'list_extension.repos') +REPOS_MULTIPLE_EXTENDS_FILE = os.path.join( + os.path.dirname(__file__), 'list_multiple_extension.repos' +) REPOS_EXTENDS_LOOP_FILE = os.path.join( os.path.dirname(__file__), 'loop_extension.repos' ) @@ -351,6 +354,10 @@ def test_import_extends_loop(self): run_command('import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.']) self.assertIn(b'Circular import detected:', e.exception.output) + def test_import_multiple_extends(self): + """Test import with multiple extends functionality.""" + self.import_common('import_multiple_extends', REPOS_MULTIPLE_EXTENDS_FILE) + def test_validate(self): output = run_command('validate', ['--input', REPOS_FILE]) expected = get_expected_output('validate') diff --git a/vcs2l/commands/import_.py b/vcs2l/commands/import_.py index fe3fc17..cb84ff5 100644 --- a/vcs2l/commands/import_.py +++ b/vcs2l/commands/import_.py @@ -128,27 +128,37 @@ def get_repositories(yaml_file, visited_files=None): combined_repos = {} if 'extends' in root: - parent_file = root['extends'] - - # If absolute path is not valid, try relative to current file - if not os.path.isabs(parent_file): - current_dir = os.path.dirname(current_file_path) - parent_file = os.path.join(current_dir, parent_file) - - if not os.path.exists(parent_file): - raise RuntimeError(f'Parent file not found: {parent_file}') - - try: - # Recursively get repositories from parent file - with open(parent_file, 'r', encoding='utf-8') as parent_f: - parent_repos = get_repositories(parent_f, visited_files.copy()) - combined_repos.update(parent_repos) - except Exception as e: - if str(e).startswith('Circular import detected:'): - raise + parent_files = root['extends'] + # Convert single file to list for consistent processing + if isinstance(parent_files, str): + parent_files = [parent_files] + + # Check for duplicate entries in extends + if len(parent_files) != len(set(parent_files)): raise RuntimeError( - f'Error reading parent file {parent_file}: \n{str(e)}' - ) from e + f'Duplicate entries found in extends in file: {current_file_path}' + ) + + for parent_file in parent_files: + # If absolute path is not valid, try relative to current file + if not os.path.isabs(parent_file): + current_dir = os.path.dirname(current_file_path) + parent_file = os.path.join(current_dir, parent_file) + + if not os.path.exists(parent_file): + raise RuntimeError(f'Parent file not found: {parent_file}') + + try: + # Recursively get repositories from parent file + with open(parent_file, 'r', encoding='utf-8') as parent_f: + parent_repos = get_repositories(parent_f, visited_files.copy()) + combined_repos.update(parent_repos) + except Exception as e: + if str(e).startswith('Circular import detected:'): + raise + raise RuntimeError( + f'Error reading parent file {parent_file}: \n{str(e)}' + ) from e current_repos = get_repositories_from_root(root) combined_repos.update(current_repos) @@ -158,9 +168,7 @@ def get_repositories(yaml_file, visited_files=None): except FileNotFoundError as e: raise RuntimeError(f'File not found: {yaml_file}') from e except yaml.YAMLError as e: - raise RuntimeError( - f'Error parsing YAML file {yaml_file}: {str(e)}' - ) from e + raise RuntimeError(f'Error parsing YAML file {yaml_file}: {str(e)}') from e finally: # Remove current file from visited set when leaving this call visited_files.discard(current_file_path) From 138ae197784b3b429cefce8da34780abb70c94ea Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Mon, 22 Sep 2025 03:42:53 +0100 Subject: [PATCH 4/6] Define a custom exception for circular imports in vcs2l. Signed-off-by: Leander Stephen D'Souza --- vcs2l/commands/import_.py | 5 +++-- vcs2l/errors.py | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vcs2l/commands/import_.py b/vcs2l/commands/import_.py index cb84ff5..ceeb1cd 100644 --- a/vcs2l/commands/import_.py +++ b/vcs2l/commands/import_.py @@ -10,6 +10,7 @@ from vcs2l.clients import vcs2l_clients from vcs2l.clients.vcs_base import run_command from vcs2l.commands.command import Command, add_common_arguments +from vcs2l.errors import CircularImportError from vcs2l.executor import ansi, execute_jobs, output_repositories, output_results from vcs2l.streams import set_streams @@ -118,7 +119,7 @@ def get_repositories(yaml_file, visited_files=None): current_file_path = os.path.abspath(yaml_file.name) if current_file_path in visited_files: - raise RuntimeError(f'Circular import detected: {current_file_path}') + raise CircularImportError(f'Circular import detected: {current_file_path}') visited_files.add(current_file_path) @@ -154,7 +155,7 @@ def get_repositories(yaml_file, visited_files=None): parent_repos = get_repositories(parent_f, visited_files.copy()) combined_repos.update(parent_repos) except Exception as e: - if str(e).startswith('Circular import detected:'): + if isinstance(e, CircularImportError): raise raise RuntimeError( f'Error reading parent file {parent_file}: \n{str(e)}' diff --git a/vcs2l/errors.py b/vcs2l/errors.py index fc7ba30..fe88a1b 100644 --- a/vcs2l/errors.py +++ b/vcs2l/errors.py @@ -21,3 +21,10 @@ def __init__(self, min_version: str = '3.6'): f'vcs2l requires Python {min_version} or higher.' ) super().__init__(message) + + +class CircularImportError(Vcs2lError): + """Raised when a circular import is detected.""" + + def __init__(self, message: str = 'Circular import detected.'): + super().__init__(message) From 16e1be681caa699513926bbb84f6602a631ae048 Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Mon, 22 Sep 2025 03:48:20 +0100 Subject: [PATCH 5/6] Fix documentation inaccuracies. Signed-off-by: Leander Stephen D'Souza --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba74e89..7afe5df 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ For instance, consider the following three files: --- extends: - base_1.repos # Lower priority - - base_2.repos # Higher priority + - base_2.repos # Higher priority repositories: vcs2l: type: git @@ -240,7 +240,7 @@ Currently there are two ways to specify the path to the repository file passed t * For instance: `vcs import --input my.repos ` - * The extended files are searched relative to the location of `my.repos`. + * The extended files are searched relative to the file containing the `extends` key. * You do not require to be in the same directory as `my.repos` to run the command. From d89e704cd643328b72258c1040e68e4eb550751b Mon Sep 17 00:00:00 2001 From: Leander Stephen D'Souza Date: Wed, 24 Sep 2025 00:51:46 +0100 Subject: [PATCH 6/6] Apply documentation changes and improve error handling for inheritance in import command. Signed-off-by: Leander Stephen D'Souza --- README.md | 7 +++++-- vcs2l/commands/import_.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7afe5df..e59f0a9 100644 --- a/README.md +++ b/README.md @@ -248,9 +248,12 @@ Currently there are two ways to specify the path to the repository file passed t * For instance: `vcs import < my.repos ` - * The extended files are searched relative to the current working directory. + * The extended files are searched relative to the current working directory. - * Therefore, you have to be in the **same** directory as `my.repos` to run the command. In addition, all the extended files must also be relative to the current working directory. + * Therefore, you have to be in the **same** directory as `my.repos` to run the command. + + The files being directly extended by the file provided through `stdin` are relative to the current working directory. + Any other file being extended is relative to the file extending it. ### Delete set of repositories diff --git a/vcs2l/commands/import_.py b/vcs2l/commands/import_.py index ceeb1cd..5b4ae0b 100644 --- a/vcs2l/commands/import_.py +++ b/vcs2l/commands/import_.py @@ -154,9 +154,11 @@ def get_repositories(yaml_file, visited_files=None): with open(parent_file, 'r', encoding='utf-8') as parent_f: parent_repos = get_repositories(parent_f, visited_files.copy()) combined_repos.update(parent_repos) + + except CircularImportError: + raise + except Exception as e: - if isinstance(e, CircularImportError): - raise raise RuntimeError( f'Error reading parent file {parent_file}: \n{str(e)}' ) from e