Skip to content

Commit eb14aa5

Browse files
christophebedardchristophebedard
committed
Add 'extends' feature to overlay repos files when importing
1 parent ce80335 commit eb14aa5

File tree

7 files changed

+293
-2
lines changed

7 files changed

+293
-2
lines changed

‎test/import_extends.txt‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
........
2+
=== ./immutable/hash (git) ===
3+
Cloning into '.'...
4+
Note: switching to '377d5b3d03c212f015cc832fdb368f4534d0d583'.
5+
6+
You are in 'detached HEAD' state. You can look around, make experimental
7+
changes and commit them, and you can discard any commits you make in this
8+
state without impacting any branches by switching back to a branch.
9+
10+
If you want to create a new branch to retain commits you create, you may
11+
do so (now or later) by using -c with the switch command. Example:
12+
13+
git switch -c <new-branch-name>
14+
15+
Or undo this operation with:
16+
17+
git switch -
18+
19+
Turn off this advice by setting config variable advice.detachedHead to false
20+
21+
HEAD is now at 377d5b3... update changelog
22+
=== ./immutable/hash_tar (tar) ===
23+
Downloaded tarball from 'https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz' and unpacked it
24+
=== ./immutable/hash_zip (zip) ===
25+
Downloaded zipfile from 'https://github.com/dirk-thomas/vcstool/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it
26+
=== ./immutable/tag (git) ===
27+
Cloning into '.'...
28+
Note: switching to '0.2.7'.
29+
30+
You are in 'detached HEAD' state. You can look around, make experimental
31+
changes and commit them, and you can discard any commits you make in this
32+
state without impacting any branches by switching back to a branch.
33+
34+
If you want to create a new branch to retain commits you create, you may
35+
do so (now or later) by using -c with the switch command. Example:
36+
37+
git switch -c <new-branch-name>
38+
39+
Or undo this operation with:
40+
41+
git switch -
42+
43+
Turn off this advice by setting config variable advice.detachedHead to false
44+
45+
HEAD is now at 80aadd6... 0.2.7
46+
=== ./vcstool (git) ===
47+
Cloning into '.'...
48+
=== ./vcstool-custom (git) ===
49+
Cloning into '.'...
50+
Note: switching to '0.2.10'.
51+
52+
You are in 'detached HEAD' state. You can look around, make experimental
53+
changes and commit them, and you can discard any commits you make in this
54+
state without impacting any branches by switching back to a branch.
55+
56+
If you want to create a new branch to retain commits you create, you may
57+
do so (now or later) by using -c with the switch command. Example:
58+
59+
git switch -c <new-branch-name>
60+
61+
Or undo this operation with:
62+
63+
git switch -
64+
65+
Turn off this advice by setting config variable advice.detachedHead to false
66+
67+
HEAD is now at 865990f... 0.2.10
68+
=== ./vcstool-old (git) ===
69+
Cloning into '.'...
70+
Note: switching to '0.1.2'.
71+
72+
You are in 'detached HEAD' state. You can look around, make experimental
73+
changes and commit them, and you can discard any commits you make in this
74+
state without impacting any branches by switching back to a branch.
75+
76+
If you want to create a new branch to retain commits you create, you may
77+
do so (now or later) by using -c with the switch command. Example:
78+
79+
git switch -c <new-branch-name>
80+
81+
Or undo this operation with:
82+
83+
git switch -
84+
85+
Turn off this advice by setting config variable advice.detachedHead to false
86+
87+
HEAD is now at 0ac0d6f... 0.1.2
88+
=== ./without_version (git) ===
89+
Cloning into '.'...

‎test/list_extends.repos‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
extends: list_extends2.repos
2+
repositories:
3+
immutable/hash_tar:
4+
type: tar
5+
url: https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz
6+
version: vcstool-afb4946c6a96aef37ad7770382b321beff0e0f26
7+
vcstool-custom:
8+
type: git
9+
url: https://github.com/dirk-thomas/vcstool.git
10+
version: 0.2.10
11+
vcstool-old:
12+
type: git
13+
url: https://github.com/dirk-thomas/vcstool.git
14+
version: 0.1.2

‎test/list_extends2.repos‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
extends: list.repos
2+
repositories:
3+
immutable/tag:
4+
type: git
5+
url: https://github.com/dirk-thomas/vcstool.git
6+
version: 0.2.7
7+
vcstool-old:
8+
type: git
9+
url: https://github.com/dirk-thomas/vcstool.git
10+
version: 0.1.1

‎test/list_extends_loop.repos‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extends: list_extends_loop2.repos
2+
repositories:
3+
vcstool:
4+
type: git
5+
url: https://github.com/dirk-thomas/vcstool.git
6+
version: master

‎test/list_extends_loop2.repos‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extends: list_extends_loop.repos
2+
repositories:
3+
vcstool:
4+
type: git
5+
url: https://github.com/dirk-thomas/vcstool.git
6+
version: 0.1.27

‎test/test_commands.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
REPOS_FILE_URL = \
1212
'https://raw.githubusercontent.com/dirk-thomas/vcstool/master/test/list.repos' # noqa: E501
1313
REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos')
14+
REPOS_EXTENDS_FILE = os.path.join(
15+
os.path.dirname(__file__), 'list_extends.repos')
16+
REPOS_EXTENDS_LOOP_FILE = os.path.join(
17+
os.path.dirname(__file__), 'list_extends_loop.repos')
1418
TEST_WORKSPACE = os.path.join(
1519
os.path.dirname(os.path.dirname(__file__)), 'test_workspace')
1620

@@ -321,6 +325,119 @@ def test_status(self):
321325
expected = get_expected_output('status')
322326
self.assertEqual(output, expected)
323327

328+
def test_import_extends(self):
329+
workdir = os.path.join(TEST_WORKSPACE, 'import-extends')
330+
os.makedirs(workdir)
331+
try:
332+
output = run_command(
333+
'import', ['--input', REPOS_EXTENDS_FILE, '.'],
334+
subfolder='import-extends')
335+
# the actual output contains absolute paths
336+
output = output.replace(
337+
b'repository in ' + workdir.encode() + b'/',
338+
b'repository in ./')
339+
expected = get_expected_output('import_extends')
340+
# newer git versions don't append three dots after the commit hash
341+
assert output == expected or \
342+
output == expected.replace(b'... ', b' ')
343+
finally:
344+
rmtree(workdir)
345+
346+
def test_import_extends_loop(self):
347+
with self.assertRaises(subprocess.CalledProcessError) as e:
348+
run_command(
349+
'import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.'])
350+
self.assertIn(
351+
b'Infinite loop in repos extensions', e.exception.output)
352+
353+
def test_import_extends_merge(self):
354+
from vcstool.commands.import_ import merge_repositories
355+
# only one set of repos
356+
repos = [
357+
{
358+
'some/path': {
359+
'type': 'git',
360+
'url': 'https://github.com/user/repo',
361+
'version': 'master',
362+
},
363+
},
364+
]
365+
merged_repos = merge_repositories(repos)
366+
self.assertDictEqual(repos[0], merged_repos)
367+
# multiple sets repos
368+
repos = [
369+
{
370+
'a/b/c': {
371+
'type': 'git',
372+
'url': 'https://github.com/a/bc',
373+
'version': '1.0.0',
374+
},
375+
'd/e': {
376+
'type': 'hg',
377+
'url': 'very.old.com',
378+
'version': '-1',
379+
},
380+
'f': {
381+
'type': 'svn',
382+
'url': 'https://gitlab.com/f/f',
383+
'version': 'master',
384+
},
385+
'g/h': {
386+
'type': 'git',
387+
'url': 'https://gitlab.com/g/h',
388+
'version': '2.7',
389+
},
390+
},
391+
{
392+
'a/b/c': {
393+
'type': 'git',
394+
'url': 'https://github.com/a/bc',
395+
'version': 'master',
396+
},
397+
'i/j': {
398+
'type': 'git',
399+
'url': 'https://some.website',
400+
'version': '42',
401+
},
402+
},
403+
{
404+
'g/h': {
405+
'type': 'git',
406+
'url': 'https://gitlab.com/custom/h',
407+
'version': '2.8',
408+
},
409+
},
410+
]
411+
expected_merged_repos = {
412+
'a/b/c': {
413+
'type': 'git',
414+
'url': 'https://github.com/a/bc',
415+
'version': 'master',
416+
},
417+
'd/e': {
418+
'type': 'hg',
419+
'url': 'very.old.com',
420+
'version': '-1',
421+
},
422+
'f': {
423+
'type': 'svn',
424+
'url': 'https://gitlab.com/f/f',
425+
'version': 'master',
426+
},
427+
'g/h': {
428+
'type': 'git',
429+
'url': 'https://gitlab.com/custom/h',
430+
'version': '2.8',
431+
},
432+
'i/j': {
433+
'type': 'git',
434+
'url': 'https://some.website',
435+
'version': '42',
436+
},
437+
}
438+
merged_repos = merge_repositories(repos)
439+
self.assertDictEqual(expected_merged_repos, merged_repos)
440+
324441

325442
def run_command(command, args=None, subfolder=None):
326443
repo_root = os.path.dirname(os.path.dirname(__file__))

‎vcstool/commands/import_.py‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,61 @@ def file_or_url_type(value):
7979
value, headers={'User-Agent': 'vcstool/' + vcstool_version})
8080

8181

82-
def get_repositories(yaml_file):
82+
def load_yaml_file(yaml_file):
8383
try:
84-
root = yaml.safe_load(yaml_file)
84+
return yaml.safe_load(yaml_file)
8585
except yaml.YAMLError as e:
8686
raise RuntimeError('Input data is not valid yaml format: %s' % e)
8787

88+
89+
def get_repositories(yaml_file):
90+
root = load_yaml_file(yaml_file)
91+
repos = get_repositories_from_root(root)
92+
if 'extends' not in root or not repos:
93+
return repos
94+
repos_list = [repos]
95+
# If the initial file is passed through --input, consider the extended
96+
# file path as being relative to it. Otherwise, if the initial file is
97+
# passed through stdin, use curdir as the base path.
98+
file_path = os.path.abspath(yaml_file.name) \
99+
if yaml_file.name != '<stdin>' else None
100+
base_rel_path = os.path.dirname(file_path) \
101+
if file_path else os.path.abspath(os.path.curdir)
102+
file_paths = [file_path] if file_path else []
103+
while 'extends' in root:
104+
extended_file_path = os.path.join(base_rel_path, root['extends'])
105+
if any(
106+
os.path.samefile(extended_file_path, path)
107+
for path in file_paths
108+
):
109+
raise RuntimeError(
110+
'Infinite loop in repos extensions: %s' % file_paths)
111+
base_rel_path = os.path.dirname(extended_file_path)
112+
file_paths.append(extended_file_path)
113+
try:
114+
with open(extended_file_path, 'r') as extended_file:
115+
root = load_yaml_file(extended_file)
116+
except IOError:
117+
raise RuntimeError(
118+
'Could not find extended file: %s' % extended_file_path)
119+
repos_list.append(get_repositories_from_root(root))
120+
repos_list.reverse()
121+
return merge_repositories(repos_list)
122+
123+
124+
def merge_repositories(repositories):
125+
base_repos = repositories[0]
126+
# merge second set of repos into first, then third one into that, etc.
127+
for extension_repos in repositories[1:]:
128+
for path, attributes in extension_repos.items():
129+
if path in base_repos:
130+
base_repos[path].update(attributes)
131+
else:
132+
base_repos[path] = attributes
133+
return base_repos
134+
135+
136+
def get_repositories_from_root(root):
88137
try:
89138
repositories = root['repositories']
90139
return get_repos_in_vcstool_format(repositories)

0 commit comments

Comments
 (0)