Skip to content

Commit 520c97f

Browse files
dmsimarddmsimard
committed
build-release: add support for running sanity tests
- The number of tests is growing and it's getting a bit unyieldy. Break them up a bit so we can add more tests easily. - Add an "antsibull_run_sanity" argument to loop over the collections in the release tarball to run "ansible-test sanity --docker" on them.
1 parent 423693d commit 520c97f

File tree

7 files changed

+170
-105
lines changed

7 files changed

+170
-105
lines changed

‎.github/workflows/antsibull-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
matrix:
1818
include:
1919
- name: Ansible 5 with default settings
20-
options: '-e antsibull_ansible_version=5.99.0'
20+
options: '-e antsibull_ansible_version=5.99.0 -e antsibull_run_sanity=true'
2121
- name: Ansible 6 with ansible-core devel
2222
options: '-e antsibull_ansible_version=6.99.0 -e antsibull_build_file=ansible-6.build -e antsibull_data_dir="{{ antsibull_data_git_dir }}/6" -e antsibull_ansible_git_version=devel'
2323

‎playbooks/nested-ansible-tests.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
hosts: localhost
55
gather_facts: no
66
vars:
7-
# These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests.yaml
7+
# These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests/nested.yaml
88
antsibull_sdist_dir: "{{ playbook_dir | dirname }}/build"
99
antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv"
1010
antsibull_ansible_git_dir: "{{ antsibull_sdist_dir }}/ansible"

‎roles/build-release/defaults/main.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv"
4646
# Whether or not to start from scratch with a new venv if one exists
4747
antsibull_venv_cleanup: true
4848

49+
# Whether to run "ansible-test sanity --docker" with podman on packaged collections
50+
# This is disabled by default since it is time consuming to run.
51+
antsibull_run_sanity: false
52+
4953
# TODO:
5054
# --dest-data-dir (Directory to write .build and .deps files to, as well as changelog and porting guide if applicable. Defaults to --data-dir)
5155
# --collection-cache (Directory of cached collection tarballs. Will be used if a collection tarball to be downloaded exists in here, and will be populated when downloading new tarballs.)

‎roles/build-release/tasks/tests.yaml‎

Lines changed: 7 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -20,108 +20,12 @@
2020
virtualenv: "{{ antsibull_ansible_venv }}"
2121
virtualenv_command: "{{ ansible_python.executable }} -m venv"
2222

23-
# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >=
24-
- block:
25-
# Note: the value is either _ansible_base_version or _ansible_core_version depending on the version
26-
# ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps
27-
- name: Retrieve the expected minimum version of ansible-core
28-
shell: >-
29-
grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}'
30-
changed_when: false
31-
register: _expected_ansible_core
23+
- name: Test and validate that we have the expected versions of collections, ansible-core and ansible
24+
include_tasks: tests/versions.yaml
3225

33-
- name: Retrieve the installed version of ansible-core
34-
shell: >-
35-
{{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}'
36-
changed_when: false
37-
register: _installed_ansible_core
26+
- name: Install the built ansible in venv and run integration tests with it
27+
include_tasks: tests/nested.yaml
3828

39-
- name: Validate the version of ansible-core
40-
ansible.builtin.assert:
41-
that:
42-
- _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=')
43-
success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}"
44-
fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}"
45-
46-
- block:
47-
- name: Retrieve expected versions of Ansible and collections
48-
command: cat {{ antsibull_data_dir }}/{{ _deps_file }}
49-
changed_when: false
50-
register: _expected_versions
51-
52-
- name: Retrieve collections that should be included in the package
53-
shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#"
54-
changed_when: false
55-
register: _included_collections
56-
57-
- name: Retrieve the installed version of ansible with pip
58-
shell: >-
59-
{{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}'
60-
changed_when: false
61-
register: _ansible_version_pypi
62-
63-
- name: Retrieve the builtin reported version of ansible
64-
command: >-
65-
{{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)'
66-
changed_when: false
67-
register: _ansible_version_builtin
68-
69-
- name: Validate the version of ansible
70-
ansible.builtin.assert:
71-
that:
72-
- "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout"
73-
- _ansible_version_pypi.stdout == _ansible_version_builtin.stdout
74-
success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'"
75-
fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'"
76-
77-
- name: Retrieve installed collections
78-
environment:
79-
# In case we happen to be testing with devel, don't print a warning about it
80-
ANSIBLE_DEVEL_WARNING: false
81-
# Until https://github.com/ansible/ansible/pull/70173 is backported and released
82-
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections"
83-
# List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file
84-
shell: >-
85-
{{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }'
86-
changed_when: false
87-
register: _installed_collections
88-
89-
- name: Validate that the installed collections are the expected ones
90-
ansible.builtin.assert:
91-
that:
92-
- item in _expected_versions.stdout
93-
success_msg: "{{ item }} matches {{ _deps_file }}"
94-
fail_msg: "{{ item }} does not match {{ _deps_file }}"
95-
loop: "{{ _installed_collections.stdout_lines }}"
96-
97-
- name: Validate that included collections are packaged
98-
ansible.builtin.assert:
99-
that:
100-
- item in _installed_collections.stdout
101-
success_msg: "{{ item }} is in ansible.in and inside the package"
102-
fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?"
103-
loop: "{{ _included_collections.stdout_lines }}"
104-
105-
- block:
106-
- name: Create a temporary COLLECTIONS_PATH
107-
file:
108-
path: "{{ antsibull_sdist_dir }}/ansible_collections"
109-
state: directory
110-
111-
- name: Install community.general for tests using 'a_module' and 'collection_version'
112-
environment:
113-
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
114-
command: >-
115-
{{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general
116-
117-
- name: Run nested Ansible tests with the Ansible we just built
118-
environment:
119-
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
120-
command: >-
121-
{{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local
122-
-e antsibull_sdist_dir="{{ antsibull_sdist_dir }}"
123-
-e antsibull_ansible_venv="{{ antsibull_ansible_venv }}"
124-
-e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}"
125-
-e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}"
126-
-e _python_version="{{ _python_version }}"
127-
{{ playbook_dir }}/nested-ansible-tests.yaml
29+
- name: Run ansible-test sanity --docker on installed collections with podman
30+
include_tasks: tests/sanity.yaml
31+
when: antsibull_run_sanity | bool

‎roles/build-release/tasks/tests/nested.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
- block:
2+
- name: Create a temporary COLLECTIONS_PATH
3+
file:
4+
path: "{{ antsibull_sdist_dir }}/ansible_collections"
5+
state: directory
6+
7+
- name: Install community.general for tests using 'a_module' and 'collection_version'
8+
environment:
9+
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
10+
command: >-
11+
{{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general
12+
13+
- name: Run nested Ansible tests with the Ansible we just built
14+
environment:
15+
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
16+
command: >-
17+
{{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local
18+
-e antsibull_sdist_dir="{{ antsibull_sdist_dir }}"
19+
-e antsibull_ansible_venv="{{ antsibull_ansible_venv }}"
20+
-e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}"
21+
-e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}"
22+
-e _python_version="{{ _python_version }}"
23+
{{ playbook_dir }}/nested-ansible-tests.yaml

‎roles/build-release/tasks/tests/sanity.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
- name: Get list of installed packages to verify if podman is installed
2+
package_facts:
3+
manager: "auto"
4+
# This is noisy (and potentially sensitive) to print on stdout
5+
no_log: true
6+
7+
# This is so we can otherwise run unprivileged if podman is already installed
8+
- when: "'podman' not in ansible_facts['packages'].keys()"
9+
become: true
10+
block:
11+
- name: Install podman
12+
package:
13+
name: podman
14+
state: present
15+
rescue:
16+
- name: Could not install podman
17+
fail:
18+
msg: |
19+
Failed to elevate privileges and install podman.
20+
Install podman manually or run ansible-playbook with elevated privileges.
21+
22+
# The tests are no longer installed on ansible>=6.0, use the ones from the release tarball
23+
- name: Unzip the release tarball
24+
ansible.builtin.unarchive:
25+
src: "{{ _release_archive }}"
26+
dest: "{{ antsibull_sdist_dir }}"
27+
remote_src: yes
28+
29+
# ansible.builtin.find doesn't have mindepth
30+
# https://github.com/ansible/ansible/issues/36369
31+
- name: Find collection directories
32+
vars:
33+
_extracted_release: "{{ antsibull_sdist_dir }}/ansible-{{ antsibull_ansible_version }}"
34+
command: find {{ _extracted_release }}/ansible_collections -mindepth 2 -maxdepth 2 -type d
35+
register: _collection_directories
36+
37+
# Note: unless we run a git init first, ansible-test won't be able to pick up tests because it's under
38+
# a .gitignore directory (antsibull/build)
39+
- name: Run sanity tests and collect results at the end
40+
shell: "git init .; {{ antsibull_ansible_venv }}/bin/ansible-test sanity --docker"
41+
ignore_errors: yes
42+
args:
43+
chdir: "{{ item }}"
44+
loop: "{{ _collection_directories.stdout_lines }}"
45+
register: _sanity_tests
46+
47+
# TODO: Loop over results and return the failures
48+
- name: Return results of sanity tests
49+
debug:
50+
var: _sanity_tests

‎roles/build-release/tasks/tests/versions.yaml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
---
2+
# Validate that we have the expected versions for the Ansible collections as well ansible-core and ansible
3+
4+
# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >=
5+
- block:
6+
# Note: the value is either _ansible_base_version or _ansible_core_version depending on the version
7+
# ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps
8+
- name: Retrieve the expected minimum version of ansible-core
9+
shell: >-
10+
grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}'
11+
changed_when: false
12+
register: _expected_ansible_core
13+
14+
- name: Retrieve the installed version of ansible-core
15+
shell: >-
16+
{{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}'
17+
changed_when: false
18+
register: _installed_ansible_core
19+
20+
- name: Validate the version of ansible-core
21+
ansible.builtin.assert:
22+
that:
23+
- _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=')
24+
success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}"
25+
fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}"
26+
27+
- block:
28+
- name: Retrieve expected versions of Ansible and collections
29+
command: cat {{ antsibull_data_dir }}/{{ _deps_file }}
30+
changed_when: false
31+
register: _expected_versions
32+
33+
- name: Retrieve collections that should be included in the package
34+
shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#"
35+
changed_when: false
36+
register: _included_collections
37+
38+
- name: Retrieve the installed version of ansible with pip
39+
shell: >-
40+
{{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}'
41+
changed_when: false
42+
register: _ansible_version_pypi
43+
44+
- name: Retrieve the builtin reported version of ansible
45+
command: >-
46+
{{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)'
47+
changed_when: false
48+
register: _ansible_version_builtin
49+
50+
- name: Validate the version of ansible
51+
ansible.builtin.assert:
52+
that:
53+
- "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout"
54+
- _ansible_version_pypi.stdout == _ansible_version_builtin.stdout
55+
success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'"
56+
fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'"
57+
58+
- name: Retrieve installed collections
59+
environment:
60+
# In case we happen to be testing with devel, don't print a warning about it
61+
ANSIBLE_DEVEL_WARNING: false
62+
# Until https://github.com/ansible/ansible/pull/70173 is backported and released
63+
ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections"
64+
# List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file
65+
shell: >-
66+
{{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }'
67+
changed_when: false
68+
register: _installed_collections
69+
70+
- name: Validate that the installed collections are the expected ones
71+
ansible.builtin.assert:
72+
that:
73+
- item in _expected_versions.stdout
74+
success_msg: "{{ item }} matches {{ _deps_file }}"
75+
fail_msg: "{{ item }} does not match {{ _deps_file }}"
76+
loop: "{{ _installed_collections.stdout_lines }}"
77+
78+
- name: Validate that included collections are packaged
79+
ansible.builtin.assert:
80+
that:
81+
- item in _installed_collections.stdout
82+
success_msg: "{{ item }} is in ansible.in and inside the package"
83+
fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?"
84+
loop: "{{ _included_collections.stdout_lines }}"

0 commit comments

Comments
 (0)