diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 00000000..caf4d8c1 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,19 @@ +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# profile: safety +quiet: true +strict: false +verbosity: 0 +offline: true diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore new file mode 100644 index 00000000..3217e5c7 --- /dev/null +++ b/.ansible-lint-ignore @@ -0,0 +1,268 @@ +# This file contains ignores rule violations for ansible-lint + +# galaxy.yml galaxy[no-changelog] +roles/grafana/tasks/Grafana-CentOS.yml fqcn[action-core] + +# Scheduled for removal +extensions/molecule/rdbms_server_postgresql_14/molecule.yml yaml[truthy] +extensions/molecule/rdbms_server_postgresql_14/requirements.yml schema[requirements] +extensions/molecule/rdbms_server_postgresql_14_tls/molecule.yml yaml[truthy] +extensions/molecule/rdbms_server_postgresql_14_tls/prepare.yml package-latest +extensions/molecule/rdbms_server_postgresql_14_tls/requirements.yml schema[requirements] +extensions/molecule/rdbms_server_postgresql_default/requirements.yml schema[requirements] +playbooks/pvc_base_postfix.yml name[missing] +playbooks/pvc_base_prereqs_ext.yml name[missing] +playbooks/pvc_base_teardown.yml name[missing] +roles/auto_repo_mirror/defaults/main.yml var-naming[no-role-prefix] +roles/auto_repo_mirror/tasks/inject.yml fqcn[action-core] +roles/auto_repo_mirror/tasks/inject.yml ignore-errors +roles/auto_repo_mirror/tasks/inject.yml jinja[spacing] +roles/auto_repo_mirror/tasks/inject.yml risky-file-permissions +roles/auto_repo_mirror/tasks/inject.yml yaml[line-length] +roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml yaml[line-length] +roles/auto_repo_mirror/tasks/populate_from_upstream.yml ignore-errors +roles/auto_repo_mirror/tasks/populate_from_upstream.yml risky-file-permissions +roles/auto_repo_mirror/tasks/update_mirror_cache.yml risky-file-permissions +roles/auto_repo_mirror/tasks/update_mirror_cache.yml yaml[line-length] +roles/auto_repo_mirror/vars/main.yml var-naming[no-role-prefix] +roles/blackbox/tasks/main.yml no-handler +roles/blackbox/tasks/main.yml risky-file-permissions +roles/common/defaults/main.yml jinja[spacing] +roles/common/defaults/main.yml yaml[line-length] +roles/common/meta/main.yml schema[meta] +roles/data/meta/main.yml schema[meta] +roles/data/tasks/initialize.yml yaml[line-length] +roles/data/tasks/main.yml name[missing] +roles/data/tasks/setup_aws.yml risky-file-permissions +roles/data/tasks/setup_aws.yml yaml[line-length] +roles/data/tasks/teardown_aws_policies.yml yaml[line-length] +roles/dynamic_inventory/defaults/main.yml jinja[spacing] +roles/dynamic_inventory/defaults/main.yml var-naming[no-role-prefix] +roles/dynamic_inventory/tasks/create_static_inventory.yml jinja[spacing] +roles/dynamic_inventory/tasks/create_static_inventory.yml risky-file-permissions +roles/dynamic_inventory/tasks/refresh_inventory.yml risky-file-permissions +roles/dynamic_inventory/tasks/retire_static_inventory.yml risky-file-permissions +roles/freeipa_client/defaults/main.yml var-naming[no-role-prefix] +roles/freeipa_client/meta/main.yml schema[meta] +roles/freeipa_client/tasks/main.yml ignore-errors +roles/freeipa_client/tasks/main.yml no-changed-when +roles/freeipa_client/tasks/main.yml risky-file-permissions +roles/freeipa_host_group/meta/main.yml schema[meta] +roles/freeipa_server/defaults/main.yml var-naming[no-role-prefix] +roles/freeipa_server/defaults/main.yml yaml[comments] +roles/freeipa_server/meta/main.yml schema[meta] +roles/freeipa_server/tasks/main.yml package-latest +roles/freeipa_server/tasks/main.yml risky-file-permissions +roles/freeipa_server/vars/RedHat-7.yml var-naming[no-role-prefix] +roles/freeipa_server/vars/RedHat-8.yml var-naming[no-role-prefix] +roles/freeipa_server/vars/RedHat-9.yml var-naming[no-role-prefix] +roles/freeipa_server/vars/default.yml var-naming[no-role-prefix] +roles/grafana/defaults/main.yml var-naming[no-role-prefix] +roles/grafana/tasks/Grafana-CentOS.yml risky-file-permissions +roles/grafana/tasks/Grafana-Ubuntu.yml risky-file-permissions +roles/grafana/vars/CentOS.yml var-naming[no-role-prefix] +roles/info/meta/main.yml schema[meta] +roles/info/tasks/main.yml jinja[spacing] +roles/info/tasks/main.yml risky-file-permissions +roles/infrastructure/defaults/main.yml jinja[spacing] +roles/infrastructure/defaults/main.yml var-naming[no-role-prefix] +roles/infrastructure/defaults/main.yml yaml[comments] +roles/infrastructure/defaults/main.yml yaml[line-length] +roles/infrastructure/meta/main.yml schema[meta] +roles/infrastructure/tasks/initialize_aws.yml no-changed-when +roles/infrastructure/tasks/initialize_aws.yml yaml[line-length] +roles/infrastructure/tasks/initialize_aws_terraform.yml risky-file-permissions +roles/infrastructure/tasks/initialize_azure.yml jinja[spacing] +roles/infrastructure/tasks/initialize_azure.yml no-changed-when +roles/infrastructure/tasks/initialize_base.yml ignore-errors +roles/infrastructure/tasks/initialize_gcp.yml args[module] +roles/infrastructure/tasks/initialize_gcp.yml no-changed-when +roles/infrastructure/tasks/initialize_setup_aws.yml no-changed-when +roles/infrastructure/tasks/initialize_setup_aws.yml var-naming[no-reserved] +roles/infrastructure/tasks/initialize_setup_aws.yml yaml[line-length] +roles/infrastructure/tasks/initialize_teardown_aws.yml no-changed-when +roles/infrastructure/tasks/initialize_teardown_aws.yml yaml[comments] +roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml risky-file-permissions +roles/infrastructure/tasks/main.yml name[missing] +roles/infrastructure/tasks/setup_aws.yml risky-file-permissions +roles/infrastructure/tasks/setup_aws_compute.yml jinja[spacing] +roles/infrastructure/tasks/setup_aws_compute.yml name[template] +roles/infrastructure/tasks/setup_aws_compute.yml yaml[line-length] +roles/infrastructure/tasks/setup_aws_network.yml no-changed-when +roles/infrastructure/tasks/setup_aws_network.yml yaml[comments] +roles/infrastructure/tasks/setup_aws_network.yml yaml[line-length] +roles/infrastructure/tasks/setup_aws_network_prefix_list.yml no-changed-when +roles/infrastructure/tasks/setup_aws_storage.yml no-changed-when +roles/infrastructure/tasks/setup_aws_utility_service.yml no-changed-when +roles/infrastructure/tasks/setup_azure_network.yml ignore-errors +roles/infrastructure/tasks/setup_azure_network.yml jinja[spacing] +roles/infrastructure/tasks/setup_azure_network.yml no-changed-when +roles/infrastructure/tasks/setup_gcp_network.yml args[module] +roles/infrastructure/tasks/setup_gcp_network.yml no-changed-when +roles/infrastructure/tasks/setup_gcp_storage.yml args[module] +roles/infrastructure/tasks/setup_terraform.yml risky-file-permissions +roles/infrastructure/tasks/teardown_aws_compute.yml no-changed-when +roles/infrastructure/tasks/teardown_aws_network.yml no-changed-when +roles/infrastructure/tasks/teardown_aws_network.yml yaml[comments] +roles/infrastructure/tasks/teardown_azure_storage.yml var-naming[no-reserved] +roles/infrastructure/tasks/teardown_gcp_network.yml args[module] +roles/infrastructure/tasks/teardown_gcp_network.yml no-changed-when +roles/infrastructure/tasks/teardown_gcp_storage.yml no-changed-when +roles/infrastructure/tasks/teardown_terraform.yml risky-file-permissions +roles/infrastructure/tasks/validate_aws.yml no-changed-when +roles/infrastructure/tasks/validate_aws.yml yaml[comments] +roles/infrastructure/tasks/validate_aws_terraform.yml no-changed-when +roles/infrastructure/tasks/validate_aws_terraform.yml yaml[line-length] +roles/infrastructure/tasks/validate_azure.yml yaml[line-length] +roles/infrastructure/tests/test.yml syntax-check[specific] +roles/infrastructure/vars/main.yml jinja[spacing] +roles/infrastructure/vars/main.yml var-naming[no-role-prefix] +roles/init_deployment/defaults/main.yml var-naming[no-role-prefix] +roles/init_deployment/tasks/marshall.yml jinja[invalid] +roles/init_deployment/tasks/marshall.yml name[template] +roles/init_deployment/tasks/prep_pvc.yml args[module] +roles/init_deployment/tasks/runlevels.yml yaml[line-length] +roles/init_deployment/tasks/validate.yml no-jinja-when +roles/init_deployment/vars/basic_cluster.yml schema[vars] +roles/init_deployment/vars/basic_cluster.yml var-naming[no-reserved] +roles/init_deployment/vars/basic_cluster.yml var-naming[no-role-prefix] +roles/mount/meta/main.yml schema[meta] +roles/nodeexporter/defaults/main.yml var-naming[no-role-prefix] +roles/nodeexporter/tasks/main.yml no-handler +roles/nodeexporter/tasks/main.yml risky-file-permissions +roles/platform/defaults/main.yml jinja[spacing] +roles/platform/defaults/main.yml var-naming[no-role-prefix] +roles/platform/defaults/main.yml yaml[line-length] +roles/platform/meta/main.yml schema[meta] +roles/platform/molecule/default/molecule.yml yaml[truthy] +roles/platform/molecule/level0/molecule.yml yaml[truthy] +roles/platform/molecule/shared/prepare.yml no-changed-when +roles/platform/molecule/shared/prepare.yml risky-file-permissions +roles/platform/tasks/aws_policy_download.yml risky-file-permissions +roles/platform/tasks/initialize_aws.yml no-changed-when +roles/platform/tasks/initialize_aws_terraform.yml risky-file-permissions +roles/platform/tasks/initialize_azure.yml ignore-errors +roles/platform/tasks/initialize_azure.yml no-changed-when +roles/platform/tasks/initialize_azure.yml yaml[line-length] +roles/platform/tasks/initialize_gcp.yml args[module] +roles/platform/tasks/initialize_setup_azure.yml risky-file-permissions +roles/platform/tasks/initialize_setup_base.yml no-changed-when +roles/platform/tasks/initialize_setup_base.yml var-naming[no-reserved] +roles/platform/tasks/initialize_setup_gcp.yml no-changed-when +roles/platform/tasks/initialize_setup_gcp.yml yaml[line-length] +roles/platform/tasks/initialize_teardown_aws_terraform.yml risky-file-permissions +roles/platform/tasks/initialize_teardown_azure.yml yaml[line-length] +roles/platform/tasks/initialize_teardown_gcp.yml no-changed-when +roles/platform/tasks/main.yml name[missing] +roles/platform/tasks/setup_aws_authz.yml jinja[spacing] +roles/platform/tasks/setup_aws_authz.yml no-changed-when +roles/platform/tasks/setup_aws_terraform_authz.yml risky-file-permissions +roles/platform/tasks/setup_azure_authz.yml no-changed-when +roles/platform/tasks/setup_azure_authz.yml yaml[line-length] +roles/platform/tasks/setup_base.yml yaml[line-length] +roles/platform/tasks/setup_gcp_authz.yml jinja[spacing] +roles/platform/tasks/setup_gcp_authz.yml no-changed-when +roles/platform/tasks/setup_gcp_authz.yml yaml[line-length] +roles/platform/tasks/setup_gcp_env.yml yaml[line-length] +roles/platform/tasks/teardown_aws_authz.yml no-changed-when +roles/platform/tasks/teardown_aws_terraform_authz.yml risky-file-permissions +roles/platform/tasks/teardown_azure_authz.yml no-changed-when +roles/platform/tasks/teardown_gcp_authz.yml jinja[spacing] +roles/platform/tasks/teardown_gcp_authz.yml no-changed-when +roles/platform/tasks/validate.yml no-changed-when +roles/platform/tasks/validate_aws_terraform.yml yaml[line-length] +roles/platform/vars/main.yml var-naming[no-role-prefix] +roles/prometheus/tasks/main.yml no-handler +roles/prometheus/tasks/main.yml risky-file-permissions +roles/provision/defaults/main.yml yaml[comments] +roles/provision/meta/main.yml schema[meta] +roles/provision/tasks/present.yml risky-file-permissions +roles/provision/tasks/present.yml var-naming[no-reserved] +roles/rdbms/client/mysql_connector/defaults/main.yml var-naming[no-role-prefix] +roles/rdbms/client/mysql_connector/tasks/main.yml ignore-errors +roles/rdbms/client/mysql_connector/tasks/main.yml no-changed-when +roles/rdbms/client/oracle_connector/defaults/main.yml var-naming[no-role-prefix] +roles/rdbms/client/postgresql_connector/defaults/main.yml var-naming[no-role-prefix] +roles/rdbms/client/postgresql_connector/tasks/main.yml literal-compare +roles/rdbms/client/postgresql_connector/tasks/main.yml no-changed-when +roles/rdbms/server/defaults/main.yml var-naming[no-role-prefix] +roles/rdbms/server/handlers/main.yml no-changed-when +roles/rdbms/server/meta/main.yml schema[meta] +roles/rdbms/server/tasks/mariadb/RedHat.yml command-instead-of-module +roles/rdbms/server/tasks/mariadb/RedHat.yml no-changed-when +roles/rdbms/server/tasks/mysql/RedHat.yml command-instead-of-module +roles/rdbms/server/tasks/mysql/RedHat.yml no-changed-when +roles/rdbms/server/tasks/mysql/RedHat.yml fqcn[action-core] +roles/rdbms/server/tasks/postgresql/Debian.yml package-latest +roles/rdbms/server/tasks/postgresql/Debian.yml risky-file-permissions +roles/rdbms/server/tasks/postgresql/template_fix.yml no-changed-when +roles/rdbms/server/vars/mariadb/Debian.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mariadb/RedHat-7.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mariadb/RedHat-8.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mariadb/RedHat-9.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mariadb/common.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mysql/Debian.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mysql/RedHat-7.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mysql/RedHat-8.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mysql/RedHat-9.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/mysql/common.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/postgresql/Debian.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/postgresql/RedHat.yml jinja[spacing] +roles/rdbms/server/vars/postgresql/RedHat.yml var-naming[no-role-prefix] +roles/rdbms/server/vars/postgresql/common.yml var-naming[no-role-prefix] +roles/rdbms_server/defaults/main.yml var-naming[no-role-prefix] +roles/rdbms_server/handlers/main.yml no-changed-when +roles/rdbms_server/meta/main.yml schema[meta] +roles/rdbms_server/tasks/mariadb/RedHat.yml command-instead-of-module +roles/rdbms_server/tasks/mariadb/RedHat.yml no-changed-when +roles/rdbms_server/tasks/mysql/RedHat.yml command-instead-of-module +roles/rdbms_server/tasks/mysql/RedHat.yml no-changed-when +roles/rdbms_server/tasks/mysql/RedHat.yml fqcn[action-core] +roles/rdbms_server/tasks/postgresql/Debian.yml package-latest +roles/rdbms_server/tasks/postgresql/Debian.yml risky-file-permissions +roles/rdbms_server/tasks/postgresql/RedHat.yml yaml[line-length] +roles/rdbms_server/tasks/postgresql/template_fix.yml no-changed-when +roles/rdbms_server/vars/mariadb/Debian.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mariadb/RedHat-7.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mariadb/RedHat-8.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mariadb/RedHat-9.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mariadb/common.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mysql/Debian.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mysql/RedHat-7.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mysql/RedHat-8.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mysql/RedHat-9.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/mysql/common.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/postgresql/Debian.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/postgresql/RedHat.yml jinja[spacing] +roles/rdbms_server/vars/postgresql/RedHat.yml var-naming[no-role-prefix] +roles/rdbms_server/vars/postgresql/common.yml var-naming[no-role-prefix] +roles/runtime/defaults/main.yml jinja[spacing] +roles/runtime/defaults/main.yml var-naming[no-role-prefix] +roles/runtime/meta/main.yml meta-no-tags +roles/runtime/meta/main.yml schema[meta] +roles/runtime/molecule/default/molecule.yml yaml[truthy] +roles/runtime/molecule/level0/molecule.yml yaml[truthy] +roles/runtime/molecule/shared/prepare.yml no-changed-when +roles/runtime/molecule/shared/prepare.yml risky-file-permissions +roles/runtime/tasks/initialize_base.yml jinja[spacing] +roles/runtime/tasks/initialize_base.yml jinja[invalid] +roles/runtime/tasks/initialize_base.yml var-naming[no-reserved] +roles/runtime/tasks/initialize_base.yml yaml[line-length] +roles/runtime/tasks/initialize_setup_azure.yml yaml[line-length] +roles/runtime/tasks/main.yml name[missing] +roles/runtime/tasks/setup_aws.yml yaml[comments] +roles/runtime/tasks/setup_base.yml yaml[line-length] +roles/runtime/vars/main.yml var-naming[no-role-prefix] +roles/sequence/meta/main.yml schema[meta] +roles/sudoers/meta/main.yml schema[meta] +roles/tls_fetch_ca_certs/defaults/main.yml var-naming[no-role-prefix] +roles/tls_generate_csr/defaults/main.yml var-naming[no-role-prefix] +roles/tls_install_certs/defaults/main.yml var-naming[no-role-prefix] +roles/tls_install_certs/tasks/main.yml no-changed-when +roles/tls_signing/defaults/main.yml var-naming[no-role-prefix] +tests/integration/integration_config.yml yaml[comments] +tests/integration/integration_config.yml yaml[truthy] +tests/integration/targets/light_duty_l1/tasks/main.yml var-naming[no-role-prefix] +tests/integration/targets/light_duty_l1/vars/cdp.yml jinja[spacing] +tests/integration/targets/medium_duty_l1/tasks/main.yml var-naming[no-role-prefix] +tests/integration/targets/medium_duty_l1/vars/cdp.yml jinja[spacing] diff --git a/.config/molecule/config.yml b/.config/molecule/config.yml index 4d78a1b9..74f61eb8 100644 --- a/.config/molecule/config.yml +++ b/.config/molecule/config.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 00000000..9514543e --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Lint Collection + +on: + pull_request: + push: + branches: [main, devel] +jobs: + pre-commit: + name: Execute ansible-lint + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up Ansible + run: | + sudo update-alternatives --install /usr/bin/python python $(which python3) 1 + python -m pip install --upgrade pip + python -m pip install "ansible-core<2.17" ansible-builder bindep pycodestyle voluptuous pylint pyyaml + + - name: Install Ansible collections and roles + run: | + mkdir -p /usr/share/ansible/collections /usr/share/ansible/roles + ansible-galaxy collection install . -p /usr/share/ansible/collections + ansible-galaxy collection install -r requirements.yml -p /usr/share/ansible/collections + ansible-galaxy role install -r requirements.yml -p /usr/share/ansible/roles + + - name: Set up Ansible collection dependencies + run: | + ansible-builder introspect --write-pip final_python.txt --write-bindep final_bindep.txt /usr/share/ansible/collections + [[ -f final_python.txt ]] && pip install -r final_python.txt || echo "No Python dependencies found." + [[ -f final_bindep.txt ]] && bindep --file final_bindep.txt || echo "No system dependencies found." + + - name: Run ansible-lint + uses: ansible/ansible-lint@main + with: + setup_python: false + env: + ANSIBLE_COLLECTION_PATHS: /usr/share/ansible/collections + ANSIBLE_ROLE_PATHS: /usr/share/ansible/roles diff --git a/.github/workflows/label_pr.yml b/.github/workflows/label_pr.yml index dfcd25a9..9ee76a5c 100644 --- a/.github/workflows/label_pr.yml +++ b/.github/workflows/label_pr.yml @@ -55,7 +55,7 @@ jobs: let fs = require('fs'); fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); - - name: 'Unzip artifact' + - name: "Unzip artifact" run: unzip pr_number.zip - name: Read the PR number diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 1b6688dc..1563fcc4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,4 +1,5 @@ -# Copyright 2024 Cloudera, Inc. +--- +# Copyright 2025 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,17 +13,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Execute Precommit Linting and Checks +name: Check Collection on: pull_request: push: branches: [main, devel] - jobs: pre-commit: + name: Execute pre-commit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: pre-commit/action@v3.0.1 + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up pre-commit and Ansible + run: | + # sudo update-alternatives --install /usr/bin/python python $(which python3) 1 + python -m pip install --upgrade pip + python -m pip install pre-commit + # python -m pip install "ansible-core<2.17" ansible-builder bindep pycodestyle voluptuous pylint pyyaml ansible-lint + + # - name: Install Ansible collections and roles + # run: | + # mkdir -p /usr/share/ansible/collections /usr/share/ansible/roles + # ansible-galaxy collection install . -p /usr/share/ansible/collections + # ansible-galaxy collection install -r requirements.yml -p /usr/share/ansible/collections + # ansible-galaxy role install -r requirements.yml -p /usr/share/ansible/roles + + # - name: Set up Ansible collection dependencies + # run: | + # ansible-builder introspect --write-pip final_python.txt --write-bindep final_bindep.txt /usr/share/ansible/collections + # [[ -f final_python.txt ]] && pip install -r final_python.txt || echo "No Python dependencies found." + # [[ -f final_bindep.txt ]] && bindep --file final_bindep.txt || echo "No system dependencies found." + + - name: Run pre-commit + run: | + pre-commit run -a --show-diff-on-failure --color=always + # env: + # ANSIBLE_COLLECTION_PATHS: /usr/share/ansible/collections + # ANSIBLE_ROLE_PATHS: /usr/share/ansible/roles diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml index 53e98007..13ff9147 100644 --- a/.github/workflows/publish_docs.yml +++ b/.github/workflows/publish_docs.yml @@ -19,10 +19,9 @@ name: Publish documentation on: push: branches: - - 'main' + - "main" workflow_dispatch: - jobs: build-ansible-docs: name: Build Ansible Docs diff --git a/.github/workflows/publish_galaxy.yml b/.github/workflows/publish_galaxy.yml index b61ebc9c..3f39b8ba 100644 --- a/.github/workflows/publish_galaxy.yml +++ b/.github/workflows/publish_galaxy.yml @@ -19,7 +19,6 @@ name: Publish to Ansible Galaxy on: release: types: [published] - jobs: galaxy_release: runs-on: ubuntu-latest diff --git a/.github/workflows/reset_pr.yml b/.github/workflows/reset_pr.yml index b3446f67..5f4797b3 100644 --- a/.github/workflows/reset_pr.yml +++ b/.github/workflows/reset_pr.yml @@ -23,9 +23,9 @@ on: - synchronize - ready_for_review branches: - - 'release/**' - - 'devel' - - 'devel-pvc-base' + - "release/**" + - "devel" + - "devel-pvc-base" jobs: reset: diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index 893726fa..7081b119 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -19,8 +19,8 @@ name: Validate Pull Request on: pull_request: branches: - - 'release/**' - - 'devel' + - "release/**" + - "devel" jobs: validate: @@ -32,8 +32,8 @@ jobs: - name: Setup Python and caching uses: actions/setup-python@v4 with: - python-version: '3.9' - cache: 'pip' + python-version: "3.9" + cache: "pip" - name: Set up Ansible collections run: | @@ -59,7 +59,7 @@ jobs: - name: Report installed Python dependencies run: pip freeze - - name: Validate collection + - name: Test collection run: | pushd /usr/share/ansible/collections/ansible_collections/cloudera/exe #ansible-lint diff --git a/.github/workflows/validate_pr_docs.yml b/.github/workflows/validate_pr_docs.yml index 9009d854..b3294f67 100644 --- a/.github/workflows/validate_pr_docs.yml +++ b/.github/workflows/validate_pr_docs.yml @@ -19,15 +19,14 @@ name: Validate Pull Request documentation on: pull_request: branches: - - 'release/**' - - 'devel' + - "release/**" + - "devel" workflow_dispatch: - jobs: validate-docs: name: Validate Ansible Docs - uses: cloudera-labs/github-actions/.github/workflows/lint-ansible-docs.yml@v1 + uses: cloudera-labs/github-actions/.github/workflows/lint-ansible-docs.yml@v2 with: antsibull-log-upload: true collection-namespace: cloudera diff --git a/.gitignore b/.gitignore index e1b0788e..10bcde78 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,6 @@ cython_debug/ # UV package manager uv.lock + +# Ansible tests +.ansible diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cda92b6..5cf04faa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,5 @@ -# Copyright 2024 Cloudera, Inc. +--- +# Copyright 2025 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,13 +15,40 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + - id: check-toml + - id: check-json - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-merge-conflict + - id: check-symlinks + - id: debug-statements + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: detect-private-key + - id: forbid-submodules + # - id: name-tests-test + - repo: https://github.com/asottile/add-trailing-comma.git + rev: v3.2.0 + hooks: + - id: add-trailing-comma + name: ensure trailing commas + args: + - --py36-plus - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 25.1.0 hooks: - id: black + name: lint python + # - repo: https://github.com/ansible/ansible-lint + # rev: v25.6.1 + # hooks: + # - id: ansible-lint + # name: lint ansible + # language: system diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..d410863f --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,409 @@ +============================== +Cloudera.Cluster Release Notes +============================== + +.. contents:: Topics + +v2.4.1 +====== + +Minor Changes +------------- + +- Add pre-commit hooks, workflow, and instructions (https://github.com/cloudera-labs/cloudera.exe/pull/188) +- migrate rdbms role - fixes for rhel9 (https://github.com/cloudera-labs/cloudera.exe/pull/181) + +Bugfixes +-------- + +- Update guard condition on dynamic inventory AMI lookup (https://github.com/cloudera-labs/cloudera.exe/pull/191) + +v2.4.0 +====== + +Minor Changes +------------- + +- Add analytics to API documents (https://github.com/cloudera-labs/cloudera.exe/pull/183) +- Add workflow and steps to validate for and publish to Ansible Galaxy (https://github.com/cloudera-labs/cloudera.exe/pull/184) +- Update role READMEs to enable Ansible Galaxy publication (https://github.com/cloudera-labs/cloudera.exe/pull/185) +- Update to version 2.4.0 (https://github.com/cloudera-labs/cloudera.exe/pull/186) + +New Roles +--------- + +- cloudera.exe.blackbox - Deploy Blackbox exporter. +- cloudera.exe.grafana - Deploy Grafana. +- cloudera.exe.nodeexporter - Deploy Node exporter. +- cloudera.exe.prometheus - Deploy Prometheus. + +v2.3.1 +====== + +Minor Changes +------------- + +- Add Blackbox Role (https://github.com/cloudera-labs/cloudera.exe/pull/178) +- Add Monitoring roles (https://github.com/cloudera-labs/cloudera.exe/pull/174) +- Add minor changes to Monitoring Roles (https://github.com/cloudera-labs/cloudera.exe/pull/177) +- Adds RHEL9 support for free_ipaserver & free_ipaclient (https://github.com/cloudera-labs/cloudera.exe/pull/176) +- Don't run GPC VPC discovery tasks when the subnet was already specified. (https://github.com/cloudera-labs/cloudera.exe/pull/149) + +v2.3.0 +====== + +Minor Changes +------------- + +- Allow skipping GCP availability zones validation. (https://github.com/cloudera-labs/cloudera.exe/pull/150) +- GCP: Add support for specifying the backups storage bucket. (https://github.com/cloudera-labs/cloudera.exe/pull/172) +- Move listing cross account keys to teardown playbook. (https://github.com/cloudera-labs/cloudera.exe/pull/147) +- Update AWS SG rules to use Prefix List for extra CIDR block access (https://github.com/cloudera-labs/cloudera.exe/pull/168) +- Variables that are set in roles/runtime/tasks/initialize_setup_gcp.yml are never used. (https://github.com/cloudera-labs/cloudera.exe/pull/148) + +Bugfixes +-------- + +- Remove duplicate namespace entry in freeipa_server role (https://github.com/cloudera-labs/cloudera.exe/pull/170) + +v2.2.0 +====== + +Minor Changes +------------- + +- Add PostgreSQL Connector install to pvc_base_prereqs_ext Playbook (https://github.com/cloudera-labs/cloudera.exe/pull/167) + +v2.1.0 +====== + +Bugfixes +-------- + +- Remove PVC Base teardown environment vars (https://github.com/cloudera-labs/cloudera.exe/pull/165) + +v2.0.1 +====== + +Minor Changes +------------- + +- Add PvC infra provision role (https://github.com/cloudera-labs/cloudera.exe/pull/159) +- Add storage volume mount role (https://github.com/cloudera-labs/cloudera.exe/pull/160) + +Bugfixes +-------- + +- Fixes for FreeIPA client and server roles (https://github.com/cloudera-labs/cloudera.exe/pull/158) +- Update pip requirements for the latest 2.12.* point releases (https://github.com/cloudera-labs/cloudera.exe/pull/162) + +New Roles +--------- + +- cloudera.exe.mount - Mount partitions. +- cloudera.exe.provision - Provision Cloudera-specific inventory. + +v2.0.0 +====== + +Minor Changes +------------- + +- Add Ansible documentation generation resources and workflows (https://github.com/cloudera-labs/cloudera.exe/pull/151) +- Add GCP region zones to CDP Env creation (https://github.com/cloudera-labs/cloudera.exe/pull/143) +- Add cloudera-deploy playbooks (https://github.com/cloudera-labs/cloudera.exe/pull/146) +- Add freeipa roles for PvC pre_setup RHEL only (https://github.com/cloudera-labs/cloudera.exe/pull/144) +- Update release/v2.0.0 (#153) (https://github.com/cloudera-labs/cloudera.exe/pull/155) +- Update release/v2.0.0 (https://github.com/cloudera-labs/cloudera.exe/pull/153) + +Bugfixes +-------- + +- Remove "virtual" collection dependencies (https://github.com/cloudera-labs/cloudera.exe/pull/156) +- Update check for MSI consistency (https://github.com/cloudera-labs/cloudera.exe/pull/145) + +New Playbooks +------------- + +- cloudera.exe.pbc_infra_setup - Set up CDP Public Cloud infrastructure +- cloudera.exe.pbc_infra_teardown - Tear down CDP Public Cloud infrastructure +- cloudera.exe.pbc_setup - Set up CDP Public Cloud platform resources +- cloudera.exe.pbc_teardown - Tear down CDP Public Cloud platform resources +- cloudera.exe.pvc_base_postfix - Post-deployment updates for CDP Private Cloud +- cloudera.exe.pvc_base_prereqs_ext - Set up external dependencies for CDP Private Cloud +- cloudera.exe.pvc_base_prereqs_int - Set up internal dependencies for CDP Private Cloud +- cloudera.exe.pvc_base_setup - Set up CDP Private Cloud +- cloudera.exe.pvc_base_teardown - Tear down CDP Private Cloud + +New Roles +--------- + +- cloudera.exe.auto_repo_mirror - Populate repository mirror. +- cloudera.exe.dynamic_inventory - Provision dynamic inventory. +- cloudera.exe.freeipa_client - Deploy FreeIPA clients. +- cloudera.exe.freeipa_server - Deploy FreeIPA server. +- cloudera.exe.init_deployment - Marshal deployment configuration. + +v1.7.5 +====== + +Minor Changes +------------- + +- Added subnet filters to the df_service module. (https://github.com/cloudera-labs/cloudera.exe/pull/118) +- RAZ Implementation for Azure (https://github.com/cloudera-labs/cloudera.exe/pull/111) +- Rebase of devel-pvc-update onto devel (https://github.com/cloudera-labs/cloudera.exe/pull/141) + +Bugfixes +-------- + +- Fix unset variable in runtime deployment for DW VW config (https://github.com/cloudera-labs/cloudera.exe/pull/136) +- Fixing regression due to recent changes to DataFlow runtime. (https://github.com/cloudera-labs/cloudera.exe/pull/137) + +v1.7.4 +====== + +Bugfixes +-------- + +- Update bindep installation and operations (https://github.com/cloudera-labs/cloudera.exe/pull/140) + +v1.7.3 +====== + +Minor Changes +------------- + +- Add support to choosing the GCP subnet to deploy to. (https://github.com/cloudera-labs/cloudera.exe/pull/132) +- PR validation workflows and ansible-builder support (https://github.com/cloudera-labs/cloudera.exe/pull/139) + +v1.7.2 +====== + +Minor Changes +------------- + +- Add import of DF Custom Flows to runtime role (https://github.com/cloudera-labs/cloudera.exe/pull/116) +- Allow skipping of GCP Service and IAM management (https://github.com/cloudera-labs/cloudera.exe/pull/130) +- CDW Round 47 (https://github.com/cloudera-labs/cloudera.exe/pull/102) +- Fixes for RHEL8.6 support and Dynamic Inventory (https://github.com/cloudera-labs/cloudera.exe/pull/127) +- Improve GCP APIs Services check and Enable (https://github.com/cloudera-labs/cloudera.exe/pull/129) +- Refactor Terraform into pure-TF resource files and Jinja tfvars (https://github.com/cloudera-labs/cloudera.exe/pull/125) +- Update GCP for L2 networking deployment (https://github.com/cloudera-labs/cloudera.exe/pull/115) +- Update collection version to 2.0.0-alpha1 (https://github.com/cloudera-labs/cloudera.exe/pull/121) +- WIP PvC Prereqs and Control Plane merge (https://github.com/cloudera-labs/cloudera.exe/pull/119) + +Bugfixes +-------- + +- Fix Azure deployment (https://github.com/cloudera-labs/cloudera.exe/pull/128) +- Fix git branch in collection dependency (https://github.com/cloudera-labs/cloudera.exe/pull/123) +- Hotfix- Update CentOS 7 AMI search terms (https://github.com/cloudera-labs/cloudera.exe/pull/133) +- Update collection dependency for PVC development (https://github.com/cloudera-labs/cloudera.exe/pull/122) + +v1.7.1 +====== + +Bugfixes +-------- + +- Change lookup search for Azure Service Principal Object ID (https://github.com/cloudera-labs/cloudera.exe/pull/120) + +v1.7.0 +====== + +Minor Changes +------------- + +- Initial commit for ansible-test support (https://github.com/cloudera-labs/cloudera.exe/pull/63) +- RAZ impl in exe (https://github.com/cloudera-labs/cloudera.exe/pull/107) +- Remove calls to the unsupported cloudera.cloud.env_auth (https://github.com/cloudera-labs/cloudera.exe/pull/117) + +v1.6.2 +====== + +Bugfixes +-------- + +- Fix MSI teardown to delete MSIs (https://github.com/cloudera-labs/cloudera.exe/pull/108) +- Support configurable AWS ARN partition for policies (https://github.com/cloudera-labs/cloudera.exe/pull/113) + +v1.6.1 +====== + +Bugfixes +-------- + +- Update parameters for EC2 module (https://github.com/cloudera-labs/cloudera.exe/pull/110) + +v1.6.0 +====== + +Minor Changes +------------- + +- Add Terraform deployment engine for cloud resources (https://github.com/cloudera-labs/cloudera.exe/pull/56) +- Azure AuthZ/Single Resource Group Work - EXE (https://github.com/cloudera-labs/cloudera.exe/pull/68) +- Convert terraform related global variables to a dictionary (https://github.com/cloudera-labs/cloudera.exe/pull/100) +- Map common__azure_sp_login_env to infra (https://github.com/cloudera-labs/cloudera.exe/pull/101) +- Pin collection dependencies to single versions (https://github.com/cloudera-labs/cloudera.exe/pull/98) +- Support AWSCLI v2 (https://github.com/cloudera-labs/cloudera.exe/pull/81) +- Support for DataFlow Deployments (https://github.com/cloudera-labs/cloudera.exe/pull/82) +- Support the use of other CDP control planes (https://github.com/cloudera-labs/cloudera.exe/pull/91) +- Update Azure MSI and role assignment handling (https://github.com/cloudera-labs/cloudera.exe/pull/80) +- Update config docs (https://github.com/cloudera-labs/cloudera.exe/pull/96) +- fix ec2 dynamic inventory and el8 deployment (https://github.com/cloudera-labs/cloudera.exe/pull/94) + +Bugfixes +-------- + +- Fix AWS ELB teardown (https://github.com/cloudera-labs/cloudera.exe/pull/97) +- Fix default Azure Netapp volume size (https://github.com/cloudera-labs/cloudera.exe/pull/79) +- Fix dynamic inventory public IP check (https://github.com/cloudera-labs/cloudera.exe/pull/99) +- Fix failed_when condition for GCP Service Accounts Policies (https://github.com/cloudera-labs/cloudera.exe/pull/106) +- Hotfix for Issue #83 (https://github.com/cloudera-labs/cloudera.exe/pull/84) +- Rearrange teardown tasks for GCP (https://github.com/cloudera-labs/cloudera.exe/pull/93) +- Update Azure NetApp management and add NFS protocol version (https://github.com/cloudera-labs/cloudera.exe/pull/86) +- Use infra__security_group_vpce_name as variable for VPC Endpoint SG (https://github.com/cloudera-labs/cloudera.exe/pull/104) + +v1.5.2 +====== + +Bugfixes +-------- + +- Fix bug with __infra_aws_storage_tags_list (https://github.com/cloudera-labs/cloudera.exe/pull/74) +- Fix invalid subnet variables for CDW creation (https://github.com/cloudera-labs/cloudera.exe/pull/77) +- region statement missing from modify-vpc-endpoint awscli call (https://github.com/cloudera-labs/cloudera.exe/pull/75) + +v1.5.1 +====== + +Bugfixes +-------- + +- Fix reference to undefined storage tags variable (https://github.com/cloudera-labs/cloudera.exe/pull/73) + +v1.5.0 +====== + +Minor Changes +------------- + +- AWS VPC Endpoint Support (https://github.com/cloudera-labs/cloudera.exe/pull/54) +- Add GCP support to FreeIPA host group role (https://github.com/cloudera-labs/cloudera.exe/pull/61) +- Add Ubuntu 20.04 focal fossa as optional OS for dynamic inventory (https://github.com/cloudera-labs/cloudera.exe/pull/69) +- Add network discovery and assignment functions (https://github.com/cloudera-labs/cloudera.exe/pull/62) +- Add role, policy, and storage tagging to AWS (https://github.com/cloudera-labs/cloudera.exe/pull/55) +- Add selectable distribution support for cloudera.cluster (https://github.com/cloudera-labs/cloudera.exe/pull/51) +- Add support for CDE (https://github.com/cloudera-labs/cloudera.exe/pull/58) +- Add support for CDE (part 2 - virtual clusters) (https://github.com/cloudera-labs/cloudera.exe/pull/60) +- Allow optional deletion of GCP Custom roles during teardown (https://github.com/cloudera-labs/cloudera.exe/pull/44) +- Enhancement to sudoers role to add groups and work with user sync (https://github.com/cloudera-labs/cloudera.exe/pull/50) +- Extensible tagging for Cloudera Experiences (https://github.com/cloudera-labs/cloudera.exe/pull/48) +- Molecule test harness for platform role (https://github.com/cloudera-labs/cloudera.exe/pull/59) +- Move DFX Beta implementation to GA process (https://github.com/cloudera-labs/cloudera.exe/pull/47) +- Update streams messaging default template (https://github.com/cloudera-labs/cloudera.exe/pull/65) + +Bugfixes +-------- + +- Add guard conditionals for CDE setup (https://github.com/cloudera-labs/cloudera.exe/pull/66) +- Add missing CDF configurations (https://github.com/cloudera-labs/cloudera.exe/pull/64) +- Fix AWS network creation error when no tags are defined (https://github.com/cloudera-labs/cloudera.exe/pull/46) +- Fix AWS network discovery (https://github.com/cloudera-labs/cloudera.exe/pull/72) + +v1.4.2 +====== + +v1.4.1 +====== + +v1.4.0 +====== + +Minor Changes +------------- + +- AWS Level 2 networking (including shared resources) (https://github.com/cloudera-labs/cloudera.exe/pull/32) +- Add Centos8 to Dynamic Inventory options (https://github.com/cloudera-labs/cloudera.exe/pull/25) +- Changes for DF-beta (https://github.com/cloudera-labs/cloudera.exe/pull/20) +- Ciao dynamo (https://github.com/cloudera-labs/cloudera.exe/pull/33) +- Improve Azure deployment stability (https://github.com/cloudera-labs/cloudera.exe/pull/34) +- Improve GCP teardown idempotence (https://github.com/cloudera-labs/cloudera.exe/pull/39) +- Improve network security port determination logic (https://github.com/cloudera-labs/cloudera.exe/pull/29) +- Improve purge functionality with further edge cases (https://github.com/cloudera-labs/cloudera.exe/pull/35) +- Improve teardown and support purge mode, other minor fixes (https://github.com/cloudera-labs/cloudera.exe/pull/24) +- Remove initialize tasks in sudoers role (https://github.com/cloudera-labs/cloudera.exe/pull/42) +- Support Private Networks (https://github.com/cloudera-labs/cloudera.exe/pull/15) +- Update Azure Teardown - Currently broken (https://github.com/cloudera-labs/cloudera.exe/pull/18) +- Update ML Workspace setup to use definition of a single instance group (https://github.com/cloudera-labs/cloudera.exe/pull/40) +- Update env setup to include passing freeipa instance count. Add some … (https://github.com/cloudera-labs/cloudera.exe/pull/38) + +Bugfixes +-------- + +- Correct references to AWS policy documents (https://github.com/cloudera-labs/cloudera.exe/pull/30) +- Correcting Idbroker Role policy definitions for AWS (https://github.com/cloudera-labs/cloudera.exe/pull/41) +- Fix L1 networking teardown when purge is used (https://github.com/cloudera-labs/cloudera.exe/pull/43) +- Fix default opdb teardown (https://github.com/cloudera-labs/cloudera.exe/pull/22) +- Fix unused DWX variable and more accurate datahub definition filters (https://github.com/cloudera-labs/cloudera.exe/pull/19) + +v1.3.1 +====== + +v1.3.0 +====== + +Minor Changes +------------- + +- Add support for DFX Tech Preview (https://github.com/cloudera-labs/cloudera.exe/pull/12) + +Bugfixes +-------- + +- Reopening PR after revert on Cloudera Labs (https://github.com/cloudera-labs/cloudera.exe/pull/16) + +v1.2.1 +====== + +v1.2.0 +====== + +Minor Changes +------------- + +- Add tasks for retrieving datahub definitions and filtering by datalak… (https://github.com/cloudera-labs/cloudera.exe/pull/9) +- Improve Azure Storage Account name check to be more informative (https://github.com/cloudera-labs/cloudera.exe/pull/13) +- New Roles to facilitate creation of FreeIPA sudoers group and rule (https://github.com/cloudera-labs/cloudera.exe/pull/6) +- Remove extraneous user_ports from Extra security group (https://github.com/cloudera-labs/cloudera.exe/pull/14) + +New Roles +--------- + +- cloudera.exe.freeipa_host_group - Create FreeIPA host group. +- cloudera.exe.sudoers - Create sudo rule. + +v1.1.2 +====== + +v1.1.1 +====== + +v1.1.0 +====== + +v1.0.0 +====== + +New Roles +--------- + +- cloudera.exe.common - Shared configuration variables. +- cloudera.exe.data - Manage external data locations. +- cloudera.exe.info - Retrieve CDP details. +- cloudera.exe.infrastructure - Deploy cloud provider infrastructure. +- cloudera.exe.infrastructure - Execute run-level operations for CDP Public Cloud. +- cloudera.exe.platform - Deploy CDP Public Cloud core services. +- cloudera.exe.runtime - Deploy CDP Public Cloud Experiences. diff --git a/changelogs/.plugin-cache.yaml b/changelogs/.plugin-cache.yaml new file mode 100644 index 00000000..c5e83e72 --- /dev/null +++ b/changelogs/.plugin-cache.yaml @@ -0,0 +1,124 @@ +objects: + role: + auto_repo_mirror: + description: Populate repository mirror + name: auto_repo_mirror + version_added: 2.0.0 + blackbox: + description: Deploy Blackbox exporter. + name: blackbox + version_added: 2.4.0 + common: + description: Shared configuration variables + name: common + version_added: 1.0.0 + data: + description: Manage external data locations + name: data + version_added: 1.0.0 + dynamic_inventory: + description: Provision dynamic inventory + name: dynamic_inventory + version_added: 2.0.0 + freeipa_client: + description: Deploy FreeIPA clients. + name: freeipa_client + version_added: 2.0.0 + freeipa_host_group: + description: Create FreeIPA host group + name: freeipa_host_group + version_added: 1.2.0 + freeipa_server: + description: Deploy FreeIPA server. + name: freeipa_server + version_added: 2.0.0 + grafana: + description: Deploy Grafana. + name: grafana + version_added: 2.4.0 + info: + description: Retrieve CDP details + name: info + version_added: 1.0.0 + infrastructure: + description: Deploy cloud provider infrastructure + name: infrastructure + version_added: 1.0.0 + init_deployment: + description: Marshal deployment configuration. + name: init_deployment + version_added: 2.0.0 + mount: + description: Mount partitions. + name: mount + version_added: 2.0.1 + nodeexporter: + description: Deploy Node exporter. + name: nodeexporter + version_added: 2.4.0 + platform: + description: Deploy CDP Public Cloud core services + name: platform + version_added: 1.0.0 + prometheus: + description: Deploy Prometheus. + name: prometheus + version_added: 2.4.0 + provision: + description: Provision Cloudera-specific inventory. + name: provision + version_added: 2.0.1 + rdbms_server: + description: Install standalone RDBMS instance + name: rdbms_server + version_added: null + runtime: + description: Deploy CDP Public Cloud Experiences + name: runtime + version_added: 1.0.0 + sequence: + description: Execute run-level operations for CDP Public Cloud + name: sequence + version_added: 1.0.0 + sudoers: + description: Create sudo rule + name: sudoers + version_added: 1.2.0 + tls_fetch_ca_certs: + description: Bring CA root and intermediate cert back to controller + name: tls_fetch_ca_certs + version_added: 3.0.0 + tls_generate_csr: + description: Generates a CSR on each host and copies it back to the Ansible + controller + name: tls_generate_csr + version_added: 3.0.0 + tls_install_certs: + description: Copy and install the signed TLS certificates to each cluster + name: tls_install_certs + version_added: 3.0.0 + tls_signing: + description: Sign of CSRs by a CA Server + name: tls_signing + version_added: 3.0.0 +plugins: + become: {} + cache: {} + callback: {} + cliconf: {} + connection: {} + filter: + combine_onto: + description: combine two dictionaries + name: combine_onto + version_added: 1.0.0 + httpapi: {} + inventory: {} + lookup: {} + module: {} + netconf: {} + shell: {} + strategy: {} + test: {} + vars: {} +version: 2.4.1 diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml new file mode 100644 index 00000000..32191f3e --- /dev/null +++ b/changelogs/changelog.yaml @@ -0,0 +1,347 @@ +--- +ancestor: +releases: + 2.4.1: + changes: + bugfixes: + - Update guard condition on dynamic inventory AMI lookup (https://github.com/cloudera-labs/cloudera.exe/pull/191) + minor_changes: + - Add pre-commit hooks, workflow, and instructions (https://github.com/cloudera-labs/cloudera.exe/pull/188) + - migrate rdbms role - fixes for rhel9 (https://github.com/cloudera-labs/cloudera.exe/pull/181) + release_date: '2024-08-28' + 2.4.0: + changes: + minor_changes: + - Update to version 2.4.0 (https://github.com/cloudera-labs/cloudera.exe/pull/186) + - Update role READMEs to enable Ansible Galaxy publication (https://github.com/cloudera-labs/cloudera.exe/pull/185) + - Add workflow and steps to validate for and publish to Ansible Galaxy (https://github.com/cloudera-labs/cloudera.exe/pull/184) + - Add analytics to API documents (https://github.com/cloudera-labs/cloudera.exe/pull/183) + objects: + role: + - description: Deploy Blackbox exporter. + name: blackbox + namespace: + - description: Deploy Grafana. + name: grafana + namespace: + - description: Deploy Node exporter. + name: nodeexporter + namespace: + - description: Deploy Prometheus. + name: prometheus + namespace: + release_date: '2024-05-21' + 2.3.1: + changes: + minor_changes: + - Add Blackbox Role (https://github.com/cloudera-labs/cloudera.exe/pull/178) + - Add minor changes to Monitoring Roles (https://github.com/cloudera-labs/cloudera.exe/pull/177) + - Adds RHEL9 support for free_ipaserver & free_ipaclient (https://github.com/cloudera-labs/cloudera.exe/pull/176) + - Add Monitoring roles (https://github.com/cloudera-labs/cloudera.exe/pull/174) + - Don't run GPC VPC discovery tasks when the subnet was already specified. + (https://github.com/cloudera-labs/cloudera.exe/pull/149) + release_date: '2024-04-08' + 2.3.0: + changes: + minor_changes: + - 'GCP: Add support for specifying the backups storage bucket. (https://github.com/cloudera-labs/cloudera.exe/pull/172)' + - Update AWS SG rules to use Prefix List for extra CIDR block access (https://github.com/cloudera-labs/cloudera.exe/pull/168) + - Allow skipping GCP availability zones validation. (https://github.com/cloudera-labs/cloudera.exe/pull/150) + - Variables that are set in roles/runtime/tasks/initialize_setup_gcp.yml are + never used. (https://github.com/cloudera-labs/cloudera.exe/pull/148) + - Move listing cross account keys to teardown playbook. (https://github.com/cloudera-labs/cloudera.exe/pull/147) + bugfixes: + - Remove duplicate namespace entry in freeipa_server role (https://github.com/cloudera-labs/cloudera.exe/pull/170) + release_date: '2023-12-21' + 2.2.0: + changes: + minor_changes: + - Add PostgreSQL Connector install to pvc_base_prereqs_ext Playbook (https://github.com/cloudera-labs/cloudera.exe/pull/167) + release_date: '2023-11-20' + 2.1.0: + changes: + bugfixes: + - Remove PVC Base teardown environment vars (https://github.com/cloudera-labs/cloudera.exe/pull/165) + release_date: '2023-11-02' + 2.0.1: + changes: + bugfixes: + - Update pip requirements for the latest 2.12.* point releases (https://github.com/cloudera-labs/cloudera.exe/pull/162) + - Fixes for FreeIPA client and server roles (https://github.com/cloudera-labs/cloudera.exe/pull/158) + minor_changes: + - Add storage volume mount role (https://github.com/cloudera-labs/cloudera.exe/pull/160) + - Add PvC infra provision role (https://github.com/cloudera-labs/cloudera.exe/pull/159) + objects: + role: + - description: Mount partitions. + name: mount + namespace: + - description: Provision Cloudera-specific inventory. + name: provision + namespace: + release_date: '2023-10-05' + 2.0.0: + changes: + minor_changes: + - Update release/v2.0.0 (#153) (https://github.com/cloudera-labs/cloudera.exe/pull/155) + - Update release/v2.0.0 (https://github.com/cloudera-labs/cloudera.exe/pull/153) + - Add Ansible documentation generation resources and workflows (https://github.com/cloudera-labs/cloudera.exe/pull/151) + - Add cloudera-deploy playbooks (https://github.com/cloudera-labs/cloudera.exe/pull/146) + - Add freeipa roles for PvC pre_setup RHEL only (https://github.com/cloudera-labs/cloudera.exe/pull/144) + - Add GCP region zones to CDP Env creation (https://github.com/cloudera-labs/cloudera.exe/pull/143) + bugfixes: + - Remove "virtual" collection dependencies (https://github.com/cloudera-labs/cloudera.exe/pull/156) + - Update check for MSI consistency (https://github.com/cloudera-labs/cloudera.exe/pull/145) + objects: + role: + - description: Populate repository mirror. + name: auto_repo_mirror + namespace: + - description: Provision dynamic inventory. + name: dynamic_inventory + namespace: + - description: Deploy FreeIPA clients. + name: freeipa_client + namespace: + - description: Deploy FreeIPA server. + name: freeipa_server + namespace: + - description: Marshal deployment configuration. + name: init_deployment + namespace: + playbook: + - name: pbc_infra_setup + description: Set up CDP Public Cloud infrastructure + namespace: null + - name: pbc_infra_teardown + description: Tear down CDP Public Cloud infrastructure + namespace: null + - name: pbc_setup + description: Set up CDP Public Cloud platform resources + namespace: null + - name: pbc_teardown + description: Tear down CDP Public Cloud platform resources + namespace: null + - name: pvc_base_postfix + description: Post-deployment updates for CDP Private Cloud + namespace: null + - name: pvc_base_prereqs_ext + description: Set up external dependencies for CDP Private Cloud + namespace: null + - name: pvc_base_prereqs_int + description: Set up internal dependencies for CDP Private Cloud + namespace: null + - name: pvc_base_setup + description: Set up CDP Private Cloud + namespace: null + - name: pvc_base_teardown + description: Tear down CDP Private Cloud + namespace: null + release_date: '2023-09-28' + 1.7.5: + changes: + minor_changes: + - Rebase of devel-pvc-update onto devel (https://github.com/cloudera-labs/cloudera.exe/pull/141) + - Added subnet filters to the df_service module. (https://github.com/cloudera-labs/cloudera.exe/pull/118) + - RAZ Implementation for Azure (https://github.com/cloudera-labs/cloudera.exe/pull/111) + bugfixes: + - Fixing regression due to recent changes to DataFlow runtime. (https://github.com/cloudera-labs/cloudera.exe/pull/137) + - Fix unset variable in runtime deployment for DW VW config (https://github.com/cloudera-labs/cloudera.exe/pull/136) + release_date: '2023-06-09' + 1.7.4: + changes: + bugfixes: + - Update bindep installation and operations (https://github.com/cloudera-labs/cloudera.exe/pull/140) + release_date: '2023-02-03' + 1.7.3: + changes: + minor_changes: + - PR validation workflows and ansible-builder support (https://github.com/cloudera-labs/cloudera.exe/pull/139) + - Add support to choosing the GCP subnet to deploy to. (https://github.com/cloudera-labs/cloudera.exe/pull/132) + release_date: '2023-02-02' + 1.7.2: + changes: + minor_changes: + - Allow skipping of GCP Service and IAM management (https://github.com/cloudera-labs/cloudera.exe/pull/130) + - Improve GCP APIs Services check and Enable (https://github.com/cloudera-labs/cloudera.exe/pull/129) + - Fixes for RHEL8.6 support and Dynamic Inventory (https://github.com/cloudera-labs/cloudera.exe/pull/127) + - Refactor Terraform into pure-TF resource files and Jinja tfvars (https://github.com/cloudera-labs/cloudera.exe/pull/125) + - Update collection version to 2.0.0-alpha1 (https://github.com/cloudera-labs/cloudera.exe/pull/121) + - WIP PvC Prereqs and Control Plane merge (https://github.com/cloudera-labs/cloudera.exe/pull/119) + - Add import of DF Custom Flows to runtime role (https://github.com/cloudera-labs/cloudera.exe/pull/116) + - Update GCP for L2 networking deployment (https://github.com/cloudera-labs/cloudera.exe/pull/115) + - CDW Round 47 (https://github.com/cloudera-labs/cloudera.exe/pull/102) + bugfixes: + - Hotfix- Update CentOS 7 AMI search terms (https://github.com/cloudera-labs/cloudera.exe/pull/133) + - Fix Azure deployment (https://github.com/cloudera-labs/cloudera.exe/pull/128) + - Fix git branch in collection dependency (https://github.com/cloudera-labs/cloudera.exe/pull/123) + - Update collection dependency for PVC development (https://github.com/cloudera-labs/cloudera.exe/pull/122) + release_date: '2022-11-15' + 1.7.1: + changes: + bugfixes: + - Change lookup search for Azure Service Principal Object ID (https://github.com/cloudera-labs/cloudera.exe/pull/120) + release_date: '2022-08-04' + 1.7.0: + changes: + minor_changes: + - Remove calls to the unsupported cloudera.cloud.env_auth (https://github.com/cloudera-labs/cloudera.exe/pull/117) + - RAZ impl in exe (https://github.com/cloudera-labs/cloudera.exe/pull/107) + - Initial commit for ansible-test support (https://github.com/cloudera-labs/cloudera.exe/pull/63) + release_date: '2022-08-02' + 1.6.2: + changes: + bugfixes: + - Support configurable AWS ARN partition for policies (https://github.com/cloudera-labs/cloudera.exe/pull/113) + - Fix MSI teardown to delete MSIs (https://github.com/cloudera-labs/cloudera.exe/pull/108) + release_date: '2022-05-09' + 1.6.1: + changes: + bugfixes: + - Update parameters for EC2 module (https://github.com/cloudera-labs/cloudera.exe/pull/110) + release_date: '2022-04-14' + 1.6.0: + changes: + minor_changes: + - Map common__azure_sp_login_env to infra (https://github.com/cloudera-labs/cloudera.exe/pull/101) + - Convert terraform related global variables to a dictionary (https://github.com/cloudera-labs/cloudera.exe/pull/100) + - Pin collection dependencies to single versions (https://github.com/cloudera-labs/cloudera.exe/pull/98) + - Update config docs (https://github.com/cloudera-labs/cloudera.exe/pull/96) + - fix ec2 dynamic inventory and el8 deployment (https://github.com/cloudera-labs/cloudera.exe/pull/94) + - Support the use of other CDP control planes (https://github.com/cloudera-labs/cloudera.exe/pull/91) + - Support for DataFlow Deployments (https://github.com/cloudera-labs/cloudera.exe/pull/82) + - Support AWSCLI v2 (https://github.com/cloudera-labs/cloudera.exe/pull/81) + - Update Azure MSI and role assignment handling (https://github.com/cloudera-labs/cloudera.exe/pull/80) + - Azure AuthZ/Single Resource Group Work - EXE (https://github.com/cloudera-labs/cloudera.exe/pull/68) + - Add Terraform deployment engine for cloud resources (https://github.com/cloudera-labs/cloudera.exe/pull/56) + bugfixes: + - Fix failed_when condition for GCP Service Accounts Policies (https://github.com/cloudera-labs/cloudera.exe/pull/106) + - Use infra__security_group_vpce_name as variable for VPC Endpoint SG (https://github.com/cloudera-labs/cloudera.exe/pull/104) + - Fix dynamic inventory public IP check (https://github.com/cloudera-labs/cloudera.exe/pull/99) + - Fix AWS ELB teardown (https://github.com/cloudera-labs/cloudera.exe/pull/97) + - Rearrange teardown tasks for GCP (https://github.com/cloudera-labs/cloudera.exe/pull/93) + - Update Azure NetApp management and add NFS protocol version (https://github.com/cloudera-labs/cloudera.exe/pull/86) + - 'Hotfix for Issue #83 (https://github.com/cloudera-labs/cloudera.exe/pull/84)' + - Fix default Azure Netapp volume size (https://github.com/cloudera-labs/cloudera.exe/pull/79) + release_date: '2022-04-07' + 1.5.2: + changes: + bugfixes: + - Fix invalid subnet variables for CDW creation (https://github.com/cloudera-labs/cloudera.exe/pull/77) + - region statement missing from modify-vpc-endpoint awscli call (https://github.com/cloudera-labs/cloudera.exe/pull/75) + - Fix bug with __infra_aws_storage_tags_list (https://github.com/cloudera-labs/cloudera.exe/pull/74) + release_date: '2021-12-07' + 1.5.1: + changes: + bugfixes: + - Fix reference to undefined storage tags variable (https://github.com/cloudera-labs/cloudera.exe/pull/73) + release_date: '2021-11-30' + 1.5.0: + changes: + bugfixes: + - Fix AWS network discovery (https://github.com/cloudera-labs/cloudera.exe/pull/72) + - Add guard conditionals for CDE setup (https://github.com/cloudera-labs/cloudera.exe/pull/66) + - Add missing CDF configurations (https://github.com/cloudera-labs/cloudera.exe/pull/64) + - Fix AWS network creation error when no tags are defined (https://github.com/cloudera-labs/cloudera.exe/pull/46) + minor_changes: + - Add Ubuntu 20.04 focal fossa as optional OS for dynamic inventory (https://github.com/cloudera-labs/cloudera.exe/pull/69) + - Update streams messaging default template (https://github.com/cloudera-labs/cloudera.exe/pull/65) + - Add network discovery and assignment functions (https://github.com/cloudera-labs/cloudera.exe/pull/62) + - Add GCP support to FreeIPA host group role (https://github.com/cloudera-labs/cloudera.exe/pull/61) + - Add support for CDE (part 2 - virtual clusters) (https://github.com/cloudera-labs/cloudera.exe/pull/60) + - Molecule test harness for platform role (https://github.com/cloudera-labs/cloudera.exe/pull/59) + - Add support for CDE (https://github.com/cloudera-labs/cloudera.exe/pull/58) + - Add role, policy, and storage tagging to AWS (https://github.com/cloudera-labs/cloudera.exe/pull/55) + - AWS VPC Endpoint Support (https://github.com/cloudera-labs/cloudera.exe/pull/54) + - Add selectable distribution support for cloudera.cluster (https://github.com/cloudera-labs/cloudera.exe/pull/51) + - Enhancement to sudoers role to add groups and work with user sync (https://github.com/cloudera-labs/cloudera.exe/pull/50) + - Extensible tagging for Cloudera Experiences (https://github.com/cloudera-labs/cloudera.exe/pull/48) + - Move DFX Beta implementation to GA process (https://github.com/cloudera-labs/cloudera.exe/pull/47) + - Allow optional deletion of GCP Custom roles during teardown (https://github.com/cloudera-labs/cloudera.exe/pull/44) + release_date: '2021-11-29' + 1.4.2: + release_date: '2021-09-22' + 1.4.1: + release_date: '2021-09-22' + 1.4.0: + changes: + bugfixes: + - Fix L1 networking teardown when purge is used (https://github.com/cloudera-labs/cloudera.exe/pull/43) + - Correcting Idbroker Role policy definitions for AWS (https://github.com/cloudera-labs/cloudera.exe/pull/41) + - Correct references to AWS policy documents (https://github.com/cloudera-labs/cloudera.exe/pull/30) + - Fix default opdb teardown (https://github.com/cloudera-labs/cloudera.exe/pull/22) + - Fix unused DWX variable and more accurate datahub definition filters (https://github.com/cloudera-labs/cloudera.exe/pull/19) + minor_changes: + - Remove initialize tasks in sudoers role (https://github.com/cloudera-labs/cloudera.exe/pull/42) + - Update ML Workspace setup to use definition of a single instance group (https://github.com/cloudera-labs/cloudera.exe/pull/40) + - Improve GCP teardown idempotence (https://github.com/cloudera-labs/cloudera.exe/pull/39) + - Update env setup to include passing freeipa instance count. Add some … (https://github.com/cloudera-labs/cloudera.exe/pull/38) + - Improve purge functionality with further edge cases (https://github.com/cloudera-labs/cloudera.exe/pull/35) + - Improve Azure deployment stability (https://github.com/cloudera-labs/cloudera.exe/pull/34) + - Ciao dynamo (https://github.com/cloudera-labs/cloudera.exe/pull/33) + - AWS Level 2 networking (including shared resources) (https://github.com/cloudera-labs/cloudera.exe/pull/32) + - Improve network security port determination logic (https://github.com/cloudera-labs/cloudera.exe/pull/29) + - Add Centos8 to Dynamic Inventory options (https://github.com/cloudera-labs/cloudera.exe/pull/25) + - Improve teardown and support purge mode, other minor fixes (https://github.com/cloudera-labs/cloudera.exe/pull/24) + - Changes for DF-beta (https://github.com/cloudera-labs/cloudera.exe/pull/20) + - Update Azure Teardown - Currently broken (https://github.com/cloudera-labs/cloudera.exe/pull/18) + - Support Private Networks (https://github.com/cloudera-labs/cloudera.exe/pull/15) + release_date: '2021-09-11' + 1.3.1: + release_date: '2021-06-15' + 1.3.0: + changes: + bugfixes: + - Reopening PR after revert on Cloudera Labs (https://github.com/cloudera-labs/cloudera.exe/pull/16) + minor_changes: + - Add support for DFX Tech Preview (https://github.com/cloudera-labs/cloudera.exe/pull/12) + release_date: '2021-06-15' + 1.2.1: + release_date: '2021-06-11' + 1.2.0: + changes: + minor_changes: + - Remove extraneous user_ports from Extra security group (https://github.com/cloudera-labs/cloudera.exe/pull/14) + - Improve Azure Storage Account name check to be more informative (https://github.com/cloudera-labs/cloudera.exe/pull/13) + - Add tasks for retrieving datahub definitions and filtering by datalak… (https://github.com/cloudera-labs/cloudera.exe/pull/9) + - New Roles to facilitate creation of FreeIPA sudoers group and rule (https://github.com/cloudera-labs/cloudera.exe/pull/6) + objects: + role: + - description: Create FreeIPA host group. + name: freeipa_host_group + namespace: + - description: Create sudo rule. + name: sudoers + namespace: + release_date: '2021-06-10' + 1.1.2: + release_date: '2021-06-03' + 1.1.1: + release_date: '2021-06-02' + 1.1.0: + release_date: '2021-05-26' + 1.0.0: + objects: + role: + - description: Shared configuration variables. + name: common + namespace: + - description: Manage external data locations. + name: data + namespace: + - description: Retrieve CDP details. + name: info + namespace: + - description: Deploy cloud provider infrastructure. + name: infrastructure + namespace: + - description: Execute run-level operations for CDP Public Cloud. + name: infrastructure + namespace: + - description: Deploy CDP Public Cloud core services. + name: platform + namespace: + - description: Deploy CDP Public Cloud Experiences. + name: runtime + namespace: + release_date: '2021-05-03' diff --git a/changelogs/config.yaml b/changelogs/config.yaml new file mode 100644 index 00000000..db9bef5d --- /dev/null +++ b/changelogs/config.yaml @@ -0,0 +1,37 @@ +add_plugin_period: true +changelog_nice_yaml: false +changelog_sort: alphanumerical +changes_file: changelog.yaml +changes_format: combined +ignore_other_fragment_extensions: true +keep_fragments: false +mention_ancestor: true +new_plugins_after_name: removed_features +notesdir: fragments +output: + - file: CHANGELOG.rst + format: rst +prelude_section_name: release_summary +prelude_section_title: Release Summary +sanitize_changelog: true +sections: + - - major_changes + - Major Changes + - - minor_changes + - Minor Changes + - - breaking_changes + - Breaking Changes / Porting Guide + - - deprecated_features + - Deprecated Features + - - removed_features + - Removed Features (previously deprecated) + - - security_fixes + - Security Fixes + - - bugfixes + - Bugfixes + - - known_issues + - Known Issues +title: Cloudera.Cluster +trivial_section_name: trivial +use_fqcn: true +vcs: auto diff --git a/docs/configuration.yml b/docs/configuration.yml index 958a44fd..8629b0a1 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -81,7 +81,7 @@ df: parameters: - name: value: - assetReferences: [str, ...] + assetReferences: [str, "..."] kpis: - metricId: componentId: @@ -505,5 +505,5 @@ data: aws: suffix: teardown: - delete_policies: - delete_roles: + delete_policies: + delete_roles: diff --git a/docs/links.yml b/docs/links.yml index 6a5373cd..fb665e1a 100644 --- a/docs/links.yml +++ b/docs/links.yml @@ -3,7 +3,7 @@ edit_on_github: repository: cloudera-labs/cloudera.exe branch: main - path_prefix: '' + path_prefix: "" extra_links: - description: Submit a bug report diff --git a/extensions/molecule/default/converge.yml b/extensions/molecule/default/converge.yml index 4ebf68fe..424bf1ae 100644 --- a/extensions/molecule/default/converge.yml +++ b/extensions/molecule/default/converge.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +15,7 @@ - name: Default test hosts: all - gather_facts: yes + gather_facts: true tasks: - name: Heartbeat ansible.builtin.ping: diff --git a/extensions/molecule/default/molecule.yml b/extensions/molecule/default/molecule.yml index 2e704f0f..be8f9d61 100644 --- a/extensions/molecule/default/molecule.yml +++ b/extensions/molecule/default/molecule.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/extensions/molecule/default/requirements.yml b/extensions/molecule/default/requirements.yml index 48342277..963e90f2 100644 --- a/extensions/molecule/default/requirements.yml +++ b/extensions/molecule/default/requirements.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/extensions/molecule/rdbms_server_postgresql_14/molecule.yml b/extensions/molecule/rdbms_server_postgresql_14/molecule.yml index b8045b50..53471e6e 100644 --- a/extensions/molecule/rdbms_server_postgresql_14/molecule.yml +++ b/extensions/molecule/rdbms_server_postgresql_14/molecule.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/extensions/molecule/rdbms_server_postgresql_14/requirements.yml b/extensions/molecule/rdbms_server_postgresql_14/requirements.yml index b5f72813..2920ccd1 100644 --- a/extensions/molecule/rdbms_server_postgresql_14/requirements.yml +++ b/extensions/molecule/rdbms_server_postgresql_14/requirements.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +14,8 @@ # limitations under the License. collections: - - containers.podman - - community.postgresql + - name: containers.podman + - name: community.postgresql roles: - - geerlingguy.postgresql + - name: geerlingguy.postgresql diff --git a/extensions/molecule/rdbms_server_postgresql_14_tls/cleanup.yml b/extensions/molecule/rdbms_server_postgresql_14_tls/cleanup.yml index 2624b09a..3d2d0191 100644 --- a/extensions/molecule/rdbms_server_postgresql_14_tls/cleanup.yml +++ b/extensions/molecule/rdbms_server_postgresql_14_tls/cleanup.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +16,7 @@ - name: Remove CA certificate hosts: localhost connection: local - gather_facts: no + gather_facts: false tasks: - name: Remove CA private key ansible.builtin.file: diff --git a/extensions/molecule/rdbms_server_postgresql_14_tls/prepare.yml b/extensions/molecule/rdbms_server_postgresql_14_tls/prepare.yml index ea317ad0..ed09c2b4 100644 --- a/extensions/molecule/rdbms_server_postgresql_14_tls/prepare.yml +++ b/extensions/molecule/rdbms_server_postgresql_14_tls/prepare.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +20,7 @@ - name: Create CA private key community.crypto.openssl_privatekey: path: ./ca-certificate.key - mode: 0600 + mode: "0600" - name: Create CSR for CA certificate community.crypto.openssl_csr_pipe: @@ -27,7 +28,7 @@ common_name: Molecule CA use_common_name_for_san: false basic_constraints: - - 'CA:TRUE' + - "CA:TRUE" basic_constraints_critical: true key_usage: - keyCertSign @@ -40,11 +41,11 @@ csr_content: "{{ __ca_csr.csr }}" privatekey_path: ca-certificate.key provider: selfsigned - mode: 0644 + mode: "0644" - name: Set up TLS certificates hosts: all - gather_facts: yes + gather_facts: true tasks: - name: Update pip ansible.builtin.pip: @@ -77,20 +78,20 @@ ansible.builtin.file: path: /opt/security/pki state: directory - mode: 0755 + mode: "0755" - name: Copy CA certificate from controller ansible.builtin.copy: src: ca-certificate.pem dest: /opt/security/pki/ca-certificate.pem - mode: 0644 + mode: "0644" - name: Create host private key community.crypto.openssl_privatekey: path: /opt/security/pki/host.key - return_content: yes + return_content: true group: postgres - mode: 0640 + mode: "0640" register: __key - name: Write unencrypted host private key @@ -98,7 +99,7 @@ dest: /opt/security/pki/host.key.unenc content: "{{ __key.privatekey }}" group: postgres - mode: 0640 + mode: "0640" - name: Create CSR for host certificate community.crypto.openssl_csr_pipe: @@ -121,5 +122,5 @@ ansible.builtin.copy: dest: /opt/security/pki/host.pem content: "{{ __cert.certificate }}" - mode: 0640 + mode: "0640" group: postgres diff --git a/extensions/molecule/rdbms_server_postgresql_14_tls/requirements.yml b/extensions/molecule/rdbms_server_postgresql_14_tls/requirements.yml index b5f72813..2920ccd1 100644 --- a/extensions/molecule/rdbms_server_postgresql_14_tls/requirements.yml +++ b/extensions/molecule/rdbms_server_postgresql_14_tls/requirements.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +14,8 @@ # limitations under the License. collections: - - containers.podman - - community.postgresql + - name: containers.podman + - name: community.postgresql roles: - - geerlingguy.postgresql + - name: geerlingguy.postgresql diff --git a/extensions/molecule/rdbms_server_postgresql_14_tls/verify.yml b/extensions/molecule/rdbms_server_postgresql_14_tls/verify.yml index f6fb87c0..eb1c3420 100644 --- a/extensions/molecule/rdbms_server_postgresql_14_tls/verify.yml +++ b/extensions/molecule/rdbms_server_postgresql_14_tls/verify.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,18 +15,18 @@ - name: Confirm local database access hosts: all - gather_facts: no + gather_facts: false tasks: - name: Ping database locally community.postgresql.postgresql_ping: - become: yes + become: true become_user: postgres register: postgres_user failed_when: not postgres_user.is_available - name: Confirm external database access hosts: all - gather_facts: no + gather_facts: false tasks: - name: Ping database as authorized user community.postgresql.postgresql_ping: diff --git a/extensions/molecule/rdbms_server_postgresql_default/converge.yml b/extensions/molecule/rdbms_server_postgresql_default/converge.yml index 649235b4..bc6b6cf7 100644 --- a/extensions/molecule/rdbms_server_postgresql_default/converge.yml +++ b/extensions/molecule/rdbms_server_postgresql_default/converge.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/extensions/molecule/rdbms_server_postgresql_default/prepare.yml b/extensions/molecule/rdbms_server_postgresql_default/prepare.yml index 0dc02c41..f5ae6118 100644 --- a/extensions/molecule/rdbms_server_postgresql_default/prepare.yml +++ b/extensions/molecule/rdbms_server_postgresql_default/prepare.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,5 +15,5 @@ - name: Set up TLS certificates hosts: all - gather_facts: yes + gather_facts: true tasks: diff --git a/extensions/molecule/rdbms_server_postgresql_default/requirements.yml b/extensions/molecule/rdbms_server_postgresql_default/requirements.yml index 74f1f387..3e818cb1 100644 --- a/extensions/molecule/rdbms_server_postgresql_default/requirements.yml +++ b/extensions/molecule/rdbms_server_postgresql_default/requirements.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +14,7 @@ # limitations under the License. collections: - - containers.podman - - community.postgresql + - name: containers.podman + - name: community.postgresql roles: - - geerlingguy.postgresql + - name: geerlingguy.postgresql diff --git a/extensions/molecule/rdbms_server_postgresql_default/side_effect.yml b/extensions/molecule/rdbms_server_postgresql_default/side_effect.yml index c4772ba4..8300df0c 100644 --- a/extensions/molecule/rdbms_server_postgresql_default/side_effect.yml +++ b/extensions/molecule/rdbms_server_postgresql_default/side_effect.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +15,13 @@ - name: Add test users to PostgreSQL hosts: all - gather_facts: yes - become: yes + gather_facts: true + become: true tasks: - name: Create test database community.postgresql.postgresql_db: name: test + become: true become_user: postgres - name: Add authorized user @@ -28,6 +30,7 @@ name: user_one password: authorized comment: Authorized User + become: true become_user: postgres - name: Add non-authorized user @@ -36,4 +39,5 @@ name: user_two password: "" comment: Unauthorized User + become: true become_user: postgres diff --git a/extensions/molecule/rdbms_server_postgresql_default/verify.yml b/extensions/molecule/rdbms_server_postgresql_default/verify.yml index 9703ea0a..1fa13c51 100644 --- a/extensions/molecule/rdbms_server_postgresql_default/verify.yml +++ b/extensions/molecule/rdbms_server_postgresql_default/verify.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,18 +23,18 @@ - name: Confirm local database access hosts: all - gather_facts: no + gather_facts: false tasks: - name: Ping database locally community.postgresql.postgresql_ping: - become: yes + become: true become_user: postgres register: postgres_user failed_when: not postgres_user.is_available - name: Confirm external database access hosts: all - gather_facts: no + gather_facts: false tasks: - name: Ping database as authorized user community.postgresql.postgresql_ping: diff --git a/galaxy.yml b/galaxy.yml index 6e81a780..04bde9c8 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,6 +1,4 @@ ---- - -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -namespace: cloudera -name: exe -version: 2.4.1 -readme: README.md +--- + +namespace: cloudera +name: exe +version: 3.0.0-rc.1 +readme: README.md authors: - Jim Enright @jenright - Ronald Suplina @rsuplina @@ -25,32 +25,32 @@ authors: description: > A set of roles, modules, and plugins that encapsulate best practices and opinionated - deployment and management processes for Cloudera Data Platform (CDP) Public Cloud - and Private Cloud deployments. + deployment and management processes for Cloudera on cloud and on premise deployments. license_file: LICENSE tags: -- cloudera -- cdp -- azure -- aws -- gcp -- private_cloud -- public_cloud -- data_service -- cloudera_manager -- cm + - application + - cloud + - tools + - cloudera + - cdp + - cdh + - private_cloud + - on_premise + - data_services + - cloudera_manager + - cm dependencies: - 'ansible.netcommon': '2.5.1' - 'community.general': '4.5.0' + "ansible.netcommon": "2.5.1" + "community.general": "4.5.0" -repository: https://github.com/cloudera-labs/cloudera.exe -homepage: https://github.com/cloudera-labs/cloudera.exe -issues: https://github.com/cloudera-labs/cloudera.exe/issues -documentation: https://cloudera-labs.github.io/cloudera.exe +repository: https://github.com/cloudera-labs/cloudera.exe +homepage: https://github.com/cloudera-labs/cloudera.exe +issues: https://github.com/cloudera-labs/cloudera.exe/issues +documentation: https://cloudera-labs.github.io/cloudera.exe build_ignore: -- '.*' -- docs -- docsrc -- site + - ".*" + - docs + - docsrc + - site diff --git a/meta/runtime.yml b/meta/runtime.yml index 87e89dc5..0c3577cd 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -1,6 +1,4 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -requires_ansible: ">=2.10" +--- + +requires_ansible: ">=2.15.0" plugin_routing: role: diff --git a/playbooks/pbc_infra_setup.yml b/playbooks/pbc_infra_setup.yml index a5e1bc57..4a3f25a1 100644 --- a/playbooks/pbc_infra_setup.yml +++ b/playbooks/pbc_infra_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set up CDP Public Cloud infrastructure (Ansible-based) hosts: "{{ target | default('localhost') }}" environment: "{{ globals.env_vars }}" - gather_facts: yes + gather_facts: true tasks: - name: Validate CDP Public Cloud infrastructure configuration ansible.builtin.import_role: diff --git a/playbooks/pbc_infra_teardown.yml b/playbooks/pbc_infra_teardown.yml index 7ad9f610..74f0a1b3 100644 --- a/playbooks/pbc_infra_teardown.yml +++ b/playbooks/pbc_infra_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Tear down CDP Public Cloud infrastructure (Ansible-based) hosts: "{{ target | default('localhost') }}" environment: "{{ globals.env_vars }}" - gather_facts: yes + gather_facts: true tasks: - name: Validate CDP Public Cloud infrastructure configuration ansible.builtin.import_role: diff --git a/playbooks/pbc_setup.yml b/playbooks/pbc_setup.yml index 803c9a21..aabfeac5 100644 --- a/playbooks/pbc_setup.yml +++ b/playbooks/pbc_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +16,8 @@ - name: Set up CDP Public Cloud hosts: "{{ target | default('localhost') }}" environment: "{{ globals.env_vars }}" - gather_facts: yes + gather_facts: true tasks: - # TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, # including those that would otherwise be 'always' - in this context; 'always' # should be reserved for the initialization of cloudera-deploy diff --git a/playbooks/pbc_teardown.yml b/playbooks/pbc_teardown.yml index 159e3703..74ed9834 100644 --- a/playbooks/pbc_teardown.yml +++ b/playbooks/pbc_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +16,8 @@ - name: Tear down CDP Public Cloud hosts: "{{ target | default('localhost') }}" environment: "{{ globals.env_vars }}" - gather_facts: yes + gather_facts: true tasks: - # TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, # including those that would otherwise be 'always' - in this context; 'always' # should be reserved for the initialization of cloudera-deploy diff --git a/playbooks/pvc_base_postfix.yml b/playbooks/pvc_base_postfix.yml index 1cd63dc1..2efb0c20 100644 --- a/playbooks/pvc_base_postfix.yml +++ b/playbooks/pvc_base_postfix.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ # STARTBLOCK # Fix Auto-TLS - name: Auto-TLS Services Setup hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.cms_tls when: autotls is defined and autotls == True @@ -29,12 +28,12 @@ # STARTBLOCK # Setup HDFS Encryption - name: Setup KTS HA hosts: "{{ target | default('localhost') }}" - become: yes - gather_facts: no + become: true + gather_facts: false tasks: - ansible.builtin.include_role: name: cloudera.cluster.deployment.services.kts_high_availability - public: yes + public: true apply: tags: - kts @@ -48,8 +47,8 @@ - name: Handle KMS services hosts: "{{ target | default('localhost') }}" - gather_facts: no - become: yes + gather_facts: false + become: true roles: - role: cloudera.cluster.deployment.services.kms when: "'kms_servers' in groups" @@ -61,8 +60,8 @@ - name: Handle KMS services hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no - become: no + gather_facts: false + become: false roles: - role: cloudera.cluster.operations.refresh_ranger_kms_repo when: "'kms_servers' in groups" @@ -72,7 +71,7 @@ - name: Restart and re-deploy stale client configs hosts: "{{ target | default('localhost') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.operations.restart_stale when: "'kms_servers' in groups" @@ -82,12 +81,12 @@ - kms - restart_stale - full_cluster - # ENDBLOCK # Setup HDFS Encryption +# ENDBLOCK # Setup HDFS Encryption - # STARTBLOCK # WXM Setup +# STARTBLOCK # WXM Setup - name: Handle WXM Setup hosts: "{{ target | default('cluster_master_nodes[0]') }}" - gather_facts: yes + gather_facts: true tags: - wxm - full_cluster @@ -97,43 +96,43 @@ - use_wxm | default(False) - altus_key_id | length > 0 - altus_private_key | length > 0 - import_role: + ansible.builtin.import_role: name: cloudera.cluster.deployment.services.wxm # ENDBLOCK # WXM Setup - name: Post-Install for PvC on all cluster hosts hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - gather_facts: yes + gather_facts: true tags: - pvc tasks: - name: Add missing ExtJS for Oozie UI - include_role: + ansible.builtin.include_role: name: cloudera.cluster.config.services.oozie_ui when: oozie_service_exists | default(false) - name: Post-Install Cloudera Manager and Cluster hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: yes + gather_facts: true tags: - pvc tasks: - name: Refresh CM Services Info - include_role: + ansible.builtin.include_role: name: cloudera.cluster.cloudera_manager.services_info - public: yes + public: true vars: cluster_name: "{{ cluster_name_base }}" - name: Fix Hue ticket lifetime for Free IPA - include_role: + ansible.builtin.include_role: name: cloudera.cluster.config.services.hue_ticket_lifetime when: - hue_service_exists | default(false) - krb5_kdc_type == 'Red Hat IPA' - name: Set Cloudera Manager session timeout to 30d - include_role: + ansible.builtin.include_role: name: cloudera.cluster.cloudera_manager.session_timeout ## TODO Believe this is version specific, in 7.1.7Spx solr plugin is NOT missing, needs work @@ -145,7 +144,7 @@ # - solr_service_exists | default(false) - name: Add Solr urls to Knox - include_role: + ansible.builtin.include_role: name: cloudera.cluster.config.services.solr_knox when: knox_service_exists | default(false) and solr_service_exists | default(false) @@ -153,7 +152,7 @@ when: - kms_service_exists | default(False) - (autotls | default(False)) - include_role: + ansible.builtin.include_role: name: cloudera.cluster.config.services.kms_tls ## TODO Believe this is version specific, in 7.1.7SPx Ranger policies are duplicated, needs work diff --git a/playbooks/pvc_base_prereqs_ext.yml b/playbooks/pvc_base_prereqs_ext.yml index 4051c468..7522821e 100644 --- a/playbooks/pvc_base_prereqs_ext.yml +++ b/playbooks/pvc_base_prereqs_ext.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ # STARTBLOCK # Verify Inventory and Definition - name: Verify inventory [verify_inventory] hosts: "{{ target | default('localhost') }}" - gather_facts: no + gather_facts: false roles: - cloudera.cluster.verify.inventory tags: @@ -28,7 +27,7 @@ - name: Verify definition [verify_definition] hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: yes + gather_facts: true roles: - cloudera.cluster.verify.definition tags: @@ -40,7 +39,7 @@ # Moved before parcel verification to allow rehosting - name: Install custom parcel repository hosts: "{{ target | default('custom_repo') }}" - become: yes + become: true roles: - cloudera.cluster.infrastructure.custom_repo tags: @@ -51,7 +50,7 @@ # Moved from verify_parcels to reduce duplication - name: Verify definition [verify_parcels_and_roles] hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: yes + gather_facts: true roles: - cloudera.cluster.verify.parcels_and_roles tags: @@ -64,7 +63,7 @@ # STARTBLOCK # Prepare Nodes - name: Apply OS pre-requisite configurations hosts: "{{ target | default('cloudera_manager, cluster, ca_server, ecs_nodes') }}" - become: yes + become: true roles: - cloudera.cluster.prereqs.os tags: @@ -74,8 +73,8 @@ - name: Apply OS Prereqs to ECS Nodes hosts: "{{ target | default('ecs_nodes') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true tags: - pvc - os @@ -88,8 +87,8 @@ - name: Create local user accounts hosts: "{{ target | default('cloudera_manager, cluster') }}" - become: yes - gather_facts: no + become: true + gather_facts: false roles: - cloudera.cluster.prereqs.user_accounts tags: @@ -99,8 +98,8 @@ - name: Create local users on ECS Nodes hosts: "{{ target | default('ecs_nodes') }}" - become: yes - gather_facts: yes + become: true + gather_facts: true tags: - pvc - users @@ -113,7 +112,7 @@ - name: Install JDK hosts: "{{ target | default('cloudera_manager, cluster, tls, krb5_server, ecs_nodes') }}" - become: yes + become: true roles: - cloudera.cluster.prereqs.jdk tags: @@ -128,8 +127,8 @@ # DB Connectors - name: Install MySQL Connector hosts: "{{ target | default('cloudera_manager, cluster') }}" - gather_facts: no - become: yes + gather_facts: false + become: true roles: - role: cloudera.exe.rdbms.client.mysql_connector when: database_type == 'mysql' or database_type == 'mariadb' @@ -139,8 +138,8 @@ - name: Install Oracle Connector hosts: "{{ target | default('cloudera_manager, cluster') }}" - gather_facts: no - become: yes + gather_facts: false + become: true roles: - role: cloudera.exe.rdbms.client.oracle_connector when: database_type == 'oracle' @@ -150,8 +149,8 @@ - name: Install PostgreSQL Connector hosts: cloudera_manager, cluster - gather_facts: no - become: yes + gather_facts: false + become: true roles: - role: cloudera.exe.rdbms.client.postgresql_connector when: database_type == 'postgresql' @@ -163,8 +162,8 @@ # STARTBLOCK # Create Cluster Service Infrastructure - name: Install Kerberos Server hosts: "{{ target | default('krb5_server') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true roles: - role: cloudera.cluster.infrastructure.krb5_server tags: @@ -176,8 +175,8 @@ - name: Setup KRB5 clients hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true roles: - role: cloudera.cluster.infrastructure.krb5_client when: @@ -192,7 +191,7 @@ - name: Install CA server hosts: "{{ target | default('ca_server') }}" - become: yes + become: true roles: - cloudera.cluster.infrastructure.ca_server tags: @@ -202,7 +201,7 @@ - name: Install HAProxy hosts: "{{ target | default('haproxy') }}" - become: yes + become: true roles: - cloudera.cluster.infrastructure.haproxy tags: @@ -214,8 +213,8 @@ - name: Build TLS keystores and truststores hosts: "{{ target | default('tls') }}" - become: yes - gather_facts: no + become: true + gather_facts: false roles: - cloudera.cluster.security.tls_generate_csr - cloudera.cluster.security.tls_signing @@ -229,10 +228,10 @@ - name: Delete temp directories hosts: "{{ target | default('localhost') }}" - become: no - gather_facts: no + become: false + gather_facts: false tasks: - - file: + - ansible.builtin.file: path: "{{ [local_temp_dir, dir] | path_join }}" state: absent loop: @@ -247,8 +246,8 @@ # STARTBLOCK # NiFi TLS - name: Setup symlinks for NiFi TLS keystore and truststore hosts: "{{ target | default('cluster') }}" - become: yes - gather_facts: no + become: true + gather_facts: false roles: - role: cloudera.cluster.security.tls_nifi when: > @@ -260,7 +259,7 @@ # STARTBLOCK # Install Cluster Service Infrastructure II - name: Install RDBMS hosts: "{{ target | default('db_server') }}" - become: yes + become: true roles: - cloudera.exe.rdbms.server tags: diff --git a/playbooks/pvc_base_prereqs_int.yml b/playbooks/pvc_base_prereqs_int.yml index c6e3ee20..37540245 100644 --- a/playbooks/pvc_base_prereqs_int.yml +++ b/playbooks/pvc_base_prereqs_int.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ # STARTBLOCK # Install Cloudera Manager - name: Install Cloudera Manager daemons hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - become: yes + become: true any_errors_fatal: true roles: - role: cloudera.cluster.cloudera_manager.daemons @@ -28,7 +27,7 @@ - name: Install Cloudera Manager server hosts: "{{ target | default('cloudera_manager') }}" - become: yes + become: true roles: - role: cloudera.cluster.cloudera_manager.server tags: @@ -38,7 +37,7 @@ - name: Install Cloudera Manager License hosts: "{{ target | default('cloudera_manager') }}" - become: yes + become: true roles: - role: cloudera.cluster.cloudera_manager.license tags: @@ -49,7 +48,7 @@ - name: Install Cloudera Manager agents hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - become: yes + become: true any_errors_fatal: true roles: - role: cloudera.cluster.cloudera_manager.agent @@ -60,8 +59,8 @@ - name: Configure Cloudera Manager server for TLS hosts: "{{ target | default('cloudera_manager') }}" - become: yes - gather_facts: no + become: true + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.server_tls when: tls | default(False) or manual_tls_cert_distribution | default(False) @@ -72,7 +71,7 @@ - name: Configure Cloudera Manager agents hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - become: yes + become: true any_errors_fatal: true roles: - cloudera.cluster.cloudera_manager.agent_config @@ -83,7 +82,7 @@ - name: Configure Cloudera Manager server hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - cloudera.cluster.cloudera_manager.config tags: @@ -93,7 +92,7 @@ - name: Configure Cloudera Manager auth and accounts hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.external_auth - role: cloudera.cluster.cloudera_manager.external_account @@ -105,7 +104,7 @@ # STARTBLOCK # Cloudera Manager Password - name: Configure Cloudera Manager Password hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.admin_password.set tags: @@ -115,7 +114,7 @@ - name: Check Cloudera Manager admin password hosts: "{{ target | default('cloudera_manager, cluster') }}" - gather_facts: no + gather_facts: false roles: - cloudera.cluster.cloudera_manager.api_client tags: @@ -127,7 +126,7 @@ # STARTBLOCK # Prepare Security - name: Enable Auto-TLS hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.autotls when: autotls is defined and autotls == True @@ -137,7 +136,7 @@ - name: Install prerequisite packages for Kerberos hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - become: yes + become: true roles: - role: cloudera.cluster.prereqs.kerberos when: @@ -150,7 +149,7 @@ - name: Configure Cloudera Manager server for Kerberos hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.kerberos when: krb5_kdc_host is defined or 'krb5_server' in groups @@ -162,11 +161,11 @@ # STARTBLOCK # Configure CM - name: Restart Cloudera Manager Agents hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - gather_facts: no - become: yes + gather_facts: false + become: true tasks: - name: Restart Cloudera Manager Agents - meta: noop + ansible.builtin.meta: noop notify: - restart cloudera-scm-agent tags: @@ -175,8 +174,8 @@ - name: Ensure that the agents are heartbeating hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" - gather_facts: yes - any_errors_fatal: yes + gather_facts: true + any_errors_fatal: true roles: - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat when: cloudera_manager_agent_wait_for_heartbeat | default(True) @@ -187,7 +186,7 @@ - name: Deploy Cloudera Management Service hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: no + gather_facts: false roles: - cloudera.cluster.deployment.services.mgmt tags: @@ -197,8 +196,8 @@ - name: Preload parcels from custom repo to Cloudera Manager hosts: "{{ target | default('cloudera_manager') }}" - become: yes - gather_facts: no + become: true + gather_facts: false roles: - role: cloudera.cluster.cloudera_manager.preload_parcels when: "'custom_repo' in groups" diff --git a/playbooks/pvc_base_setup.yml b/playbooks/pvc_base_setup.yml index af646c2e..8d725fb8 100644 --- a/playbooks/pvc_base_setup.yml +++ b/playbooks/pvc_base_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ # STARTBLOCK # Install Cluster - name: Deploy clusters hosts: "{{ target | default('cloudera_manager') }}" - gather_facts: yes + gather_facts: true roles: - cloudera.cluster.deployment.cluster tags: diff --git a/playbooks/pvc_base_teardown.yml b/playbooks/pvc_base_teardown.yml index ecc55c72..a1ee3b8f 100644 --- a/playbooks/pvc_base_teardown.yml +++ b/playbooks/pvc_base_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,8 +17,8 @@ # Teardown CA - name: Teardown CA server hosts: "{{ target | default('ca_server') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true tasks: - ansible.builtin.include_role: name: cloudera.cluster.infrastructure.ca_certs @@ -29,8 +28,8 @@ - name: Teardown security artifact directories hosts: "{{ target | default('tls') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true roles: - role: cloudera.cluster.security.tls_clean when: "'tls' in groups" @@ -40,8 +39,8 @@ # Teardown Cluster - name: Teardown ECS hosts: "{{ target | default('ecs_nodes') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true tasks: - ansible.builtin.include_role: name: cloudera.cluster.teardown @@ -53,8 +52,8 @@ - name: Cluster Teardown Process hosts: "{{ target | default('all') }}" - gather_facts: yes - become: yes + gather_facts: true + become: true any_errors_fatal: true roles: - cloudera.cluster.teardown diff --git a/playbooks/pvc_renew_certs.yml b/playbooks/pvc_renew_certs.yml index fe1476d6..619f89ee 100644 --- a/playbooks/pvc_renew_certs.yml +++ b/playbooks/pvc_renew_certs.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +27,7 @@ ansible.builtin.file: path: "{{ [__pvc_tls_tempdir.path, item] | path_join }}" state: directory - mode: '0755' + mode: "0755" loop: - csrs - certs @@ -36,15 +35,14 @@ - name: Play 1 - Generate CSR on each host hosts: "{{ target | default('cluster') }}" - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - - name: Call tls_generate_csr role - ansible.builtin.import_role: - name: cloudera.exe.tls_generate_csr - vars: - local_csrs_dir: "{{ (hostvars['localhost']['__pvc_tls_tempdir']['path'], 'csrs') | path_join }}" + - name: Call tls_generate_csr role + ansible.builtin.import_role: + name: cloudera.exe.tls_generate_csr + vars: + local_csrs_dir: "{{ (hostvars['localhost']['__pvc_tls_tempdir']['path'], 'csrs') | path_join }}" - name: Get the list of CSRs to sign hosts: localhost @@ -64,10 +62,9 @@ - name: Play 2 - Sign the CSR hosts: ca_server - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - name: Call tls_signing role ansible.builtin.import_role: name: cloudera.exe.tls_signing @@ -78,51 +75,47 @@ - name: Play 3 - Install the sign certs on each host hosts: "{{ target | default('cluster') }}" - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - - name: Call tls_install_certs role - ansible.builtin.import_role: - name: cloudera.exe.tls_install_certs - vars: - local_tls_signed_certs_dir: "{{ (hostvars['localhost']['__pvc_tls_tempdir']['path'], 'certs') | path_join }}" + - name: Call tls_install_certs role + ansible.builtin.import_role: + name: cloudera.exe.tls_install_certs + vars: + local_tls_signed_certs_dir: "{{ (hostvars['localhost']['__pvc_tls_tempdir']['path'], 'certs') | path_join }}" - name: Post 1 - Restart CM Server service hosts: cloudera_manager - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - - name: Restart CM Server service - when: - - restart_services | default(False) - ansible.builtin.service: - name: cloudera-scm-server - state: restarted + - name: Restart CM Server service + when: + - restart_services | default(False) + ansible.builtin.service: + name: cloudera-scm-server + state: restarted - name: Post 2 - Restart DB Server service hosts: db_server - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - - name: Restart DB Server service - when: - - restart_services | default(False) - ansible.builtin.service: - name: "{{ db_service_name }}" - state: reloaded + - name: Restart DB Server service + when: + - restart_services | default(False) + ansible.builtin.service: + name: "{{ db_service_name }}" + state: reloaded - name: Post 3 - Restart CM Agent service hosts: cluster - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - - - name: Restart CM Agent service - when: - - restart_services | default(False) - ansible.builtin.service: - name: cloudera-scm-agent - state: restarted + - name: Restart CM Agent service + when: + - restart_services | default(False) + ansible.builtin.service: + name: cloudera-scm-agent + state: restarted diff --git a/plugins/filter/core_exe.py b/plugins/filter/core_exe.py index 4e8b29f4..94aa4258 100644 --- a/plugins/filter/core_exe.py +++ b/plugins/filter/core_exe.py @@ -20,59 +20,60 @@ __metaclass__ = type DOCUMENTATION = """ - name: combine_onto - author: Webster Mudge (@wmudge) - short_description: combine two dictionaries - description: - - Create a dictionary (hash/associative array) as a result of merging existing dictionaries. - - This is the reverse of the C(ansible.builtin.combine) filter. - positional: _input, _dicts - options: - _input: - description: - - First dictionary to combine. - type: dict - required: True - _dicts: - description: - - The list of dictionaries to combine - type: list - elements: dict - required: True - recursive: - description: - - If V(True), merge elements recursively. - type: boolean - default: False - list_merge: - description: Behavior when encountering list elements. - type: str - default: replace - choices: - replace: overwrite older entries with newer ones - keep: discard newer entries - append: append newer entries to the older ones - prepend: insert newer entries in front of the older ones - append_rp: append newer entries to the older ones, overwrite duplicates - prepend_rp: insert newer entries in front of the older ones, discard duplicates +name: combine_onto +author: Webster Mudge (@wmudge) +short_description: combine two dictionaries +description: + - Create a dictionary (hash/associative array) as a result of merging existing dictionaries. + - This is the reverse of the C(ansible.builtin.combine) filter. +version_added: 1.0.0 +positional: _input, _dicts +options: + _input: + description: + - First dictionary to combine. + type: dict + required: True + _dicts: + description: + - The list of dictionaries to combine + type: list + elements: dict + required: True + recursive: + description: + - If V(True), merge elements recursively. + type: boolean + default: False + list_merge: + description: Behavior when encountering list elements. + type: str + default: replace + choices: + replace: overwrite older entries with newer ones + keep: discard newer entries + append: append newer entries to the older ones + prepend: insert newer entries in front of the older ones + append_rp: append newer entries to the older ones, overwrite duplicates + prepend_rp: insert newer entries in front of the older ones, discard duplicates """ EXAMPLES = """ - # ab => {'a':1, 'b':2, 'c': 4} - ab: {{ {'a':1, 'b':2} | cloudera.exe.combine_onto({'b':3, 'c':4}) }} +# ab => {'a':1, 'b':2, 'c': 4} +ab: "{{ {'a':1, 'b':2} | cloudera.exe.combine_onto({'b':3, 'c':4}) }}" - many: "{{ dict1 | cloudera.exe.combine_onto(dict2, dict3, dict4) }}" +many: "{{ dict1 | cloudera.exe.combine_onto(dict2, dict3, dict4) }}" - # defaults => {'a':{'b':3, 'c':4}, 'd': 5} - # customization => {'a':{'c':20}} - # final => {'a':{'b':3, 'c':20}, 'd': 5} - final: "{{ customization | cloudera.exe.combine_onto(defaults, recursive=true) }}" +# defaults => {'a':{'b':3, 'c':4}, 'd': 5} +# customization => {'a':{'c':20}} +# final => {'a':{'b':3, 'c':20}, 'd': 5} +final: "{{ customization | cloudera.exe.combine_onto(defaults, recursive=true) }}" """ RETURN = """ - _value: - description: Resulting merge of supplied dictionaries. - type: dict +_value: + description: Resulting merge of supplied dictionaries. + type: dict """ from ansible.errors import AnsibleFilterError @@ -89,7 +90,7 @@ def combine_onto(*terms, **kwargs): list_merge = kwargs.pop("list_merge", "replace") if kwargs: raise AnsibleFilterError( - "'recursive' and 'list_merge' are the only valid keyword arguments" + "'recursive' and 'list_merge' are the only valid keyword arguments", ) # allow the user to do `[dict1, dict2, ...] | combine` diff --git a/pyproject.toml b/pyproject.toml index 878d2118..5d452195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,82 @@ [project] name = "exe" +dynamic = ["version"] description = "cloudera.exe Ansible collection" readme = "README.md" -dependencies = [ - "ansible-core~=2.12.0" -] -requires-python = "~=3.8.0" -dynamic = ["version"] +requires-python = ">=3.8" +license = "Apache-2.0" +keywords = [] + +authors = [{ name = "Webster Mudge", email = "wmudge@cloudera.com" }] +classifiers = [] +dependencies = [] + +[tool.hatch.build.targets.wheel] +bypass-selection = true [tool.hatch.version] path = "galaxy.yml" pattern = "version:\\s+(?P[\\d\\.]+)" +scheme = "semver" +validate-bump = true [tool.hatch.envs.default] +python = "3.12" +detached = true dependencies = [ "pre-commit", "coverage[toml]", "pytest", - "pytest-cov", + "molecule", + "molecule-plugins", + "molecule-plugins[ec2]", + "ansible-core<2.17", # For RHEL 8 support + "ansible-lint", + "antsibull-docs >= 2.0.0, < 3.0.0", + "netaddr", + "psycopg2-binary", + "cryptography", + "jmespath", ] -installer = "uv" -[tool.pytest.ini_options] -testpaths = [ - "tests", +[tool.hatch.envs.default.scripts] +lint = [ + "pre-commit run -a", + "antsibull-docs lint-collection-docs --plugin-docs --validate-collection-refs=all --skip-rstcheck .", ] + +[tool.hatch.envs.docs] +python = "3.12" +detached = true +extra-dependencies = [ + "ansible-core<2.17", # For RHEL 8 support + "antsibull-docs @ git+https://github.com/cloudera-labs/antsibull-docs@cldr-docsite#egg=antsibull-docs", + "ansible-pygments", + "sphinx", + "sphinx-ansible-theme >= 0.9.0", + "antsichaut", +] + +[tool.hatch.envs.docs.scripts] +build = "docsbuild/build.sh" +changelog = [ + # Read the version in galaxy.yml via hatch itself (normalizes release candidates, etc.) + # Use 'hatch version' to manage the version, i.e. 'hatch version major,rc' + "antsibull-changelog release --version $(hatch version)", + "antsichaut --since_version=latest", + "antsibull-changelog generate", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] filterwarnings = [ "ignore:AnsibleCollectionFinder has already been configured", "ignore:'crypt' is deprecated and slated for removal in Python 3.13:DeprecationWarning", ] [build-system] -requires = ["hatchling"] +requires = [ + "hatchling", + "hatch-semver", +] build-backend = "hatchling.build" diff --git a/requirements.txt b/requirements.txt index 5ecc8933..f6747d17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ netaddr psycopg2-binary cryptography +jmespath diff --git a/requirements.yml b/requirements.yml index 374c0267..fafce2f6 100644 --- a/requirements.yml +++ b/requirements.yml @@ -22,34 +22,43 @@ roles: version: master collections: - - name: cloudera.cloud - version: 2.4.0 - - name: cloudera.cluster - version: 4.4.0 - - name: ansible.netcommon - version: 2.5.1 - - name: community.general - version: 4.5.0 + - name: https://github.com/cloudera-labs/cloudera.cloud + type: git + version: devel + - name: https://github.com/wmudge/cloudera.cluster + type: git + version: feature/galaxy-dependencies + - name: amazon.aws - version: 3.0.0 - - name: community.aws - version: 3.0.1 - - name: azure.azcollection - version: 1.11.0 - - name: netapp.azure - version: 21.10.0 - - name: google.cloud - version: 1.0.2 + version: 9.5.0 # Handles ansible<2.17.0 + + - name: ansible.netcommon + version: 8.0.1 - name: ansible.posix - version: 1.3.0 - # Is included via cloudera.cluster - # - name: community.crypto - # version: 2.2.1 + version: 2.0.0 + + - name: azure.azcollection + version: 3.6.0 + + - name: cloud.terraform + version: 4.0.0 + + - name: community.aws + version: 9.3.0 # Handles ansible<2.17.0 + - name: community.crypto + version: 2.26.3 # Handles ansible<2.17.0 + - name: community.general + version: 11.1.0 - name: community.mysql - version: 3.8.0 + version: 3.13.0 # Handles ansible<2.17.0 - name: community.postgresql - version: 3.3.0 + version: 4.0.1 # Handles ansible<2.17.0 + - name: freeipa.ansible_freeipa - version: 1.11.1 - - name: cloud.terraform - version: 3.0.0 + version: 1.14.7 + + - name: google.cloud + version: 1.6.0 + + - name: netapp.azure + version: 21.10.1 diff --git a/roles/auto_repo_mirror/defaults/main.yml b/roles/auto_repo_mirror/defaults/main.yml index 982e1aa4..05a34c23 100644 --- a/roles/auto_repo_mirror/defaults/main.yml +++ b/roles/auto_repo_mirror/defaults/main.yml @@ -1,3 +1,4 @@ -default_enable_auto_repo_mirror: no +--- +default_enable_auto_repo_mirror: false default_download_link_expiry: 3600 default_auto_repo_mirror_prefix: cache diff --git a/roles/freeipa_host_group/meta/main.yml b/roles/auto_repo_mirror/meta/argument_specs.yml similarity index 56% rename from roles/freeipa_host_group/meta/main.yml rename to roles/auto_repo_mirror/meta/argument_specs.yml index c9344213..f1a05cee 100644 --- a/roles/freeipa_host_group/meta/main.yml +++ b/roles/auto_repo_mirror/meta/argument_specs.yml @@ -12,30 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -galaxy_info: - author: Jim Enright (jenright@cloudera.com) - description: > - Creation of FreeIPA host group. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - freeipa - -dependencies: ['cloudera.exe.common'] +argument_specs: + main: + short_description: Populate repository mirror + description: Populate repository mirror. + author: Cloudera Labs + version_added: 2.0.0 diff --git a/roles/auto_repo_mirror/tasks/inject.yml b/roles/auto_repo_mirror/tasks/inject.yml index f64c42ad..5bf6d8c8 100644 --- a/roles/auto_repo_mirror/tasks/inject.yml +++ b/roles/auto_repo_mirror/tasks/inject.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,7 +45,7 @@ when: - init__auto_repo_mirror_ini_key is defined - use_auto_repo_mirror | default(default_enable_auto_repo_mirror) | bool - ignore_errors: yes + ignore_errors: true ansible.builtin.set_fact: __auto_repo_mirror_ini_entry: "{{ lookup('ini', __ini_lookup) }}" vars: @@ -68,8 +67,8 @@ - name: Inject Parcel cache entries to Repository URLs ansible.builtin.replace: name: "{{ __tmp_cluster_file }}" - regexp: '^(\s+\-\s)https://archive\.cloudera\.com(\/.+)$' - replace: '\1http://{{ groups.custom_repo | first }}\2' + regexp: "^(\\s+\\-\\s)https://archive\\.cloudera\\.com(\\/.+)$" + replace: "\\1http://{{ groups.custom_repo | first }}\\2" - name: Set Cluster Definition file to Temp file with parcel cache entries ansible.builtin.set_fact: @@ -91,31 +90,21 @@ loop_control: loop_var: __cluster_repo_path_item ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __auto_repo_mirror_ini_entry - | select('search', __cluster_repo_path_item) - | select('search', init__parcel_distro) + init__urls_to_sign: "{{ init__urls_to_sign | default([]) + __auto_repo_mirror_ini_entry | select('search', __cluster_repo_path_item) | select('search', init__parcel_distro) | list }}" - name: Include Cloudera Manager Tarball ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __auto_repo_mirror_ini_entry - | select('search', 'repo-as-tarball') - | select('search', cloudera_manager_version) - | select('search', cm_distro_select[init__parcel_distro]['version'] | string + '.tar') - | list }}" - -# TODO: Filter to relevent version manifests, not all manifests, just to be tidy + init__urls_to_sign: "{{ init__urls_to_sign | default([]) + __auto_repo_mirror_ini_entry | select('search', 'repo-as-tarball') | select('search', cloudera_manager_version) + | select('search', cm_distro_select[init__parcel_distro]['version'] | string + '.tar') | list }}" + + # TODO: Filter to relevent version manifests, not all manifests, just to be tidy - name: Ensure manifest is included in Download Mirror URLs if present loop: "{{ init__cluster_repo_entries }}" loop_control: loop_var: __cluster_repo_path_item ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __auto_repo_mirror_ini_entry - | select('search', 'manifest.json') - | list }}" + init__urls_to_sign: "{{ init__urls_to_sign | default([]) + __auto_repo_mirror_ini_entry | select('search', 'manifest.json') | list }}" - name: Get AWS Specific download URIs when: @@ -127,10 +116,10 @@ loop: "{{ init__urls_to_sign }}" loop_control: loop_var: __s3_bucket_uri - amazon.aws.aws_s3: + amazon.aws.s3_object: bucket: "{{ __s3_bucket_uri | regex_replace('^.+//(.+)\\.s3.+$', '\\1') }}" object: "{{ __s3_bucket_uri | regex_replace('^.+amazonaws\\.com(.+)$', '\\1') }}" - ignore_nonexistent_bucket: yes + ignore_nonexistent_bucket: true expiry: "{{ download_link_expiry | default(default_download_link_expiry) }}" mode: geturl diff --git a/roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml b/roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml index 45370f3e..b116fd51 100644 --- a/roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml +++ b/roles/auto_repo_mirror/tasks/parse_definition_for_mirror_targets.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ # Read in cluster definition without jinja parsing= - name: Fetch repositories from cluster definition ansible.builtin.set_fact: - init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}" + init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file) | from_yaml | json_query('clusters[*].repositories') | flatten }}" - name: Check that a Cloudera License is presented if mirroring from files behind Cloudera Subscription when: init__preparse_repo_listing | select('search', 'archive.cloudera.com/p') | length > 0 @@ -36,7 +35,7 @@ - name: Get Parcel Manifests ansible.builtin.include_role: name: cloudera.cluster.deployment.repometa - public: yes + public: true vars: repositories: "{{ __init_parcel_repos }}" cluster_os_distribution: "{{ init__parcel_distro }}" @@ -48,7 +47,7 @@ - name: Filter Parcels by distro ansible.builtin.set_fact: - __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}" + __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term) | list }}" - name: Prepare target Download Mirror listing with parcels and attendant files when: __filtered_parcel_urls | length > 0 @@ -56,7 +55,8 @@ loop_control: loop_var: __filtered_parcel_item ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets | default(__init_tarball_links) + [__filtered_parcel_item, __filtered_parcel_item + '.sha1', __filtered_parcel_item + '.sha', __filtered_parcel_item + '.sha256', __filtered_parcel_item.replace(__filtered_parcel_item | basename, 'manifest.json') ] }}" + init__file_mirror_targets: "{{ init__file_mirror_targets | default(__init_tarball_links) + [__filtered_parcel_item, __filtered_parcel_item + '.sha1', __filtered_parcel_item + + '.sha', __filtered_parcel_item + '.sha256', __filtered_parcel_item.replace(__filtered_parcel_item | basename, 'manifest.json')] }}" # Explicitly set version from parcel distro as Ansible controller could be different OS from target cluster - name: Determine Cloudera-Manager Distro and Version @@ -69,14 +69,15 @@ ansible.builtin.include_role: role: cloudera.cluster.cloudera_manager.repo vars: - install_repo_on_host: no + install_repo_on_host: false clusters: [] cloudera_manager_distro_name: "{{ init__cloudera_manager_distro_name }}" cloudera_manager_distro_version: "{{ init__cloudera_manager_distro_version }}" - name: Add Cloudera Manager Repo to File Mirror list ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets + [cloudera_manager_repo_url | regex_replace('^(.+\\/(\\d\\.\\d\\.\\d)\\/)(\\w+)\\/.+$', '\\1' + 'repo-as-tarball/cm' + '\\2' + '-' + '\\3' + '.tar.gz')] }}" + init__file_mirror_targets: "{{ init__file_mirror_targets + [cloudera_manager_repo_url | regex_replace('^(.+\\/(\\d\\.\\d\\.\\d)\\/)(\\w+)\\/.+$', '\\1' + 'repo-as-tarball/cm' + + '\\2' + '-' + '\\3' + '.tar.gz')] }}" - name: Include CSDs if set when: @@ -94,14 +95,15 @@ - name: Prepare Localised Download Mirror utility bucket name ansible.builtin.set_fact: - init__auto_repo_mirror_bucket_name: "{{ utility_bucket_name | default([ auto_repo_mirror_prefix | default(default_auto_repo_mirror_prefix), __aws_caller_info.account, globals.region ] | join('-') ) }}" + init__auto_repo_mirror_bucket_name: "{{ utility_bucket_name | default([auto_repo_mirror_prefix | default(default_auto_repo_mirror_prefix), __aws_caller_info.account, + globals.region] | join('-')) }}" - name: List current target cache contents if any exist register: __auto_repo_mirror_lookup_initial failed_when: - __auto_repo_mirror_lookup_initial.s3_keys is not defined - "'cannot be found' not in __auto_repo_mirror_lookup_initial.msg" - amazon.aws.aws_s3: + amazon.aws.aws3_object: bucket: "{{ init__auto_repo_mirror_bucket_name }}" mode: list @@ -115,7 +117,7 @@ - name: Set Download Mirror details in Globals ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __auto_repo_mirror_spec, recursive=True ) }}" + globals: "{{ globals | default({}) | combine(__auto_repo_mirror_spec, recursive=True) }}" vars: __auto_repo_mirror_spec: auto_repo_mirror_targets: "{{ init__file_mirror_targets }}" diff --git a/roles/auto_repo_mirror/tasks/populate_from_upstream.yml b/roles/auto_repo_mirror/tasks/populate_from_upstream.yml index fc11c402..eaf1b8bb 100644 --- a/roles/auto_repo_mirror/tasks/populate_from_upstream.yml +++ b/roles/auto_repo_mirror/tasks/populate_from_upstream.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +33,7 @@ url: "{{ __mirror_fetch_item }}" dest: "/tmp/{{ globals.utility_bucket_name }}{{ __mirror_fetch_item | urlsplit('path') }}" url_username: "{{ cloudera_manager_repo_username | default(omit) }}" - url_password: "{{ cloudera_manager_repo_password | default(omit) }}" + url_password: "{{ cloudera_manager_repo_password | default(omit) }}" - name: Track async downloads to completion [ This may take up to an hour the first time for multi-gb Parcels ] loop: "{{ __auto_repo_mirror_rehost_results.results }}" @@ -44,7 +43,7 @@ until: __async_download_results.finished is defined and __async_download_results.finished delay: 30 retries: 240 - async_status: + ansible.builtin.async_status: jid: "{{ __download_async_item.ansible_job_id }}" failed_when: - __download_async_item.failed == True @@ -57,26 +56,26 @@ loop_control: loop_var: __parcel_unpack_item ansible.builtin.unarchive: - extra_opts: [ --strip-components=1 ] - remote_src: yes + extra_opts: [--strip-components=1] + remote_src: true src: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') }}" dest: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') | dirname }}" - keep_newer: yes + keep_newer: true - name: Upload Download Mirror for AWS to S3 when: globals.infra_type == 'aws' block: # Prepare to sync cache dir to S3 - name: Setup System Rhel8 - ignore_errors: true # newer versions of rhel8 do not need or have epel-release + ignore_errors: true # newer versions of rhel8 do not need or have epel-release when: - ansible_os_family == 'RedHat' - ansible_distribution_major_version | int >= 8 - become: yes + become: true ansible.builtin.package: lock_timeout: 180 name: "{{ __package_item }}" - update_cache: yes + update_cache: true state: present loop_control: loop_var: __package_item @@ -88,7 +87,7 @@ when: - ansible_os_family == 'RedHat' - ansible_distribution_major_version | int < 8 - become: yes + become: true ansible.builtin.package: name: "{{ __package_item }}" state: present @@ -102,9 +101,9 @@ - name: Setup system Debian when: ansible_os_family == "Debian" block: - - name: enable Debian Repos - become: yes - apt_repository: + - name: Enable Debian Repos + become: true + ansible.builtin.apt_repository: repo: "{{ __repo_item }}" loop_control: loop_var: __repo_item @@ -114,14 +113,14 @@ - "deb http://security.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-security universe" - name: Install Pip on Debian - become: yes + become: true ansible.builtin.apt: - update_cache: yes + update_cache: true name: python3-pip state: present - name: Prepare host for s3 actions - become: yes + become: true ansible.builtin.pip: name: "{{ __pip_item }}" loop_control: @@ -131,7 +130,7 @@ - "{{ (ansible_python_version[0] == '2') | ternary('boto3 >= 1.4.4,<1.18', 'boto3 >= 1.20.0') }}" - name: Sync downloaded Files paths to S3 cache bucket - become: yes + become: true community.aws.s3_sync: bucket: "{{ globals.utility_bucket_name }}" file_root: "/tmp/{{ globals.utility_bucket_name }}" diff --git a/roles/auto_repo_mirror/tasks/prepare_auto_repo_mirror.yml b/roles/auto_repo_mirror/tasks/prepare_auto_repo_mirror.yml index 54082038..e615e7b4 100644 --- a/roles/auto_repo_mirror/tasks/prepare_auto_repo_mirror.yml +++ b/roles/auto_repo_mirror/tasks/prepare_auto_repo_mirror.yml @@ -1,5 +1,4 @@ --- - - name: Fetch necessary variables from Ansible Controller ansible.builtin.set_fact: globals: "{{ hostvars['localhost']['globals'] }}" diff --git a/roles/auto_repo_mirror/tasks/update_mirror_cache.yml b/roles/auto_repo_mirror/tasks/update_mirror_cache.yml index 3ed1472c..f71a108e 100644 --- a/roles/auto_repo_mirror/tasks/update_mirror_cache.yml +++ b/roles/auto_repo_mirror/tasks/update_mirror_cache.yml @@ -1,5 +1,4 @@ --- - - name: Refresh Listing of target cache contents when: - init__auto_repo_mirror_bucket_name is defined @@ -8,7 +7,7 @@ failed_when: - __auto_repo_mirror_lookup_initial.s3_keys is not defined - "'cannot be found' not in __auto_repo_mirror_lookup_initial.msg" - amazon.aws.aws_s3: + amazon.aws.s3_object: bucket: "{{ init__auto_repo_mirror_bucket_name }}" mode: list @@ -18,7 +17,8 @@ loop_control: loop_var: __auto_repo_mirror_s3_urls_item ansible.builtin.set_fact: - __auto_repo_mirror_url_listing: "{{ __auto_repo_mirror_url_listing | default([]) + [['https:/', init__auto_repo_mirror_bucket_name + '.s3.amazonaws.com', __auto_repo_mirror_s3_urls_item ] | join('/') ] }}" + __auto_repo_mirror_url_listing: "{{ __auto_repo_mirror_url_listing | default([]) + [['https:/', init__auto_repo_mirror_bucket_name + '.s3.amazonaws.com', __auto_repo_mirror_s3_urls_item] + | join('/')] }}" - name: Persist Download Mirror to Definition path when: diff --git a/roles/auto_repo_mirror/vars/main.yml b/roles/auto_repo_mirror/vars/main.yml index 415bc858..66a58fa7 100644 --- a/roles/auto_repo_mirror/vars/main.yml +++ b/roles/auto_repo_mirror/vars/main.yml @@ -1,3 +1,4 @@ +--- cm_distro_select: el7: name: redhat diff --git a/roles/blackbox/defaults/main.yml b/roles/blackbox/defaults/main.yml index 4b36b8ca..acdbaa1f 100644 --- a/roles/blackbox/defaults/main.yml +++ b/roles/blackbox/defaults/main.yml @@ -13,8 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - blackbox_tarball_url: https://github.com/prometheus/blackbox_exporter/releases/download/v0.24.0/blackbox_exporter-0.24.0.linux-amd64.tar.gz blackbox_directory: /etc/blackbox blackbox_bin_directory: /usr/local/bin diff --git a/roles/common/meta/main.yml b/roles/blackbox/meta/argument_specs.yml similarity index 54% rename from roles/common/meta/main.yml rename to roles/blackbox/meta/argument_specs.yml index 571441ae..72543250 100644 --- a/roles/common/meta/main.yml +++ b/roles/blackbox/meta/argument_specs.yml @@ -12,31 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -galaxy_info: - role_name: platform - namespace: cloudera - author: Webster Mudge (wmudge@cloudera.com) - description: > - Shared configuration variables managed by role dependency. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - -dependencies: [] +argument_specs: + main: + short_description: Deploy Blackbox exporter. + description: Deploy Blackbox exporter for Prometheus. + author: Ronald Suplina + version_added: 2.4.0 diff --git a/roles/blackbox/tasks/main.yml b/roles/blackbox/tasks/main.yml index 7ad5206b..a7bee2d1 100644 --- a/roles/blackbox/tasks/main.yml +++ b/roles/blackbox/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +33,7 @@ src: "{{ __blackbox_tmp.path }}/{{ blackbox_tarball_file }}" dest: "{{ blackbox_directory }}" extra_opts: --strip-components=1 - remote_src: yes + remote_src: true - name: Remove the temporary directory when: __blackbox_tmp is defined @@ -49,23 +48,23 @@ - name: Create Blackbox user ansible.builtin.user: name: "{{ blackbox_user }}" - system: True + system: true - name: Set ownership of all files inside /etc/blackbox ansible.builtin.file: path: "{{ blackbox_directory }}" owner: "{{ blackbox_user }}" group: "{{ blackbox_group }}" - recurse: yes + recurse: true - name: Copy blackbox binary to /usr/local/bin ansible.builtin.copy: - remote_src: yes + remote_src: true src: "{{ blackbox_directory }}/blackbox_exporter" dest: "{{ blackbox_bin_directory }}/blackbox_exporter" owner: "{{ blackbox_user }}" group: "{{ blackbox_group }}" - mode: '0755' + mode: "0755" - name: Create Blackbox service template ansible.builtin.template: @@ -76,13 +75,12 @@ - name: Start and enable Blackbox service when: __blackbox_service.changed block: + - name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true - - name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: yes - - - name: Start Blackbox service - ansible.builtin.systemd: - name: blackbox - state: started - enabled: yes + - name: Start Blackbox service + ansible.builtin.systemd: + name: blackbox + state: started + enabled: true diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 759ef3a8..a39b59d4 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -19,158 +19,166 @@ # Role prefix is 'common__' # Labels -common__namespace: "{{ globals.name_prefix | default([globals.namespace_root | default('cdp'), globals.namespace_uid | default(ansible_date_time.date | replace('-',''))] | join('-')) }}" - -common__namespace_unique_suffix: "{{ globals.labels.namespace_suffix | default(common__infra_type[:2]) }}" -common__xaccount_suffix: "{{ globals.labels.cross_account | default('xaccount') }}" -common__credential_suffix: "{{ globals.labels.credential | default('cred') }}" -common__env_suffix: "{{ globals.labels.env | default('env') }}" -common__datalake_suffix: "{{ globals.labels.datalake | default('dl') }}" -common__table_suffix: "{{ globals.labels.table | default('table') }}" -common__vpc_name_suffix: "{{ globals.labels.vpc | default('net') }}" -common__vpc_subnet_suffix: "{{ globals.labels.subnet | default('sbnt') }}" -common__vpc_svcnet_suffix: "{{ globals.labels.service_network | default('svcnet') }}" -common__public_suffix: "{{ globals.labels.public | default('pub') }}" -common__private_suffix: "{{ globals.labels.private | default('pvt') }}" -common__security_group_knox_suffix: "{{ globals.labels.knox | default('knox') }}" -common__security_group_default_suffix: "{{ globals.labels.default | default('default') }}" -common__security_group_vpce_suffix: "{{ globals.labels.vpce | default('vpce') }}" -common__role_suffix: "{{ globals.labels.role | default('role') }}" -common__policy_suffix: "{{ globals.labels.policy | default('policy') }}" -common__storage_suffix: "{{ globals.labels.storage | default('storage') }}" -common__identity_suffix: "{{ globals.labels.identity | default('identity') }}" -common__idbroker_suffix: "{{ globals.labels.idbroker | default('idbroker') }}" -common__logs_suffix: "{{ globals.labels.logs | default('logs') }}" -common__data_suffix: "{{ globals.labels.data | default('data') }}" -common__external_data_suffix: "{{ globals.labels.external_data | default('external') }}" -common__datalake_admin_suffix: "{{ globals.labels.datalake_admin | default('dladmin') }}" -common__ranger_audit_suffix: "{{ globals.labels.ranger_audit | default('audit') }}" -common__raz_suffix: "{{ globals.labels.raz | default('raz') }}" -common__cml_suffix: "{{ globals.labels.cml | default('cml') }}" -common__cde_suffix: "{{ globals.labels.cde | default('cde') }}" -common__igw_suffix: "{{ globals.labels.internet_gateway | default('igw') }}" -common__app_suffix: "{{ globals.labels.app | default('app') }}" -common__group_suffix: "{{ globals.labels.group | default('group') }}" -common__admin_suffix: "{{ globals.labels.admin | default('admin') }}" -common__user_suffix: "{{ globals.labels.user | default('user') }}" -common__ngw_suffix: "{{ globals.labels.nat_gateway | default('ngw') }}" - -common__unique_storage_name_suffix: "{{ globals.storage.name | default((common__region + common__aws_profile) if 'aws' in common__infra_type else common__region) }}" +common__namespace: "{{ globals.name_prefix | default([globals.namespace_root | default('cdp'), globals.namespace_uid | default(ansible_date_time.date | replace('-',''))] + | join('-')) }}" + +common__namespace_unique_suffix: "{{ globals.labels.namespace_suffix | default(common__infra_type[:2]) }}" +common__xaccount_suffix: "{{ globals.labels.cross_account | default('xaccount') }}" +common__credential_suffix: "{{ globals.labels.credential | default('cred') }}" +common__env_suffix: "{{ globals.labels.env | default('env') }}" +common__datalake_suffix: "{{ globals.labels.datalake | default('dl') }}" +common__table_suffix: "{{ globals.labels.table | default('table') }}" +common__vpc_name_suffix: "{{ globals.labels.vpc | default('net') }}" +common__vpc_subnet_suffix: "{{ globals.labels.subnet | default('sbnt') }}" +common__vpc_svcnet_suffix: "{{ globals.labels.service_network | default('svcnet') }}" +common__public_suffix: "{{ globals.labels.public | default('pub') }}" +common__private_suffix: "{{ globals.labels.private | default('pvt') }}" +common__security_group_knox_suffix: "{{ globals.labels.knox | default('knox') }}" +common__security_group_default_suffix: "{{ globals.labels.default | default('default') }}" +common__security_group_vpce_suffix: "{{ globals.labels.vpce | default('vpce') }}" +common__role_suffix: "{{ globals.labels.role | default('role') }}" +common__policy_suffix: "{{ globals.labels.policy | default('policy') }}" +common__storage_suffix: "{{ globals.labels.storage | default('storage') }}" +common__identity_suffix: "{{ globals.labels.identity | default('identity') }}" +common__idbroker_suffix: "{{ globals.labels.idbroker | default('idbroker') }}" +common__logs_suffix: "{{ globals.labels.logs | default('logs') }}" +common__data_suffix: "{{ globals.labels.data | default('data') }}" +common__external_data_suffix: "{{ globals.labels.external_data | default('external') }}" +common__datalake_admin_suffix: "{{ globals.labels.datalake_admin | default('dladmin') }}" +common__ranger_audit_suffix: "{{ globals.labels.ranger_audit | default('audit') }}" +common__raz_suffix: "{{ globals.labels.raz | default('raz') }}" +common__cml_suffix: "{{ globals.labels.cml | default('cml') }}" +common__cde_suffix: "{{ globals.labels.cde | default('cde') }}" +common__igw_suffix: "{{ globals.labels.internet_gateway | default('igw') }}" +common__app_suffix: "{{ globals.labels.app | default('app') }}" +common__group_suffix: "{{ globals.labels.group | default('group') }}" +common__admin_suffix: "{{ globals.labels.admin | default('admin') }}" +common__user_suffix: "{{ globals.labels.user | default('user') }}" +common__ngw_suffix: "{{ globals.labels.nat_gateway | default('ngw') }}" + +common__unique_storage_name_suffix: "{{ globals.storage.name | default((common__region + common__aws_profile) if 'aws' in common__infra_type else common__region) + }}" # CDP Control Plane Region -common__cdp_control_plane_region: "{{ globals.cdp_region | default('us-west-1') }}" -common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" +common__cdp_control_plane_region: "{{ globals.cdp_region | default('us-west-1') }}" +common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" # Infra -common__infra_deployment_engine: "{{ globals.infra_deployment_engine | default('ansible') }}" -common__aws_profile: "{{ globals.aws_profile | default('') }}" -common__infra_type: "{{ globals.infra_type | default('aws') }}" -common__public_key_file: "{{ globals.ssh.public_key_file | default('') }}" -common__namespace_cdp: "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}" -common__tags: "{{ globals.tags | default({}) }}" -common__public_key_id: "{{ globals.ssh.public_key_id | default('') }}" -common__public_key_text: "{{ globals.ssh.public_key_text | default('') }}" -common__region: "{{ globals.region | default(common__region_default[common__infra_type]) }}" -common__storage_name: "{{ infra.storage.name | default([common__namespace, common__unique_storage_name_suffix[::2] | replace('-','')] | join('-')) }}" +common__infra_deployment_engine: "{{ globals.infra_deployment_engine | default('ansible') }}" +common__aws_profile: "{{ globals.aws_profile | default('') }}" +common__infra_type: "{{ globals.infra_type | default('aws') }}" +common__public_key_file: "{{ globals.ssh.public_key_file | default('') }}" +common__namespace_cdp: "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}" +common__tags: "{{ globals.tags | default({}) }}" +common__public_key_id: "{{ globals.ssh.public_key_id | default('') }}" +common__public_key_text: "{{ globals.ssh.public_key_text | default('') }}" +common__region: "{{ globals.region | default(common__region_default[common__infra_type]) }}" +common__storage_name: "{{ infra.storage.name | default([common__namespace, common__unique_storage_name_suffix[::2] | replace('-','')] | join('-')) }}" # Terraform -common__terraform_base_dir: "{{ globals.terraform.base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}" +common__terraform_base_dir: "{{ globals.terraform.base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}" # The processed Jinja template files for Terraform are placed in common__terraform_template_dir -common__terraform_template_dir: "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}" +common__terraform_template_dir: "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}" # A timestamped artefact directory storing a copy of the Terraform code from each run -common__terraform_artefact_dir: "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}" +common__terraform_artefact_dir: "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}" # Terraform apply/destroy run from under this directory -common__terraform_workspace_dir: "{{ [common__terraform_base_dir, 'workspace'] | path_join }}" +common__terraform_workspace_dir: "{{ [common__terraform_base_dir, 'workspace'] | path_join }}" -common__terraform_allowed_state_storage: "['local', 'remote_s3']" -common__terraform_state_storage: "{{ globals.terraform.state_storage | default('local') }}" -common__terraform_remote_state_bucket: "{{ globals.terraform.remote_state_bucket | default('') }}" +common__terraform_allowed_state_storage: "['local', 'remote_s3']" +common__terraform_state_storage: "{{ globals.terraform.state_storage | default('local') }}" +common__terraform_remote_state_bucket: "{{ globals.terraform.remote_state_bucket | default('') }}" common__terraform_remote_state_lock_table: "{{ globals.terraform.remote_state_lock_table | default('') }}" -common__vpc_name: "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}" -common__vpc_public_subnet_cidrs: "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}" -common__vpc_private_subnet_cidrs: "{{ infra.vpc.private_subnets | default(['10.10.96.0/19', '10.10.128.0/19', '10.10.160.0/19']) }}" -common__vpc_private_subnets_suffix: "{{ infra.vpc.public_subnets_suffix | default([common__vpc_subnet_suffix, common__private_suffix] | join('-')) }}" -common__vpc_public_subnets_suffix: "{{ infra.vpc.private_subnets_suffix | default([common__vpc_subnet_suffix, common__public_suffix] | join('-')) }}" +common__vpc_name: "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}" +common__vpc_public_subnet_cidrs: "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}" +common__vpc_private_subnet_cidrs: "{{ infra.vpc.private_subnets | default(['10.10.96.0/19', '10.10.128.0/19', '10.10.160.0/19']) }}" +common__vpc_private_subnets_suffix: "{{ infra.vpc.public_subnets_suffix | default([common__vpc_subnet_suffix, common__private_suffix] | join('-')) }}" +common__vpc_public_subnets_suffix: "{{ infra.vpc.private_subnets_suffix | default([common__vpc_subnet_suffix, common__public_suffix] | join('-')) }}" -common__security_group_knox_name: "{{ infra.security_group.knox.name | default([common__namespace, common__security_group_knox_name_suffix] | join('-')) }}" -common__security_group_default_name: "{{ infra.security_group.default.name | default([common__namespace, common__security_group_default_name_suffix] | join('-')) }}" -common__security_group_vpce_name: "{{ infra.security_group.vpce.name | default([common__namespace, common__security_group_vpce_name_suffix] | join('-')) }}" +common__security_group_knox_name: "{{ infra.security_group.knox.name | default([common__namespace, common__security_group_knox_name_suffix] | join('-')) }}" +common__security_group_default_name: "{{ infra.security_group.default.name | default([common__namespace, common__security_group_default_name_suffix] | join('-')) + }}" +common__security_group_vpce_name: "{{ infra.security_group.vpce.name | default([common__namespace, common__security_group_vpce_name_suffix] | join('-')) }}" -common__security_group_knox_name_suffix: "{{ infra.security_group.knox.suffix | default(common__security_group_knox_suffix) }}" +common__security_group_knox_name_suffix: "{{ infra.security_group.knox.suffix | default(common__security_group_knox_suffix) }}" common__security_group_default_name_suffix: "{{ infra.security_group.default.suffix | default(common__security_group_default_suffix) }}" -common__security_group_vpce_name_suffix: "{{ infra.security_group.vpce.suffix | default(common__security_group_vpce_suffix) }}" +common__security_group_vpce_name_suffix: "{{ infra.security_group.vpce.suffix | default(common__security_group_vpce_suffix) }}" -common__ml_path: "{{ infra.storage.path.ml | default('datasci') }}" -common__de_path: "{{ infra.storage.path.de | default('dataeng') }}" -common__logs_path: "{{ infra.storage.path.logs | default(common__logs_suffix) }}" -common__data_path: "{{ infra.storage.path.data | default(common__data_suffix) }}" -common__backups_path: "{{ infra.storage.path.backups | default(common__logs_path) }}" -common__ranger_audit_path: "{{ infra.storage.path.ranger_audit | default('ranger/audit') }}" +common__ml_path: "{{ infra.storage.path.ml | default('datasci') }}" +common__de_path: "{{ infra.storage.path.de | default('dataeng') }}" +common__logs_path: "{{ infra.storage.path.logs | default(common__logs_suffix) }}" +common__data_path: "{{ infra.storage.path.data | default(common__data_suffix) }}" +common__backups_path: "{{ infra.storage.path.backups | default(common__logs_path) }}" +common__ranger_audit_path: "{{ infra.storage.path.ranger_audit | default('ranger/audit') }}" # AWS Infra -common__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | default('') }}" -common__aws_prefix_list_suffix: "{{ infra.aws.prefix_list.suffix | default('prefix') }}" -common__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" -common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" -common__aws_region: "{{ infra.aws.region | default('eu-west-1') }}" -common__aws_role_suffix: "{{ infra.aws.role.suffix | default(common__role_suffix) }}" - -common__aws_datalake_admin_role_name: "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}" -common__aws_datalake_admin_suffix: "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" -common__aws_idbroker_role_name: "{{ env.aws.role.name.idbroker | default([common__namespace, common__aws_idbroker_suffix, common__aws_role_suffix] | join('-')) }}" -common__aws_idbroker_suffix: "{{ env.aws.role.label.idbroker | default(common__idbroker_suffix) }}" +common__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | default('') }}" +common__aws_prefix_list_suffix: "{{ infra.aws.prefix_list.suffix | default('prefix') }}" +common__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" +common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" +common__aws_region: "{{ infra.aws.region | default('eu-west-1') }}" +common__aws_role_suffix: "{{ infra.aws.role.suffix | default(common__role_suffix) }}" + +common__aws_datalake_admin_role_name: "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] + | join('-')) }}" +common__aws_datalake_admin_suffix: "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" +common__aws_idbroker_role_name: "{{ env.aws.role.name.idbroker | default([common__namespace, common__aws_idbroker_suffix, common__aws_role_suffix] | join('-')) }}" +common__aws_idbroker_suffix: "{{ env.aws.role.label.idbroker | default(common__idbroker_suffix) }}" # Azure Infra -common__azure_storage_name: "{{ infra.azure.storage.name | default(common__storage_name | replace('-','')) }}" - -common__azure_account_suffix: "{{ infra.azure.netapp.account.suffix | default('account') }}" -common__azure_netapp_suffix: "{{ infra.azure.netapp.suffix | default('netapp') }}" -common__azure_pool_suffix: "{{ infra.azure.netapp.pool.suffix | default('pool') }}" -common__azure_volume_suffix: "{{ infra.azure.netapp.volume.suffix | default('vol') }}" - -common__azure_metagroup_name_suffix: "{{ infra.azure.metagroup.suffix | default('rmgp') }}" -common__azure_metagroup_name: "{{ infra.azure.metagroup.name | default([common__namespace, common__azure_metagroup_name_suffix] | join('-')) }}" -common__azure_region: "{{ infra.azure.region | default('westeurope') }}" -common__azure_sp_login_from_env: "{{ infra.azure.sp_login_from_env | default(False) }}" -common__azure_netapp_account_name: "{{ infra.azure.netapp.account.name | default([common__namespace, common__azure_netapp_suffix, common__azure_account_suffix] | join('-'))}}" -common__azure_netapp_pool_name: "{{ infra.azure.netapp.pool.name | default([common__namespace, common__azure_netapp_suffix, common__azure_pool_suffix] | join('-')) }}" -common__azure_netapp_vol_name: "{{ infra.azure.netapp.volume.name | default([common__namespace, common__azure_netapp_suffix, common__azure_volume_suffix] | join('-')) }}" -common__azure_netapp_nfs_version: "{{ infra.azure.netapp.nfs.version | default('3') }}" +common__azure_storage_name: "{{ infra.azure.storage.name | default(common__storage_name | replace('-','')) }}" + +common__azure_account_suffix: "{{ infra.azure.netapp.account.suffix | default('account') }}" +common__azure_netapp_suffix: "{{ infra.azure.netapp.suffix | default('netapp') }}" +common__azure_pool_suffix: "{{ infra.azure.netapp.pool.suffix | default('pool') }}" +common__azure_volume_suffix: "{{ infra.azure.netapp.volume.suffix | default('vol') }}" + +common__azure_metagroup_name_suffix: "{{ infra.azure.metagroup.suffix | default('rmgp') }}" +common__azure_metagroup_name: "{{ infra.azure.metagroup.name | default([common__namespace, common__azure_metagroup_name_suffix] | join('-')) }}" +common__azure_region: "{{ infra.azure.region | default('westeurope') }}" +common__azure_sp_login_from_env: "{{ infra.azure.sp_login_from_env | default(False) }}" +common__azure_netapp_account_name: "{{ infra.azure.netapp.account.name | default([common__namespace, common__azure_netapp_suffix, common__azure_account_suffix] | + join('-'))}}" +common__azure_netapp_pool_name: "{{ infra.azure.netapp.pool.name | default([common__namespace, common__azure_netapp_suffix, common__azure_pool_suffix] | join('-')) + }}" +common__azure_netapp_vol_name: "{{ infra.azure.netapp.volume.name | default([common__namespace, common__azure_netapp_suffix, common__azure_volume_suffix] | join('-')) + }}" +common__azure_netapp_nfs_version: "{{ infra.azure.netapp.nfs.version | default('3') }}" # GCP Infra -common__gcp_project: "{{ infra.gcp.project | default('gcp-se') }}" -common__gcp_region: "{{ infra.gcp.region | default('europe-west1') }}" -common__gcp_subnet_id: "{{ infra.gcp.vpc.subnet_id | default(None) }}" +common__gcp_project: "{{ infra.gcp.project | default('gcp-se') }}" +common__gcp_region: "{{ infra.gcp.region | default('europe-west1') }}" +common__gcp_subnet_id: "{{ infra.gcp.vpc.subnet_id | default(None) }}" # Plat -common__xaccount_credential_suffix: "{{ env.cdp.credential.suffix | default(common__xaccount_suffix) }}" -common__xaccount_credential_name_suffix: "{{ env.cdp.credential.name_suffix | default(common__credential_suffix) }}" -common__xaccount_credential_name: "{{ env.cdp.credential.name | default([common__namespace_cdp, common__xaccount_credential_suffix, common__xaccount_credential_name_suffix] | join('-')) }}" +common__xaccount_credential_suffix: "{{ env.cdp.credential.suffix | default(common__xaccount_suffix) }}" +common__xaccount_credential_name_suffix: "{{ env.cdp.credential.name_suffix | default(common__credential_suffix) }}" +common__xaccount_credential_name: "{{ env.cdp.credential.name | default([common__namespace_cdp, common__xaccount_credential_suffix, common__xaccount_credential_name_suffix] + | join('-')) }}" -common__env_name: "{{ env.name | default([common__namespace_cdp, common__env_name_suffix] | join('-')) }}" -common__env_name_suffix: "{{ env.suffix | default(common__env_suffix) }}" +common__env_name: "{{ env.name | default([common__namespace_cdp, common__env_name_suffix] | join('-')) }}" +common__env_name_suffix: "{{ env.suffix | default(common__env_suffix) }}" -common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" -common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" -common__tunnel: "{{ env.tunnel | default(False) }}" -common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" -common__use_public_ip: "{{ env.public_endpoint_access | default(not common__tunnel) }}" +common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" +common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" +common__tunnel: "{{ env.tunnel | default(False) }}" +common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" +common__use_public_ip: "{{ env.public_endpoint_access | default(not common__tunnel) }}" -common__env_admin_password: "{{ globals.admin_password | mandatory }}" -common__aws_policy_urls_default_root: "https://raw.githubusercontent.com/hortonworks/cloudbreak/master/cloud-aws-common/src/main/resources/definitions/cdp" +common__env_admin_password: "{{ globals.admin_password | mandatory }}" +common__aws_policy_urls_default_root: "https://raw.githubusercontent.com/hortonworks/cloudbreak/master/cloud-aws-common/src/main/resources/definitions/cdp" # Deploy -common__setup_runtime: "{{ ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) | bool }}" -common__setup_plat: "{{ env is defined or sequence__setup_runtime | default(False) | bool }}" -common__setup_infra: "{{ infra is defined or sequence__setup_plat | default(False) | bool }}" -common__setup_base: "{{ mgmt is defined | default(False) | bool }}" - -common__include_ml: "{{ ml is defined | bool }}" -common__include_dw: "{{ dw is defined | bool }}" -common__include_de: "{{ de is defined | bool }}" -common__include_df: "{{ df is defined | bool }}" -common__include_datahub: "{{ datahub is defined | bool }}" -common__include_opdb: "{{ opdb is defined | bool }}" +common__setup_runtime: "{{ ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) | bool }}" +common__setup_plat: "{{ env is defined or sequence__setup_runtime | default(False) | bool }}" +common__setup_infra: "{{ infra is defined or sequence__setup_plat | default(False) | bool }}" +common__setup_base: "{{ mgmt is defined | default(False) | bool }}" + +common__include_ml: "{{ ml is defined | bool }}" +common__include_dw: "{{ dw is defined | bool }}" +common__include_de: "{{ de is defined | bool }}" +common__include_df: "{{ df is defined | bool }}" +common__include_datahub: "{{ datahub is defined | bool }}" +common__include_opdb: "{{ opdb is defined | bool }}" # Teardown -common__force_teardown: "{{ globals.force_teardown | default(False) }}" # WARNING: This will purge your namespace and anything related to it, use with extreme caution +common__force_teardown: "{{ globals.force_teardown | default(False) }}" # WARNING: This will purge your namespace and anything related to it, use with extreme caution diff --git a/roles/freeipa_client/meta/main.yml b/roles/common/meta/argument_specs.yml similarity index 53% rename from roles/freeipa_client/meta/main.yml rename to roles/common/meta/argument_specs.yml index b82d2075..4723817f 100644 --- a/roles/freeipa_client/meta/main.yml +++ b/roles/common/meta/argument_specs.yml @@ -1,5 +1,3 @@ ---- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,30 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -galaxy_info: - role_name: freeipa_client - namespace: cloudera - author: Webster Mudge Jim Enright Chuck Levesque - description: > - Deployment of FreeIPA clients for Cloudera Data Platform (CDP) Base and ECS - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - freeipa +argument_specs: + main: + short_description: Shared configuration variables + description: Shared configuration variables managed by role dependency. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml index b017182b..9440b7c2 100644 --- a/roles/common/vars/main.yml +++ b/roles/common/vars/main.yml @@ -20,6 +20,6 @@ common__region_default: gcp: "{{ common__gcp_region }}" common__cdp_control_planes: - us-west-1: "crn:altus:iam:us-west-1:altus" - eu-1: "crn:altus:iam:eu-1:altus" - ap-1: "crn:altus:iam:ap-1:altus" + us-west-1: "crn:altus:iam:us-west-1:altus" + eu-1: "crn:altus:iam:eu-1:altus" + ap-1: "crn:altus:iam:ap-1:altus" diff --git a/roles/data/defaults/main.yml b/roles/data/defaults/main.yml index f3be63bf..127b0e42 100644 --- a/roles/data/defaults/main.yml +++ b/roles/data/defaults/main.yml @@ -18,27 +18,27 @@ # Role prefix is 'data__' -data__infra_type: "{{ common__infra_type }}" -data__region: "{{ common__region }}" -data__namespace: "{{ common__namespace }}" +data__infra_type: "{{ common__infra_type }}" +data__region: "{{ common__region }}" +data__namespace: "{{ common__namespace }}" -data__storage: "{{ data.storage | default([]) }}" -data__external_data_suffix: "{{ common__external_data_suffix }}" -data__policy_suffix: "{{ data.policy.suffix | default(common__policy_suffix) }}" -data__role_suffix: "{{ data.role.suffix | default(common__role_suffix) }}" +data__storage: "{{ data.storage | default([]) }}" +data__external_data_suffix: "{{ common__external_data_suffix }}" +data__policy_suffix: "{{ data.policy.suffix | default(common__policy_suffix) }}" +data__role_suffix: "{{ data.role.suffix | default(common__role_suffix) }}" -data__teardown_deletes_policies: "{{ data.teardown.delete_policies | default(False) }}" -data__teardown_deletes_roles: "{{ data.teardown.delete_roles | default(False) }}" +data__teardown_deletes_policies: "{{ data.teardown.delete_policies | default(False) }}" +data__teardown_deletes_roles: "{{ data.teardown.delete_roles | default(False) }}" # AWS -data__aws_policy_suffix: "{{ data.policy.aws.suffix | default(data__policy_suffix) }}" -data__aws_read_only_policy_suffix: "{{ data.policy.aws.read_only.suffix | default('read-only') }}" -data__aws_read_write_policy_suffix: "{{ data.policy.aws.read_write.suffix | default('read-write') }}" +data__aws_policy_suffix: "{{ data.policy.aws.suffix | default(data__policy_suffix) }}" +data__aws_read_only_policy_suffix: "{{ data.policy.aws.read_only.suffix | default('read-only') }}" +data__aws_read_write_policy_suffix: "{{ data.policy.aws.read_write.suffix | default('read-write') }}" -data__aws_role_suffix: "{{ data.role.aws.suffix | default(data__role_suffix) }}" +data__aws_role_suffix: "{{ data.role.aws.suffix | default(data__role_suffix) }}" -data__aws_read_only_policy_url: "{{ data.policy.aws.read_only.url | default(data__aws_policy_urls_default['read_only']) }}" -data__aws_read_write_policy_url: "{{ data.policy.aws.read_write.url | default(data__aws_policy_urls_default['read_write']) }}" +data__aws_read_only_policy_url: "{{ data.policy.aws.read_only.url | default(data__aws_policy_urls_default['read_only']) }}" +data__aws_read_write_policy_url: "{{ data.policy.aws.read_write.url | default(data__aws_policy_urls_default['read_write']) }}" -data__aws_idbroker_role_name: "{{ common__aws_idbroker_role_name }}" -data__aws_datalake_admin_role_name: "{{ common__aws_datalake_admin_role_name }}" +data__aws_idbroker_role_name: "{{ common__aws_idbroker_role_name }}" +data__aws_datalake_admin_role_name: "{{ common__aws_datalake_admin_role_name }}" diff --git a/roles/sequence/meta/main.yml b/roles/data/meta/argument_specs.yml similarity index 53% rename from roles/sequence/meta/main.yml rename to roles/data/meta/argument_specs.yml index 416e5ccb..ac1f22ad 100644 --- a/roles/sequence/meta/main.yml +++ b/roles/data/meta/argument_specs.yml @@ -12,30 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -galaxy_info: - author: Webster Mudge (wmudge@cloudera.com) - description: > - Runlevel execution using tags for deploying and managing Cloudera Data Platform - (CDP) Public Cloud. - company: Cloudera, Inc. - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - -dependencies: ['cloudera.exe.common'] +argument_specs: + main: + short_description: Manage external data locations + description: | + Management of cloud provider roles and policies supporting external data locations + for Cloudera Data Platform (CDP) Public Cloud deployments and Environment. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/data/meta/main.yml b/roles/data/meta/main.yml deleted file mode 100644 index 59cec4ac..00000000 --- a/roles/data/meta/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: Webster Mudge (wmudge@cloudera.com) - description: > - Management of cloud provider roles and policies supporting external data locations - for Cloudera Data Platform (CDP) Public Cloud deployments and Environment. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - aws - - gcloud - - azure - -dependencies: ['cloudera.exe.common'] diff --git a/roles/data/tasks/initialize.yml b/roles/data/tasks/initialize.yml index eff61332..c7012b2b 100644 --- a/roles/data/tasks/initialize.yml +++ b/roles/data/tasks/initialize.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,8 @@ - name: Set facts for data location types ansible.builtin.set_fact: __data_storage_read_only: "{{ data__storage | selectattr('read_only', 'defined') | selectattr('read_only') | list }}" - __data_storage_read_write: "{{ data__storage | selectattr('read_only', 'defined') | rejectattr('read_only') | list | union(data__storage | selectattr('read_only', 'undefined') | list) }}" + __data_storage_read_write: "{{ data__storage | selectattr('read_only', 'defined') | rejectattr('read_only') | list | union(data__storage | selectattr('read_only', + 'undefined') | list) }}" - name: Initialize cloud provider details ansible.builtin.include_tasks: "initialize_{{ data__infra_type | lower }}.yml" diff --git a/roles/data/tasks/initialize_aws.yml b/roles/data/tasks/initialize_aws.yml index 9f97430a..e469e602 100644 --- a/roles/data/tasks/initialize_aws.yml +++ b/roles/data/tasks/initialize_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +14,7 @@ # limitations under the License. - name: Query for the AWS ID Broker role - community.aws.iam_role_info: + amazon.aws.iam_role_info: region: "{{ data__region }}" name: "{{ data__aws_idbroker_role_name }}" register: __data_idbroker_role_info @@ -25,7 +24,7 @@ that: - __data_idbroker_role_info.iam_roles | length > 0 fail_msg: "Unable to find AWS ID Broker role '{{ data__aws_idbroker_role_name }}'" - quiet: yes + quiet: true - name: Set fact for the AWS ID Broker role ARN ansible.builtin.set_fact: diff --git a/roles/data/tasks/main.yml b/roles/data/tasks/main.yml index 92624c88..b05cd65f 100644 --- a/roles/data/tasks/main.yml +++ b/roles/data/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include_tasks: validate.yml -- include_tasks: initialize.yml -- include_tasks: setup.yml +- ansible.builtin.include_tasks: validate.yml +- ansible.builtin.include_tasks: initialize.yml +- ansible.builtin.include_tasks: setup.yml diff --git a/roles/data/tasks/setup.yml b/roles/data/tasks/setup.yml index d1d4aede..001aa488 100644 --- a/roles/data/tasks/setup.yml +++ b/roles/data/tasks/setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/data/tasks/setup_aws.yml b/roles/data/tasks/setup_aws.yml index 43211cfd..7041340a 100644 --- a/roles/data/tasks/setup_aws.yml +++ b/roles/data/tasks/setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,13 +23,15 @@ - name: Set fact for AWS data access policy name ansible.builtin.set_fact: - data__aws_data_access_policy_name: "{{ __data_storage.policy.name | default([data__namespace, data__external_data_suffix, __data_aws_policy_label, __data_storage.policy.suffix | default(data__aws_policy_suffix)] | join('-')) }}" + data__aws_data_access_policy_name: "{{ __data_storage.policy.name | default([data__namespace, data__external_data_suffix, __data_aws_policy_label, __data_storage.policy.suffix + | default(data__aws_policy_suffix)] | join('-')) }}" vars: __data_aws_policy_label: "{{ __data_storage['read_only'] | default(False) | ternary(data__aws_read_only_policy_suffix, data__aws_read_write_policy_suffix) }}" - name: Set fact for data storage location declarations ansible.builtin.set_fact: - __data_aws_data_access_locations: "{{ __data_aws_data_access_locations | default({}) | combine({ data__aws_data_access_policy_name: locations | unique }, recursive=True, list_merge='append') }}" + __data_aws_data_access_locations: "{{ __data_aws_data_access_locations | default({}) | combine({data__aws_data_access_policy_name: locations | unique}, recursive=True, + list_merge='append') }}" vars: locations: - "arn:aws:s3:::{{ __data_storage_location }}" @@ -42,20 +43,22 @@ - name: Set fact for AWS data access policy Resource with storage locations ansible.builtin.set_fact: - __data_aws_data_access_policy_contents: "{{ document | combine({ 'Statement' : [modified_statement] | union(remaining_sid_statements) | union(remaining_other_statements) }, recursive=True) }}" + __data_aws_data_access_policy_contents: "{{ document | combine({'Statement': [modified_statement] | union(remaining_sid_statements) | union(remaining_other_statements)}, + recursive=True) }}" vars: sid: "{{ __data_storage['read_only'] | default(False) | ternary('AllowListingOfDataLakeFolder', 'VisualEditor3') }}" document: "{{ lookup('file', __aws_data_access_policy_document.dest) | from_json }}" target_statement: "{{ document.Statement | selectattr('Sid', 'defined') | selectattr('Sid', 'equalto', sid) | first }}" remaining_sid_statements: "{{ document.Statement | selectattr('Sid', 'defined') | rejectattr('Sid', 'equalto', sid) }}" remaining_other_statements: "{{ document.Statement | selectattr('Sid', 'undefined') }}" - modified_statement: "{{ target_statement | combine({ 'Resource': __data_aws_data_access_locations[data__aws_data_access_policy_name] }, recursive=True) }}" + modified_statement: "{{ target_statement | combine({'Resource': __data_aws_data_access_locations[data__aws_data_access_policy_name]}, recursive=True) }}" - name: Create AWS managed policy for data access storage locations - community.aws.iam_managed_policy: + amazon.aws.iam_managed_policy: region: "{{ data__region }}" policy_name: "{{ data__aws_data_access_policy_name }}" - policy_description: "{{ __data_storage['read_only'] | default(False) | ternary('Read-only', 'Read-write') }} external data storage policy for {{ data__namespace }}" + policy_description: "{{ __data_storage['read_only'] | default(False) | ternary('Read-only', 'Read-write') }} external data storage policy for {{ data__namespace + }}" policy: "{{ __data_aws_data_access_policy_contents | to_json }}" state: present @@ -67,11 +70,11 @@ __data_role_name_default: "{{ [data__namespace, data__external_data_suffix, __data_storage.role.suffix | default(data__aws_role_suffix)] | join('-') }}" - name: Attach AWS data access storage policy to external data AWS role - community.aws.iam_role: + amazon.aws.iam_role: region: "{{ data__region }}" name: "{{ __data_aws_access_role_name }}" - purge_policies: no - purge_tags: no + purge_policies: false + purge_tags: false assume_role_policy_document: Version: "2012-10-17" Statement: diff --git a/roles/data/tasks/teardown.yml b/roles/data/tasks/teardown.yml index 567d1e9d..1460d3c4 100644 --- a/roles/data/tasks/teardown.yml +++ b/roles/data/tasks/teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/data/tasks/teardown_aws_policies.yml b/roles/data/tasks/teardown_aws_policies.yml index 9c6b9b5a..99ad3d78 100644 --- a/roles/data/tasks/teardown_aws_policies.yml +++ b/roles/data/tasks/teardown_aws_policies.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,8 @@ block: - name: Set fact for AWS external policy name ansible.builtin.set_fact: - __data_aws_external_policy_name: "{{ __data_storage.policy.name | default([data__namespace, data__external_data_suffix, __data_storage_policy_label, __data_storage_policy_suffix] | join('-')) }}" + __data_aws_external_policy_name: "{{ __data_storage.policy.name | default([data__namespace, data__external_data_suffix, __data_storage_policy_label, __data_storage_policy_suffix] + | join('-')) }}" vars: __data_storage_readonly: "{{ __data_storage['read_only'] | default(False) }}" __data_storage_policy_label: "{{ __data_storage_readonly | ternary(data__aws_read_only_policy_suffix, data__aws_read_write_policy_suffix) }}" @@ -35,7 +35,7 @@ __data_role_name_default: "{{ [data__namespace, data__external_data_suffix, __data_storage.role.suffix | default(data__aws_role_suffix)] | join('-') }}" - name: Query the external data AWS role - community.aws.iam_role_info: + amazon.aws.iam_role_info: region: "{{ data__region }}" name: "{{ __data_aws_access_role_name }}" register: __data_aws_access_role_info @@ -43,17 +43,18 @@ - name: Set facts for existing managed policies for external data AWS role when: __data_aws_access_role_info.iam_roles | length > 0 ansible.builtin.set_fact: - __data_aws_access_role_remaining_policies: "{{ __data_aws_access_role.managed_policies | rejectattr('policy_name', 'equalto', __data_aws_external_policy_name) | list }}" + __data_aws_access_role_remaining_policies: "{{ __data_aws_access_role.managed_policies | rejectattr('policy_name', 'equalto', __data_aws_external_policy_name) + | list }}" vars: __data_aws_access_role: "{{ __data_aws_access_role_info.iam_roles | first }}" - name: Detach AWS external storage policy from external data AWS role when: __data_aws_access_role_info.iam_roles | length > 0 - community.aws.iam_role: + amazon.aws.iam_role: region: "{{ data__region }}" name: "{{ __data_aws_access_role_name }}" - purge_policies: yes - purge_tags: no + purge_policies: true + purge_tags: false assume_role_policy_document: Version: "2012-10-17" Statement: @@ -65,7 +66,7 @@ state: present - name: Delete AWS external storage policy - community.aws.iam_managed_policy: + amazon.aws.iam_managed_policy: region: "{{ data__region }}" policy_name: "{{ __data_aws_external_policy_name }}" state: absent diff --git a/roles/data/tasks/teardown_aws_roles.yml b/roles/data/tasks/teardown_aws_roles.yml index 28a79bc9..53750507 100644 --- a/roles/data/tasks/teardown_aws_roles.yml +++ b/roles/data/tasks/teardown_aws_roles.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/data/vars/main.yml b/roles/data/vars/main.yml index c90a255a..1c5f27ac 100644 --- a/roles/data/vars/main.yml +++ b/roles/data/vars/main.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -data__aws_policy_urls_default_root: "{{ common__aws_policy_urls_default_root }}" +data__aws_policy_urls_default_root: "{{ common__aws_policy_urls_default_root }}" data__aws_policy_urls_default: - read_only: "{{ data__aws_policy_urls_default_root }}/aws-cdp-bucket-access-policy.json" - read_write: "{{ data__aws_policy_urls_default_root }}/aws-cdp-datalake-admin-s3-policy.json" + read_only: "{{ data__aws_policy_urls_default_root }}/aws-cdp-bucket-access-policy.json" + read_write: "{{ data__aws_policy_urls_default_root }}/aws-cdp-datalake-admin-s3-policy.json" diff --git a/roles/dynamic_inventory/meta/argument_specs.yml b/roles/dynamic_inventory/meta/argument_specs.yml new file mode 100644 index 00000000..fcd9df1b --- /dev/null +++ b/roles/dynamic_inventory/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Provision dynamic inventory + description: Provision dynamic inventory. + author: Cloudera Labs + version_added: 2.0.0 diff --git a/roles/dynamic_inventory/tasks/create_static_inventory.yml b/roles/dynamic_inventory/tasks/create_static_inventory.yml index 01e16c9a..0485dcc1 100644 --- a/roles/dynamic_inventory/tasks/create_static_inventory.yml +++ b/roles/dynamic_inventory/tasks/create_static_inventory.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,8 +28,8 @@ loop_var: __infra_inventory_compute_item ansible.builtin.replace: name: "{{ __tmp_dynamic_inventory_artefact }}" - regexp: '(\s+){{ __infra_inventory_compute_item.0 }}(\s+)' - replace: '\1{{ __infra_inventory_compute_item.1 }}\2' + regexp: "(\\s+){{ __infra_inventory_compute_item.0 }}(\\s+)" + replace: "\\1{{ __infra_inventory_compute_item.1 }}\\2" - name: Stat the Temporary Artefact ansible.builtin.stat: @@ -54,7 +53,7 @@ ansible.builtin.copy: src: "{{ __tmp_inventory_static.stat.path }}" dest: "{{ init__dynamic_inventory_artefact }}" - force: yes + force: true - name: Remove Temporary Artefact file ansible.builtin.file: diff --git a/roles/dynamic_inventory/tasks/parse_inventory_template.yml b/roles/dynamic_inventory/tasks/parse_inventory_template.yml index 128909b9..e71598fb 100644 --- a/roles/dynamic_inventory/tasks/parse_inventory_template.yml +++ b/roles/dynamic_inventory/tasks/parse_inventory_template.yml @@ -1,3 +1,4 @@ +--- # Read in Dynamic Inventory - name: Seek Inventory Template in Definition Path register: __di_template_stat @@ -12,18 +13,18 @@ - __di_template_stat.stat.exists block: - name: Load in Dynamic Inventory Template - include_tasks: refresh_inventory.yml + ansible.builtin.include_tasks: refresh_inventory.yml vars: include_inventory_file: "{{ __di_template_stat.stat.path }}" - name: Print Dynamic Inventory groups to debug at Verbosity 3 - debug: + ansible.builtin.debug: msg: "{{ groups }}" verbosity: 3 - name: Check expected minimum host groups appear in Inventory ansible.builtin.assert: - quiet: yes + quiet: true that: - groups.cluster is defined - groups.cloudera_manager is defined @@ -37,7 +38,7 @@ - name: Set Dynamic Inventory host count in Globals when: __dynamic_inventory_host_list | length > 0 ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __di_entry | default(omit), recursive=True ) }}" + globals: "{{ globals | default({}) | combine(__di_entry | default(omit), recursive=True) }}" loop_control: loop_var: __di_entry loop: @@ -47,4 +48,4 @@ os: "{{ init__parcel_distro }}" always: - name: Remove Dynamic Inventory Template from current inventory - include_tasks: refresh_inventory.yml + ansible.builtin.include_tasks: refresh_inventory.yml diff --git a/roles/dynamic_inventory/tasks/refresh_inventory.yml b/roles/dynamic_inventory/tasks/refresh_inventory.yml index 0572ecf6..daa91a94 100644 --- a/roles/dynamic_inventory/tasks/refresh_inventory.yml +++ b/roles/dynamic_inventory/tasks/refresh_inventory.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,8 +36,7 @@ dest: "{{ __tmp_add_inventory_file }}" - name: Refresh inventory - meta: refresh_inventory - + ansible.builtin.meta: refresh_inventory - name: Remove temporary static inventory file when: - __add_inventory_static.stat.exists diff --git a/roles/dynamic_inventory/tasks/retire_static_inventory.yml b/roles/dynamic_inventory/tasks/retire_static_inventory.yml index 9eda6ef8..33db4709 100644 --- a/roles/dynamic_inventory/tasks/retire_static_inventory.yml +++ b/roles/dynamic_inventory/tasks/retire_static_inventory.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,13 +14,13 @@ # limitations under the License. - name: Check for a Static Inventory file - stat: + ansible.builtin.stat: path: "{{ init__dynamic_inventory_artefact }}" register: __inventory_static - name: Create a backup when: __inventory_static.stat.exists - copy: + ansible.builtin.copy: src: "{{ __inventory_static.stat.path }}" dest: "{{ [init__dynamic_inventory_artefact | splitext | first, ansible_date_time.epoch] | join('.') }}" diff --git a/roles/freeipa_client/defaults/main.yml b/roles/freeipa_client/defaults/main.yml index 25ec0b38..8d25c3c1 100644 --- a/roles/freeipa_client/defaults/main.yml +++ b/roles/freeipa_client/defaults/main.yml @@ -22,4 +22,4 @@ # ipaadmin_password: enable_dns: false -fallback_nameservers: [ "8.8.8.8" ] +fallback_nameservers: ["8.8.8.8"] diff --git a/roles/freeipa_client/handlers/main.yml b/roles/freeipa_client/handlers/main.yml index b2c3b6d1..2b0545c2 100644 --- a/roles/freeipa_client/handlers/main.yml +++ b/roles/freeipa_client/handlers/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: restart network +- name: Restart network ansible.builtin.service: name: NetworkManager state: restarted diff --git a/roles/freeipa_client/meta/argument_specs.yml b/roles/freeipa_client/meta/argument_specs.yml new file mode 100644 index 00000000..6be983a8 --- /dev/null +++ b/roles/freeipa_client/meta/argument_specs.yml @@ -0,0 +1,23 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy FreeIPA clients. + description: Deployment of FreeIPA clients for Cloudera Data Platform (CDP) Base and ECS. + author: + - Webster Mudge + - Jim Enright + - Chuck Levesque + version_added: 2.0.0 diff --git a/roles/freeipa_client/tasks/main.yml b/roles/freeipa_client/tasks/main.yml index 62fcb6d7..5e64b3d4 100644 --- a/roles/freeipa_client/tasks/main.yml +++ b/roles/freeipa_client/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +25,8 @@ when: ansible_facts['os_family'] == 'RedHat' block: - name: Check for existence of /etc/cloud/cloud.cfg - ansible.builtin.stat: path=/etc/cloud/cloud.cfg + ansible.builtin.stat: + path: /etc/cloud/cloud.cfg register: cloud_cfg - name: Set cloud-init to preserve hostname (RHEL) @@ -65,7 +65,7 @@ notify: restart network - name: Disable nm-cloud-setup if present - ignore_errors: yes + ignore_errors: true ansible.builtin.command: "{{ __nm_cloud_setup_disable_item }}" loop_control: loop_var: __nm_cloud_setup_disable_item @@ -89,7 +89,8 @@ notify: restart network - name: Check for existence of /etc/dhcp/dhclient.conf - ansible.builtin.stat: path=/etc/dhcp/dhclient.conf + ansible.builtin.stat: + path: /etc/dhcp/dhclient.conf register: dhclient_conf - name: Set /etc/dhcp/dhclient.conf for domain search and name servers @@ -121,4 +122,4 @@ ipaclient_servers: "{{ ipa_hosts }}" ipaserver_setup_dns: "{{ enable_dns }}" ipasssd_enable_dns_updates: "{{ enable_dns }}" - ipaclient_mkhomedir: yes + ipaclient_mkhomedir: true diff --git a/roles/freeipa_host_group/defaults/main.yml b/roles/freeipa_host_group/defaults/main.yml index 18a60da9..0f5cf008 100644 --- a/roles/freeipa_host_group/defaults/main.yml +++ b/roles/freeipa_host_group/defaults/main.yml @@ -19,9 +19,9 @@ # Role prefix is 'freeipa_host_group__' # Variables used as inputs in main.yml -freeipa_host_group__env_name: "{{ common__env_name }}" -freeipa_host_group__infra_type: "{{ common__infra_type }}" -freeipa_host_group__region: "{{ common__region }}" +freeipa_host_group__env_name: "{{ common__env_name }}" +freeipa_host_group__infra_type: "{{ common__infra_type }}" +freeipa_host_group__region: "{{ common__region }}" freeipa_host_group__gcp_project: "{{ common__gcp_project }}" diff --git a/roles/freeipa_host_group/meta/argument_specs.yml b/roles/freeipa_host_group/meta/argument_specs.yml new file mode 100644 index 00000000..e05a4a8c --- /dev/null +++ b/roles/freeipa_host_group/meta/argument_specs.yml @@ -0,0 +1,21 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Create FreeIPA host group + description: | + Creation of FreeIPA host group. + author: Jim Enright (jenright@cloudera.com) + version_added: 1.2.0 diff --git a/roles/freeipa_host_group/tasks/main.yml b/roles/freeipa_host_group/tasks/main.yml index 52464c58..281e2528 100644 --- a/roles/freeipa_host_group/tasks/main.yml +++ b/roles/freeipa_host_group/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +29,7 @@ when: freeipa_host_group__infra_type == "aws" block: - name: Gather EC2 instance information using ID - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ freeipa_host_group__region }}" instance_ids: - "{{ __freeipa_server_instance_id }}" @@ -56,7 +55,7 @@ project: "{{ freeipa_host_group__gcp_project }}" # Filter on the freeipa instance name with the timestamp stripped filters: - - "name : {{ __freeipa_server_instance_id | regex_replace('[^-]+$', '') }}*" + - "name : {{ __freeipa_server_instance_id | regex_replace('[^-]+$', '') }}*" register: __gcp_freeipa_address_info - name: Set facts for the FreeIPA server IP diff --git a/roles/freeipa_server/defaults/main.yml b/roles/freeipa_server/defaults/main.yml index b7335cfd..59b18bb8 100644 --- a/roles/freeipa_server/defaults/main.yml +++ b/roles/freeipa_server/defaults/main.yml @@ -22,7 +22,7 @@ #ipadm_password: # ipaserver_recursion_acl_cidr: -ipaserver_resolv_nameservers: [ '8.8.8.8' ] +ipaserver_resolv_nameservers: ["8.8.8.8"] ipaserver_server_recursion: true enable_dns: false -needs_python2: True +needs_python2: true diff --git a/roles/freeipa_server/handlers/main.yml b/roles/freeipa_server/handlers/main.yml index a2a89031..ea9ec813 100644 --- a/roles/freeipa_server/handlers/main.yml +++ b/roles/freeipa_server/handlers/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: restart network +- name: Restart network ansible.builtin.service: name: NetworkManager state: restarted -- name: restart dns +- name: Restart dns ansible.builtin.service: name: named-pkcs11 state: restarted diff --git a/roles/freeipa_server/meta/argument_specs.yml b/roles/freeipa_server/meta/argument_specs.yml new file mode 100644 index 00000000..7526bb2d --- /dev/null +++ b/roles/freeipa_server/meta/argument_specs.yml @@ -0,0 +1,23 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy FreeIPA server. + description: Deployment of sidecar FreeIPA Server for Cloudera Data Platform (CDP) Base and ECS. + author: + - Webster Mudge + - Jim Enright + - Chuck Levesque + version_added: 2.0.0 diff --git a/roles/freeipa_server/meta/main.yml b/roles/freeipa_server/meta/main.yml deleted file mode 100644 index a6b4e5a2..00000000 --- a/roles/freeipa_server/meta/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - role_name: freeipa_server - namespace: cloudera - author: Webster Mudge Jim Enright Chuck Levesque - description: > - Deployment of sidecar FreeIPA Server for Cloudera Data Platform (CDP) Base and ECS - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - freeipa diff --git a/roles/freeipa_server/tasks/main.yml b/roles/freeipa_server/tasks/main.yml index 9371c0d9..1001844f 100644 --- a/roles/freeipa_server/tasks/main.yml +++ b/roles/freeipa_server/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +33,7 @@ ansible.builtin.package: lock_timeout: 180 name: python2 - update_cache: yes + update_cache: true state: present when: needs_python2 @@ -42,7 +41,7 @@ ansible.builtin.package: lock_timeout: 180 name: python3 - update_cache: yes + update_cache: true state: present - name: Permissive SELinux @@ -62,7 +61,8 @@ when: ansible_facts['os_family'] == 'RedHat' block: - name: Check for existence of /etc/cloud/cloud.cfg - ansible.builtin.stat: path=/etc/cloud/cloud.cfg + ansible.builtin.stat: + path: /etc/cloud/cloud.cfg register: cloud_cfg - name: Set cloud-init to preserve hostname (RHEL) @@ -75,7 +75,8 @@ notify: restart network - name: Check for existence of /etc/NetworkManager/conf.d - ansible.builtin.stat: path=/etc/NetworkManager/conf.d + ansible.builtin.stat: + path: /etc/NetworkManager/conf.d register: nm_conf - name: Set /etc/NetworkManager/conf.d/disable-resolve.conf-managing.conf (RHEL) @@ -115,11 +116,12 @@ # Generated by Ansible 127.0.0.1 localhost {{ ansible_default_ipv4.address }} {{ inventory_hostname }} {{ inventory_hostname_short }} - backup: yes + backup: true notify: restart network - name: Check for existence of /etc/dhcp/dhclient.conf - ansible.builtin.stat: path=/etc/dhcp/dhclient.conf + ansible.builtin.stat: + path: /etc/dhcp/dhclient.conf register: dhclient_conf - name: Set /etc/dhcp/dhclient.conf for domain search and name servers @@ -150,8 +152,8 @@ - name: Configure DNS recursion for priv & pub IPs when: - - ipaserver_setup_dns - - ipaserver_server_recursion + - ipaserver_setup_dns + - ipaserver_server_recursion block: - name: Set up DNS recursion when: ipaserver_recursion_acl_cidr is defined @@ -178,7 +180,7 @@ block: - name: Update global DNS freeipa.ansible_freeipa.ipadnsconfig: - allow_sync_ptr: yes + allow_sync_ptr: true forward_policy: only ipaadmin_password: "{{ ipaadmin_password }}" rescue: diff --git a/roles/freeipa_server/vars/RedHat-7.yml b/roles/freeipa_server/vars/RedHat-7.yml index 40e3bee0..fd5e8156 100644 --- a/roles/freeipa_server/vars/RedHat-7.yml +++ b/roles/freeipa_server/vars/RedHat-7.yml @@ -14,5 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -ipaserver_packages: [ "ipa-server", "libselinux-python" ] -needs_python2: True +ipaserver_packages: ["ipa-server", "libselinux-python"] +needs_python2: true diff --git a/roles/freeipa_server/vars/RedHat-8.yml b/roles/freeipa_server/vars/RedHat-8.yml index b0c6a201..33bb7d08 100644 --- a/roles/freeipa_server/vars/RedHat-8.yml +++ b/roles/freeipa_server/vars/RedHat-8.yml @@ -14,5 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -ipaserver_packages: [ "@idm:DL1/server" ] -needs_python2: True +ipaserver_packages: ["@idm:DL1/server"] +needs_python2: true diff --git a/roles/freeipa_server/vars/RedHat-9.yml b/roles/freeipa_server/vars/RedHat-9.yml index d2de9dee..7aa439fa 100644 --- a/roles/freeipa_server/vars/RedHat-9.yml +++ b/roles/freeipa_server/vars/RedHat-9.yml @@ -14,6 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -ipaserver_packages: [ "ipa-server" ] -ipaserver_packages_dns: [ "ipa-server-dns" ] -needs_python2: False +ipaserver_packages: ["ipa-server"] +ipaserver_packages_dns: ["ipa-server-dns"] +needs_python2: false diff --git a/roles/freeipa_server/vars/default.yml b/roles/freeipa_server/vars/default.yml index 47961661..c32494e2 100644 --- a/roles/freeipa_server/vars/default.yml +++ b/roles/freeipa_server/vars/default.yml @@ -14,5 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -ipaserver_packages: [ "ipa-server", "python3-libselinux" ] -needs_python2: True +ipaserver_packages: ["ipa-server", "python3-libselinux"] +needs_python2: true diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml new file mode 100644 index 00000000..dd2801d7 --- /dev/null +++ b/roles/grafana/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy Grafana. + description: Deploy Grafana for Prometheus visualization. + author: Ronald Suplina + version_added: 2.4.0 diff --git a/roles/grafana/tasks/Grafana-CentOS.yml b/roles/grafana/tasks/Grafana-CentOS.yml index eced3c2a..c0fbcd8b 100644 --- a/roles/grafana/tasks/Grafana-CentOS.yml +++ b/roles/grafana/tasks/Grafana-CentOS.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/grafana/tasks/Grafana-Ubuntu.yml b/roles/grafana/tasks/Grafana-Ubuntu.yml index 3cd0cd52..715b0b3b 100644 --- a/roles/grafana/tasks/Grafana-Ubuntu.yml +++ b/roles/grafana/tasks/Grafana-Ubuntu.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +18,7 @@ - name: Update package list ansible.builtin.apt: - update_cache: yes + update_cache: true - name: Install a grafana .deb package ansible.builtin.apt: diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 92d23a4d..081b4b0e 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +12,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - - name: Include distribution specific tasks ansible.builtin.include_tasks: "Grafana-{{ ansible_facts['distribution'] }}.yml" diff --git a/roles/info/defaults/main.yml b/roles/info/defaults/main.yml index 54e89f06..c53e9795 100644 --- a/roles/info/defaults/main.yml +++ b/roles/info/defaults/main.yml @@ -18,7 +18,7 @@ # Role prefix is 'info__' -info__env_name: "{{ common__env_name }}" +info__env_name: "{{ common__env_name }}" -info__create_deployment_details: "{{ globals.artifacts.create_deployment_details | default(True) }}" -info__artifacts_directory: "{{ globals.artifacts.directory | default('') }}" +info__create_deployment_details: "{{ globals.artifacts.create_deployment_details | default(True) }}" +info__artifacts_directory: "{{ globals.artifacts.directory | default('') }}" diff --git a/roles/info/meta/argument_specs.yml b/roles/info/meta/argument_specs.yml new file mode 100644 index 00000000..7b683b3f --- /dev/null +++ b/roles/info/meta/argument_specs.yml @@ -0,0 +1,22 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Retrieve CDP details + description: | + Collection and publishing of available services and general information for Cloudera Data + Platform (CDP) Public Cloud or Private Cloud deployments. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/info/meta/main.yml b/roles/info/meta/main.yml deleted file mode 100644 index 6d115b78..00000000 --- a/roles/info/meta/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: Webster Mudge (wmudge@cloudera.com) - description: > - Collection and publishing of available services and general information for Cloudera Data - Platform (CDP) Public Cloud or Private Cloud deployments. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - aws - - gcloud - - azure - -dependencies: ['cloudera.exe.common'] diff --git a/roles/info/tasks/main.yml b/roles/info/tasks/main.yml index bc9a48d1..17a843a4 100644 --- a/roles/info/tasks/main.yml +++ b/roles/info/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +43,7 @@ - "{{ __artifacts_directory.stat.isdir }}" - "{{ __artifacts_directory.stat.writeable }}" fail_msg: "Artifacts directory is either not a directory or is not writeable" - quiet: yes + quiet: true - name: Set fact for the artifacts directory path ansible.builtin.set_fact: @@ -77,12 +76,12 @@ - name: Query CDP Operational DBs cloudera.cloud.opdb_info: - env: "{{ info__env_name }}" + env: "{{ info__env_name }}" register: __opdb_info - name: Query CDP DFX cloudera.cloud.df_service_info: - name: "{{ info__env_name }}" + name: "{{ info__env_name }}" register: __df_info - name: Set facts for the CDP deployment details diff --git a/roles/infrastructure/defaults/main.yml b/roles/infrastructure/defaults/main.yml index 28cdf665..a8a40dfa 100644 --- a/roles/infrastructure/defaults/main.yml +++ b/roles/infrastructure/defaults/main.yml @@ -19,162 +19,164 @@ # Role prefix is 'infra__' # Labels -infra__namespace: "{{ common__namespace }}" +infra__namespace: "{{ common__namespace }}" -infra__vpc_subnet_suffix: "{{ common__vpc_subnet_suffix }}" -infra__vpc_svcnet_suffix: "{{ common__vpc_svcnet_suffix }}" -infra__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" -infra__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" +infra__vpc_subnet_suffix: "{{ common__vpc_subnet_suffix }}" +infra__vpc_svcnet_suffix: "{{ common__vpc_svcnet_suffix }}" +infra__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" +infra__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" # Deployment type -infra__deployment_engine: "{{ common__infra_deployment_engine }}" +infra__deployment_engine: "{{ common__infra_deployment_engine }}" # Location of output from template module which creates Terraform -infra__terraform_template_dir: "{{ common__terraform_template_dir }}" -infra__terraform_artefact_dir: "{{ common__terraform_artefact_dir }}" -infra__terraform_workspace_dir: "{{ common__terraform_workspace_dir }}" +infra__terraform_template_dir: "{{ common__terraform_template_dir }}" +infra__terraform_artefact_dir: "{{ common__terraform_artefact_dir }}" +infra__terraform_workspace_dir: "{{ common__terraform_workspace_dir }}" -infra__terraform_allowed_state_storage: "{{ common__terraform_allowed_state_storage }}" -infra__terraform_state_storage: "{{ common__terraform_state_storage }}" -infra__terraform_remote_state_bucket: "{{ common__terraform_remote_state_bucket }}" +infra__terraform_allowed_state_storage: "{{ common__terraform_allowed_state_storage }}" +infra__terraform_state_storage: "{{ common__terraform_state_storage }}" +infra__terraform_remote_state_bucket: "{{ common__terraform_remote_state_bucket }}" infra__terraform_remote_state_lock_table: "{{ common__terraform_remote_state_lock_table }}" # Infra -infra__type: "{{ common__infra_type }}" -infra__tunnel: "{{ common__tunnel }}" -infra__public_endpoint_access: "{{ common__public_endpoint_access }}" +infra__type: "{{ common__infra_type }}" +infra__tunnel: "{{ common__tunnel }}" +infra__public_endpoint_access: "{{ common__public_endpoint_access }}" # Dynamic Inventory for Clusters -infra__private_key_file: "{{ globals.ssh.private_key_file | default('') }}" -infra__dynamic_inventory_count: "{{ globals.dynamic_inventory.vm.count | default(0) }}" -infra__dynamic_inventory_os: "{{ globals.dynamic_inventory.vm.os | default('el7') }}" -infra__dynamic_inventory_vm_suffix: "{{ infra.dynamic_inventory.vm.suffix | default('vm') }}" -infra__dynamic_inventory_vm_type: "{{ infra.dynamic_inventory.vm.type | default('std') }}" -infra__dynamic_inventory_storage_type: "{{ infra.dynamic_inventory.storage.type | default('std') }}" -infra__dynamic_inventory_storage_size: "{{ infra.dynamic_inventory.storage.size | default('200') }}" -infra__dynamic_inventory_tag_key: "{{ infra.dynamic_inventory.tag_key | default('cldr_deploy_namespace') }}" -infra__dynamic_inventory_tag_value: "{{ infra.dynamic_inventory.tag_value | default(infra__namespace) }}" -infra__dynamic_inventory_tag: "{{ infra.dynamic_inventory.tag | default({infra__dynamic_inventory_tag_key: infra__dynamic_inventory_tag_value }) }}" +infra__private_key_file: "{{ globals.ssh.private_key_file | default('') }}" +infra__dynamic_inventory_count: "{{ globals.dynamic_inventory.vm.count | default(0) }}" +infra__dynamic_inventory_os: "{{ globals.dynamic_inventory.vm.os | default('el7') }}" +infra__dynamic_inventory_vm_suffix: "{{ infra.dynamic_inventory.vm.suffix | default('vm') }}" +infra__dynamic_inventory_vm_type: "{{ infra.dynamic_inventory.vm.type | default('std') }}" +infra__dynamic_inventory_storage_type: "{{ infra.dynamic_inventory.storage.type | default('std') }}" +infra__dynamic_inventory_storage_size: "{{ infra.dynamic_inventory.storage.size | default('200') }}" +infra__dynamic_inventory_tag_key: "{{ infra.dynamic_inventory.tag_key | default('cldr_deploy_namespace') }}" +infra__dynamic_inventory_tag_value: "{{ infra.dynamic_inventory.tag_value | default(infra__namespace) }}" +infra__dynamic_inventory_tag: "{{ infra.dynamic_inventory.tag | default({infra__dynamic_inventory_tag_key: infra__dynamic_inventory_tag_value }) }}" infra__dynamic_inventory_delete_storage: "{{ infra.dynamic_inventory.storage.delete | default('yes') }}" # TODO: consider moving to globals? -infra__teardown_deletes_data: "{{ infra.teardown.delete_data | default(False) }}" -infra__teardown_deletes_ssh_key: "{{ infra.teardown.delete_ssh_key | default(False) }}" -infra__teardown_deletes_network: "{{ infra.teardown.delete_network | default(True) }}" -infra__teardown_auto_repo_mirror: "{{ infra.teardown.delete_mirror | default(False) }}" - -infra__region: "{{ common__region }}" - -infra__storage_name: "{{ common__storage_name }}" - -infra__logs_path: "{{ common__logs_path }}" -infra__data_path: "{{ common__data_path }}" -infra__backups_path: "{{ common__backups_path }}" -infra__ranger_audit_path: "{{ common__ranger_audit_path }}" - -infra__public_key_path: "{{ globals.ssh.key_path | default('~/.ssh') }}" -infra__public_key_id: "{{ common__public_key_id }}" -infra__public_key_file: "{{ common__public_key_file }}" -infra__public_key_text: "{{ common__public_key_text }}" - -infra__tags: "{{ common__tags }}" - -infra__vpc_name: "{{ common__vpc_name }}" -infra__vpc_cidr: "{{ infra.vpc.cidr | default('10.10.0.0/16') }}" -infra__vpc_svcnet_cidr: "{{ infra.vpc.service_network.subnet | default('10.10.192.0/28') }}" -infra__vpc_svcnet_name: "{{ infra.vpc.service_network.name | default([infra__namespace, infra__vpc_svcnet_suffix] | join('-')) }}" - -infra__vpc_public_subnet_cidrs: "{{ common__vpc_public_subnet_cidrs }}" -infra__vpc_private_subnet_cidrs: "{{ common__vpc_private_subnet_cidrs }}" - -infra__allow_ssh_access: "{{ infra.vpc.enable_ssh | default(True) }}" -infra__vpc_cloud_ports: "{{ infra.vpc.cloud_ports | default([443]) }}" -infra__vpc_base_ports: "{{ infra.vpc.base_ports | default([7180, 7183]) }}" -infra__vpc_ssh_ports: "{{ infra.vpc.ssh_ports | default([22]) }}" -infra__vpc_extra_ports: "{{ infra.vpc.extra_ports | default(common__setup_base | ternary(infra__vpc_base_ports, []) | union( common__setup_plat | ternary(infra__vpc_cloud_ports, [])) | union(infra__allow_ssh_access | ternary(infra__vpc_ssh_ports, [])) ) }}" -infra__vpc_extra_cidr: "{{ infra.vpc.extra_cidr | default([]) }}" -infra__vpc_user_ports: "{{ infra.vpc.user_ports | default([infra__all_ports_security_rule[infra__type]]) }}" -infra__vpc_user_cidr: "{{ infra.vpc.user_cidr | default([]) }}" -infra__vpc_tunneled_cidr: "{{ infra.vpc.tunneled_cidr | default([]) }}" - -infra__aws_vpc_id: "{{ common__aws_vpc_id }}" -infra__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" -infra__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" - -infra__security_group_knox_name: "{{ common__security_group_knox_name }}" +infra__teardown_deletes_data: "{{ infra.teardown.delete_data | default(False) }}" +infra__teardown_deletes_ssh_key: "{{ infra.teardown.delete_ssh_key | default(False) }}" +infra__teardown_deletes_network: "{{ infra.teardown.delete_network | default(True) }}" +infra__teardown_auto_repo_mirror: "{{ infra.teardown.delete_mirror | default(False) }}" + +infra__region: "{{ common__region }}" + +infra__storage_name: "{{ common__storage_name }}" + +infra__logs_path: "{{ common__logs_path }}" +infra__data_path: "{{ common__data_path }}" +infra__backups_path: "{{ common__backups_path }}" +infra__ranger_audit_path: "{{ common__ranger_audit_path }}" + +infra__public_key_path: "{{ globals.ssh.key_path | default('~/.ssh') }}" +infra__public_key_id: "{{ common__public_key_id }}" +infra__public_key_file: "{{ common__public_key_file }}" +infra__public_key_text: "{{ common__public_key_text }}" + +infra__tags: "{{ common__tags }}" + +infra__vpc_name: "{{ common__vpc_name }}" +infra__vpc_cidr: "{{ infra.vpc.cidr | default('10.10.0.0/16') }}" +infra__vpc_svcnet_cidr: "{{ infra.vpc.service_network.subnet | default('10.10.192.0/28') }}" +infra__vpc_svcnet_name: "{{ infra.vpc.service_network.name | default([infra__namespace, infra__vpc_svcnet_suffix] | join('-')) }}" + +infra__vpc_public_subnet_cidrs: "{{ common__vpc_public_subnet_cidrs }}" +infra__vpc_private_subnet_cidrs: "{{ common__vpc_private_subnet_cidrs }}" + +infra__allow_ssh_access: "{{ infra.vpc.enable_ssh | default(True) }}" +infra__vpc_cloud_ports: "{{ infra.vpc.cloud_ports | default([443]) }}" +infra__vpc_base_ports: "{{ infra.vpc.base_ports | default([7180, 7183]) }}" +infra__vpc_ssh_ports: "{{ infra.vpc.ssh_ports | default([22]) }}" +infra__vpc_extra_ports: "{{ infra.vpc.extra_ports | default(common__setup_base | ternary(infra__vpc_base_ports, []) | union( common__setup_plat | ternary(infra__vpc_cloud_ports, + [])) | union(infra__allow_ssh_access | ternary(infra__vpc_ssh_ports, [])) ) }}" +infra__vpc_extra_cidr: "{{ infra.vpc.extra_cidr | default([]) }}" +infra__vpc_user_ports: "{{ infra.vpc.user_ports | default([infra__all_ports_security_rule[infra__type]]) }}" +infra__vpc_user_cidr: "{{ infra.vpc.user_cidr | default([]) }}" +infra__vpc_tunneled_cidr: "{{ infra.vpc.tunneled_cidr | default([]) }}" + +infra__aws_vpc_id: "{{ common__aws_vpc_id }}" +infra__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" +infra__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" + +infra__security_group_knox_name: "{{ common__security_group_knox_name }}" infra__security_group_default_name: "{{ common__security_group_default_name }}" -infra__security_group_vpce_name: "{{ common__security_group_vpce_name }}" +infra__security_group_vpce_name: "{{ common__security_group_vpce_name }}" -infra__ml_deploy: "{{ common__include_ml }}" -infra__ml_path: "{{ common__ml_path }}" +infra__ml_deploy: "{{ common__include_ml }}" +infra__ml_path: "{{ common__ml_path }}" -infra__de_deploy: "{{ common__include_de }}" -infra__de_path: "{{ common__de_path }}" +infra__de_deploy: "{{ common__include_de }}" +infra__de_path: "{{ common__de_path }}" # AWS -infra__aws_profile: "{{ common__aws_profile }}" -infra__aws_vpc_az_count: "{{ infra.aws.vpc.az_count | default(3) }}" -infra__aws_igw_name: "{{ infra.aws.vpc.internet_gateway.name | default([infra__namespace, infra__aws_igw_suffix] | join('-')) }}" -infra__aws_igw_suffix: "{{ infra.aws.vpc.internet_gateway.suffix | default(common__igw_suffix) }}" - -infra__aws_prefix_list_suffix: "{{ infra.aws.vpc.labels.prefix_list_suffix | default(common__aws_prefix_list_suffix) }}" -infra__aws_extra_prefix_list_name: "{{ infra.aws.extra_prefix_list_name | default([infra__namespace, 'extra', infra__aws_prefix_list_suffix] | join('-')) }}" -infra__aws_public_route_table_suffix: "{{ infra.aws.vpc.labels.public_route_table_suffix | default('public-rtb') }}" +infra__aws_profile: "{{ common__aws_profile }}" +infra__aws_vpc_az_count: "{{ infra.aws.vpc.az_count | default(3) }}" +infra__aws_igw_name: "{{ infra.aws.vpc.internet_gateway.name | default([infra__namespace, infra__aws_igw_suffix] | join('-')) }}" +infra__aws_igw_suffix: "{{ infra.aws.vpc.internet_gateway.suffix | default(common__igw_suffix) }}" + +infra__aws_prefix_list_suffix: "{{ infra.aws.vpc.labels.prefix_list_suffix | default(common__aws_prefix_list_suffix) }}" +infra__aws_extra_prefix_list_name: "{{ infra.aws.extra_prefix_list_name | default([infra__namespace, 'extra', infra__aws_prefix_list_suffix] | join('-')) }}" +infra__aws_public_route_table_suffix: "{{ infra.aws.vpc.labels.public_route_table_suffix | default('public-rtb') }}" infra__aws_private_route_table_suffix: "{{ infra.aws.vpc.labels.private_route_table_suffix | default('private-rtb') }}" infra__aws_public_route_table_name: "{{ infra.aws.vpc.labels.public_route_table | default([infra__namespace, infra__aws_public_route_table_suffix] | join('-')) }}" -infra__aws_private_route_table_name: "{{ infra.aws.vpc.labels.private_route_table | default([infra__namespace, infra__aws_private_route_table_suffix] | join('-')) }}" +infra__aws_private_route_table_name: "{{ infra.aws.vpc.labels.private_route_table | default([infra__namespace, infra__aws_private_route_table_suffix] | join('-')) + }}" -infra__aws_nat_gateway_name: "{{ infra.aws.vpc.nat_gateway.name | default([infra__namespace, infra__aws_nat_gateway_suffix] | join('-')) }}" -infra__aws_nat_gateway_suffix: "{{ infra.aws.vpc.nat_gateway.suffix | default(common__ngw_suffix) }}" +infra__aws_nat_gateway_name: "{{ infra.aws.vpc.nat_gateway.name | default([infra__namespace, infra__aws_nat_gateway_suffix] | join('-')) }}" +infra__aws_nat_gateway_suffix: "{{ infra.aws.vpc.nat_gateway.suffix | default(common__ngw_suffix) }}" -infra__aws_role_tags: "{{ infra.aws.role.tags | default({}) }}" -infra__aws_policy_tags: "{{ infra.aws.policy.tags | default({}) }}" -infra__aws_storage_tags: "{{ infra.aws.storage.tags | default({}) }}" +infra__aws_role_tags: "{{ infra.aws.role.tags | default({}) }}" +infra__aws_policy_tags: "{{ infra.aws.policy.tags | default({}) }}" +infra__aws_storage_tags: "{{ infra.aws.storage.tags | default({}) }}" infra__aws_bucket_object_ownership: "{{ infra.aws.storage.bucket_object_ownership | default('BucketOwnerPreferred')}}" -infra__aws_bucket_acl_permissions: "{{ infra.aws.storage.bucket_acl_permissions | default(['bucket-owner-full-control']) }}" -infra__aws_private_endpoints: "{{ infra.aws.vpc.private_endpoints | default(common__tunnel) }}" +infra__aws_bucket_acl_permissions: "{{ infra.aws.storage.bucket_acl_permissions | default(['bucket-owner-full-control']) }}" +infra__aws_private_endpoints: "{{ infra.aws.vpc.private_endpoints | default(common__tunnel) }}" # GCP -infra__gcp_project: "{{ common__gcp_project }}" +infra__gcp_project: "{{ common__gcp_project }}" -infra__gcp_storage_location_data: "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}" -infra__gcp_storage_location_logs: "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}" +infra__gcp_storage_location_data: "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}" +infra__gcp_storage_location_logs: "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}" infra__gcp_storage_location_backups: "{{ infra.gcp.storage.path.backups | default(infra__gcp_storage_location_logs) }}" -infra__gcp_cloud_router_name_suffix: "{{ infra.gcp.network.router.name_suffix | default('router') }}" -infra__gcp_cloud_router_name: "{{ infra.gcp.network.router.name | default([infra__namespace, infra__gcp_cloud_router_name_suffix] | join('-')) }}" -infra__gcp_cloud_router_asn: "{{ infra.gcp.network.router.asn | default(64514) }}" +infra__gcp_cloud_router_name_suffix: "{{ infra.gcp.network.router.name_suffix | default('router') }}" +infra__gcp_cloud_router_name: "{{ infra.gcp.network.router.name | default([infra__namespace, infra__gcp_cloud_router_name_suffix] | join('-')) }}" +infra__gcp_cloud_router_asn: "{{ infra.gcp.network.router.asn | default(64514) }}" -infra__gcp_cloud_nat_name_suffix: "{{ infra.gcp.network.nat.name_suffix | default('nat') }}" -infra__gcp_cloud_nat_name: "{{ infra.gcp.network.nat.name | default([infra__namespace, infra__gcp_cloud_nat_name_suffix] | join('-')) }}" +infra__gcp_cloud_nat_name_suffix: "{{ infra.gcp.network.nat.name_suffix | default('nat') }}" +infra__gcp_cloud_nat_name: "{{ infra.gcp.network.nat.name | default([infra__namespace, infra__gcp_cloud_nat_name_suffix] | join('-')) }}" # Azure -infra__azure_metagroup_name: "{{ common__azure_metagroup_name }}" +infra__azure_metagroup_name: "{{ common__azure_metagroup_name }}" -infra__azure_storage_name: "{{ common__azure_storage_name }}" -infra__azure_storage_class: "{{ infra.azure.storage.class | default('Standard_LRS') }}" -infra__azure_storage_kind: "{{ infra.azure.storage.type | default('StorageV2') }}" +infra__azure_storage_name: "{{ common__azure_storage_name }}" +infra__azure_storage_class: "{{ infra.azure.storage.class | default('Standard_LRS') }}" +infra__azure_storage_kind: "{{ infra.azure.storage.type | default('StorageV2') }}" -infra__azure_netapp_account_name: "{{ common__azure_netapp_account_name }}" -infra__azure_netapp_pool_name: "{{ common__azure_netapp_pool_name }}" -infra__azure_netapp_vol_name: "{{ common__azure_netapp_vol_name }}" -infra__azure_netapp_nfs_version: "{{ common__azure_netapp_nfs_version }}" +infra__azure_netapp_account_name: "{{ common__azure_netapp_account_name }}" +infra__azure_netapp_pool_name: "{{ common__azure_netapp_pool_name }}" +infra__azure_netapp_vol_name: "{{ common__azure_netapp_vol_name }}" +infra__azure_netapp_nfs_version: "{{ common__azure_netapp_nfs_version }}" -infra__azure_netapp_pool_size: "{{ infra.azure.netapp.pool.size | default(1) }}" # 4TB 'chunks' -infra__azure_netapp_pool_type: "{{ infra.azure.netapp.pool.type | default('Standard') }}" -infra__azure_netapp_vol_size: "{{ infra.azure.netapp.volume.size | default(500) }}" -infra__azure_netapp_vol_type: "{{ infra.azure.netapp.volume.type | default('Standard') }}" +infra__azure_netapp_pool_size: "{{ infra.azure.netapp.pool.size | default(1) }}" # 4TB 'chunks' +infra__azure_netapp_pool_type: "{{ infra.azure.netapp.pool.type | default('Standard') }}" +infra__azure_netapp_vol_size: "{{ infra.azure.netapp.volume.size | default(500) }}" +infra__azure_netapp_vol_type: "{{ infra.azure.netapp.volume.type | default('Standard') }}" -infra__azure_sp_login_from_env: "{{ common__azure_sp_login_from_env }}" +infra__azure_sp_login_from_env: "{{ common__azure_sp_login_from_env }}" # CDP Control Plane -infra__cdp_control_plane_ports: "{{ env.cdp.control_plane.ports | default([443, 9443]) }}" -infra__cdp_control_plane_cidr: "{{ env.cdp.control_plane.cidr | default(infra__cdp_control_plane_cidr_default) }}" +infra__cdp_control_plane_ports: "{{ env.cdp.control_plane.ports | default([443, 9443]) }}" +infra__cdp_control_plane_cidr: "{{ env.cdp.control_plane.cidr | default(infra__cdp_control_plane_cidr_default) }}" # Utility Service for Download Mirror -infra__create_utility_service: "{{ globals.create_utility_service | default('no') | bool }}" -infra__utlity_bucket_name: "{{ globals.utility_bucket_name | default('') }}" +infra__create_utility_service: "{{ globals.create_utility_service | default('no') | bool }}" +infra__utlity_bucket_name: "{{ globals.utility_bucket_name | default('') }}" #infra__auto_repo_mirror_bucket_prefix: "{{ auto_repo_mirror_prefix | default('cache') }}" # Teardown -infra__force_teardown: "{{ common__force_teardown }}" -infra__env_name: "{{ common__env_name }}" # Used for purge lookups +infra__force_teardown: "{{ common__force_teardown }}" +infra__env_name: "{{ common__env_name }}" # Used for purge lookups diff --git a/roles/infrastructure/meta/argument_specs.yml b/roles/infrastructure/meta/argument_specs.yml new file mode 100644 index 00000000..76bce668 --- /dev/null +++ b/roles/infrastructure/meta/argument_specs.yml @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy cloud provider infrastructure + description: | + Deployment and management of cloud provider and/or infrastucture provider + artifacts for Cloudera Data Platform (CDP) Public Cloud and Private Cloud, + including Private Cloud Base, in addition to artifacts required for auxillary + services, e.g. Keycloak, external data locations. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/infrastructure/meta/main.yml b/roles/infrastructure/meta/main.yml deleted file mode 100644 index 2a233bfa..00000000 --- a/roles/infrastructure/meta/main.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: Webster Mudge (wmudge@cloudera.com) - description: > - Deployment and management of cloud provider and/or infrastucture provider - artifacts for Cloudera Data Platform (CDP) Public Cloud and Private Cloud, - including Private Cloud Base, in addition to artifacts required for auxillary - services, e.g. Keycloak, external data locations. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - aws - - gcloud - - azure - - openstack - -dependencies: ['cloudera.exe.common'] diff --git a/roles/infrastructure/tasks/initialize_aws.yml b/roles/infrastructure/tasks/initialize_aws.yml index d32dbc28..16c8ae0b 100644 --- a/roles/infrastructure/tasks/initialize_aws.yml +++ b/roles/infrastructure/tasks/initialize_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,7 +47,7 @@ - __aws_private_subnets_info.subnets | map(attribute='availability_zone') | list | unique | count >= infra__aws_vpc_az_count | int - __aws_private_subnets_info.subnets | map(attribute='map_public_ip_on_launch') | reject() | length > 0 fail_msg: "The private subnets should be provided from at least 2 AZs and should have public IP addressing disabled." - quiet: yes + quiet: true - name: Generate Private Subnet details for update ansible.builtin.set_fact: @@ -59,11 +58,13 @@ loop: "{{ __aws_private_subnets_info.subnets }}" vars: entry: - name: "{{ __private_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx|string] | join('-')) }}" + name: "{{ __private_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx | string] | join('-')) + }}" cidr: "{{ __private_subnet_item.cidr_block }}" tags: "kubernetes.io/role/internal-elb": "1" - "Name": "{{ __private_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx|string] | join('-')) }}" + "Name": "{{ __private_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx | string] | join('-')) + }}" - name: Set facts for existing AWS Private Subnet IDs and associate VPC ID ansible.builtin.set_fact: @@ -85,7 +86,7 @@ - __aws_public_subnets_info.subnets | map(attribute='availability_zone') | list | unique | count >= infra__aws_vpc_az_count | int - __aws_public_subnets_info.subnets | map(attribute='map_public_ip_on_launch') | select() | length > 0 fail_msg: "The public subnets should be associated with at least 2 AZs and should have public IP addressing enabled." - quiet: yes + quiet: true - name: Confirm additional AWS Public Subnet AZ count if providing Public Endpoint access when: infra__public_endpoint_access @@ -94,7 +95,7 @@ - infra__aws_private_subnet_ids - infra__aws_public_subnet_ids | unique | count >= __aws_private_subnets_info.subnets | map(attribute='availability_zone') | list | unique | count fail_msg: "The number of public subnets should be at least as many as the number of associated AZs of the private subnets." - quiet: yes + quiet: true - name: Generate Public Subnet details for update ansible.builtin.set_fact: @@ -105,13 +106,15 @@ loop: "{{ __aws_public_subnets_info.subnets }}" vars: entry: - name: "{{ __public_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx|string] | join('-')) }}" + name: "{{ __public_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx | string] | join('-')) + }}" cidr: "{{ __public_subnet_item.cidr_block }}" map_public: "{{ __public_subnet_item.map_public_ip_on_launch }}" assign_instances_ipv6: "{{ __public_subnet_item.assign_ipv6_address_on_creation }}" tags: "kubernetes.io/role/elb": "1" - "Name": "{{ __public_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx|string] | join('-')) }}" + "Name": "{{ __public_subnet_item.tags['Name'] | default([infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx | string] | join('-')) + }}" - name: Set facts for existing AWS Public Subnet IDs ansible.builtin.set_fact: @@ -142,7 +145,8 @@ block: # Using CLI due to hardcoded module behavior - see https://github.com/boto/boto3/issues/2929 - name: Query AWS VPC ID by unique name and CIDR (CLI version) - ansible.builtin.command: "aws ec2 describe-vpcs --region {{ infra__region }} --filters Name=cidr,Values={{ infra__vpc_cidr }} Name=tag:Name,Values={{ infra__vpc_name }}" + ansible.builtin.command: "aws ec2 describe-vpcs --region {{ infra__region }} --filters Name=cidr,Values={{ infra__vpc_cidr }} Name=tag:Name,Values={{ infra__vpc_name + }}" register: __aws_vpc_list_discovered - name: Check VPC list for singular response (CLI version) @@ -174,7 +178,7 @@ - name: Fetch EC2 Instance info for Dynamic Inventory Nodes register: __infra_dynamic_inventory_discovered - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ infra__region }}" filters: "{{ __filters | items2dict }}" vars: @@ -184,7 +188,7 @@ - name: Fetch EC2 Instance info for Utility VM register: __infra_utility_compute_discovered - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ infra__region }}" filters: "{{ __filters | items2dict }}" vars: diff --git a/roles/infrastructure/tasks/initialize_aws_terraform.yml b/roles/infrastructure/tasks/initialize_aws_terraform.yml index 5ba15375..c0b5805f 100644 --- a/roles/infrastructure/tasks/initialize_aws_terraform.yml +++ b/roles/infrastructure/tasks/initialize_aws_terraform.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,13 +26,13 @@ # Copy Terraform provider file - name: Copy Terraform Provider file ansible.builtin.copy: - src: 'files/{{ infra__type }}/provider.tf' + src: "files/{{ infra__type }}/provider.tf" dest: "{{ infra__terraform_template_dir }}/infra/provider.tf" # Copy Terraform variables file - name: Copy Terraform Variables declaration file ansible.builtin.copy: - src: 'files/{{ infra__type }}/terraform_variables.tf' + src: "files/{{ infra__type }}/terraform_variables.tf" dest: "{{ infra__terraform_template_dir }}/infra/variables.tf" no_log: false @@ -39,33 +40,33 @@ # ...network resources - name: Copy Terraform resource file for network resources ansible.builtin.copy: - src: 'files/{{ infra__type }}/infra_{{ infra__type }}_network.tf' + src: "files/{{ infra__type }}/infra_{{ infra__type }}_network.tf" dest: "{{ infra__terraform_template_dir }}/infra/infra_network.tf" no_log: false # ...storage resources - name: Copy Terraform resource file for storage resources ansible.builtin.copy: - src: 'files/{{ infra__type }}/infra_{{ infra__type }}_storage.tf' + src: "files/{{ infra__type }}/infra_{{ infra__type }}_storage.tf" dest: "{{ infra__terraform_template_dir }}/infra/infra_storage.tf" no_log: false # ...compute resources - name: Generating Terraform infra file for compute resources ansible.builtin.copy: - src: 'files/{{ infra__type }}/infra_{{ infra__type }}_compute.tf' + src: "files/{{ infra__type }}/infra_{{ infra__type }}_compute.tf" dest: "{{ infra__terraform_template_dir }}/infra/infra_compute.tf" no_log: false # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'templates/{{ infra__type }}/backend_state.tf.j2' + src: "templates/{{ infra__type }}/backend_state.tf.j2" dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" # Create Terraform variable definitions from template - name: Generate Terraform Variables definition ansible.builtin.template: - src: 'templates/{{ infra__type }}/terraform.tfvars.j2' + src: "templates/{{ infra__type }}/terraform.tfvars.j2" dest: "{{ infra__terraform_template_dir }}/infra/terraform.tfvars" no_log: false diff --git a/roles/infrastructure/tasks/initialize_azure.yml b/roles/infrastructure/tasks/initialize_azure.yml index 9f10d25e..90e119e2 100644 --- a/roles/infrastructure/tasks/initialize_azure.yml +++ b/roles/infrastructure/tasks/initialize_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,25 +23,25 @@ --tenant "{{ lookup('env','AZURE_TENANT') }}" async: 30 poll: 5 - no_log: yes + no_log: true - name: Check Azure CLI Account to ensure functionality and user access register: __azure_account_info - ansible.builtin.command: "az account show" # Calling the CLI version doesn't test much of the install + ansible.builtin.command: "az account show" # Calling the CLI version doesn't test much of the install - name: Confirm Azure CLI ansible.builtin.assert: that: - "__azure_account_info.rc == 0" fail_msg: "Azure CLI failed to execute: {{ __azure_account_info.stdout }} ({{ __azure_account_info.stderr }})" - quiet: yes + quiet: true - name: Fetch Netapp subnet URI if it exists azure.azcollection.azure_rm_subnet_info: name: "{{ infra__vpc_svcnet_name }}" resource_group: "{{ infra__azure_metagroup_name }}" virtual_network_name: "{{ infra__vpc_name }}" - ignore_errors: yes # Microsoft throws errors instead of reporting a simple 'not found'. + ignore_errors: true # Microsoft throws errors instead of reporting a simple 'not found'. register: __azure_netapp_subnet_info - name: Set fact NetApp Subnet URI if exists diff --git a/roles/infrastructure/tasks/initialize_base.yml b/roles/infrastructure/tasks/initialize_base.yml index dba39c83..e2baa8f9 100644 --- a/roles/infrastructure/tasks/initialize_base.yml +++ b/roles/infrastructure/tasks/initialize_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Get public IP from Ipify community.general.ipify_facts: timeout: 20 - ignore_errors: yes + ignore_errors: true - name: Alternate Ipify when: ipify_public_ip is not defined @@ -25,7 +24,7 @@ - name: Fetch Public IP From Icanhazip ansible.builtin.uri: url: https://ipv4.icanhazip.com - return_content: yes + return_content: true register: __public_ip_info - name: Set Alternate Ipify fact @@ -48,11 +47,11 @@ loop: "{{ infra__vpc_public_subnet_cidrs }}" vars: entry: - name: "{{ [infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx|string] | join('-') }}" + name: "{{ [infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx | string] | join('-') }}" cidr: "{{ __public_subnet_item }}" tags: "kubernetes.io/role/elb": "1" - "Name": "{{ [infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx|string] | join('-') }}" + "Name": "{{ [infra__namespace, infra__vpc_public_subnets_suffix, __public_subnet_idx | string] | join('-') }}" - name: Generate Private Subnet Details for creation when: not infra__aws_private_subnet_ids @@ -64,8 +63,8 @@ loop: "{{ infra__vpc_private_subnet_cidrs }}" vars: entry: - name: "{{ [infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx|string] | join('-') }}" + name: "{{ [infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx | string] | join('-') }}" cidr: "{{ __private_subnet_item }}" tags: "kubernetes.io/role/internal-elb": "1" - "Name": "{{ [infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx|string] | join('-') }}" + "Name": "{{ [infra__namespace, infra__vpc_private_subnets_suffix, __private_subnet_idx | string] | join('-') }}" diff --git a/roles/infrastructure/tasks/initialize_gcp.yml b/roles/infrastructure/tasks/initialize_gcp.yml index a04e2f04..5eaec212 100644 --- a/roles/infrastructure/tasks/initialize_gcp.yml +++ b/roles/infrastructure/tasks/initialize_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +22,7 @@ that: - "__gcloud_cli.rc == 0" fail_msg: "GCP CLI failed to execute: {{ __gcloud_cli.stdout }} ({{ __gcloud_cli.stderr }})" - quiet: yes + quiet: true - name: Get Google VPC Info if exists google.cloud.gcp_compute_network_info: @@ -36,7 +35,7 @@ ansible.builtin.assert: that: "__gcp_vpc_info.resources is defined" fail_msg: "Gcloud Collection failed to retrieve resources, you may need to run 'gcloud auth login' or 'gcloud init': {{ __gcp_vpc_info }}" - quiet: yes + quiet: true - name: Set fact for Log, Backups and Data locations ansible.builtin.set_fact: diff --git a/roles/infrastructure/tasks/initialize_setup.yml b/roles/infrastructure/tasks/initialize_setup.yml index 1ef30df6..967f80e3 100644 --- a/roles/infrastructure/tasks/initialize_setup.yml +++ b/roles/infrastructure/tasks/initialize_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/initialize_setup_aws.yml b/roles/infrastructure/tasks/initialize_setup_aws.yml index 1aabeb0c..faf1c4b2 100644 --- a/roles/infrastructure/tasks/initialize_setup_aws.yml +++ b/roles/infrastructure/tasks/initialize_setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +21,7 @@ ansible.builtin.assert: that: "__aws_cli.rc == 0" fail_msg: "AWS CLI failed to execute: {{ __aws_cli.stdout }} ({{ __aws_cli.stderr }})" - quiet: yes + quiet: true - name: Validate access to AWS Endpoints using supplied Credentials amazon.aws.aws_caller_info: @@ -43,10 +42,10 @@ ansible.builtin.assert: that: "__aws_az_info.availability_zones | length >= infra__aws_vpc_az_count | int" fail_msg: "Not enough Availability Zones in the AWS Region, found less than {{ infra__aws_vpc_az_count }} threshold" - quiet: yes + quiet: true - name: List available AWS EC2 Instance Types in region - command: "aws ec2 describe-instance-type-offerings --region {{ infra__region }}" + ansible.builtin.command: "aws ec2 describe-instance-type-offerings --region {{ infra__region }}" register: __aws_ec2_types - name: Check required AWS EC2 Instance Types @@ -54,13 +53,13 @@ that: "__ec2_type_item.value in (__aws_ec2_types.stdout | from_json | community.general.json_query(query))" fail_msg: | EC2 Instance Type {{ __ec2_type_item.value }} is required, but not available in Region {{ infra__region }}. - You might try 'aws ec2 describe-instance-type-offerings --filters Name=instance-type,Values={{__ec2_type_item.value}}'" - quiet: yes + You might try 'aws ec2 describe-instance-type-offerings --filters Name=instance-type,Values={{ __ec2_type_item.value }}'" + quiet: true loop_control: loop_var: __ec2_type_item loop: "{{ infra__dynamic_inventory_vm_type_default[infra__type] | dict2items }}" vars: - query: 'InstanceTypeOfferings[].InstanceType' + query: "InstanceTypeOfferings[].InstanceType" - name: Find AMI for Dynamic Inventory Deployment when: infra__dynamic_inventory_count | int > 0 @@ -74,7 +73,7 @@ owners: "{{ infra__dynamic_inventory_images_default[infra__type][infra__dynamic_inventory_os].owners }}" register: __infra_aws_ami_list - - name: Filter to latest Image + - name: Filter to latest Image ansible.builtin.set_fact: __infra_aws_ami_info: "{{ __infra_aws_ami_list.images | selectattr('name', 'defined') | sort(attribute='creation_date') | last }}" @@ -142,7 +141,8 @@ - name: Set facts for dynamic inventory metadata ansible.builtin.set_fact: infra__dynamic_inventory_tags: "{{ infra__tags | combine(infra__dynamic_inventory_tag) }}" - infra__dynamic_inventory_connectors: "{{ 'ansible_user=' + infra__dynamic_inventory_images_default[infra__type][infra__dynamic_inventory_os].user + provided_ssh_private_key_file }}" + infra__dynamic_inventory_connectors: "{{ 'ansible_user=' + infra__dynamic_inventory_images_default[infra__type][infra__dynamic_inventory_os].user + provided_ssh_private_key_file + }}" vars: provided_ssh_private_key_file: "{{ (infra__private_key_file == '') | ternary('', ' ansible_ssh_private_key_file=' + infra__private_key_file) }}" diff --git a/roles/infrastructure/tasks/initialize_setup_azure.yml b/roles/infrastructure/tasks/initialize_setup_azure.yml index a932afaa..3d2d4c7d 100644 --- a/roles/infrastructure/tasks/initialize_setup_azure.yml +++ b/roles/infrastructure/tasks/initialize_setup_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Prepare Azure Security Group Rules ansible.builtin.set_fact: infra__azure_security_group_rules: - - name: 'AllowCallerAccess' + - name: "AllowCallerAccess" protocol: Tcp # TODO Add back in infra__vpc_user_cidr as an union of source addresses source_address_prefix: @@ -32,7 +31,7 @@ infra__azure_security_group_rules: "{{ infra__azure_security_group_rules | union(rule) }}" vars: rule: - - name: 'AllowCDPControlPlane' + - name: "AllowCDPControlPlane" protocol: Tcp source_address_prefix: "{{ infra__cdp_control_plane_cidr }}" destination_port_range: "{{ infra__cdp_control_plane_ports }}" @@ -46,7 +45,7 @@ infra__azure_security_group_rules: "{{ infra__azure_security_group_rules | union(rule) }}" vars: rule: - - name: 'AllowExtraAccess' + - name: "AllowExtraAccess" protocol: Tcp source_address_prefix: "{{ infra__vpc_extra_cidr }}" destination_port_range: "{{ infra__vpc_extra_ports }}" diff --git a/roles/infrastructure/tasks/initialize_setup_gcp.yml b/roles/infrastructure/tasks/initialize_setup_gcp.yml index 06d389e6..9fc20c2b 100644 --- a/roles/infrastructure/tasks/initialize_setup_gcp.yml +++ b/roles/infrastructure/tasks/initialize_setup_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/initialize_teardown.yml b/roles/infrastructure/tasks/initialize_teardown.yml index e8ab3a70..d81527ce 100644 --- a/roles/infrastructure/tasks/initialize_teardown.yml +++ b/roles/infrastructure/tasks/initialize_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/initialize_teardown_aws.yml b/roles/infrastructure/tasks/initialize_teardown_aws.yml index 6f4241d5..7683de1e 100644 --- a/roles/infrastructure/tasks/initialize_teardown_aws.yml +++ b/roles/infrastructure/tasks/initialize_teardown_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +15,7 @@ - name: List VPC Endpoints when: infra__aws_private_endpoints | bool - community.aws.ec2_vpc_endpoint_info: + amazon.aws.ec2_vpc_endpoint_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" @@ -39,7 +38,7 @@ - name: Discover all AWS VPC EC2 instances register: __infra_vpc_ec2_instances - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ infra__region }}" filters: "{{ __filters | items2dict }}" vars: @@ -53,8 +52,7 @@ - name: List AWS EKS clusters register: __infra_eks_cluster_list - command: "aws eks list-clusters --region {{ infra__region }}" - + ansible.builtin.command: "aws eks list-clusters --region {{ infra__region }}" - name: Describe all AWS EKS clusters register: __infra_eks_cluster_details ansible.builtin.command: "aws eks describe-cluster --name {{ __infra_eks_cluster_item }} --region {{ infra__region }}" @@ -122,14 +120,14 @@ #label: "{{ __infra_ec2_elb_item.load_balancer_name }}" - name: List all AWS VPC NAT gateways - community.aws.ec2_vpc_nat_gateway_info: + amazon.aws.ec2_vpc_nat_gateway_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" register: __infra_aws_nat_gateways - name: List all AWS VPC Security Groups - register: __infra_aws_sgs + register: __infra_aws_sgs amazon.aws.ec2_group_info: region: "{{ infra__region }}" filters: @@ -145,7 +143,7 @@ - name: Discover AWS RDS for forced teardown when: infra__force_teardown | bool block: - - name: list AWS RDS Instances in region + - name: List AWS RDS Instances in region community.aws.rds_instance_info: region: "{{ infra__region }}" register: __infra_aws_rds_list diff --git a/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml b/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml index 3672216c..3352677a 100644 --- a/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml +++ b/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,18 +21,18 @@ # Copy Terraform provider file - name: Copy Terraform Provider file ansible.builtin.copy: - src: 'files/{{ infra__type }}/provider.tf' + src: "files/{{ infra__type }}/provider.tf" dest: "{{ infra__terraform_template_dir }}/infra/provider.tf" # Copy Terraform variables file - name: Copy Terraform Variables declaration file ansible.builtin.copy: - src: 'files/{{ infra__type }}/terraform_variables.tf' + src: "files/{{ infra__type }}/terraform_variables.tf" dest: "{{ infra__terraform_template_dir }}/infra/variables.tf" no_log: false # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'templates/{{ infra__type }}/backend_state.tf.j2' + src: "templates/{{ infra__type }}/backend_state.tf.j2" dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" diff --git a/roles/infrastructure/tasks/main.yml b/roles/infrastructure/tasks/main.yml index 3211b7d8..61082f58 100644 --- a/roles/infrastructure/tasks/main.yml +++ b/roles/infrastructure/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include_tasks: validate.yml -- include_tasks: initialize_setup.yml -- include_tasks: setup.yml +- ansible.builtin.include_tasks: validate.yml +- ansible.builtin.include_tasks: initialize_setup.yml +- ansible.builtin.include_tasks: setup.yml diff --git a/roles/infrastructure/tasks/setup.yml b/roles/infrastructure/tasks/setup.yml index 0b179d9b..eea3a474 100644 --- a/roles/infrastructure/tasks/setup.yml +++ b/roles/infrastructure/tasks/setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,22 +21,22 @@ - name: Set up for Ansible deployment engine when: infra__deployment_engine == 'ansible' block: - - name: Set up provider-specific Infrastructure network - ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_network.yml" + - name: Set up provider-specific Infrastructure network + ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_network.yml" - - name: Set up provider-specific Infrastructure storage - ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_storage.yml" + - name: Set up provider-specific Infrastructure storage + ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_storage.yml" - - name: Set Up localised provider-specific Storage Utility Service - when: infra__create_utility_service - ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_utility_service.yml" + - name: Set Up localised provider-specific Storage Utility Service + when: infra__create_utility_service + ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_utility_service.yml" - - name: Set up provider-specific Infrastructure Compute - when: infra__dynamic_inventory_count | int > 0 - ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_compute.yml" + - name: Set up provider-specific Infrastructure Compute + when: infra__dynamic_inventory_count | int > 0 + ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_compute.yml" - name: Set up for Terraform deployment engine when: infra__deployment_engine == 'terraform' block: - - name: Set up Terraform Infrastructure artifacts - ansible.builtin.include_tasks: "setup_{{ infra__deployment_engine }}.yml" + - name: Set up Terraform Infrastructure artifacts + ansible.builtin.include_tasks: "setup_{{ infra__deployment_engine }}.yml" diff --git a/roles/infrastructure/tasks/setup_aws.yml b/roles/infrastructure/tasks/setup_aws.yml index d0d56e67..dff82044 100644 --- a/roles/infrastructure/tasks/setup_aws.yml +++ b/roles/infrastructure/tasks/setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +16,11 @@ - name: Ensure designated SSH keypair is in region amazon.aws.ec2_key: name: "{{ infra__public_key_id }}" - force: no + force: false key_material: "{{ (infra__public_key_text | length > 0) | ternary(infra__public_key_text, omit) }}" region: "{{ infra__region }}" state: present - no_log: yes + no_log: true register: __aws_ec2_ssh_keypair - name: Save the private key locally if created diff --git a/roles/infrastructure/tasks/setup_aws_compute.yml b/roles/infrastructure/tasks/setup_aws_compute.yml index b8d28b4a..61d71fa2 100644 --- a/roles/infrastructure/tasks/setup_aws_compute.yml +++ b/roles/infrastructure/tasks/setup_aws_compute.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,21 +25,21 @@ key_name: "{{ infra__public_key_id }}" instance_type: "{{ infra__dynamic_inventory_vm_type_default[infra__type][infra__dynamic_inventory_vm_type] }}" image_id: "{{ __infra_aws_ami_info.image_id }}" - ebs_optimized: yes + ebs_optimized: true volumes: - device_name: /dev/sda1 ebs: volume_type: "{{ infra__dynamic_inventory_storage_type_default[infra__type][infra__dynamic_inventory_storage_type] }}" volume_size: "{{ infra__dynamic_inventory_storage_size }}" delete_on_termination: "{{ infra__dynamic_inventory_delete_storage | bool }}" - wait: yes + wait: true state: running tags: "{{ infra__dynamic_inventory_tags }}" name: "{{ '-'.join([infra__namespace, infra__dynamic_inventory_vm_suffix, infra__dynamic_inventory_os[::2], '%02d' | format(__infra_compute_instance_item)]) }}" vpc_subnet_id: "{{ infra__aws_subnet_ids | first }}" network: - assign_public_ip: yes - delete_on_termination: yes + assign_public_ip: true + delete_on_termination: true - name: Ensure all {{ infra__dynamic_inventory_count }} instances have Public IPs assigned register: __infra_dynamic_inventory_instances @@ -48,7 +47,7 @@ region: "{{ infra__region }}" filters: "tag:Name": "{{ '-'.join([infra__namespace, infra__dynamic_inventory_vm_suffix, infra__dynamic_inventory_os[::2]]) }}-*" - instance-state-name: [ "running" ] + instance-state-name: ["running"] until: __infra_dynamic_inventory_instances.instances | selectattr('public_ip_address', 'defined') | list | count | int == infra__dynamic_inventory_count | int retries: 5 delay: 5 @@ -71,7 +70,7 @@ key_name: "{{ infra__public_key_id }}" instance_type: "{{ infra__dynamic_inventory_vm_type_default[infra__type]['sml'] }}" image_id: "{{ __infra_aws_ami_info.image_id }}" - ebs_optimized: yes + ebs_optimized: true instance_role: "{{ infra__auto_repo_mirror_role.iam_role.role_name }}" volumes: - device_name: /dev/sda1 @@ -79,13 +78,13 @@ volume_type: "{{ infra__dynamic_inventory_storage_type_default[infra__type]['std'] }}" volume_size: 100 delete_on_termination: true - wait: yes + wait: true state: running tags: "{{ infra__dynamic_inventory_tags }}" - name: "{{ '-'.join([infra__namespace, infra__region, 'utility_vm' ]) }}" + name: "{{ '-'.join([infra__namespace, infra__region, 'utility_vm']) }}" vpc_subnet_id: "{{ infra__aws_subnet_ids | first }}" network: - assign_public_ip: yes + assign_public_ip: true - name: Add Utility Instance to host group when: infra__create_utility_service diff --git a/roles/infrastructure/tasks/setup_aws_network.yml b/roles/infrastructure/tasks/setup_aws_network.yml index 557577c6..cb83b8b5 100644 --- a/roles/infrastructure/tasks/setup_aws_network.yml +++ b/roles/infrastructure/tasks/setup_aws_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ ansible.builtin.set_fact: __aws_vpc_tags: "{{ __aws_vpc_tags | default([]) | union([tag_entry]) }}" vars: - tag_entry: "Key={{ __tag.key }},Value={{ __tag.value |quote}}" + tag_entry: "Key={{ __tag.key }},Value={{ __tag.value | quote }}" loop: "{{ infra__tags | dict2items }}" loop_control: loop_var: __tag @@ -56,7 +55,7 @@ map_public: "{{ __aws_public_subnet_item.map_public }}" assign_instances_ipv6: "{{ __aws_public_subnet_item.assign_instances_ipv6 }}" state: present - purge_tags: no + purge_tags: false tags: "{{ infra__tags | combine(__aws_public_subnet_item.tags, recursive=True) }}" loop_control: loop_var: __aws_public_subnet_item @@ -70,7 +69,7 @@ vpc_id: "{{ infra__aws_vpc_id }}" cidr: "{{ __aws_private_subnet_item.cidr }}" state: present - purge_tags: no + purge_tags: false tags: "{{ infra__tags | combine(__aws_private_subnet_item.tags, recursive=True) }}" loop_control: loop_var: __aws_private_subnet_item @@ -78,15 +77,15 @@ loop: "{{ infra__vpc_private_subnets_info }}" - name: Set up AWS Public Network infrastructure - when: infra__aws_subnet_ids is undefined # L0 (fully public), L1 (semi-private), or L2 (fully-private), but not for existing network or L3 (outbound restricted) + when: infra__aws_subnet_ids is undefined # L0 (fully public), L1 (semi-private), or L2 (fully-private), but not for existing network or L3 (outbound restricted) # when: not (infra__tunnel and not infra__public_endpoint_access) and infra__aws_subnet_ids is undefined # L0 (all public) or L1 (public/private) block: - name: Create AWS Internet Gateway (IGW) - community.aws.ec2_vpc_igw: + amazon.aws.ec2_vpc_igw: vpc_id: "{{ infra__aws_vpc_id }}" region: "{{ infra__region }}" state: present - tags: "{{ infra__tags | combine({ 'Name': infra__aws_igw_name }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__aws_igw_name}, recursive=True) }}" register: __aws_igw - name: Set fact for AWS IGW ID @@ -101,7 +100,7 @@ cidr: "{{ __aws_public_subnet_item.cidr }}" state: present tags: "{{ infra__tags | combine(__aws_public_subnet_item.tags, recursive=True) }}" - map_public: yes + map_public: true az: "{{ __aws_az_info.availability_zones[__aws_subnet_index % infra__aws_vpc_az_count | int].zone_name }}" loop_control: loop_var: __aws_public_subnet_item @@ -130,7 +129,7 @@ route_table_id: "{{ __aws_route_table_id }}" lookup: id state: present - tags: "{{ infra__tags | combine({ 'Name': infra__aws_public_route_table_name }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__aws_public_route_table_name}, recursive=True) }}" routes: - dest: "0.0.0.0/0" gateway_id: "{{ infra__aws_igw_id }}" @@ -153,8 +152,8 @@ vpc_id: "{{ infra__aws_vpc_id }}" cidr: "{{ __aws_private_subnet_item.cidr }}" state: present - tags: "{{ infra__tags | combine(__aws_private_subnet_item.tags, recursive = true) }}" - map_public: no + tags: "{{ infra__tags | combine(__aws_private_subnet_item.tags, recursive=true) }}" + map_public: false az: "{{ __aws_az_info.availability_zones[__aws_subnet_index % infra__aws_vpc_az_count | int].zone_name }}" loop_control: loop_var: __aws_private_subnet_item @@ -171,25 +170,25 @@ loop: "{{ __aws_private_subnets.results }}" - name: Configure Private Route Table for the AWS VPC - when: no + when: false #when: not infra__public_endpoint_access community.aws.ec2_vpc_route_table: vpc_id: "{{ infra__aws_vpc_id }}" region: "{{ infra__region }}" - tags: "{{ infra__tags | combine({ 'Name': infra__aws_private_route_table_name }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__aws_private_route_table_name}, recursive=True) }}" subnets: "{{ infra__aws_private_subnet_ids }}" routes: [] - name: Create Network Gateways (NAT) # and allocate Elastic IP Addresses (EIP) for Public Endpoint Access #when: infra__public_endpoint_access # Might want a net new flag -- disconnected -- that will not set up any routes to outside, i.e. Level3 - community.aws.ec2_vpc_nat_gateway: + amazon.aws.ec2_vpc_nat_gateway: state: present subnet_id: "{{ __aws_public_subnet_id }}" wait: true if_exist_do_not_create: true region: "{{ infra__region }}" - tags: "{{ infra__tags | combine({ 'Name': '-'.join([infra__aws_nat_gateway_name, __aws_public_subnet_index | string]) }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': '-'.join([infra__aws_nat_gateway_name, __aws_public_subnet_index | string])}, recursive=True) }}" loop_control: loop_var: __aws_public_subnet_id index_var: __aws_public_subnet_index @@ -202,11 +201,11 @@ community.aws.ec2_vpc_route_table: vpc_id: "{{ infra__aws_vpc_id }}" region: "{{ infra__region }}" - tags: "{{ infra__tags | combine({ 'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string]) }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string])}, recursive=True) }}" subnets: "{{ __aws_private_subnet_id_item }}" routes: - - dest: "0.0.0.0/0" - nat_gateway_id: "{{ __aws_ngws.results[ __aws_private_subnet_id_index % __aws_ngws.results | length ].nat_gateway_id }}" + - dest: "0.0.0.0/0" + nat_gateway_id: "{{ __aws_ngws.results[__aws_private_subnet_id_index % __aws_ngws.results | length].nat_gateway_id }}" loop_control: loop_var: __aws_private_subnet_id_item index_var: __aws_private_subnet_id_index @@ -236,8 +235,8 @@ vpc_id: "{{ infra__aws_vpc_id }}" name: "{{ __security_group_name_item }}" description: "{{ __security_group_name_item }}" - tags: "{{ infra__tags | combine({ 'Name': __security_group_name_item }, recursive=True) }}" - rules: "{{ infra__aws_security_group_rules | union([rule])}}" + tags: "{{ infra__tags | combine({'Name': __security_group_name_item}, recursive=True) }}" + rules: "{{ infra__aws_security_group_rules | union([rule]) }}" vars: rule: proto: all @@ -265,7 +264,7 @@ state: present region: "{{ infra__region }}" vpc_id: "{{ infra__aws_vpc_id }}" - tags: "{{ infra__tags | combine({ 'Name': infra__security_group_vpce_name }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__security_group_vpce_name}, recursive=True) }}" name: "{{ infra__security_group_vpce_name }}" description: "{{ infra__security_group_vpce_name }}" rules: @@ -291,13 +290,13 @@ loop_var: __aws_route_tables - name: Create Gateway VPC Endpoints (Skip if infra__aws_private_endpoints is false) - community.aws.ec2_vpc_endpoint: + amazon.aws.ec2_vpc_endpoint: state: present region: "{{ infra__region }}" vpc_id: "{{ infra__aws_vpc_id }}" service: "{{ __infra_gateway_vpc_endpoint }}" route_table_ids: "{{ infra__route_table_ids }}" - tags: "{{ infra__tags | combine({ 'Name': infra__namespace + __infra_gateway_vpc_endpoint.split(infra__region)[1] }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__namespace + __infra_gateway_vpc_endpoint.split(infra__region)[1]}, recursive=True) }}" vars: route_table_id: "{{ route_tables.associations[0].route_table_id }}" loop: "{{ infra__aws_gateway_vpc_private_endpoints }}" @@ -306,14 +305,14 @@ register: __aws_gateway_vpc_endpoints - name: Create Interface VPC Endpoints (Skip if infra__aws_private_endpoints is false) - community.aws.ec2_vpc_endpoint: + amazon.aws.ec2_vpc_endpoint: state: present region: "{{ infra__region }}" vpc_id: "{{ infra__aws_vpc_id }}" service: "{{ __infra_interface_vpc_endpoint }}" vpc_endpoint_type: Interface wait: true - tags: "{{ infra__tags | combine({ 'Name': infra__namespace + __infra_interface_vpc_endpoint.split(infra__region)[1] }, recursive=True) }}" + tags: "{{ infra__tags | combine({'Name': infra__namespace + __infra_interface_vpc_endpoint.split(infra__region)[1]}, recursive=True) }}" loop: "{{ infra__aws_interface_vpc_private_endpoints }}" loop_control: loop_var: __infra_interface_vpc_endpoint @@ -331,7 +330,7 @@ when: - __aws_interface_vpc_endpoints is defined - __aws_interface_vpc_endpoints.results is defined - command: > + ansible.builtin.command: > aws ec2 modify-vpc-endpoint --region {{ infra__region }} --vpc-endpoint-id {{ __infra_vpce_loop_var.result.vpc_endpoint_id }} @@ -340,6 +339,6 @@ --remove-security-group-ids {{ __aws_vpc_default_sg.security_groups[0].group_id }} --private-dns-enabled loop_control: - label: "{{ __infra_vpce_loop_var.result.vpc_endpoint_id | default('') }}" # Default empty string to avoid ansible label undef error + label: "{{ __infra_vpce_loop_var.result.vpc_endpoint_id | default('') }}" # Default empty string to avoid ansible label undef error loop_var: __infra_vpce_loop_var loop: "{{ __aws_interface_vpc_endpoints.results }}" diff --git a/roles/infrastructure/tasks/setup_aws_network_prefix_list.yml b/roles/infrastructure/tasks/setup_aws_network_prefix_list.yml index 1da1a85c..9206060b 100644 --- a/roles/infrastructure/tasks/setup_aws_network_prefix_list.yml +++ b/roles/infrastructure/tasks/setup_aws_network_prefix_list.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,10 +41,7 @@ - name: Update AWS Prefix List maximum length when: infra__aws_extra_prefix_list_additions or infra__aws_extra_prefix_list_deletions ansible.builtin.command: - cmd: - aws ec2 modify-managed-prefix-list - --prefix-list-id {{ infra__aws_extra_prefix_list_id }} - --max-entries {{ infra__vpc_extra_cidr | count }} + cmd: aws ec2 modify-managed-prefix-list --prefix-list-id {{ infra__aws_extra_prefix_list_id }} --max-entries {{ infra__vpc_extra_cidr | count }} - name: Add entries to AWS Prefix List when: infra__aws_extra_prefix_list_additions diff --git a/roles/infrastructure/tasks/setup_aws_storage.yml b/roles/infrastructure/tasks/setup_aws_storage.yml index 09406836..da2a1d7d 100644 --- a/roles/infrastructure/tasks/setup_aws_storage.yml +++ b/roles/infrastructure/tasks/setup_aws_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +14,7 @@ # limitations under the License. - name: Warn user about HeadBucket Forbidden errors - debug: + ansible.builtin.debug: msg: - "If the 'Create AWS Buckets' task below fails. " - "If you see HeadBucket Forbidden errors. " @@ -47,7 +46,7 @@ region: "{{ infra__region }}" bucket: "{{ __aws_storage_object_item.bucket }}" object: "{{ __aws_storage_object_item.path }}" - mode: create # Put will not work here due to the way the s3 module is written + mode: create # Put will not work here due to the way the s3 module is written permission: "{{ infra__aws_bucket_acl_permissions }}" loop_control: loop_var: __aws_storage_object_item diff --git a/roles/infrastructure/tasks/setup_aws_utility_service.yml b/roles/infrastructure/tasks/setup_aws_utility_service.yml index ab1da6b2..1698c3b2 100644 --- a/roles/infrastructure/tasks/setup_aws_utility_service.yml +++ b/roles/infrastructure/tasks/setup_aws_utility_service.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,7 +39,7 @@ Version: "2012-10-17" Statement: - Effect: "Allow" - Sid: "{{ infra__utlity_bucket_name | replace('-','') }}" + Sid: "{{ infra__utlity_bucket_name | replace('-', '') }}" Action: - "s3:ListBucket" - "s3:PutObject" @@ -65,7 +64,7 @@ - name: Create Role to access Storage Utility Service Bucket register: infra__auto_repo_mirror_role community.aws.iam_role: - create_instance_profile: yes + create_instance_profile: true name: "{{ infra__utlity_bucket_name }}" state: present managed_policies: diff --git a/roles/infrastructure/tasks/setup_azure.yml b/roles/infrastructure/tasks/setup_azure.yml index 2d55632a..67ae1a75 100644 --- a/roles/infrastructure/tasks/setup_azure.yml +++ b/roles/infrastructure/tasks/setup_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/setup_azure_network.yml b/roles/infrastructure/tasks/setup_azure_network.yml index 30b46a02..16b0d4b3 100644 --- a/roles/infrastructure/tasks/setup_azure_network.yml +++ b/roles/infrastructure/tasks/setup_azure_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +19,8 @@ location: "{{ infra__region }}" name: "{{ infra__vpc_name }}" address_prefixes_cidr: "{{ [infra__vpc_cidr] }}" - purge_address_prefixes: yes - purge_dns_servers: yes + purge_address_prefixes: true + purge_dns_servers: true tags: "{{ infra__tags }}" state: present register: __azure_vnet_info @@ -36,7 +35,7 @@ location: "{{ infra__region }}" name: "{{ __azure_network_security_group_name_item }}" tags: "{{ infra__tags }}" - purge_rules: yes + purge_rules: true rules: "{{ infra__azure_security_group_rules }}" state: present loop_control: @@ -65,7 +64,7 @@ - { service: "Microsoft.Sql" } - { service: "Microsoft.Storage" } state: present - ignore_errors: yes # Not idempotent + ignore_errors: true # Not idempotent loop_control: loop_var: __azure_subnet_item loop: "{{ infra__vpc_public_subnets_info | union(infra__vpc_private_subnets_info) }}" @@ -77,7 +76,7 @@ loop_var: __azure_subnet_item loop: "{{ infra__vpc_public_subnets_info | union(infra__vpc_private_subnets_info) }}" - # TODO: Check new collection release to see if Delegations are now included +# TODO: Check new collection release to see if Delegations are now included - name: Create Azure NetApp Svcnet if deploying CML when: - infra__ml_deploy diff --git a/roles/infrastructure/tasks/setup_azure_storage.yml b/roles/infrastructure/tasks/setup_azure_storage.yml index 79467175..5f8cfbe5 100644 --- a/roles/infrastructure/tasks/setup_azure_storage.yml +++ b/roles/infrastructure/tasks/setup_azure_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,22 +15,22 @@ - name: Request Azure Storage Account Creation with HNS register: __infra_az_stor_acccount_test - ignore_errors: yes + ignore_errors: true azure.azcollection.azure_rm_resource: state: present resource_group: "{{ infra__azure_metagroup_name }}" provider: Storage resource_type: storageAccounts resource_name: "{{ infra__azure_storage_name }}" - api_version: '2019-04-01' - idempotency: yes + api_version: "2019-04-01" + idempotency: true body: sku: name: "{{ infra__azure_storage_class }}" kind: "{{ infra__azure_storage_kind }}" properties: - isHnsEnabled: yes - location: "{{ infra__region }}" + isHnsEnabled: true + location: "{{ infra__region }}" tags: "{{ infra__tags }}" - name: Check Storage Account creation result for failure @@ -41,13 +40,13 @@ block: - name: Test Azure Storage Account creation for StorageAccountAlreadyTaken ansible.builtin.assert: - quiet: yes + quiet: true that: '"StorageAccountAlreadyTaken" not in __infra_az_stor_acccount_test.module_stderr' fail_msg: "Your proposed Storage Account Name {{ infra__azure_storage_name }} is already taken, please use another" - name: Fail on any other Azure Storage Account Creation Error ansible.builtin.assert: - quiet: yes + quiet: true that: '"StorageAccountAlreadyTaken" in __infra_az_stor_acccount_test.module_stderr' fail_msg: "Failed to Create Azure Storage Account with unanticipated error: {{ __infra_az_stor_acccount_test }}" diff --git a/roles/infrastructure/tasks/setup_gcp_network.yml b/roles/infrastructure/tasks/setup_gcp_network.yml index 6341bd88..562f8acb 100644 --- a/roles/infrastructure/tasks/setup_gcp_network.yml +++ b/roles/infrastructure/tasks/setup_gcp_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +17,7 @@ google.cloud.gcp_compute_network: project: "{{ infra__gcp_project }}" name: "{{ infra__vpc_name }}" - auto_create_subnetworks: 'false' + auto_create_subnetworks: "false" state: present register: __gcp_vpc_info @@ -29,7 +28,7 @@ name: "{{ __gcp_subnet_item.name }}" network: "{{ __gcp_vpc_info }}" ip_cidr_range: "{{ __gcp_subnet_item.cidr }}" - private_ip_google_access: yes + private_ip_google_access: true state: present loop_control: loop_var: __gcp_subnet_item @@ -68,14 +67,12 @@ register: __gcp_peer_vnet_info - name: Create VPC Peering for CloudSQL - command: > + ansible.builtin.command: > gcloud services vpc-peerings connect --project={{ infra__gcp_project }} --network={{ infra__vpc_name }} --service=servicenetworking.googleapis.com --ranges={{ infra__vpc_svcnet_name }} - -# Cloud Router and Cloud NAT for L2 networking - name: Create a Cloud Router when: - infra__tunnel diff --git a/roles/infrastructure/tasks/setup_gcp_storage.yml b/roles/infrastructure/tasks/setup_gcp_storage.yml index f6fbc47c..4a69c9db 100644 --- a/roles/infrastructure/tasks/setup_gcp_storage.yml +++ b/roles/infrastructure/tasks/setup_gcp_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/setup_terraform.yml b/roles/infrastructure/tasks/setup_terraform.yml index c798644f..ce6b7c08 100644 --- a/roles/infrastructure/tasks/setup_terraform.yml +++ b/roles/infrastructure/tasks/setup_terraform.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +27,7 @@ community.general.terraform: project_path: "{{ infra__terraform_workspace_dir }}/infra" state: "present" - force_init: yes + force_init: true register: tf_result retries: 3 delay: 10 @@ -48,7 +47,7 @@ # Get information about Dynamic Inventory VMs if created via Terraform - name: Fetch EC2 Instance info for Dynamic Inventory Nodes register: __infra_dynamic_inventory_discovered - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ infra__region }}" filters: "{{ __filters | items2dict }}" vars: @@ -71,7 +70,7 @@ when: infra__create_utility_service block: - name: Discover the Utility Instance details - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ infra__region }}" filters: "{{ __filters | items2dict }}" vars: @@ -82,7 +81,7 @@ - name: Add discovered Utility Instance to host group ansible.builtin.add_host: - hostname: "{{__infra_utility_compute_discovered.instances[0].public_ip_address}}" + hostname: "{{ __infra_utility_compute_discovered.instances[0].public_ip_address }}" ansible_user: "{{ infra__dynamic_inventory_images_default[infra__type][infra__dynamic_inventory_os].user }}" ansible_ssh_private_key_file: "{{ (infra__private_key_file == '') | ternary(omit, infra__private_key_file) }}" groupname: cldr_utility diff --git a/roles/infrastructure/tasks/teardown.yml b/roles/infrastructure/tasks/teardown.yml index c29bd6ff..c037af27 100644 --- a/roles/infrastructure/tasks/teardown.yml +++ b/roles/infrastructure/tasks/teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,23 +16,23 @@ - name: Teardown for Ansible deployment engine when: infra__deployment_engine == 'ansible' block: - - name: Teardown provider-specific Infrastructure compute - ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_compute.yml" + - name: Teardown provider-specific Infrastructure compute + ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_compute.yml" - - name: Teardown provider-specific Infrastructure storage - ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_storage.yml" + - name: Teardown provider-specific Infrastructure storage + ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_storage.yml" - - name: Teardown provider-specific Infrastructure network - ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_network.yml" + - name: Teardown provider-specific Infrastructure network + ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_network.yml" - - name: Teardown provider-specific Utility Services - ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_utility_service.yml" + - name: Teardown provider-specific Utility Services + ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_utility_service.yml" - - name: Teardown provider-specific Infrastructure artifacts - ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}.yml" + - name: Teardown provider-specific Infrastructure artifacts + ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}.yml" - name: Teardown for Terraform deployment engine when: infra__deployment_engine == 'terraform' block: - - name: Teardown Terraform Infrastructure artifacts - ansible.builtin.include_tasks: "teardown_{{ infra__deployment_engine }}.yml" + - name: Teardown Terraform Infrastructure artifacts + ansible.builtin.include_tasks: "teardown_{{ infra__deployment_engine }}.yml" diff --git a/roles/infrastructure/tasks/teardown_aws.yml b/roles/infrastructure/tasks/teardown_aws.yml index fc7c75f0..3ff3949c 100644 --- a/roles/infrastructure/tasks/teardown_aws.yml +++ b/roles/infrastructure/tasks/teardown_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/teardown_aws_compute.yml b/roles/infrastructure/tasks/teardown_aws_compute.yml index e9d0c9c3..cf1d70dd 100644 --- a/roles/infrastructure/tasks/teardown_aws_compute.yml +++ b/roles/infrastructure/tasks/teardown_aws_compute.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +32,7 @@ - __infra_vpc_eks_cluster_names | length > 0 community.aws.aws_eks_cluster: name: "{{ __infra_eks_remove_item }}" - wait: yes + wait: true state: absent loop: "{{ __infra_vpc_eks_cluster_names }}" loop_control: @@ -47,7 +46,7 @@ - infra__force_teardown | bool - __infra_aws_rds_instances is defined - __infra_aws_rds_instances | length > 0 - command: > + ansible.builtin.command: > aws rds modify-db-instance --db-instance-identifier {{ __infra_rds_protect_item.db_instance_identifier }} --no-deletion-protection @@ -64,7 +63,7 @@ command: delete region: "{{ infra__region }}" instance_name: "{{ __infra_rds_remove_item.db_instance_identifier }}" - wait: yes + wait: true loop: "{{ __infra_aws_rds_instances }}" loop_control: loop_var: __infra_rds_remove_item @@ -108,6 +107,6 @@ when: infra__discovered_compute_inventory | count > 0 amazon.aws.ec2_instance: region: "{{ infra__region }}" - wait: yes + wait: true state: absent instance_ids: "{{ infra__discovered_compute_inventory | community.general.json_query('[*].instance_id') | list | unique }}" diff --git a/roles/infrastructure/tasks/teardown_aws_network.yml b/roles/infrastructure/tasks/teardown_aws_network.yml index 6abd42f2..73574069 100644 --- a/roles/infrastructure/tasks/teardown_aws_network.yml +++ b/roles/infrastructure/tasks/teardown_aws_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,14 +48,14 @@ when: infra__force_teardown | bool block: - name: List VPC Endpoints - community.aws.ec2_vpc_endpoint_info: + amazon.aws.ec2_vpc_endpoint_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" register: existing_endpoints - name: Delete VPC Endpoints - community.aws.ec2_vpc_endpoint: + amazon.aws.ec2_vpc_endpoint: state: absent vpc_endpoint_id: "{{ endpoint.vpc_endpoint_id }}" region: "{{ infra__region }}" @@ -66,7 +65,7 @@ label: "{{ endpoint.vpc_endpoint_id }}" - name: Wait for VPC Endpoint Deletion - community.aws.ec2_vpc_endpoint_info: + amazon.aws.ec2_vpc_endpoint_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" @@ -87,7 +86,7 @@ amazon.aws.elb_classic_lb: name: "{{ __infra_elb_remove_item }}" state: absent - wait: yes + wait: true async: 3600 # 1 hour timeout poll: 0 register: __elb_teardowns_info @@ -112,7 +111,7 @@ - __infra_aws_nat_gateways is defined - __infra_aws_nat_gateways.result is defined - __infra_aws_nat_gateways.result | length > 0 - community.aws.ec2_vpc_nat_gateway: + amazon.aws.ec2_vpc_nat_gateway: state: absent region: "{{ infra__region }}" wait: true @@ -188,7 +187,7 @@ - name: Remove AWS Internet Gateway (IGW) when: (infra__tunnel and infra__public_endpoint_access) or infra__force_teardown | bool - community.aws.ec2_vpc_igw: + amazon.aws.ec2_vpc_igw: region: "{{ infra__region }}" vpc_id: "{{ infra__aws_vpc_id }}" state: absent @@ -216,7 +215,7 @@ when: infra__aws_private_endpoints | bool block: - name: Delete VPC Endpoints - community.aws.ec2_vpc_endpoint: + amazon.aws.ec2_vpc_endpoint: state: absent vpc_endpoint_id: "{{ __infra_vpc_endpoint.vpc_endpoint_id }}" region: "{{ infra__region }}" @@ -226,7 +225,7 @@ label: "{{ __infra_vpc_endpoint.vpc_endpoint_id }}" - name: Wait for VPC Endpoint Deletion - community.aws.ec2_vpc_endpoint_info: + amazon.aws.ec2_vpc_endpoint_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" @@ -249,12 +248,12 @@ block: - name: Delete the AWS Private Route Table #when: not infra__public_endpoint_access - when: no + when: false community.aws.ec2_vpc_route_table: vpc_id: "{{ infra__aws_vpc_id }}" region: "{{ infra__region }}" lookup: tag - tags: "{{ { 'Name': infra__aws_private_route_table_name } }}" + tags: "{{ {'Name': infra__aws_private_route_table_name} }}" state: absent - name: Delete the AWS Private Route Tables # for Public Endpoint Access @@ -263,7 +262,7 @@ vpc_id: "{{ infra__aws_vpc_id }}" region: "{{ infra__region }}" lookup: tag - tags: "{{ { 'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string])} }}" + tags: "{{ {'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string])} }}" state: absent loop_control: index_var: __aws_private_subnet_id_index @@ -271,7 +270,7 @@ - name: List all managed AWS NAT Gateways # for Public Endpoint Access # when: infra__public_endpoint_access - community.aws.ec2_vpc_nat_gateway_info: + amazon.aws.ec2_vpc_nat_gateway_info: region: "{{ infra__region }}" filters: vpc-id: "{{ infra__aws_vpc_id }}" @@ -279,7 +278,7 @@ - name: Delete associated AWS NAT Gateways # when: infra__public_endpoint_access - community.aws.ec2_vpc_nat_gateway: + amazon.aws.ec2_vpc_nat_gateway: state: absent region: "{{ infra__region }}" wait: true @@ -296,7 +295,8 @@ when: __aws_ngw_teardown is defined and __aws_ngw_teardown.results is defined and __aws_ngw_teardown.results | count > 0 ansible.builtin.fail: msg: "Failed to delete a NAT gateway" - failed_when: __aws_ngw_teardown_item.rc is defined and __aws_ngw_teardown_item.rc != 1 and ('InvalidAllocationID.NotFound' in __aws_ngw_teardown_item.module_stderr) + failed_when: __aws_ngw_teardown_item.rc is defined and __aws_ngw_teardown_item.rc != 1 and ('InvalidAllocationID.NotFound' in + __aws_ngw_teardown_item.module_stderr) loop_control: loop_var: __aws_ngw_teardown_item loop: "{{ __aws_ngw_teardown.results }}" @@ -316,7 +316,7 @@ loop: "{{ infra__vpc_public_subnet_cidrs | union(infra__vpc_private_subnet_cidrs) }}" - name: Remove AWS Internet Gateway (IGW) - community.aws.ec2_vpc_igw: + amazon.aws.ec2_vpc_igw: region: "{{ infra__region }}" vpc_id: "{{ infra__aws_vpc_id }}" state: absent diff --git a/roles/infrastructure/tasks/teardown_aws_storage.yml b/roles/infrastructure/tasks/teardown_aws_storage.yml index e7add0b0..9153631b 100644 --- a/roles/infrastructure/tasks/teardown_aws_storage.yml +++ b/roles/infrastructure/tasks/teardown_aws_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +19,7 @@ region: "{{ infra__region }}" name: "{{ __aws_storage_location_item.bucket }}" state: absent - force: yes + force: true loop_control: loop_var: __aws_storage_location_item loop: "{{ infra__aws_storage_locations }}" @@ -31,7 +30,7 @@ region: "{{ infra__region }}" name: "{{ infra__utlity_bucket_name }}" state: absent - force: yes + force: true - name: Remove AWS EFS File Systems, if Discovered during Purge when: @@ -41,7 +40,7 @@ community.aws.efs: state: absent id: "{{ __infra_efs_item.file_system_id }}" - wait: yes + wait: true loop: "{{ __infra_efs_fs.results | community.general.json_query('[*].efs') | flatten }}" loop_control: loop_var: __infra_efs_item diff --git a/roles/infrastructure/tasks/teardown_aws_utility_service.yml b/roles/infrastructure/tasks/teardown_aws_utility_service.yml index 48fa3006..d66f5511 100644 --- a/roles/infrastructure/tasks/teardown_aws_utility_service.yml +++ b/roles/infrastructure/tasks/teardown_aws_utility_service.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,8 +18,8 @@ community.aws.iam_role: region: "{{ infra__region }}" name: "{{ infra__utlity_bucket_name }}" - purge_policies: yes - delete_instance_profile: yes + purge_policies: true + delete_instance_profile: true state: absent - name: Remove CDP Data Access Policies diff --git a/roles/infrastructure/tasks/teardown_azure.yml b/roles/infrastructure/tasks/teardown_azure.yml index b8fe5623..af856b8e 100644 --- a/roles/infrastructure/tasks/teardown_azure.yml +++ b/roles/infrastructure/tasks/teardown_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,5 +26,5 @@ delay: 5 azure.azcollection.azure_rm_resourcegroup_info: name: "{{ infra__azure_metagroup_name }}" - list_resources: yes + list_resources: true register: __azure_resource_group_teardown diff --git a/roles/infrastructure/tasks/teardown_azure_network.yml b/roles/infrastructure/tasks/teardown_azure_network.yml index 32fae211..aa570ede 100644 --- a/roles/infrastructure/tasks/teardown_azure_network.yml +++ b/roles/infrastructure/tasks/teardown_azure_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/teardown_azure_storage.yml b/roles/infrastructure/tasks/teardown_azure_storage.yml index ee05df42..c2e36c47 100644 --- a/roles/infrastructure/tasks/teardown_azure_storage.yml +++ b/roles/infrastructure/tasks/teardown_azure_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,14 +15,14 @@ - name: Remove Main Azure Storage Account during Teardown when: - - infra__teardown_deletes_data - - infra__azure_metagroup_uri | length > 0 + - infra__teardown_deletes_data + - infra__azure_metagroup_uri | length > 0 azure.azcollection.azure_rm_storageaccount: resource_group: "{{ infra__azure_metagroup_name }}" location: "{{ infra__region }}" force_delete_nonempty: "{{ infra__teardown_deletes_data }}" state: absent - name: "{{ infra__storage_name }}" # HNS Property currently NA in Stock Ansible + name: "{{ infra__storage_name }}" # HNS Property currently NA in Stock Ansible - name: Remove Azure Netapp Storage during Teardown when: diff --git a/roles/infrastructure/tasks/teardown_gcp_network.yml b/roles/infrastructure/tasks/teardown_gcp_network.yml index f4ecf510..e9f73a9b 100644 --- a/roles/infrastructure/tasks/teardown_gcp_network.yml +++ b/roles/infrastructure/tasks/teardown_gcp_network.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/teardown_gcp_storage.yml b/roles/infrastructure/tasks/teardown_gcp_storage.yml index f2016867..1be1ce77 100644 --- a/roles/infrastructure/tasks/teardown_gcp_storage.yml +++ b/roles/infrastructure/tasks/teardown_gcp_storage.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,5 +22,5 @@ loop: "{{ infra__gcp_storage_locations }}" loop_control: loop_var: __gcp_storage_location_item - command: > + ansible.builtin.command: > gsutil -m rm -r gs://{{ __gcp_storage_location_item }} diff --git a/roles/infrastructure/tasks/teardown_terraform.yml b/roles/infrastructure/tasks/teardown_terraform.yml index 15157283..57be45fa 100644 --- a/roles/infrastructure/tasks/teardown_terraform.yml +++ b/roles/infrastructure/tasks/teardown_terraform.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +22,7 @@ community.general.terraform: project_path: "{{ infra__terraform_workspace_dir }}/infra" state: "absent" - force_init: yes + force_init: true register: tf_result retries: 3 delay: 10 diff --git a/roles/infrastructure/tasks/validate.yml b/roles/infrastructure/tasks/validate.yml index f6680fc7..0b885d57 100644 --- a/roles/infrastructure/tasks/validate.yml +++ b/roles/infrastructure/tasks/validate.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/infrastructure/tasks/validate_aws.yml b/roles/infrastructure/tasks/validate_aws.yml index 8a9e02cb..c59c81b7 100644 --- a/roles/infrastructure/tasks/validate_aws.yml +++ b/roles/infrastructure/tasks/validate_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ - "__auth_item is string" - "__auth_item | trim | length > 0" fail_msg: "AWS authentication parameter, '{{ __auth_item }}', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __auth_item loop: @@ -33,7 +32,7 @@ # TODO: Revisit this logic, we could just provide a blank VPC for L0, L1, or L2 - name: Validate existing AWS Subnet details if provided for a private network #when: infra__aws_vpc_id != "" - when: no + when: false block: - name: Check for non-empty AWS private subnets when: infra__aws_private_subnet_ids | unique | count < 3 diff --git a/roles/infrastructure/tasks/validate_aws_terraform.yml b/roles/infrastructure/tasks/validate_aws_terraform.yml index 56379916..10e17448 100644 --- a/roles/infrastructure/tasks/validate_aws_terraform.yml +++ b/roles/infrastructure/tasks/validate_aws_terraform.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ - "__auth_item is string" - "__auth_item | trim | length > 0" fail_msg: "AWS authentication parameter, '{{ __auth_item }}', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __auth_item loop: @@ -32,45 +31,44 @@ - name: Confirm that required Terraform variables are defined block: - - name: Check infra__terraform_template_dir - ansible.builtin.assert: - that: - - "infra__terraform_template_dir is defined" - - "infra__terraform_template_dir | length > 0" - fail_msg: "Required infra__terraform_template_dir variable for Terraform is not valid." - quiet: yes + - name: Check infra__terraform_template_dir + ansible.builtin.assert: + that: + - "infra__terraform_template_dir is defined" + - "infra__terraform_template_dir | length > 0" + fail_msg: "Required infra__terraform_template_dir variable for Terraform is not valid." + quiet: true - - name: Check infra__terraform_workspace_dir - ansible.builtin.assert: - that: - - "infra__terraform_workspace_dir is defined" - - "infra__terraform_workspace_dir | length > 0" - fail_msg: "Required infra__terraform_workspace_dir variable for Terraform is not valid." - quiet: yes + - name: Check infra__terraform_workspace_dir + ansible.builtin.assert: + that: + - "infra__terraform_workspace_dir is defined" + - "infra__terraform_workspace_dir | length > 0" + fail_msg: "Required infra__terraform_workspace_dir variable for Terraform is not valid." + quiet: true - - name: Check infra__terraform_artefact_dir - ansible.builtin.assert: - that: - - "infra__terraform_artefact_dir is defined" - - "infra__terraform_artefact_dir | length > 0" - fail_msg: "Required infra__terraform_artefact_dir variable for Terraform is not valid." - quiet: yes + - name: Check infra__terraform_artefact_dir + ansible.builtin.assert: + that: + - "infra__terraform_artefact_dir is defined" + - "infra__terraform_artefact_dir | length > 0" + fail_msg: "Required infra__terraform_artefact_dir variable for Terraform is not valid." + quiet: true - - name: Check infra__terraform_state_storage - ansible.builtin.assert: - that: - - "infra__terraform_state_storage is defined" - - "infra__terraform_state_storage in infra__terraform_allowed_state_storage" - fail_msg: "Required infra__terraform_state_storage variable for Terraform needs to be \ - one of {{ infra__terraform_allowed_state_storage | join(', ') }}" - quiet: yes + - name: Check infra__terraform_state_storage + ansible.builtin.assert: + that: + - "infra__terraform_state_storage is defined" + - "infra__terraform_state_storage in infra__terraform_allowed_state_storage" + fail_msg: "Required infra__terraform_state_storage variable for Terraform needs to be one of {{ infra__terraform_allowed_state_storage | join(', ') }}" + quiet: true - - name: Check remote state variables are defined for remote_s3 - ansible.builtin.assert: - that: - - "infra__terraform_remote_state_bucket | length > 0" - - "infra__terraform_remote_state_lock_table | length > 0" - fail_msg: "Required infra__terraform_remote_state_bucket and infra__terraform_remote_state_lock_table variables \ - need to be defined for '{{ infra__terraform_state_storage }}' Terraform remote state" - quiet: yes - when: infra__terraform_state_storage == "remote_s3" + - name: Check remote state variables are defined for remote_s3 + ansible.builtin.assert: + that: + - "infra__terraform_remote_state_bucket | length > 0" + - "infra__terraform_remote_state_lock_table | length > 0" + fail_msg: "Required infra__terraform_remote_state_bucket and infra__terraform_remote_state_lock_table variables need to be defined for '{{ infra__terraform_state_storage + }}' Terraform remote state" + quiet: true + when: infra__terraform_state_storage == "remote_s3" diff --git a/roles/infrastructure/tasks/validate_azure.yml b/roles/infrastructure/tasks/validate_azure.yml index 11a0bc77..3af6785f 100644 --- a/roles/infrastructure/tasks/validate_azure.yml +++ b/roles/infrastructure/tasks/validate_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set public key text from file if not set when: infra__public_key_text | trim | length < 1 and infra__public_key_file ansible.builtin.set_fact: - infra__public_key_text: "{{ lookup('file', infra__public_key_file ) }}" + infra__public_key_text: "{{ lookup('file', infra__public_key_file) }}" - name: Check public key text ansible.builtin.assert: @@ -26,26 +25,28 @@ - "infra__public_key_text is string" - "infra__public_key_text | trim | length > 0" fail_msg: "Azure authentication parameter, 'infra__public_key_text', is invalid. Either set this variable or specify 'infra__public_key_file'." - quiet: yes + quiet: true - name: Check Azure Region - {{ common__azure_region }} ansible.builtin.assert: that: - "common__azure_region is match(regex1)" - fail_msg: "Azure region value 'common__azure_region' with value '{{ common__azure_region }}' is not valid, it is typically lowercase letters and numbers with no spaces. Consider running 'az account list-locations -o table' in Azure CLI for a current listing." - quiet: yes + fail_msg: "Azure region value 'common__azure_region' with value '{{ common__azure_region }}' is not valid, it is typically lowercase letters and numbers with + no spaces. Consider running 'az account list-locations -o table' in Azure CLI for a current listing." + quiet: true vars: - regex1: '^([a-z0-9]+)$' + regex1: "^([a-z0-9]+)$" - name: Check storage name - {{ infra__azure_storage_name }} ansible.builtin.assert: that: - "infra__azure_storage_name | trim | length < 25" - "infra__azure_storage_name is match(regex1)" - fail_msg: "Azure storage name parameter, 'infra__azure_storage_name' with value '{{ infra__azure_storage_name }}', is invalid. Azure requires lowercase letters and numbers only and 24 characters or less. Please check your name_prefix and any suffix modifications." - quiet: yes + fail_msg: "Azure storage name parameter, 'infra__azure_storage_name' with value '{{ infra__azure_storage_name }}', is invalid. Azure requires lowercase letters + and numbers only and 24 characters or less. Please check your name_prefix and any suffix modifications." + quiet: true vars: - regex1: '^([a-z0-9]+)$' + regex1: "^([a-z0-9]+)$" - name: Check that network subnets comply with Azure restrictions ansible.builtin.assert: @@ -53,6 +54,6 @@ - "'10.0.0.0/16' not in {{ azure_subnets }}" - "'10.244.0.0/16' not in {{ azure_subnets }}" fail_msg: "Certain subnets are reserved when deploying on Azure." - quiet: yes + quiet: true vars: azure_subnets: "{{ [infra__vpc_cidr] + infra__vpc_public_subnet_cidrs + infra__vpc_private_subnet_cidrs + [infra__vpc_svcnet_cidr] }}" diff --git a/roles/infrastructure/tasks/validate_gcp.yml b/roles/infrastructure/tasks/validate_gcp.yml index 593f1f1c..3f54dd21 100644 --- a/roles/infrastructure/tasks/validate_gcp.yml +++ b/roles/infrastructure/tasks/validate_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set public key text from file if not set when: infra__public_key_text | trim | length < 1 and infra__public_key_file ansible.builtin.set_fact: - infra__public_key_text: "{{ lookup('file', infra__public_key_file ) }}" + infra__public_key_text: "{{ lookup('file', infra__public_key_file) }}" - name: Check public key text ansible.builtin.assert: @@ -26,4 +25,4 @@ - "infra__public_key_text is string" - "infra__public_key_text | trim | length > 0" fail_msg: "GCP authentication parameter, 'infra__public_key_text', is invalid. Either set this variable or specify 'infra__public_key_file'." - quiet: yes + quiet: true diff --git a/roles/infrastructure/vars/main.yml b/roles/infrastructure/vars/main.yml index 747d3cf2..eeb9d0cc 100644 --- a/roles/infrastructure/vars/main.yml +++ b/roles/infrastructure/vars/main.yml @@ -3,67 +3,66 @@ infra__dynamic_inventory_vm_type_default: aws: - sml: 'm5.large' - std: 'm5.2xlarge' - lrg: 'm5.4xlarge' + sml: "m5.large" + std: "m5.2xlarge" + lrg: "m5.4xlarge" azure: - std: 'Standard_D8_v3' - lrg: 'Standard_D16_v3' + std: "Standard_D8_v3" + lrg: "Standard_D16_v3" gcp: - std: 'e2-standard-8' - lrg: 'e2-standard-8' + std: "e2-standard-8" + lrg: "e2-standard-8" infra__dynamic_inventory_storage_type_default: aws: - std: 'gp2' - fast: 'gp2' + std: "gp2" + fast: "gp2" azure: - std: 'StandardSSD_LRS' - fast: 'StandardSSD_LRS' + std: "StandardSSD_LRS" + fast: "StandardSSD_LRS" gcp: - std: 'pd-standard' - fast: 'pd-standard' - -infra__cdp_control_plane_cidr_default: ['52.36.110.208/32', '52.40.165.49/32', '35.166.86.177/32'] + std: "pd-standard" + fast: "pd-standard" +infra__cdp_control_plane_cidr_default: ["52.36.110.208/32", "52.40.165.49/32", "35.166.86.177/32"] infra__dynamic_inventory_images_default: aws: el7: - search: 'CentOS-7*x86_64*' - user: 'centos' - product-code: 'cvugziknvmxgqna9noibqnnsy' + search: "CentOS-7*x86_64*" + user: "centos" + product-code: "cvugziknvmxgqna9noibqnnsy" owners: - - '679593333241' + - "679593333241" el8: - search: 'RHEL-8.6*HVM-*x86_64*' - user: 'ec2-user' + search: "RHEL-8.6*HVM-*x86_64*" + user: "ec2-user" owners: - - '309956199498' + - "309956199498" bionic: - search: 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server*' - user: 'ubuntu' + search: "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server*" + user: "ubuntu" owners: - - '099720109477' + - "099720109477" focal: - search: 'ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server*' - user: 'ubuntu' + search: "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server*" + user: "ubuntu" owners: - - '099720109477' + - "099720109477" infra__all_ports_security_rule: aws: -1 azure: 0-65535 infra__aws_gateway_vpc_private_endpoints: - - 'com.amazonaws.{{infra__region}}.s3' + - "com.amazonaws.{{infra__region}}.s3" infra__aws_interface_vpc_private_endpoints: - - 'com.amazonaws.{{infra__region}}.elasticfilesystem' - - 'com.amazonaws.{{infra__region}}.ecr.dkr' - - 'com.amazonaws.{{infra__region}}.rds' - - 'com.amazonaws.{{infra__region}}.ecr.api' - - 'com.amazonaws.{{infra__region}}.sts' - - 'com.amazonaws.{{infra__region}}.ec2' - - 'com.amazonaws.{{infra__region}}.cloudformation' - - 'com.amazonaws.{{infra__region}}.elasticloadbalancing' - - 'com.amazonaws.{{infra__region}}.autoscaling' + - "com.amazonaws.{{infra__region}}.elasticfilesystem" + - "com.amazonaws.{{infra__region}}.ecr.dkr" + - "com.amazonaws.{{infra__region}}.rds" + - "com.amazonaws.{{infra__region}}.ecr.api" + - "com.amazonaws.{{infra__region}}.sts" + - "com.amazonaws.{{infra__region}}.ec2" + - "com.amazonaws.{{infra__region}}.cloudformation" + - "com.amazonaws.{{infra__region}}.elasticloadbalancing" + - "com.amazonaws.{{infra__region}}.autoscaling" diff --git a/roles/init_deployment/defaults/main.yml b/roles/init_deployment/defaults/main.yml index 483ecc2d..89e36501 100644 --- a/roles/init_deployment/defaults/main.yml +++ b/roles/init_deployment/defaults/main.yml @@ -18,9 +18,9 @@ definition_path: "./" # Default Paths -default_local_temp_dir: '/tmp' -default_sshkey_path: '~/.ssh' -default_config_path: '~/.config/cloudera-deploy' +default_local_temp_dir: "/tmp" +default_sshkey_path: "~/.ssh" +default_config_path: "~/.config/cloudera-deploy" # Default names default_name_prefix: cldr @@ -33,14 +33,14 @@ default_cluster_definition_file: "vars/basic_cluster.yml" default_definition_file: "definition.yml" default_cluster_file: "cluster.yml" -include_inventory_file: '' +include_inventory_file: "" auto_repo_mirror_file: "{{ [default_config_path, 'auto_repo_mirror.ini'] | path_join }}" # Default Deployment Controls default_infra_deployment_engine: ansible -default_infra_type: aws # azure, gcp -default_infra_region: us-east-1 # westeurope, gcp? -default_parcel_distro: el7 # el8, bionic, focal +default_infra_type: aws # azure, gcp +default_infra_region: us-east-1 # westeurope, gcp? +default_parcel_distro: el7 # el8, bionic, focal # Terraform defaults default_terraform_base_dir: "{{ [default_config_path, 'terraform'] | path_join }}" diff --git a/roles/init_deployment/meta/argument_specs.yml b/roles/init_deployment/meta/argument_specs.yml new file mode 100644 index 00000000..9bdbdb4d --- /dev/null +++ b/roles/init_deployment/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Marshal deployment configuration. + description: Marshal deployment configuration. + author: Cloudera Labs + version_added: 2.0.0 diff --git a/roles/init_deployment/tasks/main.yml b/roles/init_deployment/tasks/main.yml index 32920f2f..e7b187d9 100644 --- a/roles/init_deployment/tasks/main.yml +++ b/roles/init_deployment/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,4 +28,4 @@ - name: Explicitly finalise Initialisation to avoid duplicate init ansible.builtin.set_fact: - init__completed: True + init__completed: true diff --git a/roles/init_deployment/tasks/marshall.yml b/roles/init_deployment/tasks/marshall.yml index 283bc884..6a60c4bc 100644 --- a/roles/init_deployment/tasks/marshall.yml +++ b/roles/init_deployment/tasks/marshall.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,12 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - # Check path - name: Check a Definition path has been supplied ansible.builtin.assert: - quiet: yes + quiet: true that: - definition_path is defined - definition_path | length > 0 @@ -31,7 +28,7 @@ - name: Assert Definition Path is a directory ansible.builtin.assert: - quiet: yes + quiet: true fail_msg: "'definition_path' does not appear to point to an existing and reachable directory'" that: - __definition_path_stat.stat.isdir is defined @@ -40,8 +37,8 @@ # Set File Paths - name: Set Expected File Paths ansible.builtin.set_fact: - init__definition_file: "{{ abs_definition | default( [definition_path, definition_file | default(default_definition_file)] | path_join ) }}" - init__cluster_file: "{{ abs_cluster | default( [definition_path, cluster_file | default(default_cluster_file)] | path_join ) }}" + init__definition_file: "{{ abs_definition | default([definition_path, definition_file | default(default_definition_file)] | path_join) }}" + init__cluster_file: "{{ abs_cluster | default([definition_path, cluster_file | default(default_cluster_file)] | path_join) }}" init__auto_repo_mirror_artefact: "{{ auto_repo_mirror_file | default(auto_repo_mirror_file) }}" # Handle Definition File @@ -52,7 +49,7 @@ - name: Assert that a Definition File has been provided ansible.builtin.assert: - quiet: yes + quiet: true that: __def_file_stat.stat.exists fail_msg: "Expected to find a definition file '{{ init__definition_file }}' in Definition Path '{{ definition_path }}'" @@ -106,7 +103,7 @@ # Merge User Profile to Globals - name: Marshal User Config into Globals ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(user_config , recursive=True) }}" + globals: "{{ globals | default({}) | combine(user_config, recursive=True) }}" vars: user_config: name_prefix: "{{ name_prefix | default(default_name_prefix) }}" @@ -137,7 +134,7 @@ - name: Set GCloud Environment Variables if needed when: globals.gcloud_credential_file is defined ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_gcp_entries, recursive=True ) }}" + globals: "{{ globals | default({}) | combine(env_gcp_entries, recursive=True) }}" vars: env_gcp_entries: env_vars: @@ -165,7 +162,7 @@ - name: Set Profile Env Vars if required ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_var_entries, recursive=True ) }}" + globals: "{{ globals | default({}) | combine(env_var_entries, recursive=True) }}" vars: env_var_entries: env_vars: diff --git a/roles/init_deployment/tasks/prep_pvc.yml b/roles/init_deployment/tasks/prep_pvc.yml index 4627d72e..b68f7a70 100644 --- a/roles/init_deployment/tasks/prep_pvc.yml +++ b/roles/init_deployment/tasks/prep_pvc.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +15,7 @@ - name: Set specific Facts for later use in Cluster Deployment ansible.builtin.set_fact: - _pre_template_cluster: "{{ lookup('file', init__cluster_definition_file ) | from_yaml }}" + _pre_template_cluster: "{{ lookup('file', init__cluster_definition_file) | from_yaml }}" preload_parcels: "{{ auto_repo_mirror_file_list | default([]) }}" custom_repo_rehost_files: "{{ auto_repo_mirror_file_list | default([]) }}" delegate_to: "{{ __play_host }}" @@ -46,7 +45,7 @@ loop: "{{ groups.all }}" loop_control: loop_var: __play_host - label : __play_host + label: __play_host - name: Include Cluster Definition override ansible.builtin.include_vars: @@ -56,10 +55,10 @@ loop: "{{ groups.all }}" loop_control: loop_var: __play_host - label : __play_host + label: __play_host - name: Create local temp directories - file: "{{ __dir }}" + ansible.builtin.file: "{{ __dir }}" loop: - path: "{{ local_temp_dir }}/csrs" state: directory diff --git a/roles/init_deployment/tasks/runlevels.yml b/roles/init_deployment/tasks/runlevels.yml index faadec4b..4d5f3531 100644 --- a/roles/init_deployment/tasks/runlevels.yml +++ b/roles/init_deployment/tasks/runlevels.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,4 +24,5 @@ - name: Determine if Specific Roles should be called ansible.builtin.set_fact: # init__call_cdp_pvc: "{{ mgmt is defined or cluster is defined }}" - init__call_cdp_pbc: "{{ env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" + init__call_cdp_pbc: "{{ env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) + }}" diff --git a/roles/init_deployment/tasks/ssh.yml b/roles/init_deployment/tasks/ssh.yml index 6998de53..c7534896 100644 --- a/roles/init_deployment/tasks/ssh.yml +++ b/roles/init_deployment/tasks/ssh.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +33,7 @@ type: rsa size: 4096 regenerate: never - force: no + force: false vars: __generated_keypair_name: "{{ globals.name_prefix + default_ssh_key_suffix }}" @@ -45,16 +44,16 @@ __generated_keypair: ssh: private_key_file: "{{ __generated_ssh_keys.filename }}" - public_key_file: "{{ [ __generated_ssh_keys.filename, 'pub' ] | join('.') }}" + public_key_file: "{{ [__generated_ssh_keys.filename, 'pub'] | join('.') }}" - name: Load SSH public key file to text when: globals.ssh.public_key_file is defined ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__public_key_globals , recursive=True) }}" + globals: "{{ globals | default({}) | combine(__public_key_globals, recursive=True) }}" vars: __public_key_globals: ssh: - public_key_text: "{{ lookup('file', globals.ssh.public_key_file ) | default(omit) }}" + public_key_text: "{{ lookup('file', globals.ssh.public_key_file) | default(omit) }}" - name: Validate SSH Private Key File has acceptable permissions when: globals.ssh.private_key_file is defined @@ -72,4 +71,4 @@ - "SSH Private Key at {{ __private_key_file_stat.stat.path }} has invalid permissions" - "Permissions are {{ __private_key_file_stat.stat.mode }}" - "Permissions should be 0400 or 0600" - quiet: yes + quiet: true diff --git a/roles/init_deployment/tasks/validate.yml b/roles/init_deployment/tasks/validate.yml index 567348d6..fbd7a70d 100644 --- a/roles/init_deployment/tasks/validate.yml +++ b/roles/init_deployment/tasks/validate.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,12 +18,12 @@ ansible.builtin.assert: that: purge|bool is sameas true or purge|bool is sameas false fail_msg: "purge key is present in definition, but not a boolean as expected" - quiet: yes + quiet: true - name: Assert user has supplied an Admin Password no_log: true ansible.builtin.assert: - quiet: yes + quiet: true that: - admin_password is defined - admin_password | length > 2 @@ -41,7 +40,7 @@ - globals.name_prefix | length < 5 - globals.name_prefix | regex_search('^[a-zA-Z]') fail_msg: "You must supply a valid Namespace" - quiet: yes + quiet: true - name: Check supplied Namespace (AWS, GCP) when: @@ -53,14 +52,14 @@ - globals.name_prefix | length < 8 - globals.name_prefix | regex_search('^[a-zA-Z]') fail_msg: "You must supply a valid Namespace" - quiet: yes + quiet: true - name: Check Deployment Engine variable ansible.builtin.assert: that: - globals.infra_deployment_engine in ['ansible', 'terraform'] fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'" - quiet: yes + quiet: true - name: Check Supplied terraform_base_dir variable when: @@ -70,7 +69,7 @@ - globals.terraform.base_dir is defined - globals.terraform.base_dir | length > 0 fail_msg: "You must supply a 'terraform_base_dir' where Terraform assets will be placed" - quiet: yes + quiet: true - name: Check Supplied terraform_auto_remote_state variable when: @@ -79,7 +78,7 @@ that: - (globals.terraform.auto_remote_state|bool is sameas true) or (globals.terraform.auto_remote_state|bool is sameas false) fail_msg: "The terraform.auto_remote_state variable must be a boolean variable" - quiet: yes + quiet: true - name: Check Admin Password is CDP Cloud compliant when: init__call_cdp_pbc | bool @@ -88,4 +87,4 @@ - admin_password is match('^(?=.*[A-Za-z])(?=.*\\d)(?=.*[@$!%*#?&])[A-Za-z\\d@$!%*#?&]{8,64}$') fail_msg: >- Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars. - quiet: yes + quiet: true diff --git a/roles/init_deployment/vars/basic_cluster.yml b/roles/init_deployment/vars/basic_cluster.yml index 9aa6facd..d02c63ce 100644 --- a/roles/init_deployment/vars/basic_cluster.yml +++ b/roles/init_deployment/vars/basic_cluster.yml @@ -34,7 +34,7 @@ clusters: NODEMANAGER: yarn_nodemanager_resource_memory_mb: 4096 yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm + yarn_nodemanager_local_dirs: /tmp/nm yarn_nodemanager_log_dirs: /var/log/nm GATEWAY: mapred_submit_replication: 3 @@ -54,7 +54,6 @@ clusters: mgmt: name: Cloudera Management Service services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - hosts: configs: host_default_proc_memswap_thresholds: diff --git a/roles/mount/meta/argument_specs.yml b/roles/mount/meta/argument_specs.yml new file mode 100644 index 00000000..d973d2a8 --- /dev/null +++ b/roles/mount/meta/argument_specs.yml @@ -0,0 +1,22 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Mount partitions. + description: | + Create and mount an LVM partition for a specified storage volume. + Includes installation of the LVM2 OS package. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 2.0.1 diff --git a/roles/mount/meta/main.yml b/roles/mount/meta/main.yml index a760df41..dc7e3aa6 100755 --- a/roles/mount/meta/main.yml +++ b/roles/mount/meta/main.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,20 +24,20 @@ galaxy_info: min_ansible_version: 2.10 platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all + - name: Debian + versions: all + - name: Fedora + versions: all + - name: GenericLinux + versions: all + - name: MacOSX + versions: all + - name: Ubuntu + versions: all galaxy_tags: - - storage - - mount - - cdp - - aws - - openstack + - storage + - mount + - cdp + - aws + - openstack diff --git a/roles/mount/tasks/main.yml b/roles/mount/tasks/main.yml index dfe58122..e53de078 100644 --- a/roles/mount/tasks/main.yml +++ b/roles/mount/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Generate map of EBS volume attachments when: mount_provider == 'aws' block: @@ -13,7 +12,7 @@ - device.key is match("nvme") - volume != "UNKNOWN" ansible.builtin.set_fact: - ebs_device_map: "{{ ebs_device_map | default({}) | combine({ volume : '/dev/' + device.key }) }}" + ebs_device_map: "{{ ebs_device_map | default({}) | combine({volume: '/dev/' + device.key}) }}" loop: "{{ ansible_devices | dict2items }}" loop_control: loop_var: device @@ -24,7 +23,7 @@ - name: Set required facts for volumes ansible.builtin.set_fact: - __storage_volumes_facts: "{{ __storage_volumes_facts | default([]) | union([storage_volume_detail]) }}" + __storage_volumes_facts: "{{ __storage_volumes_facts | default([]) | union([storage_volume_detail]) }}" vars: __device: "{{ ebs_device_map[volume.vol_id] | default(volume.device) }}" __speared_device: "{{ __device | replace('/', '-') }}" @@ -48,7 +47,7 @@ community.general.parted: device: "{{ volume.device }}" number: 1 - flags: [ lvm ] + flags: [lvm] state: present part_start: "0%" part_end: "100%" diff --git a/roles/mount/tasks/volume.yml b/roles/mount/tasks/volume.yml index aade9403..bb5d88e3 100644 --- a/roles/mount/tasks/volume.yml +++ b/roles/mount/tasks/volume.yml @@ -9,24 +9,24 @@ - name: Create logical volume community.general.lvol: vg: "{{ volume.vg_name }}" - lv: "{{ volume.lv_name }}" + lv: "{{ volume.lv_name }}" size: +100%FREE - force: yes + force: true - name: Format partition as XFS community.general.filesystem: - dev: "{{ '/'.join(['/dev',volume.vg_name, volume.lv_name]) }}" + dev: "{{ '/'.join(['/dev', volume.vg_name, volume.lv_name]) }}" fstype: xfs - name: Create the mount directory ansible.builtin.file: path: "{{ volume.mount }}" state: directory - mode: '0755' + mode: "0755" - name: Mount the logical volume - mount: + ansible.posix.mount: path: "{{ volume.mount }}" - src: "{{ '/'.join(['/dev',volume.vg_name, volume.lv_name]) }}" + src: "{{ '/'.join(['/dev', volume.vg_name, volume.lv_name]) }}" fstype: xfs state: mounted diff --git a/roles/nodeexporter/meta/argument_specs.yml b/roles/nodeexporter/meta/argument_specs.yml new file mode 100644 index 00000000..b360bab1 --- /dev/null +++ b/roles/nodeexporter/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy Node exporter. + description: Deploy Node exporter for Prometheus. + author: Ronald Suplina + version_added: 2.4.0 diff --git a/roles/nodeexporter/tasks/main.yml b/roles/nodeexporter/tasks/main.yml index b60f09f4..299c3283 100644 --- a/roles/nodeexporter/tasks/main.yml +++ b/roles/nodeexporter/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,7 +44,7 @@ extra_opts: --strip-components=1 owner: "{{ node_exporter_user }}" group: "{{ node_exporter_group }}" - remote_src: yes + remote_src: true - name: Remove the temporary directory when: __exporter_tmp is defined @@ -62,13 +61,12 @@ - name: Start and enable node_exporter service when: __exporter_service.changed block: + - name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true - - name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: yes - - - name: Enable and start node_exporter service - ansible.builtin.systemd: - name: node_exporter - state: started - enabled: yes + - name: Enable and start node_exporter service + ansible.builtin.systemd: + name: node_exporter + state: started + enabled: true diff --git a/roles/platform/defaults/main.yml b/roles/platform/defaults/main.yml index 4543cb69..5ea5607d 100644 --- a/roles/platform/defaults/main.yml +++ b/roles/platform/defaults/main.yml @@ -20,211 +20,234 @@ # Labels # Deployment type (Ansible or Terraform) -plat__infra_deployment_engine: "{{ common__infra_deployment_engine }}" +plat__infra_deployment_engine: "{{ common__infra_deployment_engine }}" # Location of output from template module which creates Terraform -plat__terraform_template_dir: "{{ common__terraform_template_dir }}" -plat__terraform_artefact_dir: "{{ common__terraform_artefact_dir }}" -plat__terraform_workspace_dir: "{{ common__terraform_workspace_dir }}" +plat__terraform_template_dir: "{{ common__terraform_template_dir }}" +plat__terraform_artefact_dir: "{{ common__terraform_artefact_dir }}" +plat__terraform_workspace_dir: "{{ common__terraform_workspace_dir }}" -plat__terraform_allowed_state_storage: "{{ common__terraform_allowed_state_storage }}" -plat__terraform_state_storage: "{{ common__terraform_state_storage }}" -plat__terraform_remote_state_bucket: "{{ common__terraform_remote_state_bucket }}" -plat__terraform_remote_state_lock_table: "{{ common__terraform_remote_state_lock_table }}" +plat__terraform_allowed_state_storage: "{{ common__terraform_allowed_state_storage }}" +plat__terraform_state_storage: "{{ common__terraform_state_storage }}" +plat__terraform_remote_state_bucket: "{{ common__terraform_remote_state_bucket }}" +plat__terraform_remote_state_lock_table: "{{ common__terraform_remote_state_lock_table }}" -plat__infra_type: "{{ common__infra_type }}" -plat__region: "{{ common__region }}" -plat__namespace: "{{ common__namespace }}" -plat__namespace_cdp: "{{ common__namespace_cdp }}" +plat__infra_type: "{{ common__infra_type }}" +plat__region: "{{ common__region }}" +plat__namespace: "{{ common__namespace }}" +plat__namespace_cdp: "{{ common__namespace_cdp }}" -plat__env_suffix: "{{ common__env_suffix }}" -plat__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" -plat__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" +plat__env_suffix: "{{ common__env_suffix }}" +plat__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" +plat__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" -plat__cdp_control_plane_crn: "{{ common__cdp_control_plane_crn }}" -plat__cdp_iam_role_suffix: "{{ plat__cdp_iam_identities.role_suffix }}" -plat__cdp_iam_resource_suffix: "{{ plat__cdp_iam_identities.resource_role_suffix }}" +plat__cdp_control_plane_crn: "{{ common__cdp_control_plane_crn }}" +plat__cdp_iam_role_suffix: "{{ plat__cdp_iam_identities.role_suffix }}" +plat__cdp_iam_resource_suffix: "{{ plat__cdp_iam_identities.resource_role_suffix }}" # Infra -plat__tags: "{{ common__tags }}" -plat__env_name: "{{ common__env_name }}" -plat__vpc_name: "{{ common__vpc_name }}" -plat__storage_name: "{{ common__storage_name }}" +plat__tags: "{{ common__tags }}" +plat__env_name: "{{ common__env_name }}" +plat__vpc_name: "{{ common__vpc_name }}" +plat__storage_name: "{{ common__storage_name }}" -plat__logs_path: "{{ common__logs_path }}" -plat__data_path: "{{ common__data_path }}" -plat__backup_path: "{{ common__backups_path }}" +plat__logs_path: "{{ common__logs_path }}" +plat__data_path: "{{ common__data_path }}" +plat__backup_path: "{{ common__backups_path }}" -plat__public_key_id: "{{ common__public_key_id }}" -plat__public_key_text: "{{ common__public_key_text }}" -plat__public_key_file: "{{ common__public_key_file }}" +plat__public_key_id: "{{ common__public_key_id }}" +plat__public_key_text: "{{ common__public_key_text }}" +plat__public_key_file: "{{ common__public_key_file }}" -plat__security_group_knox_name: "{{ common__security_group_knox_name }}" -plat__security_group_default_name: "{{ common__security_group_default_name }}" +plat__security_group_knox_name: "{{ common__security_group_knox_name }}" +plat__security_group_default_name: "{{ common__security_group_default_name }}" -plat__vpc_public_subnets: "{{ common__vpc_public_subnet_cidrs }}" -plat__vpc_private_subnets: "{{ common__vpc_private_subnet_cidrs }}" +plat__vpc_public_subnets: "{{ common__vpc_public_subnet_cidrs }}" +plat__vpc_private_subnets: "{{ common__vpc_private_subnet_cidrs }}" # Plat -plat__teardown_deletes_policies: "{{ env.teardown.delete_policies | default(True) }}" -plat__teardown_deletes_roles: "{{ env.teardown.delete_roles | default(True) }}" -plat__teardown_deletes_gcp_custom_roles: "{{ env.teardown.delete_gcp_custom_roles | default(plat__teardown_deletes_roles) }}" -plat__teardown_deletes_xaccount: "{{ env.teardown.delete_cross_account | default(True) }}" -plat__teardown_deletes_credential: "{{ env.teardown.delete_credential | default(True) }}" -plat__teardown_deletes_admin_group: "{{ env.teardown.delete_admin_group | default(True) }}" -plat__teardown_deletes_user_group: "{{ env.teardown.delete_user_group | default(True) }}" +plat__teardown_deletes_policies: "{{ env.teardown.delete_policies | default(True) }}" +plat__teardown_deletes_roles: "{{ env.teardown.delete_roles | default(True) }}" +plat__teardown_deletes_gcp_custom_roles: "{{ env.teardown.delete_gcp_custom_roles | default(plat__teardown_deletes_roles) }}" +plat__teardown_deletes_xaccount: "{{ env.teardown.delete_cross_account | default(True) }}" +plat__teardown_deletes_credential: "{{ env.teardown.delete_credential | default(True) }}" +plat__teardown_deletes_admin_group: "{{ env.teardown.delete_admin_group | default(True) }}" +plat__teardown_deletes_user_group: "{{ env.teardown.delete_user_group | default(True) }}" -plat__xacccount_credential_name: "{{ common__xaccount_credential_name }}" +plat__xacccount_credential_name: "{{ common__xaccount_credential_name }}" -plat__workload_analytics: "{{ env.workload_analytics | default(True) }}" -plat__tunnel: "{{ common__tunnel }}" -plat__public_endpoint_access: "{{ common__public_endpoint_access }}" -plat__enable_raz: "{{ env.datalake.enable_raz | default(False) }}" -plat__use_public_ip: "{{ common__use_public_ip }}" +plat__workload_analytics: "{{ env.workload_analytics | default(True) }}" +plat__tunnel: "{{ common__tunnel }}" +plat__public_endpoint_access: "{{ common__public_endpoint_access }}" +plat__enable_raz: "{{ env.datalake.enable_raz | default(False) }}" +plat__use_public_ip: "{{ common__use_public_ip }}" -plat__env_admin_password: "{{ common__env_admin_password }}" +plat__env_admin_password: "{{ common__env_admin_password }}" -plat__vpc_public_subnet_cidrs: "{{ common__vpc_public_subnet_cidrs }}" -plat__vpc_private_subnet_cidrs: "{{ common__vpc_private_subnet_cidrs }}" +plat__vpc_public_subnet_cidrs: "{{ common__vpc_public_subnet_cidrs }}" +plat__vpc_private_subnet_cidrs: "{{ common__vpc_private_subnet_cidrs }}" # CDP IAM -plat__cdp_iam_group_suffix: "{{ env.cdp.group_suffix | default(common__group_suffix) }}" -plat__cdp_iam_admin_group_suffix: "{{ env.cdp.admin_group.suffix | default(common__admin_suffix) }}" -plat__cdp_iam_admin_group_name: "{{ env.cdp.admin_group.name | default([plat__namespace_cdp, plat__env_suffix, plat__cdp_iam_admin_group_suffix, plat__cdp_iam_group_suffix] | join('-')) }}" -plat__cdp_iam_admin_group_roles: "{{ env.cdp.admin_group.roles | default([plat__cdp_iam_identities.env_admin]) }}" -plat__cdp_iam_admin_group_resource_roles: "{{ env.cdp.admin_group.resource_roles | default(plat__cdp_iam_admin_group_resource_roles_default) }}" -plat__cdp_iam_user_group_suffix: "{{ env.cdp.user_group.suffix | default(common__user_suffix) }}" -plat__cdp_iam_user_group_name: "{{ env.cdp.user_group.name | default([plat__namespace_cdp, plat__env_suffix, plat__cdp_iam_user_group_suffix, plat__cdp_iam_group_suffix] | join('-')) }}" -plat__cdp_iam_user_group_roles: "{{ env.cdp.user_group.roles | default([plat__cdp_iam_identities.env_user]) }}" -plat__cdp_iam_user_group_resource_roles: "{{ env.cdp.user_group.resource_roles | default(plat__cdp_iam_user_group_resource_roles_default) }}" - -plat__env_freeipa: "{{ env.freeipa.instance_count | default(env.enable_ha | default(false) | ternary(3, 2)) }}" - -plat__datalake_name: "{{ common__datalake_name }}" -plat__datalake_version: "{{ env.datalake.version | default(omit) }}" -plat__datalake_scale: "{{ env.datalake.scale | default(env.enable_ha | default(false) | ternary('MEDIUM_DUTY_HA', 'LIGHT_DUTY')) }}" -plat__datalake_user_sync: "{{ env.datalake.user_sync | default(True) }}" - -plat__cdp_xaccount_external_id: "{{ env.cdp.cross_account.external_id | default(False) }}" -plat__cdp_xaccount_account_id: "{{ env.cdp.cross_account.account_id | default(False) }}" +plat__cdp_iam_group_suffix: "{{ env.cdp.group_suffix | default(common__group_suffix) }}" +plat__cdp_iam_admin_group_suffix: "{{ env.cdp.admin_group.suffix | default(common__admin_suffix) }}" +plat__cdp_iam_admin_group_name: "{{ env.cdp.admin_group.name | default([plat__namespace_cdp, plat__env_suffix, plat__cdp_iam_admin_group_suffix, plat__cdp_iam_group_suffix] + | join('-')) }}" +plat__cdp_iam_admin_group_roles: "{{ env.cdp.admin_group.roles | default([plat__cdp_iam_identities.env_admin]) }}" +plat__cdp_iam_admin_group_resource_roles: "{{ env.cdp.admin_group.resource_roles | default(plat__cdp_iam_admin_group_resource_roles_default) }}" +plat__cdp_iam_user_group_suffix: "{{ env.cdp.user_group.suffix | default(common__user_suffix) }}" +plat__cdp_iam_user_group_name: "{{ env.cdp.user_group.name | default([plat__namespace_cdp, plat__env_suffix, plat__cdp_iam_user_group_suffix, plat__cdp_iam_group_suffix] + | join('-')) }}" +plat__cdp_iam_user_group_roles: "{{ env.cdp.user_group.roles | default([plat__cdp_iam_identities.env_user]) }}" +plat__cdp_iam_user_group_resource_roles: "{{ env.cdp.user_group.resource_roles | default(plat__cdp_iam_user_group_resource_roles_default) }}" + +plat__env_freeipa: "{{ env.freeipa.instance_count | default(env.enable_ha | default(false) | ternary(3, 2)) }}" + +plat__datalake_name: "{{ common__datalake_name }}" +plat__datalake_version: "{{ env.datalake.version | default(omit) }}" +plat__datalake_scale: "{{ env.datalake.scale | default(env.enable_ha | default(false) | ternary('MEDIUM_DUTY_HA', 'LIGHT_DUTY')) }}" +plat__datalake_user_sync: "{{ env.datalake.user_sync | default(True) }}" + +plat__cdp_xaccount_external_id: "{{ env.cdp.cross_account.external_id | default(False) }}" +plat__cdp_xaccount_account_id: "{{ env.cdp.cross_account.account_id | default(False) }}" # AWS -plat__aws_profile: "{{ common__aws_profile }}" -plat__aws_arn_partition: "{{ env.aws.arn_partition | default('aws') }}" -plat__aws_vpc_id: "{{ common__aws_vpc_id }}" -plat__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" -plat__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" - -plat__aws_role_suffix: "{{ common__aws_role_suffix }}" -plat__aws_policy_suffix: "{{ env.aws.policy.suffix | default(common__policy_suffix) }}" -plat__aws_storage_suffix: "{{ env.aws.storage.suffix | default(common__storage_suffix) }}" - -plat__aws_role_tags: "{{ env.aws.role.tags | default({}) }}" -plat__aws_policy_tags: "{{ env.aws.policy.tags | default({}) }}" -plat__aws_storage_tags: "{{ env.aws.storage.tags | default({}) }}" - -plat__aws_xaccount_suffix: "{{ env.aws.role.label.cross_account | default(common__xaccount_suffix) }}" -plat__aws_idbroker_suffix: "{{ common__aws_idbroker_suffix }}" -plat__aws_log_suffix: "{{ env.aws.role.label.log | default(common__logs_suffix) }}" -plat__aws_datalake_admin_suffix: "{{ common__aws_datalake_admin_suffix }}" -plat__aws_ranger_audit_suffix: "{{ env.aws.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" - -plat__aws_xaccount_role_name: "{{ env.aws.role.name.cross_account | default([plat__namespace, plat__aws_xaccount_suffix, plat__aws_role_suffix] | join('-')) }}" -plat__aws_idbroker_role_name: "{{ common__aws_idbroker_role_name }}" -plat__aws_log_role_name: "{{ env.aws.role.name.log | default([plat__namespace, plat__aws_log_suffix, plat__aws_role_suffix] | join('-')) }}" -plat__aws_datalake_admin_role_name: "{{ common__aws_datalake_admin_role_name }}" -plat__aws_ranger_audit_role_name: "{{ env.aws.role.name.ranger_audit | default([plat__namespace, plat__aws_ranger_audit_suffix, plat__aws_role_suffix] | join('-')) }}" -plat__aws_ranger_cloud_access_role_name: "{{ env.aws.role.name.ranger_cloud_access | default(common__aws_datalake_admin_role_name) }}" - -plat__aws_xaccount_policy_name: "{{ env.aws.policy.name.cross_account | default([plat__namespace, plat__aws_xaccount_suffix, plat__aws_policy_suffix] | join('-')) }}" -plat__aws_idbroker_policy_name: "{{ env.aws.policy.name.idbroker | default([plat__namespace, plat__aws_idbroker_suffix, plat__aws_policy_suffix] | join('-')) }}" -plat__aws_log_location_policy_name: "{{ env.aws.policy.name.log | default([plat__namespace, plat__aws_log_suffix, plat__aws_policy_suffix] | join('-')) }}" -plat__aws_datalake_admin_s3_policy_name: "{{ env.aws.policy.name.datalake_admin_s3 | default([plat__namespace, plat__aws_datalake_admin_suffix, plat__aws_policy_suffix] | join('-')) }}" -plat__aws_ranger_audit_s3_policy_name: "{{ env.aws.policy.name.ranger_audit_s3 | default([plat__namespace, plat__aws_ranger_audit_suffix, plat__aws_policy_suffix] | join('-')) }}" -plat__aws_bucket_access_policy_name: "{{ env.aws.policy.name.bucket_access | default([plat__namespace, plat__aws_storage_suffix, plat__aws_policy_suffix] | join('-')) }}" - -plat__aws_policy_urls: "{{ plat__aws_policy_urls_default | combine(env.aws.policy.url | default({})) }}" +plat__aws_profile: "{{ common__aws_profile }}" +plat__aws_arn_partition: "{{ env.aws.arn_partition | default('aws') }}" +plat__aws_vpc_id: "{{ common__aws_vpc_id }}" +plat__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" +plat__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" + +plat__aws_role_suffix: "{{ common__aws_role_suffix }}" +plat__aws_policy_suffix: "{{ env.aws.policy.suffix | default(common__policy_suffix) }}" +plat__aws_storage_suffix: "{{ env.aws.storage.suffix | default(common__storage_suffix) }}" + +plat__aws_role_tags: "{{ env.aws.role.tags | default({}) }}" +plat__aws_policy_tags: "{{ env.aws.policy.tags | default({}) }}" +plat__aws_storage_tags: "{{ env.aws.storage.tags | default({}) }}" + +plat__aws_xaccount_suffix: "{{ env.aws.role.label.cross_account | default(common__xaccount_suffix) }}" +plat__aws_idbroker_suffix: "{{ common__aws_idbroker_suffix }}" +plat__aws_log_suffix: "{{ env.aws.role.label.log | default(common__logs_suffix) }}" +plat__aws_datalake_admin_suffix: "{{ common__aws_datalake_admin_suffix }}" +plat__aws_ranger_audit_suffix: "{{ env.aws.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" + +plat__aws_xaccount_role_name: "{{ env.aws.role.name.cross_account | default([plat__namespace, plat__aws_xaccount_suffix, plat__aws_role_suffix] | join('-')) }}" +plat__aws_idbroker_role_name: "{{ common__aws_idbroker_role_name }}" +plat__aws_log_role_name: "{{ env.aws.role.name.log | default([plat__namespace, plat__aws_log_suffix, plat__aws_role_suffix] | join('-')) }}" +plat__aws_datalake_admin_role_name: "{{ common__aws_datalake_admin_role_name }}" +plat__aws_ranger_audit_role_name: "{{ env.aws.role.name.ranger_audit | default([plat__namespace, plat__aws_ranger_audit_suffix, plat__aws_role_suffix] | join('-')) + }}" +plat__aws_ranger_cloud_access_role_name: "{{ env.aws.role.name.ranger_cloud_access | default(common__aws_datalake_admin_role_name) }}" + +plat__aws_xaccount_policy_name: "{{ env.aws.policy.name.cross_account | default([plat__namespace, plat__aws_xaccount_suffix, plat__aws_policy_suffix] | join('-')) + }}" +plat__aws_idbroker_policy_name: "{{ env.aws.policy.name.idbroker | default([plat__namespace, plat__aws_idbroker_suffix, plat__aws_policy_suffix] | join('-')) }}" +plat__aws_log_location_policy_name: "{{ env.aws.policy.name.log | default([plat__namespace, plat__aws_log_suffix, plat__aws_policy_suffix] | join('-')) }}" +plat__aws_datalake_admin_s3_policy_name: "{{ env.aws.policy.name.datalake_admin_s3 | default([plat__namespace, plat__aws_datalake_admin_suffix, plat__aws_policy_suffix] + | join('-')) }}" +plat__aws_ranger_audit_s3_policy_name: "{{ env.aws.policy.name.ranger_audit_s3 | default([plat__namespace, plat__aws_ranger_audit_suffix, plat__aws_policy_suffix] + | join('-')) }}" +plat__aws_bucket_access_policy_name: "{{ env.aws.policy.name.bucket_access | default([plat__namespace, plat__aws_storage_suffix, plat__aws_policy_suffix] | join('-')) + }}" + +plat__aws_policy_urls: "{{ plat__aws_policy_urls_default | combine(env.aws.policy.url | default({})) }}" # GCP -plat__gcp_project: "{{ common__gcp_project }}" -plat__gcp_subnet_id: "{{ common__gcp_subnet_id }}" - -plat__gcp_availability_zones: "{{ env.gcp.availability_zones | default([]) }}" - -plat__gcp_role_suffix: "{{ env.gcp.role.suffix | default(common__role_suffix) }}" -plat__gcp_storage_suffix: "{{ env.gcp.storage.suffix | default(common__storage_suffix) }}" - -plat__gcp_xaccount_suffix: "{{ env.gcp.role.label.cross_account | default(common__xaccount_suffix) }}" -plat__gcp_log_suffix: "{{ env.gcp.role.label.log | default(common__logs_suffix) }}" -plat__gcp_identity_suffix: "{{ env.gcp.role.label.identity | default(common__identity_suffix) }}" -plat__gcp_datalake_admin_suffix: "{{ env.gcp.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" -plat__gcp_ranger_audit_suffix: "{{ env.gcp.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" -plat__gcp_idbroker_suffix: "{{ env.gcp.role.label.idbroker | default(common__idbroker_suffix) }}" - -plat__gcp_log_role_name: "{{ env.gcp.role.name.log | default([plat__namespace, plat__gcp_log_suffix, plat__gcp_role_suffix] | join('-')) | replace('-','_') }}" -plat__gcp_xaccount_identity_name: "{{ env.gcp.role.name.cross_account | default([plat__namespace, plat__gcp_xaccount_suffix, plat__gcp_identity_suffix] | join('-')) }}" -plat__gcp_log_identity_name: "{{ env.gcp.role.name.identity | default([plat__namespace, plat__gcp_log_suffix, plat__gcp_identity_suffix] | join('-')) }}" -plat__gcp_datalakeadmin_identity_name: "{{ env.gcp.role.name.datalake_admin | default([plat__namespace, plat__gcp_datalake_admin_suffix, plat__gcp_identity_suffix] | join('-')) }}" -plat__gcp_ranger_audit_identity_name: "{{ env.gcp.role.name.ranger_audit | default([plat__namespace, plat__gcp_ranger_audit_suffix, plat__gcp_identity_suffix] | join('-')) }}" -plat__gcp_idbroker_identity_name: "{{ env.gcp.role.name.idbroker | default([plat__namespace, plat__gcp_idbroker_suffix, plat__gcp_identity_suffix] | join('-')) }}" - -plat__gcp_storage_location_data: "{{ env.gcp.storage.path.data | default([plat__storage_name, plat__data_path] | join('-')) }}" -plat__gcp_storage_location_logs: "{{ env.gcp.storage.path.logs | default([plat__storage_name, plat__logs_path] | join('-')) }}" -plat__gcp_storage_location_backups: "{{ env.gcp.storage.path.backups | default(plat__gcp_storage_location_logs) }}" - -plat__gcp_xaccount_policy_bindings: "{{ env.gcp.bindings.cross_account | default(plat__gcp_xaccount_policy_bindings_default) }}" -plat__gcp_log_role_perms: "{{ env.gcp.bindings.logs | default(plat__gcp_log_policy_bindings_default) }}" - -plat__gcp_manage_identities: "{{ env.gcp.manage_identities | default(true) }}" -plat__gcp_enable_services: "{{ env.gcp.auto_enable_services | default(true) }}" -plat__gcp_check_availability_zones: "{{ infra.gcp.check_availability_zones | default(true) }}" +plat__gcp_project: "{{ common__gcp_project }}" +plat__gcp_subnet_id: "{{ common__gcp_subnet_id }}" + +plat__gcp_availability_zones: "{{ env.gcp.availability_zones | default([]) }}" + +plat__gcp_role_suffix: "{{ env.gcp.role.suffix | default(common__role_suffix) }}" +plat__gcp_storage_suffix: "{{ env.gcp.storage.suffix | default(common__storage_suffix) }}" + +plat__gcp_xaccount_suffix: "{{ env.gcp.role.label.cross_account | default(common__xaccount_suffix) }}" +plat__gcp_log_suffix: "{{ env.gcp.role.label.log | default(common__logs_suffix) }}" +plat__gcp_identity_suffix: "{{ env.gcp.role.label.identity | default(common__identity_suffix) }}" +plat__gcp_datalake_admin_suffix: "{{ env.gcp.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" +plat__gcp_ranger_audit_suffix: "{{ env.gcp.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" +plat__gcp_idbroker_suffix: "{{ env.gcp.role.label.idbroker | default(common__idbroker_suffix) }}" + +plat__gcp_log_role_name: "{{ env.gcp.role.name.log | default([plat__namespace, plat__gcp_log_suffix, plat__gcp_role_suffix] | join('-')) | replace('-','_') }}" +plat__gcp_xaccount_identity_name: "{{ env.gcp.role.name.cross_account | default([plat__namespace, plat__gcp_xaccount_suffix, plat__gcp_identity_suffix] | join('-')) + }}" +plat__gcp_log_identity_name: "{{ env.gcp.role.name.identity | default([plat__namespace, plat__gcp_log_suffix, plat__gcp_identity_suffix] | join('-')) }}" +plat__gcp_datalakeadmin_identity_name: "{{ env.gcp.role.name.datalake_admin | default([plat__namespace, plat__gcp_datalake_admin_suffix, plat__gcp_identity_suffix] + | join('-')) }}" +plat__gcp_ranger_audit_identity_name: "{{ env.gcp.role.name.ranger_audit | default([plat__namespace, plat__gcp_ranger_audit_suffix, plat__gcp_identity_suffix] | join('-')) + }}" +plat__gcp_idbroker_identity_name: "{{ env.gcp.role.name.idbroker | default([plat__namespace, plat__gcp_idbroker_suffix, plat__gcp_identity_suffix] | join('-')) }}" + +plat__gcp_storage_location_data: "{{ env.gcp.storage.path.data | default([plat__storage_name, plat__data_path] | join('-')) }}" +plat__gcp_storage_location_logs: "{{ env.gcp.storage.path.logs | default([plat__storage_name, plat__logs_path] | join('-')) }}" +plat__gcp_storage_location_backups: "{{ env.gcp.storage.path.backups | default(plat__gcp_storage_location_logs) }}" + +plat__gcp_xaccount_policy_bindings: "{{ env.gcp.bindings.cross_account | default(plat__gcp_xaccount_policy_bindings_default) }}" +plat__gcp_log_role_perms: "{{ env.gcp.bindings.logs | default(plat__gcp_log_policy_bindings_default) }}" + +plat__gcp_manage_identities: "{{ env.gcp.manage_identities | default(true) }}" +plat__gcp_enable_services: "{{ env.gcp.auto_enable_services | default(true) }}" +plat__gcp_check_availability_zones: "{{ infra.gcp.check_availability_zones | default(true) }}" # Azure -plat__azure_app_suffix: "{{ env.azure.app.suffix | default(common__app_suffix) }}" -plat__azure_role_suffix: "{{ env.azure.role.suffix | default(common__role_suffix) }}" -plat__azure_policy_suffix: "{{ env.azure.policy.suffix | default(common__policy_suffix) }}" -plat__azure_storage_suffix: "{{ env.azure.storage.suffix | default(common__storage_suffix) }}" - -plat__azure_xaccount_suffix: "{{ env.azure.role.label.xaccount | default(common__xaccount_suffix) }}" -plat__azure_log_suffix: "{{ env.azure.role.label.log | default(common__logs_suffix) }}" -plat__azure_data_suffix: "{{ env.azure.role.label.data | default(common__data_suffix) }}" -plat__azure_identity_suffix: "{{ env.azure.role.label.identity | default(common__identity_suffix) }}" -plat__azure_datalake_admin_suffix: "{{ env.azure.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" -plat__azure_ranger_audit_suffix: "{{ env.azure.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" -plat__azure_raz_suffix: "{{ env.azure.role.label.raz | default(common__raz_suffix) }}" -plat__azure_idbroker_suffix: "{{ env.azure.role.label.idbroker | default(common__idbroker_suffix) }}" - -plat__azure_owner_name_suffix: "{{ env.azure.role.name_suffix.owner | default('owner') }}" -plat__azure_contributor_name_suffix: "{{ env.azure.role.name_suffix.contributor | default('contributor') }}" -plat__azure_operator_name_suffix: "{{ env.azure.role.name_suffix.operator | default('operator') }}" -plat__azure_admin_name_suffix: "{{ env.azure.role.name_suffix.admin | default(common__admin_suffix) }}" -plat__azure_user_name_suffix: "{{ env.azure.role.name_suffix.user | default(common__user_suffix) }}" -plat__azure_assignment_name_suffix: "{{ env.azure.role.name_suffix.assignment | default('assignment') }}" - -plat__azure_metagroup_name: "{{ common__azure_metagroup_name }}" -plat__azure_storage_name: "{{ common__azure_storage_name }}" - -plat__azure_consistency_wait: "{{ env.azure.app.wait | default(30) }}" -plat__azure_xaccount_use_custom_role: "{{ env.azure.use_custom_role | default(False) }}" -plat__azure_xaccount_rg_scope: "{{ env.azure.rg_scope | default(False) }}" -plat__azure_single_resource_group: "{{ env.azure.single_resource_group | default(False) }}" -plat__azure_xaccount_app_name: "{{ env.azure.app.name | default([plat__namespace, plat__azure_xaccount_suffix, plat__azure_app_suffix] | join('-')) }}" -plat__azure_xaccount_role_name: "{{ env.azure.role.name.cross_account | default([plat__namespace, plat__azure_xaccount_suffix, plat__azure_role_suffix] | join('-')) }}" -plat__azure_policy_url: "{{ env.azure.policy.url | default('https://raw.githubusercontent.com/cloudera-labs/snippets/main/policies/azure/cloudbreak_minimal_multiple_rgs_v1.json') }}" - -plat__azure_log_identity_name: "{{ env.azure.role.name.log | default([plat__namespace, plat__azure_log_suffix, plat__azure_identity_suffix] | join('-')) }}" -plat__azure_datalakeadmin_identity_name: "{{ env.azure.role.name.datalake_admin | default([plat__namespace, plat__azure_datalake_admin_suffix, plat__azure_identity_suffix] | join('-')) }}" -plat__azure_ranger_audit_identity_name: "{{ env.azure.role.name.ranger_audit | default([plat__namespace, plat__azure_ranger_audit_suffix, plat__azure_identity_suffix] | join('-')) }}" -plat__azure_raz_identity_name: "{{ env.azure.role.name.raz | default([plat__namespace, plat__azure_raz_suffix, plat__azure_identity_suffix] | join('-')) }}" -plat__azure_idbroker_identity_name: "{{ env.azure.role.name.idbroker | default([plat__namespace, plat__azure_idbroker_suffix, plat__azure_identity_suffix] | join('-')) }}" - -plat__azure_xaccount_contributor_assn_name: "{{ env.azure.role.assignment.cross_account.contributor | default('-'.join([plat__namespace, plat__azure_xaccount_suffix, plat__azure_contributor_name_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_xaccount_role_assn_name: "{{ env.azure.role.assignment.cross_account.role | default('-'.join([plat__namespace, plat__azure_xaccount_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_vmcontributor_idbroker_assn_name: "{{ env.azure.role.assignment.idbroker.vmcontributor | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, plat__azure_idbroker_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_mgdidentop_idbroker_assn_name: "{{ env.azure.role.assignment.idbroker.mgdidentop | default('-'.join([plat__namespace, plat__azure_operator_name_suffix, plat__azure_idbroker_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_storageowner_datalakeadmin_logs_assn_name: "{{ env.azure.role.assignment.datalake_admin.logs.storageowner | default('-'.join([plat__namespace, plat__azure_owner_name_suffix, plat__azure_log_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_storageowner_datalakeadmin_data_assn_name: "{{ env.azure.role.assignment.datalake_admin.data.storageowner | default('-'.join([plat__namespace, plat__azure_owner_name_suffix, plat__azure_data_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_storagecontr_log_assn_name: "{{ env.azure.role.assignment.log.storagecontr | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, plat__azure_log_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" -plat__azure_storagecontr_ranger_audit_assn_name: "{{ env.azure.role.assignment.ranger_audit.storagecontr | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, plat__azure_ranger_audit_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" - -plat__azure_stor_logs_uri: "{{ env.azure.storage.path.logs | default('abfs://' + plat__logs_path + '@' + plat__azure_storage_name + '.dfs.core.windows.net') }}" -plat__azure_stor_data_uri: "{{ env.azure.storage.path.data | default('abfs://' + plat__data_path + '@' + plat__azure_storage_name + '.dfs.core.windows.net') }}" +plat__azure_app_suffix: "{{ env.azure.app.suffix | default(common__app_suffix) }}" +plat__azure_role_suffix: "{{ env.azure.role.suffix | default(common__role_suffix) }}" +plat__azure_policy_suffix: "{{ env.azure.policy.suffix | default(common__policy_suffix) }}" +plat__azure_storage_suffix: "{{ env.azure.storage.suffix | default(common__storage_suffix) }}" + +plat__azure_xaccount_suffix: "{{ env.azure.role.label.xaccount | default(common__xaccount_suffix) }}" +plat__azure_log_suffix: "{{ env.azure.role.label.log | default(common__logs_suffix) }}" +plat__azure_data_suffix: "{{ env.azure.role.label.data | default(common__data_suffix) }}" +plat__azure_identity_suffix: "{{ env.azure.role.label.identity | default(common__identity_suffix) }}" +plat__azure_datalake_admin_suffix: "{{ env.azure.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" +plat__azure_ranger_audit_suffix: "{{ env.azure.role.label.ranger_audit | default(common__ranger_audit_suffix) }}" +plat__azure_raz_suffix: "{{ env.azure.role.label.raz | default(common__raz_suffix) }}" +plat__azure_idbroker_suffix: "{{ env.azure.role.label.idbroker | default(common__idbroker_suffix) }}" + +plat__azure_owner_name_suffix: "{{ env.azure.role.name_suffix.owner | default('owner') }}" +plat__azure_contributor_name_suffix: "{{ env.azure.role.name_suffix.contributor | default('contributor') }}" +plat__azure_operator_name_suffix: "{{ env.azure.role.name_suffix.operator | default('operator') }}" +plat__azure_admin_name_suffix: "{{ env.azure.role.name_suffix.admin | default(common__admin_suffix) }}" +plat__azure_user_name_suffix: "{{ env.azure.role.name_suffix.user | default(common__user_suffix) }}" +plat__azure_assignment_name_suffix: "{{ env.azure.role.name_suffix.assignment | default('assignment') }}" + +plat__azure_metagroup_name: "{{ common__azure_metagroup_name }}" +plat__azure_storage_name: "{{ common__azure_storage_name }}" + +plat__azure_consistency_wait: "{{ env.azure.app.wait | default(30) }}" +plat__azure_xaccount_use_custom_role: "{{ env.azure.use_custom_role | default(False) }}" +plat__azure_xaccount_rg_scope: "{{ env.azure.rg_scope | default(False) }}" +plat__azure_single_resource_group: "{{ env.azure.single_resource_group | default(False) }}" +plat__azure_xaccount_app_name: "{{ env.azure.app.name | default([plat__namespace, plat__azure_xaccount_suffix, plat__azure_app_suffix] | join('-')) }}" +plat__azure_xaccount_role_name: "{{ env.azure.role.name.cross_account | default([plat__namespace, plat__azure_xaccount_suffix, plat__azure_role_suffix] | join('-')) + }}" +plat__azure_policy_url: "{{ env.azure.policy.url | default('https://raw.githubusercontent.com/cloudera-labs/snippets/main/policies/azure/cloudbreak_minimal_multiple_rgs_v1.json') + }}" + +plat__azure_log_identity_name: "{{ env.azure.role.name.log | default([plat__namespace, plat__azure_log_suffix, plat__azure_identity_suffix] | join('-')) }}" +plat__azure_datalakeadmin_identity_name: "{{ env.azure.role.name.datalake_admin | default([plat__namespace, plat__azure_datalake_admin_suffix, plat__azure_identity_suffix] + | join('-')) }}" +plat__azure_ranger_audit_identity_name: "{{ env.azure.role.name.ranger_audit | default([plat__namespace, plat__azure_ranger_audit_suffix, plat__azure_identity_suffix] + | join('-')) }}" +plat__azure_raz_identity_name: "{{ env.azure.role.name.raz | default([plat__namespace, plat__azure_raz_suffix, plat__azure_identity_suffix] | join('-')) }}" +plat__azure_idbroker_identity_name: "{{ env.azure.role.name.idbroker | default([plat__namespace, plat__azure_idbroker_suffix, plat__azure_identity_suffix] | join('-')) + }}" + +plat__azure_xaccount_contributor_assn_name: "{{ env.azure.role.assignment.cross_account.contributor | default('-'.join([plat__namespace, plat__azure_xaccount_suffix, + plat__azure_contributor_name_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_xaccount_role_assn_name: "{{ env.azure.role.assignment.cross_account.role | default('-'.join([plat__namespace, plat__azure_xaccount_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) + | to_uuid )}}" +plat__azure_vmcontributor_idbroker_assn_name: "{{ env.azure.role.assignment.idbroker.vmcontributor | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, + plat__azure_idbroker_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_mgdidentop_idbroker_assn_name: "{{ env.azure.role.assignment.idbroker.mgdidentop | default('-'.join([plat__namespace, plat__azure_operator_name_suffix, + plat__azure_idbroker_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_storageowner_datalakeadmin_logs_assn_name: "{{ env.azure.role.assignment.datalake_admin.logs.storageowner | default('-'.join([plat__namespace, plat__azure_owner_name_suffix, + plat__azure_log_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_storageowner_datalakeadmin_data_assn_name: "{{ env.azure.role.assignment.datalake_admin.data.storageowner | default('-'.join([plat__namespace, plat__azure_owner_name_suffix, + plat__azure_data_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_storagecontr_log_assn_name: "{{ env.azure.role.assignment.log.storagecontr | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, plat__azure_log_suffix, + plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" +plat__azure_storagecontr_ranger_audit_assn_name: "{{ env.azure.role.assignment.ranger_audit.storagecontr | default('-'.join([plat__namespace, plat__azure_contributor_name_suffix, + plat__azure_ranger_audit_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}" + +plat__azure_stor_logs_uri: "{{ env.azure.storage.path.logs | default('abfs://' + plat__logs_path + '@' + plat__azure_storage_name + '.dfs.core.windows.net') }}" +plat__azure_stor_data_uri: "{{ env.azure.storage.path.data | default('abfs://' + plat__data_path + '@' + plat__azure_storage_name + '.dfs.core.windows.net') }}" diff --git a/roles/platform/meta/argument_specs.yml b/roles/platform/meta/argument_specs.yml new file mode 100644 index 00000000..1ef794b0 --- /dev/null +++ b/roles/platform/meta/argument_specs.yml @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy CDP Public Cloud core services + description: | + Deployment and management of Cloudera Data Platform (CDP) Public Cloud core + services, e.g. Environments, Datalakes, Shared Data Experiences (SDX). In + addition to the CDP services, the role handles the deployment and management + of the cloud provider artifacts required for CDP operations, e.g. roles, policies. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/platform/meta/main.yml b/roles/platform/meta/main.yml deleted file mode 100644 index ad7c8e4f..00000000 --- a/roles/platform/meta/main.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - role_name: platform - namespace: cloudera - author: Webster Mudge (wmudge@cloudera.com) - description: > - Deployment and management of Cloudera Data Platform (CDP) Public Cloud core - services, e.g. Environments, Datalakes, Shared Data Experiences (SDX). In - addition to the CDP services, the role handles the deployment and management - of the cloud provider artifacts required for CDP operations, e.g. roles, policies. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - aws - - gcloud - - azure - -dependencies: ['cloudera.exe.common'] diff --git a/roles/platform/molecule/default/collections.yml b/roles/platform/molecule/default/collections.yml index b05a6f87..366ccf0e 100644 --- a/roles/platform/molecule/default/collections.yml +++ b/roles/platform/molecule/default/collections.yml @@ -1,6 +1,5 @@ --- roles: [] - collections: - name: https://github.com/cloudera-labs/cloudera.cloud type: git diff --git a/roles/platform/molecule/default/molecule.yml b/roles/platform/molecule/default/molecule.yml index c9bdde63..5b3e02f4 100644 --- a/roles/platform/molecule/default/molecule.yml +++ b/roles/platform/molecule/default/molecule.yml @@ -46,7 +46,7 @@ provisioner: aws: region: ${FOUNDRY_AWS_REGION:-us-east-2} env: - tunnel: yes # L1 Networking + tunnel: yes # L1 Networking public_endpoint_access: yes verifier: name: ansible diff --git a/roles/platform/molecule/default/verify.yml b/roles/platform/molecule/default/verify.yml index 79044cd0..a5cfa75e 100644 --- a/roles/platform/molecule/default/verify.yml +++ b/roles/platform/molecule/default/verify.yml @@ -5,6 +5,6 @@ hosts: all gather_facts: false tasks: - - name: Example assertion - assert: - that: true + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/roles/platform/molecule/level0/molecule.yml b/roles/platform/molecule/level0/molecule.yml index 8b9ff31e..7340ada5 100644 --- a/roles/platform/molecule/level0/molecule.yml +++ b/roles/platform/molecule/level0/molecule.yml @@ -46,7 +46,7 @@ provisioner: aws: region: ${FOUNDRY_AWS_REGION:-us-east-2} env: - tunnel: no # L0 Networking + tunnel: no # L0 Networking verifier: name: ansible lint: | diff --git a/roles/platform/molecule/level0/verify.yml b/roles/platform/molecule/level0/verify.yml index 79044cd0..a5cfa75e 100644 --- a/roles/platform/molecule/level0/verify.yml +++ b/roles/platform/molecule/level0/verify.yml @@ -5,6 +5,6 @@ hosts: all gather_facts: false tasks: - - name: Example assertion - assert: - that: true + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/roles/platform/molecule/shared/cleanup.yml b/roles/platform/molecule/shared/cleanup.yml index c962cce8..594d7f24 100644 --- a/roles/platform/molecule/shared/cleanup.yml +++ b/roles/platform/molecule/shared/cleanup.yml @@ -1,5 +1,4 @@ --- - - name: Cleanup CDP deployment and SSH key hosts: localhost tasks: @@ -36,7 +35,7 @@ community.general.terraform: project_path: "{{ molecule_scenario_directory }}/deployment/" state: absent - force_init: yes + force_init: true - name: Delete the deployment state directory when: __deployment.stat.exists diff --git a/roles/platform/molecule/shared/prepare.yml b/roles/platform/molecule/shared/prepare.yml index 7232528e..af110c7a 100644 --- a/roles/platform/molecule/shared/prepare.yml +++ b/roles/platform/molecule/shared/prepare.yml @@ -1,5 +1,4 @@ --- - - name: Prepare hosts: localhost tasks: @@ -35,13 +34,13 @@ community.general.terraform: project_path: "{{ molecule_scenario_directory }}/deployment/" state: present - force_init: yes + force_init: true - name: Create the testing SSH key amazon.aws.ec2_key: name: "{{ [globals.name_prefix, lookup('password', '/dev/null chars=ascii_lowercase,digits length=8')] | join('-') }}" region: "{{ infra.aws.region }}" - force: yes + force: true register: __aws_ssh_key - name: Save the private key locally diff --git a/roles/platform/tasks/aws_policy_download.yml b/roles/platform/tasks/aws_policy_download.yml index 31690ad0..6a07495d 100644 --- a/roles/platform/tasks/aws_policy_download.yml +++ b/roles/platform/tasks/aws_policy_download.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/aws_policy_regex.yml b/roles/platform/tasks/aws_policy_regex.yml index af6ab1e8..e858e109 100644 --- a/roles/platform/tasks/aws_policy_regex.yml +++ b/roles/platform/tasks/aws_policy_regex.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,23 +17,23 @@ - name: Replace placeholder values in AWS default policy documents ansible.builtin.replace: - path: "{{ __aws_policy_document_item.dest }}" # Outer loop variable (is the result of get_url) + path: "{{ __aws_policy_document_item.dest }}" # Outer loop variable (is the result of get_url) regexp: "{{ __aws_policy_regexp_item.regexp }}" replace: "{{ __aws_policy_regexp_item.replace }}" loop_control: loop_var: __aws_policy_regexp_item loop: - - regexp: '\${LOGS_LOCATION_BASE}' + - regexp: "\\${LOGS_LOCATION_BASE}" replace: "{{ plat__storage_name }}" - - regexp: '\${LOGS_BUCKET}' + - regexp: "\\${LOGS_BUCKET}" replace: "{{ plat__storage_name }}" - - regexp: '\${STORAGE_LOCATION_BASE}' + - regexp: "\\${STORAGE_LOCATION_BASE}" replace: "{{ plat__storage_name }}" - - regexp: '\${DATALAKE_BUCKET}' + - regexp: "\\${DATALAKE_BUCKET}" replace: "{{ plat__storage_name }}" - - regexp: '\${STORAGE_LOCATION_BASE}' + - regexp: "\\${STORAGE_LOCATION_BASE}" replace: "{{ plat__storage_name }}" - - regexp: '\${DATALAKE_BUCKET}' + - regexp: "\\${DATALAKE_BUCKET}" replace: "{{ plat__storage_name }}" - - regexp: '\${ARN_PARTITION}' + - regexp: "\\${ARN_PARTITION}" replace: "{{ plat__aws_arn_partition }}" diff --git a/roles/platform/tasks/initialize_aws.yml b/roles/platform/tasks/initialize_aws.yml index e9fb714f..6cada989 100644 --- a/roles/platform/tasks/initialize_aws.yml +++ b/roles/platform/tasks/initialize_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/initialize_aws_terraform.yml b/roles/platform/tasks/initialize_aws_terraform.yml index 21c003fd..990c5d0d 100644 --- a/roles/platform/tasks/initialize_aws_terraform.yml +++ b/roles/platform/tasks/initialize_aws_terraform.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,13 +44,13 @@ # Copy Terraform provider file - name: Copy Terraform Provider file ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/provider.tf' + src: "files/{{ plat__infra_type }}/provider.tf" dest: "{{ plat__terraform_template_dir }}/plat/provider.tf" # Copy Terraform variables file - name: Copy Terraform Variables declaration file ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/terraform_variables.tf' + src: "files/{{ plat__infra_type }}/terraform_variables.tf" dest: "{{ plat__terraform_template_dir }}/plat/variables.tf" no_log: false @@ -59,26 +58,26 @@ # ...policies - name: Generate Terraform authz file for policies ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_policies.tf' + src: "files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_policies.tf" dest: "{{ plat__terraform_template_dir }}/plat/plat_authz_policies.tf" no_log: false # ...roles - name: Generate Terraform authz file for roles ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_roles.tf' + src: "files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_roles.tf" dest: "{{ plat__terraform_template_dir }}/plat/plat_authz_roles.tf" no_log: false # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'templates/{{ plat__infra_type }}/backend_state.tf.j2' + src: "templates/{{ plat__infra_type }}/backend_state.tf.j2" dest: "{{ plat__terraform_template_dir }}/plat/backend_state.tf" # Create Terraform variable definitions from template - name: Generate Terraform Variables definition ansible.builtin.template: - src: 'templates/{{ infra__type }}/terraform.tfvars.j2' + src: "templates/{{ infra__type }}/terraform.tfvars.j2" dest: "{{ plat__terraform_template_dir }}/plat/terraform.tfvars" no_log: false diff --git a/roles/platform/tasks/initialize_azure.yml b/roles/platform/tasks/initialize_azure.yml index 3fc38fae..ae2be228 100644 --- a/roles/platform/tasks/initialize_azure.yml +++ b/roles/platform/tasks/initialize_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +15,7 @@ - name: Check Azure CLI Account to ensure functionality and user access register: __azure_account_info - ansible.builtin.command: "az account show" # Calling the CLI version doesn't test much of the install + ansible.builtin.command: "az account show" # Calling the CLI version doesn't test much of the install - name: Set Azure Caller Information ansible.builtin.set_fact: @@ -66,18 +65,20 @@ - name: Check that we found a valid Service Principal for the Azure App ansible.builtin.assert: - quiet: yes + quiet: true that: __azure_application_service_principals_list.stdout | from_json | length == 1 - fail_msg: "Expected exactly one result from Azure Service Principal query for UUID {{ plat__azure_xaccount_app_uuid}}, got {{ __azure_application_service_principals_list.stdout | from_json | length }} instead" + fail_msg: "Expected exactly one result from Azure Service Principal query for UUID {{ plat__azure_xaccount_app_uuid }}, got {{ __azure_application_service_principals_list.stdout + | from_json | length }} instead" success_msg: "Found New Azure Cross Account App in directory matching UUID {{ plat__azure_xaccount_app_uuid }}, using for Cross Account Credential Creation" - name: Set Service Principal Object UUID for Azure App ansible.builtin.set_fact: - plat__azure_application_service_principal_objuuid: "{{ __azure_application_service_principals_list.stdout | from_json | community.general.json_query('[0].id') }}" + plat__azure_application_service_principal_objuuid: "{{ __azure_application_service_principals_list.stdout | from_json | community.general.json_query('[0].id') + }}" - name: Check that Azure Service Principal ID is now set ansible.builtin.assert: - quiet: yes + quiet: true that: plat__azure_application_service_principal_objuuid | length > 0 fail_msg: "Azure Service Principal Object ID appears to be length 0, please check and try again" @@ -93,11 +94,10 @@ - name: Fetch Azure Security Group Info register: __azure_sg_info - command: "az network nsg list" - + ansible.builtin.command: "az network nsg list" - name: Extract URIs for Azure Security Groups when: __azure_sg_info | length > 0 - ignore_errors: True # We do not want to fail if collecting facts and nsg are not already present + ignore_errors: true # We do not want to fail if collecting facts and nsg are not already present ansible.builtin.set_fact: __azure_sec_group_knox_uri: "{{ __azure_sg_info.stdout | from_json | community.general.json_query(__azure_jq_knox) | first }}" __azure_sec_group_default_uri: "{{ __azure_sg_info.stdout | from_json | community.general.json_query(__azure_jq_default) | first }}" @@ -114,7 +114,7 @@ loop: "{{ plat__vpc_public_subnet_cidrs }}" vars: entry: - name: "{{ [plat__namespace, plat__vpc_public_subnets_suffix, __public_subnet_idx|string] | join('-') }}" + name: "{{ [plat__namespace, plat__vpc_public_subnets_suffix, __public_subnet_idx | string] | join('-') }}" cidr: "{{ __public_subnet_item }}" tags: "kubernetes.io/role/elb": "1" @@ -128,7 +128,7 @@ loop: "{{ plat__vpc_private_subnet_cidrs }}" vars: entry: - name: "{{ [plat__namespace, plat__vpc_private_subnets_suffix, __private_subnet_idx|string] | join('-') }}" + name: "{{ [plat__namespace, plat__vpc_private_subnets_suffix, __private_subnet_idx | string] | join('-') }}" cidr: "{{ __private_subnet_item }}" tags: "kubernetes.io/role/internal-elb": "1" @@ -143,8 +143,10 @@ - name: Set fact for Azure Storage URIs ansible.builtin.set_fact: plat__azure_storage_account_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}" - plat__azure_logpath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ plat__azure_log_suffix }}" - plat__azure_datapath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ plat__azure_data_suffix }}" + plat__azure_logpath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ + plat__azure_log_suffix }}" + plat__azure_datapath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ + plat__azure_data_suffix }}" - name: Set fact for default Azure MSIs ansible.builtin.set_fact: diff --git a/roles/platform/tasks/initialize_gcp.yml b/roles/platform/tasks/initialize_gcp.yml index 19e13ab6..31cc263f 100644 --- a/roles/platform/tasks/initialize_gcp.yml +++ b/roles/platform/tasks/initialize_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/initialize_setup.yml b/roles/platform/tasks/initialize_setup.yml index 0ca1843a..4b5b7482 100644 --- a/roles/platform/tasks/initialize_setup.yml +++ b/roles/platform/tasks/initialize_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/initialize_setup_aws.yml b/roles/platform/tasks/initialize_setup_aws.yml index bef02808..456e657a 100644 --- a/roles/platform/tasks/initialize_setup_aws.yml +++ b/roles/platform/tasks/initialize_setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,7 +62,7 @@ ansible.builtin.assert: that: __aws_vpc_info.vpcs | length == 1 fail_msg: "No AWS VPC discovered" - quiet: yes + quiet: true - name: Set fact for AWS VPC ID when: __aws_vpc_info is defined @@ -89,7 +88,7 @@ ansible.builtin.assert: that: __aws_subnets_info.subnets | length > 0 fail_msg: "No subnets discovered for AWS VPC" - quiet: yes + quiet: true - name: Set fact for AWS Public Subnet IDs if established by Infrastructure when: not plat__aws_public_subnet_ids and infra__aws_public_subnet_ids is defined diff --git a/roles/platform/tasks/initialize_setup_azure.yml b/roles/platform/tasks/initialize_setup_azure.yml index 42d61b3b..7f5a0c48 100644 --- a/roles/platform/tasks/initialize_setup_azure.yml +++ b/roles/platform/tasks/initialize_setup_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/initialize_setup_base.yml b/roles/platform/tasks/initialize_setup_base.yml index cc554adc..830d1268 100644 --- a/roles/platform/tasks/initialize_setup_base.yml +++ b/roles/platform/tasks/initialize_setup_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,11 +22,11 @@ that: - "__cdp_cli.rc == 0" fail_msg: "CDP CLI failed to execute: {{ __cdp_cli.stdout }} ({{ __cdp_cli.stderr }})" - quiet: yes + quiet: true - name: Query CDP Caller to confirm access to Endpoints cloudera.cloud.iam_user_info: - current_user: yes + current_user: true register: __cdp_iam_current_user_info - name: Set facts for CDP Caller Workload Username and CRN diff --git a/roles/platform/tasks/initialize_setup_gcp.yml b/roles/platform/tasks/initialize_setup_gcp.yml index 906f8167..01d0a595 100644 --- a/roles/platform/tasks/initialize_setup_gcp.yml +++ b/roles/platform/tasks/initialize_setup_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,28 +31,30 @@ ansible.builtin.assert: that: - plat__gcp_availability_zones is subset(plat__gcp_availability_zones_discovered) - fail_msg: "The following zone(s) are not available in region '{{ plat__region }}': {{ plat__gcp_availability_zones | difference(plat__gcp_availability_zones_discovered) | join(', ') }}" - quiet: yes + fail_msg: "The following zone(s) are not available in region '{{ plat__region }}': {{ plat__gcp_availability_zones | difference(plat__gcp_availability_zones_discovered) + | join(', ') }}" + quiet: true # https://docs.cloudera.com/management-console/cloud/requirements-gcp/topics/mc-gcp_apis.html - name: Ensure Google Services Enabled when: plat__gcp_enable_services | bool block: - name: Fetch list of enabled GCP Services - command: > + ansible.builtin.command: > gcloud services list --enabled --project {{ plat__gcp_project }} register: __gcp_services_info - name: Determine list of missing Services - set_fact: - __plat_gcp_services_to_enable: "{{ __plat_gcp_services_to_enable | default([]) + ([__gcp_service_item] if __gcp_service_item not in __gcp_services_info.stdout else []) | unique }}" + ansible.builtin.set_fact: + __plat_gcp_services_to_enable: "{{ __plat_gcp_services_to_enable | default([]) + ([__gcp_service_item] if __gcp_service_item not in __gcp_services_info.stdout + else []) | unique }}" loop: "{{ plat__gcp_required_services }}" loop_control: loop_var: __gcp_service_item - name: Enable missing GCP Service APIs when: __plat_gcp_services_to_enable | length > 0 - command: > + ansible.builtin.command: > gcloud services enable --quiet {{ __gcp_enable_item }} loop: "{{ __plat_gcp_services_to_enable }}" loop_control: diff --git a/roles/platform/tasks/initialize_teardown.yml b/roles/platform/tasks/initialize_teardown.yml index 7e7a7fbb..3125f5bd 100644 --- a/roles/platform/tasks/initialize_teardown.yml +++ b/roles/platform/tasks/initialize_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +29,7 @@ - plat__env_info.environments[0].descendants.opdb | length == 0 - plat__env_info.environments[0].descendants.de | length == 0 fail_msg: "Environment {{ plat__env_name }} has one or more child services registered, please check and try again" - quiet: yes + quiet: true - name: Include provider-specific initialization base ansible.builtin.include_tasks: "initialize_{{ plat__infra_type | lower }}.yml" diff --git a/roles/platform/tasks/initialize_teardown_aws_terraform.yml b/roles/platform/tasks/initialize_teardown_aws_terraform.yml index 486c6249..1d046e57 100644 --- a/roles/platform/tasks/initialize_teardown_aws_terraform.yml +++ b/roles/platform/tasks/initialize_teardown_aws_terraform.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,20 +21,20 @@ # Copy Terraform provider - name: Copy Terraform Provider file ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/provider.tf' + src: "files/{{ plat__infra_type }}/provider.tf" dest: "{{ plat__terraform_template_dir }}/plat/provider.tf" # Copy Terraform variables file - name: Copy Terraform Variables declaration file ansible.builtin.copy: - src: 'files/{{ plat__infra_type }}/terraform_variables.tf' + src: "files/{{ plat__infra_type }}/terraform_variables.tf" dest: "{{ plat__terraform_template_dir }}/plat/variables.tf" no_log: false # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'templates/{{ plat__infra_type }}/backend_state.tf.j2' + src: "templates/{{ plat__infra_type }}/backend_state.tf.j2" dest: "{{ plat__terraform_template_dir }}/plat/backend_state.tf" - name: Create a temporary directory for policy documents diff --git a/roles/platform/tasks/initialize_teardown_azure.yml b/roles/platform/tasks/initialize_teardown_azure.yml index 547d1032..76f62814 100644 --- a/roles/platform/tasks/initialize_teardown_azure.yml +++ b/roles/platform/tasks/initialize_teardown_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +18,7 @@ resource_group: "{{ plat__azure_metagroup_name }}" provider: ManagedIdentity resource_type: userAssignedIdentities - api_version: '2018-11-30' + api_version: "2018-11-30" register: __azure_identity_list - name: Retrieve Azure role assignments for Subscription @@ -32,12 +31,15 @@ - name: Set list of role assignments to delete ansible.builtin.set_fact: - __plat_azure_role_assignment_list: "{{ __plat_azure_role_assignments_discovered.roleassignments | selectattr('assignee_object_id', 'in', role_assignment_assignee_list) | list if __plat_azure_role_assignments_discovered.roleassignments is defined else [] }}" + __plat_azure_role_assignment_list: "{{ __plat_azure_role_assignments_discovered.roleassignments | selectattr('assignee_object_id', 'in', role_assignment_assignee_list) + | list if __plat_azure_role_assignments_discovered.roleassignments is defined else [] }}" vars: - msi_principal_id_list: "{{ __azure_identity_list.response | rejectattr('name', 'undefined') | selectattr('name', 'in', identity_names) | map(attribute='properties') | map(attribute='principalId') | list }}" + msi_principal_id_list: "{{ __azure_identity_list.response | rejectattr('name', 'undefined') | selectattr('name', 'in', identity_names) | map(attribute='properties') + | map(attribute='principalId') | list }}" identity_names: - "{{ plat__azure_idbroker_identity_name }}" - "{{ plat__azure_datalakeadmin_identity_name }}" - "{{ plat__azure_log_identity_name }}" - "{{ plat__azure_ranger_audit_identity_name }}" - role_assignment_assignee_list: "{{ msi_principal_id_list | union([plat__azure_application_service_principal_objuuid] if plat__azure_application_service_principal_objuuid is defined else []) }}" + role_assignment_assignee_list: "{{ msi_principal_id_list | union([plat__azure_application_service_principal_objuuid] if plat__azure_application_service_principal_objuuid + is defined else []) }}" diff --git a/roles/platform/tasks/initialize_teardown_gcp.yml b/roles/platform/tasks/initialize_teardown_gcp.yml index 1deb15cf..a3e71ae4 100644 --- a/roles/platform/tasks/initialize_teardown_gcp.yml +++ b/roles/platform/tasks/initialize_teardown_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,11 +19,10 @@ - __gcp_xaccount_sa_discovered.rc == 1 - "'NOT_FOUND:' not in __gcp_xaccount_sa_discovered.stderr" - "'Permission iam.serviceAccountKeys.list' not in __gcp_xaccount_sa_discovered.stderr" - command: > + ansible.builtin.command: > gcloud iam service-accounts keys list --iam-account "{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" --format="json" - - name: Set discovered Cross Account Service Account keys if exists when: - __gcp_xaccount_sa_discovered is defined diff --git a/roles/platform/tasks/main.yml b/roles/platform/tasks/main.yml index 3211b7d8..61082f58 100644 --- a/roles/platform/tasks/main.yml +++ b/roles/platform/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include_tasks: validate.yml -- include_tasks: initialize_setup.yml -- include_tasks: setup.yml +- ansible.builtin.include_tasks: validate.yml +- ansible.builtin.include_tasks: initialize_setup.yml +- ansible.builtin.include_tasks: setup.yml diff --git a/roles/platform/tasks/setup.yml b/roles/platform/tasks/setup.yml index 8cb7c869..eeaf4e0a 100644 --- a/roles/platform/tasks/setup.yml +++ b/roles/platform/tasks/setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/setup_aws_authz.yml b/roles/platform/tasks/setup_aws_authz.yml index 3cafb280..458c2392 100644 --- a/roles/platform/tasks/setup_aws_authz.yml +++ b/roles/platform/tasks/setup_aws_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -174,8 +173,8 @@ region: "{{ plat__region }}" name: "{{ __aws_service_role_details_item.name }}" description: "{{ __aws_service_role_details_item.description }} role for {{ plat__namespace }}" - create_instance_profile: yes - purge_policies: yes + create_instance_profile: true + purge_policies: true assume_role_policy_document: Version: "2012-10-17" Statement: @@ -224,8 +223,8 @@ region: "{{ plat__region }}" name: "{{ __aws_data_access_role_details_item.name }}" description: "{{ __aws_data_access_role_details_item.description }} role for {{ plat__namespace }}" - create_instance_profile: yes - purge_policies: yes + create_instance_profile: true + purge_policies: true assume_role_policy_document: Version: "2012-10-17" Statement: @@ -268,7 +267,7 @@ failed_when: __aws_data_access_role_tags_item.failed - name: Ensure AWS Instance Profiles are attached to CDP Roles - command: > + ansible.builtin.command: > aws iam add-role-to-instance-profile --instance-profile-name {{ __aws_role_item }} --role-name {{ __aws_role_item }} diff --git a/roles/platform/tasks/setup_aws_datalake.yml b/roles/platform/tasks/setup_aws_datalake.yml index 963ad5aa..85b22bc6 100644 --- a/roles/platform/tasks/setup_aws_datalake.yml +++ b/roles/platform/tasks/setup_aws_datalake.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +26,7 @@ state: present - name: Retrieve AWS EC2 Instance details for CDP Datalake - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ plat__region }}" filters: "tag:Name": "{{ plat__namespace }}*" diff --git a/roles/platform/tasks/setup_aws_env.yml b/roles/platform/tasks/setup_aws_env.yml index c883daf6..9a20eb2e 100644 --- a/roles/platform/tasks/setup_aws_env.yml +++ b/roles/platform/tasks/setup_aws_env.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/setup_aws_idbroker.yml b/roles/platform/tasks/setup_aws_idbroker.yml index 43eba5ab..02656f3c 100644 --- a/roles/platform/tasks/setup_aws_idbroker.yml +++ b/roles/platform/tasks/setup_aws_idbroker.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set CDP IDBroker Mappings for AWS cloudera.cloud.env_idbroker: name: "{{ plat__env_name }}" - sync: no + sync: false data_access: "{{ plat__aws_datalake_admin_role_arn }}" ranger_audit: "{{ plat__aws_ranger_audit_role_arn }}" ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(plat__aws_ranger_cloud_access_role_arn, omit) }}" diff --git a/roles/platform/tasks/setup_aws_terraform_authz.yml b/roles/platform/tasks/setup_aws_terraform_authz.yml index 3ec0fa5d..ebf7d3f4 100644 --- a/roles/platform/tasks/setup_aws_terraform_authz.yml +++ b/roles/platform/tasks/setup_aws_terraform_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +27,7 @@ community.general.terraform: project_path: "{{ plat__terraform_workspace_dir }}/plat" state: "present" - force_init: yes + force_init: true register: tf_result retries: 3 delay: 10 diff --git a/roles/platform/tasks/setup_azure_authz.yml b/roles/platform/tasks/setup_azure_authz.yml index c855296a..0d72246a 100644 --- a/roles/platform/tasks/setup_azure_authz.yml +++ b/roles/platform/tasks/setup_azure_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,15 +16,15 @@ - name: Handle Azure Cross Account Role register: __azure_xaccount_role_info when: plat__azure_xaccount_use_custom_role | bool - azure.azcollection.azure_rm_roledefinition: # This version fails idempotence if a description is set + azure.azcollection.azure_rm_roledefinition: # This version fails idempotence if a description is set state: present name: "{{ plat__azure_xaccount_role_name }}" assignable_scopes: "/subscriptions/{{ plat__azure_subscription_id }}" permissions: - - actions: "{{ lookup('file', __azure_policy_document.dest ) | from_json | community.general.json_query('Actions') }}" - data_actions: "{{ lookup('file', __azure_policy_document.dest ) | from_json | community.general.json_query('DataActions') }}" - not_actions: "{{ lookup('file', __azure_policy_document.dest ) | from_json | community.general.json_query('NotActions') }}" - not_data_actions: "{{ lookup('file', __azure_policy_document.dest ) | from_json | community.general.json_query('NotDataActions') }}" + - actions: "{{ lookup('file', __azure_policy_document.dest) | from_json | community.general.json_query('Actions') }}" + data_actions: "{{ lookup('file', __azure_policy_document.dest) | from_json | community.general.json_query('DataActions') }}" + not_actions: "{{ lookup('file', __azure_policy_document.dest) | from_json | community.general.json_query('NotActions') }}" + not_data_actions: "{{ lookup('file', __azure_policy_document.dest) | from_json | community.general.json_query('NotDataActions') }}" - name: Ensure Azure Cross Account App and Credential are Deployed when: plat__azure_xaccount_app_uuid is undefined or plat__xacccount_credential_name not in plat__cdp_credentials_list @@ -40,31 +39,29 @@ when: plat__azure_xaccount_app_uuid is defined block: - name: Issue Azure App Delete command - command: "az ad app delete --id {{ plat__azure_xaccount_app_uuid }}" - + ansible.builtin.command: "az ad app delete --id {{ plat__azure_xaccount_app_uuid }}" - name: Wait for consistency ansible.builtin.pause: seconds: "{{ plat__azure_consistency_wait }}" - name: Request Azure Cross Account App Creation - no_log: True + no_log: true register: __azure_xaccount_app_info - command: > + ansible.builtin.command: > az ad sp create-for-rbac --name {{ plat__azure_xaccount_app_name }} --role {{ plat__azure_xaccount_use_custom_role | ternary(__azure_xaccount_role_info.id, plat__azure_roles.contrib) }} --scope {{ plat__azure_xaccount_rg_scope | ternary(plat__azure_metagroup_uri, plat__azure_subscription_uri) }} - - name: Register Azure Cross Account App info - no_log: True + no_log: true ansible.builtin.set_fact: - __azure_xaccount_app_pword: "{{ __azure_xaccount_app_info.stdout | from_json | community.general.json_query('password') }}" - plat__azure_xaccount_app_uuid: "{{ __azure_xaccount_app_info.stdout | from_json | community.general.json_query('appId') }}" + __azure_xaccount_app_pword: "{{ __azure_xaccount_app_info.stdout | from_json | community.general.json_query('password') }}" + plat__azure_xaccount_app_uuid: "{{ __azure_xaccount_app_info.stdout | from_json | community.general.json_query('appId') }}" - name: Validate that the Azure Cross Account App info has been set - no_log: True + no_log: true ansible.builtin.assert: - quiet: yes + quiet: true that: - __azure_xaccount_app_pword | length > 0 - plat__azure_xaccount_app_uuid | length > 0 @@ -83,18 +80,20 @@ - name: Check that we found a valid Service Principal for the Azure App ansible.builtin.assert: - quiet: yes + quiet: true that: __azure_application_service_principals_list.stdout | from_json | length == 1 - fail_msg: "Expected exactly one result from Azure Service Principal query for UUID {{ plat__azure_xaccount_app_uuid}}, got {{ __azure_application_service_principals_list.stdout | from_json | length }} instead" + fail_msg: "Expected exactly one result from Azure Service Principal query for UUID {{ plat__azure_xaccount_app_uuid }}, got {{ __azure_application_service_principals_list.stdout + | from_json | length }} instead" success_msg: "Found New Azure Cross Account App in directory matching UUID {{ plat__azure_xaccount_app_uuid }}, using for Cross Account Credential Creation" - name: Set Service Principal Object ID for new Azure App ansible.builtin.set_fact: - plat__azure_application_service_principal_objuuid: "{{ __azure_application_service_principals_list.stdout | from_json | community.general.json_query('[0].id') }}" + plat__azure_application_service_principal_objuuid: "{{ __azure_application_service_principals_list.stdout | from_json | community.general.json_query('[0].id') + }}" - name: Check that Azure Service Principal ID is now set ansible.builtin.assert: - quiet: yes + quiet: true that: plat__azure_application_service_principal_objuuid | length > 0 fail_msg: "Azure Service Principal Object ID appears to be length 0, please check and try again" @@ -117,8 +116,8 @@ provider: ManagedIdentity resource_type: userAssignedIdentities resource_name: "{{ __azure_msi_item }}" - api_version: '2018-11-30' - idempotency: yes + api_version: "2018-11-30" + idempotency: true state: present body: location: "{{ plat__region }}" @@ -129,7 +128,7 @@ resource_group: "{{ plat__azure_metagroup_name }}" provider: ManagedIdentity resource_type: userAssignedIdentities - api_version: '2018-11-30' + api_version: "2018-11-30" register: __azure_identity_list delay: 5 retries: 120 # 10 mins @@ -147,18 +146,24 @@ - name: Refresh listing of Azure Role Definitions register: __azure_role_definition_info - no_log: yes # Extremely verbose output + no_log: true # Extremely verbose output azure.azcollection.azure_rm_roledefinition_info: scope: "{{ plat__azure_subscription_uri }}" - name: Extract Azure Role Definition IDs ansible.builtin.set_fact: - __azure_virtualmachine_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.vmcnt) | map(attribute='id') | list | first }}" - __azure_managedidentity_optr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.miop) | map(attribute='id') | list | first }}" - __azure_storageblobdata_ownr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storown) | map(attribute='id') | list | first }}" - __azure_storageblobdata_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storcnt) | map(attribute='id') | list | first }}" - __azure_contributor_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.contrib) | map(attribute='id') | list | first }}" - __azure_storageblob_delegator_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.stordel) | map(attribute='id') | list | first }}" + __azure_virtualmachine_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.vmcnt) | map(attribute='id') + | list | first }}" + __azure_managedidentity_optr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.miop) | map(attribute='id') + | list | first }}" + __azure_storageblobdata_ownr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storown) | map(attribute='id') + | list | first }}" + __azure_storageblobdata_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storcnt) | map(attribute='id') + | list | first }}" + __azure_contributor_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.contrib) | map(attribute='id') + | list | first }}" + __azure_storageblob_delegator_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.stordel) | map(attribute='id') + | list | first }}" - name: Process Azure Role Assignments register: __infra_az_sp_assign_result diff --git a/roles/platform/tasks/setup_azure_datalake.yml b/roles/platform/tasks/setup_azure_datalake.yml index c9499133..d11a30b5 100644 --- a/roles/platform/tasks/setup_azure_datalake.yml +++ b/roles/platform/tasks/setup_azure_datalake.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/setup_azure_env.yml b/roles/platform/tasks/setup_azure_env.yml index 6a0c743a..c574d198 100644 --- a/roles/platform/tasks/setup_azure_env.yml +++ b/roles/platform/tasks/setup_azure_env.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/setup_azure_idbroker.yml b/roles/platform/tasks/setup_azure_idbroker.yml index 01351d7c..9018cde0 100644 --- a/roles/platform/tasks/setup_azure_idbroker.yml +++ b/roles/platform/tasks/setup_azure_idbroker.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set IDBroker Mappings for CDP on Azure cloudera.cloud.env_idbroker: name: "{{ plat__env_name }}" - sync: no + sync: false data_access: "{{ __azure_datalakeadmin_identity.id }}" ranger_audit: "{{ __azure_ranger_audit_identity.id }}" ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(__azure_raz_identity.id, omit) }}" diff --git a/roles/platform/tasks/setup_base.yml b/roles/platform/tasks/setup_base.yml index a6c6bff9..ecfdb734 100644 --- a/roles/platform/tasks/setup_base.yml +++ b/roles/platform/tasks/setup_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +25,8 @@ - name: Set fact for CDP Admin Group Resource Role assignments ansible.builtin.set_fact: - plat__cdp_env_admin_group_resource_role_assignments: "{{ plat__cdp_env_admin_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" + plat__cdp_env_admin_group_resource_role_assignments: "{{ plat__cdp_env_admin_group_resource_role_assignments | default([]) | union([resource_role_assignment]) + }}" vars: resource_role_assignment: resource: "{{ plat__cdp_env_crn }}" @@ -100,14 +100,14 @@ ansible.builtin.uri: url: "{{ plat__cdp_datalake_cm_api.rstrip('/') }}/version" timeout: 60 - return_content: no + return_content: false status_code: 200 url_username: "{{ plat__cdp_workload_username }}" url_password: "{{ plat__env_admin_password }}" validate_certs: false - run_once: yes + run_once: true register: __cdp_datalake_cm_api_response - ignore_errors: yes + ignore_errors: true - name: Request User Sync for CDP Datalake, if needed when: __cdp_datalake_cm_api_response.status != 200 diff --git a/roles/platform/tasks/setup_gcp_authz.yml b/roles/platform/tasks/setup_gcp_authz.yml index fb34320e..2614a32f 100644 --- a/roles/platform/tasks/setup_gcp_authz.yml +++ b/roles/platform/tasks/setup_gcp_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,14 +35,13 @@ loop_control: loop_var: __gcp_sa_binding_item loop: "{{ plat__gcp_xaccount_policy_bindings }}" - command: > + ansible.builtin.command: > gcloud projects add-iam-policy-binding {{ plat__gcp_project }} --member="serviceAccount:{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" --role={{ __gcp_sa_binding_item |quote }} --no-user-output-enabled --condition=None - - name: Generate Key for Google Cross Account Service Account google.cloud.gcp_iam_service_account_key: service_account: "{{ __gcp_xaccount_sa_info }}" @@ -98,7 +96,7 @@ that: - "'includedPermissions:' in __gcp_custom_log_role_info.stdout" - "'NOT_FOUND' not in __gcp_custom_log_role_info.stderr" - quiet: yes + quiet: true fail_msg: | Custom Log Role {{ plat__gcp_log_role_name }} could not be created or undeleted. It is likely that the unique role_id was marked for deletion recently and you are in the GCloud no-reuse window. @@ -136,14 +134,16 @@ condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}'" - member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.storage_admin }}" - condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}-fulladmin'" + condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name + }}-fulladmin'" - member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.storage_object_admin }}" condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}'" # Ranger Audit - member: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.storage_object_admin }}" - condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}-fulladmin'" + condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name + }}-fulladmin'" # ID Broker / Assumer Role - member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.iam_workload_identity_user }}" @@ -154,13 +154,12 @@ - member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.iam_service_account_token_creator }}" condition: "None" - command: > + ansible.builtin.command: > gcloud projects add-iam-policy-binding {{ plat__gcp_project }} --member={{ __gcp_binding_item.member |quote }} --role={{ __gcp_binding_item.role |quote }} --condition={{ __gcp_binding_item.condition |quote }} - - name: Add Service Accounts to Storage Policies for Buckets loop_control: loop_var: __gcp_pol_item @@ -173,7 +172,7 @@ bucket: "{{ plat__gcp_storage_location_data }}" - account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin" bucket: "{{ plat__gcp_storage_location_data }}" - command: > + ansible.builtin.command: > gsutil iam ch {{ __gcp_pol_item.account |quote }} gs://{{ __gcp_pol_item.bucket |quote }} diff --git a/roles/platform/tasks/setup_gcp_datalake.yml b/roles/platform/tasks/setup_gcp_datalake.yml index 2f7ed61a..793d7ceb 100644 --- a/roles/platform/tasks/setup_gcp_datalake.yml +++ b/roles/platform/tasks/setup_gcp_datalake.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/setup_gcp_env.yml b/roles/platform/tasks/setup_gcp_env.yml index aac400c0..77748a18 100644 --- a/roles/platform/tasks/setup_gcp_env.yml +++ b/roles/platform/tasks/setup_gcp_env.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +27,7 @@ backup_location: "gs://{{ plat__gcp_storage_location_backups }}" vpc_id: "{{ plat__vpc_name }}" subnet_ids: - - "{{ plat__gcp_subnet_id if plat__gcp_subnet_id else plat__gcp_subnets_discovered[0].name }}" # TODO - Check in validation_gcp.yml -- CDP on GCP only supports a single subnet deployment + - "{{ plat__gcp_subnet_id if plat__gcp_subnet_id else plat__gcp_subnets_discovered[0].name }}" # TODO - Check in validation_gcp.yml -- CDP on GCP only supports a single subnet deployment project: "{{ plat__gcp_project }}" tunnel: "{{ plat__tunnel }}" workload_analytics: "{{ plat__workload_analytics }}" diff --git a/roles/platform/tasks/setup_gcp_idbroker.yml b/roles/platform/tasks/setup_gcp_idbroker.yml index 2213d1eb..ce9d3219 100644 --- a/roles/platform/tasks/setup_gcp_idbroker.yml +++ b/roles/platform/tasks/setup_gcp_idbroker.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set IDBroker Mappings for CDP on GCP cloudera.cloud.env_idbroker: name: "{{ plat__env_name }}" - sync: no + sync: false data_access: "{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" ranger_audit: "{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" mappings: diff --git a/roles/platform/tasks/teardown.yml b/roles/platform/tasks/teardown.yml index 9dbe33e3..90e99223 100644 --- a/roles/platform/tasks/teardown.yml +++ b/roles/platform/tasks/teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/platform/tasks/teardown_aws_authz.yml b/roles/platform/tasks/teardown_aws_authz.yml index 371a7313..cb4a7659 100644 --- a/roles/platform/tasks/teardown_aws_authz.yml +++ b/roles/platform/tasks/teardown_aws_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +20,8 @@ community.aws.iam_role: region: "{{ plat__region }}" name: "{{ __aws_data_access_role_name_item }}" - purge_policies: yes - delete_instance_profile: yes + purge_policies: true + delete_instance_profile: true state: absent loop_control: loop_var: __aws_data_access_role_name_item @@ -34,8 +33,8 @@ community.aws.iam_role: region: "{{ plat__region }}" name: "{{ __aws_service_role_name_item }}" - purge_policies: yes - delete_instance_profile: yes + purge_policies: true + delete_instance_profile: true state: absent loop_control: loop_var: __aws_service_role_name_item @@ -44,7 +43,7 @@ - "{{ plat__aws_log_role_name }}" - name: Ensure AWS Instance Profiles are detached from CDP Roles - command: > + ansible.builtin.command: > aws iam remove-role-from-instance-profile --instance-profile-name {{ __aws_role_item }} --role-name {{ __aws_role_item }} @@ -85,7 +84,7 @@ - name: Remove CDP Cross Account Credential for AWS when: plat__teardown_deletes_credential cloudera.cloud.env_cred: - name: "{{ plat__xacccount_credential_name }}" # TODO: Make specific to AWS Teardown as credentials can be for multiple environments + name: "{{ plat__xacccount_credential_name }}" # TODO: Make specific to AWS Teardown as credentials can be for multiple environments state: absent - name: Tear down AWS Cross Acount diff --git a/roles/platform/tasks/teardown_aws_terraform_authz.yml b/roles/platform/tasks/teardown_aws_terraform_authz.yml index c504dc12..378d32f8 100644 --- a/roles/platform/tasks/teardown_aws_terraform_authz.yml +++ b/roles/platform/tasks/teardown_aws_terraform_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Remove CDP Cross Account Credential for AWS when: plat__teardown_deletes_credential cloudera.cloud.env_cred: - name: "{{ plat__xacccount_credential_name }}" # TODO: Make specific to AWS Teardown as credentials can be for multiple environments + name: "{{ plat__xacccount_credential_name }}" # TODO: Make specific to AWS Teardown as credentials can be for multiple environments state: absent - name: Ensure the Terraform workspace directory exists @@ -29,7 +28,7 @@ community.general.terraform: project_path: "{{ plat__terraform_workspace_dir }}/plat" state: "absent" - force_init: yes + force_init: true register: tf_result retries: 3 delay: 10 diff --git a/roles/platform/tasks/teardown_azure_authz.yml b/roles/platform/tasks/teardown_azure_authz.yml index 27a777a0..cc2262bc 100644 --- a/roles/platform/tasks/teardown_azure_authz.yml +++ b/roles/platform/tasks/teardown_azure_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +20,7 @@ loop_control: loop_var: __plat_azure_role_item label: __plat_azure_role_item.name - loop: - "{{ __plat_azure_role_assignment_list }}" + loop: "{{ __plat_azure_role_assignment_list }}" - name: Request deletion of Azure Managed Identities when: @@ -36,12 +34,11 @@ provider: ManagedIdentity resource_type: userAssignedIdentities resource_name: "{{ __azure_msi_item }}" - api_version: '2018-11-30' - idempotency: yes + api_version: "2018-11-30" + idempotency: true state: absent loop: "{{ plat__azure_msis }}" - - name: Wait for MSIs to be delisted - Non RAZ when: - plat__azure_metagroup_uri is defined @@ -50,7 +47,7 @@ resource_group: "{{ plat__azure_metagroup_name }}" provider: ManagedIdentity resource_type: userAssignedIdentities - api_version: '2018-11-30' + api_version: "2018-11-30" register: __azure_identity_list delay: 5 retries: 10 @@ -62,7 +59,6 @@ vars: discovered_msi_list: "{{ __azure_identity_list.response | map(attribute='name') | list }}" - - name: Wait for MSIs to be delisted - RAZ when: - plat__azure_metagroup_uri is defined @@ -72,7 +68,7 @@ resource_group: "{{ plat__azure_metagroup_name }}" provider: ManagedIdentity resource_type: userAssignedIdentities - api_version: '2018-11-30' + api_version: "2018-11-30" register: __azure_identity_list delay: 5 retries: 10 @@ -81,7 +77,6 @@ vars: discovered_msi_list: "{{ __azure_identity_list.response | map(attribute='name') | list }}" - - name: Remove CDP Cross Account Credential for Azure when: plat__teardown_deletes_credential cloudera.cloud.env_cred: @@ -90,10 +85,9 @@ - name: Tear down Azure AD App Registration when: plat__teardown_deletes_xaccount and ( plat__azure_xaccount_app_uuid is defined ) and ( plat__azure_xaccount_app_uuid | length > 0 ) - command: > - az ad app delete - --id {{ plat__azure_xaccount_app_uuid }} - + ansible.builtin.command: > + az ad app delete + --id {{ plat__azure_xaccount_app_uuid }} - name: Tear down Custom Role when: plat__teardown_deletes_roles azure.azcollection.azure_rm_roledefinition: diff --git a/roles/platform/tasks/teardown_gcp_authz.yml b/roles/platform/tasks/teardown_gcp_authz.yml index a737e9d4..79cafc41 100644 --- a/roles/platform/tasks/teardown_gcp_authz.yml +++ b/roles/platform/tasks/teardown_gcp_authz.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,11 +29,10 @@ loop: "{{ plat__gcp_xaccount_keys }}" loop_control: loop_var: __gcp_xaccount_key_item - command: > + ansible.builtin.command: > gcloud iam service-accounts keys delete {{ __gcp_xaccount_key_item.name.split('/')[-1] }} --iam-account "{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" - - name: Remove GCP Cross Account Service Account when: plat__teardown_deletes_xaccount google.cloud.gcp_iam_service_account: @@ -82,13 +80,12 @@ role: "{{ plat__gcp_roles.iam_service_account_user }}" - member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" role: "{{ plat__gcp_roles.iam_service_account_token_creator }}" - command: > + ansible.builtin.command: > gcloud projects remove-iam-policy-binding {{ plat__gcp_project }} --member={{ __gcp_binding_item.member |quote }} --role={{ __gcp_binding_item.role |quote }} --all - - name: Tear down GCP Storage Policies when: plat__teardown_deletes_policies register: __gcp_storage_policy_teardown @@ -104,11 +101,10 @@ bucket: "{{ plat__gcp_storage_location_data }}" - account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" bucket: "{{ plat__gcp_storage_location_data }}" - command: > + ansible.builtin.command: > gsutil iam ch -d {{ __gcp_pol_item.account |quote }} gs://{{ __gcp_pol_item.bucket |quote }} - - name: Tear down Operational GCP Service Accounts when: plat__teardown_deletes_roles loop_control: diff --git a/roles/platform/tasks/validate.yml b/roles/platform/tasks/validate.yml index 84354474..f122dbb5 100644 --- a/roles/platform/tasks/validate.yml +++ b/roles/platform/tasks/validate.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ - "{{ __auth_fact_item }} is string" - "{{ __auth_fact_item }} | trim | length > 0" fail_msg: "Authentication parameter, '{{ __auth_fact_item }}', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __auth_fact_item loop: @@ -34,7 +33,7 @@ fail_msg: >- Namespace exceeds 18 chars. Resulting hostnames may exceed 64 chars and cause errors in some systems like Hue. Please use a shorter namespace. - quiet: yes + quiet: true - name: Confirm mutually-inclusive CDP Cross Account details when: (plat__cdp_xaccount_external_id is string) != (plat__cdp_xaccount_account_id is string) @@ -46,7 +45,7 @@ ansible.builtin.assert: that: "{{ __xaccount_item }} | length > 0" fail_msg: "CDP Cross Account parameter, '{{ __xaccount_item }}'', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __xaccount_item loop: @@ -62,7 +61,7 @@ fail_msg: >- Error: '{{ lookup('vars', __group_fact_item) }}'. CDP Group name must be less than 32 characters, must begin with either a letter or underscore, and contain only alphanumeric characters, dashes, and underscores. - quiet: yes + quiet: true loop_control: loop_var: __group_fact_item loop: diff --git a/roles/platform/tasks/validate_aws.yml b/roles/platform/tasks/validate_aws.yml index 920c4ef0..7450c880 100644 --- a/roles/platform/tasks/validate_aws.yml +++ b/roles/platform/tasks/validate_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ - "{{ __auth_fact_item }} is string" - "{{ __auth_fact_item }} | trim | length > 0" fail_msg: "Authentication parameter, '{{ __auth_fact_item }}', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __auth_fact_item loop: diff --git a/roles/platform/tasks/validate_aws_terraform.yml b/roles/platform/tasks/validate_aws_terraform.yml index 60fb75fc..19a086c7 100644 --- a/roles/platform/tasks/validate_aws_terraform.yml +++ b/roles/platform/tasks/validate_aws_terraform.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +20,7 @@ - "{{ __auth_fact_item }} is string" - "{{ __auth_fact_item }} | trim | length > 0" fail_msg: "Authentication parameter, '{{ __auth_fact_item }}', is invalid." - quiet: yes + quiet: true loop_control: loop_var: __auth_fact_item loop: @@ -29,45 +28,44 @@ - name: Confirm that required Terraform variables are defined block: - - name: Check plat__terraform_template_dir - ansible.builtin.assert: - that: - - "plat__terraform_template_dir is defined" - - "plat__terraform_template_dir | length > 0" - fail_msg: "Required plat__terraform_template_dir variable for Terraform is not valid." - quiet: yes + - name: Check plat__terraform_template_dir + ansible.builtin.assert: + that: + - "plat__terraform_template_dir is defined" + - "plat__terraform_template_dir | length > 0" + fail_msg: "Required plat__terraform_template_dir variable for Terraform is not valid." + quiet: true - - name: Check plat__terraform_workspace_dir - ansible.builtin.assert: - that: - - "plat__terraform_workspace_dir is defined" - - "plat__terraform_workspace_dir | length > 0" - fail_msg: "Required plat__terraform_workspace_dir variable for Terraform is not valid." - quiet: yes + - name: Check plat__terraform_workspace_dir + ansible.builtin.assert: + that: + - "plat__terraform_workspace_dir is defined" + - "plat__terraform_workspace_dir | length > 0" + fail_msg: "Required plat__terraform_workspace_dir variable for Terraform is not valid." + quiet: true - - name: Check plat__terraform_artefact_dir - ansible.builtin.assert: - that: - - "plat__terraform_artefact_dir is defined" - - "plat__terraform_artefact_dir | length > 0" - fail_msg: "Required plat__terraform_artefact_dir variable for Terraform is not valid." - quiet: yes + - name: Check plat__terraform_artefact_dir + ansible.builtin.assert: + that: + - "plat__terraform_artefact_dir is defined" + - "plat__terraform_artefact_dir | length > 0" + fail_msg: "Required plat__terraform_artefact_dir variable for Terraform is not valid." + quiet: true - - name: Check plat__terraform_state_storage - ansible.builtin.assert: - that: - - "plat__terraform_state_storage is defined" - - "plat__terraform_state_storage in plat__terraform_allowed_state_storage" - fail_msg: "Required plat__terraform_state_storage variable for Terraform needs to be \ - one of {{ plat__terraform_allowed_state_storage | join(', ') }}" - quiet: yes + - name: Check plat__terraform_state_storage + ansible.builtin.assert: + that: + - "plat__terraform_state_storage is defined" + - "plat__terraform_state_storage in plat__terraform_allowed_state_storage" + fail_msg: "Required plat__terraform_state_storage variable for Terraform needs to be one of {{ plat__terraform_allowed_state_storage | join(', ') }}" + quiet: true - - name: Check remote state variables are defined for remote_s3 - ansible.builtin.assert: - that: - - "plat__terraform_remote_state_bucket | length > 0" - - "plat__terraform_remote_state_lock_table | length > 0" - fail_msg: "Required plat__terraform_remote_state_bucket and plat__terraform_remote_state_lock_table variables \ - need to be defined for '{{ plat__terraform_state_storage }}' Terraform remote state" - quiet: yes - when: plat__terraform_state_storage == "remote_s3" + - name: Check remote state variables are defined for remote_s3 + ansible.builtin.assert: + that: + - "plat__terraform_remote_state_bucket | length > 0" + - "plat__terraform_remote_state_lock_table | length > 0" + fail_msg: "Required plat__terraform_remote_state_bucket and plat__terraform_remote_state_lock_table variables need to be defined for '{{ plat__terraform_state_storage + }}' Terraform remote state" + quiet: true + when: plat__terraform_state_storage == "remote_s3" diff --git a/roles/platform/tasks/validate_azure.yml b/roles/platform/tasks/validate_azure.yml index 25165df8..bf627202 100644 --- a/roles/platform/tasks/validate_azure.yml +++ b/roles/platform/tasks/validate_azure.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set public key text from file if not set when: plat__public_key_text | trim | length < 1 and plat__public_key_file ansible.builtin.set_fact: - plat__public_key_text: "{{ lookup('file', plat__public_key_file ) }}" + plat__public_key_text: "{{ lookup('file', plat__public_key_file) }}" - name: Check public key text ansible.builtin.assert: @@ -26,4 +25,4 @@ - "plat__public_key_text is string" - "plat__public_key_text | trim | length > 0" fail_msg: "Azure authentication parameter, 'plat__public_key_text', is invalid." - quiet: yes + quiet: true diff --git a/roles/platform/tasks/validate_gcp.yml b/roles/platform/tasks/validate_gcp.yml index 823125ae..b1d81311 100644 --- a/roles/platform/tasks/validate_gcp.yml +++ b/roles/platform/tasks/validate_gcp.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +16,7 @@ - name: Set public key text from file if not set when: plat__public_key_text | trim | length < 1 and plat__public_key_file ansible.builtin.set_fact: - plat__public_key_text: "{{ lookup('file', plat__public_key_file ) }}" + plat__public_key_text: "{{ lookup('file', plat__public_key_file) }}" - name: Check public key text ansible.builtin.assert: @@ -26,4 +25,4 @@ - "plat__public_key_text is string" - "plat__public_key_text | trim | length > 0" fail_msg: "GCP authentication parameter, 'plat__public_key_text', is invalid." - quiet: yes + quiet: true diff --git a/roles/platform/vars/main.yml b/roles/platform/vars/main.yml index f7f57f28..28e4d390 100644 --- a/roles/platform/vars/main.yml +++ b/roles/platform/vars/main.yml @@ -17,10 +17,10 @@ # Vars for platform plat__aws_policy_urls_default_root: "{{ common__aws_policy_urls_default_root }}" plat__aws_policy_urls_default: - log: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-log-policy.json" - ranger_audit_s3: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-ranger-audit-s3-policy.json" - datalake_admin_s3: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-datalake-admin-s3-policy.json" - bucket_access: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-bucket-access-policy.json" + log: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-log-policy.json" + ranger_audit_s3: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-ranger-audit-s3-policy.json" + datalake_admin_s3: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-datalake-admin-s3-policy.json" + bucket_access: "{{ plat__aws_policy_urls_default_root }}/aws-cdp-bucket-access-policy.json" plat__gcp_roles: storage_admin: roles/storage.admin @@ -76,18 +76,17 @@ plat__gcp_xaccount_policy_bindings_default: - "roles/owner" plat__gcp_log_policy_bindings_default: - - 'storage.buckets.get' - - 'storage.buckets.create' + - "storage.buckets.get" + - "storage.buckets.create" plat__azure_roles: - vmcnt: 'Virtual Machine Contributor' - miop: 'Managed Identity Operator' - storown: 'Storage Blob Data Owner' - storcnt: 'Storage Blob Data Contributor' - contrib: 'Contributor' - owner: 'Owner' - stordel: 'Storage Blob Delegator' - + vmcnt: "Virtual Machine Contributor" + miop: "Managed Identity Operator" + storown: "Storage Blob Data Owner" + storcnt: "Storage Blob Data Contributor" + contrib: "Contributor" + owner: "Owner" + stordel: "Storage Blob Delegator" plat__gcp_required_services: - compute.googleapis.com diff --git a/roles/prometheus/handlers/main.yml b/roles/prometheus/handlers/main.yml index 4b246266..3425e35a 100644 --- a/roles/prometheus/handlers/main.yml +++ b/roles/prometheus/handlers/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: restart prometheus +- name: Restart prometheus ansible.builtin.service: name: prometheus state: restarted diff --git a/roles/prometheus/meta/argument_specs.yml b/roles/prometheus/meta/argument_specs.yml new file mode 100644 index 00000000..97c05bc4 --- /dev/null +++ b/roles/prometheus/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy Prometheus. + description: Deploy Prometheus. + author: Ronald Suplina + version_added: 2.4.0 diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index e0e0a762..06475717 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +33,7 @@ src: "{{ __prometheus_tmp.path }}/{{ prometheus_tarball_file }}" dest: "{{ prometheus_directory }}" extra_opts: --strip-components=1 - remote_src: yes + remote_src: true - name: Remove the temporary directory when: __prometheus_tmp is defined @@ -45,7 +44,7 @@ - name: Create Prometheus user ansible.builtin.user: name: "{{ prometheus_user }}" - system: True + system: true - name: Create directory for Prometheus TSDB ansible.builtin.file: @@ -53,14 +52,14 @@ owner: "{{ prometheus_user }}" group: "{{ prometheus_group }}" state: directory - recurse: yes + recurse: true - name: Set ownership of all files inside /etc/prometheus ansible.builtin.file: path: "{{ prometheus_directory }}" owner: "{{ prometheus_user }}" group: "{{ prometheus_group }}" - recurse: yes + recurse: true - name: Create Prometheus service template ansible.builtin.template: @@ -71,16 +70,15 @@ - name: Start and enable prometheus service when: __prometheus_service.changed block: + - name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true - - name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: yes - - - name: Enable and start prometheus service - ansible.builtin.systemd: - name: prometheus - state: started - enabled: yes + - name: Enable and start prometheus service + ansible.builtin.systemd: + name: prometheus + state: started + enabled: true - name: Update Prometheus configuration ansible.builtin.template: diff --git a/roles/provision/defaults/main.yml b/roles/provision/defaults/main.yml index 914c913e..0cf38005 100644 --- a/roles/provision/defaults/main.yml +++ b/roles/provision/defaults/main.yml @@ -9,7 +9,7 @@ provision_inventory_file: "{{ undef(hint='Static inventory file') }}" # inventor provision_state_storage: local # remote_s3 # provision_remote_storage_s3_region: # provision_remote_storage_s3_bucket: -provision_create_remote_storage: False +provision_create_remote_storage: false provision_name_prefix: "{{ undef(hint='Deployment name prefix') }}" provision_domain_suffix: "{{ undef(hint='DNS domain suffix') }}" @@ -17,7 +17,6 @@ provision_ssh_keypair_label: "{{ undef(hint='SSH keypair label') }}" provision_ssh_keypair_public_key: "{{ undef(hint='SSH keypair public key text') }}" provision_owner_email: "{{ undef(hint='Resource owner email') }}" provision_tags: {} - provision_aws_ec2_region: "{{ undef(hint='AWS EC2 region') }}" #provision_aws_ec2_default_ami_filters: "{{ undef(hint='AWS EC2 filters for default AMI') }}" #provision_aws_ec2_default_ami_owners: "{{ undef(hint='AWS EC2 AMI owner filter') }}" diff --git a/roles/provision/meta/argument_specs.yml b/roles/provision/meta/argument_specs.yml new file mode 100644 index 00000000..ad70ec28 --- /dev/null +++ b/roles/provision/meta/argument_specs.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Provision Cloudera-specific inventory. + description: Provision Cloudera-specific inventory. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 2.0.1 diff --git a/roles/provision/meta/main.yml b/roles/provision/meta/main.yml index 1d1af1ca..dab1ef88 100755 --- a/roles/provision/meta/main.yml +++ b/roles/provision/meta/main.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,20 +23,20 @@ galaxy_info: min_ansible_version: 2.10 platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all + - name: Debian + versions: all + - name: Fedora + versions: all + - name: GenericLinux + versions: all + - name: MacOSX + versions: all + - name: Ubuntu + versions: all galaxy_tags: - - storage - - mount - - cdp - - aws - - openstack + - storage + - mount + - cdp + - aws + - openstack diff --git a/roles/provision/tasks/absent.yml b/roles/provision/tasks/absent.yml index fbf88524..be82b414 100644 --- a/roles/provision/tasks/absent.yml +++ b/roles/provision/tasks/absent.yml @@ -1,5 +1,4 @@ --- - - name: Examine the local Terraform project directory ansible.builtin.stat: path: "{{ provision_directory }}" @@ -19,4 +18,4 @@ region: "{{ provision_remote_storage_s3_region }}" name: "{{ provision_remote_storage_s3_bucket }}" state: absent - force: yes + force: true diff --git a/roles/provision/tasks/main.yml b/roles/provision/tasks/main.yml index 02e24da2..f387f440 100644 --- a/roles/provision/tasks/main.yml +++ b/roles/provision/tasks/main.yml @@ -1,4 +1,3 @@ --- - - name: Execute provisioning state ansible.builtin.include_tasks: "{{ provision_state }}.yml" diff --git a/roles/provision/tasks/present.yml b/roles/provision/tasks/present.yml index 63386455..6341f65c 100644 --- a/roles/provision/tasks/present.yml +++ b/roles/provision/tasks/present.yml @@ -1,5 +1,4 @@ --- - - name: Set up the local Terraform project directory ansible.builtin.file: state: directory @@ -22,7 +21,7 @@ - name: Generate Terraform backend state ansible.builtin.template: - src: 'backend_state.tf.j2' + src: "backend_state.tf.j2" dest: "{{ [provision_directory, 'backend_state.tf'] | path_join }}" - name: Generate Terraform variables file @@ -36,7 +35,7 @@ project_path: "{{ provision_directory }}/" parallelism: "{{ provision_terraform_parallelism | default(omit) }}" state: present - force_init: yes + force_init: true register: tf_result - name: Establish jump host IP address @@ -57,7 +56,8 @@ ansible_timeout: "{{ (jump_host_ip is defined and (jump_host_ip | length > 0)) | ternary(60, omit) }}" ansible_ssh_common_args: "{{ (jump_host_ip is defined and (jump_host_ip | length > 0)) | ternary(jump, omit) }}" vars: - jump: -o ProxyCommand="ssh -o User={{ jump_host_user }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q {{ jump_host_ip | default() }}" + jump: -o ProxyCommand="ssh -o User={{ jump_host_user }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q {{ jump_host_ip | default() + }}" loop: "{{ tf_result.outputs.nodes.value }}" loop_control: loop_var: node diff --git a/roles/rdbms/client/mysql_connector/defaults/main.yml b/roles/rdbms/client/mysql_connector/defaults/main.yml index 738d4de9..b06f4fe8 100644 --- a/roles/rdbms/client/mysql_connector/defaults/main.yml +++ b/roles/rdbms/client/mysql_connector/defaults/main.yml @@ -14,7 +14,7 @@ --- -local_temp_dir: '/tmp' +local_temp_dir: "/tmp" mysql_connector_url: https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.49.zip mysql_connector_checksum: md5:5ecd588e13f14de07faa5c67f5caf3f1 mysql_connector_download_dir: "{{ local_temp_dir }}" diff --git a/roles/rdbms/client/mysql_connector/tasks/main.yml b/roles/rdbms/client/mysql_connector/tasks/main.yml index 3c5daf15..5547c428 100644 --- a/roles/rdbms/client/mysql_connector/tasks/main.yml +++ b/roles/rdbms/client/mysql_connector/tasks/main.yml @@ -13,22 +13,21 @@ # limitations under the License. --- - - name: Download MySQL Connector/J - get_url: + ansible.builtin.get_url: url: "{{ mysql_connector_url }}" dest: "{{ mysql_connector_download_dir }}/mysql-connector-java.zip" checksum: "{{ mysql_connector_checksum }}" - mode: 0644 - become: no + mode: "0644" + become: false run_once: true delegate_to: localhost - name: Create /usr/share/java directory - file: + ansible.builtin.file: path: /usr/share/java state: directory - mode: 0755 + mode: "0755" - name: Install unzip package ansible.builtin.package: @@ -37,23 +36,23 @@ state: present - name: Extract MySQL Connector/J zip file - unarchive: + ansible.builtin.unarchive: src: "{{ mysql_connector_download_dir }}/mysql-connector-java.zip" dest: "{{ mysql_connector_extract_dir }}" exclude: - src - name: Copy MySQL Connector/J jar file to correct location - copy: + ansible.builtin.copy: src: "{{ mysql_connector_local_path }}" dest: /usr/share/java/mysql-connector-java.jar - remote_src: yes - mode: 0644 + remote_src: true + mode: "0644" ## Fix for RHEL8,9 - name: Install Mysql packages for python - PyMySQL when: - ansible_distribution == "RedHat" - ansible_distribution_major_version >= "8" - shell: /usr/local/bin/pip install PyMySQL --force-reinstall --ignore-installed + ansible.builtin.command: /usr/local/bin/pip install PyMySQL --force-reinstall --ignore-installed ignore_errors: true diff --git a/roles/rdbms/client/oracle_connector/tasks/main.yml b/roles/rdbms/client/oracle_connector/tasks/main.yml index e455a107..df831260 100644 --- a/roles/rdbms/client/oracle_connector/tasks/main.yml +++ b/roles/rdbms/client/oracle_connector/tasks/main.yml @@ -13,44 +13,42 @@ # limitations under the License. --- - - name: Setup the Oracle JDBC Driver - block: + when: + - not (skip_oracle_jdbc_driver_distribution | default(False)) + - oracle_connector_maven_url is defined + - oracle_connector_maven_url != '' + block: - name: Download Oracle Connector - maven_artifact: + community.general.maven_artifact: group_id: "{{ oracle_connector_group_id }}" artifact_id: "{{ oracle_connector_artifact_id }}" version: "{{ oracle_connector_version }}" dest: "{{ local_temp_dir }}/{{ oracle_connector_artifact_id }}-connector-java-{{ oracle_connector_version }}.jar" repository_url: "{{ oracle_connector_maven_url }}" - become: no + become: false run_once: true connection: local delegate_to: localhost - name: Create /usr/share/java directory - file: + ansible.builtin.file: path: /usr/share/java state: directory - mode: 0755 + mode: "0755" - name: Copy Oracle Connector jar file to correct location - copy: + ansible.builtin.copy: src: "{{ local_temp_dir }}/{{ oracle_connector_artifact_id }}-connector-java-{{ oracle_connector_version }}.jar" dest: /usr/share/java/oracle-connector-java.jar - mode: 0644 - - when: - - not (skip_oracle_jdbc_driver_distribution | default(False)) - - oracle_connector_maven_url is defined - - oracle_connector_maven_url != '' + mode: "0644" - name: Ensure directory for the instantclient - file: + ansible.builtin.file: path: /usr/share/oracle/instantclient/lib state: directory - mode: 0755 + mode: "0755" when: - oracle_instantclient_basic_zip is defined - oracle_instantclient_sdk_zip is defined @@ -65,17 +63,17 @@ - oracle_instantclient_sdk_zip is defined - name: Unarchive basic instantclient - unarchive: + ansible.builtin.unarchive: src: "{{ oracle_instantclient_basic_zip }}" dest: /usr/share/oracle/instantclient/lib - extra_opts: [ "-j" ] + extra_opts: ["-j"] when: oracle_instantclient_basic_zip is defined - name: Unarchive sdk instantclient - unarchive: + ansible.builtin.unarchive: src: "{{ oracle_instantclient_sdk_zip }}" dest: /usr/share/oracle/instantclient/lib - extra_opts: [ "-j" ] + extra_opts: ["-j"] when: oracle_instantclient_sdk_zip is defined - name: Install the libaio package diff --git a/roles/rdbms/client/postgresql_connector/defaults/main.yml b/roles/rdbms/client/postgresql_connector/defaults/main.yml index 39e06fa8..94770bab 100644 --- a/roles/rdbms/client/postgresql_connector/defaults/main.yml +++ b/roles/rdbms/client/postgresql_connector/defaults/main.yml @@ -14,7 +14,7 @@ --- -local_temp_dir: '/tmp' +local_temp_dir: "/tmp" postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.7.2.jar postgresql_connector_checksum: md5:bb897217989c97a463d8f571069d158a install_py3_psycopg2: false diff --git a/roles/rdbms/client/postgresql_connector/tasks/main.yml b/roles/rdbms/client/postgresql_connector/tasks/main.yml index fd318d55..27350799 100644 --- a/roles/rdbms/client/postgresql_connector/tasks/main.yml +++ b/roles/rdbms/client/postgresql_connector/tasks/main.yml @@ -13,38 +13,37 @@ # limitations under the License. --- - - name: Download PostgreSQL Connector - get_url: + ansible.builtin.get_url: url: "{{ postgresql_connector_url }}" dest: "{{ local_temp_dir }}/postgresql-connector-java.jar" checksum: "{{ postgresql_connector_checksum }}" - mode: 0644 - become: no + mode: "0644" + become: false run_once: true delegate_to: localhost - name: Create /usr/share/java directory - file: + ansible.builtin.file: path: /usr/share/java state: directory - mode: 0755 + mode: "0755" - name: Copy PostgreSQL Connector jar file to correct location - copy: + ansible.builtin.copy: src: "{{ local_temp_dir }}/postgresql-connector-java.jar" dest: /usr/share/java/postgresql-connector-java.jar - mode: 0644 + mode: "0644" # SSB will need the python3-psycopg2 connector - name: Create python3-psycopg2 directory - file: - path: "/usr/share/python3" - state: directory - mode: '777' + ansible.builtin.file: + path: "/usr/share/python3" + state: directory + mode: "777" when: install_py3_psycopg2 == true - name: Install python3-psycopg2 - shell: "pip3 install psycopg2-binary==2.8.5 -t /usr/share/python3" + ansible.builtin.command: "pip3 install psycopg2-binary==2.8.5 -t /usr/share/python3" when: install_py3_psycopg2 == true diff --git a/roles/rdbms/server/defaults/main.yml b/roles/rdbms/server/defaults/main.yml index 2fc38626..129d9742 100644 --- a/roles/rdbms/server/defaults/main.yml +++ b/roles/rdbms/server/defaults/main.yml @@ -21,7 +21,7 @@ database_version: 14 skip_rdbms_repo_setup: false # MYSQL -mysql_require_secure_transport: "OFF" # If TLS-enabled, honor or not +mysql_require_secure_transport: "OFF" # If TLS-enabled, honor or not base_dir_security_pki: "/opt/cloudera/security/pki" tls_chain_path: "{{ base_dir_security_pki }}/chain.pem" diff --git a/roles/rdbms/server/handlers/main.yml b/roles/rdbms/server/handlers/main.yml index 034c8b04..1eb10bfe 100644 --- a/roles/rdbms/server/handlers/main.yml +++ b/roles/rdbms/server/handlers/main.yml @@ -13,6 +13,5 @@ # limitations under the License. --- - -- name: yum clean metadata +- name: Yum clean metadata ansible.builtin.command: yum clean metadata diff --git a/roles/rdbms/server/meta/argument_specs.yml b/roles/rdbms/server/meta/argument_specs.yml index 29129c28..bd10906c 100644 --- a/roles/rdbms/server/meta/argument_specs.yml +++ b/roles/rdbms/server/meta/argument_specs.yml @@ -58,7 +58,8 @@ argument_specs: required: false default: "/opt/cloudera/security/pki" tls_chain_path: - description: File on the target host consisting of an ordered list of certificates, including TLS certificates and Certificate Authority (CA) certificates. + description: File on the target host consisting of an ordered list of certificates, including TLS certificates and Certificate Authority (CA) + certificates. type: path required: false default: "O(base_dir_security_pki)/chain.pem" diff --git a/roles/rdbms/server/tasks/main.yml b/roles/rdbms/server/tasks/main.yml index 387c8f59..39935024 100644 --- a/roles/rdbms/server/tasks/main.yml +++ b/roles/rdbms/server/tasks/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- - - name: DEPRECATION WARNING ansible.builtin.debug: msg: @@ -28,8 +27,8 @@ ansible.builtin.include_vars: file: "{{ item }}" with_first_found: - - "{{ database_type }}/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" - - "{{ database_type }}/{{ ansible_os_family }}.yml" + - "{{ database_type }}/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "{{ database_type }}/{{ ansible_os_family }}.yml" - name: Install database ansible.builtin.include_tasks: diff --git a/roles/rdbms/server/tasks/mariadb/Debian.yml b/roles/rdbms/server/tasks/mariadb/Debian.yml index f14ad6a3..429c8534 100644 --- a/roles/rdbms/server/tasks/mariadb/Debian.yml +++ b/roles/rdbms/server/tasks/mariadb/Debian.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/tasks/mariadb/RedHat.yml b/roles/rdbms/server/tasks/mariadb/RedHat.yml index 07a8f8bc..0214f506 100644 --- a/roles/rdbms/server/tasks/mariadb/RedHat.yml +++ b/roles/rdbms/server/tasks/mariadb/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/tasks/mysql/RedHat.yml b/roles/rdbms/server/tasks/mysql/RedHat.yml index 5bcfb60c..127c6c6a 100644 --- a/roles/rdbms/server/tasks/mysql/RedHat.yml +++ b/roles/rdbms/server/tasks/mysql/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/tasks/postgresql/Debian.yml b/roles/rdbms/server/tasks/postgresql/Debian.yml index 5bdca9d7..dc198295 100644 --- a/roles/rdbms/server/tasks/postgresql/Debian.yml +++ b/roles/rdbms/server/tasks/postgresql/Debian.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/tasks/postgresql/RedHat.yml b/roles/rdbms/server/tasks/postgresql/RedHat.yml index 404d96a6..991afabb 100644 --- a/roles/rdbms/server/tasks/postgresql/RedHat.yml +++ b/roles/rdbms/server/tasks/postgresql/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/tasks/postgresql/template_fix.yml b/roles/rdbms/server/tasks/postgresql/template_fix.yml index 6be8d86c..58685485 100644 --- a/roles/rdbms/server/tasks/postgresql/template_fix.yml +++ b/roles/rdbms/server/tasks/postgresql/template_fix.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,15 +31,15 @@ dest: "{{ __sql.path }}/utf8-template.sql" owner: postgres group: postgres - mode: 0660 + mode: "0660" - name: Run SQL to change template to UTF-8 ansible.builtin.command: "psql -f {{ __sql.path }}/utf8-template.sql" - become: yes + become: true become_user: postgres - name: Remove temporary SQL directory ansible.builtin.file: path: "{{ __sql.path }}" state: absent - become: yes + become: true diff --git a/roles/rdbms/server/vars/mysql/RedHat-7.yml b/roles/rdbms/server/vars/mysql/RedHat-7.yml index 7b05a01d..9fbb7db5 100644 --- a/roles/rdbms/server/vars/mysql/RedHat-7.yml +++ b/roles/rdbms/server/vars/mysql/RedHat-7.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/vars/mysql/RedHat-8.yml b/roles/rdbms/server/vars/mysql/RedHat-8.yml index 7cc60cd6..ad90d09f 100644 --- a/roles/rdbms/server/vars/mysql/RedHat-8.yml +++ b/roles/rdbms/server/vars/mysql/RedHat-8.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/vars/mysql/RedHat-9.yml b/roles/rdbms/server/vars/mysql/RedHat-9.yml index a6d51b03..5374e4d4 100644 --- a/roles/rdbms/server/vars/mysql/RedHat-9.yml +++ b/roles/rdbms/server/vars/mysql/RedHat-9.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms/server/vars/mysql/common.yml b/roles/rdbms/server/vars/mysql/common.yml index e03c705d..e61df6ea 100644 --- a/roles/rdbms/server/vars/mysql/common.yml +++ b/roles/rdbms/server/vars/mysql/common.yml @@ -16,8 +16,8 @@ mysql_repo_key: https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 -mysql_root_password: 'Super$ecret1' -mysql_user_password: 'Super$ecret1' +mysql_root_password: "Super$ecret1" +mysql_user_password: "Super$ecret1" mysql_daemon: mysqld mysql_log_error: /var/log/mysqld.log mysql_syslog_tag: mysqld diff --git a/roles/rdbms/server/vars/postgresql/common.yml b/roles/rdbms/server/vars/postgresql/common.yml index 6943a705..0f146f3e 100644 --- a/roles/rdbms/server/vars/postgresql/common.yml +++ b/roles/rdbms/server/vars/postgresql/common.yml @@ -16,9 +16,9 @@ postgresql_global_config_options: - option: log_directory - value: 'log' + value: "log" - option: listen_addresses - value: '*' + value: "*" - option: max_connections value: 300 - option: ssl @@ -31,8 +31,8 @@ postgresql_global_config_options: value: "{{ database_tls | bool | ternary(tls_chain_path, None) }}" postgresql_hba_entries: - - {type: local, database: all, user: postgres, auth_method: peer} - - {type: local, database: all, user: all, auth_method: peer} - - {type: host, database: all, user: all, address: '127.0.0.1/32', auth_method: md5} - - {type: host, database: all, user: all, address: '::1/128', auth_method: md5} - - {type: host, database: all, user: all, address: '0.0.0.0/0', auth_method: md5} + - { type: local, database: all, user: postgres, auth_method: peer } + - { type: local, database: all, user: all, auth_method: peer } + - { type: host, database: all, user: all, address: "127.0.0.1/32", auth_method: md5 } + - { type: host, database: all, user: all, address: "::1/128", auth_method: md5 } + - { type: host, database: all, user: all, address: "0.0.0.0/0", auth_method: md5 } diff --git a/roles/rdbms_server/defaults/main.yml b/roles/rdbms_server/defaults/main.yml index 2fc38626..129d9742 100644 --- a/roles/rdbms_server/defaults/main.yml +++ b/roles/rdbms_server/defaults/main.yml @@ -21,7 +21,7 @@ database_version: 14 skip_rdbms_repo_setup: false # MYSQL -mysql_require_secure_transport: "OFF" # If TLS-enabled, honor or not +mysql_require_secure_transport: "OFF" # If TLS-enabled, honor or not base_dir_security_pki: "/opt/cloudera/security/pki" tls_chain_path: "{{ base_dir_security_pki }}/chain.pem" diff --git a/roles/rdbms_server/handlers/main.yml b/roles/rdbms_server/handlers/main.yml index 2f942fed..cf3371a4 100644 --- a/roles/rdbms_server/handlers/main.yml +++ b/roles/rdbms_server/handlers/main.yml @@ -13,9 +13,8 @@ # limitations under the License. --- - -- name: yum clean metadata +- name: Yum clean metadata ansible.builtin.command: yum clean metadata -- name: dnf clean metadata +- name: Dnf clean metadata ansible.builtin.command: dnf clean metadata diff --git a/roles/rdbms_server/meta/argument_specs.yml b/roles/rdbms_server/meta/argument_specs.yml index a1e34737..dd4227a5 100644 --- a/roles/rdbms_server/meta/argument_specs.yml +++ b/roles/rdbms_server/meta/argument_specs.yml @@ -58,7 +58,8 @@ argument_specs: required: false default: "/opt/cloudera/security/pki" tls_chain_path: - description: File on the target host consisting of an ordered list of certificates, including TLS certificates and Certificate Authority (CA) certificates. + description: File on the target host consisting of an ordered list of certificates, including TLS certificates and Certificate Authority (CA) + certificates. type: path required: false default: "C(base_dir_security_pki)/chain.pem" diff --git a/roles/rdbms_server/tasks/main.yml b/roles/rdbms_server/tasks/main.yml index 77fdce52..691ed3a0 100644 --- a/roles/rdbms_server/tasks/main.yml +++ b/roles/rdbms_server/tasks/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- - - name: Include database type variables ansible.builtin.include_vars: file: "{{ database_type }}/common.yml" @@ -22,8 +21,8 @@ ansible.builtin.include_vars: file: "{{ item }}" with_first_found: - - "{{ database_type }}/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" - - "{{ database_type }}/{{ ansible_os_family }}.yml" + - "{{ database_type }}/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "{{ database_type }}/{{ ansible_os_family }}.yml" - name: Install database ansible.builtin.include_tasks: diff --git a/roles/rdbms_server/tasks/mariadb/Debian.yml b/roles/rdbms_server/tasks/mariadb/Debian.yml index f14ad6a3..429c8534 100644 --- a/roles/rdbms_server/tasks/mariadb/Debian.yml +++ b/roles/rdbms_server/tasks/mariadb/Debian.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/tasks/mariadb/RedHat.yml b/roles/rdbms_server/tasks/mariadb/RedHat.yml index 07a8f8bc..0214f506 100644 --- a/roles/rdbms_server/tasks/mariadb/RedHat.yml +++ b/roles/rdbms_server/tasks/mariadb/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/tasks/mysql/RedHat.yml b/roles/rdbms_server/tasks/mysql/RedHat.yml index 5bcfb60c..127c6c6a 100644 --- a/roles/rdbms_server/tasks/mysql/RedHat.yml +++ b/roles/rdbms_server/tasks/mysql/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/tasks/postgresql/Debian.yml b/roles/rdbms_server/tasks/postgresql/Debian.yml index ed20cda0..f687cc53 100644 --- a/roles/rdbms_server/tasks/postgresql/Debian.yml +++ b/roles/rdbms_server/tasks/postgresql/Debian.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/tasks/postgresql/RedHat.yml b/roles/rdbms_server/tasks/postgresql/RedHat.yml index d457c1e1..a68e5761 100644 --- a/roles/rdbms_server/tasks/postgresql/RedHat.yml +++ b/roles/rdbms_server/tasks/postgresql/RedHat.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/tasks/postgresql/template_fix.yml b/roles/rdbms_server/tasks/postgresql/template_fix.yml index 82a0468e..b4bf8398 100644 --- a/roles/rdbms_server/tasks/postgresql/template_fix.yml +++ b/roles/rdbms_server/tasks/postgresql/template_fix.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,15 +31,15 @@ dest: "{{ __sql.path }}/utf8-template.sql" owner: postgres group: postgres - mode: 0660 + mode: "0660" - name: Run SQL to change template to UTF-8 ansible.builtin.command: "psql -f {{ __sql.path }}/utf8-template.sql" - become: yes + become: true become_user: postgres - name: Remove temporary SQL directory ansible.builtin.file: path: "{{ __sql.path }}" state: absent - become: yes + become: true diff --git a/roles/rdbms_server/vars/mysql/RedHat-7.yml b/roles/rdbms_server/vars/mysql/RedHat-7.yml index 7b05a01d..9fbb7db5 100644 --- a/roles/rdbms_server/vars/mysql/RedHat-7.yml +++ b/roles/rdbms_server/vars/mysql/RedHat-7.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/vars/mysql/RedHat-8.yml b/roles/rdbms_server/vars/mysql/RedHat-8.yml index 7cc60cd6..ad90d09f 100644 --- a/roles/rdbms_server/vars/mysql/RedHat-8.yml +++ b/roles/rdbms_server/vars/mysql/RedHat-8.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/vars/mysql/RedHat-9.yml b/roles/rdbms_server/vars/mysql/RedHat-9.yml index a6d51b03..5374e4d4 100644 --- a/roles/rdbms_server/vars/mysql/RedHat-9.yml +++ b/roles/rdbms_server/vars/mysql/RedHat-9.yml @@ -1,3 +1,4 @@ +--- # Copyright 2024 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/rdbms_server/vars/mysql/common.yml b/roles/rdbms_server/vars/mysql/common.yml index e03c705d..e61df6ea 100644 --- a/roles/rdbms_server/vars/mysql/common.yml +++ b/roles/rdbms_server/vars/mysql/common.yml @@ -16,8 +16,8 @@ mysql_repo_key: https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 -mysql_root_password: 'Super$ecret1' -mysql_user_password: 'Super$ecret1' +mysql_root_password: "Super$ecret1" +mysql_user_password: "Super$ecret1" mysql_daemon: mysqld mysql_log_error: /var/log/mysqld.log mysql_syslog_tag: mysqld diff --git a/roles/rdbms_server/vars/postgresql/common.yml b/roles/rdbms_server/vars/postgresql/common.yml index 6943a705..0f146f3e 100644 --- a/roles/rdbms_server/vars/postgresql/common.yml +++ b/roles/rdbms_server/vars/postgresql/common.yml @@ -16,9 +16,9 @@ postgresql_global_config_options: - option: log_directory - value: 'log' + value: "log" - option: listen_addresses - value: '*' + value: "*" - option: max_connections value: 300 - option: ssl @@ -31,8 +31,8 @@ postgresql_global_config_options: value: "{{ database_tls | bool | ternary(tls_chain_path, None) }}" postgresql_hba_entries: - - {type: local, database: all, user: postgres, auth_method: peer} - - {type: local, database: all, user: all, auth_method: peer} - - {type: host, database: all, user: all, address: '127.0.0.1/32', auth_method: md5} - - {type: host, database: all, user: all, address: '::1/128', auth_method: md5} - - {type: host, database: all, user: all, address: '0.0.0.0/0', auth_method: md5} + - { type: local, database: all, user: postgres, auth_method: peer } + - { type: local, database: all, user: all, auth_method: peer } + - { type: host, database: all, user: all, address: "127.0.0.1/32", auth_method: md5 } + - { type: host, database: all, user: all, address: "::1/128", auth_method: md5 } + - { type: host, database: all, user: all, address: "0.0.0.0/0", auth_method: md5 } diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 9e74977b..5a013d4a 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -18,105 +18,105 @@ # Role prefix is 'run__' -run__infra_type: "{{ common__infra_type }}" -run__region: "{{ common__region }}" -run__namespace: "{{ common__namespace }}" -run__namespace_cdp: "{{ common__namespace_cdp }}" -run__env_name: "{{ common__env_name }}" -run__datalake_name: "{{ common__datalake_name }}" -run__vpc_name: "{{ common__vpc_name }}" -run__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" -run__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" -run__vpc_public_subnets_name: "{{ [run__namespace, run__vpc_public_subnets_suffix] | join('-') }}*" -run__vpc_private_subnets_name: "{{ [run__namespace, run__vpc_private_subnets_suffix] | join('-') }}*" - -run__public_endpoint_access: "{{ common__public_endpoint_access }}" - -run__gcp_project: "{{ common__gcp_project }}" +run__infra_type: "{{ common__infra_type }}" +run__region: "{{ common__region }}" +run__namespace: "{{ common__namespace }}" +run__namespace_cdp: "{{ common__namespace_cdp }}" +run__env_name: "{{ common__env_name }}" +run__datalake_name: "{{ common__datalake_name }}" +run__vpc_name: "{{ common__vpc_name }}" +run__vpc_public_subnets_suffix: "{{ common__vpc_public_subnets_suffix }}" +run__vpc_private_subnets_suffix: "{{ common__vpc_private_subnets_suffix }}" +run__vpc_public_subnets_name: "{{ [run__namespace, run__vpc_public_subnets_suffix] | join('-') }}*" +run__vpc_private_subnets_name: "{{ [run__namespace, run__vpc_private_subnets_suffix] | join('-') }}*" + +run__public_endpoint_access: "{{ common__public_endpoint_access }}" + +run__gcp_project: "{{ common__gcp_project }}" # Teardown -run__force_teardown: "{{ common__force_teardown }}" - -run__datahub_image_catalog_url: "{{ datahub.image_catalog.url | default('https://cloudbreak-imagecatalog.s3.amazonaws.com/v3-prod-cb-image-catalog.json') }}" -run__datahub_image_catalog_name: "{{ datahub.image_catalog.name | default('cdp-default') }}" -run__datahub_instance_group_base: "{{ datahub.instance_group_base | default(lookup('template', 'datahub_instance_group_base.j2') | from_yaml) }}" -run__datahub_suffix: "{{ datahub.suffix | default('dhub') }}" -run__datahub_tags: "{{ datahub.tags | default(common__tags) }}" -run__datahub_definitions: "{{ datahub.definitions | default([]) }}" -run__datahub_force_teardown: "{{ datahub.force_delete | default(run__force_teardown) }}" - -run__datahub_compute_aws: "{{ datahub.compute.aws | default({}) }}" -run__datahub_compute_azure: "{{ datahub.compute.azure | default({}) }}" -run__datahub_compute_gcp: "{{ datahub.compute.gcp | default({}) }}" - -run__datahub_storage_aws: "{{ datahub.storage.aws | default({}) }}" -run__datahub_storage_azure: "{{ datahub.storage.azure | default({}) }}" -run__datahub_storage_gcp: "{{ datahub.storage.gcp | default({}) }}" - -run__opdb_definitions: "{{ opdb.definitions | default([{}]) }}" -run__opdb_suffix: "{{ opdb.suffix | default('od') }}" - -run__ml_definitions: "{{ ml.definitions | default([{}]) }}" -run__ml_suffix: "{{ ml.suffix | default('wksp') }}" -run__ml_k8s_request_base: "{{ ml.k8s_request_base | default(lookup('template', 'ml_k8s_request_base.j2') | from_yaml) }}" -run__ml_tags: "{{ ml.tags | default(common__tags) }}" -run__ml_force_delete: "{{ ml.force_delete | default (run__force_teardown) }}" -run__ml_remove_storage: "{{ ml.remove_storage | default (run__force_teardown) }}" -run__ml_public_loadbalancer: "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}" - -run__de_definitions: "{{ de.definitions | default([{}]) }}" -run__de_suffix: "{{ de.suffix | default('de') }}" -run__de_tags: "{{ de.tags | default(common__tags) }}" -run__de_force_delete: "{{ de.force_delete | default (run__force_teardown) }}" -run__de_vc_suffix: "{{ de.vc.suffix | default('vc') }}" - -run__dw_definitions: "{{ dw.definitions | default([{}]) }}" -run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" -run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" -run__dw_tags: "{{ dw.tags | default(common__tags) }}" -run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bool }}" -run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" -run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" -run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" -run__dw_default_vw_type: "{{ dw.default_vw.type | default('hive') }}" -run__dw_default_template_type: "{{ dw.default_template.type | default('xsmall') }}" -run__dw_default_dbc_suffix: "{{ dw.default_dbc.suffix | default('dl-default') }}" -run__dw_default_dbc: "{{ dw.default_dbc.name | default([run__env_name, run__dw_default_dbc_suffix] | join('-')) }}" - -run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" -run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" -run__df_public_loadbalancer: "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}" -run__df_lb_ip_ranges: "{{ df.loadbalancer_ip_ranges | default([]) }}" -run__df_k8s_ip_ranges: "{{ df.k8s_ip_ranges | default([]) }}" -run__df_cluster_subnets: "{{ df.cluster_subnets | default(omit) }}" -run__df_cluster_subnets_filter: "{{ df.cluster_subnets_filter | default(omit) }}" -run__df_lb_subnets: "{{ df.loadbalancer_subnets | default(omit) }}" -run__df_lb_subnets_filter: "{{ df.loadbalancer_subnets_filter | default(omit) }}" -run__df_persist: "{{ df.teardown.persist | default(False) }}" -run__df_force_delete: "{{ df.force_delete | default(run__force_teardown) }}" -run__df_terminate_deployments: "{{ df.terminate_deployments | default(True) }}" -run__df_tags: "{{ df.tags | default(common__tags) }}" -run__df_deployments: "{{ df.deployments | default([]) }}" -run__df_readyflows: "{{ df.readyflows | default([]) }}" -run__df_customflows: "{{ df.customflows | default([]) }}" -run__df_delete_readyflows: "{{ df.delete_imported_readyflows | default(False) }}" +run__force_teardown: "{{ common__force_teardown }}" + +run__datahub_image_catalog_url: "{{ datahub.image_catalog.url | default('https://cloudbreak-imagecatalog.s3.amazonaws.com/v3-prod-cb-image-catalog.json') }}" +run__datahub_image_catalog_name: "{{ datahub.image_catalog.name | default('cdp-default') }}" +run__datahub_instance_group_base: "{{ datahub.instance_group_base | default(lookup('template', 'datahub_instance_group_base.j2') | from_yaml) }}" +run__datahub_suffix: "{{ datahub.suffix | default('dhub') }}" +run__datahub_tags: "{{ datahub.tags | default(common__tags) }}" +run__datahub_definitions: "{{ datahub.definitions | default([]) }}" +run__datahub_force_teardown: "{{ datahub.force_delete | default(run__force_teardown) }}" + +run__datahub_compute_aws: "{{ datahub.compute.aws | default({}) }}" +run__datahub_compute_azure: "{{ datahub.compute.azure | default({}) }}" +run__datahub_compute_gcp: "{{ datahub.compute.gcp | default({}) }}" + +run__datahub_storage_aws: "{{ datahub.storage.aws | default({}) }}" +run__datahub_storage_azure: "{{ datahub.storage.azure | default({}) }}" +run__datahub_storage_gcp: "{{ datahub.storage.gcp | default({}) }}" + +run__opdb_definitions: "{{ opdb.definitions | default([{}]) }}" +run__opdb_suffix: "{{ opdb.suffix | default('od') }}" + +run__ml_definitions: "{{ ml.definitions | default([{}]) }}" +run__ml_suffix: "{{ ml.suffix | default('wksp') }}" +run__ml_k8s_request_base: "{{ ml.k8s_request_base | default(lookup('template', 'ml_k8s_request_base.j2') | from_yaml) }}" +run__ml_tags: "{{ ml.tags | default(common__tags) }}" +run__ml_force_delete: "{{ ml.force_delete | default (run__force_teardown) }}" +run__ml_remove_storage: "{{ ml.remove_storage | default (run__force_teardown) }}" +run__ml_public_loadbalancer: "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}" + +run__de_definitions: "{{ de.definitions | default([{}]) }}" +run__de_suffix: "{{ de.suffix | default('de') }}" +run__de_tags: "{{ de.tags | default(common__tags) }}" +run__de_force_delete: "{{ de.force_delete | default (run__force_teardown) }}" +run__de_vc_suffix: "{{ de.vc.suffix | default('vc') }}" + +run__dw_definitions: "{{ dw.definitions | default([{}]) }}" +run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" +run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" +run__dw_tags: "{{ dw.tags | default(common__tags) }}" +run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bool }}" +run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" +run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" +run__dw_default_vw_type: "{{ dw.default_vw.type | default('hive') }}" +run__dw_default_template_type: "{{ dw.default_template.type | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.default_dbc.suffix | default('dl-default') }}" +run__dw_default_dbc: "{{ dw.default_dbc.name | default([run__env_name, run__dw_default_dbc_suffix] | join('-')) }}" + +run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" +run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" +run__df_public_loadbalancer: "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}" +run__df_lb_ip_ranges: "{{ df.loadbalancer_ip_ranges | default([]) }}" +run__df_k8s_ip_ranges: "{{ df.k8s_ip_ranges | default([]) }}" +run__df_cluster_subnets: "{{ df.cluster_subnets | default(omit) }}" +run__df_cluster_subnets_filter: "{{ df.cluster_subnets_filter | default(omit) }}" +run__df_lb_subnets: "{{ df.loadbalancer_subnets | default(omit) }}" +run__df_lb_subnets_filter: "{{ df.loadbalancer_subnets_filter | default(omit) }}" +run__df_persist: "{{ df.teardown.persist | default(False) }}" +run__df_force_delete: "{{ df.force_delete | default(run__force_teardown) }}" +run__df_terminate_deployments: "{{ df.terminate_deployments | default(True) }}" +run__df_tags: "{{ df.tags | default(common__tags) }}" +run__df_deployments: "{{ df.deployments | default([]) }}" +run__df_readyflows: "{{ df.readyflows | default([]) }}" +run__df_customflows: "{{ df.customflows | default([]) }}" +run__df_delete_readyflows: "{{ df.delete_imported_readyflows | default(False) }}" # Deploy -run__include_ml: "{{ common__include_ml }}" -run__include_dw: "{{ common__include_dw }}" -run__include_de: "{{ common__include_de }}" -run__include_df: "{{ common__include_df }}" -run__include_datahub: "{{ common__include_datahub }}" -run__include_opdb: "{{ common__include_opdb }}" +run__include_ml: "{{ common__include_ml }}" +run__include_dw: "{{ common__include_dw }}" +run__include_de: "{{ common__include_de }}" +run__include_df: "{{ common__include_df }}" +run__include_datahub: "{{ common__include_datahub }}" +run__include_opdb: "{{ common__include_opdb }}" # AWS -run__aws_vpc_id: "{{ common__aws_vpc_id }}" -run__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" -run__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" +run__aws_vpc_id: "{{ common__aws_vpc_id }}" +run__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" +run__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" # Azure -run__azure_metagroup_name: "{{ common__azure_metagroup_name }}" -run__azure_netapp_account_name: "{{ common__azure_netapp_account_name }}" -run__azure_netapp_pool_name: "{{ common__azure_netapp_pool_name }}" -run__azure_netapp_vol_name: "{{ common__azure_netapp_vol_name }}" -run__azure_netapp_nfs_version: "{{ common__azure_netapp_nfs_version }}" +run__azure_metagroup_name: "{{ common__azure_metagroup_name }}" +run__azure_netapp_account_name: "{{ common__azure_netapp_account_name }}" +run__azure_netapp_pool_name: "{{ common__azure_netapp_pool_name }}" +run__azure_netapp_vol_name: "{{ common__azure_netapp_vol_name }}" +run__azure_netapp_nfs_version: "{{ common__azure_netapp_nfs_version }}" diff --git a/roles/runtime/meta/argument_specs.yml b/roles/runtime/meta/argument_specs.yml new file mode 100644 index 00000000..9e8d01fb --- /dev/null +++ b/roles/runtime/meta/argument_specs.yml @@ -0,0 +1,22 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Deploy CDP Public Cloud Experiences + description: | + Deployment and management of Cloudera Data Platform (CDP) Public Cloud + Experiences, e.g. Machine Learning, Data Engineering, Data Warehousing. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/runtime/meta/main.yml b/roles/runtime/meta/main.yml deleted file mode 100644 index 5181bee3..00000000 --- a/roles/runtime/meta/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: Webster Mudge (wmudge@cloudera.com) - description: > - Deployment and management of Cloudera Data Platform (CDP) Public Cloud - Experiences, e.g. Machine Learning, Data Engineering, Data Warehousing. - company: Cloudera, Inc. - license: Apache-2.0 - - min_ansible_version: 2.10 - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - machine_learning - - data_engineering - - data_warehousing - - operationa_database - -dependencies: ['cloudera.exe.common'] diff --git a/roles/runtime/molecule/default/collections.yml b/roles/runtime/molecule/default/collections.yml index b05a6f87..366ccf0e 100644 --- a/roles/runtime/molecule/default/collections.yml +++ b/roles/runtime/molecule/default/collections.yml @@ -1,6 +1,5 @@ --- roles: [] - collections: - name: https://github.com/cloudera-labs/cloudera.cloud type: git diff --git a/roles/runtime/molecule/default/molecule.yml b/roles/runtime/molecule/default/molecule.yml index be729eee..d97c97dd 100644 --- a/roles/runtime/molecule/default/molecule.yml +++ b/roles/runtime/molecule/default/molecule.yml @@ -46,10 +46,10 @@ provisioner: aws: region: ${FOUNDRY_AWS_REGION:-us-east-2} env: - tunnel: yes # L1 Networking + tunnel: yes # L1 Networking public_endpoint_access: yes dw: - force_delete: yes # Non-force delete is inconsistent until we can filter 'compactor' VWs + force_delete: yes # Non-force delete is inconsistent until we can filter 'compactor' VWs tags: project: "${FOUNDRY_NAME_PREFIX:-r01}-CDW-testing" definitions: diff --git a/roles/runtime/molecule/default/verify.yml b/roles/runtime/molecule/default/verify.yml index 79044cd0..a5cfa75e 100644 --- a/roles/runtime/molecule/default/verify.yml +++ b/roles/runtime/molecule/default/verify.yml @@ -5,6 +5,6 @@ hosts: all gather_facts: false tasks: - - name: Example assertion - assert: - that: true + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/roles/runtime/molecule/level0/molecule.yml b/roles/runtime/molecule/level0/molecule.yml index 377dbdd4..1cd342be 100644 --- a/roles/runtime/molecule/level0/molecule.yml +++ b/roles/runtime/molecule/level0/molecule.yml @@ -46,7 +46,7 @@ provisioner: aws: region: ${FOUNDRY_AWS_REGION:-us-east-2} env: - tunnel: no # L0 Networking + tunnel: no # L0 Networking verifier: name: ansible lint: | diff --git a/roles/runtime/molecule/level0/verify.yml b/roles/runtime/molecule/level0/verify.yml index 79044cd0..a5cfa75e 100644 --- a/roles/runtime/molecule/level0/verify.yml +++ b/roles/runtime/molecule/level0/verify.yml @@ -5,6 +5,6 @@ hosts: all gather_facts: false tasks: - - name: Example assertion - assert: - that: true + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/roles/runtime/molecule/shared/cleanup.yml b/roles/runtime/molecule/shared/cleanup.yml index 14c176ee..e609ed37 100644 --- a/roles/runtime/molecule/shared/cleanup.yml +++ b/roles/runtime/molecule/shared/cleanup.yml @@ -1,5 +1,4 @@ --- - - name: Cleanup CDP deployment and SSH key hosts: localhost tasks: @@ -52,7 +51,7 @@ community.general.terraform: project_path: "{{ molecule_scenario_directory }}/deployment/" state: absent - force_init: yes + force_init: true - name: Delete the SSH key asset amazon.aws.ec2_key: diff --git a/roles/runtime/molecule/shared/prepare.yml b/roles/runtime/molecule/shared/prepare.yml index 50df3b51..f023f0cf 100644 --- a/roles/runtime/molecule/shared/prepare.yml +++ b/roles/runtime/molecule/shared/prepare.yml @@ -1,5 +1,4 @@ --- - - name: Prepare hosts: localhost tasks: @@ -35,13 +34,13 @@ community.general.terraform: project_path: "{{ molecule_scenario_directory }}/deployment/" state: present - force_init: yes + force_init: true - name: Create the testing SSH key amazon.aws.ec2_key: name: "{{ [globals.name_prefix, lookup('password', '/dev/null chars=ascii_lowercase,digits length=8')] | join('-') }}" region: "{{ infra.aws.region }}" - force: yes + force: true register: __aws_ssh_key - name: Save the private key locally diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index ef273faf..a7948001 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,7 +42,7 @@ - name: Retrieve default CDP Datalake Runtime version if required when: run__cdp_datalake_version | length < 1 cloudera.cloud.datalake_runtime_info: - default: yes + default: true register: __cdp_datalake_version_info failed_when: __cdp_datalake_version_info.versions is not defined @@ -65,12 +64,13 @@ - name: Retrieve Image Catalog File ansible.builtin.uri: url: "{{ run__datahub_image_catalog_url }}" - no_log: yes + no_log: true register: __datahub_image_catalog - name: Set fact for latest CDP Image in Catalog ansible.builtin.set_fact: - run__datahub_image_catalog_entry: "{{ __datahub_image_catalog.json.images['cdh-images'] | community.general.json_query(builds) | sort(attribute='created', reverse=True) | first }}" + run__datahub_image_catalog_entry: "{{ __datahub_image_catalog.json.images['cdh-images'] | community.general.json_query(builds) | sort(attribute='created', + reverse=True) | first }}" vars: builds: "[? version=='{{ run__cdp_datalake_version }}' && contains(keys(images), '{{ run__infra_type }}')]" @@ -80,7 +80,8 @@ - name: Set fact for available CDP Datahub templates by Datalake version ansible.builtin.set_fact: - run__datahub_available_templates: "{{ __datahub_template_info.templates | selectattr('productVersion', 'search', run__cdp_datalake_version) | map(attribute='clusterTemplateName') | list }}" + run__datahub_available_templates: "{{ __datahub_template_info.templates | selectattr('productVersion', 'search', run__cdp_datalake_version) | map(attribute='clusterTemplateName') + | list }}" - name: Retrieve available CDP Datahub definitions cloudera.cloud.datahub_definition_info: @@ -88,19 +89,21 @@ - name: Set fact for available CDP Datahub definitions by Datalake version and Cloud Platform ansible.builtin.set_fact: - run__datahub_available_definitions: "{{ __datahub_definition_info.definitions | selectattr('productVersion', 'search', run__cdp_datalake_version) | selectattr('cloudPlatform', 'search', run__infra_type.upper()) | map(attribute='clusterDefinitionName') | list }}" + run__datahub_available_definitions: "{{ __datahub_definition_info.definitions | selectattr('productVersion', 'search', run__cdp_datalake_version) | selectattr('cloudPlatform', + 'search', run__infra_type.upper()) | map(attribute='clusterDefinitionName') | list }}" - name: Construct CDP Datahub configurations ansible.builtin.set_fact: run__datahub_configs: "{{ run__datahub_configs | default([]) | union([config]) }}" vars: include: "{{ lookup('template', __datahub_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - overlay: "{{ __datahub_config.instance_groups | default(include.instance_groups | default([]) ) }}" + overlay: "{{ __datahub_config.instance_groups | default(include.instance_groups | default([])) }}" base: "{{ run__datahub_instance_group_base }}" config: - name: "{{ __datahub_config.name | default([run__namespace_cdp, __datahub_config.suffix | default(include.suffix) | default(run__datahub_suffix)] | join('-')) }}" - template: "{{ __datahub_config.template | default(include.template | default(omit) ) }}" - definition: "{{ __datahub_config.definition | default(include.definition | default(omit) ) }}" + name: "{{ __datahub_config.name | default([run__namespace_cdp, __datahub_config.suffix | default(include.suffix) | default(run__datahub_suffix)] | join('-')) + }}" + template: "{{ __datahub_config.template | default(include.template | default(omit)) }}" + definition: "{{ __datahub_config.definition | default(include.definition | default(omit)) }}" instance_groups: "{{ (overlay | length > 0) | ternary(overlay | map('cloudera.exe.combine_onto', base, recursive=True) | list, omit) }}" extension: "{{ __datahub_config.extension | default(omit) }}" subnets_filter: "{{ __datahub_config.subnets_filter | default(omit) }}" @@ -120,7 +123,7 @@ - "not ('{{ __datahub_config.name }}' | regex_search('[^-a-z0-9]+'))" fail_msg: >- CDP Datahub name must contain only lowercase letters, numbers, and hypens. - quiet: yes + quiet: true loop_control: loop_var: __datahub_config loop: "{{ run__datahub_configs }}" @@ -157,7 +160,8 @@ config: name: "{{ __ml_config.name | default([run__namespace_cdp, __ml_config.suffix | default(include.suffix) | default(run__ml_suffix)] | join('-')) }}" nfs: "{{ __ml_config.nfs | default(run__azure_netapp_nfs_mount | default(omit)) }}" - nfs_version: "{{ __ml_config.nfs_version | default((__ml_config.nfs is defined or run__azure_netapp_nfs_mount is defined) | ternary(run__azure_netapp_nfs_version, omit)) }}" + nfs_version: "{{ __ml_config.nfs_version | default((__ml_config.nfs is defined or run__azure_netapp_nfs_mount is defined) | ternary(run__azure_netapp_nfs_version, + omit)) }}" k8s_request: environmentName: "{{ run__env_name }}" instanceGroups: "{{ overlay_instance_groups | map('cloudera.exe.combine_onto', base_instance_group, recursive=True) | list }}" @@ -188,7 +192,6 @@ loop_var: __de_config label: "{{ config.name }}" - - name: Prepare for CDP DW experiences when: run__include_dw block: @@ -237,9 +240,9 @@ - __dw_vw_config.tags | dict2items | rejectattr('value', 'regex', '[^-_a-zA-Z0-9.=:+@]+') | list fail_msg: - "A tag in Data Warehouse, '{{ __dw_vw_config.name }}', does not meet requirements;" - - "current tags: {{ __dw_vw_config.tags}}." + - "current tags: {{ __dw_vw_config.tags }}." - "Allowed characters in tags are letters, numbers and the following characters: _.:/=+-@" - quiet: yes + quiet: true loop_control: loop_var: __dw_vw_config label: "{{ __dw_vw_config.name }}" @@ -257,7 +260,7 @@ vars: include: "{{ lookup('template', __df_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" config: - name: "{{ __df_config.name | default([run__namespace_cdp, __df_config.flow_name[::2] | replace(' ','') ] | join('-')) }}" + name: "{{ __df_config.name | default([run__namespace_cdp, __df_config.flow_name[::2] | replace(' ', '')] | join('-')) }}" raw: "{{ __df_config }}" loop: "{{ run__df_deployments }}" loop_control: diff --git a/roles/runtime/tasks/initialize_setup.yml b/roles/runtime/tasks/initialize_setup.yml index 599a9612..2e62f9d1 100644 --- a/roles/runtime/tasks/initialize_setup.yml +++ b/roles/runtime/tasks/initialize_setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,7 +40,7 @@ that: - run__public_subnet_ids | length == 3 fail_msg: "Must have exactly 3 public subnets when deploying CDP Data Warehouse with a public load balancer" - quiet: yes + quiet: true tags: - ml - dw diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml index 18cd9efc..bbcf021a 100644 --- a/roles/runtime/tasks/initialize_setup_aws.yml +++ b/roles/runtime/tasks/initialize_setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +28,7 @@ ansible.builtin.assert: that: __aws_vpc_info.vpcs | length == 1 fail_msg: "No AWS VPC discovered" - quiet: yes + quiet: true - name: Set fact for AWS VPC ID when: __aws_vpc_info is defined @@ -61,7 +60,7 @@ ansible.builtin.assert: that: __aws_subnets_info.subnets | length > 0 fail_msg: "No subnets discovered for AWS VPC" - quiet: yes + quiet: true - name: Set fact for AWS Public Subnet IDs if established by Infrastructure when: not run__aws_public_subnet_ids and infra__aws_public_subnet_ids is defined diff --git a/roles/runtime/tasks/initialize_setup_azure.yml b/roles/runtime/tasks/initialize_setup_azure.yml index b50bbc35..8fcb54c1 100644 --- a/roles/runtime/tasks/initialize_setup_azure.yml +++ b/roles/runtime/tasks/initialize_setup_azure.yml @@ -1,3 +1,4 @@ +--- # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,8 +57,10 @@ - name: Validate Azure NetApp Volume NFS protocol version ansible.builtin.assert: - that: ("NFSv" + run__azure_netapp_nfs_version) in __azure_netapp_nfs_info.response | first | community.general.json_query('properties.protocolTypes') - fail_msg: "Invalid NFS protocol version. 'NFSv{{ run__azure_netapp_nfs_version }}' not supported by Azure NetApp volume, '{{ __azure_netapp_nfs_info.response | map('attribute', 'id') | list | first }}'." - quiet: yes + that: ("NFSv" + run__azure_netapp_nfs_version) in __azure_netapp_nfs_info.response | first | + community.general.json_query('properties.protocolTypes') + fail_msg: "Invalid NFS protocol version. 'NFSv{{ run__azure_netapp_nfs_version }}' not supported by Azure NetApp volume, '{{ __azure_netapp_nfs_info.response + | map('attribute', 'id') | list | first }}'." + quiet: true # ansible -m azure.azcollection.azure_rm_resource_info localhost -a "resource_group=go03-rmgp provider=NetApp resource_type='netAppAccounts' resource_name='go03-netapp-account' subresource={{ subs }}" -e '{ "subs": [{ "type": "capacityPools" }] }' diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index 32a76810..ff13b023 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/runtime/tasks/main.yml b/roles/runtime/tasks/main.yml index 3211b7d8..61082f58 100644 --- a/roles/runtime/tasks/main.yml +++ b/roles/runtime/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include_tasks: validate.yml -- include_tasks: initialize_setup.yml -- include_tasks: setup.yml +- ansible.builtin.include_tasks: validate.yml +- ansible.builtin.include_tasks: initialize_setup.yml +- ansible.builtin.include_tasks: setup.yml diff --git a/roles/runtime/tasks/setup.yml b/roles/runtime/tasks/setup.yml index 4ec25988..935fc5ea 100644 --- a/roles/runtime/tasks/setup.yml +++ b/roles/runtime/tasks/setup.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index f84e20f3..6647e11b 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +18,7 @@ tags: dh block: - name: Retrieve AWS EC2 instance details - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ run__region }}" filters: "tag:Name": "{{ run__namespace }}*" @@ -47,7 +46,7 @@ aws_public_subnets: "{{ run__aws_public_subnet_ids }}" aws_private_subnets: "{{ run__aws_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present - wait: yes + wait: true register: __dw_cluster_build - name: Set fact for DW Cluster @@ -70,11 +69,11 @@ - name: Create Additional CDP DW Database catalogs when: not __dw_dbc_config.use_default_dbc cloudera.cloud.dw_database_catalog: - cluster_id : "{{ __dw_cluster_id }}" + cluster_id: "{{ __dw_cluster_id }}" name: "{{ __dw_dbc_config.name }}" load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" state: present - wait: yes + wait: true async: 3600 # 1 hour timeout poll: 0 loop: "{{ run__dw_dbc_configs }}" @@ -99,7 +98,7 @@ - name: Set CDP DW Database catalog name to id map when: __dw_dbc_build_async.database_catalog is defined ansible.builtin.set_fact: - run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}" + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({__dw_dbc_build_async.database_catalog.name: __dw_dbc_build_async.database_catalog.id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async @@ -118,7 +117,7 @@ application_configs: "{{ __dw_vw_config.configs.application_configs | default(omit) }}" ldap_groups: "{{ __dw_vw_config.configs.ldap_groups | default(omit) }}" enable_sso: "{{ __dw_vw_config.configs.enable_sso | default(omit) }}" - wait: yes + wait: true async: 3600 # 1 hour timeout poll: 0 register: __dw_vw_builds diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index a09fefb4..9238ec4e 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,17 +23,19 @@ state: present image: "{{ run__datahub_image_catalog_entry.uuid }}" catalog: "{{ run__datahub_image_catalog_name }}" - template: "{{ ('template' in __datahub_config_item) | ternary(run__datahub_available_templates | select('ansible.builtin.search', __datahub_config_item.template | default(omit)) | list | first, omit, omit) }}" - definition: "{{ ('definition' in __datahub_config_item) | ternary(run__datahub_available_definitions | select('ansible.builtin.search', __datahub_config_item.definition | default(omit)) | list | first, omit, omit) }}" + template: "{{ ('template' in __datahub_config_item) | ternary(run__datahub_available_templates | select('ansible.builtin.search', __datahub_config_item.template + | default(omit)) | list | first, omit, omit) }}" + definition: "{{ ('definition' in __datahub_config_item) | ternary(run__datahub_available_definitions | select('ansible.builtin.search', __datahub_config_item.definition + | default(omit)) | list | first, omit, omit) }}" groups: "{{ __datahub_config_item.instance_groups | default(omit) }}" extension: "{{ __datahub_config_item.extension | default(omit) }}" subnets_filter: "{{ __datahub_config_item.subnets_filter | default(omit) }}" multi_az: "{{ __datahub_config_item.multi_az | default(omit) }}" tags: "{{ __datahub_config_item.tags }}" - wait: yes + wait: true loop_control: loop_var: __datahub_config_item - label: "{{ __datahub_config_item.name | default ('None') }}" + label: "{{ __datahub_config_item.name | default('None') }}" loop: "{{ run__datahub_configs }}" async: 3600 # 1 hour timeout poll: 0 @@ -59,7 +60,7 @@ storage: "{{ __ml_config_item.raw.storage | default(omit) }}" loop_control: loop_var: __ml_config_item - label: "{{ __ml_config_item.name | default ('None') }}" + label: "{{ __ml_config_item.name | default('None') }}" loop: "{{ run__ml_configs }}" async: 3600 # 1 hour timeout poll: 0 @@ -89,7 +90,7 @@ whitelist_ips: "{{ __de_config_item.raw.whitelist_ips | default(omit) }}" loop_control: loop_var: __de_config_item - label: "{{ __de_config_item.name | default ('None') }}" + label: "{{ __de_config_item.name | default('None') }}" loop: "{{ run__de_configs }}" async: 7200 # 2 hour timeout poll: 0 @@ -102,10 +103,10 @@ name: "{{ __opdb_config_item.name }}" env: "{{ run__env_name }}" state: present - wait: yes + wait: true loop_control: loop_var: __opdb_config_item - label: "{{ __opdb_config_item.name | default ('None') }}" + label: "{{ __opdb_config_item.name | default('None') }}" loop: "{{ run__opdb_configs }}" async: 3600 # 1 hour timeout poll: 0 @@ -127,7 +128,7 @@ loadbalancer_subnets_filter: "{{ run__df_lb_subnets_filter }}" tags: "{{ run__df_tags }}" state: present - wait: no + wait: false # Wait for Service Deployments - name: Wait for CDP Datahub deployments to complete @@ -137,7 +138,7 @@ jid: "{{ __datahub_build_item.ansible_job_id }}" loop_control: loop_var: __datahub_build_item - label: "{{ __datahub_build_item.__datahub_config_item.name | default ('None') }}" + label: "{{ __datahub_build_item.__datahub_config_item.name | default('None') }}" loop: "{{ __datahub_builds.results }}" register: __datahub_builds_async until: __datahub_builds_async.finished @@ -151,7 +152,7 @@ jid: "{{ __ml_build.ansible_job_id }}" loop_control: loop_var: __ml_build - label: "{{ __ml_build.__ml_config_item.name | default ('None') }}" + label: "{{ __ml_build.__ml_config_item.name | default('None') }}" loop: "{{ __ml_builds.results }}" register: __ml_builds_async until: __ml_builds_async.finished @@ -165,7 +166,7 @@ jid: "{{ __de_build.ansible_job_id }}" loop_control: loop_var: __de_build - label: "{{ __de_build.__de_config_item.name | default ('None') }}" + label: "{{ __de_build.__de_config_item.name | default('None') }}" loop: "{{ __de_builds.results }}" register: __de_builds_async until: __de_builds_async.finished @@ -179,7 +180,7 @@ jid: "{{ __opdb_build.ansible_job_id }}" loop_control: loop_var: __opdb_build - label: "{{ __opdb_build.__opdb_config_item.name | default ('None') }}" + label: "{{ __opdb_build.__opdb_config_item.name | default('None') }}" loop: "{{ __opdb_builds.results }}" register: __opdb_builds_async until: __opdb_builds_async.finished @@ -191,7 +192,7 @@ tags: df cloudera.cloud.df_service: name: "{{ run__cdp_env_crn }}" - wait: yes + wait: true # Request Service child deployments - name: Create CDP DE Virtual clusters @@ -207,7 +208,7 @@ runtime_spot_component: "{{ __de_vc_config_item.1.runtime_spot_component | default(omit) }}" spark_version: "{{ __de_vc_config_item.1.spark_version | default(omit) }}" acl_users: "{{ __de_vc_config_item.1.acl_users | default(omit) }}" - wait: yes + wait: true async: 1800 # 30 minute timeout poll: 0 register: __de_vc_builds diff --git a/roles/runtime/tasks/teardown.yml b/roles/runtime/tasks/teardown.yml index 94865ad6..885a46f8 100644 --- a/roles/runtime/tasks/teardown.yml +++ b/roles/runtime/tasks/teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index 16c15f06..2e038fa4 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +22,7 @@ env: "{{ run__env_name }}" name: "{{ __opdb_config.name }}" state: absent - wait: yes + wait: true loop_control: loop_var: __opdb_config label: "{{ __opdb_config.name | default('opdb') }}" @@ -43,7 +42,7 @@ cluster_id: "{{ __dw_cluster_id }}" id: "{{ __vw_id.id }}" state: absent - wait: yes + wait: true loop: "{{ __dw_vw_list.virtual_warehouses | default([]) }}" loop_control: loop_var: __vw_id @@ -70,7 +69,7 @@ cluster_id: "{{ __dw_cluster_id }}" id: "{{ __dw_vw_compactor_item.__dw_vw_teardown_item.__vw_id.id }}" state: absent - wait: yes + wait: true loop: "{{ ansible_failed_result.results }}" when: __dw_vw_compactor_item.failed | bool loop_control: @@ -102,7 +101,7 @@ cluster_id: "{{ __dw_cluster_id }}" id: "{{ __dbc_id.id }}" state: absent - wait: yes + wait: true loop: "{{ __dw_dbc_list.database_catalogs | default([]) }}" loop_control: loop_var: __dbc_id @@ -122,7 +121,7 @@ force: "{{ run__df_force_delete }}" terminate: "{{ run__df_terminate_deployments }}" state: absent - wait: no + wait: false loop_control: loop_var: __df_teardown_req_item loop: "{{ run__df_service_info.services }}" @@ -174,7 +173,7 @@ name: "{{ __datahub_config_item.name }}" state: absent force: "{{ run__datahub_force_teardown }}" - wait: yes + wait: true loop_control: loop_var: __datahub_config_item label: "{{ __datahub_config_item.name | default('datahub') }}" @@ -251,7 +250,7 @@ cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent - wait: yes + wait: true force: "{{ run__dw_force_delete or run__force_teardown }}" - name: Wait for CDP OpDB deployments to decommission @@ -280,7 +279,7 @@ force: "{{ run__df_force_delete }}" terminate: "{{ run__df_terminate_deployments }}" state: absent - wait: yes + wait: true loop_control: loop_var: __df_teardown_wait_item loop: "{{ run__df_service_info.services }}" diff --git a/roles/runtime/tasks/validate.yml b/roles/runtime/tasks/validate.yml index 3ee2714c..0e4d238c 100644 --- a/roles/runtime/tasks/validate.yml +++ b/roles/runtime/tasks/validate.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/runtime/tasks/validate_base.yml b/roles/runtime/tasks/validate_base.yml index 4db384fa..3c3dc922 100644 --- a/roles/runtime/tasks/validate_base.yml +++ b/roles/runtime/tasks/validate_base.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,10 +22,10 @@ that: - "__opdb_definition.name | length < 16" fail_msg: "OpDB name exceeds 15 characters" - quiet: yes + quiet: true loop_control: loop_var: __opdb_definition - label: "{{ __opdb_definition.name | default ('None') }}" + label: "{{ __opdb_definition.name | default('None') }}" loop: "{{ run__opdb_definitions }}" - name: Check Datahub names @@ -38,8 +37,8 @@ that: - "__datahub_definition.name | length < 21" fail_msg: "Datahub name exceeds 20 characters" - quiet: yes + quiet: true loop_control: loop_var: __datahub_definition - label: "{{ __datahub_definition.name | default ('None') }}" + label: "{{ __datahub_definition.name | default('None') }}" loop: "{{ run__datahub_definitions }}" diff --git a/roles/runtime/vars/main.yml b/roles/runtime/vars/main.yml index d48aee99..795162e8 100644 --- a/roles/runtime/vars/main.yml +++ b/roles/runtime/vars/main.yml @@ -23,23 +23,23 @@ run__datahub_compute: run__datahub_compute_default: aws: - std_gp: 'm5.2xlarge' - lrg_gp: 'm5.4xlarge' - std_mem: 'r5.4xlarge' - dsk_mem: 'r5d.4xlarge' + std_gp: "m5.2xlarge" + lrg_gp: "m5.4xlarge" + std_mem: "r5.4xlarge" + dsk_mem: "r5d.4xlarge" std_gpu: "p2.8xlarge" azure: - std_gp: 'Standard_D8_v3' - lrg_gp: 'Standard_D16_v3' - std_mem: 'Standard_D16_v3' - dsk_mem: 'Standard_D8_v3' - std_gpu: 'Standard_D8_v3' + std_gp: "Standard_D8_v3" + lrg_gp: "Standard_D16_v3" + std_mem: "Standard_D16_v3" + dsk_mem: "Standard_D8_v3" + std_gpu: "Standard_D8_v3" gcp: - std_gp: 'e2-standard-8' - lrg_gp: 'e2-standard-8' - std_mem: 'e2-standard-8' - dsk_mem: 'e2-standard-8' - std_gpu: 'e2-standard-8' + std_gp: "e2-standard-8" + lrg_gp: "e2-standard-8" + std_mem: "e2-standard-8" + dsk_mem: "e2-standard-8" + std_gpu: "e2-standard-8" run__datahub_storage: aws: "{{ run__datahub_storage_default['aws'] | combine(run__datahub_storage_aws) }}" @@ -48,20 +48,19 @@ run__datahub_storage: run__datahub_storage_default: aws: - std: 'standard' - fast: 'st1' - eph: 'ephemeral' + std: "standard" + fast: "st1" + eph: "ephemeral" azure: - std: 'StandardSSD_LRS' - fast: 'StandardSSD_LRS' - eph: 'StandardSSD_LRS' + std: "StandardSSD_LRS" + fast: "StandardSSD_LRS" + eph: "StandardSSD_LRS" gcp: - std: 'pd-standard' - fast: 'pd-standard' - eph: 'pd-standard' - + std: "pd-standard" + fast: "pd-standard" + eph: "pd-standard" run__de_compute_default: - aws: 'm5.2xlarge' - azure: 'Standard_D8_v3' - gcp: 'e2-standard-8' + aws: "m5.2xlarge" + azure: "Standard_D8_v3" + gcp: "e2-standard-8" diff --git a/roles/sequence/defaults/main.yml b/roles/sequence/defaults/main.yml index aaa16bfa..a9c5330c 100644 --- a/roles/sequence/defaults/main.yml +++ b/roles/sequence/defaults/main.yml @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -sequence__setup_runtime: "{{ common__setup_runtime }}" +sequence__setup_runtime: "{{ common__setup_runtime }}" -sequence__setup_plat: "{{ common__setup_plat }}" +sequence__setup_plat: "{{ common__setup_plat }}" -sequence__setup_infra: "{{ common__setup_infra }}" +sequence__setup_infra: "{{ common__setup_infra }}" -sequence_init: "{{ sequence__setup_infra }}" +sequence_init: "{{ sequence__setup_infra }}" diff --git a/roles/sequence/meta/argument_specs.yml b/roles/sequence/meta/argument_specs.yml new file mode 100644 index 00000000..d6d51a43 --- /dev/null +++ b/roles/sequence/meta/argument_specs.yml @@ -0,0 +1,22 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Execute run-level operations for CDP Public Cloud + description: | + Run-level execution using tags for deploying and managing Cloudera Data Platform + (CDP) Public Cloud. + author: Webster Mudge (wmudge@cloudera.com) + version_added: 1.0.0 diff --git a/roles/sequence/tasks/main.yml b/roles/sequence/tasks/main.yml index 0b8a7247..57d14c11 100644 --- a/roles/sequence/tasks/main.yml +++ b/roles/sequence/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/sudoers/defaults/main.yml b/roles/sudoers/defaults/main.yml index 480d6799..30021847 100644 --- a/roles/sudoers/defaults/main.yml +++ b/roles/sudoers/defaults/main.yml @@ -19,15 +19,15 @@ # Role prefix is 'sudoers__' # Variables used as inputs in main.yml -sudoers__env_admin_password: "{{ common__env_admin_password }}" +sudoers__env_admin_password: "{{ common__env_admin_password }}" # sudoers__sudo_group_name: sudoers -sudoers__sudo_users: [] -sudoers__sudo_groups: [] # the groups that are members of the sudo_rule +sudoers__sudo_users: [] +sudoers__sudo_groups: [] # the groups that are members of the sudo_rule # Flag to determine if we want to remove (=True) or retain (=False) existing users in sudo rule -sudoers__purge_users: False +sudoers__purge_users: false # Flag to determine if we want to remove (=True) or retain (=False) existing group members in sudo rule -sudoers__purge_groups: False +sudoers__purge_groups: false -sudoers__sudorule_name: admin_all_rule +sudoers__sudorule_name: admin_all_rule diff --git a/roles/sudoers/meta/argument_specs.yml b/roles/sudoers/meta/argument_specs.yml new file mode 100644 index 00000000..19c117b0 --- /dev/null +++ b/roles/sudoers/meta/argument_specs.yml @@ -0,0 +1,23 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Create sudo rule + description: | + Create a passwordless sudo rule and add specified users and/or groups as members to this rule. + Existing group members and users can be purged or retained depending on the value of + the sudoers__purge_groups and sudoers__purge_users flags. + author: Jim Enright (jenright@cloudera.com) + version_added: 1.2.0 diff --git a/roles/sudoers/meta/main.yml b/roles/sudoers/meta/main.yml deleted file mode 100644 index 5fac316a..00000000 --- a/roles/sudoers/meta/main.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: Jim Enright (jenright@cloudera.com) - description: > - Create a passwordless sudo rule and add specified users and/or groups as members to this rule. - Existing group members and users can be purged or retained depending on the value of - the sudoers__purge_groups and sudoers__purge_users flags. - company: Cloudera - license: Apache-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: Debian - versions: all - - name: Fedora - versions: all - - name: GenericLinux - versions: all - - name: MacOSX - versions: all - - name: Ubuntu - versions: all - - galaxy_tags: - - cloudera - - cdp - - freeipa - -dependencies: ['cloudera.exe.common'] diff --git a/roles/sudoers/tasks/main.yml b/roles/sudoers/tasks/main.yml index 4afc5531..7b67ac46 100644 --- a/roles/sudoers/tasks/main.yml +++ b/roles/sudoers/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/sudoers/tasks/teardown.yml b/roles/sudoers/tasks/teardown.yml index 0406851a..498abbce 100644 --- a/roles/sudoers/tasks/teardown.yml +++ b/roles/sudoers/tasks/teardown.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/tls_fetch_ca_certs/meta/argument_specs.yml b/roles/tls_fetch_ca_certs/meta/argument_specs.yml index 5bff0f30..8d1ec99a 100644 --- a/roles/tls_fetch_ca_certs/meta/argument_specs.yml +++ b/roles/tls_fetch_ca_certs/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Fetch the named root and intermediate CA TLS Certificates from the CA Server. author: - "Jim Enright " + version_added: 3.0.0 options: ca_server_intermediate_path: description: "Path to intermediate CA cert on the CA server" diff --git a/roles/tls_fetch_ca_certs/tasks/main.yml b/roles/tls_fetch_ca_certs/tasks/main.yml index e3c14a29..abf00d43 100644 --- a/roles/tls_fetch_ca_certs/tasks/main.yml +++ b/roles/tls_fetch_ca_certs/tasks/main.yml @@ -13,12 +13,11 @@ # limitations under the License. --- - - name: Bring ca root and intermediate cert back to controller ansible.builtin.fetch: src: "{{ item.cert_path }}/{{ item.cert_filename }}" dest: "{{ local_ca_certs_dir }}/{{ item.cert_filename }}" - flat: yes + flat: true loop: - cert_path: "{{ ca_server_intermediate_path }}" cert_filename: "{{ ca_server_intermediate_cert_name }}" diff --git a/roles/tls_generate_csr/defaults/main.yml b/roles/tls_generate_csr/defaults/main.yml index b7ebffb2..8f15cf58 100644 --- a/roles/tls_generate_csr/defaults/main.yml +++ b/roles/tls_generate_csr/defaults/main.yml @@ -27,7 +27,6 @@ ca_server_attrs_general: ST: CA C: US - tls_key_password: changeme tls_key_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.key" diff --git a/roles/tls_generate_csr/meta/argument_specs.yml b/roles/tls_generate_csr/meta/argument_specs.yml index d821614b..bf50a4b3 100644 --- a/roles/tls_generate_csr/meta/argument_specs.yml +++ b/roles/tls_generate_csr/meta/argument_specs.yml @@ -22,6 +22,7 @@ argument_specs: - Once created the CSR file is copied back to the Ansibles controller. author: - "Jim Enright " + version_added: 3.0.0 options: base_dir_security: description: "Base directory for Cloudera CDP security related files" diff --git a/roles/tls_generate_csr/tasks/main.yml b/roles/tls_generate_csr/tasks/main.yml index 1e45b15d..d2b61cc8 100644 --- a/roles/tls_generate_csr/tasks/main.yml +++ b/roles/tls_generate_csr/tasks/main.yml @@ -13,12 +13,11 @@ # limitations under the License. --- - - name: Prepare directories for TLS ansible.builtin.file: state: directory path: "{{ dir }}" - mode: 0755 + mode: "0755" owner: root loop: - "{{ base_dir_security }}" @@ -32,7 +31,7 @@ - name: Derive openssl subjects from subject attribute ansible.builtin.set_fact: - openssl_subject: "{{ openssl_subject | default({}) | combine( {item.split('=')[0] : item.split('=')[1]}) }}" + openssl_subject: "{{ openssl_subject | default({}) | combine({item.split('=')[0]: item.split('=')[1]}) }}" loop: "{{ subject_attr }}" - name: Generate CSR @@ -57,4 +56,4 @@ ansible.builtin.fetch: src: "{{ tls_csr_path }}" dest: "{{ local_csrs_dir }}/" - flat: yes + flat: true diff --git a/roles/tls_install_certs/meta/argument_specs.yml b/roles/tls_install_certs/meta/argument_specs.yml index e1e9fa2b..7ed0b3cd 100644 --- a/roles/tls_install_certs/meta/argument_specs.yml +++ b/roles/tls_install_certs/meta/argument_specs.yml @@ -22,6 +22,7 @@ argument_specs: - Updates the Java keystore with the renewed certificate author: - "Jim Enright " + version_added: 3.0.0 options: openssl_path: description: "Absolute path to the C(openssl) executable" diff --git a/roles/tls_install_certs/tasks/main.yml b/roles/tls_install_certs/tasks/main.yml index 91c61e99..e7a98749 100644 --- a/roles/tls_install_certs/tasks/main.yml +++ b/roles/tls_install_certs/tasks/main.yml @@ -13,11 +13,10 @@ # limitations under the License. --- - - name: Check if signed cert is available - become: no + become: false delegate_to: localhost - stat: + ansible.builtin.stat: path: "{{ local_tls_signed_certs_dir }}/{{ inventory_hostname }}.pem" register: signed_cert @@ -33,7 +32,7 @@ ansible.builtin.copy: src: "{{ local_tls_signed_certs_dir }}/{{ inventory_hostname }}.pem" dest: "{{ tls_cert_path }}" - mode: 0644 + mode: "0644" - name: Create host agnostic link for signed certificate ansible.builtin.file: @@ -41,7 +40,7 @@ dest: "{{ tls_cert_path_generic }}" state: hard force: true - mode: 0644 + mode: "0644" owner: root group: root diff --git a/roles/tls_signing/defaults/main.yml b/roles/tls_signing/defaults/main.yml index 56bda5a9..30064687 100644 --- a/roles/tls_signing/defaults/main.yml +++ b/roles/tls_signing/defaults/main.yml @@ -15,7 +15,7 @@ --- # csrs_to_sign: -copy_from_controller: True +copy_from_controller: true ca_server_intermediate_path: /ca/intermediate ca_server_intermediate_private_key: "{{ ca_server_intermediate_path }}/private/intermediate.key.pem" @@ -26,4 +26,4 @@ ca_server_intermediate_cert: "{{ ca_server_intermediate_path_certs }}/intermedia ca_server_intermediate_path_csr: "{{ ca_server_intermediate_path }}/csr" -backup_old_certs: True +backup_old_certs: true diff --git a/roles/tls_signing/meta/argument_specs.yml b/roles/tls_signing/meta/argument_specs.yml index 9b151e3f..83a80517 100644 --- a/roles/tls_signing/meta/argument_specs.yml +++ b/roles/tls_signing/meta/argument_specs.yml @@ -23,6 +23,7 @@ argument_specs: - Upon completion the signed certs are copied back to the Ansible controller. author: - "Jim Enright " + version_added: 3.0.0 options: csrs_to_sign: description: "List of full path locations of the CSRs to sign." diff --git a/roles/tls_signing/tasks/main.yml b/roles/tls_signing/tasks/main.yml index 66887ec4..dba2a517 100644 --- a/roles/tls_signing/tasks/main.yml +++ b/roles/tls_signing/tasks/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- - # TODO: Check index.txt.attr and see if unique_subject = no for intermediate ca - name: Set fact for all CSRs to sign @@ -31,7 +30,7 @@ ansible.builtin.copy: src: "{{ item.local_file }}" dest: "{{ ca_server_intermediate_path_csr }}/{{ item.csr_filename }}" - mode: 0644 + mode: "0644" loop: "{{ __csrs_to_sign }}" - name: Sign CSRs @@ -51,5 +50,5 @@ ansible.builtin.fetch: src: "{{ ca_server_intermediate_path_certs }}/{{ item.file_suffix }}.pem" dest: "{{ local_certs_dir }}/{{ item.file_suffix }}.pem" - flat: yes + flat: true loop: "{{ __csrs_to_sign }}" diff --git a/tests/config.yml b/tests/config.yml index 10e4d65b..f10d1499 100644 --- a/tests/config.yml +++ b/tests/config.yml @@ -15,4 +15,4 @@ # limitations under the License. modules: - python_requires: '>=3.6' + python_requires: ">=3.6" diff --git a/tests/integration/targets/exec_cdp/meta/main.yml b/tests/integration/targets/exec_cdp/meta/main.yml index e0db08c2..bcd2b8ef 100644 --- a/tests/integration/targets/exec_cdp/meta/main.yml +++ b/tests/integration/targets/exec_cdp/meta/main.yml @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -dependencies: [ 'cloudera.exe.common' ] +dependencies: ["cloudera.exe.common"] diff --git a/tests/integration/targets/exec_cdp/tasks/main.yml b/tests/integration/targets/exec_cdp/tasks/main.yml index 889e2b2a..b5134b20 100644 --- a/tests/integration/targets/exec_cdp/tasks/main.yml +++ b/tests/integration/targets/exec_cdp/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/integration/targets/light_duty_l1/tasks/main.yml b/tests/integration/targets/light_duty_l1/tasks/main.yml index bba71262..84298f00 100644 --- a/tests/integration/targets/light_duty_l1/tasks/main.yml +++ b/tests/integration/targets/light_duty_l1/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,14 +29,14 @@ ansible.builtin.import_role: name: exec_cdp vars: - install: yes + install: true tags: setup - name: Set up the CDP Public Cloud environment again (idempotent) ansible.builtin.import_role: name: exec_cdp vars: - install: yes + install: true tags: setup - name: Run test tasks @@ -49,12 +48,12 @@ ansible.builtin.import_role: name: exec_cdp vars: - install: no + install: false tags: teardown - name: Tear down the CDP Public Cloud environment (idempotent) ansible.builtin.import_role: name: exec_cdp vars: - install: no + install: false tags: teardown diff --git a/tests/integration/targets/light_duty_l1/vars/cdp.yml b/tests/integration/targets/light_duty_l1/vars/cdp.yml index 37d82aaa..c35bdfc8 100644 --- a/tests/integration/targets/light_duty_l1/vars/cdp.yml +++ b/tests/integration/targets/light_duty_l1/vars/cdp.yml @@ -23,11 +23,11 @@ env: credential: name: "{{ globals.name_prefix }}-{{ globals.infra_type }}-lgt-l1-test" datalake: - scale: LIGHT_DUTY # same as env.enable_ha=False - tunnel: yes - public_endpoint_access: yes - workload_analytics: no + scale: LIGHT_DUTY # same as env.enable_ha=False + tunnel: true + public_endpoint_access: true + workload_analytics: false teardown: - delete_credential: yes + delete_credential: true azure: - single_resource_group: yes + single_resource_group: true diff --git a/tests/integration/targets/medium_duty_l1/tasks/main.yml b/tests/integration/targets/medium_duty_l1/tasks/main.yml index dfd0d889..c0252da5 100644 --- a/tests/integration/targets/medium_duty_l1/tasks/main.yml +++ b/tests/integration/targets/medium_duty_l1/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +29,7 @@ ansible.builtin.import_role: name: exec_cdp vars: - install: yes + install: true tags: setup - name: Run test tasks @@ -42,5 +41,5 @@ ansible.builtin.import_role: name: exec_cdp vars: - install: no + install: false tags: teardown diff --git a/tests/integration/targets/medium_duty_l1/vars/cdp.yml b/tests/integration/targets/medium_duty_l1/vars/cdp.yml index 12ac041d..7979755e 100644 --- a/tests/integration/targets/medium_duty_l1/vars/cdp.yml +++ b/tests/integration/targets/medium_duty_l1/vars/cdp.yml @@ -23,11 +23,11 @@ env: credential: name: "{{ globals.name_prefix }}-{{ globals.infra_type }}-md-l1-test" datalake: - scale: MEDIUM_DUTY_HA # same as env.enable_ha=True - tunnel: yes - public_endpoint_access: yes - workload_analytics: no + scale: MEDIUM_DUTY_HA # same as env.enable_ha=True + tunnel: true + public_endpoint_access: true + workload_analytics: false teardown: - delete_credential: yes + delete_credential: true azure: - single_resource_group: yes + single_resource_group: true diff --git a/tests/integration/targets/setup_infra/tasks/main.yml b/tests/integration/targets/setup_infra/tasks/main.yml index 79b9d7f7..ec5e35a8 100644 --- a/tests/integration/targets/setup_infra/tasks/main.yml +++ b/tests/integration/targets/setup_infra/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/integration/targets/teardown_infra/tasks/main.yml b/tests/integration/targets/teardown_infra/tasks/main.yml index 7ed5f1e2..e28bde09 100644 --- a/tests/integration/targets/teardown_infra/tasks/main.yml +++ b/tests/integration/targets/teardown_infra/tasks/main.yml @@ -1,5 +1,4 @@ --- - # Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/integration/targets/teardown_infra/vars/infra.yml b/tests/integration/targets/teardown_infra/vars/infra.yml index eeedcf2c..b648026f 100644 --- a/tests/integration/targets/teardown_infra/vars/infra.yml +++ b/tests/integration/targets/teardown_infra/vars/infra.yml @@ -17,10 +17,10 @@ # Default flags for teardown env: - tunnel: yes # Delete any private networking elements + tunnel: true # Delete any private networking elements infra: teardown: - delete_network: yes - delete_data: yes - delete_ssh_key: yes + delete_network: true + delete_data: true + delete_ssh_key: true