From 10841ee84369d24a99fa7193f12898ce3fe40f9b Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 19 May 2025 12:45:29 +0000 Subject: [PATCH 1/4] feat: support migration from `Consul` to `Raft` --- .github/workflows/pull_request.yml | 1 + roles/vault/README.md | 3 + roles/vault/defaults/main.yml | 24 ++++ roles/vault/tasks/main.yml | 4 + roles/vault/tasks/raft_migration.yml | 26 ++++ tests/test_vault_raft_migration.yml | 182 +++++++++++++++++++++++++++ 6 files changed, 240 insertions(+) create mode 100644 roles/vault/tasks/raft_migration.yml create mode 100644 tests/test_vault_raft_migration.yml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2afd358..fab78f9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,6 +26,7 @@ jobs: - openbao - openbao_ha - vault + - vault_raft_migration steps: - name: Github Checkout 🛎 uses: actions/checkout@v4 diff --git a/roles/vault/README.md b/roles/vault/README.md index f35a46b..07160d2 100644 --- a/roles/vault/README.md +++ b/roles/vault/README.md @@ -51,6 +51,9 @@ s (default: Omitted) * `vault_write_keys_file`: Whether to write the root token and unseal keys to a file. Default `false` * `vault_write_keys_file_host`: Host on which to write root token and unseal keys. Default `localhost` * `vault_write_keys_file_path`: Path of file to write root token and unseal keys. Default `vault-keys.json` + * `vault_storage_type`: The type of storage to be used by vault either `Consul` or `Raft`. Note if vault is already deployed with `Consul` then you can change this inconjunction with `vault_migrate_consul_to_raft` to perform a one-way migration from `Consul` to `Raft`. Default `consul`. + * `vault_migrate_consul_to_raft`: if set true perform the necessary steps to convert the storage backend from `Consul` to `Raft`. Default `false` + * `vault_raft_leaders`: List of IPs belonging to Raft leaders. Expected that the first and only entry is the IP address of the first Vault instance as this would be initialised whereas as the others will not. Root and unseal keys -------------------- diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml index 68e2424..ee40320 100644 --- a/roles/vault/defaults/main.yml +++ b/roles/vault/defaults/main.yml @@ -24,6 +24,10 @@ vault_tls_cert: "" vault_config_dir: "" +vault_raft_leaders: [] + +vault_storage_type: "consul" + vault_config: > { "cluster_name": "{{ vault_cluster_name }}", @@ -49,10 +53,17 @@ vault_config: > {% endif %} }], "storage": { + {% if vault_storage_type == 'consul' %} "consul": { "address": "127.0.0.1:{{ consul_bind_port }}", "path": "vault/" } + {% elif vault_storage_type == 'raft' %} + "raft": { + "node_id": "raft_{{ inventory_hostname }}", + "path": "/vault/file" + } + {% endif %} }, "telemetry": { "prometheus_retention_time": "30s", @@ -97,3 +108,16 @@ vault_write_keys_file: false vault_write_keys_file_host: localhost # Path of file to write root token and unseal keys. vault_write_keys_file_path: vault-keys.json + +vault_migrate_consul_to_raft: false + +vault_raft_migration_config: > + storage_source "consul" { + address = "{{ consul_bind_ip }}:{{ consul_bind_port }}" + path = "vault/" + } + storage_destination "raft" { + node_id = "raft_{{ inventory_hostname }}" + path = "/vault/file" + } + cluster_addr = "{{ vault_protocol }}://{{ vault_bind_address }}:8201" diff --git a/roles/vault/tasks/main.yml b/roles/vault/tasks/main.yml index 4072fc3..c27963a 100644 --- a/roles/vault/tasks/main.yml +++ b/roles/vault/tasks/main.yml @@ -8,5 +8,9 @@ - name: "Deploy Consul" import_tasks: consul.yml +- name: "Migrate from Consul to Raft" + import_tasks: raft_migration.yml + when: vault_migrate_consul_to_raft | bool + - name: "Deploy Vault" import_tasks: vault.yml diff --git a/roles/vault/tasks/raft_migration.yml b/roles/vault/tasks/raft_migration.yml new file mode 100644 index 0000000..a7f681c --- /dev/null +++ b/roles/vault/tasks/raft_migration.yml @@ -0,0 +1,26 @@ +--- +- name: Take Consul snapshot + ansible.builtin.command: > + docker exec {{ consul_docker_name }} /bin/sh -c "consul snapshot save /consul/data/backup_{{ ansible_date_time.iso8601_basic_short }}.snap" + become: true + register: consul_snapshot + changed_when: true + +- name: Write migration configuration + ansible.builtin.copy: + content: "{{ vault_raft_migration_config }}" + dest: "{{ vault_config_dir }}/migration.hcl" + mode: "0644" + become: true + +- name: Perform migration + ansible.builtin.command: > + docker exec {{ vault_docker_name }} /bin/sh -c "vault operator migrate -config /vault/config/migration.hcl" + become: true + changed_when: true + +- name: Change ownership of raft data directory + ansible.builtin.command: > + docker exec {{ vault_docker_name }} /bin/sh -c "chown -R vault:vault /vault/file" + become: true + changed_when: true diff --git a/tests/test_vault_raft_migration.yml b/tests/test_vault_raft_migration.yml new file mode 100644 index 0000000..9f05579 --- /dev/null +++ b/tests/test_vault_raft_migration.yml @@ -0,0 +1,182 @@ +--- +- name: Prepare for vault role + gather_facts: true + hosts: consul + vars: + vault_config_dir: "/etc/vault" + vault_log_keys: true + vault_protocol: http + vault_set_keys_fact: true + vault_write_keys_file: true + tasks: + - name: Ensure /etc/vault exists + file: + path: /etc/vault + state: directory + mode: "0700" + become: true + + - name: Include vault role + include_role: + name: vault + + - name: Include vault role (idempotence test) + include_role: + name: vault + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "vault-keys.json" + name: vault_keys + + - name: Unseal vault + include_role: + name: vault_unseal + vars: + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" + + - name: Configure PKI - create root/intermediate and generate certificates + vars: + vault_pki_certificate_subject: + - role: 'ServerCert' + common_name: "OS-CERT-TEST" + extra_params: + ttl: "8760h" + ip_sans: "127.0.0.1" + alt_names: "example.com" + exclude_cn_from_sans: true + vault_pki_certificates_directory: "/tmp/" + vault_pki_generate_certificates: true + vault_pki_intermediate_ca_name: "OS-TLS-INT" + vault_pki_intermediate_create: true + vault_pki_intermediate_roles: + - name: "ServerCert" + config: + max_ttl: 8760h + ttl: 8760h + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["HPC"] + vault_pki_root_ca_name: "OS-TLS-ROOT" + vault_pki_root_create: true + vault_pki_write_certificate_files: true + vault_pki_write_int_ca_to_file: true + vault_pki_write_pem_bundle: false + vault_pki_write_root_ca_to_file: true + vault_token: "{{ vault_keys.root_token }}" + block: + - name: Configure PKI - create root/intermediate and generate certificates + include_role: + name: vault_pki + + - name: Configure PKI - create root/intermediate and generate certificates (idempotence test) + include_role: + name: vault_pki + + - name: Configure PKI - generate certificate pem bundle + vars: + vault_pki_certificate_subject: + - role: 'ServerCert' + common_name: "OS-CERT-TEST2" + extra_params: + ttl: "8760h" + ip_sans: "192.168.38.72" + exclude_cn_from_sans: true + vault_pki_certificates_directory: "/tmp/" + vault_pki_generate_certificates: true + vault_pki_intermediate_ca_name: "OS-TLS-INT" + vault_pki_intermediate_create: false + vault_pki_root_ca_name: "OS-TLS-ROOT" + vault_pki_root_create: false + vault_pki_write_certificate_files: true + vault_pki_write_pem_bundle: true + vault_token: "{{ vault_keys.root_token }}" + block: + - name: Configure PKI - generate certificate pem bundle + include_role: + name: vault_pki + + - name: Configure PKI - generate certificate pem bundle (idempotence test) + include_role: + name: vault_pki + + - name: Validate if certificates exist + stat: + path: "/tmp/{{ item }}" + register: stat_result + failed_when: not stat_result.stat.exists + loop: + - OS-CERT-TEST.crt + - OS-CERT-TEST2.pem + + - name: Concatenate CAs + shell: | + cat /tmp/OS-TLS-ROOT.pem /tmp/OS-TLS-INT.crt > /tmp/CA-CHAIN.pem + args: + executable: /bin/bash + become: true + changed_when: true + + - name: Verify certificate chain + command: | + openssl verify -CAfile /tmp/CA-CHAIN.pem + /tmp/{{ item }} + register: verify_result + failed_when: verify_result.rc != 0 + loop: + - OS-CERT-TEST.crt + - OS-CERT-TEST2.pem + changed_when: false + + - name: Migrate vault to raft + include_role: + name: vault + vars: + vault_storage_type: raft + vault_migrate_consul_to_raft: true + + - name: Unseal vault + include_role: + name: vault_unseal + vars: + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" + + - name: Validate vault is using raft + ansible.builtin.command: > + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 vault /bin/sh -c "vault status -format=json" + register: vault_status + become: true + changed_when: false + + - name: Validate vault is using raft + ansible.builtin.assert: + that: + - vault_status.stdout | from_json | json_query('storage_type') == 'raft' + fail_msg: "Vault is not using raft storage backend" + success_msg: "Vault is using raft storage backend" + + - name: Read CA certificate from vault + community.hashi_vault.vault_read: + url: http://127.0.0.1:8200 + path: OS-TLS-ROOT/cert/ca + token: "{{ vault_keys.root_token }}" + register: vault_ca_cert + + - name: Read CA from file + ansible.builtin.slurp: + src: /tmp/OS-TLS-ROOT.pem + register: ca_chain + + - name: Validate ROOT CA + ansible.builtin.assert: + that: + - vault_ca_cert.data.data.certificate == (ca_chain.content | b64decode).rstrip('\n') + fail_msg: "ROOT CA certificate do not match" + success_msg: "ROOT CA certificate do match" From a376e1dad640baeb6a35ae357b9179af4b5a416d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 19 May 2025 13:00:38 +0000 Subject: [PATCH 2/4] ci: use `hashivault_read` to get `certificate` --- tests/test_vault_raft_migration.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_vault_raft_migration.yml b/tests/test_vault_raft_migration.yml index 9f05579..14b3d19 100644 --- a/tests/test_vault_raft_migration.yml +++ b/tests/test_vault_raft_migration.yml @@ -163,9 +163,10 @@ success_msg: "Vault is using raft storage backend" - name: Read CA certificate from vault - community.hashi_vault.vault_read: + hashivault_read: url: http://127.0.0.1:8200 - path: OS-TLS-ROOT/cert/ca + mount_point: OS-TLS-ROOT + secret: cert/ca token: "{{ vault_keys.root_token }}" register: vault_ca_cert @@ -177,6 +178,6 @@ - name: Validate ROOT CA ansible.builtin.assert: that: - - vault_ca_cert.data.data.certificate == (ca_chain.content | b64decode).rstrip('\n') + - vault_ca_cert.value.certificate == (ca_chain.content | b64decode).rstrip('\n') fail_msg: "ROOT CA certificate do not match" success_msg: "ROOT CA certificate do match" From cb178be2702ad2d8ccfe1e9012695af2009d6b2c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 19 May 2025 13:28:07 +0000 Subject: [PATCH 3/4] ci: use `ansible_facts` for `date_time` --- roles/vault/tasks/raft_migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/vault/tasks/raft_migration.yml b/roles/vault/tasks/raft_migration.yml index a7f681c..cedfda6 100644 --- a/roles/vault/tasks/raft_migration.yml +++ b/roles/vault/tasks/raft_migration.yml @@ -1,7 +1,7 @@ --- - name: Take Consul snapshot ansible.builtin.command: > - docker exec {{ consul_docker_name }} /bin/sh -c "consul snapshot save /consul/data/backup_{{ ansible_date_time.iso8601_basic_short }}.snap" + docker exec {{ consul_docker_name }} /bin/sh -c "consul snapshot save /consul/data/backup_{{ ansible_facts.date_time.iso8601_basic_short }}.snap" become: true register: consul_snapshot changed_when: true From 70ece3812052b655f5bd573e881370ca7c70e8e1 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 19 May 2025 13:31:11 +0000 Subject: [PATCH 4/4] ci: install `jmespath` for `json_query` --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index fab78f9..5c77381 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -40,7 +40,7 @@ jobs: run: | pipx uninstall ansible-core python3 -m pip install --upgrade pip - python3 -m pip install ansible-core==${{ matrix.ansible_version }}.* docker git+https://github.com/TerryHowe/ansible-modules-hashivault@c22434d887f0b8a5ac3ebda710664a027291e71c + python3 -m pip install ansible-core==${{ matrix.ansible_version }}.* docker jmespath git+https://github.com/TerryHowe/ansible-modules-hashivault@c22434d887f0b8a5ac3ebda710664a027291e71c ansible-galaxy collection build ansible-galaxy collection install *.tar.gz ansible-galaxy collection install community.general