From 4de663f1da982d75c85a4801cfa2da2216c58802 Mon Sep 17 00:00:00 2001 From: Daniel Neuberger Date: Wed, 30 Aug 2023 16:59:07 +0200 Subject: [PATCH 01/67] Add code for automatic updates Adding playbook contributed by @xtruthx I have to admit, I kept this playbook for waaaaay too long. @xtruthx sent it to me a long while back and I always planned to integrate it into the codebase of this collection. Now I had to face that I took too long and so I'm putting it up publicly so we can work on integration as a common effort. I left the playbook "as is". It stems from a different project so variable names etc. don't match. We need to fix them first. Also I'm not sure if we should really add it as a playbook or maybe better make it a task file in a role. fixes #216 --- .../task-elasticsearch-rolling-upgrade.yml | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 playbooks/task-elasticsearch-rolling-upgrade.yml diff --git a/playbooks/task-elasticsearch-rolling-upgrade.yml b/playbooks/task-elasticsearch-rolling-upgrade.yml new file mode 100644 index 00000000..1ac67e67 --- /dev/null +++ b/playbooks/task-elasticsearch-rolling-upgrade.yml @@ -0,0 +1,201 @@ +# Ansible +# +# Rolling Upgrade of Elasticsearch with security on +# Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros +# Modifications: author: Daniel Neuberger @netways.de +# latest tested with Ansible 2.9 and later + +--- +- name: Elasticsearch rolling upgrade + hosts: elasticsearch_{{ env }} + become: true + serial: 1 + vars_files: + - vars/elasticsearch/elasticsearch-{{ env }}_secrets.yml + vars: + es_disable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' + es_enable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": null }}' + es_http_port: 9200 + es_transport_port: 9300 + #desired version to upgrade to: 7.10.2 + es_version: '8.5.3' + + tasks: + # this first step is a overkill, but here + # in case the upgrade was cancelled by user mid playbook run + - name: make sure elasticsearch service is running + service: name=elasticsearch enabled=yes state=started + register: response + become: true + + - name: Wait for elasticsearch node to come back up if it was stopped + wait_for: + host: "{{ ansible_default_ipv4.address }}" + port: "{{ es_transport_port }}" + delay: 45 + when: response.changed == true + + - name: check current version + uri: + url: https://localhost:{{ es_http_port }} + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: version_found + retries: 10 + delay: 10 + + - name: Display Current Elasticsearch Version + debug: var=version_found.json.version.number + + # this step is key!!! Don't restart more nodes + # until all shards have completed recovery + - name: Wait for cluster health to return to green + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + until: "response.json.status == 'green'" + retries: 50 + delay: 30 + # when: version_found.json.version.number != '{{ es_version }}' + + + - name: Disable shard allocation for the cluster + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_disable_allocation }}' + body_format: json + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + #when: version_found.json.version.number != '{{ es_version }}' + + - name: stop non essential indexing to speed up shard recovery + uri: + url: https://localhost:{{ es_http_port }}/_flush + method: POST + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + ignore_errors: yes + + + - name: Shutdown elasticsearch service + service: name=elasticsearch enabled=yes state=stopped + become: true + + # do an "apt-get update", to ensure latest package lists + - name: apt-get update + apt: + update-cache: yes + changed_when: 0 + + # get a list of packages that have updates + - name: get list of pending upgrades + command: apt-get --simulate dist-upgrade + args: + warn: false # don't warn us about apt having its own plugin + register: apt_simulate + changed_when: 0 + + # pick out list of pending updates from command output. This essentially + # takes the above output from "apt-get --simulate dist-upgrade", and + # pipes it through "cut -f2 -d' ' | sort" + - name: parse apt-get output to get list of changed packages + set_fact: + updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' + changed_when: 0 + + # tell user about packages being updated + - name: show pending updates + debug: + var: updates + when: updates.0 is defined + + # request manual ack before proceeding with package upgrade + - pause: + when: updates.0 is defined + + # if a new kernel is incoming, remove old ones to avoid full /boot + - name: apt-get autoremove + command: apt-get -y autoremove + args: + warn: false + when: '"Inst linux-image-" in apt_simulate.stdout' + changed_when: 0 + + # do the actual apt-get dist-upgrade + - name: apt-get dist-upgrade + apt: + upgrade: dist # upgrade all packages to latest version + + # REBOOT machine after Upgrade + - name: check if reboot is required + register: reboot_required_file + stat: + path: /var/run/reboot-required + + - name: restart machine + reboot: + msg: "Reboot initiated by Ansible to update system libs/kernel as needed" + when: reboot_required_file.stat.exists == true + + - name: waiting for machine to come back + wait_for_connection: + delay: 10 + connect_timeout: 300 + when: reboot_required_file.stat.exists == true + + - name: Start elasticsearch + service: name=elasticsearch enabled=yes state=started + #when: version_found.json.version.number != '{{ es_version }}' + become: true + + - name: Wait for elasticsearch node to come back up if it was stopped + wait_for: + host: "{{ ansible_default_ipv4.address }}" + port: "{{ es_transport_port }}" + delay: 30 + + - name: Confirm the node joins the cluster + shell: "curl -k -u elastic:{{ es_api_basic_auth_password }} -s -m 2 'https://localhost:9200/_cat/nodes?h=name' | grep -E '^{{ ansible_fqdn }}$'" + register: result + until: result.rc == 0 + retries: 200 + delay: 3 + #when: version_found.json.version.number != '{{ es_version }}' + + - name: Enable shard allocation for the cluster + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_enable_allocation }}' + body_format: json + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + #when: version_found.json.version.number != es_version + + - name: Wait for cluster health to return to yellow or green + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + until: "response.json.status == 'yellow' or response.json.status == 'green'" + retries: 5 + delay: 30 From b9e204d70a41c244b2b0593788d5975ddab91fea Mon Sep 17 00:00:00 2001 From: Daniel Neuberger Date: Tue, 16 Jan 2024 17:45:38 +0100 Subject: [PATCH 02/67] Start changing playbook into taskfile --- .../task-elasticsearch-rolling-upgrade.yml | 201 ------------------ .../tasks/elasticsearch-rolling-upgrade.yml | 201 ++++++++++++++++++ 2 files changed, 201 insertions(+), 201 deletions(-) delete mode 100644 playbooks/task-elasticsearch-rolling-upgrade.yml create mode 100644 roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml diff --git a/playbooks/task-elasticsearch-rolling-upgrade.yml b/playbooks/task-elasticsearch-rolling-upgrade.yml deleted file mode 100644 index 1ac67e67..00000000 --- a/playbooks/task-elasticsearch-rolling-upgrade.yml +++ /dev/null @@ -1,201 +0,0 @@ -# Ansible -# -# Rolling Upgrade of Elasticsearch with security on -# Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros -# Modifications: author: Daniel Neuberger @netways.de -# latest tested with Ansible 2.9 and later - ---- -- name: Elasticsearch rolling upgrade - hosts: elasticsearch_{{ env }} - become: true - serial: 1 - vars_files: - - vars/elasticsearch/elasticsearch-{{ env }}_secrets.yml - vars: - es_disable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' - es_enable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": null }}' - es_http_port: 9200 - es_transport_port: 9300 - #desired version to upgrade to: 7.10.2 - es_version: '8.5.3' - - tasks: - # this first step is a overkill, but here - # in case the upgrade was cancelled by user mid playbook run - - name: make sure elasticsearch service is running - service: name=elasticsearch enabled=yes state=started - register: response - become: true - - - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: - host: "{{ ansible_default_ipv4.address }}" - port: "{{ es_transport_port }}" - delay: 45 - when: response.changed == true - - - name: check current version - uri: - url: https://localhost:{{ es_http_port }} - method: GET - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: version_found - retries: 10 - delay: 10 - - - name: Display Current Elasticsearch Version - debug: var=version_found.json.version.number - - # this step is key!!! Don't restart more nodes - # until all shards have completed recovery - - name: Wait for cluster health to return to green - uri: - url: https://localhost:{{ es_http_port }}/_cluster/health - method: GET - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: response - until: "response.json.status == 'green'" - retries: 50 - delay: 30 - # when: version_found.json.version.number != '{{ es_version }}' - - - - name: Disable shard allocation for the cluster - uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings - method: PUT - body: '{{ es_disable_allocation }}' - body_format: json - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - #when: version_found.json.version.number != '{{ es_version }}' - - - name: stop non essential indexing to speed up shard recovery - uri: - url: https://localhost:{{ es_http_port }}/_flush - method: POST - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - ignore_errors: yes - - - - name: Shutdown elasticsearch service - service: name=elasticsearch enabled=yes state=stopped - become: true - - # do an "apt-get update", to ensure latest package lists - - name: apt-get update - apt: - update-cache: yes - changed_when: 0 - - # get a list of packages that have updates - - name: get list of pending upgrades - command: apt-get --simulate dist-upgrade - args: - warn: false # don't warn us about apt having its own plugin - register: apt_simulate - changed_when: 0 - - # pick out list of pending updates from command output. This essentially - # takes the above output from "apt-get --simulate dist-upgrade", and - # pipes it through "cut -f2 -d' ' | sort" - - name: parse apt-get output to get list of changed packages - set_fact: - updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' - changed_when: 0 - - # tell user about packages being updated - - name: show pending updates - debug: - var: updates - when: updates.0 is defined - - # request manual ack before proceeding with package upgrade - - pause: - when: updates.0 is defined - - # if a new kernel is incoming, remove old ones to avoid full /boot - - name: apt-get autoremove - command: apt-get -y autoremove - args: - warn: false - when: '"Inst linux-image-" in apt_simulate.stdout' - changed_when: 0 - - # do the actual apt-get dist-upgrade - - name: apt-get dist-upgrade - apt: - upgrade: dist # upgrade all packages to latest version - - # REBOOT machine after Upgrade - - name: check if reboot is required - register: reboot_required_file - stat: - path: /var/run/reboot-required - - - name: restart machine - reboot: - msg: "Reboot initiated by Ansible to update system libs/kernel as needed" - when: reboot_required_file.stat.exists == true - - - name: waiting for machine to come back - wait_for_connection: - delay: 10 - connect_timeout: 300 - when: reboot_required_file.stat.exists == true - - - name: Start elasticsearch - service: name=elasticsearch enabled=yes state=started - #when: version_found.json.version.number != '{{ es_version }}' - become: true - - - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: - host: "{{ ansible_default_ipv4.address }}" - port: "{{ es_transport_port }}" - delay: 30 - - - name: Confirm the node joins the cluster - shell: "curl -k -u elastic:{{ es_api_basic_auth_password }} -s -m 2 'https://localhost:9200/_cat/nodes?h=name' | grep -E '^{{ ansible_fqdn }}$'" - register: result - until: result.rc == 0 - retries: 200 - delay: 3 - #when: version_found.json.version.number != '{{ es_version }}' - - - name: Enable shard allocation for the cluster - uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings - method: PUT - body: '{{ es_enable_allocation }}' - body_format: json - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: response - # next line is boolean not string, so no quotes around true - # use python truthiness - until: "response.json.acknowledged == true" - retries: 5 - delay: 30 - #when: version_found.json.version.number != es_version - - - name: Wait for cluster health to return to yellow or green - uri: - url: https://localhost:{{ es_http_port }}/_cluster/health - method: GET - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: response - until: "response.json.status == 'yellow' or response.json.status == 'green'" - retries: 5 - delay: 30 diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml new file mode 100644 index 00000000..51586bf9 --- /dev/null +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -0,0 +1,201 @@ +# Ansible +# +# Rolling Upgrade of Elasticsearch with security on +# Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros +# Modifications: author: Daniel Neuberger @netways.de +# latest tested with Ansible 2.9 and later + +--- +#- name: Elasticsearch rolling upgrade +# hosts: elasticsearch_{{ env }} +# become: true +# serial: 1 +# vars_files: +# - vars/elasticsearch/elasticsearch-{{ env }}_secrets.yml +# vars: +# es_disable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' +# es_enable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": null }}' +# es_http_port: 9200 +# es_transport_port: 9300 +# #desired version to upgrade to: 7.10.2 +# es_version: '8.5.3' +# +# tasks: + # this first step is a overkill, but here + # in case the upgrade was cancelled by user mid playbook run +- name: make sure elasticsearch service is running + service: name=elasticsearch enabled=yes state=started + register: response + become: true + +- name: Wait for elasticsearch node to come back up if it was stopped + wait_for: + host: "{{ ansible_default_ipv4.address }}" + port: "{{ es_transport_port }}" + delay: 45 + when: response.changed == true + +- name: check current version + uri: + url: https://localhost:{{ es_http_port }} + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: version_found + retries: 10 + delay: 10 + +- name: Display Current Elasticsearch Version + debug: var=version_found.json.version.number + + # this step is key!!! Don't restart more nodes + # until all shards have completed recovery +- name: Wait for cluster health to return to green + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + until: "response.json.status == 'green'" + retries: 50 + delay: 30 + # when: version_found.json.version.number != '{{ es_version }}' + + +- name: Disable shard allocation for the cluster + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_disable_allocation }}' + body_format: json + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + #when: version_found.json.version.number != '{{ es_version }}' + +- name: stop non essential indexing to speed up shard recovery + uri: + url: https://localhost:{{ es_http_port }}/_flush + method: POST + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + ignore_errors: yes + + +- name: Shutdown elasticsearch service + service: name=elasticsearch enabled=yes state=stopped + become: true + +# do an "apt-get update", to ensure latest package lists +- name: apt-get update + apt: + update-cache: yes + changed_when: 0 + +# get a list of packages that have updates +- name: get list of pending upgrades + command: apt-get --simulate dist-upgrade + args: + warn: false # don't warn us about apt having its own plugin + register: apt_simulate + changed_when: 0 + +# pick out list of pending updates from command output. This essentially +# takes the above output from "apt-get --simulate dist-upgrade", and +# pipes it through "cut -f2 -d' ' | sort" +- name: parse apt-get output to get list of changed packages + set_fact: + updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' + changed_when: 0 + +# tell user about packages being updated +- name: show pending updates + debug: + var: updates + when: updates.0 is defined + + request manual ack before proceeding with package upgrade +- pause: + when: updates.0 is defined + +if a new kernel is incoming, remove old ones to avoid full /boot +- name: apt-get autoremove + command: apt-get -y autoremove + args: + warn: false + when: '"Inst linux-image-" in apt_simulate.stdout' + changed_when: 0 + +do the actual apt-get dist-upgrade +- name: apt-get dist-upgrade + apt: + upgrade: dist # upgrade all packages to latest version + +REBOOT machine after Upgrade +- name: check if reboot is required + register: reboot_required_file + stat: + path: /var/run/reboot-required + +- name: restart machine + reboot: + msg: "Reboot initiated by Ansible to update system libs/kernel as needed" + when: reboot_required_file.stat.exists == true + +- name: waiting for machine to come back + wait_for_connection: + delay: 10 + connect_timeout: 300 + when: reboot_required_file.stat.exists == true + +- name: Start elasticsearch + service: name=elasticsearch enabled=yes state=started + #when: version_found.json.version.number != '{{ es_version }}' + become: true + +- name: Wait for elasticsearch node to come back up if it was stopped + wait_for: + host: "{{ ansible_default_ipv4.address }}" + port: "{{ es_transport_port }}" + delay: 30 + +- name: Confirm the node joins the cluster + shell: "curl -k -u elastic:{{ es_api_basic_auth_password }} -s -m 2 'https://localhost:9200/_cat/nodes?h=name' | grep -E '^{{ ansible_fqdn }}$'" + register: result + until: result.rc == 0 + retries: 200 + delay: 3 + #when: version_found.json.version.number != '{{ es_version }}' + +- name: Enable shard allocation for the cluster + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_enable_allocation }}' + body_format: json + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + #when: version_found.json.version.number != es_version + +- name: Wait for cluster health to return to yellow or green + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + until: "response.json.status == 'yellow' or response.json.status == 'green'" + retries: 5 + delay: 30 From eefcef95d3ff3f9590486fc9f819e191588bb44c Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 17:50:43 +0100 Subject: [PATCH 03/67] Add note about modifications by NPS --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 51586bf9..3702753d 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -3,6 +3,7 @@ # Rolling Upgrade of Elasticsearch with security on # Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros # Modifications: author: Daniel Neuberger @netways.de +# More modifications: NETWAYS Professional Services GmbH # latest tested with Ansible 2.9 and later --- From 9570a9fa49a734155910a1b5f605dbb3ee8c754e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 18:03:44 +0100 Subject: [PATCH 04/67] Move global vars to new role fixes #286 --- roles/beats/defaults/main.yml | 12 ------------ roles/beats/tasks/main.yml | 4 ++++ roles/elasticsearch/defaults/main.yml | 15 --------------- roles/elasticsearch/tasks/main.yml | 5 +++++ roles/elasticstack/defaults/main.yml | 17 +++++++++++++++++ roles/kibana/defaults/main.yml | 10 ---------- roles/kibana/tasks/main.yml | 4 ++++ roles/logstash/defaults/main.yml | 17 ----------------- roles/logstash/tasks/main.yml | 4 ++++ roles/repos/defaults/main.yml | 9 --------- roles/repos/tasks/main.yml | 4 ++++ 11 files changed, 38 insertions(+), 63 deletions(-) create mode 100644 roles/elasticstack/defaults/main.yml delete mode 100644 roles/repos/defaults/main.yml diff --git a/roles/beats/defaults/main.yml b/roles/beats/defaults/main.yml index 829bda5e..9c205c7e 100644 --- a/roles/beats/defaults/main.yml +++ b/roles/beats/defaults/main.yml @@ -6,7 +6,6 @@ beats_auditbeat: false beats_metricbeat: false beats_target_hosts: - localhost -elasticstack_beats_port: 5044 beats_logging: file beats_logpath: /var/log/beats beats_loglevel: info @@ -59,19 +58,8 @@ beats_metricbeat_modules: - system beats_metricbeat_loadbalance: true -elasticstack_release: 8 -elasticstack_full_stack: true -elasticstack_variant: elastic -elasticstack_security: true - -elasticstack_ca_dir: /opt/es-ca -elasticstack_ca_pass: PleaseChangeMe -elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords -elasticstack_elasticsearch_http_port: 9200 beats_cert_validity_period: 1095 beats_cert_expiration_buffer: "+30d" beats_cert_will_expire_soon: false -# Variables for debugging and development -elasticstack_override_beats_tls: false diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index f242c0a8..02209e93 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Include global role + ansible.builtin.import_role: + name: netways.elasticstack.elasticstack + - name: Include OS specific vars ansible.builtin.include_vars: '{{ item }}' with_first_found: diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index 5e8de21c..e39b0a2a 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -34,23 +34,8 @@ elasticsearch_jna_workaround: false # They follow a different naming scheme to show that they are global # to our set of Elastic Stack related Ansible roles -# elasticstack_ca: First host in the `elasticsearch` group -elasticstack_ca_dir: /opt/es-ca -elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticsearch_initialized_file: "{{ elasticstack_initial_passwords | dirname }}/cluster_initialized" -elasticstack_ca_name: "CN=Elastic Certificate Tool Autogenerated CA" -elasticstack_ca_pass: PleaseChangeMe -elasticstack_ca_validity_period: 1095 elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually elasticsearch_cert_validity_period: 1095 -elasticstack_ca_expiration_buffer: 30 elasticsearch_cert_expiration_buffer: 30 -elasticstack_ca_will_expire_soon: false elasticsearch_cert_will_expire_soon: false - -# "global" variables for all roles - -elasticstack_release: 8 -elasticstack_full_stack: true -elasticstack_variant: elastic -elasticstack_elasticsearch_http_port: 9200 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index a8751815..0e072d32 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,4 +1,9 @@ --- + +- name: Include global role + ansible.builtin.import_role: + name: netways.elasticstack.elasticstack + - name: Check-set-parameters ansible.builtin.include_tasks: elasticsearch-parameters.yml diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml new file mode 100644 index 00000000..f0ba5c06 --- /dev/null +++ b/roles/elasticstack/defaults/main.yml @@ -0,0 +1,17 @@ +elasticstack_beats_port: 5044 +elasticstack_ca_dir: /opt/es-ca +elasticstack_ca_expiration_buffer: 30 +elasticstack_ca_name: "CN=Elastic Certificate Tool Autogenerated CA" +elasticstack_ca_pass: PleaseChangeMe +elasticstack_ca_validity_period: 1095 +elasticstack_ca_will_expire_soon: false +elasticstack_elasticsearch_http_port: 9200 +elasticstack_enable_repos: true +elasticstack_full_stack: true +elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords +elasticstack_kibana_port: 5601 +elasticstack_override_beats_tls: false +elasticstack_release: 8 +elasticstack_rpm_workaround: false +elasticstack_security: true +elasticstack_variant: elastic diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml index a08a2acd..60e15804 100644 --- a/roles/kibana/defaults/main.yml +++ b/roles/kibana/defaults/main.yml @@ -9,18 +9,8 @@ kibana_tls: false kibana_tls_cert: /etc/kibana/certs/cert.pem kibana_tls_key: /etc/kibana/certs/key.pem kibana_tls_key_passphrase: PleaseChangeMe -elasticstack_ca_dir: /opt/es-ca -elasticstack_ca_pass: PleaseChangeMe -elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords kibana_cert_expiration_buffer: 30 kibana_cert_validity_period: 1095 kibana_cert_will_expire_soon: false kibana_sniff_on_start: false kibana_sniff_on_connection_fault: false - -# "global" variables for all roles -elasticstack_release: 8 -elasticstack_full_stack: true -elasticstack_variant: elastic -elasticstack_elasticsearch_http_port: 9200 -elasticstack_kibana_port: 5601 diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index a182b088..a0f18afa 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Include global role + ansible.builtin.import_role: + name: netways.elasticstack.elasticstack + - name: Include OS specific vars ansible.builtin.include_vars: '{{ item }}' with_first_found: diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index ac876fba..490a01d6 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -68,20 +68,3 @@ logstash_ident_field_name: "[netways][instance]" logstash_pipeline_identifier: true logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false - -elasticstack_ca_dir: /opt/es-ca -elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords -elasticstack_ca_pass: PleaseChangeMe - -# "global" variables for all roles - -elasticstack_release: 8 -elasticstack_full_stack: true -elasticstack_variant: elastic -elasticstack_security: true -elasticstack_elasticsearch_http_port: 9200 -elasticstack_beats_port: 5044 - -# Variables for debugging and development - -elasticstack_override_beats_tls: false diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 1dcee30b..45824608 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Include global role + ansible.builtin.import_role: + name: netways.elasticstack.elasticstack + - name: Include OS specific vars ansible.builtin.include_vars: '{{ item }}' with_first_found: diff --git a/roles/repos/defaults/main.yml b/roles/repos/defaults/main.yml deleted file mode 100644 index 9230985e..00000000 --- a/roles/repos/defaults/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# defaults file for elastic-repos -elasticstack_release: 8 -elasticstack_full_stack: true -elasticstack_variant: elastic - -elasticstack_rpm_workaround: false - -elasticstack_enable_repos: true diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index b78cfbcf..340b4c43 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Include global role + ansible.builtin.import_role: + name: netways.elasticstack.elasticstack + - name: Check for versions ansible.builtin.fail: msg: "No OSS versions later than 7 are available" From 6a11bf5a7188a81269ab1823f5d31c768f72cb40 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 18:15:15 +0100 Subject: [PATCH 05/67] Fix lint in defaults --- roles/elasticstack/defaults/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 2b0e1627..d681d146 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -1,3 +1,5 @@ +--- + elasticstack_beats_port: 5044 elasticstack_ca_dir: /opt/es-ca elasticstack_ca_pass: PleaseChangeMe @@ -6,9 +8,12 @@ elasticstack_enable_repos: true elasticstack_full_stack: true elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_kibana_port: 5601 -elasticstack_no_log: true elasticstack_override_beats_tls: false elasticstack_release: 8 elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic + +# only for debugging +# +elasticstack_no_log: true From df106c54c8a31163fcdf324d5aefc3d5050cbbfe Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 18:16:32 +0100 Subject: [PATCH 06/67] lint --- roles/beats/defaults/main.yml | 2 -- roles/elasticsearch/defaults/main.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/roles/beats/defaults/main.yml b/roles/beats/defaults/main.yml index 89d6a7e5..b52689f7 100644 --- a/roles/beats/defaults/main.yml +++ b/roles/beats/defaults/main.yml @@ -60,5 +60,3 @@ beats_metricbeat_loadbalance: true beats_cert_validity_period: 1095 beats_cert_expiration_buffer: "+30d" beats_cert_will_expire_soon: false - - diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index 6e3faeb2..3fdfbffc 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -45,4 +45,3 @@ elasticsearch_freshstart: changed: false elasticsearch_freshstart_security: changed: false - From 6f97e74024903a01c6a0163ac22cd1e294004244 Mon Sep 17 00:00:00 2001 From: Daniel Neuberger Date: Tue, 16 Jan 2024 18:25:47 +0100 Subject: [PATCH 07/67] Clean up lint --- .../tasks/elasticsearch-rolling-upgrade.yml | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 3702753d..9ea57e73 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -1,5 +1,5 @@ # Ansible -# +# # Rolling Upgrade of Elasticsearch with security on # Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros # Modifications: author: Daniel Neuberger @netways.de @@ -30,14 +30,14 @@ become: true - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: - host: "{{ ansible_default_ipv4.address }}" + wait_for: + host: "{{ ansible_default_ipv4.address }}" port: "{{ es_transport_port }}" delay: 45 when: response.changed == true - name: check current version - uri: + uri: url: https://localhost:{{ es_http_port }} method: GET user: elastic @@ -53,8 +53,8 @@ # this step is key!!! Don't restart more nodes # until all shards have completed recovery - name: Wait for cluster health to return to green - uri: - url: https://localhost:{{ es_http_port }}/_cluster/health + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health method: GET user: elastic password: "{{ es_api_basic_auth_password }}" @@ -67,10 +67,10 @@ - name: Disable shard allocation for the cluster - uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings method: PUT - body: '{{ es_disable_allocation }}' + body: '{{ es_disable_allocation }}' body_format: json user: elastic password: "{{ es_api_basic_auth_password }}" @@ -91,52 +91,52 @@ service: name=elasticsearch enabled=yes state=stopped become: true -# do an "apt-get update", to ensure latest package lists +# do an "apt-get update", to ensure latest package lists - name: apt-get update - apt: - update-cache: yes + apt: + update-cache: yes changed_when: 0 - + # get a list of packages that have updates -- name: get list of pending upgrades +- name: get list of pending upgrades command: apt-get --simulate dist-upgrade - args: + args: warn: false # don't warn us about apt having its own plugin - register: apt_simulate - changed_when: 0 - + register: apt_simulate + changed_when: 0 + # pick out list of pending updates from command output. This essentially # takes the above output from "apt-get --simulate dist-upgrade", and -# pipes it through "cut -f2 -d' ' | sort" +# pipes it through "cut -f2 -d' ' | sort" - name: parse apt-get output to get list of changed packages - set_fact: + set_fact: updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' changed_when: 0 - + # tell user about packages being updated -- name: show pending updates - debug: +- name: show pending updates + debug: var: updates - when: updates.0 is defined - - request manual ack before proceeding with package upgrade -- pause: - when: updates.0 is defined + when: updates.0 is defined + +# request manual ack before proceeding with package upgrade +- pause: + when: updates.0 is defined -if a new kernel is incoming, remove old ones to avoid full /boot +# if a new kernel is incoming, remove old ones to avoid full /boot - name: apt-get autoremove - command: apt-get -y autoremove - args: - warn: false + command: apt-get -y autoremove + args: + warn: false when: '"Inst linux-image-" in apt_simulate.stdout' - changed_when: 0 - -do the actual apt-get dist-upgrade -- name: apt-get dist-upgrade - apt: + changed_when: 0 + +# do the actual apt-get dist-upgrade +- name: apt-get dist-upgrade + apt: upgrade: dist # upgrade all packages to latest version -REBOOT machine after Upgrade +# REBOOT machine after Upgrade - name: check if reboot is required register: reboot_required_file stat: @@ -159,8 +159,8 @@ REBOOT machine after Upgrade become: true - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: - host: "{{ ansible_default_ipv4.address }}" + wait_for: + host: "{{ ansible_default_ipv4.address }}" port: "{{ es_transport_port }}" delay: 30 @@ -173,10 +173,10 @@ REBOOT machine after Upgrade #when: version_found.json.version.number != '{{ es_version }}' - name: Enable shard allocation for the cluster - uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings - method: PUT - body: '{{ es_enable_allocation }}' + uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_enable_allocation }}' body_format: json user: elastic password: "{{ es_api_basic_auth_password }}" @@ -190,8 +190,8 @@ REBOOT machine after Upgrade #when: version_found.json.version.number != es_version - name: Wait for cluster health to return to yellow or green - uri: - url: https://localhost:{{ es_http_port }}/_cluster/health + uri: + url: https://localhost:{{ es_http_port }}/_cluster/health method: GET user: elastic password: "{{ es_api_basic_auth_password }}" From 1b47673b4666eb5343f61d4654ddd1f56e43a4e5 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 18:32:17 +0100 Subject: [PATCH 08/67] Move OS specific vars to global role --- roles/beats/tasks/main.yml | 6 ------ roles/beats/vars/Debian.yml | 4 ---- roles/beats/vars/RedHat.yml | 4 ---- roles/beats/vars/main.yml | 2 -- roles/elasticstack/tasks/main.yml | 8 ++++++++ roles/elasticstack/vars/Debian.yml | 3 +++ roles/elasticstack/vars/RedHat.yml | 3 +++ roles/{logstash => elasticstack}/vars/main.yml | 0 roles/kibana/tasks/main.yml | 6 ------ roles/kibana/vars/Debian.yml | 4 ---- roles/kibana/vars/RedHat.yml | 4 ---- roles/kibana/vars/main.yml | 2 -- roles/logstash/tasks/main.yml | 6 ------ roles/logstash/vars/Debian.yml | 4 ---- roles/logstash/vars/RedHat.yml | 4 ---- 15 files changed, 14 insertions(+), 46 deletions(-) delete mode 100644 roles/beats/vars/Debian.yml delete mode 100644 roles/beats/vars/RedHat.yml delete mode 100644 roles/beats/vars/main.yml create mode 100644 roles/elasticstack/tasks/main.yml create mode 100644 roles/elasticstack/vars/Debian.yml create mode 100644 roles/elasticstack/vars/RedHat.yml rename roles/{logstash => elasticstack}/vars/main.yml (100%) delete mode 100644 roles/kibana/vars/Debian.yml delete mode 100644 roles/kibana/vars/RedHat.yml delete mode 100644 roles/kibana/vars/main.yml delete mode 100644 roles/logstash/vars/Debian.yml delete mode 100644 roles/logstash/vars/RedHat.yml diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index d76634e4..a9ca668f 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -4,12 +4,6 @@ ansible.builtin.import_role: name: netways.elasticstack.elasticstack -- name: Include OS specific vars - ansible.builtin.include_vars: '{{ item }}' - with_first_found: - - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - - '{{ ansible_os_family }}.yml' - - name: Update apt cache. ansible.builtin.apt: update_cache: yes diff --git a/roles/beats/vars/Debian.yml b/roles/beats/vars/Debian.yml deleted file mode 100644 index 1713160e..00000000 --- a/roles/beats/vars/Debian.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/default/elasticsearch -elasticstack_versionseparator: "=" diff --git a/roles/beats/vars/RedHat.yml b/roles/beats/vars/RedHat.yml deleted file mode 100644 index d12aa3b5..00000000 --- a/roles/beats/vars/RedHat.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/sysconfig/elasticsearch -elasticstack_versionseparator: "-" diff --git a/roles/beats/vars/main.yml b/roles/beats/vars/main.yml deleted file mode 100644 index ea359dc8..00000000 --- a/roles/beats/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# vars file for beats diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml new file mode 100644 index 00000000..031d6d22 --- /dev/null +++ b/roles/elasticstack/tasks/main.yml @@ -0,0 +1,8 @@ +--- + +- name: Include OS specific vars + ansible.builtin.include_vars: '{{ item }}' + with_first_found: + - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' + - '{{ ansible_os_family }}.yml' + diff --git a/roles/elasticstack/vars/Debian.yml b/roles/elasticstack/vars/Debian.yml new file mode 100644 index 00000000..3d9e31b5 --- /dev/null +++ b/roles/elasticstack/vars/Debian.yml @@ -0,0 +1,3 @@ +--- + +elasticstack_versionseparator: "=" diff --git a/roles/elasticstack/vars/RedHat.yml b/roles/elasticstack/vars/RedHat.yml new file mode 100644 index 00000000..a8d601fe --- /dev/null +++ b/roles/elasticstack/vars/RedHat.yml @@ -0,0 +1,3 @@ +--- + +elasticstack_versionseparator: "-" diff --git a/roles/logstash/vars/main.yml b/roles/elasticstack/vars/main.yml similarity index 100% rename from roles/logstash/vars/main.yml rename to roles/elasticstack/vars/main.yml diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index b9447251..783b476f 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -11,12 +11,6 @@ changed_when: false when: ansible_os_family == 'Debian' -- name: Include OS specific vars - ansible.builtin.include_vars: '{{ item }}' - with_first_found: - - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - - '{{ ansible_os_family }}.yml' - - name: Set common password for common certificates ansible.builtin.set_fact: kibana_tls_key_passphrase: "{{ elasticstack_cert_pass }}" diff --git a/roles/kibana/vars/Debian.yml b/roles/kibana/vars/Debian.yml deleted file mode 100644 index 1713160e..00000000 --- a/roles/kibana/vars/Debian.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/default/elasticsearch -elasticstack_versionseparator: "=" diff --git a/roles/kibana/vars/RedHat.yml b/roles/kibana/vars/RedHat.yml deleted file mode 100644 index d12aa3b5..00000000 --- a/roles/kibana/vars/RedHat.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/sysconfig/elasticsearch -elasticstack_versionseparator: "-" diff --git a/roles/kibana/vars/main.yml b/roles/kibana/vars/main.yml deleted file mode 100644 index dcb62ac2..00000000 --- a/roles/kibana/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# vars file for kibana diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 9b044b9c..fa8e806c 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -11,12 +11,6 @@ changed_when: false when: ansible_os_family == 'Debian' -- name: Include OS specific vars - ansible.builtin.include_vars: '{{ item }}' - with_first_found: - - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - - '{{ ansible_os_family }}.yml' - - name: Prepare for whole stack roles if used when: - elasticstack_full_stack | bool diff --git a/roles/logstash/vars/Debian.yml b/roles/logstash/vars/Debian.yml deleted file mode 100644 index 1713160e..00000000 --- a/roles/logstash/vars/Debian.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/default/elasticsearch -elasticstack_versionseparator: "=" diff --git a/roles/logstash/vars/RedHat.yml b/roles/logstash/vars/RedHat.yml deleted file mode 100644 index d12aa3b5..00000000 --- a/roles/logstash/vars/RedHat.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -elasticsearch_sysconfig_file: /etc/sysconfig/elasticsearch -elasticstack_versionseparator: "-" From 0ae36bb251c14489cc2af3db0e20f3039a27930a Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 18:56:41 +0100 Subject: [PATCH 09/67] Fix lint --- .../tasks/elasticsearch-rolling-upgrade.yml | 114 ++++++++++-------- roles/elasticsearch/vars/Debian.yml | 1 - roles/elasticsearch/vars/RedHat.yml | 1 - 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 9ea57e73..d47e2c57 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -24,20 +24,23 @@ # tasks: # this first step is a overkill, but here # in case the upgrade was cancelled by user mid playbook run -- name: make sure elasticsearch service is running - service: name=elasticsearch enabled=yes state=started +- name: Make sure elasticsearch service is running + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: started register: response become: true - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: + ansible.builtin.wait_for: host: "{{ ansible_default_ipv4.address }}" port: "{{ es_transport_port }}" delay: 45 - when: response.changed == true + when: response.changed | bool -- name: check current version - uri: +- name: Check current version + ansible.builtin.uri: url: https://localhost:{{ es_http_port }} method: GET user: elastic @@ -48,12 +51,13 @@ delay: 10 - name: Display Current Elasticsearch Version - debug: var=version_found.json.version.number + ansible.builtin.debug: + var: version_found.json.version.number # this step is key!!! Don't restart more nodes # until all shards have completed recovery - name: Wait for cluster health to return to green - uri: + ansible.builtin.uri: url: https://localhost:{{ es_http_port }}/_cluster/health method: GET user: elastic @@ -67,7 +71,7 @@ - name: Disable shard allocation for the cluster - uri: + ansible.builtin.uri: url: https://localhost:{{ es_http_port }}/_cluster/settings method: PUT body: '{{ es_disable_allocation }}' @@ -77,103 +81,117 @@ validate_certs: no #when: version_found.json.version.number != '{{ es_version }}' -- name: stop non essential indexing to speed up shard recovery - uri: +- name: Stop non essential indexing to speed up shard recovery + ansible.builtin.uri: url: https://localhost:{{ es_http_port }}/_flush method: POST user: elastic password: "{{ es_api_basic_auth_password }}" validate_certs: no - ignore_errors: yes + failed_when: false - name: Shutdown elasticsearch service - service: name=elasticsearch enabled=yes state=stopped + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: stopped become: true # do an "apt-get update", to ensure latest package lists -- name: apt-get update - apt: +- name: Apt-get update + ansible.builtin.apt: update-cache: yes - changed_when: 0 + changed_when: false # get a list of packages that have updates -- name: get list of pending upgrades - command: apt-get --simulate dist-upgrade - args: - warn: false # don't warn us about apt having its own plugin +- name: Get list of pending upgrades + ansible.builtin.command: apt-get --simulate dist-upgrade register: apt_simulate - changed_when: 0 + changed_when: false # pick out list of pending updates from command output. This essentially # takes the above output from "apt-get --simulate dist-upgrade", and # pipes it through "cut -f2 -d' ' | sort" -- name: parse apt-get output to get list of changed packages - set_fact: +- name: Parse apt-get output to get list of changed packages + ansible.builtin.set_fact: updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' - changed_when: 0 + changed_when: false # tell user about packages being updated -- name: show pending updates - debug: +- name: Show pending updates + ansible.builtin.debug: var: updates when: updates.0 is defined # request manual ack before proceeding with package upgrade -- pause: +- name: Wait for interaction + ansible.builtin.pause: when: updates.0 is defined # if a new kernel is incoming, remove old ones to avoid full /boot -- name: apt-get autoremove - command: apt-get -y autoremove - args: - warn: false +- name: Apt-get autoremove + ansible.builtin.command: apt-get -y autoremove when: '"Inst linux-image-" in apt_simulate.stdout' - changed_when: 0 + changed_when: false # do the actual apt-get dist-upgrade -- name: apt-get dist-upgrade - apt: +- name: Apt-get dist-upgrade + ansible.builtin.apt: upgrade: dist # upgrade all packages to latest version # REBOOT machine after Upgrade -- name: check if reboot is required - register: reboot_required_file +- name: Check if reboot is required + ansible.builtin.register: reboot_required_file stat: path: /var/run/reboot-required -- name: restart machine - reboot: +- name: Restart machine + ansible.builtin.reboot: msg: "Reboot initiated by Ansible to update system libs/kernel as needed" - when: reboot_required_file.stat.exists == true + when: reboot_required_file.stat.exists | bool -- name: waiting for machine to come back - wait_for_connection: +- name: Waiting for machine to come back + ansible.builtin.wait_for_connection: delay: 10 connect_timeout: 300 - when: reboot_required_file.stat.exists == true + when: reboot_required_file.stat.exists | bool - name: Start elasticsearch - service: name=elasticsearch enabled=yes state=started + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: started #when: version_found.json.version.number != '{{ es_version }}' become: true - name: Wait for elasticsearch node to come back up if it was stopped - wait_for: + ansible.builtin.wait_for: host: "{{ ansible_default_ipv4.address }}" port: "{{ es_transport_port }}" delay: 30 -- name: Confirm the node joins the cluster - shell: "curl -k -u elastic:{{ es_api_basic_auth_password }} -s -m 2 'https://localhost:9200/_cat/nodes?h=name' | grep -E '^{{ ansible_fqdn }}$'" +- name: Confirm the node joins the cluster # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + curl + -k + -u elastic:{{ es_api_basic_auth_password }} + -s + -m 2 + 'https://localhost:9200/_cat/nodes?h=name' + | grep + -E + '^{{ ansible_fqdn }}$' register: result until: result.rc == 0 retries: 200 delay: 3 + changed_when: false #when: version_found.json.version.number != '{{ es_version }}' - name: Enable shard allocation for the cluster - uri: + ansible.builtin.uri: url: https://localhost:{{ es_http_port }}/_cluster/settings method: PUT body: '{{ es_enable_allocation }}' @@ -190,7 +208,7 @@ #when: version_found.json.version.number != es_version - name: Wait for cluster health to return to yellow or green - uri: + ansible.builtin.uri: url: https://localhost:{{ es_http_port }}/_cluster/health method: GET user: elastic diff --git a/roles/elasticsearch/vars/Debian.yml b/roles/elasticsearch/vars/Debian.yml index 1713160e..bb0878c1 100644 --- a/roles/elasticsearch/vars/Debian.yml +++ b/roles/elasticsearch/vars/Debian.yml @@ -1,4 +1,3 @@ --- elasticsearch_sysconfig_file: /etc/default/elasticsearch -elasticstack_versionseparator: "=" diff --git a/roles/elasticsearch/vars/RedHat.yml b/roles/elasticsearch/vars/RedHat.yml index d12aa3b5..f0dbc02a 100644 --- a/roles/elasticsearch/vars/RedHat.yml +++ b/roles/elasticsearch/vars/RedHat.yml @@ -1,4 +1,3 @@ --- elasticsearch_sysconfig_file: /etc/sysconfig/elasticsearch -elasticstack_versionseparator: "-" From 839e60344b04a9065606910085ee2daadf6fe59e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 19:12:01 +0100 Subject: [PATCH 10/67] Set default for elasticstack_ca_will_expire_soon --- roles/elasticstack/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index d681d146..f93fc91f 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -13,6 +13,7 @@ elasticstack_release: 8 elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic +elasticstack_ca_will_expire_soon: false # only for debugging # From 91d588ba282225bdce8f99d430adc8dd854b3c2d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 19:23:48 +0100 Subject: [PATCH 11/67] Move elasticsearch_ca naming to global role --- roles/beats/tasks/main.yml | 12 ------------ .../elasticsearch/tasks/elasticsearch-security.yml | 9 --------- roles/elasticstack/tasks/main.yml | 6 ++++++ roles/kibana/tasks/kibana-security.yml | 11 ----------- roles/logstash/tasks/logstash-security.yml | 13 ------------- 5 files changed, 6 insertions(+), 45 deletions(-) diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index a9ca668f..319c9eb8 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -23,18 +23,6 @@ - elasticstack_variant != "oss" - not elasticstack_override_beats_tls | bool - - name: Set elasticstack_ca variable if not already done by user - ansible.builtin.set_fact: - elasticstack_ca: "{{ groups['elasticsearch'][0] }}" - when: - - beats_security | bool - - elasticstack_ca is undefined - - groups['elasticsearch'] is defined - tags: - - certificates - - renew_ca - - renew_beats_cert - - name: Set beats_ca_dir if whole stack is used ansible.builtin.set_fact: beats_ca_dir: "/etc/beats/certs" diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 48bcb2aa..f4051651 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -12,15 +12,6 @@ - renew_kibana_cert - renew_es_cert -- name: Set elasticstack_ca variable if not already done by user - ansible.builtin.set_fact: - elasticstack_ca: "{{ groups['elasticsearch'][0] }}" - when: elasticstack_ca is undefined - tags: - - certificates - - renew_ca - - renew_es_cert - - name: Ensure ca exists ansible.builtin.stat: path: "{{ elasticstack_ca_dir }}/elastic-stack-ca.p12" diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index 031d6d22..6a4afdef 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -6,3 +6,9 @@ - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - '{{ ansible_os_family }}.yml' +- name: Set elasticstack_ca variable if not already done by user + ansible.builtin.set_fact: + elasticstack_ca: "{{ groups['elasticsearch'][0] }}" + when: + - elasticstack_ca is undefined + - groups['elasticsearch'][0] is defined diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index 4bb14fbd..c19b00ed 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -11,17 +11,6 @@ - renew_ca - renew_kibana_cert -- name: Set elasticstack_ca variable if not already done by user - ansible.builtin.set_fact: - elasticstack_ca: "{{ groups['elasticsearch'][0] }}" - when: - - elasticstack_ca is undefined - - groups['elasticsearch'] is defined - tags: - - certificates - - renew_ca - - renew_kibana_cert - - name: Ensure kibana certificate exists ansible.builtin.stat: path: "/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12" diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 5a93e9d9..7721cbe7 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -11,19 +11,6 @@ - renew_ca - renew_logstash_cert -- name: Set elasticstack_ca variable if not already done by user - ansible.builtin.set_fact: - elasticstack_ca: "{{ groups['elasticsearch'][0] }}" - when: - - elasticstack_ca is undefined - - groups['elasticsearch'] is defined - tags: - - certificates - - configuration - - logstash_configuration - - renew_ca - - renew_logstash_cert - - name: Ensure logstash certificate exists ansible.builtin.stat: path: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12" From e7a4f357d2e01419332231bb14fef7c5eba4712d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 19:30:48 +0100 Subject: [PATCH 12/67] Set name of Elasticsearch CA --- roles/elasticstack/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index f93fc91f..ab66686f 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -2,6 +2,7 @@ elasticstack_beats_port: 5044 elasticstack_ca_dir: /opt/es-ca +elasticstack_ca_name: "CN=Elastic Certificate Tool Autogenerated CA" elasticstack_ca_pass: PleaseChangeMe elasticstack_elasticsearch_http_port: 9200 elasticstack_enable_repos: true From 4f07aeb25be2617b48338e170b3f3775d44a7a83 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 19:40:37 +0100 Subject: [PATCH 13/67] Fix defaults for global role --- roles/elasticstack/defaults/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index ab66686f..4c809a72 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -1,9 +1,11 @@ --- - elasticstack_beats_port: 5044 elasticstack_ca_dir: /opt/es-ca +elasticstack_ca_expiration_buffer: 30 elasticstack_ca_name: "CN=Elastic Certificate Tool Autogenerated CA" elasticstack_ca_pass: PleaseChangeMe +elasticstack_ca_validity_period: 1095 +elasticstack_ca_will_expire_soon: false elasticstack_elasticsearch_http_port: 9200 elasticstack_enable_repos: true elasticstack_full_stack: true @@ -14,8 +16,6 @@ elasticstack_release: 8 elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic -elasticstack_ca_will_expire_soon: false -# only for debugging -# + # for debugging only elasticstack_no_log: true From 19367c9a75f74d598c25b00a290f3d874398d2e3 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 16 Jan 2024 19:54:19 +0100 Subject: [PATCH 14/67] Skip global role if it ran already --- roles/beats/tasks/main.yml | 2 ++ roles/elasticsearch/tasks/main.yml | 2 ++ roles/elasticstack/tasks/main.yml | 4 ++++ roles/kibana/tasks/main.yml | 2 ++ roles/logstash/tasks/main.yml | 2 ++ roles/repos/tasks/main.yml | 2 ++ 6 files changed, 14 insertions(+) diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 319c9eb8..19731745 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -3,6 +3,8 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack + when: + - not elasticstack_globals_set | bool - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 6dce5021..ff7fa678 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -3,6 +3,8 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack + when: + - not elasticstack_globals_set | bool - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index 6a4afdef..af336369 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -12,3 +12,7 @@ when: - elasticstack_ca is undefined - groups['elasticsearch'][0] is defined + +- name: Set elasticstack_globals_set for other roles to skip this role + ansible.builtin.set_fact: + elasticstack_globals_set: true diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 783b476f..4802f1bb 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -3,6 +3,8 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack + when: + - not elasticstack_globals_set | bool - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index fa8e806c..160755e8 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -3,6 +3,8 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack + when: + - not elasticstack_globals_set | bool - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index 340b4c43..c060b41b 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -3,6 +3,8 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack + when: + - not elasticstack_globals_set | bool - name: Check for versions ansible.builtin.fail: From 61ff0a131baa06ce67fb0cc1932b4918f8bb1732 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 11:27:55 +0100 Subject: [PATCH 15/67] Stupid logical mistake --- roles/beats/tasks/main.yml | 6 ++++-- roles/elasticsearch/tasks/main.yml | 6 ++++-- roles/kibana/tasks/main.yml | 6 ++++-- roles/logstash/tasks/main.yml | 6 ++++-- roles/repos/tasks/main.yml | 6 ++++-- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 19731745..827c3ba5 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -3,8 +3,10 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: - - not elasticstack_globals_set | bool + when: > + not elasticstack_globals_set | bool + or + elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index ff7fa678..daf85d35 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -3,8 +3,10 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: - - not elasticstack_globals_set | bool + when: > + not elasticstack_globals_set | bool + or + elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 4802f1bb..8f9a04db 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -3,8 +3,10 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: - - not elasticstack_globals_set | bool + when: > + not elasticstack_globals_set | bool + or + elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 160755e8..ef0a70e4 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -3,8 +3,10 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: - - not elasticstack_globals_set | bool + when: > + not elasticstack_globals_set | bool + or + elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index c060b41b..c4812ab1 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -3,8 +3,10 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: - - not elasticstack_globals_set | bool + when: > + not elasticstack_globals_set | bool + or + elasticstack_globals_set is undefined - name: Check for versions ansible.builtin.fail: From 931d6d1ecfac86a78f579c53a1ea303ede62c3b4 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 15:02:16 +0100 Subject: [PATCH 16/67] Set version of ES on elasticstack_ca as target for all components --- .../tasks/elasticsearch-version.yml | 32 +++++++++++++++++++ roles/elasticsearch/tasks/main.yml | 4 +++ .../tasks/elasticstack_versions.yml | 9 ++++++ roles/elasticstack/tasks/main.yml | 3 ++ 4 files changed, 48 insertions(+) create mode 100644 roles/elasticsearch/tasks/elasticsearch-version.yml create mode 100644 roles/elasticstack/tasks/elasticstack_versions.yml diff --git a/roles/elasticsearch/tasks/elasticsearch-version.yml b/roles/elasticsearch/tasks/elasticsearch-version.yml new file mode 100644 index 00000000..1506c16f --- /dev/null +++ b/roles/elasticsearch/tasks/elasticsearch-version.yml @@ -0,0 +1,32 @@ +--- + +- name: Check for Elasticsearch package (RedHat) # noqa: risky-shell-pipe + shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + rpm -q elasticsearch | + cut -d- -f2 + register: elasticsearch_version_output + faiiled_when: false + changed_when: false + when: + - ansible_os_family == "RedHat" + +- name: Check for Elasticsearch package (Debian) # noqa: risky-shell-pipe + shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + dpkg -s elasticsearch | + grep Version | + cut -d: -f2 | + cut -d- -f1 | + tr -d [:space:] + register: elasticsearch_version_output + faiiled_when: false + changed_when: false + when: + - ansible_os_family == "Debian" + +- name: Set friendly name of version variable + set_fact: + elasticsearch_current_version: "{{ elasticsearch_version_output.stdout }}" + when: + - elasticsearch_version_output is defined diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index daf85d35..7c66824e 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,5 +1,9 @@ --- +- name: Fetch current Elasticsearch version + ansible.builtin.import_tasks: elasticsearch-version.yml + when: elasitcsearch_current_version is undefined + - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack diff --git a/roles/elasticstack/tasks/elasticstack_versions.yml b/roles/elasticstack/tasks/elasticstack_versions.yml new file mode 100644 index 00000000..b3811589 --- /dev/null +++ b/roles/elasticstack/tasks/elasticstack_versions.yml @@ -0,0 +1,9 @@ +--- + +- name: Set target version to Elasticsearch on CA host + set_fact: + elasticstack_version: "{{ elasticsearch_current_version }}" + delegate_to: {{ elasticstack_ca }} + when: + - elasticsearch_current_version is defined + - if elasticstack_version is undefined diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index af336369..80df6624 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -13,6 +13,9 @@ - elasticstack_ca is undefined - groups['elasticsearch'][0] is defined +- name: Set versions for components + ansible.builtin.import_tasks: elasticstack_versions.yml + - name: Set elasticstack_globals_set for other roles to skip this role ansible.builtin.set_fact: elasticstack_globals_set: true From c162010c73cf9d0cb33fc824d0891fd32726629c Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 15:06:24 +0100 Subject: [PATCH 17/67] Rename file for naming scheme --- .../{elasticstack_versions.yml => elasticstack-versions.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/elasticstack/tasks/{elasticstack_versions.yml => elasticstack-versions.yml} (100%) diff --git a/roles/elasticstack/tasks/elasticstack_versions.yml b/roles/elasticstack/tasks/elasticstack-versions.yml similarity index 100% rename from roles/elasticstack/tasks/elasticstack_versions.yml rename to roles/elasticstack/tasks/elasticstack-versions.yml From 2b365d0e5ef07b6307880a5bfc2b59b36dd72631 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 15:07:12 +0100 Subject: [PATCH 18/67] Lint --- roles/elasticsearch/tasks/elasticsearch-version.yml | 10 +++++----- roles/elasticstack/tasks/elasticstack-versions.yml | 4 ++-- roles/elasticstack/tasks/main.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-version.yml b/roles/elasticsearch/tasks/elasticsearch-version.yml index 1506c16f..1a6c9a8a 100644 --- a/roles/elasticsearch/tasks/elasticsearch-version.yml +++ b/roles/elasticsearch/tasks/elasticsearch-version.yml @@ -1,18 +1,18 @@ --- - name: Check for Elasticsearch package (RedHat) # noqa: risky-shell-pipe - shell: > + ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; rpm -q elasticsearch | cut -d- -f2 register: elasticsearch_version_output - faiiled_when: false + failed_when: false changed_when: false when: - ansible_os_family == "RedHat" - name: Check for Elasticsearch package (Debian) # noqa: risky-shell-pipe - shell: > + ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; dpkg -s elasticsearch | grep Version | @@ -20,13 +20,13 @@ cut -d- -f1 | tr -d [:space:] register: elasticsearch_version_output - faiiled_when: false + failed_when: false changed_when: false when: - ansible_os_family == "Debian" - name: Set friendly name of version variable - set_fact: + ansible.builtin.set_fact: elasticsearch_current_version: "{{ elasticsearch_version_output.stdout }}" when: - elasticsearch_version_output is defined diff --git a/roles/elasticstack/tasks/elasticstack-versions.yml b/roles/elasticstack/tasks/elasticstack-versions.yml index b3811589..ce59227c 100644 --- a/roles/elasticstack/tasks/elasticstack-versions.yml +++ b/roles/elasticstack/tasks/elasticstack-versions.yml @@ -1,9 +1,9 @@ --- - name: Set target version to Elasticsearch on CA host - set_fact: + ansible.builtin.set_fact: elasticstack_version: "{{ elasticsearch_current_version }}" - delegate_to: {{ elasticstack_ca }} + delegate_to: "{{ elasticstack_ca }}" when: - elasticsearch_current_version is defined - if elasticstack_version is undefined diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index 80df6624..d0f25bb7 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -14,7 +14,7 @@ - groups['elasticsearch'][0] is defined - name: Set versions for components - ansible.builtin.import_tasks: elasticstack_versions.yml + ansible.builtin.import_tasks: elasticstack-versions.yml - name: Set elasticstack_globals_set for other roles to skip this role ansible.builtin.set_fact: From ec218de5edf9b5bf8447b0756f1065bcf1f65a6e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 15:29:28 +0100 Subject: [PATCH 19/67] Specify subvariable for checking --- roles/elasticsearch/tasks/elasticsearch-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-version.yml b/roles/elasticsearch/tasks/elasticsearch-version.yml index 1a6c9a8a..e80bb181 100644 --- a/roles/elasticsearch/tasks/elasticsearch-version.yml +++ b/roles/elasticsearch/tasks/elasticsearch-version.yml @@ -29,4 +29,4 @@ ansible.builtin.set_fact: elasticsearch_current_version: "{{ elasticsearch_version_output.stdout }}" when: - - elasticsearch_version_output is defined + - elasticsearch_version_output.stdout is defined From 7bebba7ce9c0e3a67d3ca841745b70b701c5b95d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 15:56:37 +0100 Subject: [PATCH 20/67] Remove reboot part of upgrade playbook --- .../tasks/elasticsearch-rolling-upgrade.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index d47e2c57..3c23a3e1 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -140,23 +140,6 @@ ansible.builtin.apt: upgrade: dist # upgrade all packages to latest version -# REBOOT machine after Upgrade -- name: Check if reboot is required - ansible.builtin.register: reboot_required_file - stat: - path: /var/run/reboot-required - -- name: Restart machine - ansible.builtin.reboot: - msg: "Reboot initiated by Ansible to update system libs/kernel as needed" - when: reboot_required_file.stat.exists | bool - -- name: Waiting for machine to come back - ansible.builtin.wait_for_connection: - delay: 10 - connect_timeout: 300 - when: reboot_required_file.stat.exists | bool - - name: Start elasticsearch ansible.builtin.service: name: elasticsearch From e49d50bfcacf3825f1934372d613a18c8d7e2b10 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 17:45:59 +0100 Subject: [PATCH 21/67] Add note about version detection --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b401cb14..242b7b6e 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ The variable `elasticstack_no_log` can be set to `false` if you want to see the ### Versioning -*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest. (default: none). +*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest. (default: none). If you already have an installation of Elastic Stack, this collection will query the version of Elasticsearch on the CA host and use it for all further installations in the same setup. (Only if you run the `elasticsearch` role before all others) *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) From ff13c9ef98acbe8c513579cfa3462d310941653e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 17:52:18 +0100 Subject: [PATCH 22/67] Remove obsolete vars directory from repos --- roles/repos/vars/main.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 roles/repos/vars/main.yml diff --git a/roles/repos/vars/main.yml b/roles/repos/vars/main.yml deleted file mode 100644 index ed97d539..00000000 --- a/roles/repos/vars/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- From 1cdb58391b73ca3f8366c9c6f54837e7f371de67 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 18:30:49 +0100 Subject: [PATCH 23/67] Use module instead of shell --- roles/elasticsearch/tasks/main.yml | 6 +++--- roles/elasticstack/tasks/elasticstack-versions.yml | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 7c66824e..a9c7ecc0 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,8 +1,8 @@ --- -- name: Fetch current Elasticsearch version - ansible.builtin.import_tasks: elasticsearch-version.yml - when: elasitcsearch_current_version is undefined +#- name: Fetch current Elasticsearch version +# ansible.builtin.import_tasks: elasticsearch-version.yml +# when: elasitcsearch_current_version is undefined - name: Include global role ansible.builtin.import_role: diff --git a/roles/elasticstack/tasks/elasticstack-versions.yml b/roles/elasticstack/tasks/elasticstack-versions.yml index ce59227c..e58dca42 100644 --- a/roles/elasticstack/tasks/elasticstack-versions.yml +++ b/roles/elasticstack/tasks/elasticstack-versions.yml @@ -1,9 +1,13 @@ --- +- name: Gather package facts + ansible.builtin.package_facts: + manager: auto + - name: Set target version to Elasticsearch on CA host ansible.builtin.set_fact: - elasticstack_version: "{{ elasticsearch_current_version }}" + elasticstack_version: "{{ ansible_facts.packages['elasticsearch'].version }}" delegate_to: "{{ elasticstack_ca }}" when: - - elasticsearch_current_version is defined + - ansible_facts.packages['elasticsearch'].version is defined - if elasticstack_version is undefined From b94c2cc504e685162a0499d4de8e2fd707789d4d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 18:50:39 +0100 Subject: [PATCH 24/67] Remove overly complicated version check --- .../tasks/elasticsearch-version.yml | 32 ------------------- roles/elasticsearch/tasks/main.yml | 4 --- 2 files changed, 36 deletions(-) delete mode 100644 roles/elasticsearch/tasks/elasticsearch-version.yml diff --git a/roles/elasticsearch/tasks/elasticsearch-version.yml b/roles/elasticsearch/tasks/elasticsearch-version.yml deleted file mode 100644 index e80bb181..00000000 --- a/roles/elasticsearch/tasks/elasticsearch-version.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- - -- name: Check for Elasticsearch package (RedHat) # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - rpm -q elasticsearch | - cut -d- -f2 - register: elasticsearch_version_output - failed_when: false - changed_when: false - when: - - ansible_os_family == "RedHat" - -- name: Check for Elasticsearch package (Debian) # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - dpkg -s elasticsearch | - grep Version | - cut -d: -f2 | - cut -d- -f1 | - tr -d [:space:] - register: elasticsearch_version_output - failed_when: false - changed_when: false - when: - - ansible_os_family == "Debian" - -- name: Set friendly name of version variable - ansible.builtin.set_fact: - elasticsearch_current_version: "{{ elasticsearch_version_output.stdout }}" - when: - - elasticsearch_version_output.stdout is defined diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index a9c7ecc0..daf85d35 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,9 +1,5 @@ --- -#- name: Fetch current Elasticsearch version -# ansible.builtin.import_tasks: elasticsearch-version.yml -# when: elasitcsearch_current_version is undefined - - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack From 00a4fcf67b6af15f50a261dc94ac768ce2396d3c Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Jan 2024 18:56:16 +0100 Subject: [PATCH 25/67] Call upgrade taskfile when new version > current version --- roles/elasticsearch/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index daf85d35..c8afa3f3 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -129,6 +129,12 @@ replace(' ', '') }} +- name: Update Elasitcsearch if needed + ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml + when: + - elasticsearch_version is defined + - elasticsearch_version is version( ansible_package_facts['elasticsearch'].version, '>') + - name: Install Elasticsearch - rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" From 00179878ce4bb2c7dad2d23d786cdc87702af7a8 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 10:38:04 +0100 Subject: [PATCH 26/67] Remove upgrades for other packages This collection focuses on Elastic Stack. So we remove all code that deals with other packages. --- .../tasks/elasticsearch-rolling-upgrade.yml | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 3c23a3e1..7953bdda 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -98,43 +98,6 @@ state: stopped become: true -# do an "apt-get update", to ensure latest package lists -- name: Apt-get update - ansible.builtin.apt: - update-cache: yes - changed_when: false - -# get a list of packages that have updates -- name: Get list of pending upgrades - ansible.builtin.command: apt-get --simulate dist-upgrade - register: apt_simulate - changed_when: false - -# pick out list of pending updates from command output. This essentially -# takes the above output from "apt-get --simulate dist-upgrade", and -# pipes it through "cut -f2 -d' ' | sort" -- name: Parse apt-get output to get list of changed packages - ansible.builtin.set_fact: - updates: '{{ apt_simulate.stdout_lines | select("match", "^Inst ") | list | splitpart(1, " ") | list | sort }}' - changed_when: false - -# tell user about packages being updated -- name: Show pending updates - ansible.builtin.debug: - var: updates - when: updates.0 is defined - -# request manual ack before proceeding with package upgrade -- name: Wait for interaction - ansible.builtin.pause: - when: updates.0 is defined - -# if a new kernel is incoming, remove old ones to avoid full /boot -- name: Apt-get autoremove - ansible.builtin.command: apt-get -y autoremove - when: '"Inst linux-image-" in apt_simulate.stdout' - changed_when: false - # do the actual apt-get dist-upgrade - name: Apt-get dist-upgrade ansible.builtin.apt: From 464b663912ca1b91741b79283f51d026e4cc29f7 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 10:45:39 +0100 Subject: [PATCH 27/67] Upgrade shutdown nodes right away --- .../tasks/elasticsearch-rolling-upgrade.yml | 130 ++++++++---------- 1 file changed, 57 insertions(+), 73 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 7953bdda..9a995ba0 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -22,81 +22,65 @@ # es_version: '8.5.3' # # tasks: - # this first step is a overkill, but here - # in case the upgrade was cancelled by user mid playbook run -- name: Make sure elasticsearch service is running - ansible.builtin.service: - name: elasticsearch - enabled: yes - state: started - register: response - become: true - -- name: Wait for elasticsearch node to come back up if it was stopped - ansible.builtin.wait_for: - host: "{{ ansible_default_ipv4.address }}" - port: "{{ es_transport_port }}" - delay: 45 - when: response.changed | bool - -- name: Check current version - ansible.builtin.uri: - url: https://localhost:{{ es_http_port }} - method: GET - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: version_found - retries: 10 - delay: 10 - -- name: Display Current Elasticsearch Version - ansible.builtin.debug: - var: version_found.json.version.number - - # this step is key!!! Don't restart more nodes - # until all shards have completed recovery -- name: Wait for cluster health to return to green - ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/health - method: GET - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - register: response - until: "response.json.status == 'green'" - retries: 50 - delay: 30 - # when: version_found.json.version.number != '{{ es_version }}' - -- name: Disable shard allocation for the cluster - ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings - method: PUT - body: '{{ es_disable_allocation }}' - body_format: json - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - #when: version_found.json.version.number != '{{ es_version }}' - -- name: Stop non essential indexing to speed up shard recovery - ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_flush - method: POST - user: elastic - password: "{{ es_api_basic_auth_password }}" - validate_certs: no - failed_when: false - - -- name: Shutdown elasticsearch service - ansible.builtin.service: +- name: Check for running Elasticsearch service + ansible.builtin.systemd: name: elasticsearch - enabled: yes - state: stopped - become: true + register: elasticsearch_running + +- name: Be careful about upgrade when Elasticsearch is running + when: + - elasticsearch_running.status.ActiveState == "active" + block: + + - name: Wait for elasticsearch node to come back up if it was stopped + ansible.builtin.wait_for: + host: "{{ ansible_default_ipv4.address }}" + port: "{{ es_transport_port }}" + delay: 45 + when: response.changed | bool + + # this step is key!!! Don't restart more nodes + # until all shards have completed recovery + - name: Wait for cluster health to return to green + ansible.builtin.uri: + url: https://localhost:{{ es_http_port }}/_cluster/health + method: GET + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + register: response + until: "response.json.status == 'green'" + retries: 50 + delay: 30 + # when: version_found.json.version.number != '{{ es_version }}' + + - name: Disable shard allocation for the cluster + ansible.builtin.uri: + url: https://localhost:{{ es_http_port }}/_cluster/settings + method: PUT + body: '{{ es_disable_allocation }}' + body_format: json + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + #when: version_found.json.version.number != '{{ es_version }}' + + - name: Stop non essential indexing to speed up shard recovery + ansible.builtin.uri: + url: https://localhost:{{ es_http_port }}/_flush + method: POST + user: elastic + password: "{{ es_api_basic_auth_password }}" + validate_certs: no + failed_when: false + + - name: Shutdown elasticsearch service + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: stopped + become: true # do the actual apt-get dist-upgrade - name: Apt-get dist-upgrade From 242276d38660bca5392b33039c139662dad5caf7 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 16:06:38 +0100 Subject: [PATCH 28/67] Only start Elasticsearch if it was running before --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 9a995ba0..01a0e806 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -94,6 +94,8 @@ state: started #when: version_found.json.version.number != '{{ es_version }}' become: true + when: + - elasticsearch_running.status.ActiveState == "active" - name: Wait for elasticsearch node to come back up if it was stopped ansible.builtin.wait_for: From 1f463da83167df6a01e8a3bf73a38fa63cb6131f Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 16:07:20 +0100 Subject: [PATCH 29/67] Remove redundant become --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 01a0e806..8ce8d2ad 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -80,7 +80,6 @@ name: elasticsearch enabled: yes state: stopped - become: true # do the actual apt-get dist-upgrade - name: Apt-get dist-upgrade @@ -93,7 +92,6 @@ enabled: yes state: started #when: version_found.json.version.number != '{{ es_version }}' - become: true when: - elasticsearch_running.status.ActiveState == "active" From b248e0894d6a844868263b8a279db8644af135c5 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 17:08:58 +0100 Subject: [PATCH 30/67] Restrict execution of upgrade playbook to one at a time --- .../tasks/elasticsearch-rolling-upgrade.yml | 56 ++++++------------- roles/elasticsearch/tasks/main.yml | 1 + 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 8ce8d2ad..8a844fe3 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -7,21 +7,6 @@ # latest tested with Ansible 2.9 and later --- -#- name: Elasticsearch rolling upgrade -# hosts: elasticsearch_{{ env }} -# become: true -# serial: 1 -# vars_files: -# - vars/elasticsearch/elasticsearch-{{ env }}_secrets.yml -# vars: -# es_disable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' -# es_enable_allocation: '{ "persistent": { "cluster.routing.allocation.enable": null }}' -# es_http_port: 9200 -# es_transport_port: 9300 -# #desired version to upgrade to: 7.10.2 -# es_version: '8.5.3' -# -# tasks: - name: Check for running Elasticsearch service ansible.builtin.systemd: @@ -35,8 +20,8 @@ - name: Wait for elasticsearch node to come back up if it was stopped ansible.builtin.wait_for: - host: "{{ ansible_default_ipv4.address }}" - port: "{{ es_transport_port }}" + host: "localhost" + port: "{{ elasticstack_elasticsearch_http_port }}" delay: 45 when: response.changed | bool @@ -44,34 +29,32 @@ # until all shards have completed recovery - name: Wait for cluster health to return to green ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/health + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" method: GET user: elastic - password: "{{ es_api_basic_auth_password }}" + password: "{{ elasticstack_password.stdout }}" validate_certs: no register: response until: "response.json.status == 'green'" retries: 50 delay: 30 - # when: version_found.json.version.number != '{{ es_version }}' - name: Disable shard allocation for the cluster ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" method: PUT - body: '{{ es_disable_allocation }}' + body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' body_format: json user: elastic - password: "{{ es_api_basic_auth_password }}" + password: "{{ elasticstack_password.stdout }}" validate_certs: no - #when: version_found.json.version.number != '{{ es_version }}' - name: Stop non essential indexing to speed up shard recovery ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_flush + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_flush" method: POST user: elastic - password: "{{ es_api_basic_auth_password }}" + password: "{{ elasticstack_password.stdout }}" validate_certs: no failed_when: false @@ -91,14 +74,13 @@ name: elasticsearch enabled: yes state: started - #when: version_found.json.version.number != '{{ es_version }}' when: - elasticsearch_running.status.ActiveState == "active" - name: Wait for elasticsearch node to come back up if it was stopped ansible.builtin.wait_for: - host: "{{ ansible_default_ipv4.address }}" - port: "{{ es_transport_port }}" + host: "localhost" + port: "{{ elasticstack_elasticsearch_http_port }}" delay: 30 - name: Confirm the node joins the cluster # noqa: risky-shell-pipe @@ -106,10 +88,10 @@ if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; curl -k - -u elastic:{{ es_api_basic_auth_password }} + -u elastic:{{ elasticstack_password.stdout }} -s -m 2 - 'https://localhost:9200/_cat/nodes?h=name' + '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' | grep -E '^{{ ansible_fqdn }}$' @@ -118,16 +100,15 @@ retries: 200 delay: 3 changed_when: false - #when: version_found.json.version.number != '{{ es_version }}' - name: Enable shard allocation for the cluster ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/settings + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" method: PUT - body: '{{ es_enable_allocation }}' + body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' body_format: json user: elastic - password: "{{ es_api_basic_auth_password }}" + password: "{{ elasticstack_password.stdout }}" validate_certs: no register: response # next line is boolean not string, so no quotes around true @@ -135,14 +116,13 @@ until: "response.json.acknowledged == true" retries: 5 delay: 30 - #when: version_found.json.version.number != es_version - name: Wait for cluster health to return to yellow or green ansible.builtin.uri: - url: https://localhost:{{ es_http_port }}/_cluster/health + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" method: GET user: elastic - password: "{{ es_api_basic_auth_password }}" + password: "{{ elasticstack_password.stdout }}" validate_certs: no register: response until: "response.json.status == 'yellow' or response.json.status == 'green'" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index c8afa3f3..526ed880 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -131,6 +131,7 @@ - name: Update Elasitcsearch if needed ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml + throttle: 1 when: - elasticsearch_version is defined - elasticsearch_version is version( ansible_package_facts['elasticsearch'].version, '>') From b215d31f6ba8c1a099affeb53200f3317bfb7462 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 18 Jan 2024 17:27:19 +0100 Subject: [PATCH 31/67] Fetch elastic password for upgrades --- roles/elasticsearch/tasks/main.yml | 2 ++ .../tasks/elasticstack-passwords.yml | 18 ++++++++++++++++++ roles/elasticstack/tasks/main.yml | 5 +++++ 3 files changed, 25 insertions(+) create mode 100644 roles/elasticstack/tasks/elasticstack-passwords.yml diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 526ed880..fd72984f 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -134,6 +134,8 @@ throttle: 1 when: - elasticsearch_version is defined + - ansible_package_facts['elasticsearch'].version is defined + - elasticstack_password.stdout is defined - elasticsearch_version is version( ansible_package_facts['elasticsearch'].version, '>') - name: Install Elasticsearch - rpm diff --git a/roles/elasticstack/tasks/elasticstack-passwords.yml b/roles/elasticstack/tasks/elasticstack-passwords.yml new file mode 100644 index 00000000..1f2bc2a1 --- /dev/null +++ b/roles/elasticstack/tasks/elasticstack-passwords.yml @@ -0,0 +1,18 @@ +--- + +- name: Check for passwords being set + ansible.builtin.stat: + path: "{{ elasticstack_initial_passwords }}" + delegate_to: "{{ elasticstack_ca }}" + register: elasticsearch_passwords_file + +- name: Fetch Elastic password # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + grep "PASSWORD elastic" {{ elasticstack_initial_passwords }} | + awk {' print $4 '} + register: elasticstack_password + changed_when: false + no_log: "{{ elasticstack_no_log }}" + delegate_to: "{{ elasticstack_ca }}" + when: elasticsearch_passwords_file.stat.exists | bool diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index d0f25bb7..2082afd2 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -16,6 +16,11 @@ - name: Set versions for components ansible.builtin.import_tasks: elasticstack-versions.yml +- name: Fetch passwords if passwords are initialized + ansible.builtin.import_tasks: elasticstack-passwords.yml + when: + - elasticstack_password.stdout is undefined + - name: Set elasticstack_globals_set for other roles to skip this role ansible.builtin.set_fact: elasticstack_globals_set: true From 52440783e7e1ff98f4f1f850788953930b6f3b36 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 19 Jan 2024 12:22:51 +0100 Subject: [PATCH 32/67] Replace package installation with more general one --- .../tasks/elasticsearch-rolling-upgrade.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 8a844fe3..dabf1e00 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -64,10 +64,19 @@ enabled: yes state: stopped -# do the actual apt-get dist-upgrade -- name: Apt-get dist-upgrade - ansible.builtin.apt: - upgrade: dist # upgrade all packages to latest version +- name: Install Elasticsearch - rpm + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + enablerepo: + - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + when: + - ansible_os_family == "RedHat" + +- name: Install Elasticsearch - deb + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + when: + - ansible_os_family == "Debian" - name: Start elasticsearch ansible.builtin.service: From c659dee7a3c1b3691fab6c031db8904436edb53b Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 19 Jan 2024 19:11:45 +0100 Subject: [PATCH 33/67] Fix some errors in variable names --- roles/beats/tasks/main.yml | 4 ---- roles/elasticsearch/tasks/main.yml | 10 +++------- roles/elasticstack/tasks/elasticstack-versions.yml | 6 +++--- roles/elasticstack/tasks/main.yml | 2 -- roles/kibana/tasks/main.yml | 4 ---- roles/logstash/tasks/main.yml | 4 ---- roles/repos/tasks/main.yml | 4 ---- 7 files changed, 6 insertions(+), 28 deletions(-) diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 827c3ba5..319c9eb8 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -3,10 +3,6 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: > - not elasticstack_globals_set | bool - or - elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index fd72984f..d365c1b6 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -3,10 +3,6 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: > - not elasticstack_globals_set | bool - or - elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: @@ -133,10 +129,10 @@ ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml throttle: 1 when: - - elasticsearch_version is defined - - ansible_package_facts['elasticsearch'].version is defined + - elasticsstack_version is defined + - ansible_facts.packages['elasticsearch'][0].version is defined - elasticstack_password.stdout is defined - - elasticsearch_version is version( ansible_package_facts['elasticsearch'].version, '>') + - elasticstack_version is version( ansible_package_facts['elasticsearch'][0].version, '>') - name: Install Elasticsearch - rpm ansible.builtin.package: diff --git a/roles/elasticstack/tasks/elasticstack-versions.yml b/roles/elasticstack/tasks/elasticstack-versions.yml index e58dca42..0f5421f8 100644 --- a/roles/elasticstack/tasks/elasticstack-versions.yml +++ b/roles/elasticstack/tasks/elasticstack-versions.yml @@ -6,8 +6,8 @@ - name: Set target version to Elasticsearch on CA host ansible.builtin.set_fact: - elasticstack_version: "{{ ansible_facts.packages['elasticsearch'].version }}" + elasticstack_version: "{{ ansible_facts.packages['elasticsearch'][0].version }}" delegate_to: "{{ elasticstack_ca }}" when: - - ansible_facts.packages['elasticsearch'].version is defined - - if elasticstack_version is undefined + - ansible_facts.packages['elasticsearch'][0].version is defined + - elasticstack_version is undefined diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index 2082afd2..968a4f52 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -18,8 +18,6 @@ - name: Fetch passwords if passwords are initialized ansible.builtin.import_tasks: elasticstack-passwords.yml - when: - - elasticstack_password.stdout is undefined - name: Set elasticstack_globals_set for other roles to skip this role ansible.builtin.set_fact: diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 8f9a04db..783b476f 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -3,10 +3,6 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: > - not elasticstack_globals_set | bool - or - elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index ef0a70e4..fa8e806c 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -3,10 +3,6 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: > - not elasticstack_globals_set | bool - or - elasticstack_globals_set is undefined - name: Update apt cache. ansible.builtin.apt: diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index c4812ab1..340b4c43 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -3,10 +3,6 @@ - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack - when: > - not elasticstack_globals_set | bool - or - elasticstack_globals_set is undefined - name: Check for versions ansible.builtin.fail: From d1d0e2db9b8dcfa6d359d8357e908fc81cc5f513 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 19 Jan 2024 19:36:15 +0100 Subject: [PATCH 34/67] Damn typo --- roles/elasticsearch/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index d365c1b6..2bf6201c 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -129,10 +129,10 @@ ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml throttle: 1 when: - - elasticsstack_version is defined + - elasticstack_version is defined - ansible_facts.packages['elasticsearch'][0].version is defined - elasticstack_password.stdout is defined - - elasticstack_version is version( ansible_package_facts['elasticsearch'][0].version, '>') + - elasticstack_version is version( ansible_facts.packages['elasticsearch'][0].version, '>') - name: Install Elasticsearch - rpm ansible.builtin.package: From 1cb757107dee72ab074dcd2ed0728f8cbfd9aa54 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 19 Jan 2024 19:44:51 +0100 Subject: [PATCH 35/67] Directly upgrade nodes that are down --- .../tasks/elasticsearch-rolling-upgrade.yml | 167 ++++++++++-------- roles/elasticsearch/tasks/main.yml | 1 - 2 files changed, 90 insertions(+), 78 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index dabf1e00..f9511ab1 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -13,18 +13,31 @@ name: elasticsearch register: elasticsearch_running +- name: Update stopped services right away + when: + - elasticsearch_running.status.ActiveState == "inactive" + block: + - name: Install Elasticsearch - rpm + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + enablerepo: + - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + when: + - ansible_os_family == "RedHat" + + - name: Install Elasticsearch - deb + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + when: + - ansible_os_family == "Debian" + + - name: Be careful about upgrade when Elasticsearch is running when: - elasticsearch_running.status.ActiveState == "active" + throttle: 1 block: - - name: Wait for elasticsearch node to come back up if it was stopped - ansible.builtin.wait_for: - host: "localhost" - port: "{{ elasticstack_elasticsearch_http_port }}" - delay: 45 - when: response.changed | bool - # this step is key!!! Don't restart more nodes # until all shards have completed recovery - name: Wait for cluster health to return to green @@ -64,76 +77,76 @@ enabled: yes state: stopped -- name: Install Elasticsearch - rpm - ansible.builtin.package: - name: "{{ elasticsearch_package }}" - enablerepo: - - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' - when: - - ansible_os_family == "RedHat" + - name: Install Elasticsearch - rpm + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + enablerepo: + - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + when: + - ansible_os_family == "RedHat" -- name: Install Elasticsearch - deb - ansible.builtin.package: - name: "{{ elasticsearch_package }}" - when: - - ansible_os_family == "Debian" + - name: Install Elasticsearch - deb + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + when: + - ansible_os_family == "Debian" -- name: Start elasticsearch - ansible.builtin.service: - name: elasticsearch - enabled: yes - state: started - when: - - elasticsearch_running.status.ActiveState == "active" + - name: Start elasticsearch + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: started + when: + - elasticsearch_running.status.ActiveState == "active" + + - name: Wait for elasticsearch node to come back up if it was stopped + ansible.builtin.wait_for: + host: "localhost" + port: "{{ elasticstack_elasticsearch_http_port }}" + delay: 30 + + - name: Confirm the node joins the cluster # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + curl + -k + -u elastic:{{ elasticstack_password.stdout }} + -s + -m 2 + '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' + | grep + -E + '^{{ ansible_fqdn }}$' + register: result + until: result.rc == 0 + retries: 200 + delay: 3 + changed_when: false -- name: Wait for elasticsearch node to come back up if it was stopped - ansible.builtin.wait_for: - host: "localhost" - port: "{{ elasticstack_elasticsearch_http_port }}" - delay: 30 - -- name: Confirm the node joins the cluster # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl - -k - -u elastic:{{ elasticstack_password.stdout }} - -s - -m 2 - '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' - | grep - -E - '^{{ ansible_fqdn }}$' - register: result - until: result.rc == 0 - retries: 200 - delay: 3 - changed_when: false - -- name: Enable shard allocation for the cluster - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" - method: PUT - body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' - body_format: json - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - # next line is boolean not string, so no quotes around true - # use python truthiness - until: "response.json.acknowledged == true" - retries: 5 - delay: 30 - -- name: Wait for cluster health to return to yellow or green - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" - method: GET - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - until: "response.json.status == 'yellow' or response.json.status == 'green'" - retries: 5 - delay: 30 + - name: Enable shard allocation for the cluster + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + method: PUT + body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' + body_format: json + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + + - name: Wait for cluster health to return to yellow or green + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" + method: GET + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + until: "response.json.status == 'yellow' or response.json.status == 'green'" + retries: 5 + delay: 30 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 2bf6201c..540692e7 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -127,7 +127,6 @@ - name: Update Elasitcsearch if needed ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml - throttle: 1 when: - elasticstack_version is defined - ansible_facts.packages['elasticsearch'][0].version is defined From 2865d5d5be9b7791ea0b19ccfa487edabc348e98 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Mon, 12 Feb 2024 18:35:34 +0100 Subject: [PATCH 36/67] Apply workaround for seria in include tasks Copied from https://github.com/ansible/ansible/issues/12170#issuecomment-372263149 thanks, @hryamzik you really saved my butt here --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 5 ++++- roles/elasticsearch/tasks/main.yml | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index f9511ab1..72abb1fe 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -8,6 +8,10 @@ --- +- name: Set connection protocol to https + ansible.builtin.set_fact: + elasticsearch_http_protocol: "https" + - name: Check for running Elasticsearch service ansible.builtin.systemd: name: elasticsearch @@ -35,7 +39,6 @@ - name: Be careful about upgrade when Elasticsearch is running when: - elasticsearch_running.status.ActiveState == "active" - throttle: 1 block: # this step is key!!! Don't restart more nodes diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 540692e7..62ef74c2 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -126,8 +126,10 @@ }} - name: Update Elasitcsearch if needed - ansible.builtin.import_tasks: elasticsearch-rolling-upgrade.yml + ansible.builtin.include_tasks: elasticsearch-rolling-upgrade.yml + with_items: "{{ groups['elasticsearch'] }}" when: + - "hostvars[item].inventory_hostname == inventory_hostname" - elasticstack_version is defined - ansible_facts.packages['elasticsearch'][0].version is defined - elasticstack_password.stdout is defined From 7c4b38d3e8e35fa71ca267f3e8b0a55aed6f30e2 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Mon, 12 Feb 2024 19:22:16 +0100 Subject: [PATCH 37/67] Workaround for "back in cluster" check --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 72abb1fe..200daa8a 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -119,7 +119,7 @@ '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' | grep -E - '^{{ ansible_fqdn }}$' + '^{{ ansible_hostname }}$' register: result until: result.rc == 0 retries: 200 From 49b34d4ae8d5264ed6adf46b851ee678894f28cc Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 13:30:21 +0100 Subject: [PATCH 38/67] Fix typo thanks, @dgoetz --- roles/elasticsearch/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 62ef74c2..b9b18b81 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -125,7 +125,7 @@ replace(' ', '') }} -- name: Update Elasitcsearch if needed +- name: Update Elasticsearch if needed ansible.builtin.include_tasks: elasticsearch-rolling-upgrade.yml with_items: "{{ groups['elasticsearch'] }}" when: From 55e5de465168b5adad672dd60442bbcbb3b9ca8b Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 16:40:21 +0100 Subject: [PATCH 39/67] Introduce (and set) elasticsearch_nodename variable --- docs/role-elasticsearch.md | 1 + roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 2 +- roles/elasticsearch/tasks/main.yml | 6 ++++++ roles/elasticsearch/templates/elasticsearch.yml.j2 | 6 +++--- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md index 98e2f8c9..1f283150 100644 --- a/docs/role-elasticsearch.md +++ b/docs/role-elasticsearch.md @@ -13,6 +13,7 @@ Role Variables -------------- * *elasticsearch_node_types*: List of types of this very node. Please refer to [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for details. (default: not set. allowed value: array of types) ++ *elasticsearch_nodename*': Node name of the Elasticsearch node. (default: Hostname of the node as seen by Ansible) * *elasticsearch_clustername*: Name the Elasticsearch Cluster (default: `elasticsearch`) * *elasticsearch_heap*: Heapsize for Elasticsearch. (Half of free memory on host. Maximum 30GB. (default: Half of hosts memory. Min 1GB, Max 30GB) * *elasticsearch_tls_key_passphrase*: Passphrase for elasticsearch certificates (default: `PleaseChangeMeIndividually`) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 200daa8a..6cc42209 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -119,7 +119,7 @@ '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' | grep -E - '^{{ ansible_hostname }}$' + '^{{ elasticsearch_nodename }}$' register: result until: result.rc == 0 retries: 200 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index b9b18b81..b93d4c5f 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -20,6 +20,12 @@ - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml' - '{{ ansible_os_family }}.yml' +- name: Set node name if not overriden by user + ansible.builtin.set_fact: + elasticsearch_nodename: "{{ ansible_hostname }}" + when: + - elasticsearch_nodename is undefined + - name: Set common password for common certificates ansible.builtin.set_fact: elasticsearch_tls_key_passphrase: "{{ elasticstack_cert_pass }}" diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 index 1ae60acb..d276d1bb 100644 --- a/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,4 +1,4 @@ -node.name: "{{ ansible_hostname }}" +node.name: "{{ elasticsearch_nodename }}" path.data: {{ elasticsearch_datapath }} path.logs: {{ elasticsearch_logpath }} cluster.name: "{{ elasticsearch_clustername }}" @@ -31,11 +31,11 @@ discovery.seed_hosts: [ {% for host in groups['elasticsearch'] %} {% if not elaticsearch_cluster_set_up | bool and groups['elasticsearch'] | length > 1 %} {% if elasticsearch_node_types is defined %} cluster.initial_master_nodes: [ {% for host in groups['elasticsearch_role_master'] %} -"{{ hostvars[host].ansible_hostname }}"{% if not loop.last %},{% endif %} +"{{ hostvars[host]['elasticsearch_nodename'] }}"{% if not loop.last %},{% endif %} {% endfor %} ] {% else %} cluster.initial_master_nodes: [ {% for host in groups['elasticsearch'] %} -"{{ hostvars[host].ansible_hostname }}"{% if not loop.last %},{% endif %} +"{{ hostvars[host]['elasticsearch_nodename'] }}"{% if not loop.last %},{% endif %} {% endfor %} ] {% endif %} {% endif %} From 5d92f584734f77522f2bbff37635a0700e15c76f Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 17:20:26 +0100 Subject: [PATCH 40/67] Add rolling restart for handler --- roles/elasticsearch/handlers/main.yml | 7 +- .../handlers/rolling-restart.yml | 115 ++++++++++++++++++ 2 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 roles/elasticsearch/handlers/rolling-restart.yml diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index ff3b5ab5..2cfd161c 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -1,11 +1,10 @@ --- # handlers file for elasticsearch - name: Restart Elasticsearch - ansible.builtin.service: - name: elasticsearch - state: restarted - daemon_reload: yes + ansible.builtin.include_tasks: rolling-restart.yml + with_items: "{{ groups['elasticsearch'] }}" when: + - "hostvars[item].inventory_hostname == inventory_hostname" - elasticsearch_enable | bool - not elasticsearch_freshstart.changed | bool - not elasticsearch_freshstart_security.changed | bool diff --git a/roles/elasticsearch/handlers/rolling-restart.yml b/roles/elasticsearch/handlers/rolling-restart.yml new file mode 100644 index 00000000..b6ab4191 --- /dev/null +++ b/roles/elasticsearch/handlers/rolling-restart.yml @@ -0,0 +1,115 @@ +# Ansible +# +# Rolling restart of Elasticsearch with security on +# Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros +# Modifications: author: Daniel Neuberger @netways.de +# More modifications: NETWAYS Professional Services GmbH +# latest tested with Ansible 2.9 and later + +--- + +- name: Set connection protocol to https + ansible.builtin.set_fact: + elasticsearch_http_protocol: "https" + +- name: Check for running Elasticsearch service + ansible.builtin.systemd: + name: elasticsearch + register: elasticsearch_running + +- name: Be careful about upgrade when Elasticsearch is running + when: + - elasticsearch_running.status.ActiveState == "active" + block: + + # this step is key!!! Don't restart more nodes + # until all shards have completed recovery + - name: Wait for cluster health to return to green + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" + method: GET + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + until: "response.json.status == 'green'" + retries: 50 + delay: 30 + + - name: Disable shard allocation for the cluster + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + method: PUT + body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' + body_format: json + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + + - name: Stop non essential indexing to speed up shard recovery + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_flush" + method: POST + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + failed_when: false + + - name: Shutdown elasticsearch service + ansible.builtin.service: + name: elasticsearch + enabled: yes + daemon_reload: yes + state: restarted + + - name: Wait for elasticsearch node to come back up if it was stopped + ansible.builtin.wait_for: + host: "localhost" + port: "{{ elasticstack_elasticsearch_http_port }}" + delay: 30 + + - name: Confirm the node joins the cluster # noqa: risky-shell-pipe + ansible.builtin.shell: > + if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; + curl + -k + -u elastic:{{ elasticstack_password.stdout }} + -s + -m 2 + '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' + | grep + -E + '^{{ elasticsearch_nodename }}$' + register: result + until: result.rc == 0 + retries: 200 + delay: 3 + changed_when: false + + - name: Enable shard allocation for the cluster + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + method: PUT + body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' + body_format: json + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + + - name: Wait for cluster health to return to yellow or green + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" + method: GET + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + until: "response.json.status == 'yellow' or response.json.status == 'green'" + retries: 5 + delay: 30 From 07bc9e4c8a8bf9db31df4ab6c828e4cd8c28732c Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 17:31:30 +0100 Subject: [PATCH 41/67] Enable shard allocation before checks This will help with recovering from broken updates --- .../elasticsearch/handlers/rolling-restart.yml | 18 +++++++++++++++++- .../tasks/elasticsearch-rolling-upgrade.yml | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/roles/elasticsearch/handlers/rolling-restart.yml b/roles/elasticsearch/handlers/rolling-restart.yml index b6ab4191..b94e09ae 100644 --- a/roles/elasticsearch/handlers/rolling-restart.yml +++ b/roles/elasticsearch/handlers/rolling-restart.yml @@ -17,11 +17,27 @@ name: elasticsearch register: elasticsearch_running -- name: Be careful about upgrade when Elasticsearch is running +- name: Restart a single node with extra checks when: - elasticsearch_running.status.ActiveState == "active" block: + - name: Enable shard allocation for the cluster + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + method: PUT + body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' + body_format: json + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + # this step is key!!! Don't restart more nodes # until all shards have completed recovery - name: Wait for cluster health to return to green diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 6cc42209..d352181f 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -41,6 +41,22 @@ - elasticsearch_running.status.ActiveState == "active" block: + - name: Enable shard allocation for the cluster + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + method: PUT + body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' + body_format: json + user: elastic + password: "{{ elasticstack_password.stdout }}" + validate_certs: no + register: response + # next line is boolean not string, so no quotes around true + # use python truthiness + until: "response.json.acknowledged == true" + retries: 5 + delay: 30 + # this step is key!!! Don't restart more nodes # until all shards have completed recovery - name: Wait for cluster health to return to green From 9ae5b16b5fcd983dbc0239688a8159c613a77782 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 17:37:59 +0100 Subject: [PATCH 42/67] Re-Add repo key --- roles/elasticstack/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 4c809a72..b69f3240 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -13,6 +13,7 @@ elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_kibana_port: 5601 elasticstack_override_beats_tls: false elasticstack_release: 8 +elasticstack_repo_key: https://artifacts.elastic.co/GPG-KEY-elasticsearch elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic From 96eb6fc2e3aa7f43eaaaed63420d5348499df702 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 18:18:31 +0100 Subject: [PATCH 43/67] Restart all applications after upgrade --- roles/beats/tasks/auditbeat.yml | 6 ++++++ roles/beats/tasks/filebeat.yml | 6 ++++++ roles/beats/tasks/metricbeat.yml | 6 ++++++ roles/kibana/tasks/main.yml | 6 ++++++ roles/logstash/tasks/main.yml | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/roles/beats/tasks/auditbeat.yml b/roles/beats/tasks/auditbeat.yml index f8a0a16a..79096455 100644 --- a/roles/beats/tasks/auditbeat.yml +++ b/roles/beats/tasks/auditbeat.yml @@ -16,6 +16,8 @@ name: "{{ beats_auditbeat_package }}" enablerepo: - 'elastic-{{ elasticstack_release }}.x' + notify: + - Restart Auditbeat when: - ansible_os_family == "RedHat" - elasticstack_full_stack | bool @@ -23,6 +25,8 @@ - name: Install Auditbeat - rpm - standalone ansible.builtin.package: name: "{{ beats_auditbeat_package }}" + notify: + - Restart Auditbeat when: - ansible_os_family == "RedHat" - not elasticstack_full_stack | bool @@ -30,6 +34,8 @@ - name: Install Auditbeat - deb ansible.builtin.package: name: "{{ beats_auditbeat_package }}" + notify: + - Restart Auditbeat when: - ansible_os_family == "Debian" diff --git a/roles/beats/tasks/filebeat.yml b/roles/beats/tasks/filebeat.yml index 965bf1ca..4c90c9ed 100644 --- a/roles/beats/tasks/filebeat.yml +++ b/roles/beats/tasks/filebeat.yml @@ -15,6 +15,8 @@ name: "{{ beats_filebeat_package }}" enablerepo: - 'elastic-{{ elasticstack_release }}.x' + notify: + - Restart Filebeat when: - ansible_os_family == "RedHat" - elasticstack_full_stack | bool @@ -22,6 +24,8 @@ - name: Install Filebeat - rpm - standalone ansible.builtin.package: name: "{{ beats_filebeat_package }}" + notify: + - Restart Filebeat when: - ansible_os_family == "RedHat" - not elasticstack_full_stack | bool @@ -29,6 +33,8 @@ - name: Install Filebeat - deb ansible.builtin.package: name: "{{ beats_filebeat_package }}" + notify: + - Restart Filebeat when: - ansible_os_family == "Debian" diff --git a/roles/beats/tasks/metricbeat.yml b/roles/beats/tasks/metricbeat.yml index e65d6094..c261a962 100644 --- a/roles/beats/tasks/metricbeat.yml +++ b/roles/beats/tasks/metricbeat.yml @@ -16,6 +16,8 @@ name: "{{ beats_metricbeat_package }}" enablerepo: - 'elastic-{{ elasticstack_release }}.x' + notify: + - Restart Metricbeat when: - ansible_os_family == "RedHat" - elasticstack_full_stack | bool @@ -23,6 +25,8 @@ - name: Install Metricbeat - rpm - standalone ansible.builtin.package: name: "{{ beats_metricbeat_package }}" + notify: + - Restart Metricbeat when: - ansible_os_family == "RedHat" - not elasticstack_full_stack | bool @@ -30,6 +34,8 @@ - name: Install Metricbeat - deb ansible.builtin.package: name: "{{ beats_metricbeat_package }}" + notify: + - Restart Metricbeat when: - ansible_os_family == "Debian" diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 30afb89e..621d0432 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -48,6 +48,8 @@ name: "{{ kibana_package }}" enablerepo: - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + notify: + - Restart Kibana when: - ansible_os_family == "RedHat" - elasticstack_full_stack | bool @@ -55,6 +57,8 @@ - name: Install Kibana - rpm - standalone ansible.builtin.package: name: "{{ kibana_package }}" + notify: + - Restart Kibana when: - ansible_os_family == "RedHat" - not elasticstack_full_stack | bool @@ -62,6 +66,8 @@ - name: Install Kibana - deb ansible.builtin.package: name: "{{ kibana_package }}" + notify: + - Restart Kibana when: - ansible_os_family == "Debian" diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index dcdb10d5..927ad3f3 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -73,6 +73,8 @@ name: "{{ logstash_package }}" enablerepo: - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + notify: + - Restart Logstash when: - ansible_os_family == "RedHat" - elasticstack_full_stack | bool @@ -80,6 +82,8 @@ - name: Install Logstash - rpm - standalone ansible.builtin.package: name: "{{ logstash_package }}" + notify: + - Restart Logstash when: - ansible_os_family == "RedHat" - not elasticstack_full_stack | bool @@ -87,6 +91,8 @@ - name: Install Logstash - deb ansible.builtin.package: name: "{{ logstash_package }}" + notify: + - Restart Logstash when: - ansible_os_family == "Debian" From a257c5641a90d117299096127aeec923da8af3af Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 18:33:03 +0100 Subject: [PATCH 44/67] Build Logstash version different for deb fixes #205 --- .../templates/elasticsearch.yml.j2 | 1 + roles/logstash/tasks/main.yml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 index de23effa..427697a8 100644 --- a/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,3 +1,4 @@ +# test {{ ansible_managed | comment }} node.name: "{{ elasticsearch_nodename }}" diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 927ad3f3..a0043d8d 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -67,6 +67,23 @@ string if elasticstack_version is defined else '') | replace(' ', '') }} + when: + - ansible_os_family != "Debian" + +- name: Construct exact name of Logstas package + ansible.builtin.set_fact: + logstash_package: > + {{ + 'logstash' + + '1:' + + ('-oss' if elasticstack_variant == 'oss' else '') + + (elasticstack_versionseparator + + elasticstack_version | + string if elasticstack_version is defined else '') | + replace(' ', '') + }} + when: + - ansible_os_family == "Debian" - name: Install Logstash - rpm - full stack ansible.builtin.package: From b7edd49085e1b632b39758287ab7547b7d05f175 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 18:44:00 +0100 Subject: [PATCH 45/67] Typo --- roles/logstash/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index a0043d8d..631ec9b4 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -75,9 +75,9 @@ logstash_package: > {{ 'logstash' + - '1:' + ('-oss' if elasticstack_variant == 'oss' else '') + - (elasticstack_versionseparator + + (1: + + elasticstack_versionseparator + elasticstack_version | string if elasticstack_version is defined else '') | replace(' ', '') From 0ce2cda2036f84690138357fb63f6c72a0df6d5e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 18:47:04 +0100 Subject: [PATCH 46/67] Typo --- roles/logstash/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 631ec9b4..973ddd8f 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -76,7 +76,7 @@ {{ 'logstash' + ('-oss' if elasticstack_variant == 'oss' else '') + - (1: + + ('1:' + elasticstack_versionseparator + elasticstack_version | string if elasticstack_version is defined else '') | From 1d1537d78650a340e830495e74fb7cf3a849f0c8 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 13 Feb 2024 18:57:34 +0100 Subject: [PATCH 47/67] NGAH --- roles/logstash/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 973ddd8f..bf5caae0 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -76,8 +76,8 @@ {{ 'logstash' + ('-oss' if elasticstack_variant == 'oss' else '') + - ('1:' + - elasticstack_versionseparator + + (elasticstack_versionseparator + + '1:' + elasticstack_version | string if elasticstack_version is defined else '') | replace(' ', '') From 4b08e5483e0a4d293e76465533c85aeda533cd38 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 14 Feb 2024 13:17:55 +0100 Subject: [PATCH 48/67] Revert prototype for rolling restart --- roles/elasticsearch/handlers/main.yml | 7 +- .../handlers/rolling-restart.yml | 131 ------------------ 2 files changed, 4 insertions(+), 134 deletions(-) delete mode 100644 roles/elasticsearch/handlers/rolling-restart.yml diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index 2cfd161c..ff3b5ab5 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -1,10 +1,11 @@ --- # handlers file for elasticsearch - name: Restart Elasticsearch - ansible.builtin.include_tasks: rolling-restart.yml - with_items: "{{ groups['elasticsearch'] }}" + ansible.builtin.service: + name: elasticsearch + state: restarted + daemon_reload: yes when: - - "hostvars[item].inventory_hostname == inventory_hostname" - elasticsearch_enable | bool - not elasticsearch_freshstart.changed | bool - not elasticsearch_freshstart_security.changed | bool diff --git a/roles/elasticsearch/handlers/rolling-restart.yml b/roles/elasticsearch/handlers/rolling-restart.yml deleted file mode 100644 index b94e09ae..00000000 --- a/roles/elasticsearch/handlers/rolling-restart.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Ansible -# -# Rolling restart of Elasticsearch with security on -# Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros -# Modifications: author: Daniel Neuberger @netways.de -# More modifications: NETWAYS Professional Services GmbH -# latest tested with Ansible 2.9 and later - ---- - -- name: Set connection protocol to https - ansible.builtin.set_fact: - elasticsearch_http_protocol: "https" - -- name: Check for running Elasticsearch service - ansible.builtin.systemd: - name: elasticsearch - register: elasticsearch_running - -- name: Restart a single node with extra checks - when: - - elasticsearch_running.status.ActiveState == "active" - block: - - - name: Enable shard allocation for the cluster - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" - method: PUT - body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' - body_format: json - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - # next line is boolean not string, so no quotes around true - # use python truthiness - until: "response.json.acknowledged == true" - retries: 5 - delay: 30 - - # this step is key!!! Don't restart more nodes - # until all shards have completed recovery - - name: Wait for cluster health to return to green - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" - method: GET - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - until: "response.json.status == 'green'" - retries: 50 - delay: 30 - - - name: Disable shard allocation for the cluster - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" - method: PUT - body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' - body_format: json - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - - - name: Stop non essential indexing to speed up shard recovery - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_flush" - method: POST - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - failed_when: false - - - name: Shutdown elasticsearch service - ansible.builtin.service: - name: elasticsearch - enabled: yes - daemon_reload: yes - state: restarted - - - name: Wait for elasticsearch node to come back up if it was stopped - ansible.builtin.wait_for: - host: "localhost" - port: "{{ elasticstack_elasticsearch_http_port }}" - delay: 30 - - - name: Confirm the node joins the cluster # noqa: risky-shell-pipe - ansible.builtin.shell: > - if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; - curl - -k - -u elastic:{{ elasticstack_password.stdout }} - -s - -m 2 - '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' - | grep - -E - '^{{ elasticsearch_nodename }}$' - register: result - until: result.rc == 0 - retries: 200 - delay: 3 - changed_when: false - - - name: Enable shard allocation for the cluster - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" - method: PUT - body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' - body_format: json - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - # next line is boolean not string, so no quotes around true - # use python truthiness - until: "response.json.acknowledged == true" - retries: 5 - delay: 30 - - - name: Wait for cluster health to return to yellow or green - ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" - method: GET - user: elastic - password: "{{ elasticstack_password.stdout }}" - validate_certs: no - register: response - until: "response.json.status == 'yellow' or response.json.status == 'green'" - retries: 5 - delay: 30 From 4f2e7a0731d05e9f96d9452b16cf3fbe78c9726e Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 14 Feb 2024 13:21:24 +0100 Subject: [PATCH 49/67] Test old version picking system --- roles/logstash/tasks/main.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index bf5caae0..0d83e75c 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -67,23 +67,23 @@ string if elasticstack_version is defined else '') | replace(' ', '') }} - when: - - ansible_os_family != "Debian" - -- name: Construct exact name of Logstas package - ansible.builtin.set_fact: - logstash_package: > - {{ - 'logstash' + - ('-oss' if elasticstack_variant == 'oss' else '') + - (elasticstack_versionseparator + - '1:' + - elasticstack_version | - string if elasticstack_version is defined else '') | - replace(' ', '') - }} - when: - - ansible_os_family == "Debian" +# when: +# - ansible_os_family != "Debian" +# +#- name: Construct exact name of Logstas package +# ansible.builtin.set_fact: +# logstash_package: > +# {{ +# 'logstash' + +# ('-oss' if elasticstack_variant == 'oss' else '') + +# (elasticstack_versionseparator + +# '1:' + +# elasticstack_version | +# string if elasticstack_version is defined else '') | +# replace(' ', '') +# }} +# when: +# - ansible_os_family == "Debian" - name: Install Logstash - rpm - full stack ansible.builtin.package: From d1d5cafc9aa502c44fbe0e1a3b8ba5e84dd42ec4 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 12:03:33 +0100 Subject: [PATCH 50/67] Fix package name creation on .deb --- .github/workflows/test_roles_pr.yml | 1 + roles/elasticstack/tasks/main.yml | 7 ++++++ roles/logstash/tasks/main.yml | 37 +++++++++++++++-------------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 67855fa4..d9a93fee 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -14,6 +14,7 @@ on: - debug pull_request: merge_group: + push: jobs: lint_full: diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index 968a4f52..fd43894c 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -13,6 +13,13 @@ - elasticstack_ca is undefined - groups['elasticsearch'][0] is defined +- name: Set elasticstack_ca variable if not already set to Elasticsearch server + ansible.builtin.set_fact: + elasticstack_ca: "{{ groups['logstash'][0] }}" + when: + - elasticstack_ca is undefined + - groups['logstash'][0] is defined + - name: Set versions for components ansible.builtin.import_tasks: elasticstack-versions.yml diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 0d83e75c..b2245ae9 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -56,7 +56,7 @@ - renew_ca - renew_logstash_cert -- name: Construct exact name of Logstas package +- name: Construct exact name of Logstash package ansible.builtin.set_fact: logstash_package: > {{ @@ -67,23 +67,24 @@ string if elasticstack_version is defined else '') | replace(' ', '') }} -# when: -# - ansible_os_family != "Debian" -# -#- name: Construct exact name of Logstas package -# ansible.builtin.set_fact: -# logstash_package: > -# {{ -# 'logstash' + -# ('-oss' if elasticstack_variant == 'oss' else '') + -# (elasticstack_versionseparator + -# '1:' + -# elasticstack_version | -# string if elasticstack_version is defined else '') | -# replace(' ', '') -# }} -# when: -# - ansible_os_family == "Debian" + when: + - ansible_os_family != "Debian" + +- name: Construct exact name of Logstas package + ansible.builtin.set_fact: + logstash_package: > + {{ + 'logstash' + + ('-oss' if elasticstack_variant == 'oss' else '') + + (elasticstack_versionseparator + + '1:' + + elasticstack_version + + '-1' | + string if elasticstack_version is defined else '') | + replace(' ', '') + }} + when: + - ansible_os_family == "Debian" - name: Install Logstash - rpm - full stack ansible.builtin.package: From 76f271cf48c586f37bb73b567ab86a07dd541bc9 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 12:09:46 +0100 Subject: [PATCH 51/67] Typo --- roles/logstash/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index b2245ae9..afd5f720 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -70,7 +70,7 @@ when: - ansible_os_family != "Debian" -- name: Construct exact name of Logstas package +- name: Construct exact name of Logstash package ansible.builtin.set_fact: logstash_package: > {{ From c0988706eeed20d1b9c685ba2122eeadb3ae09cf Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 12:53:32 +0100 Subject: [PATCH 52/67] Remove directive we don't need anymore --- .github/workflows/test_roles_pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index d9a93fee..67855fa4 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -14,7 +14,6 @@ on: - debug pull_request: merge_group: - push: jobs: lint_full: From 506aa6de2b1d0075c1477490a49f00421985dbd2 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:04:11 +0100 Subject: [PATCH 53/67] Handle single instances of Elasticsearch during update --- .../tasks/elasticsearch-rolling-upgrade.yml | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index d352181f..83e5442c 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -21,24 +21,49 @@ when: - elasticsearch_running.status.ActiveState == "inactive" block: - - name: Install Elasticsearch - rpm + - name: Install Elasticsearch - rpm fullstack ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" + - elasticstack_full_stack | bool - - name: Install Elasticsearch - deb + - name: Install Elasticsearch ansible.builtin.package: name: "{{ elasticsearch_package }}" when: - - ansible_os_family == "Debian" + - ansible_os_family == "Debian" or + not elasticstack_full_stack | bool + +- name: Update single instances without extra caution + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + when: + - ansible_os_family == "Debian" or + not elasticstack_full_stack | bool + - groups['elasticsearch'] | length == 1 + notify: + - Restart Elasticsearch + +- name: Update single instances without extra caution - rpm fullstack + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + enablerepo: + - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + when: + - ansible_os_family == "RedHat" + - elasticstack_full_stack | bool + - groups['elasticsearch'] | length == 1 + notify: + - Restart Elasticsearch - name: Be careful about upgrade when Elasticsearch is running when: - elasticsearch_running.status.ActiveState == "active" + - groups['elasticsearch'] | length > 1 block: - name: Enable shard allocation for the cluster From 2a659ff20c4e64525da04bf52ecf1cc0769e9521 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:12:25 +0100 Subject: [PATCH 54/67] Lint --- molecule/elasticstack_default/verify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index 110673d4..3a2e8430 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -135,4 +135,3 @@ success_msg: "'{{ item }}' was found in nodes.content" with_inventory_hostnames: all when: groups['elasticsearch'] | length > 1 - From ff70c58f88cc3c13af8d04121eb10bdae4d81b56 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:18:59 +0100 Subject: [PATCH 55/67] Enable repo only with rpm and full_stack --- .../elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 83e5442c..c1e453c3 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -121,19 +121,21 @@ enabled: yes state: stopped - - name: Install Elasticsearch - rpm + - name: Install Elasticsearch - rpm fullstack ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" + - elasticstack_full_stack | bool - - name: Install Elasticsearch - deb + - name: Install Elasticsearch ansible.builtin.package: name: "{{ elasticsearch_package }}" when: - - ansible_os_family == "Debian" + - ansible_os_family == "Debian" or + not elasticstack_full_stack | bool - name: Start elasticsearch ansible.builtin.service: From dde6c86aafa73ffcdf973926e4ae91bd187993a0 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:27:38 +0100 Subject: [PATCH 56/67] Streamline installation of common packages and dependencies --- roles/beats/tasks/beats-security.yml | 12 ---------- .../tasks/elasticsearch-security.yml | 12 ---------- roles/elasticstack/tasks/main.yml | 3 +++ roles/elasticstack/tasks/packages.yml | 22 +++++++++++++++++++ roles/kibana/tasks/kibana-security.yml | 11 ---------- roles/logstash/tasks/logstash-security.yml | 11 ---------- 6 files changed, 25 insertions(+), 46 deletions(-) create mode 100644 roles/elasticstack/tasks/packages.yml diff --git a/roles/beats/tasks/beats-security.yml b/roles/beats/tasks/beats-security.yml index ef034ee3..0352fd73 100644 --- a/roles/beats/tasks/beats-security.yml +++ b/roles/beats/tasks/beats-security.yml @@ -1,17 +1,5 @@ --- -- name: Install packages for security tasks - ansible.builtin.package: - name: - - unzip - - python3-cryptography - - openssl - tags: - - certificates - - renew_ca - - renew_kibana_cert - - renew_beats_cert - - name: Ensure beats certificate exists ansible.builtin.stat: path: "/etc/beats/certs/{{ inventory_hostname }}-beats.crt" diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 9a4cc9b3..381e3d41 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -1,17 +1,5 @@ --- -- name: Install packages for security tasks - ansible.builtin.package: - name: - - unzip - - python3-cryptography - - openssl - tags: - - certificates - - renew_ca - - renew_kibana_cert - - renew_es_cert - - name: Ensure ca exists ansible.builtin.stat: path: "{{ elasticstack_ca_dir }}/elastic-stack-ca.p12" diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index fd43894c..88aa3315 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -29,3 +29,6 @@ - name: Set elasticstack_globals_set for other roles to skip this role ansible.builtin.set_fact: elasticstack_globals_set: true + +- name: Install common packages and dependencies + ansible.builtin.import_tasks: packages.yml diff --git a/roles/elasticstack/tasks/packages.yml b/roles/elasticstack/tasks/packages.yml new file mode 100644 index 00000000..36a2f3f7 --- /dev/null +++ b/roles/elasticstack/tasks/packages.yml @@ -0,0 +1,22 @@ +--- + +- name: Update apt cache. + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false + when: ansible_os_family == 'Debian' + +- name: Install packages for security tasks + ansible.builtin.package: + name: + - unzip + - python3-cryptography + - openssl + tags: + - certificates + - renew_ca + - renew_kibana_cert + - renew_beats_cert + - renew_es_cert + - renew_logstash_cert diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index c19b00ed..553b74c7 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -1,16 +1,5 @@ --- -- name: Install packages for security tasks - ansible.builtin.package: - name: - - unzip - - python3-cryptography - - openssl - tags: - - certificates - - renew_ca - - renew_kibana_cert - - name: Ensure kibana certificate exists ansible.builtin.stat: path: "/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12" diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 7721cbe7..e83b6c07 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -1,16 +1,5 @@ --- -- name: Install packages for security tasks - ansible.builtin.package: - name: - - unzip - - python3-cryptography - - openssl - tags: - - certificates - - renew_ca - - renew_logstash_cert - - name: Ensure logstash certificate exists ansible.builtin.stat: path: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12" From 3f9f6f4bb85437d5b3293175dcc4b19301912fbe Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:34:23 +0100 Subject: [PATCH 57/67] Add a few explanatory comments --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index c1e453c3..a4dcc0a9 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -8,6 +8,9 @@ --- +# For now we support upgrade only for clusters with security enabled +# If you positively need support for safely upgrading clusters without security, +# feel free to open an issue at https://github.com/NETWAYS/ansible-collection-elasticstack/issues - name: Set connection protocol to https ansible.builtin.set_fact: elasticsearch_http_protocol: "https" @@ -66,6 +69,8 @@ - groups['elasticsearch'] | length > 1 block: + # Usually we should not need this step. It's only there to recover from broken upgrade plays + # Without this step the cluster would never recover and the play would always fail - name: Enable shard allocation for the cluster ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" @@ -96,6 +101,7 @@ retries: 50 delay: 30 + # Disabling shard allocation right after enabling it seems redundant. Please see above for details. - name: Disable shard allocation for the cluster ansible.builtin.uri: url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" From 35e31c13974093f34896bfa482b8aeb0939b7ec8 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Thu, 15 Feb 2024 13:46:08 +0100 Subject: [PATCH 58/67] Update Readme --- README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 98568d44..d68149a8 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,13 @@ You will want to have reliable DNS resolution or enter all hosts of the stack in The variable `elasticstack_no_log` can be set to `false` if you want to see the output of all tasks. It defaults to `true` because some tasks could reveal passwords in production. -### Versioning +### Versions and upgrades -*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest. (default: none). If you already have an installation of Elastic Stack, this collection will query the version of Elasticsearch on the CA host and use it for all further installations in the same setup. (Only if you run the `elasticsearch` role before all others) +*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest on new setups. (default: none). If you already have an installation of Elastic Stack, this collection will query the version of Elasticsearch on the CA host and use it for all further installations in the same setup. (Only if you run the `elasticsearch` role before all others) Example: `7.17.2` -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) +*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) Make sure it corresponds to `elasticstack_version` if you set both. -For OSS version see `elasticstack_variant` below. **IMPORTANT** Do not change the version once you have set up the stack. There are unpredictable effects to be expected when using this for upgrades. And upgrade mechanism is already on it's way. (default: none. Example: `7.17.2`) +For OSS version see `elasticstack_variant` below. *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) @@ -99,6 +99,14 @@ roles: elasticstack_version: 8.8.1 ``` +#### Upgrades #### + +Set `elasticstack_version` to the version you want to upgrade to. Positively do read and understand Elastics changelog and "breaking changes" of your target version and all between your current and the target version. Do not use unless you have a valid backup. + +If an upgrade fails, you can try re-running the collection with the same settings. There are several tasks that can provide "self-healing". Please do not rely on these mechanisms, they are more of a "convenience recovery" for easier steps. + +The collection will make sure to upgrade Elasticsearch nodes one by one. + ### Default Passwords Default passwords can be seen during generation, or found later in `/usr/share/elasticsearch/initial_passwords` From 83243b36680f32819e4816b5944722fe406db059 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Sat, 17 Feb 2024 12:01:32 +0100 Subject: [PATCH 59/67] Update sponsoring note --- NOTICE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE.md b/NOTICE.md index 7d43b66e..e8b18e4c 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -4,4 +4,4 @@ Here's a list of sponsors who contributed by having the collection improved via outsourcing to NETWAYS. -* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster. +* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster and the ingetration of rolling upgrades. From a728f623cbc2fff94198a9db40f2db5b38085453 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 8 Mar 2024 14:09:06 +0100 Subject: [PATCH 60/67] Make single update a block --- .../tasks/elasticsearch-rolling-upgrade.yml | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index a4dcc0a9..c310c756 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -41,26 +41,28 @@ not elasticstack_full_stack | bool - name: Update single instances without extra caution - ansible.builtin.package: - name: "{{ elasticsearch_package }}" when: - - ansible_os_family == "Debian" or - not elasticstack_full_stack | bool - groups['elasticsearch'] | length == 1 - notify: - - Restart Elasticsearch - -- name: Update single instances without extra caution - rpm fullstack - ansible.builtin.package: - name: "{{ elasticsearch_package }}" - enablerepo: - - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' - when: - - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool - - groups['elasticsearch'] | length == 1 - notify: - - Restart Elasticsearch + block: + - name: Update single instances without extra caution - deb + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + when: + - ansible_os_family == "Debian" or + not elasticstack_full_stack | bool + notify: + - Restart Elasticsearch + + - name: Update single instances without extra caution - rpm fullstack + ansible.builtin.package: + name: "{{ elasticsearch_package }}" + enablerepo: + - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' + when: + - ansible_os_family == "RedHat" + - elasticstack_full_stack | bool + notify: + - Restart Elasticsearch - name: Be careful about upgrade when Elasticsearch is running From e28c823dacc9e2d62db9a12475dea3feba0111da Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 8 Mar 2024 14:14:49 +0100 Subject: [PATCH 61/67] Match elasticsearch_api_host with rest of code --- .../tasks/elasticsearch-rolling-upgrade.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index c310c756..0a0fdfbe 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -75,7 +75,7 @@ # Without this step the cluster would never recover and the play would always fail - name: Enable shard allocation for the cluster ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" method: PUT body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' body_format: json @@ -93,7 +93,7 @@ # until all shards have completed recovery - name: Wait for cluster health to return to green ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" method: GET user: elastic password: "{{ elasticstack_password.stdout }}" @@ -106,7 +106,7 @@ # Disabling shard allocation right after enabling it seems redundant. Please see above for details. - name: Disable shard allocation for the cluster ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" method: PUT body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}' body_format: json @@ -116,7 +116,7 @@ - name: Stop non essential indexing to speed up shard recovery ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_flush" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_flush" method: POST user: elastic password: "{{ elasticstack_password.stdout }}" @@ -155,7 +155,7 @@ - name: Wait for elasticsearch node to come back up if it was stopped ansible.builtin.wait_for: - host: "localhost" + host: "{{ elasticsearch_api_host }}" port: "{{ elasticstack_elasticsearch_http_port }}" delay: 30 @@ -167,7 +167,7 @@ -u elastic:{{ elasticstack_password.stdout }} -s -m 2 - '{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' + '{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name' | grep -E '^{{ elasticsearch_nodename }}$' @@ -179,7 +179,7 @@ - name: Enable shard allocation for the cluster ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings" method: PUT body: '{ "persistent": { "cluster.routing.allocation.enable": null }}' body_format: json @@ -195,7 +195,7 @@ - name: Wait for cluster health to return to yellow or green ansible.builtin.uri: - url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health" method: GET user: elastic password: "{{ elasticstack_password.stdout }}" From 3d0787c1fb37522087ecf2257951b263ffee44d4 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 8 Mar 2024 14:23:47 +0100 Subject: [PATCH 62/67] Add faster upgrades for non-prod clusters --- docs/role-elasticsearch.md | 4 ++++ roles/elasticsearch/defaults/main.yml | 7 +++---- .../tasks/elasticsearch-rolling-upgrade.yml | 12 ++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md index 19a8d40f..4a45d9cf 100644 --- a/docs/role-elasticsearch.md +++ b/docs/role-elasticsearch.md @@ -54,6 +54,10 @@ This variable activates a workaround to start on systems that have certain harde * *elasticsearch_seed_hosts*: Set elasticsearch seed hosts * *elasticsearch_security_enrollment*: Controls enrollment (of nodes and Kibana) to a local node that’s been autoconfigured for security. +The following variable was only integrated to speed up upgrades of non-production clusters. Use with caution and at your own risk: + +* *elasticsearch_unsafe_upgrade_restart*: This will still perform rolling upgrades, but will first update the package and then restart the service. In contrast the default behaviour is to stop the service, do the upgrade and then start again. (default: `false`) + These variables are identical over all our elastic related roles, hence the different naming schemes. * *elasticstack_ca*: Set to the inventory hostname of the host that should house the CA for certificates for inter-node communication. (default: First node in the `elasticsearch` host group) diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index e9bf1921..34089ad6 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -31,10 +31,6 @@ elasticsearch_heap_dump_path: "/var/lib/elasticsearch" elasticsearch_jna_workaround: false -# The following variables are to be used when activating security -# They follow a different naming scheme to show that they are global -# to our set of Elastic Stack related Ansible roles - elasticsearch_initialized_file: "{{ elasticstack_initial_passwords | dirname }}/cluster_initialized" elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually elasticsearch_cert_validity_period: 1095 @@ -42,6 +38,9 @@ elasticsearch_cert_expiration_buffer: 30 elasticsearch_cert_will_expire_soon: false elasticsearch_ssl_verification_mode: full +# use this only for non-prod environments and at your own risk! +elasticsearch_unsafe_upgrade_restart: false + # only used internally elasticsearch_freshstart: changed: false diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 0a0fdfbe..dbdcc5d0 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -128,6 +128,8 @@ name: elasticsearch enabled: yes state: stopped + when: + - not elasticsearch_unsafe_upgrade_restart | bool - name: Install Elasticsearch - rpm fullstack ansible.builtin.package: @@ -152,6 +154,16 @@ state: started when: - elasticsearch_running.status.ActiveState == "active" + - not elasticsearch_unsafe_upgrade_restart | bool + + - name: Restart elasticsearch (fast, for non-prod) + ansible.builtin.service: + name: elasticsearch + enabled: yes + state: restarted + when: + - elasticsearch_running.status.ActiveState == "active" + - elasticsearch_unsafe_upgrade_restart | bool - name: Wait for elasticsearch node to come back up if it was stopped ansible.builtin.wait_for: From 7bc3a0b35d3e59e3f46878eb9e9c36a220e8dd4f Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 8 Mar 2024 14:26:41 +0100 Subject: [PATCH 63/67] Lint --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index dbdcc5d0..25dedceb 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -52,7 +52,7 @@ not elasticstack_full_stack | bool notify: - Restart Elasticsearch - + - name: Update single instances without extra caution - rpm fullstack ansible.builtin.package: name: "{{ elasticsearch_package }}" From 31089d9fd5c5f2f0133d7747a44fc6627efdb8b0 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 22 Mar 2024 16:32:28 +0100 Subject: [PATCH 64/67] Make docs about nodename more clear --- docs/role-elasticsearch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md index 4a45d9cf..37f642e4 100644 --- a/docs/role-elasticsearch.md +++ b/docs/role-elasticsearch.md @@ -13,7 +13,7 @@ Role Variables -------------- * *elasticsearch_node_types*: List of types of this very node. Please refer to [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for details. (default: not set. allowed value: array of types) -+ *elasticsearch_nodename*': Node name of the Elasticsearch node. (default: Hostname of the node as seen by Ansible) ++ *elasticsearch_nodename*': Node name of the Elasticsearch node. (default: value of `ansible_hostname`) * *elasticsearch_clustername*: Name the Elasticsearch Cluster (default: `elasticsearch`) * *elasticsearch_heap*: Heapsize for Elasticsearch. (Half of free memory on host. Maximum 30GB. (default: Half of hosts memory. Min 1GB, Max 30GB) * *elasticsearch_tls_key_passphrase*: Passphrase for elasticsearch certificates (default: `PleaseChangeMeIndividually`) From 53a65b33c8ae4ee6eaf144dc496e9745f68c58c6 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 22 Mar 2024 16:35:33 +0100 Subject: [PATCH 65/67] Be more clear about Installation task --- .../elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 25dedceb..17251e21 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -24,7 +24,7 @@ when: - elasticsearch_running.status.ActiveState == "inactive" block: - - name: Install Elasticsearch - rpm fullstack + - name: Install Elasticsearch - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: @@ -33,7 +33,7 @@ - ansible_os_family == "RedHat" - elasticstack_full_stack | bool - - name: Install Elasticsearch + - name: Install Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: @@ -44,7 +44,7 @@ when: - groups['elasticsearch'] | length == 1 block: - - name: Update single instances without extra caution - deb + - name: Update single instances without extra caution - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: @@ -53,7 +53,7 @@ notify: - Restart Elasticsearch - - name: Update single instances without extra caution - rpm fullstack + - name: Update single instances without extra caution - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: From d2b7458e12ba2175c022437bda4ef1f4abd3d0c1 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 22 Mar 2024 16:51:07 +0100 Subject: [PATCH 66/67] A bit more clarification --- .../elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 17251e21..3b42cdd2 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -24,7 +24,7 @@ when: - elasticsearch_running.status.ActiveState == "inactive" block: - - name: Install Elasticsearch - rpm with managed repositories + - name: Update stopped Elasticsearch - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: @@ -33,7 +33,7 @@ - ansible_os_family == "RedHat" - elasticstack_full_stack | bool - - name: Install Elasticsearch - deb or unmanaged repositories rpm + - name: Update stopped Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: @@ -131,7 +131,7 @@ when: - not elasticsearch_unsafe_upgrade_restart | bool - - name: Install Elasticsearch - rpm fullstack + - name: Update Elasticsearch - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" enablerepo: @@ -140,7 +140,7 @@ - ansible_os_family == "RedHat" - elasticstack_full_stack | bool - - name: Install Elasticsearch + - name: Update Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: From a3d021ef8d74876a040b15d7a8ce6c650fb14a6a Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 17 Apr 2024 17:41:28 +0200 Subject: [PATCH 67/67] Fix remaining legacy group names --- roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 4 ++-- roles/elasticsearch/tasks/main.yml | 2 +- roles/elasticstack/defaults/main.yml | 1 + roles/elasticstack/tasks/main.yml | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 3b42cdd2..19801a76 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -42,7 +42,7 @@ - name: Update single instances without extra caution when: - - groups['elasticsearch'] | length == 1 + - groups[elasticstack_elasticsearch_group_name] | length == 1 block: - name: Update single instances without extra caution - deb or unmanaged repositories rpm ansible.builtin.package: @@ -68,7 +68,7 @@ - name: Be careful about upgrade when Elasticsearch is running when: - elasticsearch_running.status.ActiveState == "active" - - groups['elasticsearch'] | length > 1 + - groups[elasticstack_elasticsearch_group_name] | length > 1 block: # Usually we should not need this step. It's only there to recover from broken upgrade plays diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 942fef98..7d99d877 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -133,7 +133,7 @@ - name: Update Elasticsearch if needed ansible.builtin.include_tasks: elasticsearch-rolling-upgrade.yml - with_items: "{{ groups['elasticsearch'] }}" + with_items: "{{ groups[elasticstack_elasticsearch_group_name] }}" when: - "hostvars[item].inventory_hostname == inventory_hostname" - elasticstack_version is defined diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index d9cc567f..bb35596e 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -2,6 +2,7 @@ elasticstack_elasticsearch_group_name: elasticsearch elasticstack_logstash_group_name: logstash +elasticstack_kibana_group_name: kibana elasticstack_beats_port: 5044 elasticstack_ca_dir: /opt/es-ca diff --git a/roles/elasticstack/tasks/main.yml b/roles/elasticstack/tasks/main.yml index bae82161..e8797325 100644 --- a/roles/elasticstack/tasks/main.yml +++ b/roles/elasticstack/tasks/main.yml @@ -15,10 +15,10 @@ - name: Set elasticstack_ca variable if not already set to Elasticsearch server ansible.builtin.set_fact: - elasticstack_ca: "{{ groups['logstash'][0] }}" + elasticstack_ca: "{{ groups[elasticstack_logstash_group_name][0] }}" when: - elasticstack_ca is undefined - - groups['logstash'][0] is defined + - groups[elasticstack_logstash_group_name][0] is defined - name: Set versions for components ansible.builtin.import_tasks: elasticstack-versions.yml