diff --git a/etc/kayobe/ansible/drac-cancel-all-racadm.yml b/etc/kayobe/ansible/drac-cancel-all-racadm.yml new file mode 100644 index 000000000..988789882 --- /dev/null +++ b/etc/kayobe/ansible/drac-cancel-all-racadm.yml @@ -0,0 +1,7 @@ +--- + +- hosts: baremetal-compute,overcloud + gather_facts: false + tasks: + - command: /opt/dell/srvadmin/bin/idracadm7 -r {{ ipmi_address }} -u root -p {{ secrets_ipmi_password }} jobqueue delete -i JID_CLEARALL_FORCE + delegate_to: localhost diff --git a/etc/kayobe/ansible/drac-cancel-jobs.yml b/etc/kayobe/ansible/drac-cancel-jobs.yml new file mode 100644 index 000000000..5a15a8c2c --- /dev/null +++ b/etc/kayobe/ansible/drac-cancel-jobs.yml @@ -0,0 +1,23 @@ +--- +- hosts: overcloud,baremetal-compute + gather_facts: false + connection: local + + collections: + - dellemc.openmanage + + vars: + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + drac_export_path: "{{ playbook_dir }}/idrac_export/" + ansible_python_interpreter: "{{ ansible_playbook_python }}" + + tasks: + - name: Cancel all drac jobs + dellemc.openmanage.idrac_lifecycle_controller_jobs: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + validate_certs: false + delegate_to: localhost + when: ipmi_address is defined diff --git a/etc/kayobe/ansible/drac-firmware-update.yml b/etc/kayobe/ansible/drac-firmware-update.yml new file mode 100644 index 000000000..e95eaee17 --- /dev/null +++ b/etc/kayobe/ansible/drac-firmware-update.yml @@ -0,0 +1,85 @@ +--- +- hosts: baremetal-compute,overcloud + gather_facts: false + connection: local + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}" + + collections: + - dellemc.openmanage + + vars: + # Run with ``-e dell_drm_apply_update=true`` to apply the firmware updates. + dell_drm_apply_update: False + dell_drm_address: "https://{{ oob_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + # Look at the README for more details, but you need a repo + # created in drm, and exported via a webserver: + # drm --create -r=idrac_repo_joint --inputplatformlist=R640,R6525 + dell_drm_repo: "{{ ukaea_dell_drm_repo }}" + controller_host: "{{ groups['controllers'][0] }}" + venv: "{{ virtualenv_path }}/openstack-cli" + tasks: + - name: Set maintenance mode + command: |- + docker exec bifrost_deploy bash -c ' + OS_CLOUD=bifrost openstack baremetal node maintenance set \ + --reason "Running drac-firmware-update.yml" \ + {{ inventory_hostname }}' + become: true + delegate_to: "{{ groups.seed.0 }}" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}" + when: '"overcloud" in group_names' + + - name: Set maintenance + command: "{{ venv }}/bin/openstack baremetal node maintenance set {{ inventory_hostname }} --reason updates" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[controller_host].ansible_host }}" + ansible_python_interpreter: "/opt/kayobe/venvs/openstack-cli/bin/python3" + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ controller_host }}" + when: '"overcloud" not in group_names' + + - name: Update firmware from repository on a HTTP + idrac_firmware: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + reboot: True + job_wait: True + apply_update: "{{ dell_drm_apply_update | bool }}" + share_name: "{{ dell_drm_address }}" + catalog_file_name: "{{ dell_drm_repo }}" + ignore_cert_warning: True + validate_certs: false + register: idrac_firmware_output + delegate_to: localhost + when: ipmi_address is defined + - debug: + msg: "{{ idrac_firmware_output }}" + + - name: Unset maintenance mode + command: |- + docker exec bifrost_deploy bash -c ' + OS_CLOUD=bifrost openstack baremetal node maintenance unset \ + {{ inventory_hostname }}' + become: true + delegate_to: "{{ groups.seed.0 }}" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}" + when: "'overcloud' in group_names" + + - name: Unset maintenance + command: "{{ venv }}/bin/openstack baremetal node maintenance unset {{ inventory_hostname }}" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[controller_host].ansible_host }}" + ansible_python_interpreter: "/opt/kayobe/venvs/openstack-cli/bin/python3" + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ controller_host }}" + when: "'overcloud' not in group_names" + diff --git a/etc/kayobe/ansible/drac-ping.yml b/etc/kayobe/ansible/drac-ping.yml new file mode 100644 index 000000000..36fc2bec4 --- /dev/null +++ b/etc/kayobe/ansible/drac-ping.yml @@ -0,0 +1,7 @@ +--- + +- hosts: baremetal-compute + gather_facts: false + tasks: + - command: ping -c 1 {{ ipmi_address }} + delegate_to: localhost diff --git a/etc/kayobe/ansible/drac-profile-apply-idrac.yml b/etc/kayobe/ansible/drac-profile-apply-idrac.yml new file mode 100644 index 000000000..e3c0adf7f --- /dev/null +++ b/etc/kayobe/ansible/drac-profile-apply-idrac.yml @@ -0,0 +1,44 @@ +--- +- hosts: + - overcloud + - baremetal-compute + # Seemed to break connectivity + - !lustre + gather_facts: false + connection: local + + collections: + - dellemc.openmanage + + vars: + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + # Annoyingly updating eventfilters requires a reboot + drac_shutdown_type: "Graceful" # NoReboot or Graceful or Forced + drac_export_path: "{{ playbook_dir }}" + ansible_python_interpreter: "{{ ansible_playbook_python }}" + + tasks: + - name: Template golden config + template: + src: server-profiles/golden_IDRAC.json.j2 + dest: "server-profiles/golden_IDRAC_{{ inventory_hostname }}.json" + mode: 0600 + + - name: Import Server Configuration Profile (IDRAC) + idrac_server_config_profile: + command: "import" + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + share_name: "{{ drac_export_path }}" + scp_file: "server-profiles/golden_IDRAC_{{ inventory_hostname}}.json" + job_wait: true + end_host_power_state: "On" + shutdown_type: "{{ drac_shutdown_type }}" + scp_components: ["IDRAC", "EventFilters"] + validate_certs: false + timeout: 60 + register: idrac_bios_output + delegate_to: localhost + when: ipmi_address is defined diff --git a/etc/kayobe/ansible/drac-profile-apply.yml b/etc/kayobe/ansible/drac-profile-apply.yml new file mode 100644 index 000000000..dd0538fbc --- /dev/null +++ b/etc/kayobe/ansible/drac-profile-apply.yml @@ -0,0 +1,100 @@ +--- +- hosts: + - overcloud + - baremetal-compute + gather_facts: false + connection: local + + collections: + - dellemc.openmanage + + vars: + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + + # If you re-apply RAID, this recreates the disk! + apply_raid: False + drac_shutdown_type: "Graceful" # NoReboot or Graceful or Forced + drac_export_path: "{{ playbook_dir }}" + drac_profile_query_ironic: true + ansible_python_interpreter: "{{ ansible_playbook_python }}" + + tasks: + #FIXME(wszumski): Unserstand what this does + # - name: Import Server Configuration Profile (NIC) + # idrac_server_config_profile: + # command: "import" + # idrac_ip: "{{ ipmi_address }}" + # idrac_user: "{{ secrets_ipmi_username }}" + # idrac_password: "{{ secrets_ipmi_password }}" + # share_name: "{{ drac_export_path }}" + # scp_file: "golden_BIOS_no_embedded_pxe.json" + # job_wait: True + # end_host_power_state: "On" + # shutdown_type: "{{ drac_shutdown_type }}" + # scp_components: "NIC" + # register: idrac_nic_output + # delegate_to: localhost + # when: ipmi_address is defined + # + + - block: + - name: Gather ironic state + openstack.cloud.baremetal_node_info: + name: "{{ inventory_hostname }}" + register: node + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + ansible_connection: local + - name: Fail if hardware fault flag set + fail: + msg: >- + Node has been flagged as having a hardware fault. Please clear the extra/kayobe_hardware_fault + flag with `openstack baremetal node unset --extra kayobe_hardware_fault_` if the fault has + been resolved. + when: node.nodes.0.extra | map('regex_search', '^kayobe_hardware_fault') | select | list | default([]) | truthy + when: drac_profile_query_ironic | bool + + - name: Import Server Configuration Profile (BIOS) + idrac_server_config_profile: + command: "import" + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + share_name: "{{ drac_export_path }}" + scp_file: "server-profiles/golden_BIOS_{{ drac_profile }}.json" + job_wait: True + end_host_power_state: "On" + shutdown_type: "{{ drac_shutdown_type }}" + scp_components: "BIOS" + validate_certs: false + timeout: 60 + register: idrac_bios_output + delegate_to: localhost + when: ipmi_address is defined + + - name: Import Server Configuration Profile (RAID) + idrac_server_config_profile: + command: "import" + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + share_name: "{{ drac_export_path }}" + scp_file: "server-profiles/golden_RAID_{{ drac_profile }}.json" + job_wait: True + end_host_power_state: "On" + shutdown_type: "{{ drac_shutdown_type }}" + validate_certs: false + register: idrac_bios_output + register: idrac_raid_output + delegate_to: localhost + when: ipmi_address is defined and apply_raid + + - name: Reset iDRAC if bios settings were updated + idrac_reset: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + validate_certs: false + register: idrac_bios_output + when: ipmi_address is defined and idrac_bios_output.changed diff --git a/etc/kayobe/ansible/drac-profile-export.yml b/etc/kayobe/ansible/drac-profile-export.yml new file mode 100644 index 000000000..8068b32a1 --- /dev/null +++ b/etc/kayobe/ansible/drac-profile-export.yml @@ -0,0 +1,81 @@ +--- +- hosts: overcloud,baremetal-compute + gather_facts: false + connection: local + + collections: + - dellemc.openmanage + + vars: + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + drac_export_path: "{{ playbook_dir }}/idrac_export/" + ansible_python_interpreter: "{{ ansible_playbook_python }}" + + tasks: + - name: Get System Inventory + idrac_system_info: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + validate_certs: false + delegate_to: localhost + register: idrac_system_info + when: ipmi_address is defined + + - set_fact: + model_name: "{{ idrac_system_info['system_info']['System'][0]['Model'] | replace('PowerEdge ','') }}" + - name: get model + debug: + msg: "{{ model_name }}" + + - name: Get Installed Firmware Inventory + dellemc.openmanage.idrac_firmware_info: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + validate_certs: false + delegate_to: localhost + register: idrac_firmware_output + when: ipmi_address is defined + + - name: List installed components with firmware listed + debug: + msg: "{{ inventory_hostname }} hardware: {{ idrac_firmware_output['firmware_info']['Firmware'] | map(attribute='FQDD') | list | sort }}" + + - name: Extract key firmware versions + debug: + msg: "{{ inventory_hostname }}:{{ item }}: {{ idrac_firmware_output['firmware_info']['Firmware'] | selectattr('FQDD', 'equalto', item) | map(attribute='VersionString') | list }}" + loop: + - "BIOS.Setup.1-1" + - "iDRAC.Embedded.1-1" + - "NIC.Slot.2-1-1" # Mellaonx NIC + + - name: Ensure output dir exists + file: + state: directory + path: "{{ drac_export_path }}" + delegate_to: localhost + run_once: True + + - name: Export Server Configuration Profile + idrac_server_config_profile: + command: "export" + export_format: "JSON" + export_use: "Clone" + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + share_name: "{{ drac_export_path }}" + scp_file: "{{ inventory_hostname }}_{{ item }}.json" + job_wait: "True" + scp_components: "{{ item }}" + validate_certs: false + register: idrac_profile_output + delegate_to: localhost + when: ipmi_address is defined + loop: + - "BIOS" + - "NIC" + - "RAID" + - "ALL" diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 3aceec6d4..49ff809d8 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -12,6 +12,8 @@ collections: version: 2.7.1 - name: stackhpc.kayobe_workflows version: 1.2.0 + - name: dellemc.openmanage + version: 9.10.0 roles: - src: stackhpc.vxlan version: 1.1.0 diff --git a/requirements.txt b/requirements.txt index 13bccc4bb..770538012 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.1.0.6 ansible-modules-hashivault>=5.3.0 jmespath +omsdk==1.2.503 +# FIXME: I have no recollection of why I pinned these +pysnmp==4.4.12 +urllib3<2