Adds tools for idracs

etc/kayobe/ansible/drac-cancel-all-racadm.yml

@@ -0,0 +1,7 @@
+---
+
+- hosts: baremetal-compute,overcloud
+  gather_facts: false
+  tasks:
+    - command: /opt/dell/srvadmin/bin/idracadm7 -r {{ ipmi_address }} -u root -p {{ secrets_ipmi_password }}  jobqueue delete -i JID_CLEARALL_FORCE
+      delegate_to: localhost

etc/kayobe/ansible/drac-cancel-jobs.yml

@@ -0,0 +1,23 @@
+---
+- hosts: overcloud,baremetal-compute
+  gather_facts: false
+  connection: local
+
+  collections:
+   - dellemc.openmanage
+
+  vars:
+      secrets_ipmi_username: "{{ ipmi_admin_user }}"
+      secrets_ipmi_password: "{{ ipmi_admin_password }}"
+      drac_export_path: "{{ playbook_dir }}/idrac_export/"
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+  tasks:
+    - name: Cancel all drac jobs
+      dellemc.openmanage.idrac_lifecycle_controller_jobs:
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        validate_certs: false
+      delegate_to: localhost
+      when: ipmi_address is defined

etc/kayobe/ansible/drac-firmware-update.yml

@@ -0,0 +1,85 @@
+---
+- hosts: baremetal-compute,overcloud
+  gather_facts: false
+  connection: local
+  serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}"
+
+  collections:
+   - dellemc.openmanage
+
+  vars:
+    # Run with ``-e dell_drm_apply_update=true`` to apply the firmware updates.
+    dell_drm_apply_update: False
+    dell_drm_address: "https://{{ oob_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}"
+    secrets_ipmi_username: "{{ ipmi_admin_user }}"
+    secrets_ipmi_password: "{{ ipmi_admin_password }}"
+    # Look at the README for more details, but you need a repo
+    # created in drm, and exported via a webserver:
+    # drm --create -r=idrac_repo_joint --inputplatformlist=R640,R6525
+    dell_drm_repo: "{{ ukaea_dell_drm_repo }}"
+    controller_host: "{{ groups['controllers'][0] }}"
+    venv: "{{ virtualenv_path }}/openstack-cli"
+  tasks:
+    - name: Set maintenance mode
+      command: |-
+        docker exec bifrost_deploy bash -c '
+          OS_CLOUD=bifrost openstack baremetal node maintenance set \
+            --reason "Running drac-firmware-update.yml" \
+            {{ inventory_hostname }}'
+      become: true
+      delegate_to: "{{ groups.seed.0 }}"
+      vars:
+        ansible_connection: ssh
+        ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}"
+      when: '"overcloud" in group_names'
+
+    - name: Set maintenance
+      command: "{{ venv }}/bin/openstack baremetal node maintenance set {{ inventory_hostname }} --reason updates"
+      vars:
+        ansible_connection: ssh
+        ansible_host: "{{ hostvars[controller_host].ansible_host }}"
+        ansible_python_interpreter: "/opt/kayobe/venvs/openstack-cli/bin/python3"
+      environment: "{{ openstack_auth_env }}"
+      delegate_to: "{{ controller_host }}"
+      when: '"overcloud" not in group_names'
+
+    - name: Update firmware from repository on a HTTP
+      idrac_firmware:
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        reboot: True
+        job_wait: True
+        apply_update: "{{ dell_drm_apply_update | bool }}"
+        share_name: "{{ dell_drm_address }}"
+        catalog_file_name: "{{ dell_drm_repo }}"
+        ignore_cert_warning: True
+        validate_certs: false
+      register: idrac_firmware_output
+      delegate_to: localhost
+      when: ipmi_address is defined
+    - debug:
+        msg: "{{ idrac_firmware_output }}"
+
+    - name: Unset maintenance mode
+      command: |-
+        docker exec bifrost_deploy bash -c '
+          OS_CLOUD=bifrost openstack baremetal node maintenance unset \
+            {{ inventory_hostname }}'
+      become: true
+      delegate_to: "{{ groups.seed.0 }}"
+      vars:
+        ansible_connection: ssh
+        ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}"
+      when: "'overcloud' in group_names"
+
+    - name: Unset maintenance
+      command: "{{ venv }}/bin/openstack baremetal node maintenance unset {{ inventory_hostname }}"
+      vars:
+        ansible_connection: ssh
+        ansible_host: "{{ hostvars[controller_host].ansible_host }}"
+        ansible_python_interpreter: "/opt/kayobe/venvs/openstack-cli/bin/python3"
+      environment: "{{ openstack_auth_env }}"
+      delegate_to: "{{ controller_host }}"
+      when: "'overcloud' not in group_names"
+

etc/kayobe/ansible/drac-ping.yml

@@ -0,0 +1,7 @@
+---
+
+- hosts: baremetal-compute
+  gather_facts: false
+  tasks:
+    - command: ping -c 1 {{ ipmi_address }}
+      delegate_to: localhost

etc/kayobe/ansible/drac-profile-apply-idrac.yml

@@ -0,0 +1,44 @@
+---
+- hosts:
+    - overcloud
+    - baremetal-compute
+    # Seemed to break connectivity
+    - !lustre
+  gather_facts: false
+  connection: local
+
+  collections:
+   - dellemc.openmanage
+
+  vars:
+      secrets_ipmi_username: "{{ ipmi_admin_user }}"
+      secrets_ipmi_password: "{{ ipmi_admin_password }}"
+      # Annoyingly updating eventfilters requires a reboot
+      drac_shutdown_type: "Graceful" # NoReboot or Graceful or Forced
+      drac_export_path: "{{ playbook_dir }}"
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+  tasks:
+    - name: Template golden config
+      template:
+        src: server-profiles/golden_IDRAC.json.j2
+        dest: "server-profiles/golden_IDRAC_{{ inventory_hostname }}.json"
+        mode: 0600
+
+    - name: Import Server Configuration Profile (IDRAC)
+      idrac_server_config_profile:
+        command: "import"
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        share_name: "{{ drac_export_path }}"
+        scp_file: "server-profiles/golden_IDRAC_{{ inventory_hostname}}.json"
+        job_wait: true
+        end_host_power_state: "On"
+        shutdown_type: "{{ drac_shutdown_type }}"
+        scp_components: ["IDRAC", "EventFilters"]
+        validate_certs: false
+        timeout: 60
+      register: idrac_bios_output
+      delegate_to: localhost
+      when: ipmi_address is defined

etc/kayobe/ansible/drac-profile-apply.yml

@@ -0,0 +1,100 @@
+---
+- hosts:
+    - overcloud
+    - baremetal-compute  
+  gather_facts: false
+  connection: local
+
+  collections:
+   - dellemc.openmanage
+
+  vars:
+      secrets_ipmi_username: "{{ ipmi_admin_user }}"
+      secrets_ipmi_password: "{{ ipmi_admin_password }}"
+
+      # If you re-apply RAID, this recreates the disk!
+      apply_raid: False
+      drac_shutdown_type: "Graceful" # NoReboot or Graceful or Forced
+      drac_export_path: "{{ playbook_dir }}"
+      drac_profile_query_ironic: true
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+  tasks:
+    #FIXME(wszumski): Unserstand what this does
+    # - name: Import Server Configuration Profile (NIC)
+    #   idrac_server_config_profile:
+    #     command: "import"
+    #     idrac_ip: "{{ ipmi_address }}"
+    #     idrac_user: "{{ secrets_ipmi_username }}"
+    #     idrac_password: "{{ secrets_ipmi_password }}"
+    #     share_name: "{{ drac_export_path }}"
+    #     scp_file: "golden_BIOS_no_embedded_pxe.json"
+    #     job_wait: True
+    #     end_host_power_state: "On"
+    #     shutdown_type: "{{ drac_shutdown_type }}"
+    #     scp_components: "NIC"
+    #   register: idrac_nic_output
+    #   delegate_to: localhost
+    #   when: ipmi_address is defined
+    #
+
+    - block: 
+      - name: Gather ironic state
+        openstack.cloud.baremetal_node_info:
+          name: "{{ inventory_hostname }}"
+        register: node
+        vars:
+          ansible_python_interpreter: "{{ ansible_playbook_python }}"
+          ansible_connection: local
+      - name: Fail if hardware fault flag set
+        fail:
+          msg: >-
+            Node has been flagged as having a hardware fault. Please clear the extra/kayobe_hardware_fault
+            flag with `openstack baremetal node unset --extra kayobe_hardware_fault_<type>` if the fault has
+            been resolved.
+        when: node.nodes.0.extra | map('regex_search', '^kayobe_hardware_fault') | select | list | default([]) | truthy
+      when: drac_profile_query_ironic | bool
+
+    - name: Import Server Configuration Profile (BIOS)
+      idrac_server_config_profile:
+        command: "import"
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        share_name: "{{ drac_export_path }}"
+        scp_file: "server-profiles/golden_BIOS_{{ drac_profile }}.json"
+        job_wait: True
+        end_host_power_state: "On"
+        shutdown_type: "{{ drac_shutdown_type }}"
+        scp_components: "BIOS"
+        validate_certs: false
+        timeout: 60
+      register: idrac_bios_output
+      delegate_to: localhost
+      when: ipmi_address is defined
+
+    - name: Import Server Configuration Profile (RAID)
+      idrac_server_config_profile:
+        command: "import"
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        share_name: "{{ drac_export_path }}"
+        scp_file: "server-profiles/golden_RAID_{{ drac_profile }}.json"
+        job_wait: True
+        end_host_power_state: "On"
+        shutdown_type: "{{ drac_shutdown_type }}"
+        validate_certs: false
+      register: idrac_bios_output
+      register: idrac_raid_output
+      delegate_to: localhost
+      when: ipmi_address is defined and apply_raid
+
+    - name: Reset iDRAC if bios settings were updated
+      idrac_reset:
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        validate_certs: false
+      register: idrac_bios_output
+      when: ipmi_address is defined and idrac_bios_output.changed

etc/kayobe/ansible/drac-profile-export.yml

@@ -0,0 +1,81 @@
+---
+- hosts: overcloud,baremetal-compute
+  gather_facts: false
+  connection: local
+
+  collections:
+   - dellemc.openmanage
+
+  vars:
+      secrets_ipmi_username: "{{ ipmi_admin_user }}"
+      secrets_ipmi_password: "{{ ipmi_admin_password }}"
+      drac_export_path: "{{ playbook_dir }}/idrac_export/"
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
+
+  tasks:
+    - name: Get System Inventory
+      idrac_system_info:
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        validate_certs: false
+      delegate_to: localhost
+      register: idrac_system_info
+      when: ipmi_address is defined
+
+    - set_fact:
+        model_name: "{{ idrac_system_info['system_info']['System'][0]['Model'] | replace('PowerEdge ','') }}"
+    - name: get model
+      debug:
+        msg: "{{ model_name }}"
+
+    - name: Get Installed Firmware Inventory
+      dellemc.openmanage.idrac_firmware_info:
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        validate_certs: false
+      delegate_to: localhost
+      register: idrac_firmware_output
+      when: ipmi_address is defined
+
+    - name: List installed components with firmware listed
+      debug:
+        msg: "{{ inventory_hostname }} hardware: {{ idrac_firmware_output['firmware_info']['Firmware'] | map(attribute='FQDD') | list | sort }}"
+
+    - name: Extract key firmware versions
+      debug:
+        msg: "{{ inventory_hostname }}:{{ item }}: {{ idrac_firmware_output['firmware_info']['Firmware'] | selectattr('FQDD', 'equalto', item) | map(attribute='VersionString') | list }}"
+      loop:
+        - "BIOS.Setup.1-1"
+        - "iDRAC.Embedded.1-1"
+        - "NIC.Slot.2-1-1" # Mellaonx NIC
+
+    - name: Ensure output dir exists
+      file:
+        state: directory
+        path: "{{ drac_export_path }}"
+      delegate_to: localhost
+      run_once: True
+
+    - name: Export Server Configuration Profile
+      idrac_server_config_profile:
+        command: "export"
+        export_format: "JSON"
+        export_use: "Clone"
+        idrac_ip: "{{ ipmi_address }}"
+        idrac_user: "{{ secrets_ipmi_username }}"
+        idrac_password: "{{ secrets_ipmi_password }}"
+        share_name: "{{ drac_export_path }}"
+        scp_file: "{{ inventory_hostname }}_{{ item }}.json"
+        job_wait: "True"
+        scp_components: "{{ item }}"
+        validate_certs: false
+      register: idrac_profile_output
+      delegate_to: localhost
+      when: ipmi_address is defined
+      loop:
+        - "BIOS"
+        - "NIC"
+        - "RAID"
+        - "ALL"

etc/kayobe/ansible/requirements.yml

@@ -12,6 +12,8 @@ collections:
     version: 2.7.1
   - name: stackhpc.kayobe_workflows
     version: 1.2.0
+  - name: dellemc.openmanage
+    version: 9.10.0
 roles:
   - src: stackhpc.vxlan
     version: 1.1.0

requirements.txt

@@ -1,3 +1,7 @@
 kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.1.0.6
 ansible-modules-hashivault>=5.3.0
 jmespath
+omsdk==1.2.503
+# FIXME: I have no recollection of why I pinned these
+pysnmp==4.4.12
+urllib3<2