Changes from 500 VMs Hybrid work

.gitignore

@@ -16,4 +16,5 @@ out
 gen
 .idea/
 .idea/workspace.xml
-
+*.log
+*.orig

ansible/.gitignore

@@ -1 +1,2 @@
 smcipmitool.tar.gz
+*.sw*

ansible/copy-pull-secret.yml

@@ -0,0 +1,15 @@
+---
+# Copy pull secret playbook
+#
+# This playbook is used to copy the pull secret to the nodes in the cluster.
+# It is used to updae the pull secret on nodes to pull images from the Red Hat registry.
+#
+# Example Usage:
+#
+# ansible-playbook ansible/copy-pull-secret.yml
+#
+
+- name: Copies pull secret to nodes
+  hosts: hv_vm
+  roles:
+  - copy-pull-secret

ansible/hv-vm-start-one.yml

@@ -0,0 +1,6 @@
+---
+- name: start one VMs
+  gather_facts: false
+  hosts: hv
+  roles:
+  - hv-vm-start

ansible/hv-vm-stop-all.yml

@@ -0,0 +1,5 @@
+---
+- name: destroy all VMs
+  hosts: hv
+  roles:
+  - hv-vm-destroy

ansible/mno-add-vm-workers.yml

@@ -0,0 +1,121 @@
+---
+# Create and deploy a cluster with the Assisted Installer
+#
+# Example Usage:
+#
+# ansible-playbook -i ansible/inventory/cloud42.local ansible/mno-deploy.yml
+#
+
+- name: Prep cluster to add hosts
+  hosts: bastion
+  vars_files:
+  - vars/lab.yml
+  - vars/all.yml
+  gather_facts: false
+  tasks:
+  - name: Set assisted installer connection
+    set_fact:
+      assisted_installer_host: "{{ groups['bastion'][0] }}"
+      assisted_installer_port: "8090"
+
+  - name: Get cluster status
+    uri:
+      url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/clusters/{{ ai_cluster_id }}"
+      method: GET
+      body_format: json
+      status_code: [200]
+      return_content: true
+    register: cluster_data
+    failed_when: cluster_data.json.status not in ['installed', 'adding-hosts']
+
+  - name: Set cluster status to adding-hosts
+    uri:
+      url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/clusters/{{ ai_cluster_id }}/actions/allow-add-workers"
+      method: POST
+      body_format: json
+      status_code: [201, 202]
+    when: cluster_data.json.status == 'installed'
+
+  - name: Get infra-env
+    uri:
+      url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/clusters/{{ ai_cluster_id }}"
+      method: GET
+      body_format: json
+      status_code: [200]
+      return_content: true
+    register: infra_env_return
+
+  - name: Set ai_infraenv_id
+    set_fact:
+      ai_infraenv_id: "{{ infra_env_return.json.hosts[0].infra_env_id }}"
+
+  - name: Get infra-env static_network_config
+    uri:
+      url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/infra-envs/{{ ai_infraenv_id }}"
+      method: GET
+      body_format: json
+      status_code: [200]
+      return_content: true
+    register: infra_env_return
+
+  - name: Set ai_infraenv_static_config
+    set_fact:
+      ai_infraenv_static_config: "{{ infra_env_return.json.static_network_config }}"
+
+  - name: Set empty static network configuration
+    set_fact:
+      static_network_config: []
+
+  - name: Generate Static Network Config for VMs
+    ansible.builtin.include_role:
+      name: create-ai-cluster
+      tasks_from: static_network_config
+    vars:
+      hybrid_worker_count: "{{ add_worker_count }}"
+    loop: "{{ groups['hv_vm'][:hybrid_worker_count | int] }}"
+
+  - name: show ai_infraenv_static_config
+    debug:
+      var: ai_infraenv_static_config
+
+  - name: show static_network_config
+    debug:
+      var: static_network_config
+
+  - name: Set static network composite
+    set_fact:
+      static_network_config_comp: "{{ static_network_config + ai_infraenv_static_config }}"
+
+  - name: show static_network_config composite
+    debug:
+      var: static_network_config_comp
+
+  - name: Update static config
+    uri:
+      url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/infra-envs/{{ ai_infraenv_id }}"
+      body: {
+        "static_network_config": "{{ static_network_config + ai_infraenv_static_config }}"
+      }
+      method: PATCH
+      body_format: json
+      status_code: [201]
+      return_content: true
+
+
+- name: Boot / Install VMs
+  hosts: bastion
+  vars_files:
+  - vars/lab.yml
+  - vars/all.yml
+  roles:
+  - generate-discovery-iso
+  - role: boot-iso
+    vars:
+      inventory_group: hv_vm
+      index: "{{ add_worker_count }}"
+      virtual_media_iso: "discovery.iso"
+  - role: wait-hosts-discovered
+    vars:
+      inventory_nodes: "{{ groups['hv_vm'][:add_worker_count|int] }}"
+      discover_nodes: "{{ groups['hv_vm'][:add_worker_count|int] }}"
+  - add-hosts-install

ansible/roles/copy-pull-secret/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Copy pull secret
+  copy:
+    src: pull-secret.json
+    dest: "/var/lib/kubelet/config.json"
+  become: true
+- name: touch force update
+  file:
+    path: /run/machine-config-daemon-force
+    state: touch
+  become: true

ansible/roles/hv-install/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+# hv-install default vars
+
+# Hugepages configuration for hypervisors
+enable_hugepages: false
+
+# Hugepage size: 2M or 1G
+hugepage_size: "1G"
+
+# Number of hugepages to allocate (e.g., 32 for 32GB of 1G hugepages)
+hugepage_count: 32
+
+# Additional kernel parameters for performance tuning
+additional_kernel_params: []
+
+# Number of hugepages per node (e.g. total / 2)
+hugepages_count_per_node: 190

ansible/roles/hv-install/tasks/main.yml

@@ -21,6 +21,55 @@
     name: sushy-tools
     version: 1.2.0
 
+- name: Configure hugepages support
+  when: enable_hugepages
+  block:
+
+  - name: Run grubby to add hugepages arguments
+    command: grubby --update-kernel=ALL --args="default_hugepagesz={{ hugepage_size }} hugepagesz={{ hugepage_size }}"
+    register: grub_updated
+
+  - name: Set reboot required flag
+    set_fact:
+      hugepages_reboot_required: true
+    when: grub_updated.changed
+
+  - name: Create hugetlb-gigantic-pages.service file
+    copy:
+      dest: /usr/lib/systemd/system/hugetlb-gigantic-pages.service
+      content: |
+        [Unit]
+        Description=HugeTLB Gigantic Pages Reservation
+        DefaultDependencies=no
+        Before=dev-hugepages.mount
+        ConditionPathExists=/sys/devices/system/node
+        ConditionKernelCommandLine=hugepagesz=1G
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/lib/systemd/hugetlb-reserve-pages.sh
+
+        [Install]
+        WantedBy=sysinit.target
+
+  - name: Create hugetlb-reserve-pages.sh
+    template:
+      src: hugetlb-reserve-pages.sh.j2
+      dest: /usr/lib/systemd/hugetlb-reserve-pages.sh
+      mode: "0755"
+    register: hugetlb_script
+
+  - name: Set reboot required flag
+    set_fact:
+      hugepages_reboot_required: true
+    when: hugetlb_script.changed
+
+  - name: Enable hugetlb-gigantic-pages.service
+    systemd:
+      enabled: true
+      name: hugetlb-gigantic-pages.service
+
 - name: Get coredns
   get_url:
     validate_certs: false
@@ -65,3 +114,21 @@
     state: started
     enabled: true
     name: ksmtuned
+
+- name: Reboot hypervisor for hugepages configuration
+  when:
+  - enable_hugepages
+  - hugepages_reboot_required | default(false)
+  block:
+  - name: Reboot hypervisor
+    reboot:
+      msg: "Rebooting to apply hugepages configuration"
+      reboot_timeout: 600
+
+  - name: Verify hugepages are configured
+    shell: cat /proc/meminfo | grep -E "HugePages_Total|HugePages_Free|Hugepagesize"
+    register: hugepages_status
+
+  - name: Display hugepages status
+    debug:
+      msg: "{{ hugepages_status.stdout_lines }}"

ansible/roles/hv-install/templates/hugetlb-reserve-pages.sh.j2

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+nodes_path=/sys/devices/system/node/
+if [ ! -d $nodes_path ]; then
+    echo "ERROR: $nodes_path does not exist"
+    exit 1
+fi
+
+reserve_pages()
+{
+    echo $1 > $nodes_path/$2/hugepages/hugepages-1048576kB/nr_hugepages
+}
+
+reserve_pages {{ hugepages_count_per_node }} node0
+reserve_pages {{ hugepages_count_per_node }} node1

ansible/roles/hv-vm-create/defaults/main.yml

@@ -10,7 +10,7 @@ vnuma_enabled: false
 vnuma_memory_placement: "static"
 vnuma_cpu_placement: "static"
 
-# Manual vNUMA configuration 
+# Manual vNUMA configuration
 # vnuma_nodes:
 #   - id: 0
 #     cpus: "0-3"
@@ -21,4 +21,19 @@ vnuma_cpu_placement: "static"
 
 # vNUMA topology settings
 vnuma_memory_mode: "strict"  # strict, preferred, interleave
-vnuma_cpu_mode: "strict"     # strict, preferred                                                
+vnuma_cpu_mode: "strict"     # strict, preferred
+
+# Hugepages configuration for VMs
+vm_hugepages: false
+
+# Hugepage size for VMs: 2M or 1G
+vm_hugepage_size: "1G"
+
+# Number of hugepages to allocate per VM (auto-calculated based on VM memory if not specified)
+vm_hugepage_count:
+
+# Hugepage mount path in VMs
+vm_hugepage_mount: "/mnt/hugepages"
+
+# Enable IGB NICs for VMs
+vm_igb_nics: false

ansible/roles/hv-vm-create/tasks/main.yml

@@ -20,6 +20,33 @@
   set_fact:
     hv_vm_cpu_count: "{{ hostvars[inventory_hostname]['cpus'] }}"
 
+- name: Configure VM hugepages
+  when: vm_hugepages
+  block:
+    - name: Calculate hugepages needed for VM if not specified
+      set_fact:
+        calculated_vm_hugepage_count: "{{ (hostvars[inventory_hostname]['memory'] | int) // (vm_hugepage_size[:-1] | int) }}"
+      when: vm_hugepage_count is not defined or vm_hugepage_count == ""
+
+    - name: Set hugepage count for VM
+      set_fact:
+        vm_hugepages_needed: "{{ vm_hugepage_count if vm_hugepage_count is defined and vm_hugepage_count != '' else calculated_vm_hugepage_count }}"
+
+    - name: Check host hugepages availability
+      shell: |
+        grep -E "HugePages_Free.*{{ vm_hugepage_size }}" /proc/meminfo | awk '{print $2}' || echo "0"
+      register: host_hugepages_free
+      delegate_to: "{{ hostvars[inventory_hostname]['ansible_host'] }}"
+
+    - name: Validate sufficient hugepages available
+      fail:
+        msg: "Not enough {{ vm_hugepage_size }} hugepages available on host {{ hostvars[inventory_hostname]['ansible_host'] }}. Need: {{ vm_hugepages_needed }}, Available: {{ host_hugepages_free.stdout }}"
+      when: (host_hugepages_free.stdout | int) < (vm_hugepages_needed | int)
+
+    - name: Display hugepages configuration for VM
+      debug:
+        msg: "VM {{ inventory_hostname }} will use {{ vm_hugepages_needed }} {{ vm_hugepage_size }} hugepages ({{ (vm_hugepages_needed | int) * (vm_hugepage_size[:-1] | int) }}G total)"
+
 - name: Set vNUMA configuration tasks
   when: vnuma_enabled
   block:

ansible/roles/hv-vm-create/templates/kvm-def.xml.j2

@@ -3,6 +3,13 @@
   <uuid>{{ hostvars[inventory_hostname]['domain_uuid'] }}</uuid>
   <memory unit='GiB'>{{ hostvars[inventory_hostname]['memory'] }}</memory>
   <currentMemory unit='GiB'>{{ hostvars[inventory_hostname]['memory'] }}</currentMemory>
+{% if vm_hugepages %}
+  <memoryBacking>
+    <hugepages>
+      <page size='{{ vm_hugepage_size[:-1] }}' unit='{{ vm_hugepage_size[-1] }}'/>
+    </hugepages>
+  </memoryBacking>
+{% endif %}
   <vcpu placement='static'>{{ hv_vm_cpu_count | int }}</vcpu>
   <os>
     <type arch='x86_64' machine='pc-q35-rhel7.6.0'>hvm</type>
@@ -11,6 +18,9 @@
   <features>
     <acpi/>
     <apic/>
+{% if vm_igb_nics | default(false) %}
+    <ioapic driver='qemu'/>
+{% endif %}
   </features>
 {% if vnuma_enabled %}
   <cpu mode='host-model' check='partial'>
@@ -125,6 +135,20 @@
 {% endif %}
       <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
     </interface>
+{% if vm_igb_nics | default(false) %}
+{% for i in range(1, 6) %}
+    <interface type='bridge'>
+{% set mac_prefix = "%s:%02x" | format('52:54:00',i) %}
+      <mac address='{{ mac_prefix | community.general.random_mac(seed=inventory_hostname) }}'/>
+      <source bridge='br0'/>
+      <model type='igb'/>
+      <address type='pci' domain='0x0000' bus='{{ "0x%02x" | format(i + 4) }}' slot='0x00' function='0x0'/>
+    </interface>
+{% endfor %}
+    <iommu model='intel'>
+      <driver intremap='on'/>
+    </iommu>
+{% endif %}
     <serial type='pty'>
       <target type='isa-serial' port='0'>
         <model name='isa-serial'/>