Merge pull request #693 from stackhpc/2023.1-zed-merge

2023.1: zed merge

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient
+          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
 
       - name: Build a Rocky Linux 9 overcloud host image
         id: build_rocky_9

.github/workflows/stackhpc-container-image-build.yml

@@ -145,6 +145,10 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
+      - name: Prune local Kolla container images over 1 week old
+        run: |
+          sudo docker image prune --all --force --filter until=168h --filter="label=kolla_version"
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
@@ -180,17 +184,16 @@ jobs:
         run: |
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images
 
+      - name: Fail if no images have been built
+        run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
+
       - name: Upload container images artifact
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.distro }} container images
           path: ${{ matrix.distro }}-container-images
           retention-days: 7
 
-      - name: Prune local Kolla container images over 1 week old
-        run: |
-          sudo docker image prune --all --force --filter until=168h --filter="label=kolla_version"
-
   sync-container-repositories:
     name: Trigger container image repository sync
     needs:

doc/source/configuration/release-train.rst

@@ -186,6 +186,16 @@ promoted to production:
 
    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml
 
+Synchronising all Kolla container images can take a long time. A limited list
+of images can be synchronised using the ``stackhpc_pulp_images_kolla_filter``
+variable, which accepts a whitespace-separated list of regular expressions
+matching Kolla image names. Usage is similar to ``kolla-build`` CLI arguments.
+For example:
+
+.. code-block:: console
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"'
+
 Initial seed deployment
 -----------------------
 

doc/source/configuration/vault.rst

@@ -229,6 +229,18 @@ Enable the required TLS variables in kayobe and kolla
 
       kayobe overcloud service deploy
 
+   If VM provisioning fails with an error with this format:
+
+   .. code-block::
+
+      Unable to establish connection to http://<kolla internal vip/fqdn>:9696/v2.0/ports/some-sort-of-uuid: Connection aborted
+
+   Restart the nova-compute container on all hypervisors:
+
+   .. code-block::
+
+      kayobe overcloud host command run --command "docker restart nova_compute" --become --show-output -l compute
+
 Barbican integration
 ====================
 

doc/source/configuration/walled-garden.rst

@@ -77,7 +77,8 @@ proxy:
      - "127.0.0.1"
      - "localhost"
      - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}"
-     - "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}"
+     - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}"
+     - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[inventory_hostname] }}"
      - "{{ kolla_external_fqdn }}"
      - "{{ kolla_internal_fqdn }}"
 

doc/source/configuration/wazuh.rst

@@ -17,8 +17,8 @@ The short version
 #. Deploy the Wazuh agents: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml``
 
 
-Wazuh Manager
-=============
+Wazuh Manager Host
+==================
 
 Provision using infra-vms
 -------------------------
@@ -315,7 +315,7 @@ Encrypt the keys (and remember to commit to git):
 ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key``
 
 Verification
-==============
+------------
 
 The Wazuh portal should be accessible on port 443 of the Wazuh
 manager’s IPs (using HTTPS, with the root CA cert in ``etc/kayobe/ansible/wazuh/certificates/wazuh-certificates/root-ca.pem``).
@@ -327,11 +327,9 @@ Troubleshooting
 
 Logs are in ``/var/log/wazuh-indexer/wazuh.log``. There are also logs in the journal.
 
-============
 Wazuh agents
 ============
 
-
 Wazuh agent playbook is located in ``etc/kayobe/ansible/wazuh-agent.yml``.
 
 Wazuh agent variables file is located in ``etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent``.
@@ -345,13 +343,13 @@ Deploy the Wazuh agents:
 ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml``
 
 Verification
-=============
+------------
 
 The Wazuh agents should register with the Wazuh manager. This can be verified via the agents page in Wazuh Portal.
 Check CIS benchmark output in agent section.
 
-Additional resources:
-=====================
+Additional resources
+--------------------
 
 For times when you need to upgrade wazuh with elasticsearch to version with opensearch or you just need to deinstall all wazuh components:
 Wazuh purge script: https://github.com/stackhpc/wazuh-server-purge

etc/kayobe/ansible/ovn-fix-chassis-priorities.yml

@@ -0,0 +1,69 @@
+---
+# Sometimes, typically after restarting OVN services, the priorities of entries
+# in the ha_chassis and gateway_chassis tables in the OVN northbound database
+# can become misaligned. This results in broken routing for external (bare
+# metal/SR-IOV) ports.
+
+# This playbook can be used to fix the issue by realigning the priorities of
+# the table entries. It does so by assigning the highest priority to the
+# "first" (sorted alphabetically) OVN NB DB host. This results in all gateways
+# being scheduled to a single host, but is less complicated than trying to
+# balance them (and it's also not clear to me how to map between individual
+# ha_chassis and gateway_chassis entries).
+
+# The playbook can be run as follows:
+# kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ovn-fix-chassis-priorities.yml
+
+# If the 'controllers' group does not align with the group used to deploy the
+# OVN NB DB, this can be overridden by passing the following:
+# '-e ovn_nb_db_group=some_other_group'
+
+- name: Find OVN DB DB Leader
+  hosts: "{{ ovn_nb_db_group | default('controllers') }}"
+  tasks:
+    - name: Find the OVN NB DB leader
+      command: docker exec -it ovn_nb_db ovn-nbctl get-connection
+      changed_when: false
+      failed_when: false
+      register: ovn_check_result
+      check_mode: no
+
+    - name: Group hosts by leader/follower role
+      group_by:
+        key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}"
+      changed_when: false
+
+    - name: Assert one leader exists
+      assert:
+        that:
+          - groups['ovn_nb_leader'] | default([]) | length == 1
+
+- name: Fix OVN chassis priorities
+  hosts: ovn_nb_leader
+  vars:
+    ovn_nb_db_group: controllers
+    ovn_nb_db_hosts_sorted: "{{ query('inventory_hostnames', ovn_nb_db_group) | sort | list }}"
+    ha_chassis_max_priority: 32767
+    gateway_chassis_max_priority: "{{ ovn_nb_db_hosts_sorted | length }}"
+  tasks:
+    - name: Fix ha_chassis priorities
+      command: >-
+        docker exec -it ovn_nb_db
+        bash -c '
+        ovn-nbctl find ha_chassis chassis_name={{ item }} |
+        awk '\''$1 == "_uuid" { print $3 }'\'' |
+        while read uuid; do ovn-nbctl set ha_chassis $uuid priority={{ priority }}; done'
+      loop: "{{ ovn_nb_db_hosts_sorted }}"
+      vars:
+        priority: "{{ ha_chassis_max_priority | int - ovn_nb_db_hosts_sorted.index(item) }}"
+
+    - name: Fix gateway_chassis priorities
+      command: >-
+        docker exec -it ovn_nb_db
+        bash -c '
+        ovn-nbctl find gateway_chassis chassis_name={{ item }} |
+        awk '\''$1 == "_uuid" { print $3 }'\'' |
+        while read uuid; do ovn-nbctl set gateway_chassis $uuid priority={{ priority }}; done'
+      loop: "{{ ovn_nb_db_hosts_sorted }}"
+      vars:
+        priority: "{{ gateway_chassis_max_priority | int - ovn_nb_db_hosts_sorted.index(item) }}"

etc/kayobe/ansible/reboot.yml

@@ -1,7 +1,7 @@
 ---
 - name: Reboot the host
   hosts: seed-hypervisor:seed:overcloud:infra-vms
-  serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(0, true) }}"
+  serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}"
   tags:
     - reboot
   tasks:

etc/kayobe/environments/aufn-ceph/kolla.yml

@@ -13,5 +13,4 @@ kolla_overcloud_inventory_custom_services: "{{ lookup('template', kayobe_env_con
 ###############################################################################
 # Kolla feature flag configuration.
 kolla_enable_cinder: true
-kolla_enable_ovn: true
 kolla_enable_neutron_provider_networks: true

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -23,7 +23,6 @@ kolla_enable_mariabackup: true
 kolla_enable_neutron_sriov: true
 kolla_enable_octavia: true
 kolla_enable_opensearch: true
-kolla_enable_ovn: true
 kolla_enable_prometheus: true
 kolla_enable_redis: true