|
| 1 | +#! /usr/bin/bash |
| 2 | + |
| 3 | +set -ex |
| 4 | + |
| 5 | +RED='\033[0;31m' |
| 6 | +GREEN='\033[0;32m' |
| 7 | + |
| 8 | +RABBITMQ_SERVICES_TO_RESTART=barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia |
| 9 | +RABBITMQ_CONTAINER_NAME=rabbitmq |
| 10 | + |
| 11 | +if [[ ! $KAYOBE_CONFIG_PATH ]]; then |
| 12 | + echo "${RED}Environment variable \$KAYOBE_CONFIG_PATH is not defined" |
| 13 | + echo "${RED}Ensure your environment is set up to run kayobe commands" |
| 14 | + exit 2 |
| 15 | +fi |
| 16 | + |
| 17 | +if [[ ! "$1" = "--skip-checks" ]]; then |
| 18 | + # Fail if clocks are not synced |
| 19 | + if ! ( kayobe overcloud host command run -l controllers -b --command "timedatectl status | grep 'synchronized: yes'" ); then |
| 20 | + echo "${RED}Failed precheck: Time not synced on controllers" |
| 21 | + echo "${RED}Use 'timedatectl status' to check sync state" |
| 22 | + echo "${RED}Either wait for sync or use 'chronyc makestep'" |
| 23 | + exit 1 |
| 24 | + fi |
| 25 | + kayobe overcloud service configuration generate --node-config-dir /tmp/rabbit-migration --kolla-tags none |
| 26 | + # Fail if any new feature flags are not set |
| 27 | + if ! ( grep 'om_enable_queue_manager: true' $KOLLA_CONFIG_PATH/globals.yml && \ |
| 28 | + grep 'om_enable_rabbitmq_quorum_queues: true' $KOLLA_CONFIG_PATH/globals.yml && \ |
| 29 | + grep 'om_enable_rabbitmq_transient_quorum_queue: true' $KOLLA_CONFIG_PATH/globals.yml && \ |
| 30 | + grep 'om_enable_rabbitmq_stream_fanout: true' $KOLLA_CONFIG_PATH/globals.yml ); then |
| 31 | + echo "${RED}Failed precheck: The following must be enabled: om_enable_queue_manager, om_enable_rabbitmq_quorum_queues, om_enable_rabbitmq_transient_quorum_queue, om_enable_rabbitmq_stream_fanout" |
| 32 | + exit 1 |
| 33 | + fi |
| 34 | +fi |
| 35 | + |
| 36 | +# Generate new config, stop services using rabbit, and reset rabbit state |
| 37 | +kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck |
| 38 | +kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt $RABBITMQ_SERVICES_TO_RESTART |
| 39 | +kayobe kolla ansible run rabbitmq-reset-state |
| 40 | + |
| 41 | +if [[ ! "$1" = "--skip-checks" ]]; then |
| 42 | + # Fail if any queues still exist |
| 43 | + sleep 20 |
| 44 | + # Note(mattcrees): We turn the text grey here so the failed Ansible calls don't freak anyone out |
| 45 | + CURRENTTERM=${TERM} |
| 46 | + export TERM=xterm-mono |
| 47 | + if ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues name --silent | grep -v '^$'" ); then |
| 48 | + export TERM=${CURRENTTERM} |
| 49 | + echo -e "${RED}Failed check: RabbitMQ has not stopped properly, queues still exist" |
| 50 | + exit 1 |
| 51 | + fi |
| 52 | + # Fail if any exchanges still exist (excluding those starting with 'amq.') |
| 53 | + if ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_exchanges name --silent | grep -v '^$' | grep -v '^amq.'" ); then |
| 54 | + export TERM=${CURRENTTERM} |
| 55 | + echo -e "${RED}Failed check: RabbitMQ has not stopped properly, exchanges still exist" |
| 56 | + exit 1 |
| 57 | + fi |
| 58 | + export TERM=${CURRENTTERM} |
| 59 | +fi |
| 60 | + |
| 61 | +# Redeploy with all durable-type queues enabled |
| 62 | +kayobe kolla ansible run deploy-containers -kt $RABBITMQ_SERVICES_TO_RESTART |
| 63 | + |
| 64 | +if [[ ! "$1" = "--skip-checks" ]]; then |
| 65 | + sleep 60 |
| 66 | + # Assert that all queues are durable |
| 67 | + if ! ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues durable --silent | grep false" > /dev/null 2>&1 ); then |
| 68 | + echo -e "${GREEN}Queues migrated successfully" |
| 69 | + else |
| 70 | + echo -e "${RED}Failed post-check: A controller has non-durable queues" |
| 71 | + fi |
| 72 | +fi |
0 commit comments