K8SPXC-1647: Use extended exit codes for garbd

build/backup/backup.sh

@@ -62,57 +62,38 @@ function request_streaming() {
 	set +o errexit
 	log 'INFO' 'Garbd was started'
 	garbd \
-		--address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
-		--donor "$NODE_NAME" \
-		--group "$PXC_SERVICE" \
-		--options "$GARBD_OPTS" \
-		--sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
-		--recv-script="/opt/percona/backup/run_backup.sh" 2>&1 | tee /tmp/garbd.log
-
-	local sst_info_path
-	if [[ -n $S3_BUCKET || -n $AZURE_CONTAINER_NAME ]]; then
-		sst_info_path="/tmp/${SST_INFO_NAME}"
-	else
-		sst_info_path="${BACKUP_DIR}/${SST_INFO_NAME}"
-	fi
-
-	MYSQL_VERSION=$(parse_ini 'mysql-version' "$sst_info_path")
-	if ! check_for_version "$MYSQL_VERSION" '8.0.0'; then
-		if grep 'State transfer request failed' /tmp/garbd.log; then
-			exit 1
-		fi
-		if grep 'WARN: Protocol violation. JOIN message sender ... (garb) is not in state transfer' /tmp/garbd.log; then
-			exit 1
-		fi
-		if grep 'WARN: Rejecting JOIN message from ... (garb): new State Transfer required.' /tmp/garbd.log; then
-			exit 1
-		fi
-		if grep 'INFO: Shifting CLOSED -> DESTROYED (TO: -1)' /tmp/garbd.log; then
-			exit 1
-		fi
-		if ! grep 'INFO: Sending state transfer request' /tmp/garbd.log; then
-			exit 1
-		fi
-	else
-		if grep 'Will never receive state. Need to abort' /tmp/garbd.log; then
-			exit 1
-		fi
-
-		if grep 'Donor is no longer in the cluster, interrupting script' /tmp/garbd.log; then
-			exit 1
-		elif grep 'failed: Invalid argument' /tmp/garbd.log; then
-			exit 1
-		fi
-	fi
-
-	if [ -f '/tmp/backup-is-completed' ]; then
-		log 'INFO' 'Backup was finished successfully'
-		exit 0
-	fi
-
-	log 'ERROR' 'Backup was finished unsuccessful'
-
-	exit 1
+	--address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
+	--donor "$NODE_NAME" \
+	--group "$PXC_SERVICE" \
+	--options "$GARBD_OPTS" \
+	--sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
+	--extended-exit-codes \
+	--wait-for-recv-script-exit \
+	--recv-script="/opt/percona/backup/run_backup.sh"
+	GARBD_EXIT_CODE=$?
+
+	case ${GARBD_EXIT_CODE} in
+	0)
+	    log 'INFO' 'Backup was finished successfully'
+	    exit 0
+	    ;;
+	100)
+	    log 'ERROR' 'Backup was unsuccessful: Generic failure'
+	    exit 1
+	    ;;
+	101)
+	    log 'ERROR' 'Backup was unsuccessful: Donor disappeared'
+	    exit 1
+	    ;;
+	102)
+	    log 'ERROR' 'Backup was unsuccessful: SST request failure'
+	    exit 1
+	    ;;
+	*)
+	    log 'ERROR' "Backup was unsuccessful: garbd exited with ${GARBD_EXIT_CODE}"
+	    exit 1
+	    ;;
+	esac
 }
 
 # TODO: should i remove it?

build/backup/run_backup.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-set -o errexit
 set -o xtrace
+set -o errexit
+set -o pipefail
 set -m
 
 LIB_PATH='/opt/percona/backup/lib/pxc'
@@ -44,7 +45,7 @@
 }
 
 # shellcheck disable=SC2317
 handle_sigterm() {
 	if ((FIRST_RECEIVED == 0)); then
 		pid_s=$(ps -C socat -o pid= || true)
 		if [ -n "${pid_s}" ]; then
@@ -96,7 +97,6 @@
 		log 'INFO' "Socat(2) returned $?"
 	fi
 
-	trap '' 15
 	stat xtrabackup.stream
 	if (($(stat -c%s xtrabackup.stream) < 5000000)); then
 		log 'ERROR' 'Backup is empty'
@@ -126,20 +126,33 @@
 	fi
 	vault_store /tmp/${SST_INFO_NAME}
 
+	# this xbcloud command will fail with backup is incomplete
+	# it's expected since we only upload sst_info
+	set +o pipefail
 	# shellcheck disable=SC2086
 	xbstream -C /tmp -c ${SST_INFO_NAME} $XBSTREAM_EXTRA_ARGS \
-		| xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" 2>&1 \
+		| xbcloud put --storage=s3 \
+			--md5 \
+			--parallel="$(grep -c processor /proc/cpuinfo)" \
+			$XBCLOUD_ARGS \
+			--s3-bucket="$S3_BUCKET" \
+			"$S3_BUCKET_PATH.$SST_INFO_NAME" 2>&1 \
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
+	set -o pipefail
 
 	if ((SST_FAILED == 0)); then
 		# shellcheck disable=SC2086
 		socat -u "$SOCAT_OPTS" stdio \
-			| xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \
-			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
+			| xbcloud put --storage=s3 \
+				--md5 \
+				--parallel="$(grep -c processor /proc/cpuinfo)" \
+				$XBCLOUD_ARGS \
+				--s3-bucket="$S3_BUCKET" \
+				"$S3_BUCKET_PATH" 2>&1 \
+			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) &
+		wait $!
 	fi
 
-	trap '' 15
-
 	# shellcheck disable=SC2086
 	aws $AWS_S3_NO_VERIFY_SSL s3 ls "s3://$S3_BUCKET/$S3_BUCKET_PATH.md5"
 	# shellcheck disable=SC2086
@@ -172,16 +185,27 @@
 	fi
 	vault_store /tmp/${SST_INFO_NAME}
 
+	# this xbcloud command will fail with backup is incomplete
+	# it's expected since we only upload sst_info
+	set +o pipefail
 	# shellcheck disable=SC2086
 	xbstream -C /tmp -c ${SST_INFO_NAME} $XBSTREAM_EXTRA_ARGS \
-		| xbcloud put --storage=azure --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS "$BACKUP_PATH.$SST_INFO_NAME" 2>&1 \
+		| xbcloud put --storage=azure \
+			--parallel="$(grep -c processor /proc/cpuinfo)" \
+			$XBCLOUD_ARGS \
+			"$BACKUP_PATH.$SST_INFO_NAME" 2>&1 \
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
+	set -o pipefail
 
 	if ((SST_FAILED == 0)); then
 		# shellcheck disable=SC2086
 		socat -u "$SOCAT_OPTS" stdio \
-			| xbcloud put --storage=azure --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS "$BACKUP_PATH" 2>&1 \
-			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
+			| xbcloud put --storage=azure \
+				--parallel="$(grep -c processor /proc/cpuinfo)" \
+				$XBCLOUD_ARGS \
+				"$BACKUP_PATH" 2>&1 \
+		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) &
+		wait $!
 	fi
 }