Merge pull request #1752 from satya-bodapati/8.4

PXB-3768 [8.4]: Fix test failures on Jenkins

Author: satya-bodapati

storage/innobase/xtrabackup/test/suites/keyring/undo_transition_key.sh

@@ -0,0 +1,72 @@
+#
+# PXB-3762: xtrabackup --backup --transition-key crashes when an undo
+# tablespace encryption key exists in redo only (newer than the on-disk
+# undo page).
+#
+# JIRA URL: https://perconadev.atlassian.net/browse/PXB-3762
+#
+# Strategy:
+#   1. Use release-safe server knobs that reduce page flushing pressure.
+#   2. Build enough undo/redo activity on an encrypted table.
+#   3. Rotate master key immediately before backup.
+#   4. Start backup with --transition-key right away.
+#
+# This widens the race window for "redo key exists and is newer than
+# undo-page LSN", which is required to enter Condition 1 in
+# srv_undo_tablespace_read_encryption().
+#
+require_server_version_higher_than 8.4.0
+
+KEYRING_TYPE="component"
+. inc/keyring_common.sh
+. inc/keyring_file.sh
+
+# Keep dirty pages in memory longer so on-disk undo pages are more likely
+# to lag behind redo-encryption records right after key rotation.
+MYSQLD_EXTRA_MY_CNF_OPTS="${MYSQLD_EXTRA_MY_CNF_OPTS}
+innodb_buffer_pool_size=512M
+innodb_log_file_size=256M
+innodb_io_capacity=100
+innodb_io_capacity_max=200
+innodb_max_dirty_pages_pct=95
+innodb_max_dirty_pages_pct_lwm=0
+"
+
+configure_server_with_component
+
+mysql -e "CREATE TABLE t (
+  a BIGINT PRIMARY KEY AUTO_INCREMENT,
+  b LONGBLOB
+) ENGINE=InnoDB ENCRYPTION='y'" test
+mysql -e "INSERT INTO t(b) VALUES (REPEAT(UUID(), 256))" test
+
+for attempt in $(seq 1 5); do
+    vlog "=== attempt ${attempt}/5: rotate + backup + prepare + restore ==="
+
+    backup_dir=$topdir/backup.$attempt
+    mkdir -p "$backup_dir"
+
+    # Generate undo/redo churn right before rotation.
+    for i in $(seq 1 200); do
+        mysql -e "INSERT INTO t(b) VALUES (REPEAT(UUID(), 256))" test >/dev/null 2>&1
+        mysql -e "DELETE FROM t WHERE a % 13 = 0 ORDER BY a LIMIT 2" test >/dev/null 2>&1
+    done
+
+    # Rotate right before backup so redo key LSN can be newer than on-disk undo.
+    mysql -e "ALTER INSTANCE ROTATE INNODB MASTER KEY" test
+
+    record_db_state test
+
+    xtrabackup --backup --transition-key=123 --target-dir="$backup_dir"
+    xtrabackup --prepare --transition-key=123 --target-dir="$backup_dir"
+
+    stop_server
+    rm -rf "$mysql_datadir"
+    xtrabackup --copy-back --target-dir="$backup_dir"
+    cp "${instance_local_manifest}" "$mysql_datadir"
+    cp "${keyring_component_cnf}" "$mysql_datadir"
+    start_server
+
+    verify_db_state test
+    rm -rf "$backup_dir"
+done

storage/innobase/xtrabackup/test/t/PXB-2854_xbstream_memory_corruption.sh

@@ -1,4 +1,8 @@
 # PXB-2854 - Quicklz decompression memory corruption issue
+# xbstream still decompresses legacy .qp streams; skip cleanly when
+# the standalone qpress binary used to build the fixture is missing.
+
+require_qpress
 
 head -c 100000 </dev/urandom > $topdir/payload.bin
 qpress $topdir/payload.bin $topdir/payload.qp

storage/innobase/xtrabackup/test/t/bug1600656.sh

@@ -24,4 +24,10 @@ xbstream -xv -C $topdir/backup < $topdir/backup/stream.xbs
 vlog "#########################################################################"
 vlog "Verifying that streamed and 'extra copy' of xtrabackup_info do not differ"
 
-diff -q $topdir/backup/xtrabackup_info $topdir/lsndir/xtrabackup_info
+# The (uncompressed_)backup_size lines are sampled at different times
+# in the two copies, so they intentionally differ. Strip them before
+# diffing and verify everything else matches.
+diff <(sed -E '/^(uncompressed_)?backup_size = /d' \
+           $topdir/backup/xtrabackup_info) \
+     <(sed -E '/^(uncompressed_)?backup_size = /d' \
+           $topdir/lsndir/xtrabackup_info)

storage/innobase/xtrabackup/test/t/estimate_memory.sh

@@ -92,17 +92,26 @@ redo_memory=$(grep 'redo_memory' ${backupdir}/xtrabackup_checkpoints | awk '{pri
 redo_frames=$(grep 'redo_frames' ${backupdir}/xtrabackup_checkpoints | awk '{print $3}') #pages
 total_memory=$((redo_memory + (redo_frames * 16384))) #bytes
 
-free_memory=$(cat /proc/meminfo | grep 'MemAvailable' | awk '{print $2}') #kb
-free_memory_bytes=$((free_memory * 1024)) #bytes
+# Use MemAvailable to mirror what xtrabackup's host_free_memory() returns on
+# Linux (libprocps kb_main_available). PXB-3770 tracks the libproc2 build
+# returning 0 here; on those boxes the test will still fail until the binary
+# fix lands. Log MemFree too so any failure shows the full kernel view.
+mem_avail_kb=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
+mem_free_kb=$(awk  '/^MemFree:/      {print $2}' /proc/meminfo)
+free_memory_bytes=$((mem_avail_kb * 1024)) #bytes
 free_memory_1_pct=$((free_memory_bytes  * 1 / 100)) #bytes
 free_memory_99_pct=$((free_memory_bytes  * 99 / 100)) #bytes
+vlog "gate inputs: mem_avail_kb=${mem_avail_kb}, mem_free_kb=${mem_free_kb},"
+vlog "             free_memory_99_pct=${free_memory_99_pct}, total_memory=${total_memory}"
 
 if [[ "${free_memory_99_pct}" -lt "${total_memory}" ]];
 then
   vlog "Skipping full memory workload test as 99% of free memory is lower than required by pxb"
 else
+  vlog "OS view just before --prepare:" \
+       "$(awk '/^MemFree:|^MemAvailable:/ {print $1, $2, $3}' /proc/meminfo | tr '\n' ' ')"
   vlog "Preparing backup with --use-free-memory-pct=99"
-  xtrabackup --prepare --use-free-memory-pct=99 --target-dir=${backupdir} 2> ${logfile}
+  xtrabackup --prepare --use-free-memory-pct=99 --target-dir=${backupdir} 2> >(tee "${logfile}" >&2)
 
   if grep -q "Required memory will exceed free memory configuration" ${logfile}
   then
@@ -128,7 +137,7 @@ then
 fi
 
 vlog "Preparing backup with --use-free-memory-pct=1"
-xtrabackup --prepare --use-free-memory-pct=1 --target-dir=${backupdir2} 2> ${logfile2}
+xtrabackup --prepare --use-free-memory-pct=1 --target-dir=${backupdir2} 2> >(tee "${logfile2}" >&2)
 
 if ! grep -q "Required memory will exceed Free memory configuration" ${logfile2}
 then

storage/innobase/xtrabackup/test/t/pxb-3003-redo-log-purge-race.sh

@@ -11,8 +11,14 @@
 require_server_version_higher_than 8.0.29
 require_debug_pxb_version
 
+# Start at 16M capacity so we can later shrink to 8M. The shrink sets
+# target_capacity < current_capacity, the one log_files governor case
+# that reliably fires here and forces reclamation behind the registered
+# PXB consumer. Without it, none of the other governor cases trigger on
+# a release build merely because PXB has crossed the oldest file's
+# end_lsn, and MIN_FILEID would never advance.
 MYSQLD_EXTRA_MY_CNF_OPTS="
-innodb_redo_log_capacity=8M
+innodb_redo_log_capacity=16M
 "
 start_server
 
@@ -28,8 +34,15 @@ $MYSQL $MYSQL_ARGS -Ns -e "CREATE TABLE redo_log_consumer (
   str BLOB
 );" test
 
+# Ensure at least 2 redo files exist before xtrabackup starts. The
+# PFS loop makes this deterministic regardless of bootstrap redo.
 $MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO test.redo_log_consumer VALUES (NULL, REPEAT('a', 63 * 1024))"
 $MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO test.redo_log_consumer VALUES (NULL, REPEAT('a', 63 * 1024))"
+NUM_REDO_FILES=`$MYSQL $MYSQL_ARGS -Ns -e "SELECT COUNT(*) FROM performance_schema.innodb_redo_log_files"`
+while [ "${NUM_REDO_FILES}" -lt 2 ]; do
+  $MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO test.redo_log_consumer VALUES (NULL, REPEAT('a', 63 * 1024))"
+  NUM_REDO_FILES=`$MYSQL $MYSQL_ARGS -Ns -e "SELECT COUNT(*) FROM performance_schema.innodb_redo_log_files"`
+done
 
 mkdir -p $topdir/backup/
 
@@ -43,8 +56,17 @@ xb_pid=`cat $pid_file`
 
 vlog "backup pid is $job_pid"
 
+# Snapshot redo log file layout at suspend time for diagnosability.
+vlog "performance_schema.innodb_redo_log_files at suspend:"
+$MYSQL $MYSQL_ARGS -t -e \
+  "SELECT file_id, start_lsn, end_lsn, is_full FROM performance_schema.innodb_redo_log_files ORDER BY file_id" >&2
+
 INITIAL_MIN_FILEID=`$MYSQL $MYSQL_ARGS -Ns -e "SELECT MIN(file_id) FROM performance_schema.innodb_redo_log_files"`
 
+# Force file reclamation by shrinking redo capacity (see top of file).
+vlog "Shrinking innodb_redo_log_capacity to 8M to force file reclamation"
+$MYSQL $MYSQL_ARGS -Ns -e "SET GLOBAL innodb_redo_log_capacity = 8 * 1024 * 1024"
+
 run_inserts &
 insert_pid=$!
 CURRENT_MIN_FILEID=`$MYSQL $MYSQL_ARGS -Ns -e "SELECT MIN(file_id) FROM performance_schema.innodb_redo_log_files"`

storage/innobase/xtrabackup/test/t/redo_log_consumer.sh

@@ -59,18 +59,59 @@ xb_pid=`cat $pid_file`
 echo "backup pid is $job_pid"
 
 run_inserts &
-
-# ER_IB_MSG_LOG_WRITER_WAIT_ON_CONSUMER
-while ! grep -q "Redo log writer is waiting for" ${MYSQLD_ERRFILE} ; do
+inserts_pid=$!
+
+# Wait for backpressure: with 8M capacity and the registered PXB/MEB
+# consumer pinned at the catch-up LSN, the server cannot reclaim redo
+# files. Exit on the first of: a consumer-lagging warning anchored on
+# the consumer name (avoids false matches from the prior backup's
+# log_checkpointer warnings), or PFS reporting >= 3 redo files (proof
+# we're past 8M with the consumer holding it back). Bound the wait so
+# MTR fails with diagnostics instead of Jenkins killing the job.
+max_wait_s=60
+for ((i=1; i<=max_wait_s; i++)) ; do
+	if grep -qE "(Redo log writer is waiting for (PXB|MEB) redo log consumer|'(PXB|MEB)' consumer still lagging behind)" \
+			${MYSQLD_ERRFILE} ; then
+		vlog "consumer-lagging warning observed in mysql error file"
+		break
+	fi
+
+	n_files=$($MYSQL $MYSQL_ARGS -Ns -e \
+		"SELECT COUNT(*) FROM performance_schema.innodb_redo_log_files" \
+		2>/dev/null || echo 0)
+	if [ "${n_files:-0}" -ge 3 ] ; then
+		vlog "redo log accumulated ${n_files} files;" \
+			"consumer is holding the server back as expected"
+		break
+	fi
+
+	vlog "waiting for redo log backpressure" \
+		"(#files=${n_files:-?}, ${i}/${max_wait_s}s)"
 	sleep 1
-	vlog "waiting for redo log writer message in mysql error file"
 done
 
+if [ ${i} -gt ${max_wait_s} ] ; then
+	vlog "ERROR: timed out waiting for redo log backpressure"
+	vlog "performance_schema.innodb_redo_log_files at timeout:"
+	$MYSQL $MYSQL_ARGS -t -e \
+		"SELECT file_id, start_lsn, end_lsn, is_full FROM performance_schema.innodb_redo_log_files ORDER BY file_id" \
+		>&2 || true
+	vlog "tail of mysqld error file ${MYSQLD_ERRFILE}:"
+	tail -100 ${MYSQLD_ERRFILE} >&2 || true
+	kill ${inserts_pid} 2>/dev/null || true
+	kill -SIGCONT ${xb_pid} 2>/dev/null || true
+	die "redo_log_consumer: timeout waiting for backpressure"
+fi
+
 # Resume the xtrabackup process
 vlog "Resuming xtrabackup"
 kill -SIGCONT $xb_pid
 run_cmd wait $job_pid
 
+# Stop background inserts before stop_server to avoid ERROR 2002 spam.
+kill ${inserts_pid} 2>/dev/null || true
+wait ${inserts_pid} 2>/dev/null || true
+
 xtrabackup --prepare --target-dir=$topdir/backup
 
 # PXB-3147 - register redo log consumer fails with ANSI_QUOTES