From 4fde9c4854205b0d0220f0236855f192d2a2a4c5 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 8 Feb 2024 11:51:24 +0100 Subject: [PATCH 1/6] Structure examples for async and semisync replication --- examples/{ => replication-async}/.env | 0 examples/{ => replication-async}/compose-replication.yml | 0 examples/replication-semisync/.env | 6 ++++++ .../compose-replication-semisync.yml | 0 4 files changed, 6 insertions(+) rename examples/{ => replication-async}/.env (100%) rename examples/{ => replication-async}/compose-replication.yml (100%) create mode 100644 examples/replication-semisync/.env rename examples/{ => replication-semisync}/compose-replication-semisync.yml (100%) diff --git a/examples/.env b/examples/replication-async/.env similarity index 100% rename from examples/.env rename to examples/replication-async/.env diff --git a/examples/compose-replication.yml b/examples/replication-async/compose-replication.yml similarity index 100% rename from examples/compose-replication.yml rename to examples/replication-async/compose-replication.yml diff --git a/examples/replication-semisync/.env b/examples/replication-semisync/.env new file mode 100644 index 00000000..9803af2b --- /dev/null +++ b/examples/replication-semisync/.env @@ -0,0 +1,6 @@ +MARIADB_ROOT_PASSWORD=secret +MARIADB_REPLICATION_USER=repl +MARIADB_REPLICATION_PASSWORD=replicationpass +PRIMARY_name='mariadb-primary' +REPLICATION_name_1='mariadb-replica-1' +REPLICATION_name_2='mariadb-replica-2' \ No newline at end of file diff --git a/examples/compose-replication-semisync.yml b/examples/replication-semisync/compose-replication-semisync.yml similarity index 100% rename from examples/compose-replication-semisync.yml rename to examples/replication-semisync/compose-replication-semisync.yml From 7b9d423b12775a789faff22bd89056272b2c4399 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Mon, 12 Feb 2024 15:03:56 +0100 Subject: [PATCH 2/6] Automation script for MySQL 8.0 data migration --- examples/migration-8.0/.env | 7 ++ examples/migration-8.0/.gitignore | 1 + examples/migration-8.0/compose-mysql8.0.yml | 80 +++++++++++++++++++++ examples/migration-8.0/dump-mysql.sh | 28 ++++++++ examples/migration-8.0/migrate-mariadb.sh | 15 ++++ examples/migration-8.0/mysql/mysql-data.sql | 3 + 6 files changed, 134 insertions(+) create mode 100644 examples/migration-8.0/.env create mode 100644 examples/migration-8.0/.gitignore create mode 100644 examples/migration-8.0/compose-mysql8.0.yml create mode 100755 examples/migration-8.0/dump-mysql.sh create mode 100644 examples/migration-8.0/migrate-mariadb.sh create mode 100644 examples/migration-8.0/mysql/mysql-data.sql diff --git a/examples/migration-8.0/.env b/examples/migration-8.0/.env new file mode 100644 index 00000000..67f58abb --- /dev/null +++ b/examples/migration-8.0/.env @@ -0,0 +1,7 @@ +DB_ROOT_PASSWORD='secret' +DB_USER='testuser' +DB_PASSWORD='password' +DB_DATABASE='testdb' +MYSQL_name='mysql-container' +MARIADB_DUMP_name='mariadb-container-dump' +MARIADB_MIGRATED_name='mariadb-migrated-mysql8.0' \ No newline at end of file diff --git a/examples/migration-8.0/.gitignore b/examples/migration-8.0/.gitignore new file mode 100644 index 00000000..7aea0eaa --- /dev/null +++ b/examples/migration-8.0/.gitignore @@ -0,0 +1 @@ +dump-data/* \ No newline at end of file diff --git a/examples/migration-8.0/compose-mysql8.0.yml b/examples/migration-8.0/compose-mysql8.0.yml new file mode 100644 index 00000000..74563263 --- /dev/null +++ b/examples/migration-8.0/compose-mysql8.0.yml @@ -0,0 +1,80 @@ +version: "3.2" +# Anchors +x-common-variables: &common-variables + MARIADB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} + MARIADB_USER: ${DB_USER} + MARIADB_PASSWORD: ${DB_PASSWORD} + MARIADB_DB: ${DB_DATABASE} + MYSQL_CONT_NAME: ${MYSQL_name} + +x-common-attributes: &common-attributes + env_file: + - .env + image: mariadb:lts + environment: *common-variables # alias + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + start_period: 10s + interval: 20s + timeout: 20s + retries: 3 + networks: + - backend + +services: + mysql: + environment: + MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD: ${DB_PASSWORD} + MYSQL_DATABASE: ${DB_DATABASE} + container_name: ${MYSQL_name} + image: mysql:8.3.0 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent"] + interval: 20s + timeout: 20s + retries: 2 + start_period: 0s + command: --print-identified-with-as-hex=ON + volumes: + # Preload files for MySQL data + - ./mysql:/docker-entrypoint-initdb.d:z + # We have to save MySQL volume that will be used in upgrade + - dbdata:/var/lib/mysql + networks: + - backend + + mariadb-dump: + <<: *common-attributes # aliases + container_name: ${MARIADB_DUMP_name} + depends_on: + mysql: + condition: service_healthy + volumes: + - mysqldump:/etc/dump/ + - ./dump-mysql.sh:/docker-entrypoint-initdb.d/dump-mysql.sh + + mariadb-migrated-from-mysql8: + <<: *common-attributes # aliases + container_name: ${MARIADB_MIGRATED_name} + depends_on: + mariadb-dump: + condition: service_healthy + # restart: true + volumes: + - mysqldump:/etc/dump/ + - ./migrate-mariadb.sh:/docker-entrypoint-initdb.d/migrate-mariadb.sh + +volumes: + dbdata: {} + # sudo chown -R 999:999 ${PWD}/dump-data # on host before running this file + mysqldump: + driver: local + driver_opts: + type: none + device: "${PWD}/dump-data" + o: bind + +networks: + backend: \ No newline at end of file diff --git a/examples/migration-8.0/dump-mysql.sh b/examples/migration-8.0/dump-mysql.sh new file mode 100755 index 00000000..882e801f --- /dev/null +++ b/examples/migration-8.0/dump-mysql.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +echo 'MariaDB service started.' +# Run your commands and exit container +whoami # mysql +# sh -c "chown -R mysql:mysql /etc/dump" # Operation permitted +echo 'Remove files if exist' +files=("mysql-dump-data.sql.zst" "mysql-dump-users.sql.zst" "mysql-dump-stats.sql.zst" "mysql-dump-tzs.sql.zst") +for fileName in "${files[@]}"; do + if [ -f "$fileName" ]; then + echo "File ${fileName} exists. Remove it ... " + rm "$fileName" + fi +done +echo 'Dump and compress MySQL data with changed collation ...' +fileName="mysql-dump-data.sql.zst" +sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} ${MARIADB_DB} | sed 's/utf8mb4_0900/uca1400/g' | zstd > /etc/dump/${fileName}" +echo 'Dump and compress MySQL users ...' +fileName="mysql-dump-users.sql.zst" +sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=users | zstd > /etc/dump/${fileName}" +echo 'Dump and compress MySQL stats ...' +fileName="mysql-dump-stats.sql.zst" +sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=stats | zstd > /etc/dump/${fileName}" +echo 'Dump and compress MySQL timezones ...' +fileName="mysql-dump-tzs.sql.zst" +sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=timezones | zstd > /etc/dump/${fileName}" +echo 'Show MySQL 8.0 create user' +sh -c "mariadb -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} -e 'SELECT @@print_identified_with_as_hex; show create user current_user();'" \ No newline at end of file diff --git a/examples/migration-8.0/migrate-mariadb.sh b/examples/migration-8.0/migrate-mariadb.sh new file mode 100644 index 00000000..af66195c --- /dev/null +++ b/examples/migration-8.0/migrate-mariadb.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "Extract file" +oldFile="/etc/dump/mysql-dump.sql" +if [ -f "$oldFile" ]; then + echo "Old file ${oldFile} exists. Remove it ... " + rm "$oldFile" + echo "Extracting ..." +fi +sh -c "zstd -d /etc/dump/mysql-dump-data.sql.zst -o /etc/dump/mysql-dump.sql" +echo "Show data in MariaDB" +mariadb -uroot -p"${MARIADB_ROOT_PASSWORD}" -e "create database testdb;" +mariadb -uroot -p"${MARIADB_ROOT_PASSWORD}" "${MARIADB_DB}" < /etc/dump/mysql-dump.sql +mariadb -uroot -p"${MARIADB_ROOT_PASSWORD}" -e "show databases; select * from testdb.countries;" +# In order to import MySQL users we have to fix bug MDEV-33486 +# We need to load `caching_sha2_password` \ No newline at end of file diff --git a/examples/migration-8.0/mysql/mysql-data.sql b/examples/migration-8.0/mysql/mysql-data.sql new file mode 100644 index 00000000..c71d9588 --- /dev/null +++ b/examples/migration-8.0/mysql/mysql-data.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS countries; +CREATE TABLE countries(name char(20)); +INSERT INTO countries values ("Bosnia & Herzegovina"); \ No newline at end of file From de25342044cfd7b5a463e5c731a674ad6c430650 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 5 Nov 2025 18:12:49 +1100 Subject: [PATCH 3/6] Use latest stable MySQL service container --- examples/migration-8.0/compose-mysql8.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/migration-8.0/compose-mysql8.0.yml b/examples/migration-8.0/compose-mysql8.0.yml index 74563263..850693c4 100644 --- a/examples/migration-8.0/compose-mysql8.0.yml +++ b/examples/migration-8.0/compose-mysql8.0.yml @@ -29,7 +29,7 @@ services: MYSQL_PASSWORD: ${DB_PASSWORD} MYSQL_DATABASE: ${DB_DATABASE} container_name: ${MYSQL_name} - image: mysql:8.3.0 + image: mysql:8.4.7 healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent"] interval: 20s From ed7d131ad538f7e11ebe362bfa2c32d6439f490a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 5 Nov 2025 18:14:00 +1100 Subject: [PATCH 4/6] migration MySQL 8 example Make mariadb-dump a one-shot container --- examples/migration-8.0/compose-mysql8.0.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/migration-8.0/compose-mysql8.0.yml b/examples/migration-8.0/compose-mysql8.0.yml index 850693c4..675f92ae 100644 --- a/examples/migration-8.0/compose-mysql8.0.yml +++ b/examples/migration-8.0/compose-mysql8.0.yml @@ -49,8 +49,10 @@ services: <<: *common-attributes # aliases container_name: ${MARIADB_DUMP_name} depends_on: - mysql: - condition: service_healthy + mysql: + condition: service_healthy + restart: "no" + command: /docker-entrypoint-initdb.d/dump-mysql.sh volumes: - mysqldump:/etc/dump/ - ./dump-mysql.sh:/docker-entrypoint-initdb.d/dump-mysql.sh @@ -60,8 +62,7 @@ services: container_name: ${MARIADB_MIGRATED_name} depends_on: mariadb-dump: - condition: service_healthy - # restart: true + condition: service_completed_successfully volumes: - mysqldump:/etc/dump/ - ./migrate-mariadb.sh:/docker-entrypoint-initdb.d/migrate-mariadb.sh @@ -77,4 +78,4 @@ volumes: o: bind networks: - backend: \ No newline at end of file + backend: From 215b7aed18a72e816b6b3aa6e0b8219bb3945f94 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 5 Nov 2025 18:17:17 +1100 Subject: [PATCH 5/6] mysql 8 migration - handle script as command This allows the container to finish once the dump as been made to allow the next migration to start. With this, we've used the root user to allow changing of ownership of the mysqldump volume. Because of MDEV-20912 being resolved we don't need to manually change collations. --- examples/migration-8.0/compose-mysql8.0.yml | 7 +------ examples/migration-8.0/dump-mysql.sh | 16 +++++++++------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/examples/migration-8.0/compose-mysql8.0.yml b/examples/migration-8.0/compose-mysql8.0.yml index 675f92ae..11522736 100644 --- a/examples/migration-8.0/compose-mysql8.0.yml +++ b/examples/migration-8.0/compose-mysql8.0.yml @@ -52,6 +52,7 @@ services: mysql: condition: service_healthy restart: "no" + user: root command: /docker-entrypoint-initdb.d/dump-mysql.sh volumes: - mysqldump:/etc/dump/ @@ -69,13 +70,7 @@ services: volumes: dbdata: {} - # sudo chown -R 999:999 ${PWD}/dump-data # on host before running this file mysqldump: - driver: local - driver_opts: - type: none - device: "${PWD}/dump-data" - o: bind networks: backend: diff --git a/examples/migration-8.0/dump-mysql.sh b/examples/migration-8.0/dump-mysql.sh index 882e801f..19553b53 100755 --- a/examples/migration-8.0/dump-mysql.sh +++ b/examples/migration-8.0/dump-mysql.sh @@ -3,7 +3,9 @@ echo 'MariaDB service started.' # Run your commands and exit container whoami # mysql -# sh -c "chown -R mysql:mysql /etc/dump" # Operation permitted +if [ "$(whoami)" = root ]; then + chown -R mysql:mysql /etc/dump +fi echo 'Remove files if exist' files=("mysql-dump-data.sql.zst" "mysql-dump-users.sql.zst" "mysql-dump-stats.sql.zst" "mysql-dump-tzs.sql.zst") for fileName in "${files[@]}"; do @@ -14,15 +16,15 @@ for fileName in "${files[@]}"; do done echo 'Dump and compress MySQL data with changed collation ...' fileName="mysql-dump-data.sql.zst" -sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} ${MARIADB_DB} | sed 's/utf8mb4_0900/uca1400/g' | zstd > /etc/dump/${fileName}" +gosu mysql sh -c "mariadb-dump --ssl-verify-server-cert=OFF -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} ${MARIADB_DB} | zstd > /etc/dump/${fileName}" echo 'Dump and compress MySQL users ...' fileName="mysql-dump-users.sql.zst" -sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=users | zstd > /etc/dump/${fileName}" +gosu mysql sh -c "mariadb-dump --ssl-verify-server-cert=OFF -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=users | zstd > /etc/dump/${fileName}" echo 'Dump and compress MySQL stats ...' fileName="mysql-dump-stats.sql.zst" -sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=stats | zstd > /etc/dump/${fileName}" +gosu mysql sh -c "mariadb-dump --ssl-verify-server-cert=OFF -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=stats | zstd > /etc/dump/${fileName}" echo 'Dump and compress MySQL timezones ...' fileName="mysql-dump-tzs.sql.zst" -sh -c "mariadb-dump -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=timezones | zstd > /etc/dump/${fileName}" -echo 'Show MySQL 8.0 create user' -sh -c "mariadb -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} -e 'SELECT @@print_identified_with_as_hex; show create user current_user();'" \ No newline at end of file +gosu mysql sh -c "mariadb-dump --ssl-verify-server-cert=OFF -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} --system=timezones | zstd > /etc/dump/${fileName}" +echo 'Show MySQL 8.x create user' +gosu mysql sh -c "mariadb --ssl-verify-server-cert=OFF -h${MYSQL_CONT_NAME} -uroot -p${MARIADB_ROOT_PASSWORD} -e 'SELECT @@print_identified_with_as_hex; show create user current_user();'" From 14079de9d43cfb73bc4546f522505ecdd52c9201 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 5 Nov 2025 18:18:30 +1100 Subject: [PATCH 6/6] rename examples migration-8.0 to migration-8 --- examples/{migration-8.0 => migration-8}/.env | 2 +- examples/{migration-8.0 => migration-8}/.gitignore | 0 .../compose-mysql8.0.yml => migration-8/compose-mysql8.yml} | 0 examples/{migration-8.0 => migration-8}/dump-mysql.sh | 0 examples/{migration-8.0 => migration-8}/migrate-mariadb.sh | 0 examples/{migration-8.0 => migration-8}/mysql/mysql-data.sql | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename examples/{migration-8.0 => migration-8}/.env (76%) rename examples/{migration-8.0 => migration-8}/.gitignore (100%) rename examples/{migration-8.0/compose-mysql8.0.yml => migration-8/compose-mysql8.yml} (100%) rename examples/{migration-8.0 => migration-8}/dump-mysql.sh (100%) rename examples/{migration-8.0 => migration-8}/migrate-mariadb.sh (100%) rename examples/{migration-8.0 => migration-8}/mysql/mysql-data.sql (100%) diff --git a/examples/migration-8.0/.env b/examples/migration-8/.env similarity index 76% rename from examples/migration-8.0/.env rename to examples/migration-8/.env index 67f58abb..c9605b00 100644 --- a/examples/migration-8.0/.env +++ b/examples/migration-8/.env @@ -4,4 +4,4 @@ DB_PASSWORD='password' DB_DATABASE='testdb' MYSQL_name='mysql-container' MARIADB_DUMP_name='mariadb-container-dump' -MARIADB_MIGRATED_name='mariadb-migrated-mysql8.0' \ No newline at end of file +MARIADB_MIGRATED_name='mariadb-migrated-mysql8' diff --git a/examples/migration-8.0/.gitignore b/examples/migration-8/.gitignore similarity index 100% rename from examples/migration-8.0/.gitignore rename to examples/migration-8/.gitignore diff --git a/examples/migration-8.0/compose-mysql8.0.yml b/examples/migration-8/compose-mysql8.yml similarity index 100% rename from examples/migration-8.0/compose-mysql8.0.yml rename to examples/migration-8/compose-mysql8.yml diff --git a/examples/migration-8.0/dump-mysql.sh b/examples/migration-8/dump-mysql.sh similarity index 100% rename from examples/migration-8.0/dump-mysql.sh rename to examples/migration-8/dump-mysql.sh diff --git a/examples/migration-8.0/migrate-mariadb.sh b/examples/migration-8/migrate-mariadb.sh similarity index 100% rename from examples/migration-8.0/migrate-mariadb.sh rename to examples/migration-8/migrate-mariadb.sh diff --git a/examples/migration-8.0/mysql/mysql-data.sql b/examples/migration-8/mysql/mysql-data.sql similarity index 100% rename from examples/migration-8.0/mysql/mysql-data.sql rename to examples/migration-8/mysql/mysql-data.sql