diff --git a/test_scripts/ps/group-replication/README.md b/test_scripts/ps/group-replication/README.md index 489262f..d1d2a8b 100644 --- a/test_scripts/ps/group-replication/README.md +++ b/test_scripts/ps/group-replication/README.md @@ -16,9 +16,14 @@ group-replication/ ├── sysbench_helper.py # Sysbench — ephemeral sysbench load container ├── xtrabackup_helper.py # XtraBackup — full/incremental backup + restore ├── test_basic.py # smoke test: write on primary, read on every node -├── test_failover.py # primary failover + recovery under sysbench load +├── test_primary_shutdown_failover.py # primary mysqld stopped: election + auto-rejoin ├── test_scaling.py # scale up 3->5 and down 5->3 under sysbench load -└── test_backup_restore.py # XtraBackup full+incremental backup and restore +├── test_backup_restore.py # XtraBackup full+incremental backup and restore +├── test_secondary_isolation_ist.py # secondary network-partitioned, rejoins via IST +├── test_secondary_isolation_sst.py # same, binlogs purged so it rejoins via clone/SST +├── test_primary_isolation_failover.py # primary partitioned: automatic failover +├── test_majority_loss.py # both secondaries cut off: quorum loss, no writes +└── test_equal_partition.py # 4-node 2-2 split: split-brain prevention ``` ## Prerequisites @@ -46,12 +51,12 @@ The fixture brings up 3 containers (by default `ps-1`, `ps-2 `grnet-` network (e.g. `grnet-0` / `ps0-1..3` when running serially, or `grnet-gw0` / `psgw0-1..3` under pytest-xdist), bootstraps the cluster via mysqlsh, runs the tests, then removes containers, volumes, and the network. Expect ~1 minute end-to-end. -## Failover test (sysbench) +## Failover test (primary shutdown) -`test_failover.py` drives real load and exercises a primary outage: +`test_primary_shutdown_failover.py` drives real load and exercises a primary outage: ```bash -GR_VERBOSE=1 pytest -v test_failover.py +GR_VERBOSE=1 pytest -v test_primary_shutdown_failover.py ``` What it does: load initial data with sysbench (`prepare`, 4 tables × 10000 rows), @@ -61,6 +66,9 @@ restart the stopped node (it **auto-rejoins** because the framework persists `group_replication_start_on_boot=ON`), then run another 20s workload against the full cluster and compare checksums across all three. Expect ~2-3 minutes. +This kills mysqld, so the group loses the member immediately. For the variant where +the process stays alive and only its network is cut, see the partition tests below. + Sysbench notes: - Runs from the multi-arch image `pingwinator/sysbench:latest` (pulled on first use). Each sysbench command is its own one-shot `--rm` container named @@ -76,6 +84,87 @@ Relevant `GroupReplication` helpers: `get_primary()`, `stop_node()`, `rejoin_node()`, `wait_all_online()`, and `verify_checksums(database, nodes=...)` (defaults to the currently-online nodes). +## Network partition tests + +Three tests injure the cluster with `docker network disconnect` instead of +`docker stop`. That distinction is the whole point: the container keeps running, so +**mysqld stays alive and keeps its data** — the group has to cope with a member it +cannot reach rather than one that has cleanly departed. The node is put back with +`docker network connect`. Expect ~3.5 minutes per proxy for each test. + +```bash +GR_VERBOSE=1 pytest -v test_secondary_isolation_ist.py +GR_VERBOSE=1 pytest -v test_secondary_isolation_sst.py +GR_VERBOSE=1 pytest -v test_primary_isolation_failover.py +GR_VERBOSE=1 pytest -v test_majority_loss.py +GR_VERBOSE=1 pytest -v test_equal_partition.py +``` + +- **`test_secondary_isolation_ist.py`** — isolates one secondary, runs a 30s sysbench + workload so it falls behind, then heals while the donors still hold the binary logs + covering that window. Asserts the node came back by **IST**: no new row in + `performance_schema.clone_status`, and its `gtid_executed` caught up to the primary's. +- **`test_secondary_isolation_sst.py`** — the same partition, but + `purge_binary_logs()` runs on **every** surviving node before the heal, so no donor can + serve an IST. Asserts the node came back by **clone/SST**: a *new* clone row with + `STATE=Completed, ERROR_NO=0`. Purging only the primary would not be enough — GR picks + its recovery donor from any `ONLINE` member. +- **`test_primary_isolation_failover.py`** — isolates the **primary**. The two surviving + secondaries hold majority and must expel it, elect a new primary and become writable on + their own; the test measures and logs that failover window, checks the proxy re-routes + writes to the new primary, and confirms the old primary rejoins as a `SECONDARY`. + After the heal it rebuilds the proxy and checks the rejoined node is serving reads again — + it came back on a new address, and without that rebuild HAProxy drops it from the read + backend for good. It sets `group_replication_unreachable_majority_timeout=30` on the + primary beforehand: at the default of `0` a minority-blocked member never leaves the group, so + `group_replication_exit_state_action` never fires and writes hang rather than being + rejected. With the timeout set, the old primary self-ejects and goes `super_read_only`, + and the test can assert that the write it refused exists on no node afterwards. + +- **`test_majority_loss.py`** — cuts off **both** secondaries at once, so every member is + alone and no side holds a majority. The surviving primary must refuse writes rather than + accept anything it can never replicate: the test asserts writes fail both directly and + through the proxy, and that neither exists on any node afterwards. Reconnecting the + secondaries restores a 2-of-3 quorum between them, then the old primary rejoins. Like the + primary-isolation test it sets `group_replication_unreachable_majority_timeout=30` on the + primary, so it leaves the group and goes `super_read_only` instead of blocking forever. + + Recovery here is **not** automatic, which is worth knowing before writing more of these. + Reconnecting the network is not enough: a member blocked in a minority keeps its stale + view and still reports the others `UNREACHABLE`. The group has to be forced back with + `force_members()`, and XCOM refuses a forced list containing anyone it currently suspects + (*"Only alive members in the current configuration should be present in a forced + configuration list"*) — so it is forced down to a **single** member and the rest rejoin + with `restart_group_replication()`. + +- **`test_equal_partition.py`** — the only 4-node test. Splits the cluster down the middle + into two halves of two, each intact internally but holding just 2 of 4. Neither half may + accept a write or promote a primary of its own. GR cannot resolve an even split by itself: + recovery is an operator running `force_members()` on the chosen half — and unlike the + majority-loss case the forced list can name **both** its members, since they are alive and + can see each other. Roughly 4 minutes per proxy. + +Two things about splitting the cluster. `partition_group()` detaches each node from the +network individually, so the isolated nodes cannot see **each other** either — it produces N +one-node partitions. Moving a pair onto a second network does *not* fix that: a container +that changes network changes IP, and XCOM does not follow a peer to a new address, so the +moved pair lose each other too (verified). A split into two internally-connected halves +therefore uses `sever_link()`, which blackholes the other half's addresses with reject routes +inside each container, leaving every IP and process untouched. That is why node containers +run with `NET_ADMIN`. + +And one about writes during a partition. With `group_replication_unreachable_majority_timeout` +at its default `0`, a write to a primary that has lost quorum **blocks** rather than failing — +and a blocked write is not discarded. It is parked awaiting consensus, so it commits once that +half is unblocked, even if the client that issued it has gone. Assert that no write becomes +*visible* while the split is in effect, rather than that it never lands. + +Two notes for anyone writing more of these. `performance_schema.clone_status` is never +empty on a secondary — `create()` adds every node with `recoveryMethod:'clone'`, so the +only way to tell an IST from an SST is to compare snapshots taken before and after. And +`COUNT_TRANSACTIONS_REMOTE_APPLIED` stays `0` after a recovery-channel catch-up, since it +only counts what arrives once a member is already `ONLINE`; use `gtid_subset()` instead. + ## Scaling test (sysbench) `test_scaling.py` exercises elastic membership changes and data consistency checks, with a workload phase between scale operations: @@ -96,7 +185,7 @@ Relevant `GroupReplication` helpers: `scale_up(count)`, `scale_down(count)`, original members (i.e. `` — e.g. `ps0-4`, `ps0-5`, … with the default fixture). proxy is reconciled automatically after each change — MySQL Router auto-discovers members from cluster metadata; HAProxy's container is recreated so its static backend -server list matches the new membership (`_refresh_proxy()`). +server list matches the new membership (`refresh_proxy()`). ## Backup / restore test (XtraBackup) @@ -447,6 +536,11 @@ docker network rm grnet- Add files named `test_*.py` in this directory. Request the `gr_cluster` fixture and use: +The fixture's indirect parameter is normally just a proxy name (`"router"` / `"haproxy"`), +which gives a 3-node cluster. For a different size pass a `(proxy, num_nodes)` tuple, wrapped +in `pytest.param(..., id=proxy)` so the node id stays readable — see +`test_equal_partition.py`, which asks for 4 nodes. + - `gr_cluster.exec_sql("SQL;")` — run application SQL through the read/write endpoint (the router when enabled, else the primary). Use this for DDL/DML instead of targeting a node directly, so the test is proxy-agnostic. @@ -455,11 +549,76 @@ fixture and use: - `gr_cluster.get_bootstrap_node()` — name of the bootstrap node (e.g. `"ps0-1"` when running serially, or `"psgw0-1"` under pytest-xdist); for the currently-elected primary (which differs after failover) use `gr_cluster.get_primary()`. - `gr_cluster.containers` — list of all node names in start order. +- `gr_cluster.stop_node(node)` / `gr_cluster.rejoin_node(node)` — kill and restart a + node's mysqld; the group sees an immediate member loss. +- `gr_cluster.isolate_node(node)` / `gr_cluster.heal_node(node)` — network-partition a + node and heal it again. Unlike `stop_node()`, mysqld keeps running and only loses + connectivity, which is what exercises GR's expulsion, minority-block and distributed + recovery (IST/SST) paths. `heal_node()` returns `True` when GR rejoined the member on + its own and `False` when it needed an explicit `START GROUP_REPLICATION`. +- `gr_cluster.node_alive(node)` — does mysqld still answer queries? (liveness, not + membership — the point of a partition test is that this stays `True`). +- `gr_cluster.local_member_state(node)` — the node's own `MEMBER_STATE` as it sees itself + (`ONLINE` / `RECOVERING` / `ERROR` / `OFFLINE`), for the minority side of a partition. +- `gr_cluster.wait_node_isolated(node)` — poll until an isolated node sees no group + member but itself as `ONLINE`, and return its view. On timeout it returns the last view + it read rather than raising, so the caller has to assert on what comes back. The minority + side runs its own suspicion timer, so this is still needed after `wait_online_count()` + has settled. +- `gr_cluster.sever_link(group_a, group_b)` / `gr_cluster.restore_link(group_a, group_b)` — + cut two halves of the cluster off from each other with reject routes, leaving every IP and + process intact. Use this, not `partition_group()`, when both halves must stay internally + connected. Nodes in `group_b` drop out of `active_nodes`. +- `gr_cluster.partition_group(nodes)` / `gr_cluster.heal_group(nodes)` — isolate or + reconnect several nodes at once. `partition_group()` gives each node its **own** one-node + partition (they lose contact with each other too), so it cannot express a split into two + communicating sub-groups. `heal_group()` only reconnects and restores `active_nodes`; it + does not wait for `ONLINE`, because after a majority loss members come back in stages. +- `gr_cluster.wait_members_unreachable(nodes, node=...)` — poll until the given members are + seen as `UNREACHABLE` from an observer node, and return that view. Same contract as + `wait_node_isolated()`: a timeout returns the last view read rather than raising, so + assert on what comes back. +- `gr_cluster.force_members(nodes, node=...)` / `gr_cluster.restart_group_replication(node)` — + recovery from a lost quorum. Force the membership down to the single node you run it on + (XCOM rejects any member it currently suspects), then restart GR on the others so they + rejoin. `force_members()` always clears `group_replication_force_members` afterwards. +- `gr_cluster.refresh_proxy()` — rebuild HAProxy so it picks up the current backend + addresses. Reconnecting a container to the network gives it a **new IP**, and HAProxy + resolves its backends once at config-parse time, so **any** test that reconnects a node + must call this before `wait_proxy_ready()` — not only ones where that node goes on to + become the primary. A rejoined *secondary* on a stale address is health-checked out of the + read backend and silently stops serving reads, which is easy to miss because the write + path still works. No-op for MySQL Router, which resolves by name. +- `wait_all_online(node=...)` / `wait_online_count(n, node=...)` — read membership from an + explicit node. Needed when the head of `active_nodes` is not in the group. Unlike the two + above, these **raise** on timeout. +- `gr_cluster.super_read_only(node)` / `gr_cluster.wait_super_read_only(node)` — read a + node's `@@super_read_only`, or poll until it is `ON`, for the side of a partition that has + lost quorum. `wait_super_read_only()` returns whether it got there rather than raising, so + assert on the result. +- `gr_cluster.clone_status(node)` — the node's current/last clone operation as a + column→value map (`ID`, `STATE`, `ERROR_NO`, `BEGIN_TIME`), `{}` if it never cloned. + Every secondary already carries a completed row from the clone-based `addInstance` in + `create()`, so to tell an IST from an SST compare snapshots taken before and after, + rather than checking whether a row exists. +- `gr_cluster.gtid_executed(node)` / `gr_cluster.gtid_subset(node, gtid_set)` — the node's + `gtid_executed` as a single-line GTID set, and whether it is a superset of another set. + Use these to prove a node caught up: `COUNT_TRANSACTIONS_REMOTE_APPLIED` stays 0 after a + recovery-channel catch-up, since it only counts what arrives once a member is `ONLINE`. +- `gr_cluster.purge_binary_logs(nodes=None)` — `FLUSH` + `PURGE BINARY LOGS` on each node + (default: every active node), returning the resulting `gtid_purged` per node so you can + assert something was actually purged. Defaults to all nodes because GR picks its recovery + donor from any `ONLINE` member — purging only the primary still leaves a donor that can + serve an IST. - `gr_cluster.docker` — the `DockerHelper`. Common methods: - `docker.exec_mysql(node, "SQL;", database=None)` → returns `ExecResult` with `.stdout`, `.stderr`, `.returncode`, `.ok`. - `docker.exec_mysqlsh(node, "")` — same return shape, runs mysqlsh AdminAPI. - `docker.exec_command(node, "shell command")` — arbitrary `sh -c` inside the container. - `docker.stop(node)` / `docker.start(node)` — useful for failover-style tests. + - `docker.network_disconnect(network, node)` / `docker.network_connect(network, node)` — + the raw partition primitive behind `isolate_node()`/`heal_node()`; both are no-ops when + the container is already in the requested state. `docker.container_networks(node)` + reports what it is attached to right now. Skeleton: diff --git a/test_scripts/ps/group-replication/conftest.py b/test_scripts/ps/group-replication/conftest.py index a7dc394..95aba31 100644 --- a/test_scripts/ps/group-replication/conftest.py +++ b/test_scripts/ps/group-replication/conftest.py @@ -21,7 +21,9 @@ # Proxy modes the suite can run a test behind. There is intentionally no "direct" # entry: every test runs behind a proxy. Each test selects its proxies explicitly # with @pytest.mark.parametrize("gr_cluster", [...], indirect=True) — see the test -# files. The value passed (e.g. "router"/"haproxy") is the key looked up here. +# files. The value passed is either a proxy name ("router"/"haproxy", giving the +# default 3 nodes) or a (proxy, num_nodes) tuple for a differently-sized cluster; +# the proxy name is the key looked up here. PROXIES = { "router": {"mysql_router": True}, "haproxy": {"haproxy": True}, @@ -44,16 +46,35 @@ def gr_cluster(request): # @pytest.mark.parametrize("gr_cluster", [...], indirect=True). Validate it explicitly # so a test that forgets the decorator fails with a clear message instead of an opaque # AttributeError (no param) / KeyError (unknown proxy). - proxy = getattr(request, "param", None) - if proxy is None: + # + # A test needing a different cluster size passes a (proxy, num_nodes) tuple instead of a + # bare proxy name — wrapped in pytest.param(..., id=proxy), or the node id degrades to + # "gr_cluster0". Everything else about the fixture is the same either way. + param = getattr(request, "param", None) + if param is None: raise pytest.UsageError( 'gr_cluster requires a proxy via indirect parametrization, e.g. ' '@pytest.mark.parametrize("gr_cluster", ["haproxy"], indirect=True)' ) - if proxy not in PROXIES: + if isinstance(param, tuple): + if len(param) != 2: + raise pytest.UsageError( + f"gr_cluster tuple parameter must be (proxy, num_nodes); got {param!r}" + ) + proxy, num_nodes = param + else: + proxy, num_nodes = param, 3 + # isinstance before the lookup: an unhashable proxy (e.g. a list) would otherwise raise + # TypeError from inside the dict membership test rather than reporting the bad value. + if not isinstance(proxy, str) or proxy not in PROXIES: raise pytest.UsageError( f"unknown gr_cluster proxy {proxy!r}; valid options: {sorted(PROXIES)}" ) + # bool is a subclass of int, so without the isinstance guard True would pass as 1 node. + if isinstance(num_nodes, bool) or not isinstance(num_nodes, int) or num_nodes < 1: + raise pytest.UsageError( + f"gr_cluster num_nodes must be a positive integer; got {num_nodes!r}" + ) try: helper = DockerHelper() except RuntimeError as exc: @@ -71,7 +92,7 @@ def gr_cluster(request): offset = int(m.group()) if m else 0 cluster = GroupReplication( helper, - num_nodes=3, + num_nodes=num_nodes, network=f"grnet-{safe_workerid}", node_prefix=f"ps{safe_workerid}-", base_host_port=33060 + offset * 100, diff --git a/test_scripts/ps/group-replication/docker_helper.py b/test_scripts/ps/group-replication/docker_helper.py index 2510f7f..9a0a21b 100644 --- a/test_scripts/ps/group-replication/docker_helper.py +++ b/test_scripts/ps/group-replication/docker_helper.py @@ -89,6 +89,7 @@ def create( detach: bool = True, restart: str | None = None, platform: str | None = None, + cap_add: list[str] | None = None, ) -> ExecResult: """Create and start a long-lived (detached) container with the given config.""" # These containers are long-lived (no --rm), so a run that crashed before teardown @@ -110,6 +111,8 @@ def create( args.extend(["--entrypoint", entrypoint]) if restart: args.extend(["--restart", restart]) + for cap in cap_add or []: + args.extend(["--cap-add", cap]) for k, v in (environment or {}).items(): args.extend(["-e", f"{k}={v}"]) for vol in volumes or []: @@ -261,10 +264,78 @@ def network_remove(self, name: str) -> ExecResult: """Remove a container network, ignoring errors if it does not exist.""" return self._run(["network", "rm", name], check=False) + def container_networks(self, name: str) -> list[str]: + """Return the names of the networks a container is currently attached to. + + Empty means attached to nothing — the normal state of a node that + network_disconnect() has isolated, not an error. An inspect that fails (no such + container, no daemon) raises rather than returning [], so callers can act on the + answer instead of guessing: reporting a failure as "attached to nothing" would make + network_disconnect() skip the disconnect and report success, leaving a partition + test reasoning about a partition that never happened. + """ + result = self._run( + [ + "inspect", + "-f", + '{{range $net, $_ := .NetworkSettings.Networks}}{{$net}}{{"\\n"}}{{end}}', + name, + ], + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + def network_connect(self, network: str, name: str) -> ExecResult | None: + """Attach a running container to a network, doing nothing if it is already attached. + + The idempotence matters for reruns and for healing a partition that was only + partially applied: connecting twice otherwise fails with "already exists in network". + Returns None when the container was already attached. + """ + if network in self.container_networks(name): + return None + return self._run(["network", "connect", network, name]) + + def network_disconnect(self, network: str, name: str, force: bool = False) -> ExecResult | None: + """Detach a running container from a network, doing nothing if it is not attached. + + Unlike stop(), the process inside the container is untouched — it simply loses + connectivity — which is what makes this usable for network-partition tests. + Returns None when the container was not attached in the first place. + + Note: reconnecting later does not restore the container's published host port + mappings (the -p flags given at create time). Nothing in this suite reaches nodes + from the host, but a healed node is no longer reachable on its host port. + """ + if network not in self.container_networks(name): + return None + args = ["network", "disconnect"] + if force: + args.append("--force") + args.extend([network, name]) + return self._run(args) + def volume_remove(self, name: str) -> ExecResult: """Remove a container volume, ignoring errors if it does not exist.""" return self._run(["volume", "rm", name], check=False) + def container_ip(self, name: str, network: str) -> str: + """Return a container's IPv4 address on the given network, or "" if it is not attached.""" + template = f'{{{{(index .NetworkSettings.Networks "{network}").IPAddress}}}}' + result = self._run(["inspect", "-f", template, name], check=False) + return result.stdout.strip() if result.ok else "" + + def container_state(self, name: str) -> str: + """Return a container's status and restart count ("running restarts=0"), or "" if unknown. + + Handy in a timeout message: it distinguishes a container that is up but not yet + serving from one that has died or is stuck in a restart loop. + """ + result = self._run( + ["inspect", "-f", "{{.State.Status}} restarts={{.RestartCount}}", name], + check=False, + ) + return result.stdout.strip() if result.ok else "" + def container_exists(self, name: str) -> bool: """Return True if a container with the exact given name exists (running or stopped).""" result = self._run( diff --git a/test_scripts/ps/group-replication/group_replication_helper.py b/test_scripts/ps/group-replication/group_replication_helper.py index 28811a5..27112b8 100644 --- a/test_scripts/ps/group-replication/group_replication_helper.py +++ b/test_scripts/ps/group-replication/group_replication_helper.py @@ -227,35 +227,45 @@ def member_states(self, node: str) -> dict[str, tuple[str, str]]: states[parts[0]] = (parts[1], parts[2]) return states - def wait_all_online(self, timeout: int = 180) -> None: - """Wait until every expected member reports the ONLINE state, or time out.""" + def wait_all_online(self, timeout: int = 180, node: str | None = None) -> None: + """Wait until every expected member reports the ONLINE state, or time out. + + Membership is read from `node`, defaulting to the first active node. Pass one + explicitly when the head of active_nodes is not a member of the group (e.g. after a + majority loss, where a node that left the group only ever reports itself). + """ if not self.active_nodes: raise RuntimeError("No active nodes") + observer = node or self.active_nodes[0] self.log("wait for all members ONLINE") deadline = time.time() + timeout last: dict[str, tuple[str, str]] = {} while time.time() < deadline: - states = self.member_states(self.active_nodes[0]) + states = self.member_states(observer) last = states if len(states) == self.num_nodes and all(s == "ONLINE" for s, _ in states.values()): return time.sleep(2) raise RuntimeError(f"Not all members ONLINE within {timeout}s (last: {last})") - def wait_online_count(self, expected: int, timeout: int = 120) -> dict[str, tuple[str, str]]: + def wait_online_count( + self, expected: int, timeout: int = 120, node: str | None = None + ) -> dict[str, tuple[str, str]]: """Wait until exactly `expected` group members report ONLINE, returning their state map. Used after intentionally stopping a node: failure detection and expulsion take a few seconds, so the group needs a moment to settle to the smaller size before its - membership is inspected. Polls a surviving node; raises on timeout. + membership is inspected. Membership is read from `node`, defaulting to the first + active node; raises on timeout. """ if not self.active_nodes: raise RuntimeError("No active nodes") + observer = node or self.active_nodes[0] self.log(f"wait for exactly {expected} members ONLINE") deadline = time.time() + timeout last: dict[str, tuple[str, str]] = {} while time.time() < deadline: - states = self.member_states(self.active_nodes[0]) + states = self.member_states(observer) last = states if sum(1 for state, _ in states.values() if state == "ONLINE") == expected: return states @@ -271,6 +281,352 @@ def rejoin_node(self, name: str, timeout: int = 180) -> None: self.active_nodes.append(name) self.wait_all_online(timeout=timeout) + def node_alive(self, name: str) -> bool: + """Return True if mysqld on the node still answers queries (liveness, not membership). + + Probed with exec_mysql rather than `mysqladmin ping` through exec_command for the + same reason _wait_ready does: no shell is involved, so a root_password containing + shell metacharacters can't break the check. Useful during a network partition, + where the container has no connectivity but the process is expected to stay up. + """ + return self.docker.exec_mysql( + name, "SELECT 1;", password=self.root_password, check=False, timeout=15 + ).ok + + def local_member_state(self, name: str) -> str: + """Return the node's own MEMBER_STATE as it sees itself, or '' when unreadable.""" + # Matched on MEMBER_ID rather than MEMBER_HOST: MEMBER_HOST can be blank while the + # member is OFFLINE, which is exactly the state callers need to detect here. + result = self.docker.exec_mysql( + name, + "SELECT MEMBER_STATE FROM performance_schema.replication_group_members " + "WHERE MEMBER_ID=@@server_uuid;", + password=self.root_password, + check=False, + timeout=15, + ) + return result.stdout.strip() if result.ok else "" + + def isolate_node(self, name: str) -> None: + """Sever a node's network while leaving its mysqld running, and drop it from active_nodes. + + The network-partition counterpart of stop_node(): the process stays up and keeps + its data, it just can no longer reach (or be reached by) the rest of the group. + Dropping it from active_nodes keeps get_primary(), the wait_* helpers and + verify_checksums() — all of which query active_nodes[0] — pointed at a survivor. + """ + self.log(f"isolate node {name} (disconnect from {self.network})") + self.docker.network_disconnect(self.network, name) + if name in self.active_nodes: + self.active_nodes.remove(name) + + def partition_group(self, names: list[str]) -> None: + """Isolate several nodes at once, each into its own one-node partition. + + A loop over isolate_node(). Because each node is detached from the network entirely, + they lose contact with *each other* as well as with the rest of the group, so this + cannot express a split into two communicating sub-groups — use sever_link() for + that, which blackholes one half from the other with reject routes and leaves every + address intact. Moving a pair onto a second network does not work: they get new IPs, + and XCOM does not follow a peer to a new address, so the moved nodes lose each other + too. + """ + if not names: + raise ValueError("names must not be empty") + self.log(f"partition group: isolate {', '.join(names)}") + for name in names: + self.isolate_node(name) + + def heal_group(self, names: list[str]) -> None: + """Reconnect several isolated nodes and put them back in active_nodes, without waiting. + + Deliberately does not wait for ONLINE: after a majority loss the members come back + in stages — the reconnected ones regain quorum between themselves first, and a + member that left the group needs an explicit restart on top. Use heal_node() for the + single-node case where "reconnect and be ONLINE again" is one step. + """ + if not names: + raise ValueError("names must not be empty") + self.log(f"heal group: reconnect {', '.join(names)}") + for name in names: + self.docker.network_connect(self.network, name) + self._wait_ready(name) + if name not in self.active_nodes: + self.active_nodes.append(name) + + def force_members( + self, names: list[str], node: str, timeout: int = 120 + ) -> dict[str, tuple[str, str]]: + """Force a new group membership from a member that has lost quorum; return the new view. + + The documented recovery from majority loss. A member blocked in a minority does not + reconfigure on its own even once the network is back — it stays stuck with the + absent members UNREACHABLE — so the surviving set has to be imposed on it. + + XCOM refuses a forced list containing any member it currently suspects ("Only alive + members in the current configuration should be present in a forced configuration + list"), and a member it lost contact with stays suspected while the group is + blocked. In practice that means forcing down to `node` alone and bringing the rest + back with restart_group_replication(). + + group_replication_force_members is always reset to '' afterwards (including on + timeout): leaving it set makes GR reject later membership changes. + """ + if not names: + raise ValueError("names must not be empty") + addresses = ",".join(self._gr_address(name) for name in names) + self.log(f"force group membership to {addresses} via {node}") + self.docker.exec_mysql( + node, + f"SET GLOBAL group_replication_force_members={sql_str(addresses)};", + password=self.root_password, + ) + try: + return self.wait_online_count(len(names), timeout=timeout, node=node) + finally: + self.docker.exec_mysql( + node, + "SET GLOBAL group_replication_force_members='';", + password=self.root_password, + check=False, + ) + + def _peer_route(self, node: str, peers: list[str], action: str, check: bool) -> None: + """Add or delete reject routes on `node` for each peer's address on the cluster network.""" + for peer in peers: + ip = self.docker.container_ip(peer, self.network) + if not ip: + raise RuntimeError(f"could not resolve {peer}'s address on {self.network}") + self.docker.exec_command(node, f"route {action} -host {ip} reject", check=check) + + def sever_link(self, group_a: list[str], group_b: list[str]) -> None: + """Cut two halves of the cluster off from each other, leaving every IP and process intact. + + Each node blackholes the other half's addresses with a local reject route, so the + two groups cannot talk to each other while each stays fully connected internally. + That is what an even split needs, and it cannot be done by moving containers between + networks: a container that changes network changes IP, and XCOM does not follow a + peer to a new address — the moved nodes lose each other as well. + + Requires the containers to have NET_ADMIN (see _start_mysqld_node). Nodes in + `group_b` are dropped from active_nodes, so get_primary() and the wait_* helpers + keep observing `group_a`. + """ + if not group_a or not group_b: + raise ValueError("both groups must be non-empty") + self.log(f"sever link: {', '.join(group_a)} | {', '.join(group_b)}") + for node in group_a: + self._peer_route(node, group_b, "add", check=True) + for node in group_b: + self._peer_route(node, group_a, "add", check=True) + if node in self.active_nodes: + self.active_nodes.remove(node) + + def restore_link(self, group_a: list[str], group_b: list[str]) -> None: + """Remove the reject routes added by sever_link and put group_b back in active_nodes. + + Only restores connectivity: members blocked during the split keep their stale view + and still need restart_group_replication() to rejoin. + """ + if not group_a or not group_b: + raise ValueError("both groups must be non-empty") + self.log(f"restore link: {', '.join(group_a)} | {', '.join(group_b)}") + for node in group_a: + # check=False: a partially applied sever leaves nothing to delete on some nodes. + self._peer_route(node, group_b, "del", check=False) + for node in group_b: + self._peer_route(node, group_a, "del", check=False) + if node not in self.active_nodes: + self.active_nodes.append(node) + + def wait_members_unreachable( + self, names: list[str], node: str, timeout: int = 60 + ) -> dict[str, tuple[str, str]]: + """Poll until every named member is seen as UNREACHABLE from `node`; return that view. + + On timeout the last view read is returned rather than raising, so the caller can + assert on the state actually observed and report it — same contract as + wait_node_isolated(). + """ + self.log(f"wait for {', '.join(names)} to be UNREACHABLE from {node}") + deadline = time.time() + timeout + while True: + states = self.member_states(node) + if all(states.get(name, ("", ""))[0] == "UNREACHABLE" for name in names): + return states + if time.time() >= deadline: + return states + time.sleep(2) + + def heal_node(self, name: str, timeout: int = 180, rejoin_grace: int = 30) -> bool: + """Reconnect an isolated node and wait until every member is ONLINE again. + + Returns True when GR readmitted the member on its own, False when the explicit + STOP/START GROUP_REPLICATION fallback was needed. Both outcomes are valid and the + test can't control which one GR takes, so it is logged rather than asserted: a + member that was expelled can come back through group_replication_autorejoin_tries, + but one that merely sat blocked in the minority (the default + group_replication_unreachable_majority_timeout=0 never makes it leave the group) + has no auto-rejoin to trigger and only an explicit start gets it back. + """ + self.log(f"heal node {name} (reconnect to {self.network})") + survivors = [node for node in self.active_nodes if node != name] + if not survivors: + raise RuntimeError(f"No surviving node to observe {name} rejoining from") + self.docker.network_connect(self.network, name) + self._wait_ready(name) + if name not in self.active_nodes: + self.active_nodes.append(name) + + # Poll the group's view of the member, not the member's view of itself: a node + # blocked in a minority partition keeps reporting its own MEMBER_STATE as ONLINE, + # so its self-report says nothing about whether it has been readmitted. + self.log(f"wait up to {rejoin_grace}s for {name} to rejoin on its own") + deadline = time.time() + rejoin_grace + while time.time() < deadline: + state = self.member_states(survivors[0]).get(name, ("", ""))[0] + if state in ("ONLINE", "RECOVERING"): + self.wait_all_online(timeout=timeout) + return True + time.sleep(2) + + self.log(f"{name} did not auto-rejoin in {rejoin_grace}s " + f"(state {self.local_member_state(name)!r}); issuing START GROUP_REPLICATION") + self.restart_group_replication(name) + self.wait_all_online(timeout=timeout) + return False + + def restart_group_replication(self, name: str) -> None: + """Stop and start Group Replication on a node, bringing it back into the group. + + The way a member that is stuck or has left rejoins: reconnecting the network is not + enough on its own, because a member blocked in a minority keeps its stale view even + after contact is restored. + """ + self.log(f"restart group replication on {name}") + # STOP is a no-op when GR is already stopped, so failures here are not interesting. + self.docker.exec_mysql( + name, "STOP GROUP_REPLICATION;", password=self.root_password, check=False + ) + self.docker.exec_mysql(name, "START GROUP_REPLICATION;", password=self.root_password) + + def wait_node_isolated(self, name: str, timeout: int = 60) -> dict[str, tuple[str, str]]: + """Wait until an isolated node sees no group member but itself as ONLINE, returning its view. + + The minority side runs its own suspicion timer, independent of the majority's + expulsion, so this still needs polling after wait_online_count() has settled on the + surviving side. Returns the last view read, so the caller can assert on it. + """ + self.log(f"wait for {name} to see itself cut off from the group") + deadline = time.time() + timeout + while True: + states = self.member_states(name) + reachable = {host for host, (state, _) in states.items() if state == "ONLINE"} + if not reachable - {name} or time.time() >= deadline: + return states + time.sleep(2) + + def super_read_only(self, name: str) -> str: + """Return the node's @@GLOBAL.super_read_only as '1'/'0', or '' when unreadable. + + check=False so a node that is mid-restart (or otherwise unreachable) yields '' for + the caller to poll on, rather than raising. + """ + result = self.docker.exec_mysql( + name, + "SELECT @@GLOBAL.super_read_only;", + password=self.root_password, + check=False, + timeout=15, + ) + return result.stdout.strip() if result.ok else "" + + def wait_super_read_only(self, name: str, timeout: int = 90) -> bool: + """Poll until the node reports super_read_only=ON, returning whether it got there. + + Returns a bool instead of raising so the caller can build a failure message from the + surrounding diagnostics (exit_state_action, local_member_state). The default timeout + leaves room for a group_replication_unreachable_majority_timeout window plus the + expulsion that follows it. + """ + self.log(f"wait for {name} to become super_read_only") + deadline = time.time() + timeout + while True: + if self.super_read_only(name) == "1": + return True + if time.time() >= deadline: + return False + time.sleep(2) + + def clone_status(self, name: str) -> dict[str, str]: + """Return the node's current/last clone operation as a column->value map, {} if never cloned. + + performance_schema.clone_status holds at most one row — the latest operation — so + this doubles as the before/after snapshot for "did a new clone run?". Note that a + node added by create()/scale_up() already carries a completed row from the + recoveryMethod:'clone' addInstance, so an empty map is not the healthy default. + """ + columns = ["ID", "STATE", "ERROR_NO", "BEGIN_TIME"] + result = self.docker.exec_mysql( + name, + f"SELECT {', '.join(columns)} FROM performance_schema.clone_status;", + password=self.root_password, + check=False, + timeout=15, + ) + line = result.stdout.strip() + if not result.ok or not line: + return {} + return dict(zip(columns, line.split("\t"))) + + def gtid_executed(self, name: str) -> str: + """Return the node's @@GLOBAL.gtid_executed as a single-line GTID set.""" + # The client prints a newline per UUID set; GTID_SUBSET() and friends want one line. + result = self.docker.exec_mysql( + name, "SELECT @@GLOBAL.gtid_executed;", password=self.root_password, timeout=30 + ) + return result.stdout.strip().replace("\n", "") + + def gtid_subset(self, name: str, gtid_set: str) -> bool: + """Return True when the node's gtid_executed is a superset of the given GTID set.""" + result = self.docker.exec_mysql( + name, + f"SELECT GTID_SUBSET({sql_str(gtid_set)}, @@GLOBAL.gtid_executed);", + password=self.root_password, + timeout=30, + ) + return result.stdout.strip() == "1" + + def purge_binary_logs(self, nodes: list[str] | None = None) -> dict[str, str]: + """Purge the binary logs on each node, returning the resulting gtid_purged per node. + + Defaults to every active node, which is what forcing clone (SST) recovery requires: + GR picks its recovery donor from any ONLINE member, so purging only the primary + leaves the remaining secondary able to serve an IST instead. + + The returned gtid_purged is the caller's proof that something was actually purged — + an empty value means the logs are still there and no IST path has been closed off. + """ + nodes = nodes if nodes is not None else list(self.active_nodes) + if not nodes: + raise RuntimeError("No nodes to purge binary logs on") + self.log(f"purge binary logs on {', '.join(nodes)}") + purged: dict[str, str] = {} + for name in nodes: + # FLUSH first so the log holding the most recent writes stops being the active + # one; PURGE never removes the active log. The + INTERVAL 1 SECOND covers the + # file just closed by FLUSH, whose mtime is essentially NOW(). + self.docker.exec_mysql( + name, + "FLUSH BINARY LOGS; PURGE BINARY LOGS BEFORE NOW() + INTERVAL 1 SECOND;", + password=self.root_password, + ) + result = self.docker.exec_mysql( + name, "SELECT @@GLOBAL.gtid_purged;", password=self.root_password, timeout=30 + ) + purged[name] = result.stdout.strip().replace("\n", "") + return purged + def _persist_gr_settings(self, nodes: list[str]) -> None: """Persist start_on_boot and the current group_seeds list on each given node.""" start_on_boot = "ON" if self.start_on_boot else "OFF" @@ -313,7 +669,7 @@ def scale_up(self, count: int = 1) -> list[str]: self._persist_gr_settings(self.containers) self.wait_all_online() if self.proxy: - self._refresh_proxy() + self.refresh_proxy() self.wait_proxy_ready() return added @@ -357,7 +713,7 @@ def scale_down(self, count: int = 1) -> list[str]: # pinning/polling doesn't run against the transient state right after removeInstance. self.wait_all_online() if self.proxy: - self._refresh_proxy() + self.refresh_proxy() self.wait_proxy_ready() return to_remove @@ -383,8 +739,18 @@ def ro_endpoint(self) -> tuple[str, int]: secondaries = self.secondaries() return ((secondaries[0] if secondaries else self.get_primary()), 3306) - def exec_sql(self, sql: str, database: str | None = None, check: bool = True): - """Run application SQL through the read/write endpoint (via the proxy when enabled, else direct to the primary).""" + def exec_sql( + self, + sql: str, + database: str | None = None, + check: bool = True, + timeout: float | None = None, + ): + """Run application SQL through the read/write endpoint (via the proxy when enabled, else direct to the primary). + + Pass a timeout when the cluster may have no writable primary: a write through a + proxy with no live backend otherwise blocks until the outer pytest timeout. + """ if self.proxy: host, port = self.rw_endpoint() return self.docker.exec_mysql( @@ -395,6 +761,7 @@ def exec_sql(self, sql: str, database: str | None = None, check: bool = True): host=host, port=port, check=check, + timeout=timeout, ) return self.docker.exec_mysql( self.get_primary(), @@ -402,6 +769,7 @@ def exec_sql(self, sql: str, database: str | None = None, check: bool = True): password=self.root_password, database=database, check=check, + timeout=timeout, ) def _start_router(self) -> None: @@ -418,12 +786,21 @@ def _start_router(self) -> None: # correctly; shlex.quote then protects the surrounding bash -c command (it chains # bootstrap && exec) from a password with spaces/$/quotes/etc. bootstrap_uri = shlex.quote(self._instance_uri(seed)) + config = "/tmp/mysqlrouter/mysqlrouter.conf" + # Bootstrap only when there is no config yet. The container runs with + # restart=on-failure, and by the time a restart happens the bootstrap seed may be + # unreachable (a partition test can have isolated it) — re-bootstrapping then fails, + # exits non-zero and leaves the router crash-looping with its ports closed. The + # config written on first start survives a restart and already lists every metadata + # server, so a restart can simply reuse it. bootstrap = ( + f"if [ ! -f {config} ]; then " f"mysqlrouter --bootstrap {bootstrap_uri} " "--directory /tmp/mysqlrouter " "--conf-set-option=DEFAULT.unknown_config_option=warning " - "--conf-bind-address=0.0.0.0 --force " - "&& exec mysqlrouter -c /tmp/mysqlrouter/mysqlrouter.conf" + "--conf-bind-address=0.0.0.0 --force || exit 1; " + "fi; " + f"exec mysqlrouter -c {config}" ) self.docker.create( image=self.router_image, @@ -543,12 +920,16 @@ def _start_proxy(self) -> None: elif self.proxy == "haproxy": self._start_haproxy() - def _refresh_proxy(self) -> None: - """Reconcile the proxy with the current membership after a scale operation. + def refresh_proxy(self) -> None: + """Reconcile the proxy with the current node set and their current addresses. + + MySQL Router auto-discovers members from the cluster metadata and resolves them by + name, so it needs nothing. HAProxy resolves its backend server list once, when the + config is parsed at start time (see _haproxy_config), so the container is recreated. - MySQL Router auto-discovers members from the cluster metadata, so it needs nothing. - HAProxy's backend server list is baked into the config at start time (see - _haproxy_config), so the container is recreated to pick up the new node set. + Needed after a scale operation (the node set changed) and after a partition heal: + reconnecting a container to the network gives it a *new* IP, which leaves HAProxy + pointing at an address nothing answers on. """ if self.proxy == "haproxy": self.log(f"refresh HAProxy {self.haproxy_name} for new membership") @@ -602,7 +983,8 @@ def wait_proxy_ready(self, timeout: int = 120) -> None: last = routed or (result.stderr or "").strip() time.sleep(2) raise RuntimeError( - f"{self.proxy} {self.proxy_name} not ready / not routing to primary in {timeout}s (last: {last!r})" + f"{self.proxy} {self.proxy_name} not ready / not routing to primary in {timeout}s " + f"(container: {self.docker.container_state(self.proxy_name)!r}, last: {last!r})" ) def _start_mysqld_node(self, index: int) -> str: @@ -623,6 +1005,11 @@ def _start_mysqld_node(self, index: int) -> str: ports=[f"{self.base_host_port + index}:3306"], command=self._mysqld_args(server_id=index, hostname=name), restart="always", + # Lets a test blackhole individual peers from inside the container (see + # sever_link), which is the only way to split the cluster into two halves that + # each stay internally connected: moving containers between networks changes + # their IPs, and XCOM does not follow a peer to a new address. + cap_add=["NET_ADMIN"], ) self.containers.append(name) self.node_index[name] = index diff --git a/test_scripts/ps/group-replication/test_equal_partition.py b/test_scripts/ps/group-replication/test_equal_partition.py new file mode 100644 index 0000000..a16bf24 --- /dev/null +++ b/test_scripts/ps/group-replication/test_equal_partition.py @@ -0,0 +1,197 @@ +"""Group Replication equal partition and split-brain prevention test. + +This test is the only one needing a 4-node cluster. The cluster is split down the middle +into two halves of two, each half intact internally but holding only 2 of 4 — short of the +3-of-4 majority. Neither side may accept a write and neither may elect a primary of its own, +because either would be a split brain. Group Replication cannot recover from an even split +on its own: an operator has to choose a half and force its membership. + +The split is done with reject routes inside the containers (sever_link), not by moving +containers between networks. A container that changes network changes IP, and XCOM does not +follow a peer to a new address, so moving a pair onto a network of their own makes them lose +*each other* — four one-node partitions, which is scenario 1's topology, not this one. +Blackholing the other half's addresses leaves every IP and process untouched, so each half +stays internally connected and only the link across the middle is gone. + +Unlike test_majority_loss.py this leaves group_replication_unreachable_majority_timeout at +its default of 0, so the blocked members stay in the group rather than self-ejecting: +group_replication_force_members has to be run on a member that is still in the group. Writes +therefore hang rather than being rejected, which is why the write probes are bounded by a +timeout — a blocked write and a refused one are both "no write happened". +""" + +import pytest + +PROBE_TABLE = "sbtest.split_probe" +# Attempted from each half while neither has quorum. None of them may become visible while +# the split is in effect. The two that *block* (the kept half still holds the primary role, +# so GR parks the write awaiting consensus) do go on to commit once that half is unblocked — +# see the comment on the recovery assertions. The one aimed at the cut-off half is refused +# outright, because nothing there is writable, so it can never appear at all. +BLOCKED_NOTES = ("blocked-kept", "blocked-proxy") +REFUSED_NOTE = "refused-moved" +ALL_NOTES = (*BLOCKED_NOTES, REFUSED_NOTE) + + +def _probe_write(gr_cluster, node, note): + """Attempt a write on one node, bounded so a blocked (rather than refused) write returns.""" + return gr_cluster.docker.exec_mysql( + node, + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('{note}');", + password=gr_cluster.root_password, + check=False, + timeout=15, + ) + + +def _describe(result): + """Say how a write probe failed — blocked until the timeout, or refused by the server.""" + return "blocked (timed out)" if result.returncode == 124 else result.stderr.strip() + + +def _count_notes(gr_cluster, node, notes): + """Count probe rows with any of the given notes as visible on one node.""" + quoted = ", ".join(f"'{note}'" for note in notes) + return gr_cluster.docker.exec_mysql( + node, + f"SELECT COUNT(*) FROM {PROBE_TABLE} WHERE note IN ({quoted});", + password=gr_cluster.root_password, + ).stdout.strip() + + +@pytest.mark.parametrize( + "gr_cluster", + [pytest.param(("router", 4), id="router"), pytest.param(("haproxy", 4), id="haproxy")], + indirect=True, +) +def test_equal_partition(gr_cluster, sysbench): + gr_cluster.verify() + assert gr_cluster.num_nodes == 4 + + # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. + host, port = gr_cluster.rw_endpoint() + sysbench.prepare(host=host, port=port) + + # A probe table for the write attempts below. It lives in the sbtest database so the + # verify_checksums("sbtest") calls already in this test cover the probe rows too. + # GR requires a primary key on every table. + gr_cluster.exec_sql( + f"CREATE TABLE IF NOT EXISTS {PROBE_TABLE} " + "(id INT AUTO_INCREMENT PRIMARY KEY, note VARCHAR(64));" + ) + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Split 2-2, keeping the current primary on the side we will later recover. + primary = gr_cluster.get_primary() + secondaries = gr_cluster.secondaries() + kept = [primary, secondaries[0]] + moved = secondaries[1:] + assert len(moved) == 2, f"expected a 2-2 split, got kept={kept} moved={moved}" + gr_cluster.sever_link(kept, moved) + + # The kept half is intact: it still sees its own pair, and the other two are gone. + kept_view = gr_cluster.wait_members_unreachable(moved, node=primary) + assert all(kept_view.get(n, ("", ""))[0] == "UNREACHABLE" for n in moved), ( + f"expected {moved} UNREACHABLE from {primary}, got {kept_view}" + ) + assert kept_view.get(kept[1], ("", ""))[0] == "ONLINE", ( + f"{kept[1]} should still be ONLINE alongside {primary}: {kept_view}" + ) + + # ...and so is the moved half. This is what makes it an *equal* partition rather than + # four one-node partitions, and it is the check that sever_link() worked at all: the + # two cut-off nodes must still see each other. + moved_view = gr_cluster.wait_members_unreachable(kept, node=moved[0]) + assert moved_view.get(moved[1], ("", ""))[0] == "ONLINE", ( + f"{moved[0]} and {moved[1]} lost contact — this is not a 2-2 split: {moved_view}" + ) + assert all(moved_view.get(n, ("", ""))[0] == "UNREACHABLE" for n in kept), ( + f"expected {kept} UNREACHABLE from {moved[0]}, got {moved_view}" + ) + + # The whole point of a network partition versus a stop: every process is untouched. + for node in gr_cluster.containers: + assert gr_cluster.node_alive(node), f"mysqld on {node} died during the partition" + + # No split brain: the moved half must not promote a primary of its own. It holds 2 of 4, + # so it has to stay read-only and leave the role where it was. + moved_primaries = [ + h for h, (state, role) in moved_view.items() if role == "PRIMARY" and state == "ONLINE" + ] + assert not moved_primaries, ( + f"the minority half elected its own primary {moved_primaries} — split brain: {moved_view}" + ) + for node in moved: + assert gr_cluster.super_read_only(node) == "1", ( + f"{node} is writable while its half has no quorum" + ) + + # Neither half accepts a write. On the kept half the primary blocks (it still holds the + # role but cannot reach a majority); on the moved half nothing is writable at all. + for node, note in ((primary, BLOCKED_NOTES[0]), (moved[0], REFUSED_NOTE)): + rejected = _probe_write(gr_cluster, node, note) + assert not rejected.ok, f"{node} accepted a write with no quorum: {rejected.stdout!r}" + gr_cluster.log(f"{node} did not accept the write: {_describe(rejected)}") + + # ...nor does the read/write endpoint. + via_proxy = gr_cluster.exec_sql( + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('{BLOCKED_NOTES[1]}');", + check=False, + timeout=30, + ) + assert not via_proxy.ok, ( + f"{gr_cluster.proxy} accepted a write with no quorum: {via_proxy.stdout!r}" + ) + gr_cluster.log(f"{gr_cluster.proxy} did not accept the write: {_describe(via_proxy)}") + + # And none of them became visible on either side of the split. This is the guarantee + # that matters: while no half has a majority, no write is accepted anywhere. + for node in (primary, moved[0]): + seen = _count_notes(gr_cluster, node, ALL_NOTES) + assert seen == "0", f"{node} accepted {seen} write(s) while its half had no quorum" + + # GR cannot resolve an even split by itself — an operator picks a half and forces its + # membership. Both kept nodes are alive and can see each other, so unlike the + # majority-loss case the forced list can name the whole surviving half rather than a + # single node. + survivors = gr_cluster.force_members(kept, node=primary) + online = {h for h, (state, _) in survivors.items() if state == "ONLINE"} + assert online == set(kept), f"forced membership did not settle on {kept}: {survivors}" + + # That half is a quorum of its own now, so writes work again. + gr_cluster.docker.exec_mysql( + gr_cluster.get_primary(), + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('recovered');", + password=gr_cluster.root_password, + ) + + # Bring the other half back. Restoring connectivity is not enough on its own: both nodes + # are still stuck in their stale blocked view, and the forced membership left them out + # of the group entirely, so they have to restart Group Replication. + gr_cluster.restore_link(kept, moved) + for node in moved: + gr_cluster.restart_group_replication(node) + gr_cluster.wait_all_online(timeout=300, node=primary) + + # No refresh_proxy() here, unlike the tests that reconnect containers to the network: + # severing a link leaves every address unchanged, so the proxy's backends are still valid. + gr_cluster.wait_proxy_ready(timeout=300) + + # The write the cut-off half refused outright can never exist — nothing there was ever + # writable. The two that merely *blocked* are a different matter: with + # unreachable_majority_timeout at its default the primary parks such a write awaiting + # consensus rather than failing it, so unblocking the kept half lets them commit, even + # though the client that issued them is long gone. That is not a split brain — no + # conflicting write was accepted on the other side — and the checksum comparison below + # is what proves the four nodes agree on whatever did commit. + for node in gr_cluster.active_nodes: + leaked = _count_notes(gr_cluster, node, (REFUSED_NOTE,)) + assert leaked == "0", f"a refused write leaked onto {node} ({leaked} rows)" + + gr_cluster.verify() + gr_cluster.verify_checksums("sbtest", timeout=180) + + # Load against the reunited cluster; data stays consistent across all four nodes. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=180) diff --git a/test_scripts/ps/group-replication/test_majority_loss.py b/test_scripts/ps/group-replication/test_majority_loss.py new file mode 100644 index 0000000..2a1de8b --- /dev/null +++ b/test_scripts/ps/group-replication/test_majority_loss.py @@ -0,0 +1,190 @@ +"""Group Replication majority loss and quorum failure tolerance test. + +Both secondaries of a 3-node cluster are network-partitioned at the same time with +`docker network disconnect` — their mysqld keeps running, they just lose connectivity. +That leaves every member alone in a one-node partition, so no side holds a majority and +the cluster must accept no writes at all: the surviving primary goes read-only rather than +risk diverging. Reconnecting the secondaries restores a 2-of-3 quorum between them, after +which the old primary rejoins. + +Unlike the other partition tests, which always leave a writable majority behind, this one +removes the majority itself. It asserts that writes are refused both directly on the +primary and through the proxy, and that neither of those writes exists anywhere afterwards. + +The test sets group_replication_unreachable_majority_timeout=30 on the primary only. At the +default of 0 the primary would block indefinitely instead of leaving the group, never +applying group_replication_exit_state_action and so never becoming super_read_only. It is +deliberately not set cluster-wide: the secondaries have to stay in the group while blocked, +otherwise nobody holds quorum after the heal and the group would need a manual rebuild. +""" + +import pytest + +PROBE_TABLE = "sbtest.quorum_probe" +# Written while the cluster has no quorum. Both must be refused, and neither may exist +# anywhere afterwards. +REJECTED_NOTES = ("rejected-direct", "rejected-proxy") + + +@pytest.mark.parametrize("gr_cluster", ["router", "haproxy"], indirect=True) +def test_majority_loss(gr_cluster, sysbench): + gr_cluster.verify() + + # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. + host, port = gr_cluster.rw_endpoint() + sysbench.prepare(host=host, port=port) + + # A probe table for the write attempts below. It lives in the sbtest database so the + # verify_checksums("sbtest") calls already in this test cover the probe rows too. + # GR requires a primary key on every table. + gr_cluster.exec_sql( + f"CREATE TABLE IF NOT EXISTS {PROBE_TABLE} " + "(id INT AUTO_INCREMENT PRIMARY KEY, note VARCHAR(64));" + ) + gr_cluster.verify_checksums("sbtest", timeout=120) + + primary = gr_cluster.get_primary() + secondaries = gr_cluster.secondaries() + exit_action = gr_cluster.docker.exec_mysql( + primary, "SELECT @@GLOBAL.group_replication_exit_state_action;", + password=gr_cluster.root_password, + ).stdout.strip() + gr_cluster.log(f"{primary} exit_state_action={exit_action}") + + # Make the primary give up once it finds itself in an unreachable minority — see the + # module docstring for why this is set here and not on the whole cluster. + gr_cluster.docker.exec_mysql( + primary, + "SET GLOBAL group_replication_unreachable_majority_timeout=30;", + password=gr_cluster.root_password, + ) + + # Cut off both secondaries at once. Every mysqld stays running. + gr_cluster.partition_group(secondaries) + + # Assert this first: it has to land inside the ~30s before the primary leaves the group + # and stops listing the others at all. Detection takes a few seconds, so there is room. + view = gr_cluster.wait_members_unreachable(secondaries, node=primary) + unreachable = [h for h, (state, _) in view.items() if state == "UNREACHABLE"] + assert sorted(unreachable) == sorted(secondaries), ( + f"expected {secondaries} UNREACHABLE from {primary}, got {view}" + ) + + # The whole point of a network partition versus a stop: the processes are untouched. + for node in secondaries: + assert gr_cluster.node_alive(node), f"mysqld on {node} died during the partition" + + # Neither isolated node can see anyone else either, so no side holds a majority. This is + # what separates majority loss from the single-secondary isolation tests. + for node in secondaries: + node_view = gr_cluster.wait_node_isolated(node) + still_online = {h for h, (state, _) in node_view.items() if state == "ONLINE"} - {node} + assert not still_online, ( + f"isolated node {node} still sees group members as ONLINE: {node_view}" + ) + + # Having lost majority, the surviving primary must protect the data by going read-only + # rather than accepting writes it can never replicate. + assert gr_cluster.wait_super_read_only(primary), ( + f"surviving primary {primary} is not super_read_only " + f"(super_read_only={gr_cluster.super_read_only(primary)!r}, " + f"exit_state_action={exit_action!r}, " + f"member_state={gr_cluster.local_member_state(primary)!r})" + ) + + # Writes must be refused directly on the primary (expected error 1290). Asserted on + # failure rather than the error code, matching the sibling partition tests. + rejected = gr_cluster.docker.exec_mysql( + primary, + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('{REJECTED_NOTES[0]}');", + password=gr_cluster.root_password, + check=False, + timeout=15, + ) + assert not rejected.ok, f"primary {primary} accepted a write with no quorum: {rejected.stdout!r}" + gr_cluster.log(f"{primary} rejected the direct write: {rejected.stderr.strip()!r}") + + # ...and through the proxy. Deliberately after the read-only check above: until the + # primary leaves the group GR *blocks* writes rather than rejecting them, so probing + # earlier would hang instead of failing. The two proxies refuse for different reasons + # (HAProxy forwards to the read-only primary; the router has no primary to route to), + # so the assertion is on failure, not on a particular error. + via_proxy = gr_cluster.exec_sql( + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('{REJECTED_NOTES[1]}');", + check=False, + timeout=30, + ) + assert not via_proxy.ok, ( + f"{gr_cluster.proxy} accepted a write with no quorum: {via_proxy.stdout!r}" + ) + gr_cluster.log(f"{gr_cluster.proxy} rejected the write: {via_proxy.stderr.strip()!r}") + + # Reconnect the secondaries. Restoring the network is not enough on its own: a member + # blocked in a minority stays stuck with the others UNREACHABLE rather than reforming, + # so the surviving membership has to be forced onto one of them. This is the documented + # recovery from majority loss, and it is the step the scenario doc assumes happens by + # itself — it does not. + gr_cluster.heal_group(secondaries) + + # Forced down to a single member, not to both secondaries: XCOM refuses a forced list + # containing anyone it currently suspects, and each blocked node still suspects the + # others even after the network is back ("Only alive members in the current + # configuration should be present in a forced configuration list"). + seed = secondaries[0] + survivors = gr_cluster.force_members([seed], node=seed) + online = {h for h, (state, _) in survivors.items() if state == "ONLINE"} + assert online == {seed}, f"forced membership did not settle on {seed}: {survivors}" + + # Everyone else rejoins the reformed group. The remaining secondary is still stuck in + # its stale blocked view, and the old primary left the group entirely when its + # unreachable-majority timeout fired; both need Group Replication restarted. + for node in (secondaries[1], primary): + gr_cluster.restart_group_replication(node) + gr_cluster.wait_all_online(timeout=240, node=seed) + + # The cluster is whole again with exactly one primary. Which node holds it is not + # asserted: the survivors elected one of themselves while the old primary was out. + new_primary = gr_cluster.get_primary() + members = gr_cluster.member_states(new_primary) + assert sorted(members) == sorted(gr_cluster.containers), f"membership incomplete: {members}" + assert all(state == "ONLINE" for state, _ in members.values()), f"not all ONLINE: {members}" + assert sorted(role for _, role in members.values()) == ["PRIMARY", "SECONDARY", "SECONDARY"], ( + f"expected exactly one PRIMARY, got {members}" + ) + + # PS 8.4 defaults group_replication_exit_state_action to OFFLINE_MODE, so leaving the + # group put the old primary into offline mode (blocking ordinary users, not just writes). + # Recovering has to clear that, or the node is ONLINE to the group but useless to clients. + offline_mode = gr_cluster.docker.exec_mysql( + primary, "SELECT @@GLOBAL.offline_mode;", password=gr_cluster.root_password + ).stdout.strip() + assert offline_mode == "0", ( + f"{primary} is still in offline mode after rejoining (offline_mode={offline_mode!r})" + ) + + # The write endpoint has to come back too. Reconnecting a container to the network + # gives it a new IP, and HAProxy resolved its backends once at start time — so after a + # heal it is pointing at addresses nothing answers on and has to be rebuilt. (Only this + # test notices: it is the first where a node that was reconnected goes on to become the + # primary.) The generous timeout is for the same reason as in + # test_primary_isolation_failover.py: MySQL Router can be slow to reopen its RW port. + gr_cluster.refresh_proxy() + gr_cluster.wait_proxy_ready(timeout=300) + + # No data was accepted anywhere while quorum was lost. + notes = ", ".join(f"'{note}'" for note in REJECTED_NOTES) + for node in gr_cluster.active_nodes: + leaked = gr_cluster.docker.exec_mysql( + node, + f"SELECT COUNT(*) FROM {PROBE_TABLE} WHERE note IN ({notes});", + password=gr_cluster.root_password, + ).stdout.strip() + assert leaked == "0", f"a rejected write leaked onto {node} ({leaked} rows)" + + gr_cluster.verify() + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Load against the recovered cluster; data stays consistent across all three nodes. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=120) diff --git a/test_scripts/ps/group-replication/test_primary_isolation_failover.py b/test_scripts/ps/group-replication/test_primary_isolation_failover.py new file mode 100644 index 0000000..756dba9 --- /dev/null +++ b/test_scripts/ps/group-replication/test_primary_isolation_failover.py @@ -0,0 +1,220 @@ +"""Group Replication primary isolation and automatic failover test. + +The PRIMARY of a 3-node cluster is network-partitioned with `docker network disconnect` — +its mysqld keeps running, it just loses connectivity. The two surviving secondaries hold +majority, so they must detect the loss, expel the old primary, elect a new one and become +writable on their own, with the proxy following the new write endpoint. Meanwhile the +isolated old primary must refuse writes, so no split-brain is possible. After healing it +rejoins as a secondary. + +Unlike test_primary_shutdown_failover.py, mysqld is never killed here; the group has to +detect an unreachable member rather than a departed one. + +The test sets group_replication_unreachable_majority_timeout=30 on the primary before +isolating it. At the default of 0 a minority-blocked member never leaves the group, so +group_replication_exit_state_action never fires and writes hang instead of being rejected; +with the timeout set, the old primary self-ejects and goes super_read_only as expected. +""" + +import time + +import pytest + +PROBE_TABLE = "sbtest.failover_probe" +# Written to the isolated old primary, which must reject it. Its absence everywhere +# afterwards is the split-brain check. +REJECTED_NOTE = "must-not-commit" + + +@pytest.mark.parametrize("gr_cluster", ["router", "haproxy"], indirect=True) +def test_primary_isolation_failover(gr_cluster, sysbench): + gr_cluster.verify() + + # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. + host, port = gr_cluster.rw_endpoint() + sysbench.prepare(host=host, port=port) + + # A probe table for the write attempts below. It lives in the sbtest database so the + # verify_checksums("sbtest") calls already in this test cover the probe rows too. + # GR requires a primary key on every table. + gr_cluster.exec_sql( + f"CREATE TABLE IF NOT EXISTS {PROBE_TABLE} " + "(id INT AUTO_INCREMENT PRIMARY KEY, note VARCHAR(64));" + ) + gr_cluster.verify_checksums("sbtest", timeout=120) + + old_primary = gr_cluster.get_primary() + exit_action = gr_cluster.docker.exec_mysql( + old_primary, "SELECT @@GLOBAL.group_replication_exit_state_action;", + password=gr_cluster.root_password, + ).stdout.strip() + gr_cluster.log(f"{old_primary} exit_state_action={exit_action}") + + # Make the primary give up once it finds itself in an unreachable minority. At the + # default of 0 it would block indefinitely instead of leaving the group, and never + # apply exit_state_action — see the module docstring. + gr_cluster.docker.exec_mysql( + old_primary, + "SET GLOBAL group_replication_unreachable_majority_timeout=30;", + password=gr_cluster.root_password, + ) + + # Sever the network only — mysqld on the primary keeps running, unlike stop_node(). + failover_start = time.monotonic() + gr_cluster.isolate_node(old_primary) + + # The two secondaries keep majority. Wait for the group to settle to exactly 2 ONLINE + # members before asking who the primary is: for the first few seconds the survivors + # still report the old primary as the ONLINE PRIMARY, so asking early returns it. + states = gr_cluster.wait_online_count(2) + online_hosts = [host for host, (state, _) in states.items() if state == "ONLINE"] + assert old_primary not in online_hosts, ( + f"isolated primary {old_primary} still ONLINE: {states}" + ) + + # A new primary must have been elected out of the surviving majority, with no operator + # action of any kind. + new_primary = gr_cluster.get_primary() + assert new_primary != old_primary, f"primary did not change after isolating {old_primary}" + assert new_primary in gr_cluster.active_nodes + + members = gr_cluster.member_states(new_primary) + online = {host: role for host, (state, role) in members.items() if state == "ONLINE"} + assert sorted(online.values()) == ["PRIMARY", "SECONDARY"], ( + f"expected one PRIMARY and one SECONDARY among the survivors, got {members}" + ) + assert online.get(new_primary) == "PRIMARY" + + # Writes resume on the new primary. Timed from the moment of isolation — the number in + # the log is the interesting part; the assertion is only a sanity ceiling. + gr_cluster.docker.exec_mysql( + new_primary, + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('direct-after-failover');", + password=gr_cluster.root_password, + ) + failover_seconds = time.monotonic() - failover_start + gr_cluster.log( + f"failover window: {failover_seconds:.1f}s from isolating {old_primary} " + f"to first write on {new_primary}" + ) + assert failover_seconds < 120, f"failover took {failover_seconds:.1f}s" + + # The whole point of a network partition versus a stop: the process is untouched. + assert gr_cluster.node_alive(old_primary), f"mysqld on {old_primary} died during the partition" + + # From the minority side the old primary must see that it has lost the group. + old_view = gr_cluster.wait_node_isolated(old_primary) + still_online = {h for h, (state, _) in old_view.items() if state == "ONLINE"} - {old_primary} + assert not still_online, ( + f"isolated primary {old_primary} still sees group members as ONLINE: {old_view}" + ) + + # Having self-ejected, it must protect the data by going read-only... + assert gr_cluster.wait_super_read_only(old_primary), ( + f"isolated primary {old_primary} is not super_read_only " + f"(super_read_only={gr_cluster.super_read_only(old_primary)!r}, " + f"exit_state_action={exit_action!r}, " + f"member_state={gr_cluster.local_member_state(old_primary)!r})" + ) + + # ...and reject writes. Asserted on failure rather than on error 1290 specifically, so + # the check holds whether GR rejects the statement or blocks on it. + rejected = gr_cluster.docker.exec_mysql( + old_primary, + f"INSERT INTO {PROBE_TABLE} (note) VALUES ('{REJECTED_NOTE}');", + password=gr_cluster.root_password, + check=False, + timeout=15, + ) + assert not rejected.ok, ( + f"isolated primary {old_primary} accepted a write: {rejected.stdout!r}" + ) + gr_cluster.log(f"{old_primary} rejected the write: {rejected.stderr.strip()!r}") + + # The proxy must follow the election, not just direct connections: exec_sql() goes + # through the read/write endpoint, so a successful insert here is the assertion. + # Normally near-instant (~0.2s measured), but MySQL Router intermittently keeps the + # RW port closed for minutes after the primary is *blackholed* rather than cleanly + # stopped — the router process stays up (restarts=0), it just has no valid destination. + # Not reproducible with docker stop, i.e. only on this partition path. Hence the + # generous timeout; wait_proxy_ready() reports the proxy container's state on failure. + gr_cluster.wait_proxy_ready(timeout=300) + gr_cluster.exec_sql(f"INSERT INTO {PROBE_TABLE} (note) VALUES ('proxy-after-failover');") + + # Load against the new primary; the two survivors stay consistent. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Heal. The rejoin may take either the IST or the clone path depending on how much the + # old primary missed, so the budget covers the slower one and neither is asserted. + auto_rejoined = gr_cluster.heal_node(old_primary, timeout=240) + gr_cluster.log( + f"{old_primary} rejoined " + f"{'automatically' if auto_rejoined else 'via explicit START GROUP_REPLICATION'}" + ) + + # It comes back as a secondary — the election is not undone by its return. + members = gr_cluster.member_states(new_primary) + assert members.get(old_primary) == ("ONLINE", "SECONDARY"), ( + f"{old_primary} did not rejoin as a secondary: {members}" + ) + assert gr_cluster.get_primary() == new_primary, ( + f"primary moved back off {new_primary} after {old_primary} rejoined" + ) + # PS 8.4 defaults group_replication_exit_state_action to OFFLINE_MODE, so self-ejecting + # put the old primary into offline mode (blocking ordinary users, not just writes). + # Recovering has to clear that, or the node is ONLINE to the group but useless to clients. + offline_mode = gr_cluster.docker.exec_mysql( + old_primary, "SELECT @@GLOBAL.offline_mode;", password=gr_cluster.root_password + ).stdout.strip() + assert offline_mode == "0", ( + f"{old_primary} is still in offline mode after rejoining (offline_mode={offline_mode!r})" + ) + + # No split-brain: the write the isolated old primary rejected must exist nowhere. + for node in gr_cluster.active_nodes: + leaked = gr_cluster.docker.exec_mysql( + node, + f"SELECT COUNT(*) FROM {PROBE_TABLE} WHERE note='{REJECTED_NOTE}';", + password=gr_cluster.root_password, + ).stdout.strip() + assert leaked == "0", f"rejected write leaked onto {node} ({leaked} rows)" + + # The old primary came back on a *new* IP — reconnecting a container to the network + # reassigns one — and HAProxy resolved its backends once, at start time. Nothing above + # notices: the write backend is pinned to the new primary, whose address never changed. + # The read backend is what breaks, so rebuild the proxy now that the cluster is whole. + gr_cluster.refresh_proxy() + gr_cluster.wait_proxy_ready(timeout=300) + + # ...and prove it: the reconnected node has to be serving reads again. A stale backend + # shows up exactly here — HAProxy health-checks the dead address, takes that node out of + # rotation, and it never answers on the read endpoint no matter how often we ask. + ro_host, ro_port = gr_cluster.ro_endpoint() + seen: set[str] = set() + deadline = time.monotonic() + 60 + while old_primary not in seen and time.monotonic() < deadline: + probe = gr_cluster.docker.exec_mysql( + new_primary, + "SELECT @@hostname;", + password=gr_cluster.root_password, + host=ro_host, + port=ro_port, + check=False, + timeout=15, + ) + if probe.ok: + seen.add(probe.stdout.strip()) + assert old_primary in seen, ( + f"{old_primary} is not serving reads after rejoining — the {gr_cluster.proxy} read " + f"endpoint only ever answered from {sorted(seen)}" + ) + + gr_cluster.verify() + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Load against the whole cluster again; data stays consistent across all three nodes. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=120) diff --git a/test_scripts/ps/group-replication/test_failover.py b/test_scripts/ps/group-replication/test_primary_shutdown_failover.py similarity index 83% rename from test_scripts/ps/group-replication/test_failover.py rename to test_scripts/ps/group-replication/test_primary_shutdown_failover.py index 19e6690..7a19ad3 100644 --- a/test_scripts/ps/group-replication/test_failover.py +++ b/test_scripts/ps/group-replication/test_primary_shutdown_failover.py @@ -1,16 +1,19 @@ -"""Group Replication primary failover and recovery test. +"""Group Replication primary shutdown failover and recovery test. -Loads data via sysbench, stops the current primary to force the election of a new one, -then resumes writes after the failover. Brings the stopped node back, confirms it +Loads data via sysbench, stops the current primary's mysqld to force the election of a +new one, then resumes writes after the failover. Brings the stopped node back, confirms it auto-rejoins and the cluster is whole again, verifying data stays consistent across all online nodes (matching checksums) at every stage. + +Killing the process makes the group lose the member immediately. For the variant where +mysqld stays alive and only its network is severed, see test_primary_isolation_failover.py. """ import pytest @pytest.mark.parametrize("gr_cluster", ["router", "haproxy"], indirect=True) -def test_primary_failover_and_recovery(gr_cluster, sysbench): +def test_primary_shutdown_failover_and_recovery(gr_cluster, sysbench): gr_cluster.verify() # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. diff --git a/test_scripts/ps/group-replication/test_secondary_isolation_ist.py b/test_scripts/ps/group-replication/test_secondary_isolation_ist.py new file mode 100644 index 0000000..11cf519 --- /dev/null +++ b/test_scripts/ps/group-replication/test_secondary_isolation_ist.py @@ -0,0 +1,97 @@ +"""Group Replication minority node isolation with IST recovery test. + +A single secondary of a 3-node cluster is network-partitioned with `docker network disconnect` +— its mysqld keeps running, it just loses connectivity — while the primary keeps majority and +accepts writes. The node is healed before the donor can purge its binary logs, so Group +Replication catches it up with an Incremental State Transfer rather than a full clone. +Confirms mysqld stayed alive throughout the partition, that no new clone was taken, that the +node actually applied the transactions it missed, and that data is consistent across all three +nodes afterwards. +""" + +import pytest + + +@pytest.mark.parametrize("gr_cluster", ["router", "haproxy"], indirect=True) +def test_secondary_isolation_ist_recovery(gr_cluster, sysbench): + gr_cluster.verify() + + # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. + host, port = gr_cluster.rw_endpoint() + sysbench.prepare(host=host, port=port) + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Pick a secondary to partition and record its clone history before anything happens, + # so the post-rejoin comparison can tell an IST from a fresh clone. Not asserted to be + # empty: create() adds every secondary with recoveryMethod:'clone', so each already + # carries a completed clone row from cluster bootstrap. + target = gr_cluster.secondaries()[0] + clone_before = gr_cluster.clone_status(target) + + # Sever the network only — mysqld on the target keeps running, unlike stop_node(). + gr_cluster.isolate_node(target) + + # The two survivors keep majority; confirm the group settles to exactly 2 ONLINE + # members (failure detection and expulsion take a few seconds) without the target. + states = gr_cluster.wait_online_count(2) + online_hosts = [host for host, (state, _) in states.items() if state == "ONLINE"] + assert target not in online_hosts, f"isolated node {target} still ONLINE: {states}" + + # The whole point of a network partition versus a stop: the process is untouched. + assert gr_cluster.node_alive(target), f"mysqld on {target} died during the partition" + + # From the minority side the target must see that it has lost the group. + target_view = gr_cluster.wait_node_isolated(target) + still_online = {h for h, (state, _) in target_view.items() if state == "ONLINE"} - {target} + assert not still_online, ( + f"isolated node {target} still sees group members as ONLINE: {target_view}" + ) + + # Accumulate writes the target will have to catch up on. The primary never lost + # majority, so the read/write endpoint does not move during the partition window. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=30) + + # Still alive right before the heal — the partition never touched the process. + assert gr_cluster.node_alive(target), f"mysqld on {target} died during the partition" + + # The GTID set the target has to catch up on. + missed_gtids = gr_cluster.gtid_executed(gr_cluster.get_primary()) + assert not gr_cluster.gtid_subset(target, missed_gtids), ( + f"{target} was not actually behind at the end of the partition window " + "— the catch-up check below would prove nothing" + ) + + # Heal while the donor still has the binary logs covering the partition window, which + # is what keeps the IST path available (binlog_expire_logs_seconds defaults to 30 days, + # so a ~30s window can never outlive them). heal_node() reports whether GR rejoined the + # member on its own or needed an explicit START GROUP_REPLICATION. + auto_rejoined = gr_cluster.heal_node(target, timeout=120) + gr_cluster.log( + f"{target} rejoined {'automatically' if auto_rejoined else 'via explicit START GROUP_REPLICATION'}" + ) + + # IST, not SST: recovery must not have added a clone to the target's history. + clone_after = gr_cluster.clone_status(target) + assert clone_after == clone_before, ( + f"a new clone was taken on {target} (SST instead of IST):\n" + f"before: {clone_before!r}\nafter: {clone_after!r}" + ) + + # ...and the node really did replay everything it missed while partitioned. + # Deliberately not the spec's COUNT_TRANSACTIONS_REMOTE_APPLIED: that counter only + # covers transactions received from the group once a member is already ONLINE, so it + # reads 0 after a recovery-channel catch-up. The GTID set is the direct evidence. + assert gr_cluster.gtid_subset(target, missed_gtids), ( + f"{target} is still missing transactions from the partition window\n" + f"expected superset of: {missed_gtids!r}\n" + f"has: {gr_cluster.gtid_executed(target)!r}" + ) + + gr_cluster.verify() + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Load against the whole cluster again; data stays consistent across all three nodes. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=120) diff --git a/test_scripts/ps/group-replication/test_secondary_isolation_sst.py b/test_scripts/ps/group-replication/test_secondary_isolation_sst.py new file mode 100644 index 0000000..d4970a6 --- /dev/null +++ b/test_scripts/ps/group-replication/test_secondary_isolation_sst.py @@ -0,0 +1,110 @@ +"""Group Replication minority node isolation with SST (clone) recovery test. + +This is the mirror image of the IST test. A single secondary of a 3-node cluster is +network-partitioned with `docker network disconnect` — its mysqld keeps running, it just +loses connectivity — while the primary keeps majority and accepts writes. Before healing, +the binary logs are purged on every surviving node so no donor can serve an Incremental +State Transfer, forcing Group Replication to fall back to a full clone. Confirms mysqld +stayed alive throughout the partition, that a new clone really did run and completed +without error, and that data is consistent across all three nodes afterwards. + +Percona Server 8.4 uses the clone plugin for distributed recovery, so this exercises +clone-based SST rather than the XtraBackup SST path. +""" + +import pytest + + +@pytest.mark.parametrize("gr_cluster", ["router", "haproxy"], indirect=True) +def test_secondary_isolation_sst_recovery(gr_cluster, sysbench): + gr_cluster.verify() + + # Initial data load via sysbench (4 tables x 10000 rows) through the read/write endpoint. + host, port = gr_cluster.rw_endpoint() + sysbench.prepare(host=host, port=port) + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Pick a secondary to partition and record its clone history before anything happens. + # create() adds every secondary with recoveryMethod:'clone', so this is already + # populated — "clone_status has rows" would pass even for an IST, which is why the + # assertion below compares against this snapshot rather than just checking for a row. + target = gr_cluster.secondaries()[0] + clone_before = gr_cluster.clone_status(target) + + # Sever the network only — mysqld on the target keeps running, unlike stop_node(). + gr_cluster.isolate_node(target) + + # The two survivors keep majority; confirm the group settles to exactly 2 ONLINE + # members (failure detection and expulsion take a few seconds) without the target. + states = gr_cluster.wait_online_count(2) + online_hosts = [host for host, (state, _) in states.items() if state == "ONLINE"] + assert target not in online_hosts, f"isolated node {target} still ONLINE: {states}" + + # The whole point of a network partition versus a stop: the process is untouched. + assert gr_cluster.node_alive(target), f"mysqld on {target} died during the partition" + + # From the minority side the target must see that it has lost the group. + target_view = gr_cluster.wait_node_isolated(target) + still_online = {h for h, (state, _) in target_view.items() if state == "ONLINE"} - {target} + assert not still_online, ( + f"isolated node {target} still sees group members as ONLINE: {target_view}" + ) + + # Accumulate the writes the target misses. The primary never lost majority, so the + # read/write endpoint does not move during the partition window. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=30) + + # Still alive right before the heal — the partition never touched the process. + assert gr_cluster.node_alive(target), f"mysqld on {target} died during the partition" + + # The GTID set the target has to recover. Asserting it is genuinely behind first keeps + # the post-recovery catch-up check from passing vacuously. + missed_gtids = gr_cluster.gtid_executed(gr_cluster.get_primary()) + assert not gr_cluster.gtid_subset(target, missed_gtids), ( + f"{target} was not actually behind at the end of the partition window " + "— the recovery checks below would prove nothing" + ) + + # Close off the IST path. This has to happen on *every* survivor, not just the primary + # as the scenario doc suggests: GR picks its recovery donor from any ONLINE member, and + # the other secondary carries full binary logs too (--log-replica-updates=ON), so + # leaving it untouched would let it serve an IST and silently turn this into the IST + # test. A non-empty gtid_purged is the proof that something was actually purged. + purged = gr_cluster.purge_binary_logs() + empty = [node for node, gtids in purged.items() if not gtids] + assert not empty, f"binary logs were not purged on {empty} (gtid_purged still empty): {purged}" + + # With the missing transactions gone from every donor, GR has no choice but to clone. + # A full clone copies the dataset and restarts mysqld on the recipient, so it gets a + # longer budget than the IST path. + auto_rejoined = gr_cluster.heal_node(target, timeout=240) + gr_cluster.log( + f"{target} rejoined {'automatically' if auto_rejoined else 'via explicit START GROUP_REPLICATION'}" + ) + + # SST, not IST: recovery must have run a new clone, and it must have finished cleanly. + clone_after = gr_cluster.clone_status(target) + assert clone_after != clone_before, ( + f"no new clone was taken on {target} (IST instead of SST):\n" + f"before: {clone_before!r}\nafter: {clone_after!r}" + ) + assert clone_after.get("STATE") == "Completed", ( + f"clone on {target} did not complete: {clone_after}" + ) + assert clone_after.get("ERROR_NO") == "0", f"clone on {target} reported an error: {clone_after}" + + # ...and the cloned node really did come back with everything it missed. + assert gr_cluster.gtid_subset(target, missed_gtids), ( + f"{target} is still missing transactions from the partition window\n" + f"expected superset of: {missed_gtids!r}\n" + f"has: {gr_cluster.gtid_executed(target)!r}" + ) + + gr_cluster.verify() + gr_cluster.verify_checksums("sbtest", timeout=120) + + # Load against the whole cluster again; data stays consistent across all three nodes. + host, port = gr_cluster.rw_endpoint() + sysbench.run(host=host, port=port, time=20) + gr_cluster.verify_checksums("sbtest", timeout=120)