diff --git a/Cargo.lock b/Cargo.lock index af61b754f35..169259b7f66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,6 +2526,7 @@ dependencies = [ "serde_json", "strum 0.25.0", "tempfile", + "test-strategy 0.4.0", "testdir", "testresult", "thiserror", @@ -2773,7 +2774,7 @@ dependencies = [ "serde", "strum 0.25.0", "tempfile", - "test-strategy", + "test-strategy 0.3.1", "thiserror", "tokio", "tokio-stream", @@ -3052,7 +3053,7 @@ dependencies = [ "strum 0.26.3", "syncify", "tempfile", - "test-strategy", + "test-strategy 0.3.1", "thiserror", "tokio", "tokio-stream", @@ -5434,7 +5435,19 @@ checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" dependencies = [ "proc-macro2", "quote", - "structmeta-derive", + "structmeta-derive 0.2.0", + "syn 2.0.72", +] + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive 0.3.0", "syn 2.0.72", ] @@ -5449,6 +5462,17 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "strum" version = "0.25.0" @@ -5682,7 +5706,19 @@ checksum = "b8361c808554228ad09bfed70f5c823caf8a3450b6881cc3a38eb57e8c08c1d9" dependencies = [ "proc-macro2", "quote", - "structmeta", + "structmeta 0.2.0", + "syn 2.0.72", +] + +[[package]] +name = "test-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf41af45e3f54cc184831d629d41d5b2bda8297e29c81add7ae4f362ed5e01b" +dependencies = [ + "proc-macro2", + "quote", + "structmeta 0.3.0", "syn 2.0.72", ] diff --git a/iroh-willow/src/engine/actor.rs b/iroh-willow/src/engine/actor.rs index 2df1df88c33..f72862ca40b 100644 --- a/iroh-willow/src/engine/actor.rs +++ b/iroh-willow/src/engine/actor.rs @@ -265,9 +265,8 @@ impl Drop for ActorHandle { // shutdown let shutdown = move || { - if let Err(err) = inbox_tx.blocking_send(Input::Shutdown { reply: None }) { - warn!(?err, "Failed to send shutdown"); - } else if let Err(err) = handle.join() { + inbox_tx.blocking_send(Input::Shutdown { reply: None }).ok(); + if let Err(err) = handle.join() { warn!(?err, "Failed to join sync actor"); } }; diff --git a/iroh-willow/src/engine/peer_manager.rs b/iroh-willow/src/engine/peer_manager.rs index f23efc0e1db..8b43bd45cd8 100644 --- a/iroh-willow/src/engine/peer_manager.rs +++ b/iroh-willow/src/engine/peer_manager.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, future::Future, sync::Arc, time::Duration}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use futures_buffered::join_all; use futures_lite::{future::Boxed, StreamExt}; @@ -22,7 +22,7 @@ use crate::{ interest::Interests, net::{ establish, prepare_channels, terminate_gracefully, ChannelStreams, ConnHandle, ALPN, - ERROR_CODE_DUPLICATE_CONN, ERROR_CODE_FAIL, ERROR_CODE_SHUTDOWN, + ERROR_CODE_DUPLICATE_CONN, ERROR_CODE_SHUTDOWN, }, proto::wgps::AccessChallenge, session::{ @@ -118,7 +118,7 @@ pub(super) struct PeerManager { session_events_rx: StreamMap>, peers: HashMap, accept_handlers: AcceptHandlers, - conn_tasks: JoinSet<(Role, NodeId, Result)>, + conn_tasks: JoinSet<(NodeId, ConnStep)>, shutting_down: bool, } @@ -176,7 +176,7 @@ impl PeerManager { match res { Err(err) if err.is_cancelled() => continue, Err(err) => Err(err).context("conn task panicked")?, - Ok((our_role, peer, out)) => self.handle_conn_output(our_role, peer, out).await?, + Ok((peer, out)) => self.handle_conn_output(peer, out).await?, } if self.shutting_down && self.conn_tasks.is_empty() { debug!("all connections gracefully terminated"); @@ -219,27 +219,26 @@ impl PeerManager { let peer_info = self .peers .entry(peer) - .or_insert_with(|| PeerInfo::new(Role::Betty, peer)); + .or_insert_with(|| PeerInfo::new(peer)); - debug!(peer = %peer.fmt_short(), our_state=%peer_info.state, "incoming connection"); + debug!(peer = %peer.fmt_short(), our_state=%peer_info.conn_state, "incoming connection"); - let accept_conn = match peer_info.state { - PeerState::None => true, - PeerState::Pending { - ref mut cancel_dial, - .. - } => match peer_info.our_role { - Role::Betty => { + let accept_conn = match peer_info.conn_state { + ConnState::None => true, + ConnState::Establishing { + ref mut our_dial, .. + } => match our_dial { + // No dial but already establishing a previous incoming connection + None => { debug!("ignore incoming connection (already accepting)"); conn.close(ERROR_CODE_DUPLICATE_CONN, b"duplicate-already-accepting"); false } - Role::Alfie => { + // We are dialing also: abort one of the conns + Some(cancel_dial) => { if peer > self.endpoint.node_id() { debug!("incoming connection for a peer we are dialing and their connection wins, abort dial"); - if let Some(cancel_dial) = cancel_dial.take() { - cancel_dial.cancel(); - } + cancel_dial.cancel(); true } else { debug!("ignore incoming connection (already dialing and ours wins)"); @@ -248,45 +247,34 @@ impl PeerManager { } } }, - PeerState::Active { .. } => { + ConnState::Active { .. } => { debug!("ignore incoming connection (already active)"); conn.close(ERROR_CODE_DUPLICATE_CONN, b"duplicate-already-active"); false } - PeerState::Closing { .. } => true, + ConnState::Terminating { .. } => true, }; if accept_conn { debug!(peer=%peer.fmt_short(), "accept connection"); - // Take any pending intents from the previous state and merge with the new betty intent. - let mut intents = match peer_info.state { - PeerState::Pending { - ref mut intents, .. - } - | PeerState::Closing { - ref mut intents, .. - } => std::mem::take(intents), - _ => vec![], - }; - intents.push(intent); - peer_info.state = PeerState::Pending { - intents, - cancel_dial: None, - }; - peer_info.our_role = Role::Betty; + peer_info.push_intent(intent).await; // Start connection establish task. let our_nonce = AccessChallenge::generate(); let fut = async move { - let (initial_transmission, channel_streams) = - establish(&conn, Role::Betty, our_nonce).await?; - Ok(ConnStep::Ready { - conn, - initial_transmission, + let res = establish(&conn, Role::Betty, our_nonce).await; + let res = res.map(|(initial_transmission, channel_streams)| Established { channel_streams, - }) + initial_transmission, + conn, + our_role: Role::Betty, + }); + ConnStep::Established(res) }; let abort_handle = spawn_conn_task(&mut self.conn_tasks, peer_info, fut); - peer_info.abort_handle = Some(abort_handle); + peer_info.conn_state = ConnState::Establishing { + our_dial: None, + abort_handle, + }; } } @@ -294,72 +282,55 @@ impl PeerManager { let peer_info = self .peers .entry(peer) - .or_insert_with(|| PeerInfo::new(Role::Alfie, peer)); - - debug!(peer=%peer.fmt_short(), state=%peer_info.state, "submit intent"); - - match peer_info.state { - PeerState::None => { - let our_nonce = AccessChallenge::generate(); - let endpoint = self.endpoint.clone(); - let cancel_dial = CancellationToken::new(); - let cancel_dial2 = cancel_dial.clone(); - // Future that dials and establishes the connection. Can be cancelled for simultaneous connection. - let fut = async move { - debug!("connecting"); - let conn = tokio::select! { - res = endpoint.connect_by_node_id(peer, ALPN) => res, - _ = cancel_dial.cancelled() => { - debug!("dial cancelled during dial"); - return Err(ConnectionError::LocallyClosed.into()); - } - }?; - let (initial_transmission, channel_streams) = tokio::select! { - res = establish(&conn, Role::Alfie, our_nonce) => res?, - _ = cancel_dial.cancelled() => { - debug!("dial cancelled during establish"); - conn.close(ERROR_CODE_DUPLICATE_CONN, b"duplicate-your-dial-wins"); - return Err(ConnectionError::LocallyClosed.into()); - }, - }; - Ok(ConnStep::Ready { - conn, - initial_transmission, - channel_streams, - }) - }; - let abort_handle = spawn_conn_task(&mut self.conn_tasks, peer_info, fut); - peer_info.abort_handle = Some(abort_handle); - peer_info.state = PeerState::Pending { - intents: vec![intent], - cancel_dial: Some(cancel_dial2), - }; - } - PeerState::Pending { - ref mut intents, .. - } => { - intents.push(intent); - } - PeerState::Active { - ref update_tx, - ref mut intents_after_close, - .. - } => { - if let Err(err) = update_tx.send(SessionUpdate::SubmitIntent(intent)).await { - debug!("failed to submit intent into active session, queue in peer state"); - if let SessionUpdate::SubmitIntent(intent) = err.0 { - intents_after_close.push(intent); + .or_insert_with(|| PeerInfo::new(peer)); + + debug!(peer=%peer.fmt_short(), state=%peer_info.conn_state, "submit intent"); + if !peer_info.push_intent(intent).await { + self.connect_if_inactive(peer); + } + } + + fn connect_if_inactive(&mut self, peer: NodeId) { + let peer_info = self + .peers + .entry(peer) + .or_insert_with(|| PeerInfo::new(peer)); + if matches!(peer_info.conn_state, ConnState::None) { + let our_nonce = AccessChallenge::generate(); + let endpoint = self.endpoint.clone(); + let cancel_dial = CancellationToken::new(); + let cancel_dial2 = cancel_dial.clone(); + // Future that dials and establishes the connection. Can be cancelled for simultaneous connection. + let fut = async move { + debug!("connecting"); + let conn = tokio::select! { + res = endpoint.connect_by_node_id(peer, ALPN) => res, + _ = cancel_dial.cancelled() => { + debug!("dial cancelled during dial"); + return Err(ConnectionError::LocallyClosed.into()); } - } else { - trace!("intent sent to session"); - } - } - PeerState::Closing { - intents: ref mut new_intents, - .. - } => { - new_intents.push(intent); + }?; + let (initial_transmission, channel_streams) = tokio::select! { + res = establish(&conn, Role::Alfie, our_nonce) => res?, + _ = cancel_dial.cancelled() => { + debug!("dial cancelled during establish"); + conn.close(ERROR_CODE_DUPLICATE_CONN, b"duplicate-your-dial-wins"); + return Err(ConnectionError::LocallyClosed.into()); + }, + }; + Ok(Established { + conn, + initial_transmission, + channel_streams, + our_role: Role::Alfie, + }) } + .map(ConnStep::Established); + let abort_handle = spawn_conn_task(&mut self.conn_tasks, peer_info, fut); + peer_info.conn_state = ConnState::Establishing { + our_dial: Some(cancel_dial2), + abort_handle, + }; } } @@ -384,38 +355,28 @@ impl PeerManager { return; }; - let PeerState::Active { - ref mut intents_after_close, - .. - } = peer_info.state - else { - warn!("got session complete event for peer not in active state"); - return; - }; - remaining_intents.append(intents_after_close); + peer_info.pending_intents.append(&mut remaining_intents); + peer_info.session_state = SessionState::None; - peer_info.state = PeerState::Closing { - intents: remaining_intents, - }; + if peer_info.conn_state.is_none() && peer_info.pending_intents.is_empty() { + self.peers.remove(&peer); + } else if peer_info.conn_state.is_none() { + self.connect_if_inactive(peer); + } trace!("entering closing state"); } } } #[instrument("conn", skip_all, fields(peer=%peer.fmt_short()))] - async fn handle_conn_output( - &mut self, - our_role: Role, - peer: NodeId, - out: Result, - ) -> Result<()> { + async fn handle_conn_output(&mut self, peer: NodeId, out: ConnStep) -> Result<()> { let peer_info = self .peers .get_mut(&peer) .context("got conn task output for unknown peer")?; match out { - Err(err) => { - trace!(?our_role, current_state=%peer_info.state, "conn task failed: {err:#?}"); + ConnStep::Established(Err(err)) => { + debug!(current_state=%peer_info.conn_state, "conn task failed while establishing: {err:#?}"); match err.downcast_ref() { Some(ConnectionError::LocallyClosed) => { // We cancelled the connection, nothing to do. @@ -425,76 +386,51 @@ impl PeerManager { if reason.error_code == ERROR_CODE_DUPLICATE_CONN => { debug!( - "connection was cancelled by the remote: simultaneous connection and their's wins" - ); - if our_role != peer_info.our_role { - // TODO: setup a timeout to kill intents if the other conn doesn't make it. - debug!("we are still waiting for their connection to arrive"); + "connection was cancelled by the remote: simultaneous connection and their's wins" + ); + if matches!( + &peer_info.conn_state, + ConnState::Establishing { + our_dial: Some(_), + .. + }, + ) { + peer_info.conn_state = ConnState::None; } + // if our_role != peer_info.our_role { + // // TODO: setup a timeout to kill intents if the other conn doesn't make it. + // debug!("we are still waiting for their connection to arrive"); + // } } _ => { - let peer = self.peers.remove(&peer).expect("just checked"); - match peer.state { - PeerState::Pending { intents, .. } => { - warn!(?err, "connection failed while pending"); - // If we were still in pending state, terminate all pending intents. - let err = Arc::new(Error::Net(err)); - join_all( - intents - .into_iter() - .map(|intent| intent.send_abort(err.clone())), - ) - .await; - } - PeerState::Closing { intents } => { - debug!(?err, "connection failed to close gracefully"); - // If we were are in closing state, we still forward the connection error to the intents. - // This would be the place where we'd implement retries: instead of aborting the intents, resubmit them. - // Right now, we only resubmit intents that were submitted while terminating a session, and only if the session closed gracefully. - let err = Arc::new(Error::Net(err)); - join_all( - intents - .into_iter() - .map(|intent| intent.send_abort(err.clone())), - ) - .await; + peer_info.conn_state = ConnState::None; + match &peer_info.session_state { + SessionState::None => { + peer_info + .abort_pending_intents(err.context("failed while establishing")) + .await; + self.peers.remove(&peer); } - PeerState::Active { update_tx, .. } => { - warn!(?err, "connection failed while active"); - update_tx - .send(SessionUpdate::Abort(Error::ConnectionClosed(err))) - .await - .ok(); - } - PeerState::None => { - warn!(?err, "connection failed while peer is in None state"); + SessionState::Active { .. } => { + // An establishing connection failed while an old session was still not terminated. + // We log the error and keep waiting for the session to terminate. This does not happen usually but can due to timings. + warn!("establish failed while session still not closed"); } } } } } - Ok(ConnStep::Ready { + ConnStep::Established(Ok(Established { + our_role, conn, initial_transmission, channel_streams, - }) => { - let PeerState::Pending { - ref mut intents, .. - } = &mut peer_info.state - else { - debug!( - ?our_role, - "got connection ready for peer in non-pending state" - ); - conn.close(ERROR_CODE_FAIL, b"invalid-state"); - drop(conn); - // TODO: unreachable? - return Err(anyhow!( - "got connection ready for peer in non-pending state" - )); + })) => { + let SessionState::None = peer_info.session_state else { + unreachable!("session must be inactive when connection establishes"); }; - let intents = std::mem::take(intents); + let intents = std::mem::take(&mut peer_info.pending_intents); if self.shutting_down { debug!("connection became ready while shutting down, abort"); @@ -511,7 +447,6 @@ impl PeerManager { return Ok(()); } - // TODO: Here we should check again that we are not establishing a duplicate connection. debug!(?our_role, "connection ready: init session"); let (channels, fut) = prepare_channels(channel_streams)?; let conn_handle = ConnHandle { @@ -520,10 +455,9 @@ impl PeerManager { our_role, peer, }; - peer_info.our_role = our_role; let session_handle = self.actor.init_session(conn_handle, intents).await?; - let fut = fut.map_ok(move |()| ConnStep::Done { conn }); + let fut = fut.map_ok(|()| conn).map(ConnStep::Done); let abort_handle = spawn_conn_task(&mut self.conn_tasks, peer_info, fut); let SessionHandle { @@ -533,42 +467,74 @@ impl PeerManager { self.session_events_rx .insert(peer, ReceiverStream::new(event_rx)); - peer_info.state = PeerState::Active { - update_tx, - intents_after_close: vec![], - }; - peer_info.abort_handle = Some(abort_handle); + peer_info.conn_state = ConnState::Active { abort_handle }; + peer_info.session_state = SessionState::Active { update_tx }; } - Ok(ConnStep::Done { conn }) => { + ConnStep::Done(Ok(conn)) => { trace!("connection loop finished"); - let fut = async move { - terminate_gracefully(&conn).await?; - // The connection is fully closed. - drop(conn); - Ok(ConnStep::Closed) + let ConnState::Active { .. } = &peer_info.conn_state else { + unreachable!("connection state mismatch: Done comes after Active only"); + }; + if let SessionState::Active { .. } = &peer_info.session_state { + // TODO: Can this happen? + unreachable!( + "connection may not terminate gracefully while session is still active" + ); }; + let fut = async move { ConnStep::Closed(terminate_gracefully(conn).await) }; let abort_handle = spawn_conn_task(&mut self.conn_tasks, peer_info, fut); - if let PeerState::Closing { .. } = &peer_info.state { - peer_info.abort_handle = Some(abort_handle); + peer_info.conn_state = ConnState::Terminating { abort_handle }; + } + ConnStep::Done(Err(err)) => { + let ConnState::Active { .. } = &peer_info.conn_state else { + unreachable!("connection state mismatch: Done comes after Active only"); + }; + if let SessionState::Active { update_tx } = &peer_info.session_state { + warn!(?err, "connection failed while active"); + update_tx + .send(SessionUpdate::Abort(Error::ConnectionClosed(err))) + .await + .ok(); + peer_info.conn_state = ConnState::None; } else { - // TODO: What do we do with the closing abort handle in case we have a new connection already? + debug!(?err, "connection failed while on session is active"); + peer_info + .abort_pending_intents(err.context("failed while active")) + .await; + self.peers.remove(&peer); } } - Ok(ConnStep::Closed) => { - debug!("connection closed gracefully"); - let peer_info = self.peers.remove(&peer).expect("just checked"); - if let PeerState::Closing { intents } = peer_info.state { - if !intents.is_empty() { - debug!( - "resubmitting {} intents that were not yet processed", - intents.len() - ); - for intent in intents { - self.submit_intent(peer, intent).await; + ConnStep::Closed(res) => { + debug!(?res, "connection closed"); + match &peer_info.conn_state { + ConnState::Terminating { .. } => { + peer_info.conn_state = ConnState::None; + if !peer_info.pending_intents.is_empty() { + debug!("peer has pending intents, reconnect"); + match res { + Ok(()) => self.connect_if_inactive(peer), + Err(err) => { + peer_info + .abort_pending_intents( + err.context("failed while closing connection"), + ) + .await + } + } + } else if peer_info.session_state.is_none() { + debug!("removed peer"); + self.peers.remove(&peer).expect("just checked"); + } else { + debug!("keeping peer because session still closing"); } } - } else { - warn!(state=%peer_info.state, "reached closed step for peer in wrong state"); + ConnState::Establishing { .. } => { + debug!("conn is already establishing again"); + } + ConnState::Active { .. } => { + debug!("conn is already active again"); + } + ConnState::None => unreachable!("ConnState::Closed may not happen while None"), } } } @@ -578,37 +544,30 @@ impl PeerManager { async fn init_shutdown(&mut self) { self.shutting_down = true; for peer in self.peers.values() { - match &peer.state { - PeerState::None => {} - PeerState::Pending { .. } => { - // We are in pending state, which means the session has not yet been started. - // Hard-abort the task and let the other peer handle the error. - if let Some(abort_handle) = &peer.abort_handle { - abort_handle.abort(); - } - } - PeerState::Closing { .. } => {} - PeerState::Active { update_tx, .. } => { - // We are in active state. We cancel our session, which leads to graceful connection termination. - update_tx - .send(SessionUpdate::Abort(Error::ShuttingDown)) - .await - .ok(); - } + if let ConnState::Establishing { abort_handle, .. } = &peer.conn_state { + // We are in pending state, which means the session has not yet been started. + // Hard-abort the task and let the other peer handle the error. + abort_handle.abort(); + } + if let SessionState::Active { update_tx } = &peer.session_state { + // We are in active state. We cancel our session, which leads to graceful connection termination. + update_tx + .send(SessionUpdate::Abort(Error::ShuttingDown)) + .await + .ok(); } } } } fn spawn_conn_task( - conn_tasks: &mut JoinSet<(Role, NodeId, Result)>, + conn_tasks: &mut JoinSet<(NodeId, ConnStep)>, peer_info: &PeerInfo, - fut: impl Future> + Send + 'static, + fut: impl Future + Send + 'static, ) -> AbortHandle { let node_id = peer_info.node_id; - let our_role = peer_info.our_role; let fut = fut - .map(move |res| (our_role, node_id, res)) + .map(move |res| (node_id, res)) .instrument(peer_info.span.clone()); conn_tasks.spawn(fut) } @@ -616,52 +575,108 @@ fn spawn_conn_task( #[derive(Debug)] struct PeerInfo { node_id: NodeId, - our_role: Role, - abort_handle: Option, - state: PeerState, span: Span, + pending_intents: Vec, + conn_state: ConnState, + session_state: SessionState, } impl PeerInfo { - fn new(our_role: Role, peer: NodeId) -> Self { - Self { - node_id: peer, - our_role, - abort_handle: None, - state: PeerState::None, - span: error_span!("conn", peer=%peer.fmt_short()), + /// Returns `true` if the intent was pushed into the session channel and `false` if it was added to the pending intent list. + async fn push_intent(&mut self, intent: Intent) -> bool { + match &self.session_state { + SessionState::None => { + self.pending_intents.push(intent); + false + } + SessionState::Active { update_tx } => { + if let Err(err) = update_tx.send(SessionUpdate::SubmitIntent(intent)).await { + debug!("failed to submit intent into active session, queue in peer state"); + if let SessionUpdate::SubmitIntent(intent) = err.0 { + self.pending_intents.push(intent); + } + false + } else { + trace!("intent sent to session"); + true + } + } } } + + async fn abort_pending_intents(&mut self, err: anyhow::Error) { + let err = Arc::new(Error::Net(err)); + join_all( + self.pending_intents + .drain(..) + .map(|intent| intent.send_abort(err.clone())), + ) + .await; + } } -#[derive(Debug, strum::Display)] -enum PeerState { +#[derive(Debug, Default, strum::Display)] +enum SessionState { + #[default] None, - Pending { - intents: Vec, - cancel_dial: Option, - }, Active { update_tx: mpsc::Sender, - /// List of intents that we failed to submit into the session because it is closing. - intents_after_close: Vec, }, - Closing { - intents: Vec, +} + +impl SessionState { + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } +} + +#[derive(Debug, Default, strum::Display)] +enum ConnState { + #[default] + None, + Establishing { + our_dial: Option, + abort_handle: AbortHandle, }, + Active { + abort_handle: AbortHandle, + }, + Terminating { + abort_handle: AbortHandle, + }, +} + +impl ConnState { + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } +} + +impl PeerInfo { + fn new(peer: NodeId) -> Self { + Self { + node_id: peer, + span: error_span!("conn", peer=%peer.fmt_short()), + session_state: Default::default(), + conn_state: Default::default(), + pending_intents: Default::default(), + } + } +} + +#[derive(Debug)] +struct Established { + our_role: Role, + conn: Connection, + initial_transmission: InitialTransmission, + channel_streams: ChannelStreams, } #[derive(derive_more::Debug, strum::Display)] enum ConnStep { - Ready { - conn: Connection, - initial_transmission: InitialTransmission, - channel_streams: ChannelStreams, - }, - Done { - conn: Connection, - }, - Closed, + Established(anyhow::Result), + Done(anyhow::Result), + Closed(anyhow::Result<()>), } /// The internal handlers for the [`AcceptOpts]. diff --git a/iroh-willow/src/net.rs b/iroh-willow/src/net.rs index 9006c867b59..45603d6e092 100644 --- a/iroh-willow/src/net.rs +++ b/iroh-willow/src/net.rs @@ -1,6 +1,6 @@ //! Networking implementation for iroh-willow. -use std::{future::Future, time::Duration}; +use std::{future::Future, io, time::Duration}; use anyhow::{anyhow, ensure, Context as _, Result}; use futures_concurrency::future::TryJoin; @@ -10,7 +10,7 @@ use iroh_net::endpoint::{ Connection, ConnectionError, ReadError, ReadExactError, RecvStream, SendStream, VarInt, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tracing::{debug, trace}; +use tracing::{debug, trace, warn}; use crate::{ proto::wgps::{ @@ -257,6 +257,11 @@ fn prepare_channel( (sender, receiver, fut) } +/// Error code when stopping receive streams because we closed our session. +/// +/// This is currently for debugging purposes only - the other end will still see this as a connection error. +const ERROR_CODE_SESSION_CLOSED: VarInt = VarInt::from_u32(1); + async fn recv_loop( channel: Channel, mut recv_stream: RecvStream, @@ -269,9 +274,18 @@ async fn recv_loop( .await .context("failed to read from quic stream")? { - // trace!(len = buf.bytes.len(), "read"); - channel_writer.write_all(&buf.bytes[..]).await?; - // trace!(len = buf.bytes.len(), "sent"); + trace!(len = buf.bytes.len(), "read"); + match channel_writer.write_all(&buf.bytes[..]).await { + Ok(()) => { + trace!(len = buf.bytes.len(), "sent"); + } + Err(err) if err.kind() == io::ErrorKind::BrokenPipe => { + debug!("closing recv channel: session closed"); + recv_stream.stop(ERROR_CODE_SESSION_CLOSED)?; + break; + } + Err(err) => return Err(err.into()), + } } trace!(?channel, "recv: stream close"); channel_writer.close(); @@ -295,6 +309,7 @@ async fn send_loop( } trace!(?channel, "send: close writer"); send_stream.finish()?; + send_stream.stopped().await?; // We don't await SendStream::stopped, because we rely on application level closing notifications, // and make sure that the connection is closed gracefully in any case. trace!(?channel, "send: done"); @@ -321,7 +336,7 @@ async fn send_loop( /// /// Returns an error if the termination flow was aborted prematurely or if the connection was not /// closed with the expected error code. -pub(crate) async fn terminate_gracefully(conn: &Connection) -> Result<()> { +pub(crate) async fn terminate_gracefully(conn: Connection) -> Result<()> { trace!("terminating connection"); // Send a single byte on a newly opened uni stream. let mut send_stream = conn.open_uni().await?; @@ -329,7 +344,7 @@ pub(crate) async fn terminate_gracefully(conn: &Connection) -> Result<()> { send_stream.finish()?; // Wait until we either receive the goodbye byte from the other peer, or for the other peer // to close the connection with the expected error code. - match tokio::time::timeout(SHUTDOWN_TIMEOUT, wait_for_goodbye_or_graceful_close(conn)).await { + match tokio::time::timeout(SHUTDOWN_TIMEOUT, wait_for_goodbye_or_graceful_close(&conn)).await { Ok(Ok(())) => { conn.close(ERROR_CODE_OK, b"bye"); trace!("connection terminated gracefully"); @@ -574,8 +589,8 @@ mod tests { r2.unwrap(); tokio::try_join!( - terminate_gracefully(&conn_alfie), - terminate_gracefully(&conn_betty), + terminate_gracefully(conn_alfie), + terminate_gracefully(conn_betty), ) .expect("failed to close both connections gracefully"); @@ -747,8 +762,8 @@ mod tests { ); tokio::try_join!( - terminate_gracefully(&conn_alfie), - terminate_gracefully(&conn_betty), + terminate_gracefully(conn_alfie), + terminate_gracefully(conn_betty), ) .expect("failed to close both connections gracefully"); diff --git a/iroh-willow/src/proto/data_model.rs b/iroh-willow/src/proto/data_model.rs index 369e07eb1aa..1f6ceddec7d 100644 --- a/iroh-willow/src/proto/data_model.rs +++ b/iroh-willow/src/proto/data_model.rs @@ -86,6 +86,8 @@ pub type Path = willow_data_model::Path Result; + /// Debug-format the path as a lossy UTF-8 string. + fn fmt_utf8(&self) -> String; } impl PathExt for Path { @@ -104,6 +106,22 @@ impl PathExt for Path { Ok(path) } } + + fn fmt_utf8(&self) -> String { + let mut s = String::new(); + let mut iter = self.components().peekable(); + while let Some(c) = iter.next() { + if let Ok(c) = std::str::from_utf8(c.as_ref()) { + s.push_str(c); + } else { + s.push_str(&format!("<{}>", hex::encode(c.as_ref()))); + } + if iter.peek().is_some() { + s.push('/'); + } + } + s + } } #[derive(Debug, thiserror::Error)] diff --git a/iroh-willow/src/session.rs b/iroh-willow/src/session.rs index 2cf059e6c14..d12eae4f9a3 100644 --- a/iroh-willow/src/session.rs +++ b/iroh-willow/src/session.rs @@ -42,7 +42,7 @@ pub(crate) type SessionId = u64; /// To break symmetry, we refer to the peer that initiated the synchronisation session as Alfie, /// and the other peer as Betty. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] pub enum Role { /// The peer that initiated the synchronisation session. Alfie, diff --git a/iroh-willow/src/session/reconciler.rs b/iroh-willow/src/session/reconciler.rs index 8790f0181d1..695dec1c73b 100644 --- a/iroh-willow/src/session/reconciler.rs +++ b/iroh-willow/src/session/reconciler.rs @@ -4,11 +4,11 @@ use bytes::Bytes; use futures_lite::StreamExt; use genawaiter::rc::Co; use iroh_blobs::store::Store as PayloadStore; -use tracing::debug; +use tracing::{debug, trace}; use crate::{ proto::{ - data_model::PayloadDigest, + data_model::{PathExt, PayloadDigest}, grouping::{AreaExt, AreaOfInterest, Range3d}, keys::NamespaceId, wgps::{ @@ -98,11 +98,10 @@ impl Reconciler { loop { tokio::select! { Some(message) = self.recv.next() => { - tracing::trace!(?message, "tick: recv"); self.received_message(message?).await?; } Some(input) = self.targets.inbox.next() => { - tracing::trace!(?input, "tick: input"); + trace!(?input, "tick: input"); match input { Input::AoiIntersection(intersection) => { self.targets.init_target(&self.shared, intersection).await?; @@ -118,6 +117,7 @@ impl Reconciler { async fn received_message(&mut self, message: ReconciliationMessage) -> Result<(), Error> { match message { ReconciliationMessage::SendFingerprint(message) => { + trace!(range=?message.range, "recv SendFingerprint"); let target_id = message.handles(); let target = self .targets @@ -131,6 +131,7 @@ impl Reconciler { } } ReconciliationMessage::AnnounceEntries(message) => { + trace!(is_empty=?message.is_empty, range=?message.range, "recv AnnounceEntries"); let target_id = message.handles(); self.entry_state .received_announce_entries(target_id, message.is_empty)?; @@ -146,6 +147,11 @@ impl Reconciler { } } ReconciliationMessage::SendEntry(message) => { + trace!( + subspace = %message.entry.entry.subspace_id().fmt_short(), + path = %message.entry.entry.path().fmt_utf8(), + "recv SendEntry" + ); let authorised_entry = self .shared .static_tokens @@ -166,14 +172,18 @@ impl Reconciler { )?; } ReconciliationMessage::SendPayload(message) => { + trace!("recv SendPayload"); self.entry_state .received_send_payload(self.shared.store.payloads(), message.bytes) .await?; } - ReconciliationMessage::TerminatePayload(message) => { + ReconciliationMessage::TerminatePayload(ReconciliationTerminatePayload { + is_final, + }) => { + trace!(?is_final, "recv TerminatePayloade"); if let Some(completed_target) = self .entry_state - .received_terminate_payload(message.is_final) + .received_terminate_payload(is_final) .await? { let target = self @@ -528,7 +538,7 @@ impl Target { self.mark_our_next_range_pending(); } - // If we know for sure that our range is empty, we can skip creating the entry iterator alltogether. + // If we know for sure that our range is empty, we can skip creating the entry iterator. let mut iter = if is_empty { None } else { diff --git a/iroh-willow/src/store/memory.rs b/iroh-willow/src/store/memory.rs index 3b17561050f..ce36738973b 100644 --- a/iroh-willow/src/store/memory.rs +++ b/iroh-willow/src/store/memory.rs @@ -13,7 +13,9 @@ use std::task::{ready, Context, Poll, Waker}; use anyhow::Result; use futures_util::Stream; +use tracing::debug; +use crate::proto::data_model::PathExt; use crate::proto::grouping::Area; use crate::{ interest::{CapSelector, CapabilityPack}, @@ -255,6 +257,7 @@ impl EntryStore { && existing.is_newer_than(new) { // we cannot insert the entry, a newer entry exists + debug!(subspace=%entry.entry().subspace_id().fmt_short(), path=%entry.entry().path().fmt_utf8(), "skip ingest, already pruned"); return Ok(false); } if new.subspace_id() == existing.subspace_id() @@ -264,6 +267,7 @@ impl EntryStore { to_prune.push(i); } } + let pruned_count = to_prune.len(); for i in to_prune { let pruned = entries.remove(i); store.events.insert(move |id| { @@ -277,6 +281,7 @@ impl EntryStore { }); } entries.push(entry.clone()); + debug!(subspace=%entry.entry().subspace_id().fmt_short(), path=%entry.entry().path().fmt_utf8(), pruned=pruned_count, total=entries.len(), "ingest entry"); store .events .insert(|id| StoreEvent::Ingested(id, entry.clone(), origin)); diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index a98f664cac6..fce3f66a0b1 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -81,6 +81,7 @@ proptest = "1.2.0" rand_chacha = "0.3.1" regex = { version = "1.7.1", features = ["std"] } serde_json = "1.0.107" +test-strategy = "0.4.0" testdir = "0.9.1" testresult = "0.4.0" tokio = { version = "1", features = ["macros", "io-util", "rt"] } diff --git a/iroh/tests/spaces.proptest-regressions b/iroh/tests/spaces.proptest-regressions new file mode 100644 index 00000000000..156d17ee07f --- /dev/null +++ b/iroh/tests/spaces.proptest-regressions @@ -0,0 +1,30 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b247c5db7888ec8f993852033ea7d612f5a7cc5e51d6dc80cbbf0b370f1bf9df # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [])] } +cc 10758efcbd4145b23bb48a35a5a93b13f42cc71457b18e8b2f521fb66537e94e # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("gamma", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma")]), (Betty, [Write("beta", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("beta", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta")]), (Betty, [Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "beta")]), (Betty, [Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta")])] } +cc bad55ca9718ab95bc85e0ee4581fcf9ca019f10ae8cd8b1c30acd2ab7fd03a7f # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta")]), (Betty, [Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "beta")]), (Alfie, [Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "beta")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma")]), (Betty, [Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma")])] } +cc 9c1851f6773562a9d437743f7033d15df566ef3ee865533a4d197120af731891 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "gamma")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha")]), (Alfie, [])] } +cc 2bd80650f13377a3e39bbbf73c0fcf1f17b056880651abfc68e75f66a7f3c130 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta")]), (Betty, []), (Alfie, [Write("gamma", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha")]), (Alfie, [Write("alpha", "gamma")])] } +cc 48f0a951e77785086a8625b0e5afeb4a25e49fd1923e707eebc42d30c430a144 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "beta"), Write("beta", "beta")]), (Alfie, [Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha")]), (Betty, [Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta")]), (Alfie, [Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "beta")]), (Betty, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma")]), (Betty, [Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha")]), (Betty, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta")])] } +cc fd7a666da43de4a6647fd7a5b7c543c4f11abde19a3d8592d3845226bc964f1c # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [] } +cc 42fc5284840d3b6e58d5650c131a3ab6e7528fc98fb4c4fb77097e29715326f5 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [] } +cc cef4354611d13f5c0cdecbc409eb54eea1ef59512c272875d01040b187db84ea # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta")]), (Betty, []), (Alfie, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha")]), (Betty, [Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta")])] } +cc cfd6874efc9b42a5cad679512edfb09332852f4919920b2dde117e7039edff5a # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta")]), (Betty, [Write("beta", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "beta"), Write("gamma", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha")]), (Betty, [Write("gamma", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "alpha")])] } +cc 61e5a9d3c5dc02a1fe0ebb0a357d33c0c7eb5340a6b0982a9e6d187103366062 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma")]), (Betty, [Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "beta")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma")]), (Alfie, [Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta")]), (Betty, [Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta")])] } +cc 7ad26b3a87698eedb995dfcb549ddf6e730b900d794e3645e9193a3d0ba942af # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta")]), (Betty, [Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta")]), (Betty, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma")])] } +cc 66d537ccb5b41cbc39884aef2c35e3c6242a37973888a30f87d9918f3e626fca # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("gamma", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta")]), (Alfie, []), (Alfie, [Write("alpha", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma")]), (Alfie, [Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("beta", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "gamma")])] } +cc a7a7d234a4fbe2d760f1cc6000ba506da4e1a36e06ec8303a1877ff006242c84 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Betty, [Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "alpha")])] } +cc 8345ae470aaefbc75795860875a4a379be7a30a96c8108c87d0f7473278f55b8 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma")]), (Betty, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta")]), (Betty, [Write("beta", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "beta")])] } +cc f4f91399efad219908f9467397d047a6319e1111571cf08574f93fa8da8d1f06 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta")]), (Betty, [Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha")]), (Alfie, [Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("beta", "beta"), Write("beta", "beta")]), (Alfie, [Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "gamma")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "alpha")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "beta")])] } +cc d08210148ec737525059d04cef06d7c8911c32d67ecc9be9c2d9036493a6b0f8 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "alpha"), Write("gamma", "alpha")]), (X, [Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma")]), (Y, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "alpha")])] } +cc d1712b6b1cbada8a7fb7793bf1f53d562a2b2abef0842c2f533ec140da37f763 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "green"), Write("beta", "green"), Write("gamma", "green")]), (Y, []), (Y, [Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("alpha", "red")])] } +cc 52d30497f1066cca6e7f81e2e643ed6db7f7308708401f88e238bb0df583f2c1 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("gamma", "blue"), Write("gamma", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("alpha", "green")])] } +cc 81d02a44e9205146e30df256f51c3e306dbcc71ed54b8c5f6d5b9c6b011d73b6 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "red"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue")]), (Y, [Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "red")]), (X, [Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("alpha", "red")]), (X, [Write("beta", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red")]), (Y, [Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "green")]), (Y, [Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("gamma", "green")]), (X, [Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "blue")]), (Y, [Write("beta", "red"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue")]), (X, [Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "red"), Write("alpha", "green")]), (X, [Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("alpha", "red"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "red")]), (X, [Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "red"), Write("gamma", "blue")]), (X, [Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "red")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "green")]), (X, [Write("gamma", "blue")])] } +cc 1bf6826f5eed0f39830227d43f7dfad1e043474fe580ffd44ddbb396631860eb # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "blue"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "red"), Write("alpha", "blue")]), (Y, [Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "blue")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "green")]), (Y, [Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red")]), (Y, [Write("beta", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green")]), (Y, [Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "green")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "red")]), (X, [Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green")]), (X, [Write("beta", "red"), Write("alpha", "green"), Write("alpha", "green")]), (X, [Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "blue"), Write("beta", "blue")]), (X, [Write("beta", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("alpha", "red")]), (X, [Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("alpha", "red")]), (X, [Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("beta", "blue")]), (Y, [Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "blue")]), (Y, [Write("beta", "blue"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green"), Write("beta", "red"), Write("gamma", "red"), Write("gamma", "red")]), (X, [Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green")])] } +cc b3451ba74bf6d3578cd59d523609b25e6d26e72d542bfd2928106263ab0c0317 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("alpha", "blue"), Write("gamma", "red")]), (Y, [Write("beta", "green"), Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "blue")]), (X, []), (X, [Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue")]), (X, [Write("alpha", "green"), Write("gamma", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "blue")]), (Y, [Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "green"), Write("beta", "blue")]), (X, [Write("gamma", "green"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "red")]), (Y, [Write("alpha", "red"), Write("beta", "red")]), (Y, [Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "blue")]), (Y, [Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red")]), (Y, [Write("beta", "red"), Write("beta", "green"), Write("beta", "green")]), (X, [Write("gamma", "blue"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "red")]), (Y, [Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "green")])] } +cc ba16827de785794376a4a534790de30db562c8ae30d79ce9e9c113215aa7ec4b # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "red"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "green")]), (Y, [Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("beta", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "red")]), (X, [Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "green")]), (Y, [Write("gamma", "blue"), Write("beta", "blue")]), (X, [Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "green")]), (Y, [Write("beta", "blue"), Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "green")]), (Y, [Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "green"), Write("gamma", "red")]), (Y, [Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "red")]), (Y, [Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "red")]), (Y, [Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("beta", "green")]), (Y, [Write("beta", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "red")]), (Y, [Write("gamma", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "green")]), (X, [Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("gamma", "red"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("gamma", "green")])] } +cc f7035816a33aa12aab5db8a46f69789d339c4610dc800ced7d359806511a4b7a # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "green"), Write("beta", "red")]), (X, [Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "green")]), (Y, [Write("gamma", "blue"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "red")]), (X, [Write("beta", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red")]), (X, [Write("beta", "red"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("beta", "red")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "green"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "red"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "green")]), (X, [Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "red")]), (Y, [Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "green")]), (X, [Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "blue")]), (X, [Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "green")]), (X, [Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "green")]), (X, [Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green")]), (X, [Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "blue")]), (X, [Write("gamma", "green"), Write("gamma", "green")]), (Y, [Write("alpha", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "blue")]), (Y, [Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("gamma", "blue")]), (Y, [Write("beta", "green"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "red")])] } diff --git a/iroh/tests/spaces.rs b/iroh/tests/spaces.rs index c00ccac660f..d7becda3f5f 100644 --- a/iroh/tests/spaces.rs +++ b/iroh/tests/spaces.rs @@ -1,19 +1,27 @@ -use anyhow::Result; +use std::{collections::BTreeMap, time::Duration}; + +use anyhow::ensure; use futures_lite::StreamExt; -use iroh::client::{spaces::EntryForm, Iroh}; +use iroh::client::{ + spaces::{EntryForm, Space}, + Iroh, +}; use iroh_net::{key::SecretKey, NodeAddr}; use iroh_willow::{ - interest::{CapSelector, DelegateTo, RestrictArea}, + interest::{AreaOfInterestSelector, CapSelector, DelegateTo, RestrictArea}, proto::{ data_model::{Path, PathExt}, grouping::{Area, Range3d}, - keys::NamespaceKind, + keys::{NamespaceKind, UserId}, meadowcap::AccessMode, }, session::{intents::Completion, SessionMode}, store::traits::{EntryOrigin, StoreEvent}, }; -use tracing::info; +use proptest::{collection::vec, prelude::Strategy, sample::select}; +use test_strategy::proptest; +use testresult::TestResult; +use tracing::{error, info}; /// Spawn an iroh node in a separate thread and tokio runtime, and return /// the address and client. @@ -41,8 +49,220 @@ async fn spawn_node() -> (NodeAddr, Iroh) { receiver.await.unwrap() } +#[derive(Debug, Clone)] +enum Operation { + Write(String, String), +} + +fn simple_key() -> impl Strategy { + select(&["alpha", "beta", "gamma"]).prop_map(str::to_string) +} + +fn simple_value() -> impl Strategy { + select(&["red", "blue", "green"]).prop_map(str::to_string) +} + +fn simple_op() -> impl Strategy { + (simple_key(), simple_value()).prop_map(|(key, value)| Operation::Write(key, value)) +} + +fn role() -> impl Strategy { + select(&[Peer::X, Peer::Y]) +} + +#[derive(Debug, Eq, PartialEq, Clone, Copy, Ord, PartialOrd)] +enum Peer { + X, + Y, +} + +#[proptest] +fn test_get_many_weird_result( + #[strategy(vec((role(), vec(simple_op(), 0..20)), 0..20))] rounds: Vec<(Peer, Vec)>, +) { + iroh_test::logging::setup_multithreaded(); + + let res = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let mut simulated_entries: BTreeMap<(Peer, String), String> = BTreeMap::new(); + + let (addr_x, iroh_x) = spawn_node().await; + let (addr_y, iroh_y) = spawn_node().await; + let node_id_x = addr_x.node_id; + let node_id_y = addr_y.node_id; + iroh_x.net().add_node_addr(addr_y.clone()).await?; + iroh_y.net().add_node_addr(addr_x.clone()).await?; + let user_x = iroh_x.spaces().create_user().await?; + let user_y = iroh_y.spaces().create_user().await?; + info!( + "X is node {} user {}", + node_id_x.fmt_short(), + user_x.fmt_short() + ); + info!( + "Y is node {} user {}", + node_id_y.fmt_short(), + user_y.fmt_short() + ); + let space_x = iroh_x.spaces().create(NamespaceKind::Owned, user_x).await?; + + let ticket = space_x + .share(user_y, AccessMode::Write, RestrictArea::None) + .await?; + + // give betty access + let (space_y, syncs) = iroh_y + .spaces() + .import_and_sync(ticket, SessionMode::ReconcileOnce) + .await?; + + let mut completions = syncs.complete_all().await; + assert_eq!(completions.len(), 1); + let completion = completions.remove(&node_id_x).unwrap(); + assert!(completion.is_ok()); + assert_eq!(completion.unwrap(), Completion::Complete); + + let count = rounds.len(); + for (i, (peer, round)) in rounds.into_iter().enumerate() { + let i = i + 1; + let (space, user) = match peer { + Peer::X => (&space_x, user_x), + Peer::Y => (&space_y, user_y), + }; + info!(active=?peer, "[{i}/{count}] round start"); + + for Operation::Write(key, value) in round { + info!(?key, ?value, "[{i}/{count}] write"); + space + .insert_bytes( + EntryForm::new(user, Path::from_bytes(&[key.as_bytes()])?), + value.clone().into_bytes(), + ) + .await?; + simulated_entries.insert((peer, key), value); + } + + // We sync in both directions. This will only create a single session under the hood. + // Awaiting both intents ensures that the sync completed on both sides. + // Alternatively, we could sync from one side only, the result must be the same, however we miss + // an event in the client currently to know when the betty peer (accepting peer) has finished. + let fut_x = async { + space_x + .sync_once(node_id_y, AreaOfInterestSelector::Widest) + .await? + .complete() + .await?; + anyhow::Ok(()) + }; + let fut_y = async { + space_y + .sync_once(node_id_x, AreaOfInterestSelector::Widest) + .await? + .complete() + .await?; + anyhow::Ok(()) + }; + let fut = async { tokio::try_join!(fut_x, fut_y) }; + tokio::time::timeout(Duration::from_secs(10), fut).await??; + + info!("[{i}/{count}] sync complete"); + + let map_x = space_to_map(&space_x, &iroh_x, user_x, user_y).await?; + let map_y = space_to_map(&space_y, &iroh_y, user_x, user_y).await?; + ensure!( + map_x == map_y, + "states out of sync:\n{map_x:#?}\n !=\n{map_y:#?}" + ); + + ensure!( + map_x == map_y, + "states out of sync:\n{map_x:#?}\n !=\n{map_y:#?}" + ); + ensure!( + simulated_entries == map_x, + "alfie in unexpected state:\n{simulated_entries:#?}\n !=\n{map_x:#?}" + ); + // follows transitively, but still + ensure!( + simulated_entries == map_y, + "betty in unexpected state:\n{simulated_entries:#?}\n !=\n{map_y:#?}" + ); + } + + info!("completed {count} rounds successfully"); + + tokio::try_join!(iroh_x.shutdown(false), iroh_y.shutdown(false))?; + + Ok(()) + }); + if let Err(err) = &res { + error!(?err, "FAILED"); + } + res.map_err(AnyhowStdErr)?; +} + +async fn space_to_map( + space: &Space, + node: &Iroh, + user_x: UserId, + user_y: UserId, +) -> anyhow::Result> { + let role_lookup = BTreeMap::from([(user_x, Peer::X), (user_y, Peer::Y)]); + let entries = space + .get_many(Range3d::new_full()) + .await? + .try_collect::<_, _, Vec<_>>() + .await?; + let mut map: BTreeMap<(Peer, String), String> = BTreeMap::new(); + for auth_entry in entries { + let (entry, auth) = auth_entry.into_parts(); + let key_component = entry + .path() + .get_component(0) + .ok_or_else(|| anyhow::anyhow!("path component missing"))?; + let key = String::from_utf8(key_component.to_vec())?; + + let value = node.blobs().read_to_bytes(entry.payload_digest().0).await?; + + let user = auth.capability.receiver(); + let peer = role_lookup + .get(user) + .ok_or_else(|| anyhow::anyhow!("foreign write?"))?; + + map.insert((*peer, key), String::from_utf8_lossy(&value).to_string()); + } + + Ok(map) +} + +#[derive(Debug)] +struct AnyhowStdErr(anyhow::Error); + +impl std::fmt::Display for AnyhowStdErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for AnyhowStdErr { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.0.source() + } + + fn description(&self) -> &str { + "description() is deprecated; use Display" + } + + fn cause(&self) -> Option<&dyn std::error::Error> { + self.source() + } +} + #[tokio::test] -async fn spaces_smoke() -> Result<()> { +async fn spaces_smoke() -> TestResult { iroh_test::logging::setup_multithreaded(); let (alfie_addr, alfie) = spawn_node().await; let (betty_addr, betty) = spawn_node().await; @@ -138,7 +358,7 @@ async fn spaces_smoke() -> Result<()> { } #[tokio::test] -async fn spaces_subscription() -> Result<()> { +async fn spaces_subscription() -> TestResult { iroh_test::logging::setup_multithreaded(); let (alfie_addr, alfie) = spawn_node().await; let (betty_addr, betty) = spawn_node().await; @@ -224,7 +444,7 @@ async fn spaces_subscription() -> Result<()> { #[tokio::test] async fn test_restricted_area() -> testresult::TestResult { iroh_test::logging::setup_multithreaded(); - const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + const TIMEOUT: Duration = Duration::from_secs(2); let (alfie_addr, alfie) = spawn_node().await; let (betty_addr, betty) = spawn_node().await; info!("alfie is {}", alfie_addr.node_id.fmt_short());