background task

This commit is contained in:
yves committed 2026-11-20 23:22:25 +01:00
1 parent 47b73e6935
commit ce44ee48b3
7 files changed
+483 -69

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@ meow-proto = { path = "../meow-proto" }
meow-crypto = { path = "../meow-crypto" }
meow-store = { path = "../meow-store" }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
tokio = { version = "1", features = ["fs", "time", "macros"] }
tokio = { version = "1", features = ["fs", "time", "macros", "sync", "rt"] }
futures-util = "0.3"
thiserror = "2"
tracing = "0.1"
+182
View File
@@ -0,0 +1,182 @@
//! Background synchronisation (roadmap lot 5.9).
//!
//! [`MeowClient::start`] moves the client into a task that long-polls every server,
//! processes what arrives, sends queued deposits when they are due and publishes every
//! [`Event`] on the client's broadcast channel. The application keeps full access to
//! the client through [`ClientHandle::lock`].
//!
//! Network waits never hold the lock: each step is *prepared* under the lock, the
//! network calls run without it, and the results are *applied* under the lock. A send
//! from the UI therefore never waits for a 30 s long-poll or an unreachable server.
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{broadcast, watch, Mutex, MutexGuard};
use tokio::task::JoinHandle;
use crate::outbox::execute_flush;
use crate::{execute_pull, Event, MeowClient};
/// Pace of the background task.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BackgroundOptions {
/// How long each pull may wait on the servers for new messages. It also bounds how
/// late a queued deposit can be retried.
pub long_poll: Duration,
/// First delay after a failed round (all servers unreachable); doubled up to
/// [`BackgroundOptions::max_backoff`].
pub backoff: Duration,
pub max_backoff: Duration,
}
impl Default for BackgroundOptions {
fn default() -> Self {
Self {
long_poll: Duration::from_secs(30),
backoff: Duration::from_secs(2),
max_backoff: Duration::from_secs(60),
}
}
}
/// A client running in the background.
pub struct ClientHandle {
client: Arc<Mutex<MeowClient>>,
events: broadcast::Sender<Event>,
stop: watch::Sender<bool>,
task: JoinHandle<()>,
}
impl MeowClient {
/// Start background synchronisation. Subscribe with [`ClientHandle::subscribe`];
/// use the client through [`ClientHandle::lock`]; get it back with
/// [`ClientHandle::stop`].
pub fn start(self, options: BackgroundOptions) -> ClientHandle {
let events = self.events.clone();
let client = Arc::new(Mutex::new(self));
let (stop, stop_rx) = watch::channel(false);
let task = tokio::spawn(run(client.clone(), options, stop_rx));
ClientHandle {
client,
events,
stop,
task,
}
}
}
impl ClientHandle {
/// Receive every event from now on.
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
self.events.subscribe()
}
/// Exclusive access to the client (sending, contacts, history…). Keep it short:
/// the background task needs the lock between its network calls.
pub async fn lock(&self) -> MutexGuard<'_, MeowClient> {
self.client.lock().await
}
/// Stop the background task (an in-flight long-poll is abandoned; its acks will
/// simply be sent again) and give the client back.
pub async fn stop(self) -> MeowClient {
let _ = self.stop.send(true);
if let Err(e) = self.task.await {
tracing::error!("background task failed: {e}");
}
match Arc::try_unwrap(self.client) {
Ok(m) => m.into_inner(),
// Unreachable: the task is finished and the handle was consumed.
Err(_) => unreachable!("client still shared after the task ended"),
}
}
}
/// Wait for `d`, or until asked to stop. Returns `true` when stopping.
async fn pause(d: Duration, stop: &mut watch::Receiver<bool>) -> bool {
tokio::select! {
_ = tokio::time::sleep(d) => *stop.borrow(),
_ = stop.changed() => true,
}
}
async fn run(client: Arc<Mutex<MeowClient>>, opts: BackgroundOptions, mut stop: watch::Receiver<bool>) {
let mut wait = opts.long_poll;
let mut backoff = opts.backoff;
while !*stop.borrow() {
// 1. Due deposits (and expiries).
let (plan, transport) = {
let mut c = client.lock().await;
(c.prepare_flush(false), c.transport.clone())
};
if !plan.is_empty() {
let accepted = tokio::select! {
a = execute_flush(&transport, &plan) => a,
_ = stop.changed() => break,
};
let mut c = client.lock().await;
if let Err(e) = c.apply_flush(plan, accepted).await {
tracing::error!("cannot save the send queue: {e}");
}
}
{
let mut c = client.lock().await;
let pending = std::mem::take(&mut c.pending_events);
c.emit(&pending);
}
// 2. Pull (long-poll), without the lock while waiting. The wait ends in time
// for the next queued deposit to be retried.
let plan = {
let mut c = client.lock().await;
let w = match c.next_retry_in() {
Some(d) => wait.min(d.max(Duration::from_secs(1))),
None => wait,
};
c.prepare_pull(w)
};
if plan.is_empty() {
// No contact yet: just keep the queue moving.
let d = client.lock().await.next_retry_in().unwrap_or(opts.long_poll);
if pause(d.clamp(Duration::from_secs(1), opts.long_poll), &mut stop).await {
break;
}
continue;
}
let results = tokio::select! {
r = execute_pull(&transport, &plan) => r,
_ = stop.changed() => break,
};
// 3. Apply and process under the lock.
let mut c = client.lock().await;
match c.apply_pull(plan, results) {
Ok(round) => {
backoff = opts.backoff;
let got_new = !round.new.is_empty();
if got_new {
match c.process(round.new).await {
Ok(ev) => c.emit(&ev),
Err(e) => tracing::error!("cannot process received messages: {e}"),
}
}
// Something arrived (acks to send, replies to flush) or the servers
// held messages back: go again at once.
wait = if got_new || round.more {
Duration::ZERO
} else {
opts.long_poll
};
}
Err(e) => {
drop(c);
tracing::warn!("background pull failed: {e}; retrying in {backoff:?}");
if pause(backoff, &mut stop).await {
break;
}
backoff = (backoff * 2).min(opts.max_backoff);
}
}
}
}
+100 -26
View File
@@ -17,6 +17,7 @@
#![forbid(unsafe_code)]
#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))]
pub mod background;
pub mod outbox;
pub mod transport;
@@ -35,6 +36,7 @@ use meow_proto::v2::{self, request::Body as Req, response::Body as Resp};
use meow_proto::Message;
use meow_store::{Batch, RetiredMailbox, SpaceId, State as SpaceState, Store};
pub use background::{BackgroundOptions, ClientHandle};
pub use meow_core::contact::ContactState as State;
pub use meow_store::{Content, HistoryEntry, HistoryPage, PendingAck, Settings, Status};
pub use outbox::{PendingSend, RetryPolicy};
@@ -146,9 +148,43 @@ pub struct SendReport {
pub delivered_to: usize,
}
/// Events kept for slow subscribers.
const EVENT_BUFFER: usize = 256;
/// Acknowledged ids sent per mailbox and pull (server limit: 1024).
const MAX_ACKS_PER_MAILBOX: usize = 512;
/// Pull requests prepared under the client lock and sent without it.
pub(crate) struct PullPlan {
jobs: Vec<PullJob>,
wait: Duration,
}
struct PullJob {
server: ServerInfo,
/// Mailboxes pulled, with their space.
mailboxes: Vec<(usize, SignPublic)>,
/// Acks carried by the request.
sent: Vec<(usize, PendingAck)>,
request: v2::Request,
}
impl PullPlan {
pub(crate) fn is_empty(&self) -> bool {
self.jobs.is_empty()
}
}
/// Network part of a pull round (no client state).
pub(crate) async fn execute_pull(transport: &transport::Transport, plan: &PullPlan) -> Vec<Result<Resp>> {
join_all(
plan.jobs
.iter()
.map(|j| transport.call_request(&j.server, j.request.clone(), plan.wait)),
)
.await
}
#[derive(Default)]
struct PullRound {
/// Messages to process, with the index of the space owning their mailbox.
@@ -178,6 +214,8 @@ pub struct MeowClient {
/// Events produced outside `sync` (e.g. an expiry noticed while sending), handed
/// out by the next `sync` or `flush_outbox`.
pending_events: Vec<Event>,
/// Every event, for subscribers ([`MeowClient::subscribe`]).
events: tokio::sync::broadcast::Sender<Event>,
}
type Mailboxes = Vec<(SigningKeyPair, Vec<ServerInfo>)>;
@@ -213,6 +251,7 @@ impl MeowClient {
transport: transport::Transport::new()?,
retry: RetryPolicy::default(),
pending_events: Vec::new(),
events: tokio::sync::broadcast::channel(EVENT_BUFFER).0,
})
}
@@ -243,6 +282,20 @@ impl MeowClient {
&self.main().nickname
}
/// Receive every [`Event`] from now on, whatever produced it (`sync`, the background
/// task, `flush_outbox`). A receiver that falls more than 256 events behind gets
/// `RecvError::Lagged` and should reload its view (history) from the client.
pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver<Event> {
self.events.subscribe()
}
pub(crate) fn emit(&self, events: &[Event]) {
for e in events {
// No subscriber is fine.
let _ = self.events.send(e.clone());
}
}
/// Identity preferences (receipts…).
pub fn settings(&self) -> Settings {
self.main().settings
@@ -759,6 +812,8 @@ impl MeowClient {
let got_new = !r.new.is_empty();
if got_new {
events.extend(self.process(r.new).await?);
// Replies produced by processing (receipts, handshake steps).
self.flush(false).await?;
}
// Pull again (without waiting) only to acknowledge what just arrived and to
// fetch what servers held back. Redeliveries alone do not loop.
@@ -767,6 +822,8 @@ impl MeowClient {
}
wait = Duration::ZERO;
}
events.append(&mut self.pending_events);
self.emit(&events);
Ok(events)
}
@@ -774,6 +831,16 @@ impl MeowClient {
/// are returned and recorded as pending acks of the space owning their mailbox;
/// they are saved by [`Self::process`].
async fn pull_round(&mut self, wait: Duration) -> Result<PullRound> {
let plan = self.prepare_pull(wait);
if plan.is_empty() {
return Ok(PullRound::default());
}
let results = execute_pull(&self.transport, &plan).await;
self.apply_pull(plan, results)
}
/// Under the client lock: build one pull request per server (with pending acks).
pub(crate) fn prepare_pull(&mut self, wait: Duration) -> PullPlan {
// server (by signing key) → (server, [(space index, mailbox key)])
let mut jobs: BTreeMap<SignPublic, (ServerInfo, Vec<(usize, SigningKeyPair)>)> = BTreeMap::new();
for si in 0..self.spaces.len() {
@@ -808,14 +875,12 @@ impl MeowClient {
})
});
}
let mut round = PullRound::default();
if jobs.is_empty() {
return Ok(round);
}
let wait_s = wait.as_secs().min(u32::MAX as u64) as u32;
let mut sent_acks: Vec<Vec<(usize, PendingAck)>> = Vec::with_capacity(jobs.len());
let mut requests = Vec::with_capacity(jobs.len());
for (server, kps) in jobs.values() {
let mut plan = PullPlan {
jobs: Vec::with_capacity(jobs.len()),
wait,
};
for (server, kps) in jobs.into_values() {
let rid = meow_crypto::random_bytes::<16>().to_vec();
let ts = now();
let mut sent = Vec::new();
@@ -832,50 +897,60 @@ impl MeowClient {
.inspect(|a| sent.push((*si, (*a).clone())))
.map(|a| a.id.clone())
.collect();
rpc::mailbox_auth(kp, server, &rid, acks, ts)
rpc::mailbox_auth(kp, &server, &rid, acks, ts)
})
.collect();
sent_acks.push(sent);
requests.push(v2::Request {
let request = v2::Request {
request_id: rid,
timestamp: ts,
body: Some(Req::Pull(v2::Pull {
auths,
wait_seconds: wait_s,
})),
};
let mailboxes = kps.iter().map(|(si, kp)| (*si, kp.public())).collect();
plan.jobs.push(PullJob {
server,
mailboxes,
sent,
request,
});
}
let calls = jobs
.values()
.zip(requests)
.map(|((server, _), req)| self.transport.call_request(server, req, wait));
let results = join_all(calls).await;
plan
}
/// Under the client lock: record the received messages as pending acks.
pub(crate) fn apply_pull(&mut self, plan: PullPlan, results: Vec<Result<Resp>>) -> Result<PullRound> {
let mut round = PullRound::default();
let mut failures = 0;
let mut last_err = None;
for (((server, kps), r), sent) in jobs.values().zip(results).zip(sent_acks) {
let n = plan.jobs.len();
for (job, r) in plan.jobs.into_iter().zip(results) {
match r {
Ok(Resp::PullResult(p)) => {
// The server applied our acks before answering. Forgetting them
// needs no save: at worst they are sent again (acks are idempotent).
for (si, a) in &sent {
for (si, a) in &job.sent {
self.spaces[*si].state.pending_acks.retain(|x| x != a);
}
round.more |= p.more;
for m in p.messages {
let Some((si, _)) = kps
let Some((si, _)) = job
.mailboxes
.iter()
.find(|(_, kp)| kp.public().as_bytes().as_slice() == m.mailbox.as_slice())
.find(|(_, pk)| pk.as_bytes().as_slice() == m.mailbox.as_slice())
else {
tracing::debug!("server sent a message for a mailbox we did not pull");
continue;
};
let ack = PendingAck {
server: server.sign_key,
server: job.server.sign_key,
mailbox: m.mailbox.clone(),
id: m.id.clone(),
};
let sp = &mut self.spaces[*si];
let Some(sp) = self.spaces.get_mut(*si) else {
continue;
};
// Already processed (its ack was lost): only re-acknowledge.
if !sp.state.pending_acks.contains(&ack) {
sp.state.pending_acks.push(ack);
@@ -886,13 +961,13 @@ impl MeowClient {
}
Ok(_) => failures += 1,
Err(e) => {
tracing::warn!("pull on {} failed: {e}", server.url);
tracing::warn!("pull on {} failed: {e}", job.server.url);
failures += 1;
last_err = Some(e);
}
}
}
if failures == jobs.len() {
if n > 0 && failures == n {
return Err(last_err.unwrap_or_else(|| Error::Network("all servers failed".into())));
}
Ok(round)
@@ -927,7 +1002,7 @@ impl MeowClient {
}
}
async fn process(&mut self, stored: Vec<(usize, v2::StoredMessage)>) -> Result<Vec<Event>> {
pub(crate) async fn process(&mut self, stored: Vec<(usize, v2::StoredMessage)>) -> Result<Vec<Event>> {
let mailboxes_before: Vec<Mailboxes> = (0..self.spaces.len()).map(|si| self.space_mailboxes(si)).collect();
// Handshake messages first (step 2 before step 3), then ratchet messages by arrival.
let mut envs: Vec<(u32, u64, usize, Vec<u8>, v2::PeerEnvelope)> = stored
@@ -1081,9 +1156,8 @@ impl MeowClient {
}
}
// Persist (state, history, acks, queued replies) before anything leaves the device.
// The replies are sent by the caller's next flush.
self.save().await?;
self.flush(false).await?;
events.append(&mut self.pending_events);
Ok(events)
}
}
+85 -39
View File
@@ -125,9 +125,15 @@ impl MeowClient {
/// Try to deliver queued deposits: those due (or all, with `force`). Returns, per
/// item id, how many servers accepted it now. Expiries go to `pending_events`.
pub(crate) async fn flush(&mut self, force: bool) -> Result<HashMap<Vec<u8>, usize>> {
let plan = self.prepare_flush(force);
let accepted = execute_flush(&self.transport, &plan).await;
self.apply_flush(plan, accepted).await
}
/// Under the client lock: expire old items and group the due deposits.
pub(crate) fn prepare_flush(&mut self, force: bool) -> FlushPlan {
let t = now();
let events = &mut self.pending_events;
// Expired items.
let max_age = self.retry.max_age.as_secs();
for sp in &mut self.spaces {
let before = sp.state.outbox.len();
@@ -171,13 +177,6 @@ impl MeowClient {
}
}
}
if due.is_empty() {
self.save().await?;
return Ok(HashMap::new());
}
// One Post per server and batch; a rejected batch is retried item by item, so
// one bad deposit cannot block the others.
let mut batches: Vec<(ServerInfo, Batch)> = Vec::new();
for (server, items) in by_server.into_values() {
let mut cur: Batch = Vec::new();
@@ -195,30 +194,22 @@ impl MeowClient {
batches.push((server, cur));
}
}
let results = join_all(batches.iter().map(|(s, b)| self.post(s, b))).await;
let mut accepted: Vec<(usize, Vec<u8>, SignPublic)> = Vec::new();
for ((server, batch), r) in batches.iter().zip(results) {
match r {
Ok(()) => accepted.extend(batch.iter().map(|(si, id, _)| (*si, id.clone(), server.sign_key))),
Err(Error::Network(e)) => tracing::warn!("deposit on {} failed: {e}", server.url),
Err(e) if batch.len() > 1 => {
tracing::warn!("{} rejected a batch ({e}): retrying one by one", server.url);
for one in batch {
match self.post(server, std::slice::from_ref(one)).await {
Ok(()) => accepted.push((one.0, one.1.clone(), server.sign_key)),
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
}
}
}
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
}
}
FlushPlan { batches, due, at: t }
}
// Update the items.
/// Under the client lock: record what servers accepted, schedule retries, save.
pub(crate) async fn apply_flush(
&mut self,
plan: FlushPlan,
accepted: Vec<(usize, Vec<u8>, SignPublic)>,
) -> Result<HashMap<Vec<u8>, usize>> {
let mut now_accepted: HashMap<Vec<u8>, usize> = HashMap::new();
for (si, id, key) in accepted {
let sp = &mut self.spaces[si];
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
if !o.pending.iter().any(|s| s.sign_key == key) {
continue; // already recorded (concurrent flush)
}
o.pending.retain(|s| s.sign_key != key);
if o.accepted == 0 && !o.message_id.is_empty() {
sp.batch
@@ -229,12 +220,12 @@ impl MeowClient {
*now_accepted.entry(id).or_default() += 1;
}
}
for (si, id) in due {
for (si, id) in plan.due {
let retry = self.retry;
let sp = &mut self.spaces[si];
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
o.attempts += 1;
o.next_attempt = t + retry.delay(o.attempts);
o.next_attempt = plan.at + retry.delay(o.attempts);
}
sp.state.outbox.retain(|o| !o.pending.is_empty());
sp.dirty = true;
@@ -243,24 +234,79 @@ impl MeowClient {
Ok(now_accepted)
}
/// Time until the next queued deposit is due (zero if one is due now).
pub(crate) fn next_retry_in(&self) -> Option<Duration> {
let t = now();
self.spaces
.iter()
.flat_map(|sp| sp.state.outbox.iter())
.map(|o| Duration::from_secs(o.next_attempt.saturating_sub(t)))
.min()
}
/// Deliver queued deposits now, without waiting for their retry time. Returns the
/// pending events (e.g. [`Event::SendFailed`]).
pub async fn flush_outbox(&mut self) -> Result<Vec<Event>> {
self.flush(true).await?;
Ok(std::mem::take(&mut self.pending_events))
let events = std::mem::take(&mut self.pending_events);
self.emit(&events);
Ok(events)
}
}
async fn post(&self, server: &ServerInfo, batch: &[(usize, Vec<u8>, v2::Deposit)]) -> Result<()> {
let deposits = batch.iter().map(|(_, _, d)| d.clone()).collect();
match self
.transport
.call(server, Req::Post(v2::Post { deposits }), Duration::ZERO)
.await?
{
Resp::PostAck(_) => Ok(()),
_ => Err(Error::Invalid("unexpected response".into())),
/// Deposits to post, prepared under the client lock and sent without it.
pub(crate) struct FlushPlan {
batches: Vec<(ServerInfo, Batch)>,
due: Vec<(usize, Vec<u8>)>,
at: u64,
}
impl FlushPlan {
pub(crate) fn is_empty(&self) -> bool {
self.due.is_empty()
}
}
/// Network part of a flush (no client state): one Post per server and batch; a rejected
/// batch is retried item by item, so one bad deposit cannot block the others.
pub(crate) async fn execute_flush(
transport: &crate::transport::Transport,
plan: &FlushPlan,
) -> Vec<(usize, Vec<u8>, SignPublic)> {
let results = join_all(plan.batches.iter().map(|(s, b)| post(transport, s, b))).await;
let mut accepted = Vec::new();
for ((server, batch), r) in plan.batches.iter().zip(results) {
match r {
Ok(()) => accepted.extend(batch.iter().map(|(si, id, _)| (*si, id.clone(), server.sign_key))),
Err(Error::Network(e)) => tracing::warn!("deposit on {} failed: {e}", server.url),
Err(e) if batch.len() > 1 => {
tracing::warn!("{} rejected a batch ({e}): retrying one by one", server.url);
for one in batch {
match post(transport, server, std::slice::from_ref(one)).await {
Ok(()) => accepted.push((one.0, one.1.clone(), server.sign_key)),
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
}
}
}
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
}
}
accepted
}
async fn post(
transport: &crate::transport::Transport,
server: &ServerInfo,
batch: &[(usize, Vec<u8>, v2::Deposit)],
) -> Result<()> {
let deposits = batch.iter().map(|(_, _, d)| d.clone()).collect();
match transport
.call(server, Req::Post(v2::Post { deposits }), Duration::ZERO)
.await?
{
Resp::PostAck(_) => Ok(()),
_ => Err(Error::Invalid("unexpected response".into())),
}
}
#[cfg(test)]
+92
View File
@@ -615,3 +615,95 @@ async fn delivery_and_read_receipts() {
assert_eq!(statuses(&alice, "bob", true)[2], Status::Sent);
let _ = std::fs::remove_dir_all(&dir);
}
// ───────────── background task and event stream (roadmap lot 5.9) ─────────────
async fn next_message(rx: &mut tokio::sync::broadcast::Receiver<Event>) -> String {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if let Event::Message { text, .. } = rx.recv().await.unwrap() {
return text;
}
}
})
.await
.expect("no message within 10 s")
}
#[tokio::test(flavor = "multi_thread")]
async fn background_sync_publishes_events() {
use std::sync::atomic::Ordering::SeqCst;
let (url, store) = spawn_flaky_server().await;
let servers = vec![url];
let dir = tmpdir();
let mut alice = client(&dir, "alice", &servers).await;
let mut bob = client(&dir, "bob", &servers).await;
connect(&mut alice, &mut bob, "bob").await;
// Bob runs in the background with a long long-poll.
let opts = meow_client::BackgroundOptions {
long_poll: Duration::from_secs(20),
..Default::default()
};
bob.set_retry_policy(meow_client::RetryPolicy {
base: Duration::from_secs(1),
..Default::default()
});
let bob = bob.start(opts);
let mut rx = bob.subscribe();
tokio::time::sleep(Duration::from_millis(300)).await;
// Pushed to the subscriber without any explicit sync.
alice.send_text("bob", "hello background").await.unwrap();
assert_eq!(next_message(&mut rx).await, "hello background");
// The lock is free while the task waits on the server: sending is immediate.
tokio::time::sleep(Duration::from_millis(300)).await;
let start = std::time::Instant::now();
let r = bob.lock().await.send_text("alice", "reply").await.unwrap();
assert_eq!(r.delivered_to, 1);
assert!(
start.elapsed() < Duration::from_secs(3),
"send waited {:?}",
start.elapsed()
);
assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["reply"]);
// Alice's read receipt reaches Bob's subscribers too.
alice.mark_read("bob").await.unwrap();
let got = tokio::time::timeout(Duration::from_secs(10), async {
loop {
if let Event::Receipt { kind, .. } = rx.recv().await.unwrap() {
if kind == meow_client::ReceiptKind::Read {
return;
}
}
}
})
.await;
assert!(got.is_ok(), "no read receipt event");
// A message queued while the server refuses deposits is retried by the task.
store.fail_posts.store(true, SeqCst);
let r = bob.lock().await.send_text("alice", "retried").await.unwrap();
assert_eq!(r.delivered_to, 0);
store.fail_posts.store(false, SeqCst);
// The task shortens its long-poll to retry on time (no other traffic needed).
let delivered = tokio::time::timeout(Duration::from_secs(10), async {
loop {
if bob.lock().await.pending_sends().is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
})
.await;
assert!(delivered.is_ok(), "queued message not retried");
assert!(texts(&alice.sync(Duration::ZERO).await.unwrap()).contains(&"retried".to_string()));
// Stopping gives the client back, usable as before.
let mut bob = bob.stop().await;
alice.send_text("bob", "after stop").await.unwrap();
assert_eq!(texts(&bob.sync(Duration::ZERO).await.unwrap()), vec!["after stop"]);
let _ = std::fs::remove_dir_all(&dir);
}