send queue
This commit is contained in:
1 parent
a520ddb557
commit
10af480218
10 files changed
+701
-60
No files matched your search
@@ -17,6 +17,7 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))]
|
||||
|
||||
pub mod outbox;
|
||||
pub mod transport;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
@@ -36,6 +37,7 @@ use meow_store::{Batch, RetiredMailbox, SpaceId, State as SpaceState, Store};
|
||||
|
||||
pub use meow_core::contact::ContactState as State;
|
||||
pub use meow_store::{Content, HistoryEntry, HistoryPage, PendingAck};
|
||||
pub use outbox::{PendingSend, RetryPolicy};
|
||||
|
||||
/// Maximum file size sent in a single message (chunking: roadmap 5.7).
|
||||
pub const MAX_FILE_BYTES: usize = 4 * 1024 * 1024;
|
||||
@@ -119,13 +121,16 @@ pub enum Event {
|
||||
name: String,
|
||||
data: Vec<u8>,
|
||||
},
|
||||
/// A queued message expired before any server accepted it.
|
||||
SendFailed { contact: String, message_id: Vec<u8> },
|
||||
}
|
||||
|
||||
/// Outcome of a send.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SendReport {
|
||||
pub message_id: Vec<u8>,
|
||||
/// Number of servers that accepted the deposit.
|
||||
/// Number of servers that accepted the deposit right away. `0` means queued: it is
|
||||
/// retried by later sends, syncs and [`MeowClient::flush_outbox`].
|
||||
pub delivered_to: usize,
|
||||
}
|
||||
|
||||
@@ -157,6 +162,10 @@ pub struct MeowClient {
|
||||
store: std::sync::Mutex<Store>,
|
||||
spaces: Vec<Space>,
|
||||
transport: transport::Transport,
|
||||
retry: RetryPolicy,
|
||||
/// Events produced outside `sync` (e.g. an expiry noticed while sending), handed
|
||||
/// out by the next `sync` or `flush_outbox`.
|
||||
pending_events: Vec<Event>,
|
||||
}
|
||||
|
||||
type Mailboxes = Vec<(SigningKeyPair, Vec<ServerInfo>)>;
|
||||
@@ -190,6 +199,8 @@ impl MeowClient {
|
||||
dirty: false,
|
||||
}],
|
||||
transport: transport::Transport::new()?,
|
||||
retry: RetryPolicy::default(),
|
||||
pending_events: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -300,6 +311,13 @@ impl MeowClient {
|
||||
.iter()
|
||||
.map(|(kp, _)| kp.public().as_bytes().to_vec())
|
||||
.collect();
|
||||
let queued: Vec<meow_store::OutboxItem> = self.spaces[si]
|
||||
.state
|
||||
.outbox
|
||||
.iter()
|
||||
.filter(|o| o.contact == contact.id)
|
||||
.cloned()
|
||||
.collect();
|
||||
let acks: Vec<PendingAck> = self.spaces[si]
|
||||
.state
|
||||
.pending_acks
|
||||
@@ -311,12 +329,14 @@ impl MeowClient {
|
||||
let t = &mut self.spaces[target];
|
||||
t.state.contacts.push(contact);
|
||||
t.state.pending_acks.extend(acks);
|
||||
t.state.outbox.extend(queued);
|
||||
t.batch.history.extend(history);
|
||||
t.batch.files.extend(files);
|
||||
self.save_space(target)?;
|
||||
let s = &mut self.spaces[si];
|
||||
s.state.contacts.remove(ci);
|
||||
let cid = s.state.contacts.remove(ci).id;
|
||||
s.state.pending_acks.retain(|a| !mailboxes.contains(&a.mailbox));
|
||||
s.state.outbox.retain(|o| o.contact != cid);
|
||||
self.save_space(si)
|
||||
}
|
||||
|
||||
@@ -416,7 +436,8 @@ impl MeowClient {
|
||||
pub async fn remove_contact(&mut self, name: &str) -> Result<()> {
|
||||
let (si, ci) = self.locate(name)?;
|
||||
let before = self.space_mailboxes(si);
|
||||
self.spaces[si].state.contacts.remove(ci);
|
||||
let cid = self.spaces[si].state.contacts.remove(ci).id;
|
||||
self.spaces[si].state.outbox.retain(|o| o.contact != cid);
|
||||
self.retire_mailboxes(si, before);
|
||||
self.save_space(si)
|
||||
}
|
||||
@@ -506,10 +527,11 @@ impl MeowClient {
|
||||
let contact = &mut self.spaces[si].state.contacts[ci];
|
||||
let ack = invitation::finalize(contact, &msg)?;
|
||||
let deposit = messaging::deposit_for(contact, &ack)?;
|
||||
let servers = peer_servers(contact);
|
||||
let queued = outbox::item(&contact.id, vec![], &deposit, peer_servers(contact));
|
||||
let name = contact.name.clone();
|
||||
self.spaces[si].state.outbox.push(queued);
|
||||
self.save_space(si)?;
|
||||
self.deliver(&servers, vec![deposit]).await?;
|
||||
self.flush(false).await?;
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
@@ -574,8 +596,16 @@ impl MeowClient {
|
||||
let info = self.inspect_invitation(&f.payload)?;
|
||||
let answer_bytes = self.answer_invitation(&f.payload, contact_name, server_urls).await?;
|
||||
let answer = v2::InvitationMessage::decode(answer_bytes.as_slice()).map_err(meow_core::Error::from)?;
|
||||
self.deliver(&info.answer_servers, vec![invitation::answer_deposit(&info, &answer)])
|
||||
.await?;
|
||||
let (si, ci) = self.by_invitation(&info.invitation_id)?;
|
||||
let queued = outbox::item(
|
||||
&self.spaces[si].state.contacts[ci].id,
|
||||
vec![],
|
||||
&invitation::answer_deposit(&info, &answer),
|
||||
info.answer_servers.clone(),
|
||||
);
|
||||
self.spaces[si].state.outbox.push(queued);
|
||||
self.save_space(si)?;
|
||||
self.flush(false).await?;
|
||||
Ok(info.initiator_name)
|
||||
}
|
||||
|
||||
@@ -619,6 +649,12 @@ impl MeowClient {
|
||||
let env = messaging::encrypt(contact, &msg)?;
|
||||
let deposit = messaging::deposit_for(contact, &env)?;
|
||||
let servers = peer_servers(contact);
|
||||
if servers.is_empty() {
|
||||
return Err(Error::Invalid("contact has no server".into()));
|
||||
}
|
||||
let queued = outbox::item(&contact.id, msg.id.clone(), &deposit, servers);
|
||||
let item_id = queued.id.clone();
|
||||
sp.state.outbox.push(queued);
|
||||
sp.batch.history.push(HistoryEntry {
|
||||
id: msg.id.clone(),
|
||||
contact: contact.id.clone(),
|
||||
@@ -627,46 +663,16 @@ impl MeowClient {
|
||||
received_at: now(),
|
||||
content,
|
||||
});
|
||||
// Persisted (ratchet + queue) before anything is sent.
|
||||
self.save_space(si)?;
|
||||
let delivered_to = self.deliver(&servers, vec![deposit]).await?;
|
||||
let accepted = self.flush(false).await?;
|
||||
let delivered_to = accepted.get(&item_id).copied().unwrap_or(0);
|
||||
Ok(SendReport {
|
||||
message_id: msg.id,
|
||||
delivered_to,
|
||||
})
|
||||
}
|
||||
|
||||
/// Post deposits to every server; succeeds if at least one accepted them.
|
||||
async fn deliver(&self, servers: &[ServerInfo], deposits: Vec<v2::Deposit>) -> Result<usize> {
|
||||
if servers.is_empty() {
|
||||
return Err(Error::Invalid("contact has no server".into()));
|
||||
}
|
||||
let calls = servers.iter().map(|s| {
|
||||
self.transport.call(
|
||||
s,
|
||||
Req::Post(v2::Post {
|
||||
deposits: deposits.clone(),
|
||||
}),
|
||||
Duration::ZERO,
|
||||
)
|
||||
});
|
||||
let mut ok = 0;
|
||||
let mut last_err = None;
|
||||
for (s, r) in servers.iter().zip(join_all(calls).await) {
|
||||
match r {
|
||||
Ok(Resp::PostAck(_)) => ok += 1,
|
||||
Ok(_) => last_err = Some(Error::Network(format!("{}: unexpected response", s.url))),
|
||||
Err(e) => {
|
||||
tracing::warn!("deposit on {} failed: {e}", s.url);
|
||||
last_err = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
match (ok, last_err) {
|
||||
(0, Some(e)) => Err(e),
|
||||
(n, _) => Ok(n),
|
||||
}
|
||||
}
|
||||
|
||||
// ───────────────────────── sync ─────────────────────────
|
||||
|
||||
/// Pull every mailbox of every unlocked space on every server (one request per
|
||||
@@ -680,7 +686,9 @@ impl MeowClient {
|
||||
/// sends the acknowledgements and fetches what the servers held back.
|
||||
pub async fn sync(&mut self, wait: Duration) -> Result<Vec<Event>> {
|
||||
const MAX_ROUNDS: usize = 16;
|
||||
let mut events = Vec::new();
|
||||
// Messages queued while offline go first.
|
||||
self.flush(false).await?;
|
||||
let mut events = std::mem::take(&mut self.pending_events);
|
||||
let mut wait = wait;
|
||||
for round in 0..MAX_ROUNDS {
|
||||
let r = match self.pull_round(wait).await {
|
||||
@@ -880,7 +888,6 @@ impl MeowClient {
|
||||
envs.sort_by_key(|(rank, at, _, _, _)| (*rank, *at));
|
||||
|
||||
let mut events = Vec::new();
|
||||
let mut outgoing: Vec<(Vec<ServerInfo>, v2::Deposit)> = Vec::new();
|
||||
for (_, received_at, si, mailbox, env) in envs {
|
||||
let sp = &mut self.spaces[si];
|
||||
let Some(ci) = sp.state.contacts.iter().position(|c| {
|
||||
@@ -896,7 +903,10 @@ impl MeowClient {
|
||||
Some(v2::peer_envelope::Body::Invitation(inv)) if inv.step == 2 => {
|
||||
match invitation::accept(contact, &inv, now()) {
|
||||
Ok(confirm) => {
|
||||
outgoing.push((peer_servers(contact), invitation::confirm_deposit(contact, &confirm)?));
|
||||
let d = invitation::confirm_deposit(contact, &confirm)?;
|
||||
sp.state
|
||||
.outbox
|
||||
.push(outbox::item(&contact.id, vec![], &d, peer_servers(contact)));
|
||||
events.push(Event::InvitationAccepted {
|
||||
contact: contact.name.clone(),
|
||||
});
|
||||
@@ -907,7 +917,10 @@ impl MeowClient {
|
||||
Some(v2::peer_envelope::Body::Invitation(inv)) if inv.step == 3 => {
|
||||
match invitation::finalize(contact, &inv) {
|
||||
Ok(ack) => {
|
||||
outgoing.push((peer_servers(contact), messaging::deposit_for(contact, &ack)?));
|
||||
let d = messaging::deposit_for(contact, &ack)?;
|
||||
sp.state
|
||||
.outbox
|
||||
.push(outbox::item(&contact.id, vec![], &d, peer_servers(contact)));
|
||||
events.push(Event::ContactActivated {
|
||||
contact: contact.name.clone(),
|
||||
});
|
||||
@@ -973,13 +986,10 @@ impl MeowClient {
|
||||
self.retire_mailboxes(si, before);
|
||||
}
|
||||
}
|
||||
// Persist before anything leaves the device.
|
||||
// Persist (state, history, acks, queued replies) before anything leaves the device.
|
||||
self.save().await?;
|
||||
for (servers, deposit) in outgoing {
|
||||
if let Err(e) = self.deliver(&servers, vec![deposit]).await {
|
||||
tracing::warn!("could not deliver handshake message: {e}");
|
||||
}
|
||||
}
|
||||
self.flush(false).await?;
|
||||
events.append(&mut self.pending_events);
|
||||
Ok(events)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
//! Persistent send queue (roadmap lot 5.5).
|
||||
//!
|
||||
//! Every deposit (user message or handshake message) is first stored as an
|
||||
//! [`OutboxItem`] in the same save as the ratchet state that produced it, then posted
|
||||
//! to the recipient's pull servers. Servers that accept it are removed from the item;
|
||||
//! the others are retried with exponential backoff until the item expires. Deposits due
|
||||
//! at the same time are grouped into one `Post` per server.
|
||||
//!
|
||||
//! This is also where Matriochka routing will plug in (roadmap 8.2): an item will carry
|
||||
//! a route and a release time instead of a plain list of servers.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::future::join_all;
|
||||
use meow_core::contact::ContactId;
|
||||
use meow_core::server::ServerInfo;
|
||||
use meow_crypto::SignPublic;
|
||||
use meow_proto::v2::{self, request::Body as Req, response::Body as Resp};
|
||||
use meow_proto::Message;
|
||||
use meow_store::OutboxItem;
|
||||
|
||||
use crate::{now, Error, Event, MeowClient, Result};
|
||||
|
||||
/// When to retry and when to give up.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct RetryPolicy {
|
||||
/// Delay before the first retry; doubled at each attempt.
|
||||
pub base: Duration,
|
||||
/// Upper bound of the delay between two attempts.
|
||||
pub max_delay: Duration,
|
||||
/// An item older than this is dropped ([`Event::SendFailed`] if no server took it).
|
||||
pub max_age: Duration,
|
||||
}
|
||||
|
||||
impl Default for RetryPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base: Duration::from_secs(5),
|
||||
max_delay: Duration::from_secs(3600),
|
||||
max_age: Duration::from_secs(7 * 86_400),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryPolicy {
|
||||
/// Delay after `attempts` failed attempts, with ±20 % jitter.
|
||||
fn delay(&self, attempts: u32) -> u64 {
|
||||
let base = self.base.as_secs().max(1);
|
||||
let d = base
|
||||
.saturating_mul(1u64 << attempts.saturating_sub(1).min(32))
|
||||
.min(self.max_delay.as_secs().max(base));
|
||||
let spread = (d * 2 / 5).max(1);
|
||||
let r = u64::from_le_bytes(meow_crypto::random_bytes::<8>()) % (spread + 1);
|
||||
d - d / 5 + r
|
||||
}
|
||||
}
|
||||
|
||||
/// A queued deposit, as shown to the user.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PendingSend {
|
||||
pub contact: String,
|
||||
/// Empty for handshake messages.
|
||||
pub message_id: Vec<u8>,
|
||||
/// Servers that already accepted it (it is still retried on the others).
|
||||
pub accepted: u32,
|
||||
pub attempts: u32,
|
||||
pub next_attempt: u64,
|
||||
}
|
||||
|
||||
/// Deposits per `Post` and their total payload size.
|
||||
const MAX_BATCH: usize = 64;
|
||||
const MAX_BATCH_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
pub(crate) fn item(
|
||||
contact: &ContactId,
|
||||
message_id: Vec<u8>,
|
||||
deposit: &v2::Deposit,
|
||||
servers: Vec<ServerInfo>,
|
||||
) -> OutboxItem {
|
||||
let t = now();
|
||||
OutboxItem {
|
||||
id: meow_crypto::random_bytes::<16>().to_vec(),
|
||||
contact: contact.clone(),
|
||||
message_id,
|
||||
deposit: deposit.encode_to_vec(),
|
||||
pending: servers,
|
||||
accepted: 0,
|
||||
attempts: 0,
|
||||
created_at: t,
|
||||
next_attempt: t,
|
||||
}
|
||||
}
|
||||
|
||||
type Batch = Vec<(usize, Vec<u8>, v2::Deposit)>; // (space, item id, deposit)
|
||||
|
||||
impl MeowClient {
|
||||
/// Change the retry policy of the send queue.
|
||||
pub fn set_retry_policy(&mut self, policy: RetryPolicy) {
|
||||
self.retry = policy;
|
||||
}
|
||||
|
||||
/// Deposits not yet accepted by every server.
|
||||
pub fn pending_sends(&self) -> Vec<PendingSend> {
|
||||
self.spaces
|
||||
.iter()
|
||||
.flat_map(|sp| {
|
||||
sp.state.outbox.iter().map(|o| PendingSend {
|
||||
contact: sp
|
||||
.state
|
||||
.contacts
|
||||
.iter()
|
||||
.find(|c| c.id == o.contact)
|
||||
.map(|c| c.name.clone())
|
||||
.unwrap_or_default(),
|
||||
message_id: o.message_id.clone(),
|
||||
accepted: o.accepted,
|
||||
attempts: o.attempts,
|
||||
next_attempt: o.next_attempt,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Try to deliver queued deposits: those due (or all, with `force`). Returns, per
|
||||
/// item id, how many servers accepted it now. Expiries go to `pending_events`.
|
||||
pub(crate) async fn flush(&mut self, force: bool) -> Result<HashMap<Vec<u8>, usize>> {
|
||||
let t = now();
|
||||
let events = &mut self.pending_events;
|
||||
// Expired items.
|
||||
let max_age = self.retry.max_age.as_secs();
|
||||
for sp in &mut self.spaces {
|
||||
let before = sp.state.outbox.len();
|
||||
let contacts = &sp.state.contacts;
|
||||
sp.state.outbox.retain(|o| {
|
||||
let expired = t >= o.created_at.saturating_add(max_age);
|
||||
if expired && o.accepted == 0 {
|
||||
events.push(Event::SendFailed {
|
||||
contact: contacts
|
||||
.iter()
|
||||
.find(|c| c.id == o.contact)
|
||||
.map(|c| c.name.clone())
|
||||
.unwrap_or_default(),
|
||||
message_id: o.message_id.clone(),
|
||||
});
|
||||
}
|
||||
!expired
|
||||
});
|
||||
sp.dirty |= sp.state.outbox.len() != before;
|
||||
}
|
||||
|
||||
// Due deposits grouped by server, in queue order.
|
||||
let mut by_server: BTreeMap<SignPublic, (ServerInfo, Batch)> = BTreeMap::new();
|
||||
let mut due: Vec<(usize, Vec<u8>)> = Vec::new();
|
||||
for (si, sp) in self.spaces.iter().enumerate() {
|
||||
for o in sp.state.outbox.iter().filter(|o| force || o.next_attempt <= t) {
|
||||
let deposit = match v2::Deposit::decode(o.deposit.as_slice()) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
tracing::warn!("dropping corrupted queued deposit: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
due.push((si, o.id.clone()));
|
||||
for s in &o.pending {
|
||||
by_server
|
||||
.entry(s.sign_key)
|
||||
.or_insert_with(|| (s.clone(), Vec::new()))
|
||||
.1
|
||||
.push((si, o.id.clone(), deposit.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
if due.is_empty() {
|
||||
self.save().await?;
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
// One Post per server and batch; a rejected batch is retried item by item, so
|
||||
// one bad deposit cannot block the others.
|
||||
let mut batches: Vec<(ServerInfo, Batch)> = Vec::new();
|
||||
for (server, items) in by_server.into_values() {
|
||||
let mut cur: Batch = Vec::new();
|
||||
let mut size = 0;
|
||||
for it in items {
|
||||
let n = it.2.payload.len();
|
||||
if !cur.is_empty() && (cur.len() >= MAX_BATCH || size + n > MAX_BATCH_BYTES) {
|
||||
batches.push((server.clone(), std::mem::take(&mut cur)));
|
||||
size = 0;
|
||||
}
|
||||
size += n;
|
||||
cur.push(it);
|
||||
}
|
||||
if !cur.is_empty() {
|
||||
batches.push((server, cur));
|
||||
}
|
||||
}
|
||||
let results = join_all(batches.iter().map(|(s, b)| self.post(s, b))).await;
|
||||
let mut accepted: Vec<(usize, Vec<u8>, SignPublic)> = Vec::new();
|
||||
for ((server, batch), r) in batches.iter().zip(results) {
|
||||
match r {
|
||||
Ok(()) => accepted.extend(batch.iter().map(|(si, id, _)| (*si, id.clone(), server.sign_key))),
|
||||
Err(Error::Network(e)) => tracing::warn!("deposit on {} failed: {e}", server.url),
|
||||
Err(e) if batch.len() > 1 => {
|
||||
tracing::warn!("{} rejected a batch ({e}): retrying one by one", server.url);
|
||||
for one in batch {
|
||||
match self.post(server, std::slice::from_ref(one)).await {
|
||||
Ok(()) => accepted.push((one.0, one.1.clone(), server.sign_key)),
|
||||
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
||||
}
|
||||
}
|
||||
|
||||
// Update the items.
|
||||
let mut now_accepted: HashMap<Vec<u8>, usize> = HashMap::new();
|
||||
for (si, id, key) in accepted {
|
||||
if let Some(o) = self.spaces[si].state.outbox.iter_mut().find(|o| o.id == id) {
|
||||
o.pending.retain(|s| s.sign_key != key);
|
||||
o.accepted += 1;
|
||||
*now_accepted.entry(id).or_default() += 1;
|
||||
}
|
||||
}
|
||||
for (si, id) in due {
|
||||
let retry = self.retry;
|
||||
let sp = &mut self.spaces[si];
|
||||
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
|
||||
o.attempts += 1;
|
||||
o.next_attempt = t + retry.delay(o.attempts);
|
||||
}
|
||||
sp.state.outbox.retain(|o| !o.pending.is_empty());
|
||||
sp.dirty = true;
|
||||
}
|
||||
self.save().await?;
|
||||
Ok(now_accepted)
|
||||
}
|
||||
|
||||
/// Deliver queued deposits now, without waiting for their retry time. Returns the
|
||||
/// pending events (e.g. [`Event::SendFailed`]).
|
||||
pub async fn flush_outbox(&mut self) -> Result<Vec<Event>> {
|
||||
self.flush(true).await?;
|
||||
Ok(std::mem::take(&mut self.pending_events))
|
||||
}
|
||||
|
||||
async fn post(&self, server: &ServerInfo, batch: &[(usize, Vec<u8>, v2::Deposit)]) -> Result<()> {
|
||||
let deposits = batch.iter().map(|(_, _, d)| d.clone()).collect();
|
||||
match self
|
||||
.transport
|
||||
.call(server, Req::Post(v2::Post { deposits }), Duration::ZERO)
|
||||
.await?
|
||||
{
|
||||
Resp::PostAck(_) => Ok(()),
|
||||
_ => Err(Error::Invalid("unexpected response".into())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn backoff_grows_and_is_capped() {
|
||||
let p = RetryPolicy::default();
|
||||
let d1 = p.delay(1);
|
||||
assert!((4..=6).contains(&d1), "{d1}");
|
||||
let d4 = p.delay(4);
|
||||
assert!((32..=48).contains(&d4), "{d4}");
|
||||
let d30 = p.delay(30);
|
||||
assert!((2880..=4320).contains(&d30), "{d30}");
|
||||
}
|
||||
}
|
||||
@@ -193,11 +193,16 @@ async fn file_and_server_invitations_then_messaging() {
|
||||
struct FlakyStore {
|
||||
inner: MemoryStore,
|
||||
drop_acks: std::sync::atomic::AtomicBool,
|
||||
/// Refuse deposits (the server answers with an error).
|
||||
fail_posts: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl meow_server::store::ServerStore for FlakyStore {
|
||||
async fn post(&self, mb: [u8; 32], payload: Vec<u8>, now: u64) -> meow_server::store::StoreResult<()> {
|
||||
if self.fail_posts.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
return Err(meow_server::store::StoreError::Backend("down".into()));
|
||||
}
|
||||
self.inner.post(mb, payload, now).await
|
||||
}
|
||||
async fn fetch(
|
||||
@@ -230,6 +235,27 @@ impl meow_server::store::ServerStore for FlakyStore {
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_flaky_server() -> (String, Arc<FlakyStore>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let keys = meow_core::server::ServerKeys::generate();
|
||||
let (notify, _) = broadcast::channel(1024);
|
||||
let store = Arc::new(FlakyStore {
|
||||
inner: MemoryStore::new(notify.clone(), 1000),
|
||||
drop_acks: false.into(),
|
||||
fail_posts: false.into(),
|
||||
});
|
||||
let state = Arc::new(AppState {
|
||||
card: encode_card(&keys, &url, "flaky", ""),
|
||||
keys,
|
||||
store: store.clone(),
|
||||
notify,
|
||||
limits: Limits::default(),
|
||||
});
|
||||
tokio::spawn(meow_server::serve(listener, state));
|
||||
(url, store)
|
||||
}
|
||||
|
||||
/// What a client that crashed right after receiving the response would have done:
|
||||
/// pull without ever acknowledging.
|
||||
async fn raw_pull(
|
||||
@@ -266,6 +292,7 @@ async fn no_message_lost_when_the_client_dies() {
|
||||
let store = Arc::new(FlakyStore {
|
||||
inner: MemoryStore::new(notify.clone(), 1000),
|
||||
drop_acks: false.into(),
|
||||
fail_posts: false.into(),
|
||||
});
|
||||
let limits = Limits {
|
||||
max_pull_messages: 2, // force the "more" path
|
||||
@@ -415,3 +442,76 @@ async fn hidden_contacts() {
|
||||
assert_eq!(alice.history("bob").unwrap().len(), 3);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
// ───────────── persistent send queue (roadmap lot 5.5) ─────────────
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn offline_messages_are_queued_and_retried() {
|
||||
use std::sync::atomic::Ordering::SeqCst;
|
||||
let (url, store) = spawn_flaky_server().await;
|
||||
let servers = vec![url];
|
||||
let dir = tmpdir();
|
||||
let path = dir.join("bob.mwid2");
|
||||
let mut alice = client(&dir, "alice", &servers).await;
|
||||
let mut bob = client(&dir, "bob", &servers).await;
|
||||
connect(&mut alice, &mut bob, "bob").await;
|
||||
|
||||
// Server refuses deposits: sending still succeeds, the message is queued.
|
||||
store.fail_posts.store(true, SeqCst);
|
||||
let r1 = bob.send_text("alice", "queued 1").await.unwrap();
|
||||
let r2 = bob.send_text("alice", "queued 2").await.unwrap();
|
||||
assert_eq!((r1.delivered_to, r2.delivered_to), (0, 0));
|
||||
let pending = bob.pending_sends();
|
||||
assert_eq!(pending.len(), 2);
|
||||
assert!(pending.iter().all(|p| p.contact == "alice" && p.attempts >= 1));
|
||||
assert_eq!(pending[0].message_id, r1.message_id);
|
||||
assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty());
|
||||
|
||||
// The queue survives a restart; not yet due (backoff), so a sync does not send.
|
||||
drop(bob);
|
||||
let mut bob = MeowClient::open(&path, "pw").await.unwrap();
|
||||
assert_eq!(bob.pending_sends().len(), 2);
|
||||
store.fail_posts.store(false, SeqCst);
|
||||
bob.sync(Duration::ZERO).await.unwrap();
|
||||
assert_eq!(bob.pending_sends().len(), 2);
|
||||
// Forced flush: both delivered (in one Post), in order.
|
||||
assert!(bob.flush_outbox().await.unwrap().is_empty());
|
||||
assert!(bob.pending_sends().is_empty());
|
||||
assert_eq!(
|
||||
texts(&alice.sync(Duration::ZERO).await.unwrap()),
|
||||
vec!["queued 1", "queued 2"]
|
||||
);
|
||||
|
||||
// Due items are retried by the next send.
|
||||
bob.set_retry_policy(meow_client::RetryPolicy {
|
||||
base: Duration::from_secs(1),
|
||||
..Default::default()
|
||||
});
|
||||
store.fail_posts.store(true, SeqCst);
|
||||
bob.send_text("alice", "late").await.unwrap();
|
||||
store.fail_posts.store(false, SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(2500)).await;
|
||||
let r = bob.send_text("alice", "now").await.unwrap();
|
||||
assert_eq!(r.delivered_to, 1);
|
||||
assert!(bob.pending_sends().is_empty());
|
||||
assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["late", "now"]);
|
||||
|
||||
// An item that expires undelivered is reported.
|
||||
bob.set_retry_policy(meow_client::RetryPolicy {
|
||||
max_age: Duration::ZERO,
|
||||
..Default::default()
|
||||
});
|
||||
store.fail_posts.store(true, SeqCst);
|
||||
let r = bob.send_text("alice", "lost").await.unwrap();
|
||||
assert_eq!(r.delivered_to, 0);
|
||||
let ev = bob.flush_outbox().await.unwrap();
|
||||
assert_eq!(
|
||||
ev,
|
||||
vec![Event::SendFailed {
|
||||
contact: "alice".into(),
|
||||
message_id: r.message_id
|
||||
}]
|
||||
);
|
||||
assert!(bob.pending_sends().is_empty());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
Reference in new issue
Block a user