message saved if client crashes on retreival
This commit is contained in:
1 parent
6d3ea3e2a6
commit
006bedc728
11 files changed
+591
-72
No files matched your search
@@ -18,6 +18,7 @@ thiserror = "2"
|
||||
tracing = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1"
|
||||
meow-server = { path = "../meow-server" }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net"] }
|
||||
|
||||
|
||||
+154
-21
@@ -28,7 +28,8 @@ use meow_proto::v2::{self, request::Body as Req, response::Body as Resp};
|
||||
use meow_proto::Message;
|
||||
|
||||
pub use meow_core::contact::ContactState as State;
|
||||
pub use profile::{Content, HistoryEntry};
|
||||
use profile::RetiredMailbox;
|
||||
pub use profile::{Content, HistoryEntry, PendingAck};
|
||||
|
||||
/// Maximum file size sent in a single message (chunking: roadmap 5.7).
|
||||
pub const MAX_FILE_BYTES: usize = 4 * 1024 * 1024;
|
||||
@@ -109,6 +110,17 @@ pub struct SendReport {
|
||||
}
|
||||
|
||||
/// One identity, with its servers, contacts and history.
|
||||
/// Acknowledged ids sent per mailbox and pull (server limit: 1024).
|
||||
const MAX_ACKS_PER_MAILBOX: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct PullRound {
|
||||
/// Messages to process.
|
||||
new: Vec<v2::StoredMessage>,
|
||||
/// A server held messages back.
|
||||
more: bool,
|
||||
}
|
||||
|
||||
pub struct MeowClient {
|
||||
path: PathBuf,
|
||||
password: String,
|
||||
@@ -268,8 +280,10 @@ impl MeowClient {
|
||||
|
||||
pub async fn remove_contact(&mut self, name: &str) -> Result<()> {
|
||||
let i = self.contact_index(name)?;
|
||||
let before = self.contact_mailboxes();
|
||||
let c = self.profile.contacts.remove(i);
|
||||
self.profile.history.retain(|h| h.contact != c.id);
|
||||
self.retire_mailboxes(before);
|
||||
self.save().await
|
||||
}
|
||||
|
||||
@@ -521,45 +535,134 @@ impl MeowClient {
|
||||
/// Pull every mailbox on every server (one request per server), process what
|
||||
/// arrived and return the resulting events. With `wait > 0` the servers hold the
|
||||
/// request until something arrives or the delay expires (long-poll).
|
||||
///
|
||||
/// Delivery is at-least-once: servers keep messages until they are acknowledged,
|
||||
/// and a message is acknowledged (in the next pull) only once its effects are saved.
|
||||
/// After a round that brought messages, `sync` pulls again without waiting, which
|
||||
/// sends the acknowledgements and fetches what the servers held back.
|
||||
pub async fn sync(&mut self, wait: Duration) -> Result<Vec<Event>> {
|
||||
// server (by signing key) → (server, mailbox keys)
|
||||
let mut jobs: BTreeMap<SignPublic, (ServerInfo, Vec<SigningKeyPair>)> = BTreeMap::new();
|
||||
for c in &self.profile.contacts {
|
||||
for (kp, servers) in c.mailboxes() {
|
||||
for s in servers {
|
||||
jobs.entry(s.sign_key)
|
||||
.or_insert_with(|| (s.clone(), Vec::new()))
|
||||
.1
|
||||
.push(kp.clone());
|
||||
const MAX_ROUNDS: usize = 16;
|
||||
let mut events = Vec::new();
|
||||
let mut wait = wait;
|
||||
for round in 0..MAX_ROUNDS {
|
||||
let r = match self.pull_round(wait).await {
|
||||
Ok(r) => r,
|
||||
// A later round only flushes acks and backlog: keep what we already got.
|
||||
Err(e) if round > 0 => {
|
||||
tracing::warn!("follow-up pull failed: {e}");
|
||||
break;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let got_new = !r.new.is_empty();
|
||||
if got_new {
|
||||
events.extend(self.process(r.new).await?);
|
||||
}
|
||||
// Pull again (without waiting) only to acknowledge what just arrived and to
|
||||
// fetch what servers held back. Redeliveries alone do not loop.
|
||||
if !got_new && !r.more {
|
||||
break;
|
||||
}
|
||||
wait = Duration::ZERO;
|
||||
}
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// One pull per server, carrying the pending acks. New (not yet processed) messages
|
||||
/// are returned and recorded as pending acks; they are saved by [`Self::process`].
|
||||
async fn pull_round(&mut self, wait: Duration) -> Result<PullRound> {
|
||||
// server (by signing key) → (server, mailbox keys)
|
||||
let pending = &self.profile.pending_acks;
|
||||
self.profile.retired_mailboxes.retain(|r| {
|
||||
pending
|
||||
.iter()
|
||||
.any(|a| a.mailbox.as_slice() == r.key.public().as_bytes().as_slice())
|
||||
});
|
||||
let mut jobs: BTreeMap<SignPublic, (ServerInfo, Vec<SigningKeyPair>)> = BTreeMap::new();
|
||||
let retired = self
|
||||
.profile
|
||||
.retired_mailboxes
|
||||
.iter()
|
||||
.map(|r| (r.key.clone(), r.servers.clone()));
|
||||
for (kp, servers) in self.contact_mailboxes().into_iter().chain(retired) {
|
||||
for s in servers {
|
||||
jobs.entry(s.sign_key)
|
||||
.or_insert_with(|| (s.clone(), Vec::new()))
|
||||
.1
|
||||
.push(kp.clone());
|
||||
}
|
||||
}
|
||||
// Drop acks nobody can send any more (contact or server removed).
|
||||
self.profile.pending_acks.retain(|a| {
|
||||
jobs.get(&a.server).is_some_and(|(_, kps)| {
|
||||
kps.iter()
|
||||
.any(|kp| kp.public().as_bytes().as_slice() == a.mailbox.as_slice())
|
||||
})
|
||||
});
|
||||
let mut round = PullRound::default();
|
||||
if jobs.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
return Ok(round);
|
||||
}
|
||||
let wait_s = wait.as_secs().min(u32::MAX as u64) as u32;
|
||||
let calls = jobs.values().map(|(server, kps)| {
|
||||
let pending = &self.profile.pending_acks;
|
||||
let mut sent_acks: Vec<Vec<PendingAck>> = Vec::with_capacity(jobs.len());
|
||||
let mut requests = Vec::with_capacity(jobs.len());
|
||||
for (server, kps) in jobs.values() {
|
||||
let rid = meow_crypto::random_bytes::<16>().to_vec();
|
||||
let ts = now();
|
||||
let auths = kps.iter().map(|kp| rpc::mailbox_auth(kp, server, &rid, ts)).collect();
|
||||
let req = v2::Request {
|
||||
let mut sent = Vec::new();
|
||||
let auths = kps
|
||||
.iter()
|
||||
.map(|kp| {
|
||||
let mb = kp.public();
|
||||
let acks: Vec<Vec<u8>> = pending
|
||||
.iter()
|
||||
.filter(|a| a.server == server.sign_key && a.mailbox.as_slice() == mb.as_bytes().as_slice())
|
||||
.take(MAX_ACKS_PER_MAILBOX)
|
||||
.inspect(|a| sent.push((*a).clone()))
|
||||
.map(|a| a.id.clone())
|
||||
.collect();
|
||||
rpc::mailbox_auth(kp, server, &rid, acks, ts)
|
||||
})
|
||||
.collect();
|
||||
sent_acks.push(sent);
|
||||
requests.push(v2::Request {
|
||||
request_id: rid,
|
||||
timestamp: ts,
|
||||
body: Some(Req::Pull(v2::Pull {
|
||||
auths,
|
||||
wait_seconds: wait_s,
|
||||
})),
|
||||
};
|
||||
self.transport.call_request(server, req, wait)
|
||||
});
|
||||
});
|
||||
}
|
||||
let calls = jobs
|
||||
.values()
|
||||
.zip(requests)
|
||||
.map(|((server, _), req)| self.transport.call_request(server, req, wait));
|
||||
let results = join_all(calls).await;
|
||||
|
||||
let mut stored = Vec::new();
|
||||
let mut failures = 0;
|
||||
let mut last_err = None;
|
||||
for ((server, _), r) in jobs.values().zip(results) {
|
||||
for (((server, _), r), sent) in jobs.values().zip(results).zip(sent_acks) {
|
||||
match r {
|
||||
Ok(Resp::PullResult(p)) => stored.extend(p.messages),
|
||||
Ok(Resp::PullResult(p)) => {
|
||||
// The server applied our acks before answering. Forgetting them
|
||||
// needs no save: at worst they are sent again (acks are idempotent).
|
||||
self.profile.pending_acks.retain(|a| !sent.contains(a));
|
||||
round.more |= p.more;
|
||||
for m in p.messages {
|
||||
let ack = PendingAck {
|
||||
server: server.sign_key,
|
||||
mailbox: m.mailbox.clone(),
|
||||
id: m.id.clone(),
|
||||
};
|
||||
// Already processed (its ack was lost): only re-acknowledge.
|
||||
if !self.profile.pending_acks.contains(&ack) {
|
||||
self.profile.pending_acks.push(ack);
|
||||
round.new.push(m);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(_) => failures += 1,
|
||||
Err(e) => {
|
||||
tracing::warn!("pull on {} failed: {e}", server.url);
|
||||
@@ -571,10 +674,39 @@ impl MeowClient {
|
||||
if failures == jobs.len() {
|
||||
return Err(last_err.unwrap_or_else(|| Error::Network("all servers failed".into())));
|
||||
}
|
||||
self.process(stored).await
|
||||
Ok(round)
|
||||
}
|
||||
|
||||
/// Every mailbox currently used by a contact, with its pull servers.
|
||||
fn contact_mailboxes(&self) -> Vec<(SigningKeyPair, Vec<ServerInfo>)> {
|
||||
self.profile
|
||||
.contacts
|
||||
.iter()
|
||||
.flat_map(|c| c.mailboxes())
|
||||
.map(|(kp, servers)| (kp.clone(), servers.to_vec()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keep the keys of mailboxes that disappeared (`before` → now) while they still
|
||||
/// have pending acks, so the next pull can acknowledge their last messages.
|
||||
fn retire_mailboxes(&mut self, before: Vec<(SigningKeyPair, Vec<ServerInfo>)>) {
|
||||
let current: Vec<SignPublic> = self.contact_mailboxes().iter().map(|(kp, _)| kp.public()).collect();
|
||||
for (key, servers) in before {
|
||||
let pk = key.public();
|
||||
let has_acks = self
|
||||
.profile
|
||||
.pending_acks
|
||||
.iter()
|
||||
.any(|a| a.mailbox.as_slice() == pk.as_bytes().as_slice());
|
||||
let known = self.profile.retired_mailboxes.iter().any(|r| r.key.public() == pk);
|
||||
if has_acks && !known && !current.contains(&pk) {
|
||||
self.profile.retired_mailboxes.push(RetiredMailbox { key, servers });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process(&mut self, stored: Vec<v2::StoredMessage>) -> Result<Vec<Event>> {
|
||||
let mailboxes_before = self.contact_mailboxes();
|
||||
// Handshake messages first (step 2 before step 3), then ratchet messages by arrival.
|
||||
let mut envs: Vec<(u32, u64, Vec<u8>, v2::PeerEnvelope)> = stored
|
||||
.into_iter()
|
||||
@@ -686,6 +818,7 @@ impl MeowClient {
|
||||
_ => tracing::debug!("ignoring unsupported envelope"),
|
||||
}
|
||||
}
|
||||
self.retire_mailboxes(mailboxes_before);
|
||||
// Persist before anything leaves the device.
|
||||
self.save().await?;
|
||||
for (servers, deposit) in outgoing {
|
||||
|
||||
@@ -9,11 +9,12 @@ use std::path::Path;
|
||||
use meow_core::contact::{Contact, ContactId};
|
||||
use meow_core::server::ServerInfo;
|
||||
use meow_crypto::password::{self, KdfParams};
|
||||
use meow_crypto::{SignPublic, SigningKeyPair};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
const FORMAT_VERSION: u32 = 1;
|
||||
const FORMAT_VERSION: u32 = 2;
|
||||
|
||||
/// One entry of a conversation history.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -33,6 +34,24 @@ pub enum Content {
|
||||
File { name: String, size: u64 },
|
||||
}
|
||||
|
||||
/// A message received from a server and processed (its effects are in this profile),
|
||||
/// whose deletion the server has not confirmed yet. Sent with the next pull to that
|
||||
/// server; also used to ignore a redelivery of the same message.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct PendingAck {
|
||||
pub server: SignPublic,
|
||||
pub mailbox: Vec<u8>,
|
||||
pub id: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A mailbox no longer in use (temporary invitation mailbox, removed contact) that
|
||||
/// still has pending acks: its key is kept just long enough to send them.
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct RetiredMailbox {
|
||||
pub key: SigningKeyPair,
|
||||
pub servers: Vec<ServerInfo>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct Profile {
|
||||
pub format: u32,
|
||||
@@ -40,6 +59,8 @@ pub struct Profile {
|
||||
pub servers: Vec<ServerInfo>,
|
||||
pub contacts: Vec<Contact>,
|
||||
pub history: Vec<HistoryEntry>,
|
||||
pub pending_acks: Vec<PendingAck>,
|
||||
pub retired_mailboxes: Vec<RetiredMailbox>,
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
@@ -50,6 +71,8 @@ impl Profile {
|
||||
servers: vec![],
|
||||
contacts: vec![],
|
||||
history: vec![],
|
||||
pending_acks: vec![],
|
||||
retired_mailboxes: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -180,3 +180,160 @@ async fn file_and_server_invitations_then_messaging() {
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
// ───────────── at-least-once delivery (roadmap lot 3.8) ─────────────
|
||||
|
||||
/// Memory store whose acknowledgements can be "lost" on demand.
|
||||
struct FlakyStore {
|
||||
inner: MemoryStore,
|
||||
drop_acks: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl meow_server::store::ServerStore for FlakyStore {
|
||||
async fn post(&self, mb: [u8; 32], payload: Vec<u8>, now: u64) -> meow_server::store::StoreResult<()> {
|
||||
self.inner.post(mb, payload, now).await
|
||||
}
|
||||
async fn fetch(
|
||||
&self,
|
||||
mbs: &[[u8; 32]],
|
||||
limit: usize,
|
||||
) -> meow_server::store::StoreResult<Vec<meow_proto::v2::StoredMessage>> {
|
||||
self.inner.fetch(mbs, limit).await
|
||||
}
|
||||
async fn ack(&self, mb: [u8; 32], ids: &[Vec<u8>]) -> meow_server::store::StoreResult<()> {
|
||||
if self.drop_acks.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
self.inner.ack(mb, ids).await
|
||||
}
|
||||
async fn mark_request(&self, id: &[u8], ttl: Duration) -> meow_server::store::StoreResult<bool> {
|
||||
self.inner.mark_request(id, ttl).await
|
||||
}
|
||||
async fn put_invitation(
|
||||
&self,
|
||||
payload: Vec<u8>,
|
||||
password: &str,
|
||||
ttl: Duration,
|
||||
code_len: usize,
|
||||
) -> meow_server::store::StoreResult<String> {
|
||||
self.inner.put_invitation(payload, password, ttl, code_len).await
|
||||
}
|
||||
async fn get_invitation(&self, code: &str, password: &str) -> meow_server::store::StoreResult<Vec<u8>> {
|
||||
self.inner.get_invitation(code, password).await
|
||||
}
|
||||
}
|
||||
|
||||
/// What a client that crashed right after receiving the response would have done:
|
||||
/// pull without ever acknowledging.
|
||||
async fn raw_pull(
|
||||
server: &meow_core::server::ServerInfo,
|
||||
mailbox: &meow_crypto::SigningKeyPair,
|
||||
) -> Vec<meow_proto::v2::StoredMessage> {
|
||||
use meow_proto::v2::{self, request::Body as Req, response::Body as Resp};
|
||||
let rid = meow_crypto::random_bytes::<16>().to_vec();
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let req = v2::Request {
|
||||
request_id: rid.clone(),
|
||||
timestamp: ts,
|
||||
body: Some(Req::Pull(v2::Pull {
|
||||
auths: vec![meow_core::rpc::mailbox_auth(mailbox, server, &rid, vec![], ts)],
|
||||
wait_seconds: 0,
|
||||
})),
|
||||
};
|
||||
let t = meow_client::transport::Transport::new().unwrap();
|
||||
match t.call_request(server, req, Duration::ZERO).await.unwrap() {
|
||||
Resp::PullResult(p) => p.messages,
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn no_message_lost_when_the_client_dies() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let keys = meow_core::server::ServerKeys::generate();
|
||||
let (notify, _) = broadcast::channel(1024);
|
||||
let store = Arc::new(FlakyStore {
|
||||
inner: MemoryStore::new(notify.clone(), 1000),
|
||||
drop_acks: false.into(),
|
||||
});
|
||||
let limits = Limits {
|
||||
max_pull_messages: 2, // force the "more" path
|
||||
..Limits::default()
|
||||
};
|
||||
let state = Arc::new(AppState {
|
||||
card: encode_card(&keys, &url, "flaky", ""),
|
||||
keys,
|
||||
store: store.clone(),
|
||||
notify,
|
||||
limits,
|
||||
});
|
||||
tokio::spawn(meow_server::serve(listener, state));
|
||||
|
||||
let dir = tmpdir();
|
||||
let servers = vec![url];
|
||||
let mut bob = client(&dir, "bob", &servers).await;
|
||||
let mut alice = client(&dir, "alice", &servers).await;
|
||||
let offer = bob.create_invitation("alice", &[], "", 3600).await.unwrap();
|
||||
let answer = alice.answer_invitation(&offer, None, &[]).await.unwrap();
|
||||
let confirm = bob.accept_answer(&answer).await.unwrap();
|
||||
alice.finalize_invitation(&confirm).await.unwrap();
|
||||
bob.sync(Duration::ZERO).await.unwrap();
|
||||
|
||||
let server = alice.servers()[0].clone();
|
||||
let mailbox = alice.contact("bob").unwrap().keys.as_ref().unwrap().mailbox.clone();
|
||||
|
||||
// 1. A client receives messages then dies before saving: nothing is lost.
|
||||
for t in ["m1", "m2", "m3"] {
|
||||
bob.send_text("alice", t).await.unwrap();
|
||||
}
|
||||
assert_eq!(raw_pull(&server, &mailbox).await.len(), 2); // limit 2, more = true
|
||||
let ev = alice.sync(Duration::ZERO).await.unwrap();
|
||||
assert_eq!(texts(&ev), vec!["m1", "m2", "m3"]); // several rounds ("more")
|
||||
// Everything processed has been acknowledged and deleted.
|
||||
assert!(raw_pull(&server, &mailbox).await.is_empty());
|
||||
assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty());
|
||||
|
||||
// 2. The acknowledgement is lost: the redelivered message is not processed twice,
|
||||
// even after a restart (pending acks are persisted with the state).
|
||||
store.drop_acks.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
bob.send_text("alice", "m4").await.unwrap();
|
||||
assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["m4"]);
|
||||
assert_eq!(raw_pull(&server, &mailbox).await.len(), 1);
|
||||
let path = dir.join("alice.mwid2");
|
||||
drop(alice);
|
||||
let mut alice = MeowClient::open(&path, "pw", KdfParams::INSECURE_TEST).await.unwrap();
|
||||
assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty());
|
||||
store.drop_acks.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty());
|
||||
assert!(raw_pull(&server, &mailbox).await.is_empty());
|
||||
assert_eq!(alice.history("bob").unwrap().len(), 4);
|
||||
assert_eq!(store.inner.stored_messages(), 0);
|
||||
|
||||
// 3. Mailboxes that disappear still get their last messages acknowledged:
|
||||
// the temporary answer mailbox of a server invitation...
|
||||
let mut carol = client(&dir, "carol", &servers).await;
|
||||
let link = bob.publish_invitation("carol", &[], "", "", 3600).await.unwrap();
|
||||
carol.accept_published_invitation(&link, "", None, &[]).await.unwrap();
|
||||
bob.sync(Duration::ZERO).await.unwrap(); // step 2 → 3, answer mailbox retired
|
||||
carol.sync(Duration::ZERO).await.unwrap(); // step 3 → 4
|
||||
bob.sync(Duration::ZERO).await.unwrap(); // ack
|
||||
carol.send_text("bob", "hi").await.unwrap();
|
||||
assert_eq!(texts(&bob.sync(Duration::ZERO).await.unwrap()), vec!["hi"]);
|
||||
assert_eq!(store.inner.stored_messages(), 0);
|
||||
// ...and a contact removed while an ack is still pending.
|
||||
store.drop_acks.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
bob.send_text("alice", "bye").await.unwrap();
|
||||
assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["bye"]);
|
||||
store.drop_acks.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
alice.remove_contact("bob").await.unwrap();
|
||||
assert_eq!(store.inner.stored_messages(), 1);
|
||||
assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty());
|
||||
assert_eq!(store.inner.stored_messages(), 0);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
Reference in new issue
Block a user