From 006bedc7281387e92ab5011c86ff92c211fb38d9 Mon Sep 17 00:00:00 2001 From: yc Date: Fri, 20 Nov 2026 22:00:54 +0100 Subject: [PATCH] message saved if client crashes on retreival --- ARCHITECTURE.md | 45 ++++++-- Cargo.lock | 1 + ROADMAP.md | 7 +- crates/meow-client/Cargo.toml | 1 + crates/meow-client/src/lib.rs | 175 ++++++++++++++++++++++++++---- crates/meow-client/src/profile.rs | 25 ++++- crates/meow-client/tests/flow.rs | 157 +++++++++++++++++++++++++++ crates/meow-core/src/rpc.rs | 38 +++++-- crates/meow-server/src/lib.rs | 42 ++++++- crates/meow-server/src/store.rs | 162 ++++++++++++++++++++++----- proto/meow.proto | 10 +- 11 files changed, 591 insertions(+), 72 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6f88a5c..4744004 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -160,13 +160,42 @@ sautées sont bornées (`MAX_SKIP = 1000`). Le chiffrement des en-têtes est pr ### 3.5 Authentification des pulls ``` -MailboxAuth { mailbox, timestamp, signature } +MailboxAuth { mailbox, timestamp, signature, acks[] } signature = Ed25519_sign(mailbox_sk, - "meow/v2/pull" ‖ server_sign_pk ‖ mailbox ‖ timestamp ‖ request_id) + "meow/v2/pull" ‖ server_sign_pk ‖ mailbox ‖ timestamp + ‖ SHA-256(acks, préfixés par leur longueur) ‖ request_id) ``` Le serveur rejette si `|now - timestamp| > 300 s`, si la signature est invalide ou si -`request_id` a déjà été vu (cache anti-rejeu avec TTL de 600 s). +`request_id` a déjà été vu (cache anti-rejeu avec TTL de 600 s). Les `acks` étant signés, seul +le propriétaire d'une mailbox peut en supprimer les messages. + +### 3.6 Livraison au moins une fois + +Un pull **ne supprime rien**. Chaque `StoredMessage` porte un `id` attribué par le serveur, et +reste stocké jusqu'à ce qu'un pull ultérieur l'acquitte dans `MailboxAuth.acks`. + +``` +client serveur + Pull{auths(acks=[])} ─────▶ fetch (sans supprimer) + ◀───── PullResult{messages[id], more} + traitement, puis enregistrement de + l'état ET des acks en attente + Pull{auths(acks=[ids]), wait=0} ─────▶ ack(ids), puis fetch + ◀───── PullResult{…} +``` + +* Côté client, les acks en attente (`PendingAck {server, mailbox, id}`) sont enregistrés + **dans la même sauvegarde** que l'état du ratchet. Un message redélivré dont l'id est encore + en attente n'est donc pas retraité : il est simplement acquitté de nouveau. +* `sync(wait)` relance un tour sans attente tant qu'il reçoit du nouveau ou que le serveur + signale `more` (limites de 256 messages et 16 Mio par pull). Un `sync` se termine donc avec + ses acks envoyés. +* Une mailbox qui disparaît alors qu'elle a encore des acks en attente (boîte de réponse + temporaire d'une invitation, contact supprimé) est « mise en retraite » : sa clé est conservée + le temps d'envoyer ces acks. +* Effacer les acks envoyés ne demande pas de sauvegarde, car un ack est idempotent : au pire, + il est renvoyé. --- @@ -183,16 +212,18 @@ Les opérations (`oneof` de `Request`) : | Requête | Effet | |---|---| | `Post { deposits[] }` | Stocke chaque payload dans `mailbox`, avec un TTL. Retourne les ids serveur. | -| `Pull { auths[], wait_seconds }` | Vide les mailbox authentifiées. Long-poll si elles sont vides (≤ 120 s). | +| `Pull { auths[], wait_seconds }` | Supprime d'abord les messages acquittés (`auths[].acks`), puis renvoie sans les supprimer les messages restants des mailbox authentifiées (`more` si limité). Long-poll si elles sont vides (≤ 120 s). Voir §3.6. | | `InvitationPublish { payload, ttl, shortcode_len, password }` | Retourne `shortcode` et `expiry`. | | `InvitationFetch { shortcode, password }` | 3 échecs → invitation détruite. Comparaison en temps constant. | | `Ping` | Retourne l'heure du serveur (synchronisation des horodatages). | Stockage (`ServerStore`, trait asynchrone) : -* `MemoryStore` : `HashMap` + `tokio::sync::Notify`, pour le dev et les tests. -* `RedisStore` : `LIST mb:{hex}` (avec `EXPIRE`), `PUBLISH mbch:{hex}`, `SET inv:{code} EX`, - `SET invpw:{code}`, `INCR invfa:{code}`, `SET replay:{id} NX EX`. +* `MemoryStore` : `HashMap` + canal `broadcast`, pour le dev et les tests. +* `RedisStore` (Redis ≥ 5) : `meow:v2:mb:{hex}` est un **stream** (`XADD MAXLEN`, `XRANGE` pour + lire, `XDEL` pour acquitter ; l'id du stream sert d'id de message) avec `EXPIRE`, + `PUBLISH meow:v2:new`, `SET inv:{code} EX`, `SET invpw:{code}`, `INCR invfa:{code}`, + `SET rq:{id} NX EX`. --- diff --git a/Cargo.lock b/Cargo.lock index 8ef59f3..b841842 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1092,6 +1092,7 @@ dependencies = [ name = "meow-client" version = "0.2.0" dependencies = [ + "async-trait", "futures-util", "meow-core", "meow-crypto", diff --git a/ROADMAP.md b/ROADMAP.md index 79aaf6d..a8f19cb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,7 +14,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les ### Ordre conseillé pour la suite -1. **3.8** livraison au moins une fois (correction de fond, petit lot). +1. ~~**3.8** livraison au moins une fois~~ ✅ 2. **4.1 → 4.5** `meow-store` (prérequis du mobile). 3. **5.5, 5.6, 5.9** file d'envoi, accusés, flux d'événements (prérequis de l'UI). 4. **7.1 → 7.3** FFI et intégration dans bastet. @@ -24,6 +24,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les | Date | Lots | Notes | |---|---|---| +| 2026-11-20 | 3.8 | Pull non destructif + acks signés dans `MailboxAuth`, `PendingAck` enregistrés avec l'état, redélivrances dédupliquées, pagination (`more`), mise en retraite des mailbox disparues. Redis : liste → stream. Profil client au format 2 (les `.mwid2` de format 1 sont refusés). 31 tests, e2e mémoire et Redis verts, et plus aucun message résiduel dans Redis à la fin de l'e2e. | | 2026-11-20 | 0.x, 1.1–1.4, 2.1–2.4, 3.1–3.4, 4.0, 5.1–5.4, 6.1–6.5 | Squelette complet et fonctionnel. 29 tests Rust + `tests/e2e.sh` (≈ 60 appels CLI, 2 serveurs, mémoire et Redis, bascule quand un serveur tombe). Décision : protocole v2 sans compatibilité v1. | --- @@ -68,7 +69,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les | 3.5 | Limites : taille de corps, nombre de dépôts par requête, quota par mailbox, rate limiting par IP | Tests | 🟡 taille de corps et nombre de dépôts faits | | 3.6 | Packaging : Dockerfile multi-stage, unité systemd, deb/rpm (`cargo-deb`, `cargo-generate-rpm`) | Image < 30 Mo | ⬜ | | 3.7 | Statistiques agrégées `/v2/stats` (sans métadonnées par utilisateur) | | ⬜ | -| 3.8 | **Livraison au moins une fois** : aujourd'hui `Pull` *vide* la mailbox (comme en v1), donc un message est perdu si le client tombe entre la réponse et la sauvegarde. Il faut un pull qui renvoie des ids, puis un `Ack { ids }` au pull suivant pour supprimer, avec déduplication côté client par id de message | Test : coupure réseau simulée sans perte | ⬜ **prioritaire** | +| 3.8 | **Livraison au moins une fois** : aujourd'hui `Pull` *vide* la mailbox (comme en v1), donc un message est perdu si le client tombe entre la réponse et la sauvegarde. Il faut un pull qui renvoie des ids, puis un `Ack { ids }` au pull suivant pour supprimer, avec déduplication côté client par id de message | Test : coupure réseau simulée sans perte | ✅ (`no_message_lost_when_the_client_dies`) | | 3.9 | Chiffrer l'offre publiée sur le serveur avec une clé placée dans le fragment de l'URL (`…/inv/#`) : le serveur ne verrait plus le nom ni le message de l'initiateur | Test | ⬜ | ## Phase 4 — `meow-store` (persistance client) @@ -92,7 +93,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les | 5.4 | Réception : un pull par serveur regroupant toutes les mailbox, long-poll, traitement, événements | e2e | ✅ | | 5.5 | File d'envoi persistante avec retries et backoff, envoi hors ligne | Tests | ⬜ | | 5.6 | Accusés de réception et de lecture chiffrés (dans le ratchet) | e2e | ⬜ | -| 5.7 | Fichiers : découpage en morceaux, reprise, déduplication, limite de taille | e2e (`send-file`) | 🟡 envoi en un seul morceau | +| 5.7 | Fichiers : découpage en morceaux, reprise, déduplication, limite de taille. Le contenu reçu doit être écrit sur disque **avant** l'ack : aujourd'hui, la CLI écrit les fichiers après `sync`, alors que les acks sont déjà partis | e2e (`send-file`) | 🟡 envoi en un seul morceau | | 5.8 | Trafic leurre : mailbox fictives dans les pulls, ordre aléatoire, padding des requêtes | Tests | ⬜ | | 5.9 | Stream d'événements (`tokio::broadcast`) et tâche de synchronisation en arrière-plan | | ⬜ | diff --git a/crates/meow-client/Cargo.toml b/crates/meow-client/Cargo.toml index 617286b..5dd5c50 100644 --- a/crates/meow-client/Cargo.toml +++ b/crates/meow-client/Cargo.toml @@ -18,6 +18,7 @@ thiserror = "2" tracing = "0.1" [dev-dependencies] +async-trait = "0.1" meow-server = { path = "../meow-server" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net"] } diff --git a/crates/meow-client/src/lib.rs b/crates/meow-client/src/lib.rs index 65dccfd..f1d0e17 100644 --- a/crates/meow-client/src/lib.rs +++ b/crates/meow-client/src/lib.rs @@ -28,7 +28,8 @@ use meow_proto::v2::{self, request::Body as Req, response::Body as Resp}; use meow_proto::Message; pub use meow_core::contact::ContactState as State; -pub use profile::{Content, HistoryEntry}; +use profile::RetiredMailbox; +pub use profile::{Content, HistoryEntry, PendingAck}; /// Maximum file size sent in a single message (chunking: roadmap 5.7). pub const MAX_FILE_BYTES: usize = 4 * 1024 * 1024; @@ -109,6 +110,17 @@ pub struct SendReport { } /// One identity, with its servers, contacts and history. +/// Acknowledged ids sent per mailbox and pull (server limit: 1024). +const MAX_ACKS_PER_MAILBOX: usize = 512; + +#[derive(Default)] +struct PullRound { + /// Messages to process. + new: Vec, + /// A server held messages back. + more: bool, +} + pub struct MeowClient { path: PathBuf, password: String, @@ -268,8 +280,10 @@ impl MeowClient { pub async fn remove_contact(&mut self, name: &str) -> Result<()> { let i = self.contact_index(name)?; + let before = self.contact_mailboxes(); let c = self.profile.contacts.remove(i); self.profile.history.retain(|h| h.contact != c.id); + self.retire_mailboxes(before); self.save().await } @@ -521,45 +535,134 @@ impl MeowClient { /// Pull every mailbox on every server (one request per server), process what /// arrived and return the resulting events. With `wait > 0` the servers hold the /// request until something arrives or the delay expires (long-poll). + /// + /// Delivery is at-least-once: servers keep messages until they are acknowledged, + /// and a message is acknowledged (in the next pull) only once its effects are saved. + /// After a round that brought messages, `sync` pulls again without waiting, which + /// sends the acknowledgements and fetches what the servers held back. pub async fn sync(&mut self, wait: Duration) -> Result> { - // server (by signing key) → (server, mailbox keys) - let mut jobs: BTreeMap)> = BTreeMap::new(); - for c in &self.profile.contacts { - for (kp, servers) in c.mailboxes() { - for s in servers { - jobs.entry(s.sign_key) - .or_insert_with(|| (s.clone(), Vec::new())) - .1 - .push(kp.clone()); + const MAX_ROUNDS: usize = 16; + let mut events = Vec::new(); + let mut wait = wait; + for round in 0..MAX_ROUNDS { + let r = match self.pull_round(wait).await { + Ok(r) => r, + // A later round only flushes acks and backlog: keep what we already got. + Err(e) if round > 0 => { + tracing::warn!("follow-up pull failed: {e}"); + break; } + Err(e) => return Err(e), + }; + let got_new = !r.new.is_empty(); + if got_new { + events.extend(self.process(r.new).await?); + } + // Pull again (without waiting) only to acknowledge what just arrived and to + // fetch what servers held back. Redeliveries alone do not loop. + if !got_new && !r.more { + break; + } + wait = Duration::ZERO; + } + Ok(events) + } + + /// One pull per server, carrying the pending acks. New (not yet processed) messages + /// are returned and recorded as pending acks; they are saved by [`Self::process`]. + async fn pull_round(&mut self, wait: Duration) -> Result { + // server (by signing key) → (server, mailbox keys) + let pending = &self.profile.pending_acks; + self.profile.retired_mailboxes.retain(|r| { + pending + .iter() + .any(|a| a.mailbox.as_slice() == r.key.public().as_bytes().as_slice()) + }); + let mut jobs: BTreeMap)> = BTreeMap::new(); + let retired = self + .profile + .retired_mailboxes + .iter() + .map(|r| (r.key.clone(), r.servers.clone())); + for (kp, servers) in self.contact_mailboxes().into_iter().chain(retired) { + for s in servers { + jobs.entry(s.sign_key) + .or_insert_with(|| (s.clone(), Vec::new())) + .1 + .push(kp.clone()); } } + // Drop acks nobody can send any more (contact or server removed). + self.profile.pending_acks.retain(|a| { + jobs.get(&a.server).is_some_and(|(_, kps)| { + kps.iter() + .any(|kp| kp.public().as_bytes().as_slice() == a.mailbox.as_slice()) + }) + }); + let mut round = PullRound::default(); if jobs.is_empty() { - return Ok(Vec::new()); + return Ok(round); } let wait_s = wait.as_secs().min(u32::MAX as u64) as u32; - let calls = jobs.values().map(|(server, kps)| { + let pending = &self.profile.pending_acks; + let mut sent_acks: Vec> = Vec::with_capacity(jobs.len()); + let mut requests = Vec::with_capacity(jobs.len()); + for (server, kps) in jobs.values() { let rid = meow_crypto::random_bytes::<16>().to_vec(); let ts = now(); - let auths = kps.iter().map(|kp| rpc::mailbox_auth(kp, server, &rid, ts)).collect(); - let req = v2::Request { + let mut sent = Vec::new(); + let auths = kps + .iter() + .map(|kp| { + let mb = kp.public(); + let acks: Vec> = pending + .iter() + .filter(|a| a.server == server.sign_key && a.mailbox.as_slice() == mb.as_bytes().as_slice()) + .take(MAX_ACKS_PER_MAILBOX) + .inspect(|a| sent.push((*a).clone())) + .map(|a| a.id.clone()) + .collect(); + rpc::mailbox_auth(kp, server, &rid, acks, ts) + }) + .collect(); + sent_acks.push(sent); + requests.push(v2::Request { request_id: rid, timestamp: ts, body: Some(Req::Pull(v2::Pull { auths, wait_seconds: wait_s, })), - }; - self.transport.call_request(server, req, wait) - }); + }); + } + let calls = jobs + .values() + .zip(requests) + .map(|((server, _), req)| self.transport.call_request(server, req, wait)); let results = join_all(calls).await; - let mut stored = Vec::new(); let mut failures = 0; let mut last_err = None; - for ((server, _), r) in jobs.values().zip(results) { + for (((server, _), r), sent) in jobs.values().zip(results).zip(sent_acks) { match r { - Ok(Resp::PullResult(p)) => stored.extend(p.messages), + Ok(Resp::PullResult(p)) => { + // The server applied our acks before answering. Forgetting them + // needs no save: at worst they are sent again (acks are idempotent). + self.profile.pending_acks.retain(|a| !sent.contains(a)); + round.more |= p.more; + for m in p.messages { + let ack = PendingAck { + server: server.sign_key, + mailbox: m.mailbox.clone(), + id: m.id.clone(), + }; + // Already processed (its ack was lost): only re-acknowledge. + if !self.profile.pending_acks.contains(&ack) { + self.profile.pending_acks.push(ack); + round.new.push(m); + } + } + } Ok(_) => failures += 1, Err(e) => { tracing::warn!("pull on {} failed: {e}", server.url); @@ -571,10 +674,39 @@ impl MeowClient { if failures == jobs.len() { return Err(last_err.unwrap_or_else(|| Error::Network("all servers failed".into()))); } - self.process(stored).await + Ok(round) + } + + /// Every mailbox currently used by a contact, with its pull servers. + fn contact_mailboxes(&self) -> Vec<(SigningKeyPair, Vec)> { + self.profile + .contacts + .iter() + .flat_map(|c| c.mailboxes()) + .map(|(kp, servers)| (kp.clone(), servers.to_vec())) + .collect() + } + + /// Keep the keys of mailboxes that disappeared (`before` → now) while they still + /// have pending acks, so the next pull can acknowledge their last messages. + fn retire_mailboxes(&mut self, before: Vec<(SigningKeyPair, Vec)>) { + let current: Vec = self.contact_mailboxes().iter().map(|(kp, _)| kp.public()).collect(); + for (key, servers) in before { + let pk = key.public(); + let has_acks = self + .profile + .pending_acks + .iter() + .any(|a| a.mailbox.as_slice() == pk.as_bytes().as_slice()); + let known = self.profile.retired_mailboxes.iter().any(|r| r.key.public() == pk); + if has_acks && !known && !current.contains(&pk) { + self.profile.retired_mailboxes.push(RetiredMailbox { key, servers }); + } + } } async fn process(&mut self, stored: Vec) -> Result> { + let mailboxes_before = self.contact_mailboxes(); // Handshake messages first (step 2 before step 3), then ratchet messages by arrival. let mut envs: Vec<(u32, u64, Vec, v2::PeerEnvelope)> = stored .into_iter() @@ -686,6 +818,7 @@ impl MeowClient { _ => tracing::debug!("ignoring unsupported envelope"), } } + self.retire_mailboxes(mailboxes_before); // Persist before anything leaves the device. self.save().await?; for (servers, deposit) in outgoing { diff --git a/crates/meow-client/src/profile.rs b/crates/meow-client/src/profile.rs index 45a6040..70880f0 100644 --- a/crates/meow-client/src/profile.rs +++ b/crates/meow-client/src/profile.rs @@ -9,11 +9,12 @@ use std::path::Path; use meow_core::contact::{Contact, ContactId}; use meow_core::server::ServerInfo; use meow_crypto::password::{self, KdfParams}; +use meow_crypto::{SignPublic, SigningKeyPair}; use serde::{Deserialize, Serialize}; use crate::{Error, Result}; -const FORMAT_VERSION: u32 = 1; +const FORMAT_VERSION: u32 = 2; /// One entry of a conversation history. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -33,6 +34,24 @@ pub enum Content { File { name: String, size: u64 }, } +/// A message received from a server and processed (its effects are in this profile), +/// whose deletion the server has not confirmed yet. Sent with the next pull to that +/// server; also used to ignore a redelivery of the same message. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct PendingAck { + pub server: SignPublic, + pub mailbox: Vec, + pub id: Vec, +} + +/// A mailbox no longer in use (temporary invitation mailbox, removed contact) that +/// still has pending acks: its key is kept just long enough to send them. +#[derive(Clone, Serialize, Deserialize)] +pub struct RetiredMailbox { + pub key: SigningKeyPair, + pub servers: Vec, +} + #[derive(Clone, Serialize, Deserialize)] pub struct Profile { pub format: u32, @@ -40,6 +59,8 @@ pub struct Profile { pub servers: Vec, pub contacts: Vec, pub history: Vec, + pub pending_acks: Vec, + pub retired_mailboxes: Vec, } impl Profile { @@ -50,6 +71,8 @@ impl Profile { servers: vec![], contacts: vec![], history: vec![], + pending_acks: vec![], + retired_mailboxes: vec![], } } diff --git a/crates/meow-client/tests/flow.rs b/crates/meow-client/tests/flow.rs index 8611f4b..07e18b7 100644 --- a/crates/meow-client/tests/flow.rs +++ b/crates/meow-client/tests/flow.rs @@ -180,3 +180,160 @@ async fn file_and_server_invitations_then_messaging() { let _ = std::fs::remove_dir_all(&dir); } + +// ───────────── at-least-once delivery (roadmap lot 3.8) ───────────── + +/// Memory store whose acknowledgements can be "lost" on demand. +struct FlakyStore { + inner: MemoryStore, + drop_acks: std::sync::atomic::AtomicBool, +} + +#[async_trait::async_trait] +impl meow_server::store::ServerStore for FlakyStore { + async fn post(&self, mb: [u8; 32], payload: Vec, now: u64) -> meow_server::store::StoreResult<()> { + self.inner.post(mb, payload, now).await + } + async fn fetch( + &self, + mbs: &[[u8; 32]], + limit: usize, + ) -> meow_server::store::StoreResult> { + self.inner.fetch(mbs, limit).await + } + async fn ack(&self, mb: [u8; 32], ids: &[Vec]) -> meow_server::store::StoreResult<()> { + if self.drop_acks.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + self.inner.ack(mb, ids).await + } + async fn mark_request(&self, id: &[u8], ttl: Duration) -> meow_server::store::StoreResult { + self.inner.mark_request(id, ttl).await + } + async fn put_invitation( + &self, + payload: Vec, + password: &str, + ttl: Duration, + code_len: usize, + ) -> meow_server::store::StoreResult { + self.inner.put_invitation(payload, password, ttl, code_len).await + } + async fn get_invitation(&self, code: &str, password: &str) -> meow_server::store::StoreResult> { + self.inner.get_invitation(code, password).await + } +} + +/// What a client that crashed right after receiving the response would have done: +/// pull without ever acknowledging. +async fn raw_pull( + server: &meow_core::server::ServerInfo, + mailbox: &meow_crypto::SigningKeyPair, +) -> Vec { + use meow_proto::v2::{self, request::Body as Req, response::Body as Resp}; + let rid = meow_crypto::random_bytes::<16>().to_vec(); + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let req = v2::Request { + request_id: rid.clone(), + timestamp: ts, + body: Some(Req::Pull(v2::Pull { + auths: vec![meow_core::rpc::mailbox_auth(mailbox, server, &rid, vec![], ts)], + wait_seconds: 0, + })), + }; + let t = meow_client::transport::Transport::new().unwrap(); + match t.call_request(server, req, Duration::ZERO).await.unwrap() { + Resp::PullResult(p) => p.messages, + other => panic!("unexpected {other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn no_message_lost_when_the_client_dies() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let keys = meow_core::server::ServerKeys::generate(); + let (notify, _) = broadcast::channel(1024); + let store = Arc::new(FlakyStore { + inner: MemoryStore::new(notify.clone(), 1000), + drop_acks: false.into(), + }); + let limits = Limits { + max_pull_messages: 2, // force the "more" path + ..Limits::default() + }; + let state = Arc::new(AppState { + card: encode_card(&keys, &url, "flaky", ""), + keys, + store: store.clone(), + notify, + limits, + }); + tokio::spawn(meow_server::serve(listener, state)); + + let dir = tmpdir(); + let servers = vec![url]; + let mut bob = client(&dir, "bob", &servers).await; + let mut alice = client(&dir, "alice", &servers).await; + let offer = bob.create_invitation("alice", &[], "", 3600).await.unwrap(); + let answer = alice.answer_invitation(&offer, None, &[]).await.unwrap(); + let confirm = bob.accept_answer(&answer).await.unwrap(); + alice.finalize_invitation(&confirm).await.unwrap(); + bob.sync(Duration::ZERO).await.unwrap(); + + let server = alice.servers()[0].clone(); + let mailbox = alice.contact("bob").unwrap().keys.as_ref().unwrap().mailbox.clone(); + + // 1. A client receives messages then dies before saving: nothing is lost. + for t in ["m1", "m2", "m3"] { + bob.send_text("alice", t).await.unwrap(); + } + assert_eq!(raw_pull(&server, &mailbox).await.len(), 2); // limit 2, more = true + let ev = alice.sync(Duration::ZERO).await.unwrap(); + assert_eq!(texts(&ev), vec!["m1", "m2", "m3"]); // several rounds ("more") + // Everything processed has been acknowledged and deleted. + assert!(raw_pull(&server, &mailbox).await.is_empty()); + assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty()); + + // 2. The acknowledgement is lost: the redelivered message is not processed twice, + // even after a restart (pending acks are persisted with the state). + store.drop_acks.store(true, std::sync::atomic::Ordering::SeqCst); + bob.send_text("alice", "m4").await.unwrap(); + assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["m4"]); + assert_eq!(raw_pull(&server, &mailbox).await.len(), 1); + let path = dir.join("alice.mwid2"); + drop(alice); + let mut alice = MeowClient::open(&path, "pw", KdfParams::INSECURE_TEST).await.unwrap(); + assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty()); + store.drop_acks.store(false, std::sync::atomic::Ordering::SeqCst); + assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty()); + assert!(raw_pull(&server, &mailbox).await.is_empty()); + assert_eq!(alice.history("bob").unwrap().len(), 4); + assert_eq!(store.inner.stored_messages(), 0); + + // 3. Mailboxes that disappear still get their last messages acknowledged: + // the temporary answer mailbox of a server invitation... + let mut carol = client(&dir, "carol", &servers).await; + let link = bob.publish_invitation("carol", &[], "", "", 3600).await.unwrap(); + carol.accept_published_invitation(&link, "", None, &[]).await.unwrap(); + bob.sync(Duration::ZERO).await.unwrap(); // step 2 → 3, answer mailbox retired + carol.sync(Duration::ZERO).await.unwrap(); // step 3 → 4 + bob.sync(Duration::ZERO).await.unwrap(); // ack + carol.send_text("bob", "hi").await.unwrap(); + assert_eq!(texts(&bob.sync(Duration::ZERO).await.unwrap()), vec!["hi"]); + assert_eq!(store.inner.stored_messages(), 0); + // ...and a contact removed while an ack is still pending. + store.drop_acks.store(true, std::sync::atomic::Ordering::SeqCst); + bob.send_text("alice", "bye").await.unwrap(); + assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["bye"]); + store.drop_acks.store(false, std::sync::atomic::Ordering::SeqCst); + alice.remove_contact("bob").await.unwrap(); + assert_eq!(store.inner.stored_messages(), 1); + assert!(alice.sync(Duration::ZERO).await.unwrap().is_empty()); + assert_eq!(store.inner.stored_messages(), 0); + + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/meow-core/src/rpc.rs b/crates/meow-core/src/rpc.rs index 01ff24a..eb022aa 100644 --- a/crates/meow-core/src/rpc.rs +++ b/crates/meow-core/src/rpc.rs @@ -94,26 +94,44 @@ pub fn seal_response(key: &Key, resp: &v2::Response) -> Vec { .encode_to_vec() } -fn pull_message(server_sign: &SignPublic, mailbox: &SignPublic, ts: u64, request_id: &[u8]) -> Vec { +fn pull_message( + server_sign: &SignPublic, + mailbox: &SignPublic, + ts: u64, + acks: &[Vec], + request_id: &[u8], +) -> Vec { + let ack_refs: Vec<&[u8]> = acks.iter().map(Vec::as_slice).collect(); + let acks_hash = meow_crypto::sha256(&ack_refs); [ PULL_LABEL, server_sign.as_bytes(), mailbox.as_bytes(), &ts.to_be_bytes(), + &acks_hash, request_id, ] .concat() } -/// Client: prove ownership of a mailbox for one pull request. -pub fn mailbox_auth(mailbox: &SigningKeyPair, server: &ServerInfo, request_id: &[u8], now: u64) -> v2::MailboxAuth { +/// Client: prove ownership of a mailbox for one pull request, and acknowledge +/// (delete) the messages `acks` received by an earlier pull. +pub fn mailbox_auth( + mailbox: &SigningKeyPair, + server: &ServerInfo, + request_id: &[u8], + acks: Vec>, + now: u64, +) -> v2::MailboxAuth { let pk = mailbox.public(); + let signature = mailbox + .sign(&pull_message(&server.sign_key, &pk, now, &acks, request_id)) + .to_vec(); v2::MailboxAuth { mailbox: pk.to_vec(), timestamp: now, - signature: mailbox - .sign(&pull_message(&server.sign_key, &pk, now, request_id)) - .to_vec(), + signature, + acks, } } @@ -130,7 +148,7 @@ pub fn verify_mailbox_auth( return Err(Error::Expired); } mailbox.verify( - &pull_message(server_sign, &mailbox, auth.timestamp, request_id), + &pull_message(server_sign, &mailbox, auth.timestamp, &auth.acks, request_id), &auth.signature, )?; Ok(mailbox) @@ -178,9 +196,13 @@ mod tests { let keys = ServerKeys::generate(); let info = ServerInfo::from_card(&keys.card("http://s", "s", "", 1)).unwrap(); let mb = SigningKeyPair::generate(); - let auth = mailbox_auth(&mb, &info, b"rid", 1000); + let auth = mailbox_auth(&mb, &info, b"rid", vec![b"m1".to_vec()], 1000); let sp = keys.sign.public(); assert_eq!(verify_mailbox_auth(&sp, b"rid", &auth, 1100).unwrap(), mb.public()); + // Acks are covered by the signature. + let mut forged = auth.clone(); + forged.acks.push(b"m2".to_vec()); + assert!(verify_mailbox_auth(&sp, b"rid", &forged, 1100).is_err()); assert!(verify_mailbox_auth(&sp, b"other", &auth, 1100).is_err()); assert!(verify_mailbox_auth(&sp, b"rid", &auth, 1000 + AUTH_WINDOW_SECS + 1).is_err()); let other_server = ServerKeys::generate().sign.public(); diff --git a/crates/meow-server/src/lib.rs b/crates/meow-server/src/lib.rs index 327a04c..381c035 100644 --- a/crates/meow-server/src/lib.rs +++ b/crates/meow-server/src/lib.rs @@ -39,6 +39,12 @@ pub struct Limits { pub max_deposits: usize, pub max_payload_bytes: usize, pub max_mailboxes_per_pull: usize, + /// Messages returned by one pull (the client pulls again when `more` is set). + pub max_pull_messages: usize, + /// Payload bytes returned by one pull (at least one message is always returned). + pub max_pull_bytes: usize, + /// Acknowledged ids per mailbox in one pull. + pub max_acks: usize, pub max_wait: Duration, pub max_invitation_ttl: Duration, pub replay_ttl: Duration, @@ -51,6 +57,9 @@ impl Default for Limits { max_deposits: 128, max_payload_bytes: 8 * 1024 * 1024, max_mailboxes_per_pull: 512, + max_pull_messages: 256, + max_pull_bytes: 16 * 1024 * 1024, + max_acks: 1024, max_wait: Duration::from_secs(120), max_invitation_ttl: Duration::from_secs(7 * 86_400), replay_ttl: Duration::from_secs(2 * rpc::AUTH_WINDOW_SECS), @@ -196,13 +205,23 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul let sign = st.keys.sign.public(); let mut mailboxes = Vec::with_capacity(p.auths.len()); for a in &p.auths { + if a.acks.len() > st.limits.max_acks || a.acks.iter().any(|id| id.len() > 64) { + return Err((Code::BadRequest, "bad acks")); + } let mb = rpc::verify_mailbox_auth(&sign, request_id, a, now) .map_err(|_| (Code::Unauthorized, "bad mailbox auth"))?; mailboxes.push(mb.0); } - // Subscribe before draining, so a deposit landing in between is not missed. + // Acknowledged messages were persisted by the client: delete them first. + for (mb, a) in mailboxes.iter().zip(&p.auths) { + if !a.acks.is_empty() { + st.store.ack(*mb, &a.acks).await.map_err(store_err)?; + } + } + // Subscribe before reading, so a deposit landing in between is not missed. let mut rx = st.notify.subscribe(); - let mut messages = st.store.drain(&mailboxes).await.map_err(store_err)?; + let limit = st.limits.max_pull_messages; + let mut messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?; let wait = Duration::from_secs(p.wait_seconds as u64).min(st.limits.max_wait); if messages.is_empty() && !wait.is_zero() { let wanted: HashSet = mailboxes.iter().copied().collect(); @@ -217,9 +236,24 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul } }) .await; - messages = st.store.drain(&mailboxes).await.map_err(store_err)?; + messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?; } - Ok(Resp::PullResult(v2::PullResult { messages })) + // Honour the count and size budgets; always return at least one message. + let mut more = messages.len() > limit; + messages.truncate(limit); + let mut bytes = 0usize; + let keep = messages + .iter() + .position(|m| { + bytes += m.payload.len(); + bytes > st.limits.max_pull_bytes + }) + .map_or(messages.len(), |i| i.max(1)); + if keep < messages.len() { + messages.truncate(keep); + more = true; + } + Ok(Resp::PullResult(v2::PullResult { messages, more })) } /// Serve on an already bound listener until the future is dropped. diff --git a/crates/meow-server/src/store.rs b/crates/meow-server/src/store.rs index dd88bad..0cf9b29 100644 --- a/crates/meow-server/src/store.rs +++ b/crates/meow-server/src/store.rs @@ -41,10 +41,13 @@ pub const MAX_INVITATION_ATTEMPTS: i64 = 3; #[async_trait] pub trait ServerStore: Send + Sync { - /// Append a payload to a mailbox and notify waiters. + /// Append a payload to a mailbox (under a fresh id) and notify waiters. async fn post(&self, mailbox: Mailbox, payload: Vec, now: u64) -> StoreResult<()>; - /// Atomically take every message of the given mailboxes. - async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult>; + /// Read, oldest first and **without removing them**, at most `limit` messages of the + /// given mailboxes. Messages stay until acknowledged with [`ServerStore::ack`]. + async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult>; + /// Delete the given messages of a mailbox. Unknown ids are ignored (idempotent). + async fn ack(&self, mailbox: Mailbox, ids: &[Vec]) -> StoreResult<()>; /// Record a request id; returns `false` if it was already seen (replay). async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult; /// Store an invitation under a fresh shortcode. @@ -90,6 +93,7 @@ struct MemInvitation { #[derive(Default)] struct MemInner { mailboxes: HashMap>, + next_id: u64, requests: HashMap, Instant>, invitations: HashMap, } @@ -110,6 +114,11 @@ impl MemoryStore { } } + /// Number of messages currently stored (all mailboxes). + pub fn stored_messages(&self) -> usize { + self.lock().map_or(0, |g| g.mailboxes.values().map(VecDeque::len).sum()) + } + fn lock(&self) -> StoreResult> { self.inner .lock() @@ -122,11 +131,14 @@ impl ServerStore for MemoryStore { async fn post(&self, mailbox: Mailbox, payload: Vec, now: u64) -> StoreResult<()> { { let mut g = self.lock()?; + g.next_id += 1; + let id = g.next_id.to_be_bytes().to_vec(); let q = g.mailboxes.entry(mailbox).or_default(); q.push_back(v2::StoredMessage { mailbox: mailbox.to_vec(), payload, received_at: now, + id, }); while q.len() > self.max_per_mailbox { q.pop_front(); @@ -136,15 +148,28 @@ impl ServerStore for MemoryStore { Ok(()) } - async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult> { - let mut g = self.lock()?; + async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult> { + let g = self.lock()?; Ok(mailboxes .iter() - .filter_map(|m| g.mailboxes.remove(m)) + .filter_map(|m| g.mailboxes.get(m)) .flatten() + .take(limit) + .cloned() .collect()) } + async fn ack(&self, mailbox: Mailbox, ids: &[Vec]) -> StoreResult<()> { + let mut g = self.lock()?; + if let Some(q) = g.mailboxes.get_mut(&mailbox) { + q.retain(|m| !ids.contains(&m.id)); + if q.is_empty() { + g.mailboxes.remove(&mailbox); + } + } + Ok(()) + } + async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult { let mut g = self.lock()?; let now = Instant::now(); @@ -207,7 +232,8 @@ pub const NOTIFY_CHANNEL: &str = "meow:v2:new"; /// Redis-backed store. /// -/// Keys: `meow:v2:mb:{hex}` (list of encoded `StoredMessage`), `meow:v2:rq:{hex}` +/// Keys: `meow:v2:mb:{hex}` (stream; the stream id is the message id, field `m` holds the +/// encoded `StoredMessage`), `meow:v2:rq:{hex}` /// (replay cache), `meow:v2:inv:{code}`, `meow:v2:invpw:{code}`, `meow:v2:invfa:{code}`. pub struct RedisStore { conn: redis::aio::MultiplexedConnection, @@ -219,6 +245,22 @@ fn hexs(b: &[u8]) -> String { b.iter().map(|x| format!("{x:02x}")).collect() } +fn mailbox_key(mb: &Mailbox) -> String { + format!("meow:v2:mb:{}", hexs(mb)) +} + +fn is_stream_id(s: &str) -> bool { + match s.split_once('-') { + Some((ms, seq)) => { + !ms.is_empty() + && !seq.is_empty() + && (ms.len() + seq.len()) <= 40 + && (ms.bytes().chain(seq.bytes())).all(|b| b.is_ascii_digit()) + } + None => false, + } +} + impl RedisStore { /// Connect and start the notification forwarder. pub async fn connect( @@ -252,20 +294,24 @@ impl RedisStore { #[async_trait] impl ServerStore for RedisStore { async fn post(&self, mailbox: Mailbox, payload: Vec, now: u64) -> StoreResult<()> { - let key = format!("meow:v2:mb:{}", hexs(&mailbox)); + let key = mailbox_key(&mailbox); let item = v2::StoredMessage { mailbox: mailbox.to_vec(), payload, received_at: now, + id: Vec::new(), } .encode_to_vec(); let mut c = self.conn.clone(); - let max = self.max_per_mailbox as isize; let _: () = redis::pipe() .atomic() - .rpush(&key, item) - .ignore() - .ltrim(&key, -max, -1) + .cmd("XADD") + .arg(&key) + .arg("MAXLEN") + .arg(self.max_per_mailbox) + .arg("*") + .arg("m") + .arg(item) .ignore() .expire(&key, self.mailbox_ttl.as_secs() as i64) .ignore() @@ -276,27 +322,61 @@ impl ServerStore for RedisStore { Ok(()) } - async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult> { + async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult> { let mut out = Vec::new(); let mut c = self.conn.clone(); for mb in mailboxes { - let key = format!("meow:v2:mb:{}", hexs(mb)); - let (items, _): (Vec>, i64) = redis::pipe() - .atomic() - .lrange(&key, 0, -1) - .del(&key) + let left = limit.saturating_sub(out.len()); + if left == 0 { + break; + } + let key = mailbox_key(mb); + let entries: Vec<(String, Vec>)> = redis::cmd("XRANGE") + .arg(&key) + .arg("-") + .arg("+") + .arg("COUNT") + .arg(left) .query_async(&mut c) .await?; - for it in items { - match v2::StoredMessage::decode(it.as_slice()) { - Ok(m) => out.push(m), - Err(e) => tracing::warn!("dropping corrupted stored message: {e}"), + for (id, fields) in entries { + // fields = ["m", item] + let decoded = fields.get(1).map(|it| v2::StoredMessage::decode(it.as_slice())); + match decoded { + Some(Ok(mut m)) => { + m.id = id.into_bytes(); + out.push(m); + } + _ => { + tracing::warn!("dropping corrupted stored message"); + let _: i64 = redis::cmd("XDEL").arg(&key).arg(&id).query_async(&mut c).await?; + } } } } Ok(out) } + async fn ack(&self, mailbox: Mailbox, ids: &[Vec]) -> StoreResult<()> { + // Stream ids are ASCII `ms-seq`; anything else cannot exist and is skipped + // (XDEL would reject the whole command on a malformed id). + let ids: Vec<&str> = ids + .iter() + .filter_map(|i| std::str::from_utf8(i).ok()) + .filter(|i| is_stream_id(i)) + .collect(); + if ids.is_empty() { + return Ok(()); + } + let mut c = self.conn.clone(); + let _: i64 = redis::cmd("XDEL") + .arg(mailbox_key(&mailbox)) + .arg(ids) + .query_async(&mut c) + .await?; + Ok(()) + } + async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult { let mut c = self.conn.clone(); let r: Option = redis::cmd("SET") @@ -380,6 +460,15 @@ impl ServerStore for RedisStore { mod tests { use super::*; + #[test] + fn stream_ids() { + assert!(is_stream_id("1700000000000-0")); + assert!(!is_stream_id("1700000000000")); + assert!(!is_stream_id("-0")); + assert!(!is_stream_id("12-a")); + assert!(!is_stream_id("1-2-3")); + } + #[test] fn shortcodes() { let c = random_shortcode(8); @@ -392,13 +481,34 @@ mod tests { let mb = meow_crypto::random_bytes::<32>(); store.post(mb, b"one".to_vec(), 1).await.unwrap(); store.post(mb, b"two".to_vec(), 2).await.unwrap(); + store.post(mb, b"three".to_vec(), 3).await.unwrap(); assert_eq!(rx.recv().await.unwrap(), mb); - let got = store.drain(&[mb]).await.unwrap(); + let payloads = |v: &[v2::StoredMessage]| v.iter().map(|m| m.payload.clone()).collect::>(); + let got = store.fetch(&[mb], 10).await.unwrap(); assert_eq!( - got.iter().map(|m| m.payload.clone()).collect::>(), - vec![b"one".to_vec(), b"two".to_vec()] + payloads(&got), + vec![b"one".to_vec(), b"two".to_vec(), b"three".to_vec()] ); - assert!(store.drain(&[mb]).await.unwrap().is_empty()); + // Not acknowledged: delivered again. The limit is honoured. + let again = store.fetch(&[mb], 2).await.unwrap(); + assert_eq!(payloads(&again), vec![b"one".to_vec(), b"two".to_vec()]); + assert_eq!(again[0].id, got[0].id); + assert_ne!(got[0].id, got[1].id); + // Ack two of them (plus garbage, which is ignored): only "three" remains. + store + .ack(mb, &[got[0].id.clone(), got[1].id.clone(), b"bogus".to_vec()]) + .await + .unwrap(); + assert_eq!( + payloads(&store.fetch(&[mb], 10).await.unwrap()), + vec![b"three".to_vec()] + ); + // Acks are scoped to their mailbox. + let other = meow_crypto::random_bytes::<32>(); + store.ack(other, &[got[2].id.clone()]).await.unwrap(); + assert_eq!(store.fetch(&[mb], 10).await.unwrap().len(), 1); + store.ack(mb, &[got[2].id.clone()]).await.unwrap(); + assert!(store.fetch(&[mb], 10).await.unwrap().is_empty()); let rid = meow_crypto::random_bytes::<16>(); assert!(store.mark_request(&rid, Duration::from_secs(60)).await.unwrap()); diff --git a/proto/meow.proto b/proto/meow.proto index 36ff50a..60327c5 100644 --- a/proto/meow.proto +++ b/proto/meow.proto @@ -201,7 +201,10 @@ message PostAck { uint32 accepted = 1; } message MailboxAuth { bytes mailbox = 1; uint64 timestamp = 2; - bytes signature = 3; // see ARCHITECTURE.md §3.5 + bytes signature = 3; // see ARCHITECTURE.md §3.5 (covers `acks`) + // Ids of messages of this mailbox, received by an earlier pull and now safely + // persisted by the client: the server deletes them before answering. + repeated bytes acks = 4; } message Pull { repeated MailboxAuth auths = 1; uint32 wait_seconds = 2; } @@ -209,8 +212,11 @@ message StoredMessage { bytes mailbox = 1; bytes payload = 2; uint64 received_at = 3; + bytes id = 4; // server-assigned, opaque, unique within the mailbox } -message PullResult { repeated StoredMessage messages = 1; } +// Messages stay on the server until acknowledged (at-least-once delivery). +// `more`: the server held back messages (count/size limit), pull again. +message PullResult { repeated StoredMessage messages = 1; bool more = 2; } message InvitationPublish { bytes payload = 1; // encoded InvitationOffer