message saved if client crashes on retreival

This commit is contained in:
yves committed 2026-11-20 22:00:54 +01:00
1 parent 6d3ea3e2a6
commit 006bedc728
11 files changed
+591 -72

No files matched your search

+38 -4
View File
@@ -39,6 +39,12 @@ pub struct Limits {
pub max_deposits: usize,
pub max_payload_bytes: usize,
pub max_mailboxes_per_pull: usize,
/// Messages returned by one pull (the client pulls again when `more` is set).
pub max_pull_messages: usize,
/// Payload bytes returned by one pull (at least one message is always returned).
pub max_pull_bytes: usize,
/// Acknowledged ids per mailbox in one pull.
pub max_acks: usize,
pub max_wait: Duration,
pub max_invitation_ttl: Duration,
pub replay_ttl: Duration,
@@ -51,6 +57,9 @@ impl Default for Limits {
max_deposits: 128,
max_payload_bytes: 8 * 1024 * 1024,
max_mailboxes_per_pull: 512,
max_pull_messages: 256,
max_pull_bytes: 16 * 1024 * 1024,
max_acks: 1024,
max_wait: Duration::from_secs(120),
max_invitation_ttl: Duration::from_secs(7 * 86_400),
replay_ttl: Duration::from_secs(2 * rpc::AUTH_WINDOW_SECS),
@@ -196,13 +205,23 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul
let sign = st.keys.sign.public();
let mut mailboxes = Vec::with_capacity(p.auths.len());
for a in &p.auths {
if a.acks.len() > st.limits.max_acks || a.acks.iter().any(|id| id.len() > 64) {
return Err((Code::BadRequest, "bad acks"));
}
let mb = rpc::verify_mailbox_auth(&sign, request_id, a, now)
.map_err(|_| (Code::Unauthorized, "bad mailbox auth"))?;
mailboxes.push(mb.0);
}
// Subscribe before draining, so a deposit landing in between is not missed.
// Acknowledged messages were persisted by the client: delete them first.
for (mb, a) in mailboxes.iter().zip(&p.auths) {
if !a.acks.is_empty() {
st.store.ack(*mb, &a.acks).await.map_err(store_err)?;
}
}
// Subscribe before reading, so a deposit landing in between is not missed.
let mut rx = st.notify.subscribe();
let mut messages = st.store.drain(&mailboxes).await.map_err(store_err)?;
let limit = st.limits.max_pull_messages;
let mut messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?;
let wait = Duration::from_secs(p.wait_seconds as u64).min(st.limits.max_wait);
if messages.is_empty() && !wait.is_zero() {
let wanted: HashSet<Mailbox> = mailboxes.iter().copied().collect();
@@ -217,9 +236,24 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul
}
})
.await;
messages = st.store.drain(&mailboxes).await.map_err(store_err)?;
messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?;
}
Ok(Resp::PullResult(v2::PullResult { messages }))
// Honour the count and size budgets; always return at least one message.
let mut more = messages.len() > limit;
messages.truncate(limit);
let mut bytes = 0usize;
let keep = messages
.iter()
.position(|m| {
bytes += m.payload.len();
bytes > st.limits.max_pull_bytes
})
.map_or(messages.len(), |i| i.max(1));
if keep < messages.len() {
messages.truncate(keep);
more = true;
}
Ok(Resp::PullResult(v2::PullResult { messages, more }))
}
/// Serve on an already bound listener until the future is dropped.
+136 -26
View File
@@ -41,10 +41,13 @@ pub const MAX_INVITATION_ATTEMPTS: i64 = 3;
#[async_trait]
pub trait ServerStore: Send + Sync {
/// Append a payload to a mailbox and notify waiters.
/// Append a payload to a mailbox (under a fresh id) and notify waiters.
async fn post(&self, mailbox: Mailbox, payload: Vec<u8>, now: u64) -> StoreResult<()>;
/// Atomically take every message of the given mailboxes.
async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult<Vec<v2::StoredMessage>>;
/// Read, oldest first and **without removing them**, at most `limit` messages of the
/// given mailboxes. Messages stay until acknowledged with [`ServerStore::ack`].
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>>;
/// Delete the given messages of a mailbox. Unknown ids are ignored (idempotent).
async fn ack(&self, mailbox: Mailbox, ids: &[Vec<u8>]) -> StoreResult<()>;
/// Record a request id; returns `false` if it was already seen (replay).
async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult<bool>;
/// Store an invitation under a fresh shortcode.
@@ -90,6 +93,7 @@ struct MemInvitation {
#[derive(Default)]
struct MemInner {
mailboxes: HashMap<Mailbox, VecDeque<v2::StoredMessage>>,
next_id: u64,
requests: HashMap<Vec<u8>, Instant>,
invitations: HashMap<String, MemInvitation>,
}
@@ -110,6 +114,11 @@ impl MemoryStore {
}
}
/// Number of messages currently stored (all mailboxes).
pub fn stored_messages(&self) -> usize {
self.lock().map_or(0, |g| g.mailboxes.values().map(VecDeque::len).sum())
}
fn lock(&self) -> StoreResult<std::sync::MutexGuard<'_, MemInner>> {
self.inner
.lock()
@@ -122,11 +131,14 @@ impl ServerStore for MemoryStore {
async fn post(&self, mailbox: Mailbox, payload: Vec<u8>, now: u64) -> StoreResult<()> {
{
let mut g = self.lock()?;
g.next_id += 1;
let id = g.next_id.to_be_bytes().to_vec();
let q = g.mailboxes.entry(mailbox).or_default();
q.push_back(v2::StoredMessage {
mailbox: mailbox.to_vec(),
payload,
received_at: now,
id,
});
while q.len() > self.max_per_mailbox {
q.pop_front();
@@ -136,15 +148,28 @@ impl ServerStore for MemoryStore {
Ok(())
}
async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult<Vec<v2::StoredMessage>> {
let mut g = self.lock()?;
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
let g = self.lock()?;
Ok(mailboxes
.iter()
.filter_map(|m| g.mailboxes.remove(m))
.filter_map(|m| g.mailboxes.get(m))
.flatten()
.take(limit)
.cloned()
.collect())
}
async fn ack(&self, mailbox: Mailbox, ids: &[Vec<u8>]) -> StoreResult<()> {
let mut g = self.lock()?;
if let Some(q) = g.mailboxes.get_mut(&mailbox) {
q.retain(|m| !ids.contains(&m.id));
if q.is_empty() {
g.mailboxes.remove(&mailbox);
}
}
Ok(())
}
async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult<bool> {
let mut g = self.lock()?;
let now = Instant::now();
@@ -207,7 +232,8 @@ pub const NOTIFY_CHANNEL: &str = "meow:v2:new";
/// Redis-backed store.
///
/// Keys: `meow:v2:mb:{hex}` (list of encoded `StoredMessage`), `meow:v2:rq:{hex}`
/// Keys: `meow:v2:mb:{hex}` (stream; the stream id is the message id, field `m` holds the
/// encoded `StoredMessage`), `meow:v2:rq:{hex}`
/// (replay cache), `meow:v2:inv:{code}`, `meow:v2:invpw:{code}`, `meow:v2:invfa:{code}`.
pub struct RedisStore {
conn: redis::aio::MultiplexedConnection,
@@ -219,6 +245,22 @@ fn hexs(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn mailbox_key(mb: &Mailbox) -> String {
format!("meow:v2:mb:{}", hexs(mb))
}
fn is_stream_id(s: &str) -> bool {
match s.split_once('-') {
Some((ms, seq)) => {
!ms.is_empty()
&& !seq.is_empty()
&& (ms.len() + seq.len()) <= 40
&& (ms.bytes().chain(seq.bytes())).all(|b| b.is_ascii_digit())
}
None => false,
}
}
impl RedisStore {
/// Connect and start the notification forwarder.
pub async fn connect(
@@ -252,20 +294,24 @@ impl RedisStore {
#[async_trait]
impl ServerStore for RedisStore {
async fn post(&self, mailbox: Mailbox, payload: Vec<u8>, now: u64) -> StoreResult<()> {
let key = format!("meow:v2:mb:{}", hexs(&mailbox));
let key = mailbox_key(&mailbox);
let item = v2::StoredMessage {
mailbox: mailbox.to_vec(),
payload,
received_at: now,
id: Vec::new(),
}
.encode_to_vec();
let mut c = self.conn.clone();
let max = self.max_per_mailbox as isize;
let _: () = redis::pipe()
.atomic()
.rpush(&key, item)
.ignore()
.ltrim(&key, -max, -1)
.cmd("XADD")
.arg(&key)
.arg("MAXLEN")
.arg(self.max_per_mailbox)
.arg("*")
.arg("m")
.arg(item)
.ignore()
.expire(&key, self.mailbox_ttl.as_secs() as i64)
.ignore()
@@ -276,27 +322,61 @@ impl ServerStore for RedisStore {
Ok(())
}
async fn drain(&self, mailboxes: &[Mailbox]) -> StoreResult<Vec<v2::StoredMessage>> {
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
let mut out = Vec::new();
let mut c = self.conn.clone();
for mb in mailboxes {
let key = format!("meow:v2:mb:{}", hexs(mb));
let (items, _): (Vec<Vec<u8>>, i64) = redis::pipe()
.atomic()
.lrange(&key, 0, -1)
.del(&key)
let left = limit.saturating_sub(out.len());
if left == 0 {
break;
}
let key = mailbox_key(mb);
let entries: Vec<(String, Vec<Vec<u8>>)> = redis::cmd("XRANGE")
.arg(&key)
.arg("-")
.arg("+")
.arg("COUNT")
.arg(left)
.query_async(&mut c)
.await?;
for it in items {
match v2::StoredMessage::decode(it.as_slice()) {
Ok(m) => out.push(m),
Err(e) => tracing::warn!("dropping corrupted stored message: {e}"),
for (id, fields) in entries {
// fields = ["m", item]
let decoded = fields.get(1).map(|it| v2::StoredMessage::decode(it.as_slice()));
match decoded {
Some(Ok(mut m)) => {
m.id = id.into_bytes();
out.push(m);
}
_ => {
tracing::warn!("dropping corrupted stored message");
let _: i64 = redis::cmd("XDEL").arg(&key).arg(&id).query_async(&mut c).await?;
}
}
}
}
Ok(out)
}
async fn ack(&self, mailbox: Mailbox, ids: &[Vec<u8>]) -> StoreResult<()> {
// Stream ids are ASCII `ms-seq`; anything else cannot exist and is skipped
// (XDEL would reject the whole command on a malformed id).
let ids: Vec<&str> = ids
.iter()
.filter_map(|i| std::str::from_utf8(i).ok())
.filter(|i| is_stream_id(i))
.collect();
if ids.is_empty() {
return Ok(());
}
let mut c = self.conn.clone();
let _: i64 = redis::cmd("XDEL")
.arg(mailbox_key(&mailbox))
.arg(ids)
.query_async(&mut c)
.await?;
Ok(())
}
async fn mark_request(&self, request_id: &[u8], ttl: Duration) -> StoreResult<bool> {
let mut c = self.conn.clone();
let r: Option<String> = redis::cmd("SET")
@@ -380,6 +460,15 @@ impl ServerStore for RedisStore {
mod tests {
use super::*;
#[test]
fn stream_ids() {
assert!(is_stream_id("1700000000000-0"));
assert!(!is_stream_id("1700000000000"));
assert!(!is_stream_id("-0"));
assert!(!is_stream_id("12-a"));
assert!(!is_stream_id("1-2-3"));
}
#[test]
fn shortcodes() {
let c = random_shortcode(8);
@@ -392,13 +481,34 @@ mod tests {
let mb = meow_crypto::random_bytes::<32>();
store.post(mb, b"one".to_vec(), 1).await.unwrap();
store.post(mb, b"two".to_vec(), 2).await.unwrap();
store.post(mb, b"three".to_vec(), 3).await.unwrap();
assert_eq!(rx.recv().await.unwrap(), mb);
let got = store.drain(&[mb]).await.unwrap();
let payloads = |v: &[v2::StoredMessage]| v.iter().map(|m| m.payload.clone()).collect::<Vec<_>>();
let got = store.fetch(&[mb], 10).await.unwrap();
assert_eq!(
got.iter().map(|m| m.payload.clone()).collect::<Vec<_>>(),
vec![b"one".to_vec(), b"two".to_vec()]
payloads(&got),
vec![b"one".to_vec(), b"two".to_vec(), b"three".to_vec()]
);
assert!(store.drain(&[mb]).await.unwrap().is_empty());
// Not acknowledged: delivered again. The limit is honoured.
let again = store.fetch(&[mb], 2).await.unwrap();
assert_eq!(payloads(&again), vec![b"one".to_vec(), b"two".to_vec()]);
assert_eq!(again[0].id, got[0].id);
assert_ne!(got[0].id, got[1].id);
// Ack two of them (plus garbage, which is ignored): only "three" remains.
store
.ack(mb, &[got[0].id.clone(), got[1].id.clone(), b"bogus".to_vec()])
.await
.unwrap();
assert_eq!(
payloads(&store.fetch(&[mb], 10).await.unwrap()),
vec![b"three".to_vec()]
);
// Acks are scoped to their mailbox.
let other = meow_crypto::random_bytes::<32>();
store.ack(other, &[got[2].id.clone()]).await.unwrap();
assert_eq!(store.fetch(&[mb], 10).await.unwrap().len(), 1);
store.ack(mb, &[got[2].id.clone()]).await.unwrap();
assert!(store.fetch(&[mb], 10).await.unwrap().is_empty());
let rid = meow_crypto::random_bytes::<16>();
assert!(store.mark_request(&rid, Duration::from_secs(60)).await.unwrap());