bg check ajustments
This commit is contained in:
1 parent
7eed94d82c
commit
923f4e397e
14 files changed
+649
-46
No files matched your search
@@ -205,13 +205,18 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul
|
||||
let sign = st.keys.sign.public();
|
||||
let mut mailboxes = Vec::with_capacity(p.auths.len());
|
||||
for a in &p.auths {
|
||||
if a.acks.len() > st.limits.max_acks || a.acks.iter().any(|id| id.len() > 64) {
|
||||
if a.acks.len() > st.limits.max_acks || a.acks.iter().any(|id| id.len() > 64) || a.after.len() > 64 {
|
||||
return Err((Code::BadRequest, "bad acks"));
|
||||
}
|
||||
let mb = rpc::verify_mailbox_auth(&sign, request_id, a, now)
|
||||
.map_err(|_| (Code::Unauthorized, "bad mailbox auth"))?;
|
||||
mailboxes.push(mb.0);
|
||||
}
|
||||
let cursors: Vec<(Mailbox, Vec<u8>)> = mailboxes
|
||||
.iter()
|
||||
.zip(&p.auths)
|
||||
.map(|(m, a)| (*m, a.after.clone()))
|
||||
.collect();
|
||||
// Acknowledged messages were persisted by the client: delete them first.
|
||||
for (mb, a) in mailboxes.iter().zip(&p.auths) {
|
||||
if !a.acks.is_empty() {
|
||||
@@ -221,7 +226,7 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul
|
||||
// Subscribe before reading, so a deposit landing in between is not missed.
|
||||
let mut rx = st.notify.subscribe();
|
||||
let limit = st.limits.max_pull_messages;
|
||||
let mut messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?;
|
||||
let mut messages = st.store.fetch(&cursors, limit + 1).await.map_err(store_err)?;
|
||||
let wait = Duration::from_secs(p.wait_seconds as u64).min(st.limits.max_wait);
|
||||
if messages.is_empty() && !wait.is_zero() {
|
||||
let wanted: HashSet<Mailbox> = mailboxes.iter().copied().collect();
|
||||
@@ -236,7 +241,7 @@ async fn pull(st: &AppState, request_id: &[u8], p: v2::Pull, now: u64) -> HResul
|
||||
}
|
||||
})
|
||||
.await;
|
||||
messages = st.store.fetch(&mailboxes, limit + 1).await.map_err(store_err)?;
|
||||
messages = st.store.fetch(&cursors, limit + 1).await.map_err(store_err)?;
|
||||
}
|
||||
// Honour the count and size budgets; always return at least one message.
|
||||
let mut more = messages.len() > limit;
|
||||
|
||||
@@ -44,8 +44,10 @@ pub trait ServerStore: Send + Sync {
|
||||
/// Append a payload to a mailbox (under a fresh id) and notify waiters.
|
||||
async fn post(&self, mailbox: Mailbox, payload: Vec<u8>, now: u64) -> StoreResult<()>;
|
||||
/// Read, oldest first and **without removing them**, at most `limit` messages of the
|
||||
/// given mailboxes. Messages stay until acknowledged with [`ServerStore::ack`].
|
||||
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>>;
|
||||
/// given mailboxes, each with an optional cursor: only messages stored after the
|
||||
/// message whose id it is (empty: from the start). Messages stay until acknowledged
|
||||
/// with [`ServerStore::ack`]. Ids grow over time within a mailbox.
|
||||
async fn fetch(&self, mailboxes: &[(Mailbox, Vec<u8>)], limit: usize) -> StoreResult<Vec<v2::StoredMessage>>;
|
||||
/// Delete the given messages of a mailbox. Unknown ids are ignored (idempotent).
|
||||
async fn ack(&self, mailbox: Mailbox, ids: &[Vec<u8>]) -> StoreResult<()>;
|
||||
/// Record a request id; returns `false` if it was already seen (replay).
|
||||
@@ -131,7 +133,11 @@ impl ServerStore for MemoryStore {
|
||||
async fn post(&self, mailbox: Mailbox, payload: Vec<u8>, now: u64) -> StoreResult<()> {
|
||||
{
|
||||
let mut g = self.lock()?;
|
||||
g.next_id += 1;
|
||||
// Time-based so that ids keep growing across restarts (watchers keep cursors).
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_millis() as u64);
|
||||
g.next_id = (now_ms << 16).max(g.next_id + 1);
|
||||
let id = g.next_id.to_be_bytes().to_vec();
|
||||
let q = g.mailboxes.entry(mailbox).or_default();
|
||||
q.push_back(v2::StoredMessage {
|
||||
@@ -148,12 +154,12 @@ impl ServerStore for MemoryStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
|
||||
async fn fetch(&self, mailboxes: &[(Mailbox, Vec<u8>)], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
|
||||
let g = self.lock()?;
|
||||
Ok(mailboxes
|
||||
.iter()
|
||||
.filter_map(|m| g.mailboxes.get(m))
|
||||
.flatten()
|
||||
.filter_map(|(m, after)| g.mailboxes.get(m).map(|q| (q, after)))
|
||||
.flat_map(|(q, after)| q.iter().filter(move |msg| after.is_empty() || msg.id > *after))
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect())
|
||||
@@ -322,18 +328,23 @@ impl ServerStore for RedisStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch(&self, mailboxes: &[Mailbox], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
|
||||
async fn fetch(&self, mailboxes: &[(Mailbox, Vec<u8>)], limit: usize) -> StoreResult<Vec<v2::StoredMessage>> {
|
||||
let mut out = Vec::new();
|
||||
let mut c = self.conn.clone();
|
||||
for mb in mailboxes {
|
||||
for (mb, after) in mailboxes {
|
||||
let left = limit.saturating_sub(out.len());
|
||||
if left == 0 {
|
||||
break;
|
||||
}
|
||||
let key = mailbox_key(mb);
|
||||
// Exclusive start "(<id>" (Redis ≥ 6.2); an invalid cursor reads from the start.
|
||||
let start = match std::str::from_utf8(after) {
|
||||
Ok(a) if is_stream_id(a) => format!("({a}"),
|
||||
_ => "-".to_string(),
|
||||
};
|
||||
let entries: Vec<(String, Vec<Vec<u8>>)> = redis::cmd("XRANGE")
|
||||
.arg(&key)
|
||||
.arg("-")
|
||||
.arg(start)
|
||||
.arg("+")
|
||||
.arg("COUNT")
|
||||
.arg(left)
|
||||
@@ -484,13 +495,17 @@ mod tests {
|
||||
store.post(mb, b"three".to_vec(), 3).await.unwrap();
|
||||
assert_eq!(rx.recv().await.unwrap(), mb);
|
||||
let payloads = |v: &[v2::StoredMessage]| v.iter().map(|m| m.payload.clone()).collect::<Vec<_>>();
|
||||
let got = store.fetch(&[mb], 10).await.unwrap();
|
||||
let got = store.fetch(&[(mb, vec![])], 10).await.unwrap();
|
||||
assert_eq!(
|
||||
payloads(&got),
|
||||
vec![b"one".to_vec(), b"two".to_vec(), b"three".to_vec()]
|
||||
);
|
||||
// A cursor only returns what came after it.
|
||||
let after_first = store.fetch(&[(mb, got[0].id.clone())], 10).await.unwrap();
|
||||
assert_eq!(payloads(&after_first), vec![b"two".to_vec(), b"three".to_vec()]);
|
||||
assert!(store.fetch(&[(mb, got[2].id.clone())], 10).await.unwrap().is_empty());
|
||||
// Not acknowledged: delivered again. The limit is honoured.
|
||||
let again = store.fetch(&[mb], 2).await.unwrap();
|
||||
let again = store.fetch(&[(mb, vec![])], 2).await.unwrap();
|
||||
assert_eq!(payloads(&again), vec![b"one".to_vec(), b"two".to_vec()]);
|
||||
assert_eq!(again[0].id, got[0].id);
|
||||
assert_ne!(got[0].id, got[1].id);
|
||||
@@ -500,15 +515,15 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
payloads(&store.fetch(&[mb], 10).await.unwrap()),
|
||||
payloads(&store.fetch(&[(mb, vec![])], 10).await.unwrap()),
|
||||
vec![b"three".to_vec()]
|
||||
);
|
||||
// Acks are scoped to their mailbox.
|
||||
let other = meow_crypto::random_bytes::<32>();
|
||||
store.ack(other, &[got[2].id.clone()]).await.unwrap();
|
||||
assert_eq!(store.fetch(&[mb], 10).await.unwrap().len(), 1);
|
||||
assert_eq!(store.fetch(&[(mb, vec![])], 10).await.unwrap().len(), 1);
|
||||
store.ack(mb, &[got[2].id.clone()]).await.unwrap();
|
||||
assert!(store.fetch(&[mb], 10).await.unwrap().is_empty());
|
||||
assert!(store.fetch(&[(mb, vec![])], 10).await.unwrap().is_empty());
|
||||
|
||||
let rid = meow_crypto::random_bytes::<16>();
|
||||
assert!(store.mark_request(&rid, Duration::from_secs(60)).await.unwrap());
|
||||
|
||||
Reference in new issue
Block a user