background task
This commit is contained in:
1 parent
47b73e6935
commit
ce44ee48b3
7 files changed
+477
-63
No files matched your search
@@ -285,6 +285,25 @@ let mut events = client.subscribe(); // Strea
|
|||||||
* Le store applique une mise à jour de statut dans le segment qui contient le message, en
|
* Le store applique une mise à jour de statut dans le segment qui contient le message, en
|
||||||
partant du plus récent. Seuls les segments modifiés sont réécrits.
|
partant du plus récent. Seuls les segments modifiés sont réécrits.
|
||||||
|
|
||||||
|
### 5.3 Tâche de fond et flux d'événements
|
||||||
|
|
||||||
|
* `client.subscribe()` renvoie un `broadcast::Receiver<Event>`. Tout événement y passe, qu'il
|
||||||
|
vienne de `sync`, de `flush_outbox` ou de la tâche de fond. Un abonné qui prend plus de 256
|
||||||
|
événements de retard reçoit `Lagged` et doit recharger sa vue depuis le client.
|
||||||
|
* `client.start(BackgroundOptions)` déplace le client dans une tâche `tokio` et renvoie un
|
||||||
|
`ClientHandle` qui offre `subscribe()`, `lock()` (accès complet au client) et `stop()`
|
||||||
|
(rend le client).
|
||||||
|
* Boucle de la tâche : vider les dépôts dus, puis faire un long-poll sur tous les serveurs, puis
|
||||||
|
traiter et publier. Après une réception, la tâche repart aussitôt, sans attente, pour envoyer
|
||||||
|
les acks et les réponses. En cas d'échec de tous les serveurs : backoff de 2 s à 60 s.
|
||||||
|
* **Le verrou n'est jamais tenu pendant une attente réseau.** Le pull et le vidage de la file
|
||||||
|
sont découpés en trois temps : `prepare_*` sous verrou, appels réseau sans verrou, `apply_*`
|
||||||
|
sous verrou. Un envoi depuis l'UI n'attend donc ni un long-poll ni un serveur injoignable.
|
||||||
|
* Le long-poll est raccourci jusqu'au prochain réessai dû de la file : un dépôt en attente
|
||||||
|
repart à l'heure, même sans autre trafic.
|
||||||
|
* Arrêter la tâche abandonne le long-poll en cours. Ses acks n'ont pas été confirmés : ils
|
||||||
|
seront simplement renvoyés (ils sont idempotents).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. Stockage client (`meow-store`)
|
## 6. Stockage client (`meow-store`)
|
||||||
|
|||||||
+4
-3
@@ -16,7 +16,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les
|
|||||||
|
|
||||||
1. ~~**3.8** livraison au moins une fois~~ ✅
|
1. ~~**3.8** livraison au moins une fois~~ ✅
|
||||||
2. ~~**4.5** brancher `meow-store` dans le client et la CLI~~ ✅
|
2. ~~**4.5** brancher `meow-store` dans le client et la CLI~~ ✅
|
||||||
3. ~~5.5, 5.6~~ ✅, puis **5.9** flux d'événements (prérequis de l'UI), et **5.10**.
|
3. ~~5.5, 5.6, 5.9~~ ✅, puis **5.10** (inviter dans un espace caché).
|
||||||
4. **7.1 → 7.3** FFI et intégration dans bastet.
|
4. **7.1 → 7.3** FFI et intégration dans bastet.
|
||||||
5. Le reste (phases 8–10) selon les priorités.
|
5. Le reste (phases 8–10) selon les priorités.
|
||||||
|
|
||||||
@@ -24,6 +24,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les
|
|||||||
|
|
||||||
| Date | Lots | Notes |
|
| Date | Lots | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| 2026-11-20 | 5.9 | `subscribe()` (broadcast de tous les événements), `start()` → `ClientHandle { subscribe, lock, stop }`. Pull et vidage de la file découpés en préparation, réseau, application, pour ne jamais tenir le verrou pendant le réseau. Long-poll raccourci jusqu'au prochain réessai, backoff en cas de panne. `process` ne vide plus la file lui-même (c'est `sync` ou la tâche qui le fait). 44 tests, dont un test de tâche de fond (événement poussé, envoi immédiat pendant un long-poll de 20 s, accusé de lecture, réessai automatique, arrêt). |
|
||||||
| 2026-11-20 | 5.6 | `Status` par message (`Queued`, `Sent`, `Delivered`, `Read`, sans retour en arrière), mis à jour dans le segment qui contient le message. Accusés `DELIVERED` (un par contact et par lot) et `READ` (`mark_read`) dans le ratchet, désactivables par identité (`Settings`). `Event::Receipt`. CLI : `read`, `identity receipts`, statuts dans `history`, `longpoll --exit` ignore les lots ne contenant que des accusés. Correction d'un test de store instable (motif de 3 octets trouvé par hasard dans l'aléa). 43 tests, e2e avec une section « accusés ». |
|
| 2026-11-20 | 5.6 | `Status` par message (`Queued`, `Sent`, `Delivered`, `Read`, sans retour en arrière), mis à jour dans le segment qui contient le message. Accusés `DELIVERED` (un par contact et par lot) et `READ` (`mark_read`) dans le ratchet, désactivables par identité (`Settings`). `Event::Receipt`. CLI : `read`, `identity receipts`, statuts dans `history`, `longpoll --exit` ignore les lots ne contenant que des accusés. Correction d'un test de store instable (motif de 3 octets trouvé par hasard dans l'aléa). 43 tests, e2e avec une section « accusés ». |
|
||||||
| 2026-11-20 | 5.5 | `OutboxItem` enregistrés dans le store (un objet par élément et par espace), dans la même sauvegarde que le ratchet. Backoff exponentiel avec aléa, expiration après 7 jours (`Event::SendFailed`), dépôts regroupés par serveur avec repli élément par élément. Les messages de handshake passent aussi par la file. CLI : `outbox [--flush]`, « queued for … ». ARCHITECTURE : §5.1, et §7 (réseau de serveurs et Matriochka, points à trancher). 41 tests, e2e avec une section « file hors ligne ». |
|
| 2026-11-20 | 5.5 | `OutboxItem` enregistrés dans le store (un objet par élément et par espace), dans la même sauvegarde que le ratchet. Backoff exponentiel avec aléa, expiration après 7 jours (`Event::SendFailed`), dépôts regroupés par serveur avec repli élément par élément. Les messages de handshake passent aussi par la file. CLI : `outbox [--flush]`, « queued for … ». ARCHITECTURE : §5.1, et §7 (réseau de serveurs et Matriochka, points à trancher). 41 tests, e2e avec une section « file hors ligne ». |
|
||||||
| 2026-11-20 | 4.5 | `meow-client` passe sur `meow-store` (un état par espace, sauvegarde des seuls espaces modifiés, acks par espace) ; API `unlock`, `hide_contact`, `reveal_contact`, `change_password`, `history_page`, `file`. CLI : `-u`, `contact hide/reveal`, `identity password`. Store derrière un `Mutex` pour garder le client `Send + Sync` (requis pour la FFI). Le fichier `.mwid2` provisoire disparaît. 38 tests, e2e avec sections « contact caché » et « changement de mot de passe ». |
|
| 2026-11-20 | 4.5 | `meow-client` passe sur `meow-store` (un état par espace, sauvegarde des seuls espaces modifiés, acks par espace) ; API `unlock`, `hide_contact`, `reveal_contact`, `change_password`, `history_page`, `file`. CLI : `-u`, `contact hide/reveal`, `identity password`. Store derrière un `Mutex` pour garder le client `Send + Sync` (requis pour la FFI). Le fichier `.mwid2` provisoire disparaît. 38 tests, e2e avec sections « contact caché » et « changement de mot de passe ». |
|
||||||
@@ -100,7 +101,7 @@ Pour reprendre le travail : lire ce fichier, prendre le premier lot ⬜ dont les
|
|||||||
| 5.6 | Accusés de réception et de lecture chiffrés (dans le ratchet) | e2e | ✅ |
|
| 5.6 | Accusés de réception et de lecture chiffrés (dans le ratchet) | e2e | ✅ |
|
||||||
| 5.7 | Fichiers : découpage en morceaux, reprise, déduplication, limite de taille. Le contenu reçu doit être écrit sur disque **avant** l'ack : aujourd'hui, la CLI écrit les fichiers après `sync`, alors que les acks sont déjà partis | e2e (`send-file`) | 🟡 envoi en un seul morceau |
|
| 5.7 | Fichiers : découpage en morceaux, reprise, déduplication, limite de taille. Le contenu reçu doit être écrit sur disque **avant** l'ack : aujourd'hui, la CLI écrit les fichiers après `sync`, alors que les acks sont déjà partis | e2e (`send-file`) | 🟡 envoi en un seul morceau |
|
||||||
| 5.8 | Trafic leurre : mailbox fictives dans les pulls, ordre aléatoire, padding des requêtes | Tests | ⬜ |
|
| 5.8 | Trafic leurre : mailbox fictives dans les pulls, ordre aléatoire, padding des requêtes | Tests | ⬜ |
|
||||||
| 5.9 | Stream d'événements (`tokio::broadcast`) et tâche de synchronisation en arrière-plan | | ⬜ |
|
| 5.9 | Stream d'événements (`tokio::broadcast`) et tâche de synchronisation en arrière-plan | Tests | ✅ |
|
||||||
| 5.10 | Inviter directement dans un espace caché (le contact ne passe jamais par l'espace principal) | Tests | ⬜ |
|
| 5.10 | Inviter directement dans un espace caché (le contact ne passe jamais par l'espace principal) | Tests | ⬜ |
|
||||||
|
|
||||||
## Phase 6 — `meow-cli` (`meow`)
|
## Phase 6 — `meow-cli` (`meow`)
|
||||||
@@ -176,7 +177,7 @@ Correspondance des 63 exports actuels de `meowjsonlib` → API v2 : voir l'annex
|
|||||||
| `ContactsList`, `ContactGet`, `ContactUpdate`, `ContactRemove`, `ContactNameExists` | `client.contacts().list/get/update/remove/find_by_name` |
|
| `ContactsList`, `ContactGet`, `ContactUpdate`, `ContactRemove`, `ContactNameExists` | `client.contacts().list/get/update/remove/find_by_name` |
|
||||||
| `InvitationFile*`, `InvitationMessage*`, `InvitationCheck` | `client.invitations().create/answer/accept/finalize/inspect` (octets indépendants du transport) |
|
| `InvitationFile*`, `InvitationMessage*`, `InvitationCheck` | `client.invitations().create/answer/accept/finalize/inspect` (octets indépendants du transport) |
|
||||||
| `InvitationServer*` (8 fonctions) | `client.invitations().publish(offer, opts)` / `fetch(url)` : 2 appels |
|
| `InvitationServer*` (8 fonctions) | `client.invitations().publish(offer, opts)` / `fetch(url)` : 2 appels |
|
||||||
| `PollServer`, `LongPollAllServerJobs`, `SaveCheckJobs`, `ConsumeInboxFile` | `client.sync(wait)` + `client.subscribe()` (plus de fichiers inbox) |
|
| `PollServer`, `LongPollAllServerJobs`, `SaveCheckJobs`, `ConsumeInboxFile` | `client.start(opts)` → `handle.subscribe()` (ou `client.sync(wait)` ponctuel) ; plus de fichiers inbox |
|
||||||
| `CreateUserMessageAndSendJob`, `ProcessSendQueues`, `ProcessSentMessages`, `CreateAndStoreUserMessage`, `SendMessage` | `client.messages().send(contact, Outgoing{…})` (la file d'envoi est interne) |
|
| `CreateUserMessageAndSendJob`, `ProcessSendQueues`, `ProcessSentMessages`, `CreateAndStoreUserMessage`, `SendMessage` | `client.messages().send(contact, Outgoing{…})` (la file d'envoi est interne) |
|
||||||
| `LoadMessagesHistory`, `GetMessagePreview`, `GetFilePreview`, `MarkMessageProcessed` | `client.messages().history(contact, page)`, `.mark_read(id)` |
|
| `LoadMessagesHistory`, `GetMessagePreview`, `GetFilePreview`, `MarkMessageProcessed` | `client.messages().history(contact, page)`, `.mark_read(id)` |
|
||||||
| `Test*`, `TestDb`, `ActivateRemoteLogger` | supprimés |
|
| `Test*`, `TestDb`, `ActivateRemoteLogger` | supprimés |
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ meow-proto = { path = "../meow-proto" }
|
|||||||
meow-crypto = { path = "../meow-crypto" }
|
meow-crypto = { path = "../meow-crypto" }
|
||||||
meow-store = { path = "../meow-store" }
|
meow-store = { path = "../meow-store" }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
||||||
tokio = { version = "1", features = ["fs", "time", "macros"] }
|
tokio = { version = "1", features = ["fs", "time", "macros", "sync", "rt"] }
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
//! Background synchronisation (roadmap lot 5.9).
|
||||||
|
//!
|
||||||
|
//! [`MeowClient::start`] moves the client into a task that long-polls every server,
|
||||||
|
//! processes what arrives, sends queued deposits when they are due and publishes every
|
||||||
|
//! [`Event`] on the client's broadcast channel. The application keeps full access to
|
||||||
|
//! the client through [`ClientHandle::lock`].
|
||||||
|
//!
|
||||||
|
//! Network waits never hold the lock: each step is *prepared* under the lock, the
|
||||||
|
//! network calls run without it, and the results are *applied* under the lock. A send
|
||||||
|
//! from the UI therefore never waits for a 30 s long-poll or an unreachable server.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tokio::sync::{broadcast, watch, Mutex, MutexGuard};
|
||||||
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
|
use crate::outbox::execute_flush;
|
||||||
|
use crate::{execute_pull, Event, MeowClient};
|
||||||
|
|
||||||
|
/// Pace of the background task.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub struct BackgroundOptions {
|
||||||
|
/// How long each pull may wait on the servers for new messages. It also bounds how
|
||||||
|
/// late a queued deposit can be retried.
|
||||||
|
pub long_poll: Duration,
|
||||||
|
/// First delay after a failed round (all servers unreachable); doubled up to
|
||||||
|
/// [`BackgroundOptions::max_backoff`].
|
||||||
|
pub backoff: Duration,
|
||||||
|
pub max_backoff: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for BackgroundOptions {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
long_poll: Duration::from_secs(30),
|
||||||
|
backoff: Duration::from_secs(2),
|
||||||
|
max_backoff: Duration::from_secs(60),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A client running in the background.
|
||||||
|
pub struct ClientHandle {
|
||||||
|
client: Arc<Mutex<MeowClient>>,
|
||||||
|
events: broadcast::Sender<Event>,
|
||||||
|
stop: watch::Sender<bool>,
|
||||||
|
task: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MeowClient {
|
||||||
|
/// Start background synchronisation. Subscribe with [`ClientHandle::subscribe`];
|
||||||
|
/// use the client through [`ClientHandle::lock`]; get it back with
|
||||||
|
/// [`ClientHandle::stop`].
|
||||||
|
pub fn start(self, options: BackgroundOptions) -> ClientHandle {
|
||||||
|
let events = self.events.clone();
|
||||||
|
let client = Arc::new(Mutex::new(self));
|
||||||
|
let (stop, stop_rx) = watch::channel(false);
|
||||||
|
let task = tokio::spawn(run(client.clone(), options, stop_rx));
|
||||||
|
ClientHandle {
|
||||||
|
client,
|
||||||
|
events,
|
||||||
|
stop,
|
||||||
|
task,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClientHandle {
|
||||||
|
/// Receive every event from now on.
|
||||||
|
pub fn subscribe(&self) -> broadcast::Receiver<Event> {
|
||||||
|
self.events.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exclusive access to the client (sending, contacts, history…). Keep it short:
|
||||||
|
/// the background task needs the lock between its network calls.
|
||||||
|
pub async fn lock(&self) -> MutexGuard<'_, MeowClient> {
|
||||||
|
self.client.lock().await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop the background task (an in-flight long-poll is abandoned; its acks will
|
||||||
|
/// simply be sent again) and give the client back.
|
||||||
|
pub async fn stop(self) -> MeowClient {
|
||||||
|
let _ = self.stop.send(true);
|
||||||
|
if let Err(e) = self.task.await {
|
||||||
|
tracing::error!("background task failed: {e}");
|
||||||
|
}
|
||||||
|
match Arc::try_unwrap(self.client) {
|
||||||
|
Ok(m) => m.into_inner(),
|
||||||
|
// Unreachable: the task is finished and the handle was consumed.
|
||||||
|
Err(_) => unreachable!("client still shared after the task ended"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait for `d`, or until asked to stop. Returns `true` when stopping.
|
||||||
|
async fn pause(d: Duration, stop: &mut watch::Receiver<bool>) -> bool {
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::time::sleep(d) => *stop.borrow(),
|
||||||
|
_ = stop.changed() => true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run(client: Arc<Mutex<MeowClient>>, opts: BackgroundOptions, mut stop: watch::Receiver<bool>) {
|
||||||
|
let mut wait = opts.long_poll;
|
||||||
|
let mut backoff = opts.backoff;
|
||||||
|
while !*stop.borrow() {
|
||||||
|
// 1. Due deposits (and expiries).
|
||||||
|
let (plan, transport) = {
|
||||||
|
let mut c = client.lock().await;
|
||||||
|
(c.prepare_flush(false), c.transport.clone())
|
||||||
|
};
|
||||||
|
if !plan.is_empty() {
|
||||||
|
let accepted = tokio::select! {
|
||||||
|
a = execute_flush(&transport, &plan) => a,
|
||||||
|
_ = stop.changed() => break,
|
||||||
|
};
|
||||||
|
let mut c = client.lock().await;
|
||||||
|
if let Err(e) = c.apply_flush(plan, accepted).await {
|
||||||
|
tracing::error!("cannot save the send queue: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let mut c = client.lock().await;
|
||||||
|
let pending = std::mem::take(&mut c.pending_events);
|
||||||
|
c.emit(&pending);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Pull (long-poll), without the lock while waiting. The wait ends in time
|
||||||
|
// for the next queued deposit to be retried.
|
||||||
|
let plan = {
|
||||||
|
let mut c = client.lock().await;
|
||||||
|
let w = match c.next_retry_in() {
|
||||||
|
Some(d) => wait.min(d.max(Duration::from_secs(1))),
|
||||||
|
None => wait,
|
||||||
|
};
|
||||||
|
c.prepare_pull(w)
|
||||||
|
};
|
||||||
|
if plan.is_empty() {
|
||||||
|
// No contact yet: just keep the queue moving.
|
||||||
|
let d = client.lock().await.next_retry_in().unwrap_or(opts.long_poll);
|
||||||
|
if pause(d.clamp(Duration::from_secs(1), opts.long_poll), &mut stop).await {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let results = tokio::select! {
|
||||||
|
r = execute_pull(&transport, &plan) => r,
|
||||||
|
_ = stop.changed() => break,
|
||||||
|
};
|
||||||
|
|
||||||
|
// 3. Apply and process under the lock.
|
||||||
|
let mut c = client.lock().await;
|
||||||
|
match c.apply_pull(plan, results) {
|
||||||
|
Ok(round) => {
|
||||||
|
backoff = opts.backoff;
|
||||||
|
let got_new = !round.new.is_empty();
|
||||||
|
if got_new {
|
||||||
|
match c.process(round.new).await {
|
||||||
|
Ok(ev) => c.emit(&ev),
|
||||||
|
Err(e) => tracing::error!("cannot process received messages: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Something arrived (acks to send, replies to flush) or the servers
|
||||||
|
// held messages back: go again at once.
|
||||||
|
wait = if got_new || round.more {
|
||||||
|
Duration::ZERO
|
||||||
|
} else {
|
||||||
|
opts.long_poll
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
drop(c);
|
||||||
|
tracing::warn!("background pull failed: {e}; retrying in {backoff:?}");
|
||||||
|
if pause(backoff, &mut stop).await {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
backoff = (backoff * 2).min(opts.max_backoff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+100
-26
@@ -17,6 +17,7 @@
|
|||||||
#![forbid(unsafe_code)]
|
#![forbid(unsafe_code)]
|
||||||
#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))]
|
#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))]
|
||||||
|
|
||||||
|
pub mod background;
|
||||||
pub mod outbox;
|
pub mod outbox;
|
||||||
pub mod transport;
|
pub mod transport;
|
||||||
|
|
||||||
@@ -35,6 +36,7 @@ use meow_proto::v2::{self, request::Body as Req, response::Body as Resp};
|
|||||||
use meow_proto::Message;
|
use meow_proto::Message;
|
||||||
use meow_store::{Batch, RetiredMailbox, SpaceId, State as SpaceState, Store};
|
use meow_store::{Batch, RetiredMailbox, SpaceId, State as SpaceState, Store};
|
||||||
|
|
||||||
|
pub use background::{BackgroundOptions, ClientHandle};
|
||||||
pub use meow_core::contact::ContactState as State;
|
pub use meow_core::contact::ContactState as State;
|
||||||
pub use meow_store::{Content, HistoryEntry, HistoryPage, PendingAck, Settings, Status};
|
pub use meow_store::{Content, HistoryEntry, HistoryPage, PendingAck, Settings, Status};
|
||||||
pub use outbox::{PendingSend, RetryPolicy};
|
pub use outbox::{PendingSend, RetryPolicy};
|
||||||
@@ -146,9 +148,43 @@ pub struct SendReport {
|
|||||||
pub delivered_to: usize,
|
pub delivered_to: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Events kept for slow subscribers.
|
||||||
|
const EVENT_BUFFER: usize = 256;
|
||||||
|
|
||||||
/// Acknowledged ids sent per mailbox and pull (server limit: 1024).
|
/// Acknowledged ids sent per mailbox and pull (server limit: 1024).
|
||||||
const MAX_ACKS_PER_MAILBOX: usize = 512;
|
const MAX_ACKS_PER_MAILBOX: usize = 512;
|
||||||
|
|
||||||
|
/// Pull requests prepared under the client lock and sent without it.
|
||||||
|
pub(crate) struct PullPlan {
|
||||||
|
jobs: Vec<PullJob>,
|
||||||
|
wait: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PullJob {
|
||||||
|
server: ServerInfo,
|
||||||
|
/// Mailboxes pulled, with their space.
|
||||||
|
mailboxes: Vec<(usize, SignPublic)>,
|
||||||
|
/// Acks carried by the request.
|
||||||
|
sent: Vec<(usize, PendingAck)>,
|
||||||
|
request: v2::Request,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PullPlan {
|
||||||
|
pub(crate) fn is_empty(&self) -> bool {
|
||||||
|
self.jobs.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Network part of a pull round (no client state).
|
||||||
|
pub(crate) async fn execute_pull(transport: &transport::Transport, plan: &PullPlan) -> Vec<Result<Resp>> {
|
||||||
|
join_all(
|
||||||
|
plan.jobs
|
||||||
|
.iter()
|
||||||
|
.map(|j| transport.call_request(&j.server, j.request.clone(), plan.wait)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct PullRound {
|
struct PullRound {
|
||||||
/// Messages to process, with the index of the space owning their mailbox.
|
/// Messages to process, with the index of the space owning their mailbox.
|
||||||
@@ -178,6 +214,8 @@ pub struct MeowClient {
|
|||||||
/// Events produced outside `sync` (e.g. an expiry noticed while sending), handed
|
/// Events produced outside `sync` (e.g. an expiry noticed while sending), handed
|
||||||
/// out by the next `sync` or `flush_outbox`.
|
/// out by the next `sync` or `flush_outbox`.
|
||||||
pending_events: Vec<Event>,
|
pending_events: Vec<Event>,
|
||||||
|
/// Every event, for subscribers ([`MeowClient::subscribe`]).
|
||||||
|
events: tokio::sync::broadcast::Sender<Event>,
|
||||||
}
|
}
|
||||||
|
|
||||||
type Mailboxes = Vec<(SigningKeyPair, Vec<ServerInfo>)>;
|
type Mailboxes = Vec<(SigningKeyPair, Vec<ServerInfo>)>;
|
||||||
@@ -213,6 +251,7 @@ impl MeowClient {
|
|||||||
transport: transport::Transport::new()?,
|
transport: transport::Transport::new()?,
|
||||||
retry: RetryPolicy::default(),
|
retry: RetryPolicy::default(),
|
||||||
pending_events: Vec::new(),
|
pending_events: Vec::new(),
|
||||||
|
events: tokio::sync::broadcast::channel(EVENT_BUFFER).0,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -243,6 +282,20 @@ impl MeowClient {
|
|||||||
&self.main().nickname
|
&self.main().nickname
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Receive every [`Event`] from now on, whatever produced it (`sync`, the background
|
||||||
|
/// task, `flush_outbox`). A receiver that falls more than 256 events behind gets
|
||||||
|
/// `RecvError::Lagged` and should reload its view (history) from the client.
|
||||||
|
pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver<Event> {
|
||||||
|
self.events.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn emit(&self, events: &[Event]) {
|
||||||
|
for e in events {
|
||||||
|
// No subscriber is fine.
|
||||||
|
let _ = self.events.send(e.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Identity preferences (receipts…).
|
/// Identity preferences (receipts…).
|
||||||
pub fn settings(&self) -> Settings {
|
pub fn settings(&self) -> Settings {
|
||||||
self.main().settings
|
self.main().settings
|
||||||
@@ -759,6 +812,8 @@ impl MeowClient {
|
|||||||
let got_new = !r.new.is_empty();
|
let got_new = !r.new.is_empty();
|
||||||
if got_new {
|
if got_new {
|
||||||
events.extend(self.process(r.new).await?);
|
events.extend(self.process(r.new).await?);
|
||||||
|
// Replies produced by processing (receipts, handshake steps).
|
||||||
|
self.flush(false).await?;
|
||||||
}
|
}
|
||||||
// Pull again (without waiting) only to acknowledge what just arrived and to
|
// Pull again (without waiting) only to acknowledge what just arrived and to
|
||||||
// fetch what servers held back. Redeliveries alone do not loop.
|
// fetch what servers held back. Redeliveries alone do not loop.
|
||||||
@@ -767,6 +822,8 @@ impl MeowClient {
|
|||||||
}
|
}
|
||||||
wait = Duration::ZERO;
|
wait = Duration::ZERO;
|
||||||
}
|
}
|
||||||
|
events.append(&mut self.pending_events);
|
||||||
|
self.emit(&events);
|
||||||
Ok(events)
|
Ok(events)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -774,6 +831,16 @@ impl MeowClient {
|
|||||||
/// are returned and recorded as pending acks of the space owning their mailbox;
|
/// are returned and recorded as pending acks of the space owning their mailbox;
|
||||||
/// they are saved by [`Self::process`].
|
/// they are saved by [`Self::process`].
|
||||||
async fn pull_round(&mut self, wait: Duration) -> Result<PullRound> {
|
async fn pull_round(&mut self, wait: Duration) -> Result<PullRound> {
|
||||||
|
let plan = self.prepare_pull(wait);
|
||||||
|
if plan.is_empty() {
|
||||||
|
return Ok(PullRound::default());
|
||||||
|
}
|
||||||
|
let results = execute_pull(&self.transport, &plan).await;
|
||||||
|
self.apply_pull(plan, results)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Under the client lock: build one pull request per server (with pending acks).
|
||||||
|
pub(crate) fn prepare_pull(&mut self, wait: Duration) -> PullPlan {
|
||||||
// server (by signing key) → (server, [(space index, mailbox key)])
|
// server (by signing key) → (server, [(space index, mailbox key)])
|
||||||
let mut jobs: BTreeMap<SignPublic, (ServerInfo, Vec<(usize, SigningKeyPair)>)> = BTreeMap::new();
|
let mut jobs: BTreeMap<SignPublic, (ServerInfo, Vec<(usize, SigningKeyPair)>)> = BTreeMap::new();
|
||||||
for si in 0..self.spaces.len() {
|
for si in 0..self.spaces.len() {
|
||||||
@@ -808,14 +875,12 @@ impl MeowClient {
|
|||||||
})
|
})
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let mut round = PullRound::default();
|
|
||||||
if jobs.is_empty() {
|
|
||||||
return Ok(round);
|
|
||||||
}
|
|
||||||
let wait_s = wait.as_secs().min(u32::MAX as u64) as u32;
|
let wait_s = wait.as_secs().min(u32::MAX as u64) as u32;
|
||||||
let mut sent_acks: Vec<Vec<(usize, PendingAck)>> = Vec::with_capacity(jobs.len());
|
let mut plan = PullPlan {
|
||||||
let mut requests = Vec::with_capacity(jobs.len());
|
jobs: Vec::with_capacity(jobs.len()),
|
||||||
for (server, kps) in jobs.values() {
|
wait,
|
||||||
|
};
|
||||||
|
for (server, kps) in jobs.into_values() {
|
||||||
let rid = meow_crypto::random_bytes::<16>().to_vec();
|
let rid = meow_crypto::random_bytes::<16>().to_vec();
|
||||||
let ts = now();
|
let ts = now();
|
||||||
let mut sent = Vec::new();
|
let mut sent = Vec::new();
|
||||||
@@ -832,50 +897,60 @@ impl MeowClient {
|
|||||||
.inspect(|a| sent.push((*si, (*a).clone())))
|
.inspect(|a| sent.push((*si, (*a).clone())))
|
||||||
.map(|a| a.id.clone())
|
.map(|a| a.id.clone())
|
||||||
.collect();
|
.collect();
|
||||||
rpc::mailbox_auth(kp, server, &rid, acks, ts)
|
rpc::mailbox_auth(kp, &server, &rid, acks, ts)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
sent_acks.push(sent);
|
let request = v2::Request {
|
||||||
requests.push(v2::Request {
|
|
||||||
request_id: rid,
|
request_id: rid,
|
||||||
timestamp: ts,
|
timestamp: ts,
|
||||||
body: Some(Req::Pull(v2::Pull {
|
body: Some(Req::Pull(v2::Pull {
|
||||||
auths,
|
auths,
|
||||||
wait_seconds: wait_s,
|
wait_seconds: wait_s,
|
||||||
})),
|
})),
|
||||||
|
};
|
||||||
|
let mailboxes = kps.iter().map(|(si, kp)| (*si, kp.public())).collect();
|
||||||
|
plan.jobs.push(PullJob {
|
||||||
|
server,
|
||||||
|
mailboxes,
|
||||||
|
sent,
|
||||||
|
request,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let calls = jobs
|
plan
|
||||||
.values()
|
}
|
||||||
.zip(requests)
|
|
||||||
.map(|((server, _), req)| self.transport.call_request(server, req, wait));
|
|
||||||
let results = join_all(calls).await;
|
|
||||||
|
|
||||||
|
/// Under the client lock: record the received messages as pending acks.
|
||||||
|
pub(crate) fn apply_pull(&mut self, plan: PullPlan, results: Vec<Result<Resp>>) -> Result<PullRound> {
|
||||||
|
let mut round = PullRound::default();
|
||||||
let mut failures = 0;
|
let mut failures = 0;
|
||||||
let mut last_err = None;
|
let mut last_err = None;
|
||||||
for (((server, kps), r), sent) in jobs.values().zip(results).zip(sent_acks) {
|
let n = plan.jobs.len();
|
||||||
|
for (job, r) in plan.jobs.into_iter().zip(results) {
|
||||||
match r {
|
match r {
|
||||||
Ok(Resp::PullResult(p)) => {
|
Ok(Resp::PullResult(p)) => {
|
||||||
// The server applied our acks before answering. Forgetting them
|
// The server applied our acks before answering. Forgetting them
|
||||||
// needs no save: at worst they are sent again (acks are idempotent).
|
// needs no save: at worst they are sent again (acks are idempotent).
|
||||||
for (si, a) in &sent {
|
for (si, a) in &job.sent {
|
||||||
self.spaces[*si].state.pending_acks.retain(|x| x != a);
|
self.spaces[*si].state.pending_acks.retain(|x| x != a);
|
||||||
}
|
}
|
||||||
round.more |= p.more;
|
round.more |= p.more;
|
||||||
for m in p.messages {
|
for m in p.messages {
|
||||||
let Some((si, _)) = kps
|
let Some((si, _)) = job
|
||||||
|
.mailboxes
|
||||||
.iter()
|
.iter()
|
||||||
.find(|(_, kp)| kp.public().as_bytes().as_slice() == m.mailbox.as_slice())
|
.find(|(_, pk)| pk.as_bytes().as_slice() == m.mailbox.as_slice())
|
||||||
else {
|
else {
|
||||||
tracing::debug!("server sent a message for a mailbox we did not pull");
|
tracing::debug!("server sent a message for a mailbox we did not pull");
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let ack = PendingAck {
|
let ack = PendingAck {
|
||||||
server: server.sign_key,
|
server: job.server.sign_key,
|
||||||
mailbox: m.mailbox.clone(),
|
mailbox: m.mailbox.clone(),
|
||||||
id: m.id.clone(),
|
id: m.id.clone(),
|
||||||
};
|
};
|
||||||
let sp = &mut self.spaces[*si];
|
let Some(sp) = self.spaces.get_mut(*si) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
// Already processed (its ack was lost): only re-acknowledge.
|
// Already processed (its ack was lost): only re-acknowledge.
|
||||||
if !sp.state.pending_acks.contains(&ack) {
|
if !sp.state.pending_acks.contains(&ack) {
|
||||||
sp.state.pending_acks.push(ack);
|
sp.state.pending_acks.push(ack);
|
||||||
@@ -886,13 +961,13 @@ impl MeowClient {
|
|||||||
}
|
}
|
||||||
Ok(_) => failures += 1,
|
Ok(_) => failures += 1,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("pull on {} failed: {e}", server.url);
|
tracing::warn!("pull on {} failed: {e}", job.server.url);
|
||||||
failures += 1;
|
failures += 1;
|
||||||
last_err = Some(e);
|
last_err = Some(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if failures == jobs.len() {
|
if n > 0 && failures == n {
|
||||||
return Err(last_err.unwrap_or_else(|| Error::Network("all servers failed".into())));
|
return Err(last_err.unwrap_or_else(|| Error::Network("all servers failed".into())));
|
||||||
}
|
}
|
||||||
Ok(round)
|
Ok(round)
|
||||||
@@ -927,7 +1002,7 @@ impl MeowClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process(&mut self, stored: Vec<(usize, v2::StoredMessage)>) -> Result<Vec<Event>> {
|
pub(crate) async fn process(&mut self, stored: Vec<(usize, v2::StoredMessage)>) -> Result<Vec<Event>> {
|
||||||
let mailboxes_before: Vec<Mailboxes> = (0..self.spaces.len()).map(|si| self.space_mailboxes(si)).collect();
|
let mailboxes_before: Vec<Mailboxes> = (0..self.spaces.len()).map(|si| self.space_mailboxes(si)).collect();
|
||||||
// Handshake messages first (step 2 before step 3), then ratchet messages by arrival.
|
// Handshake messages first (step 2 before step 3), then ratchet messages by arrival.
|
||||||
let mut envs: Vec<(u32, u64, usize, Vec<u8>, v2::PeerEnvelope)> = stored
|
let mut envs: Vec<(u32, u64, usize, Vec<u8>, v2::PeerEnvelope)> = stored
|
||||||
@@ -1081,9 +1156,8 @@ impl MeowClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Persist (state, history, acks, queued replies) before anything leaves the device.
|
// Persist (state, history, acks, queued replies) before anything leaves the device.
|
||||||
|
// The replies are sent by the caller's next flush.
|
||||||
self.save().await?;
|
self.save().await?;
|
||||||
self.flush(false).await?;
|
|
||||||
events.append(&mut self.pending_events);
|
|
||||||
Ok(events)
|
Ok(events)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -125,9 +125,15 @@ impl MeowClient {
|
|||||||
/// Try to deliver queued deposits: those due (or all, with `force`). Returns, per
|
/// Try to deliver queued deposits: those due (or all, with `force`). Returns, per
|
||||||
/// item id, how many servers accepted it now. Expiries go to `pending_events`.
|
/// item id, how many servers accepted it now. Expiries go to `pending_events`.
|
||||||
pub(crate) async fn flush(&mut self, force: bool) -> Result<HashMap<Vec<u8>, usize>> {
|
pub(crate) async fn flush(&mut self, force: bool) -> Result<HashMap<Vec<u8>, usize>> {
|
||||||
|
let plan = self.prepare_flush(force);
|
||||||
|
let accepted = execute_flush(&self.transport, &plan).await;
|
||||||
|
self.apply_flush(plan, accepted).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Under the client lock: expire old items and group the due deposits.
|
||||||
|
pub(crate) fn prepare_flush(&mut self, force: bool) -> FlushPlan {
|
||||||
let t = now();
|
let t = now();
|
||||||
let events = &mut self.pending_events;
|
let events = &mut self.pending_events;
|
||||||
// Expired items.
|
|
||||||
let max_age = self.retry.max_age.as_secs();
|
let max_age = self.retry.max_age.as_secs();
|
||||||
for sp in &mut self.spaces {
|
for sp in &mut self.spaces {
|
||||||
let before = sp.state.outbox.len();
|
let before = sp.state.outbox.len();
|
||||||
@@ -171,13 +177,6 @@ impl MeowClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if due.is_empty() {
|
|
||||||
self.save().await?;
|
|
||||||
return Ok(HashMap::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
// One Post per server and batch; a rejected batch is retried item by item, so
|
|
||||||
// one bad deposit cannot block the others.
|
|
||||||
let mut batches: Vec<(ServerInfo, Batch)> = Vec::new();
|
let mut batches: Vec<(ServerInfo, Batch)> = Vec::new();
|
||||||
for (server, items) in by_server.into_values() {
|
for (server, items) in by_server.into_values() {
|
||||||
let mut cur: Batch = Vec::new();
|
let mut cur: Batch = Vec::new();
|
||||||
@@ -195,30 +194,22 @@ impl MeowClient {
|
|||||||
batches.push((server, cur));
|
batches.push((server, cur));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let results = join_all(batches.iter().map(|(s, b)| self.post(s, b))).await;
|
FlushPlan { batches, due, at: t }
|
||||||
let mut accepted: Vec<(usize, Vec<u8>, SignPublic)> = Vec::new();
|
|
||||||
for ((server, batch), r) in batches.iter().zip(results) {
|
|
||||||
match r {
|
|
||||||
Ok(()) => accepted.extend(batch.iter().map(|(si, id, _)| (*si, id.clone(), server.sign_key))),
|
|
||||||
Err(Error::Network(e)) => tracing::warn!("deposit on {} failed: {e}", server.url),
|
|
||||||
Err(e) if batch.len() > 1 => {
|
|
||||||
tracing::warn!("{} rejected a batch ({e}): retrying one by one", server.url);
|
|
||||||
for one in batch {
|
|
||||||
match self.post(server, std::slice::from_ref(one)).await {
|
|
||||||
Ok(()) => accepted.push((one.0, one.1.clone(), server.sign_key)),
|
|
||||||
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update the items.
|
/// Under the client lock: record what servers accepted, schedule retries, save.
|
||||||
|
pub(crate) async fn apply_flush(
|
||||||
|
&mut self,
|
||||||
|
plan: FlushPlan,
|
||||||
|
accepted: Vec<(usize, Vec<u8>, SignPublic)>,
|
||||||
|
) -> Result<HashMap<Vec<u8>, usize>> {
|
||||||
let mut now_accepted: HashMap<Vec<u8>, usize> = HashMap::new();
|
let mut now_accepted: HashMap<Vec<u8>, usize> = HashMap::new();
|
||||||
for (si, id, key) in accepted {
|
for (si, id, key) in accepted {
|
||||||
let sp = &mut self.spaces[si];
|
let sp = &mut self.spaces[si];
|
||||||
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
|
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
|
||||||
|
if !o.pending.iter().any(|s| s.sign_key == key) {
|
||||||
|
continue; // already recorded (concurrent flush)
|
||||||
|
}
|
||||||
o.pending.retain(|s| s.sign_key != key);
|
o.pending.retain(|s| s.sign_key != key);
|
||||||
if o.accepted == 0 && !o.message_id.is_empty() {
|
if o.accepted == 0 && !o.message_id.is_empty() {
|
||||||
sp.batch
|
sp.batch
|
||||||
@@ -229,12 +220,12 @@ impl MeowClient {
|
|||||||
*now_accepted.entry(id).or_default() += 1;
|
*now_accepted.entry(id).or_default() += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (si, id) in due {
|
for (si, id) in plan.due {
|
||||||
let retry = self.retry;
|
let retry = self.retry;
|
||||||
let sp = &mut self.spaces[si];
|
let sp = &mut self.spaces[si];
|
||||||
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
|
if let Some(o) = sp.state.outbox.iter_mut().find(|o| o.id == id) {
|
||||||
o.attempts += 1;
|
o.attempts += 1;
|
||||||
o.next_attempt = t + retry.delay(o.attempts);
|
o.next_attempt = plan.at + retry.delay(o.attempts);
|
||||||
}
|
}
|
||||||
sp.state.outbox.retain(|o| !o.pending.is_empty());
|
sp.state.outbox.retain(|o| !o.pending.is_empty());
|
||||||
sp.dirty = true;
|
sp.dirty = true;
|
||||||
@@ -243,17 +234,73 @@ impl MeowClient {
|
|||||||
Ok(now_accepted)
|
Ok(now_accepted)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Time until the next queued deposit is due (zero if one is due now).
|
||||||
|
pub(crate) fn next_retry_in(&self) -> Option<Duration> {
|
||||||
|
let t = now();
|
||||||
|
self.spaces
|
||||||
|
.iter()
|
||||||
|
.flat_map(|sp| sp.state.outbox.iter())
|
||||||
|
.map(|o| Duration::from_secs(o.next_attempt.saturating_sub(t)))
|
||||||
|
.min()
|
||||||
|
}
|
||||||
|
|
||||||
/// Deliver queued deposits now, without waiting for their retry time. Returns the
|
/// Deliver queued deposits now, without waiting for their retry time. Returns the
|
||||||
/// pending events (e.g. [`Event::SendFailed`]).
|
/// pending events (e.g. [`Event::SendFailed`]).
|
||||||
pub async fn flush_outbox(&mut self) -> Result<Vec<Event>> {
|
pub async fn flush_outbox(&mut self) -> Result<Vec<Event>> {
|
||||||
self.flush(true).await?;
|
self.flush(true).await?;
|
||||||
Ok(std::mem::take(&mut self.pending_events))
|
let events = std::mem::take(&mut self.pending_events);
|
||||||
|
self.emit(&events);
|
||||||
|
Ok(events)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn post(&self, server: &ServerInfo, batch: &[(usize, Vec<u8>, v2::Deposit)]) -> Result<()> {
|
/// Deposits to post, prepared under the client lock and sent without it.
|
||||||
|
pub(crate) struct FlushPlan {
|
||||||
|
batches: Vec<(ServerInfo, Batch)>,
|
||||||
|
due: Vec<(usize, Vec<u8>)>,
|
||||||
|
at: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FlushPlan {
|
||||||
|
pub(crate) fn is_empty(&self) -> bool {
|
||||||
|
self.due.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Network part of a flush (no client state): one Post per server and batch; a rejected
|
||||||
|
/// batch is retried item by item, so one bad deposit cannot block the others.
|
||||||
|
pub(crate) async fn execute_flush(
|
||||||
|
transport: &crate::transport::Transport,
|
||||||
|
plan: &FlushPlan,
|
||||||
|
) -> Vec<(usize, Vec<u8>, SignPublic)> {
|
||||||
|
let results = join_all(plan.batches.iter().map(|(s, b)| post(transport, s, b))).await;
|
||||||
|
let mut accepted = Vec::new();
|
||||||
|
for ((server, batch), r) in plan.batches.iter().zip(results) {
|
||||||
|
match r {
|
||||||
|
Ok(()) => accepted.extend(batch.iter().map(|(si, id, _)| (*si, id.clone(), server.sign_key))),
|
||||||
|
Err(Error::Network(e)) => tracing::warn!("deposit on {} failed: {e}", server.url),
|
||||||
|
Err(e) if batch.len() > 1 => {
|
||||||
|
tracing::warn!("{} rejected a batch ({e}): retrying one by one", server.url);
|
||||||
|
for one in batch {
|
||||||
|
match post(transport, server, std::slice::from_ref(one)).await {
|
||||||
|
Ok(()) => accepted.push((one.0, one.1.clone(), server.sign_key)),
|
||||||
|
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => tracing::warn!("{} rejected a deposit: {e}", server.url),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn post(
|
||||||
|
transport: &crate::transport::Transport,
|
||||||
|
server: &ServerInfo,
|
||||||
|
batch: &[(usize, Vec<u8>, v2::Deposit)],
|
||||||
|
) -> Result<()> {
|
||||||
let deposits = batch.iter().map(|(_, _, d)| d.clone()).collect();
|
let deposits = batch.iter().map(|(_, _, d)| d.clone()).collect();
|
||||||
match self
|
match transport
|
||||||
.transport
|
|
||||||
.call(server, Req::Post(v2::Post { deposits }), Duration::ZERO)
|
.call(server, Req::Post(v2::Post { deposits }), Duration::ZERO)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
@@ -261,7 +308,6 @@ impl MeowClient {
|
|||||||
_ => Err(Error::Invalid("unexpected response".into())),
|
_ => Err(Error::Invalid("unexpected response".into())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
@@ -615,3 +615,95 @@ async fn delivery_and_read_receipts() {
|
|||||||
assert_eq!(statuses(&alice, "bob", true)[2], Status::Sent);
|
assert_eq!(statuses(&alice, "bob", true)[2], Status::Sent);
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ───────────── background task and event stream (roadmap lot 5.9) ─────────────
|
||||||
|
|
||||||
|
async fn next_message(rx: &mut tokio::sync::broadcast::Receiver<Event>) -> String {
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
if let Event::Message { text, .. } = rx.recv().await.unwrap() {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("no message within 10 s")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn background_sync_publishes_events() {
|
||||||
|
use std::sync::atomic::Ordering::SeqCst;
|
||||||
|
let (url, store) = spawn_flaky_server().await;
|
||||||
|
let servers = vec![url];
|
||||||
|
let dir = tmpdir();
|
||||||
|
let mut alice = client(&dir, "alice", &servers).await;
|
||||||
|
let mut bob = client(&dir, "bob", &servers).await;
|
||||||
|
connect(&mut alice, &mut bob, "bob").await;
|
||||||
|
|
||||||
|
// Bob runs in the background with a long long-poll.
|
||||||
|
let opts = meow_client::BackgroundOptions {
|
||||||
|
long_poll: Duration::from_secs(20),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
bob.set_retry_policy(meow_client::RetryPolicy {
|
||||||
|
base: Duration::from_secs(1),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let bob = bob.start(opts);
|
||||||
|
let mut rx = bob.subscribe();
|
||||||
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||||
|
|
||||||
|
// Pushed to the subscriber without any explicit sync.
|
||||||
|
alice.send_text("bob", "hello background").await.unwrap();
|
||||||
|
assert_eq!(next_message(&mut rx).await, "hello background");
|
||||||
|
|
||||||
|
// The lock is free while the task waits on the server: sending is immediate.
|
||||||
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let r = bob.lock().await.send_text("alice", "reply").await.unwrap();
|
||||||
|
assert_eq!(r.delivered_to, 1);
|
||||||
|
assert!(
|
||||||
|
start.elapsed() < Duration::from_secs(3),
|
||||||
|
"send waited {:?}",
|
||||||
|
start.elapsed()
|
||||||
|
);
|
||||||
|
assert_eq!(texts(&alice.sync(Duration::ZERO).await.unwrap()), vec!["reply"]);
|
||||||
|
|
||||||
|
// Alice's read receipt reaches Bob's subscribers too.
|
||||||
|
alice.mark_read("bob").await.unwrap();
|
||||||
|
let got = tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
if let Event::Receipt { kind, .. } = rx.recv().await.unwrap() {
|
||||||
|
if kind == meow_client::ReceiptKind::Read {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(got.is_ok(), "no read receipt event");
|
||||||
|
|
||||||
|
// A message queued while the server refuses deposits is retried by the task.
|
||||||
|
store.fail_posts.store(true, SeqCst);
|
||||||
|
let r = bob.lock().await.send_text("alice", "retried").await.unwrap();
|
||||||
|
assert_eq!(r.delivered_to, 0);
|
||||||
|
store.fail_posts.store(false, SeqCst);
|
||||||
|
// The task shortens its long-poll to retry on time (no other traffic needed).
|
||||||
|
let delivered = tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
if bob.lock().await.pending_sends().is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(delivered.is_ok(), "queued message not retried");
|
||||||
|
assert!(texts(&alice.sync(Duration::ZERO).await.unwrap()).contains(&"retried".to_string()));
|
||||||
|
|
||||||
|
// Stopping gives the client back, usable as before.
|
||||||
|
let mut bob = bob.stop().await;
|
||||||
|
alice.send_text("bob", "after stop").await.unwrap();
|
||||||
|
assert_eq!(texts(&bob.sync(Duration::ZERO).await.unwrap()), vec!["after stop"]);
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
}
|
||||||
Reference in new issue
Block a user