fix(memcard): preserve watchdog timeout reasons across completion

This commit is contained in:
2026-10-14 10:26:30 +01:00
parent e0e01ed1de
commit bad942fb25
4 changed files with 314 additions and 25 deletions
@@ -65,6 +65,8 @@ struct Request {
control: Arc<PublicationControl>,
receiver: mpsc::Receiver<Completion>,
admitted: Instant,
watchdog_canceled_at: Option<Instant>,
worker_error: Option<Error>,
result: RequestResult,
job: AsyncWorkJobId,
released: bool,
@@ -523,6 +525,8 @@ impl Session {
control,
receiver,
admitted,
watchdog_canceled_at: None,
worker_error: None,
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
job,
released: !saves_payload,
@@ -664,6 +668,8 @@ impl Session {
control,
receiver,
admitted: now,
watchdog_canceled_at: None,
worker_error: None,
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
job,
released: false,
@@ -686,27 +692,29 @@ impl Session {
let Some(r) = &mut self.request else { return };
match r.receiver.try_recv() {
Ok(completion) => {
r.worker_error = completion.result.error;
let current =
self.mode.is_some() && r.session == self.context && r.epoch == self.epoch;
if current {
self.slots[r.local].protected = false;
if r.result.reason == RequestReason::None {
if r.result.state == RequestState::Pending {
r.result.state = match completion.result.durable {
DurableResult::Success => RequestState::Success,
DurableResult::Failure => RequestState::Failure,
DurableResult::Indeterminate => RequestState::Indeterminate,
};
r.result.reason = completion
.result
.error
.map_or(RequestReason::None, RequestReason::Storage);
r.result.reason = if r.watchdog_canceled_at.is_some() {
RequestReason::Timeout
} else {
r.worker_error.map_or(RequestReason::None, RequestReason::Storage)
};
if completion.result.durable == DurableResult::Success && r.saves_payload {
let mut s = r.source.write().unwrap();
s.descriptor =
completion.card.current().slots[self.logical[r.local] as usize];
s.dirty = false;
}
} // A timeout result is immutable; returned card requires explicit reconciliation.
} // A restart escalation is immutable; returned card requires explicit reconciliation.
if self.provisional && completion.result.durable != DurableResult::Success {
self.boot_error =
Some(completion.result.error.unwrap_or(Error::InvalidState));
@@ -755,7 +763,10 @@ impl Session {
return;
}
let elapsed = now.saturating_duration_since(r.admitted);
if elapsed >= Duration::from_millis(500) {
if elapsed >= Duration::from_millis(500) && !r.control.canceled() {
// Main-lane cancellation cause is independent of the worker's durable
// outcome. Do not relabel an earlier removal or termination cancel.
r.watchdog_canceled_at = Some(now);
r.control.cancel();
}
if elapsed >= Duration::from_secs(1) {
@@ -788,6 +799,12 @@ impl Session {
.map(|r| r.result)
.ok_or(Error::InvalidRange)
}
/// Original worker error for native diagnostics, independent of guest deadline reason.
/// Pending and RAM-only requests have no worker error. Retention follows query/release.
pub fn request_storage_error(&self, context: u64, id: u64) -> Result<Option<Error>> {
self.query(context, id)?;
Ok(self.request.as_ref().filter(|r| r.id == id).and_then(|r| r.worker_error))
}
pub fn release_request(&mut self, context: u64, id: u64) -> Result<()> {
self.query(context, id)?;
if self.deferred.is_some() {
@@ -280,8 +280,157 @@ fn async_commit_protects_only_source_and_retains_result_without_occupying_lane()
controller.stop().unwrap();
}
// The sentinel executes after the save on the single lane. Wall time only bounds
// a broken fixture; all watchdog observations use explicit simulated instants.
fn finish_queued_save(lane: Arc<crate::AsyncWorkLane>) {
let (tx, rx) = std::sync::mpsc::channel();
lane.submit_asset_load(move |_| {
tx.send(()).unwrap();
crate::services::async_work::AsyncWorkJobOutcome::Completed
})
.unwrap();
let mut controller =
crate::AsyncWorkLaneController::start(crate::AsyncWorkLaneConfig::default(), lane);
rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap();
controller.stop().unwrap();
}
fn assert_guest_query(
session: &mut super::super::session::Session,
id: u64,
state: i64,
reason: i64,
) {
use prometeu_bytecode::Value;
use prometeu_hal::{
HostReturn, host_return::GuestStringAllocator, syscalls::Syscall, vm_fault::VmFault,
};
struct NoStrings;
impl GuestStringAllocator for NoStrings {
fn byte_limit(&self) -> u64 {
0
}
fn build(
&mut self,
_: usize,
_: &mut dyn FnMut(&mut dyn std::fmt::Write) -> std::fmt::Result,
) -> std::result::Result<Arc<str>, VmFault> {
panic!("MemQuery must not allocate strings")
}
}
let mut service = super::super::MemcardService::new();
std::mem::swap(session, &mut service.session);
let args = [Value::Int64(service.session.context() as i64), Value::Int64(id as i64)];
let mut values = Vec::with_capacity(Syscall::MemQuery.results_count());
let mut strings = NoStrings;
let mut ret = HostReturn::new(&mut values, Syscall::MemQuery.results_count(), &mut strings);
service.guest(Syscall::MemQuery, &args, &mut ret, 4, 0, true).unwrap();
assert!(!ret.overflowed());
assert!(ret.failure().is_none());
assert_eq!(values, vec![Value::Int64(0), Value::Int64(state), Value::Int64(reason)]);
std::mem::swap(session, &mut service.session);
}
#[test]
fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
fn queued_memcard_watchdog_preserves_timeout_and_guest_reason() {
use super::super::session::{RequestReason, RequestState, Session};
use std::time::{Duration, Instant};
for bootstrap in [false, true] {
let (mut card, disk) = formatted();
add(&mut card, 4, 1);
let before = card.current().slots[0];
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
s.write(c, 0, 0, b"pending").unwrap();
let lane = Arc::new(crate::AsyncWorkLane::default());
let now = Instant::now();
if bootstrap {
s.begin_boot();
}
let id = s.commit(c, 0, 7, now, &lane).unwrap();
if bootstrap {
s.finish_boot(true, &lane, now).unwrap();
}
s.poll(now + Duration::from_millis(499));
assert_guest_query(&mut s, id, 0, 0);
for _ in 0..2 {
s.poll(now + Duration::from_millis(500));
}
assert_guest_query(&mut s, id, 0, 0);
assert_eq!(s.release_request(c, id), Err(Error::Busy));
assert_eq!(s.clear(c, 0), Err(Error::Busy));
finish_queued_save(lane);
s.poll(now + Duration::from_millis(501));
let result = s.query(c, id).unwrap();
assert_eq!(result.state, RequestState::Failure);
assert_eq!(result.reason, RequestReason::Timeout);
assert_eq!(s.request_storage_error(c, id), Ok(Some(Error::Canceled)));
assert_guest_query(&mut s, id, 2, 15);
assert!(s.unsaved());
assert_eq!(reboot(&disk).current().slots[0], before);
assert_eq!(s.query(c + 1, id), Err(Error::AccessDenied));
s.release_request(c, id).unwrap();
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
s.clear(c, 0).unwrap();
}
}
#[test]
fn lifecycle_cancellation_is_not_a_watchdog_timeout() {
use super::super::session::{RequestReason, RequestState, Session};
use std::time::{Duration, Instant};
for end in [false, true] {
let (mut card, _) = formatted();
add(&mut card, 4, 1);
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
let lane = Arc::new(crate::AsyncWorkLane::default());
let now = Instant::now();
let id = s.commit(c, 0, 7, now, &lane).unwrap();
if end {
s.end_session();
} else {
s.notify_removal();
}
s.poll(now + Duration::from_millis(500));
finish_queued_save(lane);
s.poll(now + Duration::from_millis(501));
if end {
assert_eq!(s.query(c, id), Err(Error::AccessDenied));
let card = s.take_idle_card().unwrap().unwrap();
let new = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
assert_ne!(new, c);
assert_eq!(s.query(new, id), Err(Error::InvalidRange));
} else {
assert_eq!(s.query(c, id).unwrap().state, RequestState::Failure);
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Storage(Error::Canceled));
assert_guest_query(&mut s, id, 2, 8);
}
}
}
#[test]
fn deferred_boot_memcard_deadline_stays_timeout() {
use super::super::session::{RequestReason, Session};
use std::time::{Duration, Instant};
let (mut card, _) = formatted();
add(&mut card, 4, 1);
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
s.begin_boot();
let lane = crate::AsyncWorkLane::default();
let now = Instant::now();
let id = s.commit(c, 0, 7, now, &lane).unwrap();
s.poll(now + Duration::from_millis(500));
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
assert_guest_query(&mut s, id, 2, 15);
assert_eq!(lane.pending_depth(), 0);
s.clear(c, 0).unwrap();
s.release_request(c, id).unwrap();
}
#[test]
fn blocked_host_memcard_watchdog_preserves_outcome_and_escalation() {
use crate::services::async_work::{
AsyncWorkLane, AsyncWorkLaneConfig, AsyncWorkLaneController,
};
@@ -290,6 +439,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
struct Blocked {
store: Box<dyn MemcardStorage>,
publication: bool,
fail_publication: bool,
entered: std::sync::mpsc::Sender<()>,
resume: std::sync::mpsc::Receiver<()>,
}
@@ -310,12 +460,24 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
if if self.publication { b.starts_with(b"PMMAP1\0\0") } else { o >= DATA_OFFSET } {
self.entered.send(()).unwrap();
self.resume.recv().unwrap();
if self.fail_publication {
return Err(MemcardIoError::IoError);
}
}
self.store.write_at(o, b)
}
}
for publication in [false, true] {
let (mut card, _) = formatted();
for (publication, fail_publication, escalate, release_early) in [
(false, false, false, false),
(true, false, false, false),
(true, true, false, false),
(false, false, true, false),
(true, false, true, false),
(true, true, true, false),
(false, false, true, true),
(true, false, true, true),
] {
let (mut card, disk) = formatted();
add(&mut card, 4, 1);
let (et, er) = std::sync::mpsc::channel();
let (rt, rr) = std::sync::mpsc::channel();
@@ -324,6 +486,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
entered: et,
resume: rr,
publication,
fail_publication,
}))
.unwrap();
let mut s = Session::default();
@@ -336,22 +499,84 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
let id = s.commit(c, 0, 7, admitted, &lane).unwrap();
er.recv_timeout(Duration::from_secs(1)).unwrap();
s.poll(admitted + Duration::from_millis(500));
s.poll(admitted + Duration::from_secs(1));
let result = s.query(c, id).unwrap();
assert_eq!(
result.state,
if publication { RequestState::Indeterminate } else { RequestState::Failure }
);
assert_eq!(result.reason, RequestReason::RestartRequired);
assert!(s.host_faulted());
s.release_request(c, id).unwrap();
assert_guest_query(&mut s, id, 0, 0);
let terminal = if escalate {
s.poll(admitted + Duration::from_secs(1));
let result = s.query(c, id).unwrap();
assert_eq!(
result.state,
if publication { RequestState::Indeterminate } else { RequestState::Failure }
);
assert_eq!(result.reason, RequestReason::RestartRequired);
assert!(s.host_faulted());
if release_early {
s.release_request(c, id).unwrap();
}
Some(result)
} else {
None
};
assert_eq!(s.write(c, 0, 0, b"unsafe"), Err(Error::Busy));
assert_eq!(s.commit(c, 0, 7, Instant::now(), &lane), Err(Error::Busy));
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::Busy));
rt.send(()).unwrap();
controller.stop().unwrap();
s.poll(Instant::now());
assert!(s.unsaved());
assert!(s.host_faulted());
s.poll(admitted + Duration::from_millis(if escalate { 1001 } else { 501 }));
if let Some(terminal) = terminal {
if release_early {
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
} else {
assert_eq!(s.query(c, id), Ok(terminal));
assert_guest_query(&mut s, id, if publication { 3 } else { 2 }, 15);
s.release_request(c, id).unwrap();
}
assert!(s.unsaved());
assert!(s.host_faulted());
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::ReadOnly));
} else {
let expected = if fail_publication {
RequestState::Indeterminate
} else if publication {
RequestState::Success
} else {
RequestState::Failure
};
assert_eq!(s.query(c, id).unwrap().state, expected);
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
assert_guest_query(
&mut s,
id,
if fail_publication {
3
} else if publication {
1
} else {
2
},
15,
);
assert_eq!(
s.request_storage_error(c, id),
Ok(if fail_publication {
Some(Error::Io(MemcardIoError::IoError))
} else if publication {
None
} else {
Some(Error::Canceled)
})
);
assert_eq!(s.unsaved(), expected != RequestState::Success);
assert_eq!(s.removal_pending(), fail_publication);
assert!(!s.host_faulted());
s.release_request(c, id).unwrap();
}
let mut durable = reboot(&disk);
let mut bytes = [0; 7];
let n = durable.read_slot(0, &mut bytes).unwrap();
if publication && !fail_publication {
assert_eq!(&bytes[..n], b"pending");
} else {
assert_eq!(n, 0);
}
}
}