fix(memcard): preserve watchdog timeout reasons across completion

This commit is contained in:
2026-10-14 10:26:30 +01:00
parent e0e01ed1de
commit bad942fb25
4 changed files with 314 additions and 25 deletions
@@ -65,6 +65,8 @@ struct Request {
control: Arc<PublicationControl>,
receiver: mpsc::Receiver<Completion>,
admitted: Instant,
watchdog_canceled_at: Option<Instant>,
worker_error: Option<Error>,
result: RequestResult,
job: AsyncWorkJobId,
released: bool,
@@ -523,6 +525,8 @@ impl Session {
control,
receiver,
admitted,
watchdog_canceled_at: None,
worker_error: None,
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
job,
released: !saves_payload,
@@ -664,6 +668,8 @@ impl Session {
control,
receiver,
admitted: now,
watchdog_canceled_at: None,
worker_error: None,
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
job,
released: false,
@@ -686,27 +692,29 @@ impl Session {
let Some(r) = &mut self.request else { return };
match r.receiver.try_recv() {
Ok(completion) => {
r.worker_error = completion.result.error;
let current =
self.mode.is_some() && r.session == self.context && r.epoch == self.epoch;
if current {
self.slots[r.local].protected = false;
if r.result.reason == RequestReason::None {
if r.result.state == RequestState::Pending {
r.result.state = match completion.result.durable {
DurableResult::Success => RequestState::Success,
DurableResult::Failure => RequestState::Failure,
DurableResult::Indeterminate => RequestState::Indeterminate,
};
r.result.reason = completion
.result
.error
.map_or(RequestReason::None, RequestReason::Storage);
r.result.reason = if r.watchdog_canceled_at.is_some() {
RequestReason::Timeout
} else {
r.worker_error.map_or(RequestReason::None, RequestReason::Storage)
};
if completion.result.durable == DurableResult::Success && r.saves_payload {
let mut s = r.source.write().unwrap();
s.descriptor =
completion.card.current().slots[self.logical[r.local] as usize];
s.dirty = false;
}
} // A timeout result is immutable; returned card requires explicit reconciliation.
} // A restart escalation is immutable; returned card requires explicit reconciliation.
if self.provisional && completion.result.durable != DurableResult::Success {
self.boot_error =
Some(completion.result.error.unwrap_or(Error::InvalidState));
@@ -755,7 +763,10 @@ impl Session {
return;
}
let elapsed = now.saturating_duration_since(r.admitted);
if elapsed >= Duration::from_millis(500) {
if elapsed >= Duration::from_millis(500) && !r.control.canceled() {
// Main-lane cancellation cause is independent of the worker's durable
// outcome. Do not relabel an earlier removal or termination cancel.
r.watchdog_canceled_at = Some(now);
r.control.cancel();
}
if elapsed >= Duration::from_secs(1) {
@@ -788,6 +799,12 @@ impl Session {
.map(|r| r.result)
.ok_or(Error::InvalidRange)
}
/// Original worker error for native diagnostics, independent of guest deadline reason.
/// Pending and RAM-only requests have no worker error. Retention follows query/release.
pub fn request_storage_error(&self, context: u64, id: u64) -> Result<Option<Error>> {
self.query(context, id)?;
Ok(self.request.as_ref().filter(|r| r.id == id).and_then(|r| r.worker_error))
}
pub fn release_request(&mut self, context: u64, id: u64) -> Result<()> {
self.query(context, id)?;
if self.deferred.is_some() {
@@ -280,8 +280,157 @@ fn async_commit_protects_only_source_and_retains_result_without_occupying_lane()
controller.stop().unwrap();
}
// The sentinel executes after the save on the single lane. Wall time only bounds
// a broken fixture; all watchdog observations use explicit simulated instants.
fn finish_queued_save(lane: Arc<crate::AsyncWorkLane>) {
let (tx, rx) = std::sync::mpsc::channel();
lane.submit_asset_load(move |_| {
tx.send(()).unwrap();
crate::services::async_work::AsyncWorkJobOutcome::Completed
})
.unwrap();
let mut controller =
crate::AsyncWorkLaneController::start(crate::AsyncWorkLaneConfig::default(), lane);
rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap();
controller.stop().unwrap();
}
fn assert_guest_query(
session: &mut super::super::session::Session,
id: u64,
state: i64,
reason: i64,
) {
use prometeu_bytecode::Value;
use prometeu_hal::{
HostReturn, host_return::GuestStringAllocator, syscalls::Syscall, vm_fault::VmFault,
};
struct NoStrings;
impl GuestStringAllocator for NoStrings {
fn byte_limit(&self) -> u64 {
0
}
fn build(
&mut self,
_: usize,
_: &mut dyn FnMut(&mut dyn std::fmt::Write) -> std::fmt::Result,
) -> std::result::Result<Arc<str>, VmFault> {
panic!("MemQuery must not allocate strings")
}
}
let mut service = super::super::MemcardService::new();
std::mem::swap(session, &mut service.session);
let args = [Value::Int64(service.session.context() as i64), Value::Int64(id as i64)];
let mut values = Vec::with_capacity(Syscall::MemQuery.results_count());
let mut strings = NoStrings;
let mut ret = HostReturn::new(&mut values, Syscall::MemQuery.results_count(), &mut strings);
service.guest(Syscall::MemQuery, &args, &mut ret, 4, 0, true).unwrap();
assert!(!ret.overflowed());
assert!(ret.failure().is_none());
assert_eq!(values, vec![Value::Int64(0), Value::Int64(state), Value::Int64(reason)]);
std::mem::swap(session, &mut service.session);
}
#[test]
fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
fn queued_memcard_watchdog_preserves_timeout_and_guest_reason() {
use super::super::session::{RequestReason, RequestState, Session};
use std::time::{Duration, Instant};
for bootstrap in [false, true] {
let (mut card, disk) = formatted();
add(&mut card, 4, 1);
let before = card.current().slots[0];
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
s.write(c, 0, 0, b"pending").unwrap();
let lane = Arc::new(crate::AsyncWorkLane::default());
let now = Instant::now();
if bootstrap {
s.begin_boot();
}
let id = s.commit(c, 0, 7, now, &lane).unwrap();
if bootstrap {
s.finish_boot(true, &lane, now).unwrap();
}
s.poll(now + Duration::from_millis(499));
assert_guest_query(&mut s, id, 0, 0);
for _ in 0..2 {
s.poll(now + Duration::from_millis(500));
}
assert_guest_query(&mut s, id, 0, 0);
assert_eq!(s.release_request(c, id), Err(Error::Busy));
assert_eq!(s.clear(c, 0), Err(Error::Busy));
finish_queued_save(lane);
s.poll(now + Duration::from_millis(501));
let result = s.query(c, id).unwrap();
assert_eq!(result.state, RequestState::Failure);
assert_eq!(result.reason, RequestReason::Timeout);
assert_eq!(s.request_storage_error(c, id), Ok(Some(Error::Canceled)));
assert_guest_query(&mut s, id, 2, 15);
assert!(s.unsaved());
assert_eq!(reboot(&disk).current().slots[0], before);
assert_eq!(s.query(c + 1, id), Err(Error::AccessDenied));
s.release_request(c, id).unwrap();
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
s.clear(c, 0).unwrap();
}
}
#[test]
fn lifecycle_cancellation_is_not_a_watchdog_timeout() {
use super::super::session::{RequestReason, RequestState, Session};
use std::time::{Duration, Instant};
for end in [false, true] {
let (mut card, _) = formatted();
add(&mut card, 4, 1);
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
let lane = Arc::new(crate::AsyncWorkLane::default());
let now = Instant::now();
let id = s.commit(c, 0, 7, now, &lane).unwrap();
if end {
s.end_session();
} else {
s.notify_removal();
}
s.poll(now + Duration::from_millis(500));
finish_queued_save(lane);
s.poll(now + Duration::from_millis(501));
if end {
assert_eq!(s.query(c, id), Err(Error::AccessDenied));
let card = s.take_idle_card().unwrap().unwrap();
let new = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
assert_ne!(new, c);
assert_eq!(s.query(new, id), Err(Error::InvalidRange));
} else {
assert_eq!(s.query(c, id).unwrap().state, RequestState::Failure);
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Storage(Error::Canceled));
assert_guest_query(&mut s, id, 2, 8);
}
}
}
#[test]
fn deferred_boot_memcard_deadline_stays_timeout() {
use super::super::session::{RequestReason, Session};
use std::time::{Duration, Instant};
let (mut card, _) = formatted();
add(&mut card, 4, 1);
let mut s = Session::default();
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
s.begin_boot();
let lane = crate::AsyncWorkLane::default();
let now = Instant::now();
let id = s.commit(c, 0, 7, now, &lane).unwrap();
s.poll(now + Duration::from_millis(500));
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
assert_guest_query(&mut s, id, 2, 15);
assert_eq!(lane.pending_depth(), 0);
s.clear(c, 0).unwrap();
s.release_request(c, id).unwrap();
}
#[test]
fn blocked_host_memcard_watchdog_preserves_outcome_and_escalation() {
use crate::services::async_work::{
AsyncWorkLane, AsyncWorkLaneConfig, AsyncWorkLaneController,
};
@@ -290,6 +439,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
struct Blocked {
store: Box<dyn MemcardStorage>,
publication: bool,
fail_publication: bool,
entered: std::sync::mpsc::Sender<()>,
resume: std::sync::mpsc::Receiver<()>,
}
@@ -310,12 +460,24 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
if if self.publication { b.starts_with(b"PMMAP1\0\0") } else { o >= DATA_OFFSET } {
self.entered.send(()).unwrap();
self.resume.recv().unwrap();
if self.fail_publication {
return Err(MemcardIoError::IoError);
}
}
self.store.write_at(o, b)
}
}
for publication in [false, true] {
let (mut card, _) = formatted();
for (publication, fail_publication, escalate, release_early) in [
(false, false, false, false),
(true, false, false, false),
(true, true, false, false),
(false, false, true, false),
(true, false, true, false),
(true, true, true, false),
(false, false, true, true),
(true, false, true, true),
] {
let (mut card, disk) = formatted();
add(&mut card, 4, 1);
let (et, er) = std::sync::mpsc::channel();
let (rt, rr) = std::sync::mpsc::channel();
@@ -324,6 +486,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
entered: et,
resume: rr,
publication,
fail_publication,
}))
.unwrap();
let mut s = Session::default();
@@ -336,22 +499,84 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
let id = s.commit(c, 0, 7, admitted, &lane).unwrap();
er.recv_timeout(Duration::from_secs(1)).unwrap();
s.poll(admitted + Duration::from_millis(500));
s.poll(admitted + Duration::from_secs(1));
let result = s.query(c, id).unwrap();
assert_eq!(
result.state,
if publication { RequestState::Indeterminate } else { RequestState::Failure }
);
assert_eq!(result.reason, RequestReason::RestartRequired);
assert!(s.host_faulted());
s.release_request(c, id).unwrap();
assert_guest_query(&mut s, id, 0, 0);
let terminal = if escalate {
s.poll(admitted + Duration::from_secs(1));
let result = s.query(c, id).unwrap();
assert_eq!(
result.state,
if publication { RequestState::Indeterminate } else { RequestState::Failure }
);
assert_eq!(result.reason, RequestReason::RestartRequired);
assert!(s.host_faulted());
if release_early {
s.release_request(c, id).unwrap();
}
Some(result)
} else {
None
};
assert_eq!(s.write(c, 0, 0, b"unsafe"), Err(Error::Busy));
assert_eq!(s.commit(c, 0, 7, Instant::now(), &lane), Err(Error::Busy));
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::Busy));
rt.send(()).unwrap();
controller.stop().unwrap();
s.poll(Instant::now());
assert!(s.unsaved());
assert!(s.host_faulted());
s.poll(admitted + Duration::from_millis(if escalate { 1001 } else { 501 }));
if let Some(terminal) = terminal {
if release_early {
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
} else {
assert_eq!(s.query(c, id), Ok(terminal));
assert_guest_query(&mut s, id, if publication { 3 } else { 2 }, 15);
s.release_request(c, id).unwrap();
}
assert!(s.unsaved());
assert!(s.host_faulted());
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::ReadOnly));
} else {
let expected = if fail_publication {
RequestState::Indeterminate
} else if publication {
RequestState::Success
} else {
RequestState::Failure
};
assert_eq!(s.query(c, id).unwrap().state, expected);
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
assert_guest_query(
&mut s,
id,
if fail_publication {
3
} else if publication {
1
} else {
2
},
15,
);
assert_eq!(
s.request_storage_error(c, id),
Ok(if fail_publication {
Some(Error::Io(MemcardIoError::IoError))
} else if publication {
None
} else {
Some(Error::Canceled)
})
);
assert_eq!(s.unsaved(), expected != RequestState::Success);
assert_eq!(s.removal_pending(), fail_publication);
assert!(!s.host_faulted());
s.release_request(c, id).unwrap();
}
let mut durable = reboot(&disk);
let mut bytes = [0; 7];
let n = durable.read_slot(0, &mut bytes).unwrap();
if publication && !fail_publication {
assert_eq!(&bytes[..n], b"pending");
} else {
assert_eq!(n, 0);
}
}
}
@@ -40,12 +40,33 @@ already completed result, which exposed an ownership leak during integration.
Cancellation and entry into publication share an atomic control state. At the
deadline, a canceled attempt that never entered publication can report failure.
Once publication may have started, the result remains indeterminate until
reconciliation. A blocked host requires restart escalation even when publication
Once publication may have started, cancellation alone cannot determine the outcome.
A worker that returns a known success before escalation still reports success;
an uncertain publication requires reconciliation. A blocked host requires restart escalation even when publication
was provably prevented: storage and source ownership still have to remain valid.
The runtime neither kills the thread nor starts a replacement worker over its
resources. The watchdog runs outside that worker.
The watchdog cause is a third observation, independent of durable outcome and
resource lifetime. Keeping it only in the terminal result loses information:
a worker's generic cancellation error can overwrite the reason the cancellation
was requested. Conversely, setting a terminal timeout before the worker returns
can accidentally suppress its eventual success. The request now records the
first watchdog cancellation separately, leaves the result Pending, and preserves
the original worker error for native diagnostics. Completion before escalation
combines the actual durable state with guest reason 15, including Success/15.
Removal or termination cancellation that happened first retains its own reason.
This correction follows accepted decision DEC-0058. Deterministic regression
fixtures hold queued work until after the simulated 500ms poll and deliver it
at 501ms. Channel barriers hold IO before or inside publication, exercising
Failure/15, Success/15, Indeterminate/15 and immutable restart results after 1s.
The tests invoke the actual guest query handler and assert all three ABI slots;
wall-clock receive limits only detect a hung fixture. They also verify deferred
and composite bootstrap saves, dirty snapshots, durable bytes after reopening,
result release and teardown isolation. These checks establish behavior under
controlled scheduling, not real-device latency.
## A checksum does not make an old reference safe
Two valid map pages can reference different generations of payload and public
@@ -236,6 +236,32 @@ not proof of rollback. If cessation is unconfirmed, isolate mount/resources,
block new work and retain source/scratch. Late completion belongs to original
reconciliation. Never kill threads unsafely, free buffers in use or create a
replacement worker. Report restart requirement when host recovery is impossible.
The first watchdog cancellation is recorded independently of the request result,
before signaling cancellation. Repeated polls preserve that cause. At 500ms the
request remains Pending with no terminal reason until the worker returns or the
1s escalation occurs. Removal or session termination that canceled first does
not become a watchdog timeout merely because a later poll crosses 500ms.
On completion before escalation, the worker's durable outcome determines
Success, Failure or Indeterminate. A recorded watchdog cancellation supplies
Timeout (guest reason 15) independently of that state; native diagnostics retain
the original worker storage error. For example:
- A save canceled while queued at 500ms, then completed at 501ms without entering
publication, reports Failure/15. The durable slot is unchanged.
- A save already inside publication at cancellation can finish successfully and
report Success/15. Its saved snapshot becomes clean; cancellation is not rollback.
- An uncertain publication reports Indeterminate/15 and requires reconciliation.
- At 1s without confirmed cessation, RestartRequired reports Failure/15 before
publication entry or Indeterminate/15 after entry. Late completion returns card
ownership for reconciliation but cannot rewrite this terminal guest result or
mark the snapshot clean. Releasing the result does not release in-use resources.
A deferred bootstrap save that has not reached the worker expires as Failure/15
at 500ms. Memory-only commits remain MemoryOnly with reason 0 and perform no IO.
The status/state/reason result slots and request identities do not change.
Fast-profile claims require measurements under asset load, including tail latency.
## 8. Player choice and lifecycle