fix(memcard): preserve watchdog timeout reasons across completion
This commit is contained in:
@@ -65,6 +65,8 @@ struct Request {
|
||||
control: Arc<PublicationControl>,
|
||||
receiver: mpsc::Receiver<Completion>,
|
||||
admitted: Instant,
|
||||
watchdog_canceled_at: Option<Instant>,
|
||||
worker_error: Option<Error>,
|
||||
result: RequestResult,
|
||||
job: AsyncWorkJobId,
|
||||
released: bool,
|
||||
@@ -523,6 +525,8 @@ impl Session {
|
||||
control,
|
||||
receiver,
|
||||
admitted,
|
||||
watchdog_canceled_at: None,
|
||||
worker_error: None,
|
||||
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
|
||||
job,
|
||||
released: !saves_payload,
|
||||
@@ -664,6 +668,8 @@ impl Session {
|
||||
control,
|
||||
receiver,
|
||||
admitted: now,
|
||||
watchdog_canceled_at: None,
|
||||
worker_error: None,
|
||||
result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None },
|
||||
job,
|
||||
released: false,
|
||||
@@ -686,27 +692,29 @@ impl Session {
|
||||
let Some(r) = &mut self.request else { return };
|
||||
match r.receiver.try_recv() {
|
||||
Ok(completion) => {
|
||||
r.worker_error = completion.result.error;
|
||||
let current =
|
||||
self.mode.is_some() && r.session == self.context && r.epoch == self.epoch;
|
||||
if current {
|
||||
self.slots[r.local].protected = false;
|
||||
if r.result.reason == RequestReason::None {
|
||||
if r.result.state == RequestState::Pending {
|
||||
r.result.state = match completion.result.durable {
|
||||
DurableResult::Success => RequestState::Success,
|
||||
DurableResult::Failure => RequestState::Failure,
|
||||
DurableResult::Indeterminate => RequestState::Indeterminate,
|
||||
};
|
||||
r.result.reason = completion
|
||||
.result
|
||||
.error
|
||||
.map_or(RequestReason::None, RequestReason::Storage);
|
||||
r.result.reason = if r.watchdog_canceled_at.is_some() {
|
||||
RequestReason::Timeout
|
||||
} else {
|
||||
r.worker_error.map_or(RequestReason::None, RequestReason::Storage)
|
||||
};
|
||||
if completion.result.durable == DurableResult::Success && r.saves_payload {
|
||||
let mut s = r.source.write().unwrap();
|
||||
s.descriptor =
|
||||
completion.card.current().slots[self.logical[r.local] as usize];
|
||||
s.dirty = false;
|
||||
}
|
||||
} // A timeout result is immutable; returned card requires explicit reconciliation.
|
||||
} // A restart escalation is immutable; returned card requires explicit reconciliation.
|
||||
if self.provisional && completion.result.durable != DurableResult::Success {
|
||||
self.boot_error =
|
||||
Some(completion.result.error.unwrap_or(Error::InvalidState));
|
||||
@@ -755,7 +763,10 @@ impl Session {
|
||||
return;
|
||||
}
|
||||
let elapsed = now.saturating_duration_since(r.admitted);
|
||||
if elapsed >= Duration::from_millis(500) {
|
||||
if elapsed >= Duration::from_millis(500) && !r.control.canceled() {
|
||||
// Main-lane cancellation cause is independent of the worker's durable
|
||||
// outcome. Do not relabel an earlier removal or termination cancel.
|
||||
r.watchdog_canceled_at = Some(now);
|
||||
r.control.cancel();
|
||||
}
|
||||
if elapsed >= Duration::from_secs(1) {
|
||||
@@ -788,6 +799,12 @@ impl Session {
|
||||
.map(|r| r.result)
|
||||
.ok_or(Error::InvalidRange)
|
||||
}
|
||||
/// Original worker error for native diagnostics, independent of guest deadline reason.
|
||||
/// Pending and RAM-only requests have no worker error. Retention follows query/release.
|
||||
pub fn request_storage_error(&self, context: u64, id: u64) -> Result<Option<Error>> {
|
||||
self.query(context, id)?;
|
||||
Ok(self.request.as_ref().filter(|r| r.id == id).and_then(|r| r.worker_error))
|
||||
}
|
||||
pub fn release_request(&mut self, context: u64, id: u64) -> Result<()> {
|
||||
self.query(context, id)?;
|
||||
if self.deferred.is_some() {
|
||||
|
||||
@@ -280,8 +280,157 @@ fn async_commit_protects_only_source_and_retains_result_without_occupying_lane()
|
||||
controller.stop().unwrap();
|
||||
}
|
||||
|
||||
// The sentinel executes after the save on the single lane. Wall time only bounds
|
||||
// a broken fixture; all watchdog observations use explicit simulated instants.
|
||||
fn finish_queued_save(lane: Arc<crate::AsyncWorkLane>) {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
lane.submit_asset_load(move |_| {
|
||||
tx.send(()).unwrap();
|
||||
crate::services::async_work::AsyncWorkJobOutcome::Completed
|
||||
})
|
||||
.unwrap();
|
||||
let mut controller =
|
||||
crate::AsyncWorkLaneController::start(crate::AsyncWorkLaneConfig::default(), lane);
|
||||
rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap();
|
||||
controller.stop().unwrap();
|
||||
}
|
||||
|
||||
fn assert_guest_query(
|
||||
session: &mut super::super::session::Session,
|
||||
id: u64,
|
||||
state: i64,
|
||||
reason: i64,
|
||||
) {
|
||||
use prometeu_bytecode::Value;
|
||||
use prometeu_hal::{
|
||||
HostReturn, host_return::GuestStringAllocator, syscalls::Syscall, vm_fault::VmFault,
|
||||
};
|
||||
struct NoStrings;
|
||||
impl GuestStringAllocator for NoStrings {
|
||||
fn byte_limit(&self) -> u64 {
|
||||
0
|
||||
}
|
||||
fn build(
|
||||
&mut self,
|
||||
_: usize,
|
||||
_: &mut dyn FnMut(&mut dyn std::fmt::Write) -> std::fmt::Result,
|
||||
) -> std::result::Result<Arc<str>, VmFault> {
|
||||
panic!("MemQuery must not allocate strings")
|
||||
}
|
||||
}
|
||||
let mut service = super::super::MemcardService::new();
|
||||
std::mem::swap(session, &mut service.session);
|
||||
let args = [Value::Int64(service.session.context() as i64), Value::Int64(id as i64)];
|
||||
let mut values = Vec::with_capacity(Syscall::MemQuery.results_count());
|
||||
let mut strings = NoStrings;
|
||||
let mut ret = HostReturn::new(&mut values, Syscall::MemQuery.results_count(), &mut strings);
|
||||
service.guest(Syscall::MemQuery, &args, &mut ret, 4, 0, true).unwrap();
|
||||
assert!(!ret.overflowed());
|
||||
assert!(ret.failure().is_none());
|
||||
assert_eq!(values, vec![Value::Int64(0), Value::Int64(state), Value::Int64(reason)]);
|
||||
std::mem::swap(session, &mut service.session);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
|
||||
fn queued_memcard_watchdog_preserves_timeout_and_guest_reason() {
|
||||
use super::super::session::{RequestReason, RequestState, Session};
|
||||
use std::time::{Duration, Instant};
|
||||
for bootstrap in [false, true] {
|
||||
let (mut card, disk) = formatted();
|
||||
add(&mut card, 4, 1);
|
||||
let before = card.current().slots[0];
|
||||
let mut s = Session::default();
|
||||
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
|
||||
s.write(c, 0, 0, b"pending").unwrap();
|
||||
let lane = Arc::new(crate::AsyncWorkLane::default());
|
||||
let now = Instant::now();
|
||||
if bootstrap {
|
||||
s.begin_boot();
|
||||
}
|
||||
let id = s.commit(c, 0, 7, now, &lane).unwrap();
|
||||
if bootstrap {
|
||||
s.finish_boot(true, &lane, now).unwrap();
|
||||
}
|
||||
s.poll(now + Duration::from_millis(499));
|
||||
assert_guest_query(&mut s, id, 0, 0);
|
||||
for _ in 0..2 {
|
||||
s.poll(now + Duration::from_millis(500));
|
||||
}
|
||||
assert_guest_query(&mut s, id, 0, 0);
|
||||
assert_eq!(s.release_request(c, id), Err(Error::Busy));
|
||||
assert_eq!(s.clear(c, 0), Err(Error::Busy));
|
||||
finish_queued_save(lane);
|
||||
s.poll(now + Duration::from_millis(501));
|
||||
let result = s.query(c, id).unwrap();
|
||||
assert_eq!(result.state, RequestState::Failure);
|
||||
assert_eq!(result.reason, RequestReason::Timeout);
|
||||
assert_eq!(s.request_storage_error(c, id), Ok(Some(Error::Canceled)));
|
||||
assert_guest_query(&mut s, id, 2, 15);
|
||||
assert!(s.unsaved());
|
||||
assert_eq!(reboot(&disk).current().slots[0], before);
|
||||
assert_eq!(s.query(c + 1, id), Err(Error::AccessDenied));
|
||||
s.release_request(c, id).unwrap();
|
||||
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
|
||||
s.clear(c, 0).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_cancellation_is_not_a_watchdog_timeout() {
|
||||
use super::super::session::{RequestReason, RequestState, Session};
|
||||
use std::time::{Duration, Instant};
|
||||
for end in [false, true] {
|
||||
let (mut card, _) = formatted();
|
||||
add(&mut card, 4, 1);
|
||||
let mut s = Session::default();
|
||||
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
|
||||
let lane = Arc::new(crate::AsyncWorkLane::default());
|
||||
let now = Instant::now();
|
||||
let id = s.commit(c, 0, 7, now, &lane).unwrap();
|
||||
if end {
|
||||
s.end_session();
|
||||
} else {
|
||||
s.notify_removal();
|
||||
}
|
||||
s.poll(now + Duration::from_millis(500));
|
||||
finish_queued_save(lane);
|
||||
s.poll(now + Duration::from_millis(501));
|
||||
if end {
|
||||
assert_eq!(s.query(c, id), Err(Error::AccessDenied));
|
||||
let card = s.take_idle_card().unwrap().unwrap();
|
||||
let new = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
|
||||
assert_ne!(new, c);
|
||||
assert_eq!(s.query(new, id), Err(Error::InvalidRange));
|
||||
} else {
|
||||
assert_eq!(s.query(c, id).unwrap().state, RequestState::Failure);
|
||||
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Storage(Error::Canceled));
|
||||
assert_guest_query(&mut s, id, 2, 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_boot_memcard_deadline_stays_timeout() {
|
||||
use super::super::session::{RequestReason, Session};
|
||||
use std::time::{Duration, Instant};
|
||||
let (mut card, _) = formatted();
|
||||
add(&mut card, 4, 1);
|
||||
let mut s = Session::default();
|
||||
let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap();
|
||||
s.begin_boot();
|
||||
let lane = crate::AsyncWorkLane::default();
|
||||
let now = Instant::now();
|
||||
let id = s.commit(c, 0, 7, now, &lane).unwrap();
|
||||
s.poll(now + Duration::from_millis(500));
|
||||
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
|
||||
assert_guest_query(&mut s, id, 2, 15);
|
||||
assert_eq!(lane.pending_depth(), 0);
|
||||
s.clear(c, 0).unwrap();
|
||||
s.release_request(c, id).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_host_memcard_watchdog_preserves_outcome_and_escalation() {
|
||||
use crate::services::async_work::{
|
||||
AsyncWorkLane, AsyncWorkLaneConfig, AsyncWorkLaneController,
|
||||
};
|
||||
@@ -290,6 +439,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
|
||||
struct Blocked {
|
||||
store: Box<dyn MemcardStorage>,
|
||||
publication: bool,
|
||||
fail_publication: bool,
|
||||
entered: std::sync::mpsc::Sender<()>,
|
||||
resume: std::sync::mpsc::Receiver<()>,
|
||||
}
|
||||
@@ -310,12 +460,24 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
|
||||
if if self.publication { b.starts_with(b"PMMAP1\0\0") } else { o >= DATA_OFFSET } {
|
||||
self.entered.send(()).unwrap();
|
||||
self.resume.recv().unwrap();
|
||||
if self.fail_publication {
|
||||
return Err(MemcardIoError::IoError);
|
||||
}
|
||||
}
|
||||
self.store.write_at(o, b)
|
||||
}
|
||||
}
|
||||
for publication in [false, true] {
|
||||
let (mut card, _) = formatted();
|
||||
for (publication, fail_publication, escalate, release_early) in [
|
||||
(false, false, false, false),
|
||||
(true, false, false, false),
|
||||
(true, true, false, false),
|
||||
(false, false, true, false),
|
||||
(true, false, true, false),
|
||||
(true, true, true, false),
|
||||
(false, false, true, true),
|
||||
(true, false, true, true),
|
||||
] {
|
||||
let (mut card, disk) = formatted();
|
||||
add(&mut card, 4, 1);
|
||||
let (et, er) = std::sync::mpsc::channel();
|
||||
let (rt, rr) = std::sync::mpsc::channel();
|
||||
@@ -324,6 +486,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
|
||||
entered: et,
|
||||
resume: rr,
|
||||
publication,
|
||||
fail_publication,
|
||||
}))
|
||||
.unwrap();
|
||||
let mut s = Session::default();
|
||||
@@ -336,22 +499,84 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() {
|
||||
let id = s.commit(c, 0, 7, admitted, &lane).unwrap();
|
||||
er.recv_timeout(Duration::from_secs(1)).unwrap();
|
||||
s.poll(admitted + Duration::from_millis(500));
|
||||
s.poll(admitted + Duration::from_secs(1));
|
||||
let result = s.query(c, id).unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
if publication { RequestState::Indeterminate } else { RequestState::Failure }
|
||||
);
|
||||
assert_eq!(result.reason, RequestReason::RestartRequired);
|
||||
assert!(s.host_faulted());
|
||||
s.release_request(c, id).unwrap();
|
||||
assert_guest_query(&mut s, id, 0, 0);
|
||||
let terminal = if escalate {
|
||||
s.poll(admitted + Duration::from_secs(1));
|
||||
let result = s.query(c, id).unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
if publication { RequestState::Indeterminate } else { RequestState::Failure }
|
||||
);
|
||||
assert_eq!(result.reason, RequestReason::RestartRequired);
|
||||
assert!(s.host_faulted());
|
||||
if release_early {
|
||||
s.release_request(c, id).unwrap();
|
||||
}
|
||||
Some(result)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
assert_eq!(s.write(c, 0, 0, b"unsafe"), Err(Error::Busy));
|
||||
assert_eq!(s.commit(c, 0, 7, Instant::now(), &lane), Err(Error::Busy));
|
||||
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::Busy));
|
||||
rt.send(()).unwrap();
|
||||
controller.stop().unwrap();
|
||||
s.poll(Instant::now());
|
||||
assert!(s.unsaved());
|
||||
assert!(s.host_faulted());
|
||||
s.poll(admitted + Duration::from_millis(if escalate { 1001 } else { 501 }));
|
||||
if let Some(terminal) = terminal {
|
||||
if release_early {
|
||||
assert_eq!(s.query(c, id), Err(Error::InvalidRange));
|
||||
} else {
|
||||
assert_eq!(s.query(c, id), Ok(terminal));
|
||||
assert_guest_query(&mut s, id, if publication { 3 } else { 2 }, 15);
|
||||
s.release_request(c, id).unwrap();
|
||||
}
|
||||
assert!(s.unsaved());
|
||||
assert!(s.host_faulted());
|
||||
assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::ReadOnly));
|
||||
} else {
|
||||
let expected = if fail_publication {
|
||||
RequestState::Indeterminate
|
||||
} else if publication {
|
||||
RequestState::Success
|
||||
} else {
|
||||
RequestState::Failure
|
||||
};
|
||||
assert_eq!(s.query(c, id).unwrap().state, expected);
|
||||
assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout);
|
||||
assert_guest_query(
|
||||
&mut s,
|
||||
id,
|
||||
if fail_publication {
|
||||
3
|
||||
} else if publication {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
},
|
||||
15,
|
||||
);
|
||||
assert_eq!(
|
||||
s.request_storage_error(c, id),
|
||||
Ok(if fail_publication {
|
||||
Some(Error::Io(MemcardIoError::IoError))
|
||||
} else if publication {
|
||||
None
|
||||
} else {
|
||||
Some(Error::Canceled)
|
||||
})
|
||||
);
|
||||
assert_eq!(s.unsaved(), expected != RequestState::Success);
|
||||
assert_eq!(s.removal_pending(), fail_publication);
|
||||
assert!(!s.host_faulted());
|
||||
s.release_request(c, id).unwrap();
|
||||
}
|
||||
let mut durable = reboot(&disk);
|
||||
let mut bytes = [0; 7];
|
||||
let n = durable.read_slot(0, &mut bytes).unwrap();
|
||||
if publication && !fail_publication {
|
||||
assert_eq!(&bytes[..n], b"pending");
|
||||
} else {
|
||||
assert_eq!(n, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+23
-2
@@ -40,12 +40,33 @@ already completed result, which exposed an ownership leak during integration.
|
||||
|
||||
Cancellation and entry into publication share an atomic control state. At the
|
||||
deadline, a canceled attempt that never entered publication can report failure.
|
||||
Once publication may have started, the result remains indeterminate until
|
||||
reconciliation. A blocked host requires restart escalation even when publication
|
||||
Once publication may have started, cancellation alone cannot determine the outcome.
|
||||
A worker that returns a known success before escalation still reports success;
|
||||
an uncertain publication requires reconciliation. A blocked host requires restart escalation even when publication
|
||||
was provably prevented: storage and source ownership still have to remain valid.
|
||||
The runtime neither kills the thread nor starts a replacement worker over its
|
||||
resources. The watchdog runs outside that worker.
|
||||
|
||||
The watchdog cause is a third observation, independent of durable outcome and
|
||||
resource lifetime. Keeping it only in the terminal result loses information:
|
||||
a worker's generic cancellation error can overwrite the reason the cancellation
|
||||
was requested. Conversely, setting a terminal timeout before the worker returns
|
||||
can accidentally suppress its eventual success. The request now records the
|
||||
first watchdog cancellation separately, leaves the result Pending, and preserves
|
||||
the original worker error for native diagnostics. Completion before escalation
|
||||
combines the actual durable state with guest reason 15, including Success/15.
|
||||
Removal or termination cancellation that happened first retains its own reason.
|
||||
|
||||
This correction follows accepted decision DEC-0058. Deterministic regression
|
||||
fixtures hold queued work until after the simulated 500ms poll and deliver it
|
||||
at 501ms. Channel barriers hold IO before or inside publication, exercising
|
||||
Failure/15, Success/15, Indeterminate/15 and immutable restart results after 1s.
|
||||
The tests invoke the actual guest query handler and assert all three ABI slots;
|
||||
wall-clock receive limits only detect a hung fixture. They also verify deferred
|
||||
and composite bootstrap saves, dirty snapshots, durable bytes after reopening,
|
||||
result release and teardown isolation. These checks establish behavior under
|
||||
controlled scheduling, not real-device latency.
|
||||
|
||||
## A checksum does not make an old reference safe
|
||||
|
||||
Two valid map pages can reference different generations of payload and public
|
||||
|
||||
@@ -236,6 +236,32 @@ not proof of rollback. If cessation is unconfirmed, isolate mount/resources,
|
||||
block new work and retain source/scratch. Late completion belongs to original
|
||||
reconciliation. Never kill threads unsafely, free buffers in use or create a
|
||||
replacement worker. Report restart requirement when host recovery is impossible.
|
||||
|
||||
The first watchdog cancellation is recorded independently of the request result,
|
||||
before signaling cancellation. Repeated polls preserve that cause. At 500ms the
|
||||
request remains Pending with no terminal reason until the worker returns or the
|
||||
1s escalation occurs. Removal or session termination that canceled first does
|
||||
not become a watchdog timeout merely because a later poll crosses 500ms.
|
||||
|
||||
On completion before escalation, the worker's durable outcome determines
|
||||
Success, Failure or Indeterminate. A recorded watchdog cancellation supplies
|
||||
Timeout (guest reason 15) independently of that state; native diagnostics retain
|
||||
the original worker storage error. For example:
|
||||
|
||||
- A save canceled while queued at 500ms, then completed at 501ms without entering
|
||||
publication, reports Failure/15. The durable slot is unchanged.
|
||||
- A save already inside publication at cancellation can finish successfully and
|
||||
report Success/15. Its saved snapshot becomes clean; cancellation is not rollback.
|
||||
- An uncertain publication reports Indeterminate/15 and requires reconciliation.
|
||||
- At 1s without confirmed cessation, RestartRequired reports Failure/15 before
|
||||
publication entry or Indeterminate/15 after entry. Late completion returns card
|
||||
ownership for reconciliation but cannot rewrite this terminal guest result or
|
||||
mark the snapshot clean. Releasing the result does not release in-use resources.
|
||||
|
||||
A deferred bootstrap save that has not reached the worker expires as Failure/15
|
||||
at 500ms. Memory-only commits remain MemoryOnly with reason 0 and perform no IO.
|
||||
The status/state/reason result slots and request identities do not change.
|
||||
|
||||
Fast-profile claims require measurements under asset load, including tail latency.
|
||||
|
||||
## 8. Player choice and lifecycle
|
||||
|
||||
Reference in New Issue
Block a user