From bad942fb25ac622fbb9b1817c37396d87689ddcd Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Wed, 14 Oct 2026 10:26:30 +0100 Subject: [PATCH] fix(memcard): preserve watchdog timeout reasons across completion --- .../src/services/memcard/session.rs | 31 ++- .../src/services/memcard/transaction/tests.rs | 257 ++++++++++++++++-- ...ion-and-resource-ownership-are-separate.md | 25 +- .../runtime/08-save-memory-and-memcard.md | 26 ++ 4 files changed, 314 insertions(+), 25 deletions(-) diff --git a/crates/console/prometeu-system/src/services/memcard/session.rs b/crates/console/prometeu-system/src/services/memcard/session.rs index 6ffda6a..9b65eef 100644 --- a/crates/console/prometeu-system/src/services/memcard/session.rs +++ b/crates/console/prometeu-system/src/services/memcard/session.rs @@ -65,6 +65,8 @@ struct Request { control: Arc, receiver: mpsc::Receiver, admitted: Instant, + watchdog_canceled_at: Option, + worker_error: Option, result: RequestResult, job: AsyncWorkJobId, released: bool, @@ -523,6 +525,8 @@ impl Session { control, receiver, admitted, + watchdog_canceled_at: None, + worker_error: None, result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None }, job, released: !saves_payload, @@ -664,6 +668,8 @@ impl Session { control, receiver, admitted: now, + watchdog_canceled_at: None, + worker_error: None, result: RequestResult { id, state: RequestState::Pending, reason: RequestReason::None }, job, released: false, @@ -686,27 +692,29 @@ impl Session { let Some(r) = &mut self.request else { return }; match r.receiver.try_recv() { Ok(completion) => { + r.worker_error = completion.result.error; let current = self.mode.is_some() && r.session == self.context && r.epoch == self.epoch; if current { self.slots[r.local].protected = false; - if r.result.reason == RequestReason::None { + if r.result.state == RequestState::Pending { r.result.state = match completion.result.durable { DurableResult::Success => RequestState::Success, DurableResult::Failure => RequestState::Failure, DurableResult::Indeterminate => RequestState::Indeterminate, }; - r.result.reason = completion - .result - .error - .map_or(RequestReason::None, RequestReason::Storage); + r.result.reason = if r.watchdog_canceled_at.is_some() { + RequestReason::Timeout + } else { + r.worker_error.map_or(RequestReason::None, RequestReason::Storage) + }; if completion.result.durable == DurableResult::Success && r.saves_payload { let mut s = r.source.write().unwrap(); s.descriptor = completion.card.current().slots[self.logical[r.local] as usize]; s.dirty = false; } - } // A timeout result is immutable; returned card requires explicit reconciliation. + } // A restart escalation is immutable; returned card requires explicit reconciliation. if self.provisional && completion.result.durable != DurableResult::Success { self.boot_error = Some(completion.result.error.unwrap_or(Error::InvalidState)); @@ -755,7 +763,10 @@ impl Session { return; } let elapsed = now.saturating_duration_since(r.admitted); - if elapsed >= Duration::from_millis(500) { + if elapsed >= Duration::from_millis(500) && !r.control.canceled() { + // Main-lane cancellation cause is independent of the worker's durable + // outcome. Do not relabel an earlier removal or termination cancel. + r.watchdog_canceled_at = Some(now); r.control.cancel(); } if elapsed >= Duration::from_secs(1) { @@ -788,6 +799,12 @@ impl Session { .map(|r| r.result) .ok_or(Error::InvalidRange) } + /// Original worker error for native diagnostics, independent of guest deadline reason. + /// Pending and RAM-only requests have no worker error. Retention follows query/release. + pub fn request_storage_error(&self, context: u64, id: u64) -> Result> { + self.query(context, id)?; + Ok(self.request.as_ref().filter(|r| r.id == id).and_then(|r| r.worker_error)) + } pub fn release_request(&mut self, context: u64, id: u64) -> Result<()> { self.query(context, id)?; if self.deferred.is_some() { diff --git a/crates/console/prometeu-system/src/services/memcard/transaction/tests.rs b/crates/console/prometeu-system/src/services/memcard/transaction/tests.rs index 9da9afc..02106ee 100644 --- a/crates/console/prometeu-system/src/services/memcard/transaction/tests.rs +++ b/crates/console/prometeu-system/src/services/memcard/transaction/tests.rs @@ -280,8 +280,157 @@ fn async_commit_protects_only_source_and_retains_result_without_occupying_lane() controller.stop().unwrap(); } +// The sentinel executes after the save on the single lane. Wall time only bounds +// a broken fixture; all watchdog observations use explicit simulated instants. +fn finish_queued_save(lane: Arc) { + let (tx, rx) = std::sync::mpsc::channel(); + lane.submit_asset_load(move |_| { + tx.send(()).unwrap(); + crate::services::async_work::AsyncWorkJobOutcome::Completed + }) + .unwrap(); + let mut controller = + crate::AsyncWorkLaneController::start(crate::AsyncWorkLaneConfig::default(), lane); + rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + controller.stop().unwrap(); +} + +fn assert_guest_query( + session: &mut super::super::session::Session, + id: u64, + state: i64, + reason: i64, +) { + use prometeu_bytecode::Value; + use prometeu_hal::{ + HostReturn, host_return::GuestStringAllocator, syscalls::Syscall, vm_fault::VmFault, + }; + struct NoStrings; + impl GuestStringAllocator for NoStrings { + fn byte_limit(&self) -> u64 { + 0 + } + fn build( + &mut self, + _: usize, + _: &mut dyn FnMut(&mut dyn std::fmt::Write) -> std::fmt::Result, + ) -> std::result::Result, VmFault> { + panic!("MemQuery must not allocate strings") + } + } + let mut service = super::super::MemcardService::new(); + std::mem::swap(session, &mut service.session); + let args = [Value::Int64(service.session.context() as i64), Value::Int64(id as i64)]; + let mut values = Vec::with_capacity(Syscall::MemQuery.results_count()); + let mut strings = NoStrings; + let mut ret = HostReturn::new(&mut values, Syscall::MemQuery.results_count(), &mut strings); + service.guest(Syscall::MemQuery, &args, &mut ret, 4, 0, true).unwrap(); + assert!(!ret.overflowed()); + assert!(ret.failure().is_none()); + assert_eq!(values, vec![Value::Int64(0), Value::Int64(state), Value::Int64(reason)]); + std::mem::swap(session, &mut service.session); +} + #[test] -fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() { +fn queued_memcard_watchdog_preserves_timeout_and_guest_reason() { + use super::super::session::{RequestReason, RequestState, Session}; + use std::time::{Duration, Instant}; + for bootstrap in [false, true] { + let (mut card, disk) = formatted(); + add(&mut card, 4, 1); + let before = card.current().slots[0]; + let mut s = Session::default(); + let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap(); + s.write(c, 0, 0, b"pending").unwrap(); + let lane = Arc::new(crate::AsyncWorkLane::default()); + let now = Instant::now(); + if bootstrap { + s.begin_boot(); + } + let id = s.commit(c, 0, 7, now, &lane).unwrap(); + if bootstrap { + s.finish_boot(true, &lane, now).unwrap(); + } + s.poll(now + Duration::from_millis(499)); + assert_guest_query(&mut s, id, 0, 0); + for _ in 0..2 { + s.poll(now + Duration::from_millis(500)); + } + assert_guest_query(&mut s, id, 0, 0); + assert_eq!(s.release_request(c, id), Err(Error::Busy)); + assert_eq!(s.clear(c, 0), Err(Error::Busy)); + finish_queued_save(lane); + s.poll(now + Duration::from_millis(501)); + let result = s.query(c, id).unwrap(); + assert_eq!(result.state, RequestState::Failure); + assert_eq!(result.reason, RequestReason::Timeout); + assert_eq!(s.request_storage_error(c, id), Ok(Some(Error::Canceled))); + assert_guest_query(&mut s, id, 2, 15); + assert!(s.unsaved()); + assert_eq!(reboot(&disk).current().slots[0], before); + assert_eq!(s.query(c + 1, id), Err(Error::AccessDenied)); + s.release_request(c, id).unwrap(); + assert_eq!(s.query(c, id), Err(Error::InvalidRange)); + s.clear(c, 0).unwrap(); + } +} + +#[test] +fn lifecycle_cancellation_is_not_a_watchdog_timeout() { + use super::super::session::{RequestReason, RequestState, Session}; + use std::time::{Duration, Instant}; + for end in [false, true] { + let (mut card, _) = formatted(); + add(&mut card, 4, 1); + let mut s = Session::default(); + let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap(); + let lane = Arc::new(crate::AsyncWorkLane::default()); + let now = Instant::now(); + let id = s.commit(c, 0, 7, now, &lane).unwrap(); + if end { + s.end_session(); + } else { + s.notify_removal(); + } + s.poll(now + Duration::from_millis(500)); + finish_queued_save(lane); + s.poll(now + Duration::from_millis(501)); + if end { + assert_eq!(s.query(c, id), Err(Error::AccessDenied)); + let card = s.take_idle_card().unwrap().unwrap(); + let new = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap(); + assert_ne!(new, c); + assert_eq!(s.query(new, id), Err(Error::InvalidRange)); + } else { + assert_eq!(s.query(c, id).unwrap().state, RequestState::Failure); + assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Storage(Error::Canceled)); + assert_guest_query(&mut s, id, 2, 8); + } + } +} + +#[test] +fn deferred_boot_memcard_deadline_stays_timeout() { + use super::super::session::{RequestReason, Session}; + use std::time::{Duration, Instant}; + let (mut card, _) = formatted(); + add(&mut card, 4, 1); + let mut s = Session::default(); + let c = s.load_card(card, 4, 1, Session::required_bytes(1).unwrap()).unwrap(); + s.begin_boot(); + let lane = crate::AsyncWorkLane::default(); + let now = Instant::now(); + let id = s.commit(c, 0, 7, now, &lane).unwrap(); + s.poll(now + Duration::from_millis(500)); + assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout); + assert_guest_query(&mut s, id, 2, 15); + assert_eq!(lane.pending_depth(), 0); + s.clear(c, 0).unwrap(); + s.release_request(c, id).unwrap(); +} + +#[test] +fn blocked_host_memcard_watchdog_preserves_outcome_and_escalation() { use crate::services::async_work::{ AsyncWorkLane, AsyncWorkLaneConfig, AsyncWorkLaneController, }; @@ -290,6 +439,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() { struct Blocked { store: Box, publication: bool, + fail_publication: bool, entered: std::sync::mpsc::Sender<()>, resume: std::sync::mpsc::Receiver<()>, } @@ -310,12 +460,24 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() { if if self.publication { b.starts_with(b"PMMAP1\0\0") } else { o >= DATA_OFFSET } { self.entered.send(()).unwrap(); self.resume.recv().unwrap(); + if self.fail_publication { + return Err(MemcardIoError::IoError); + } } self.store.write_at(o, b) } } - for publication in [false, true] { - let (mut card, _) = formatted(); + for (publication, fail_publication, escalate, release_early) in [ + (false, false, false, false), + (true, false, false, false), + (true, true, false, false), + (false, false, true, false), + (true, false, true, false), + (true, true, true, false), + (false, false, true, true), + (true, false, true, true), + ] { + let (mut card, disk) = formatted(); add(&mut card, 4, 1); let (et, er) = std::sync::mpsc::channel(); let (rt, rr) = std::sync::mpsc::channel(); @@ -324,6 +486,7 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() { entered: et, resume: rr, publication, + fail_publication, })) .unwrap(); let mut s = Session::default(); @@ -336,22 +499,84 @@ fn blocked_host_timeout_release_keeps_source_and_prevents_replacement_work() { let id = s.commit(c, 0, 7, admitted, &lane).unwrap(); er.recv_timeout(Duration::from_secs(1)).unwrap(); s.poll(admitted + Duration::from_millis(500)); - s.poll(admitted + Duration::from_secs(1)); - let result = s.query(c, id).unwrap(); - assert_eq!( - result.state, - if publication { RequestState::Indeterminate } else { RequestState::Failure } - ); - assert_eq!(result.reason, RequestReason::RestartRequired); - assert!(s.host_faulted()); - s.release_request(c, id).unwrap(); + assert_guest_query(&mut s, id, 0, 0); + let terminal = if escalate { + s.poll(admitted + Duration::from_secs(1)); + let result = s.query(c, id).unwrap(); + assert_eq!( + result.state, + if publication { RequestState::Indeterminate } else { RequestState::Failure } + ); + assert_eq!(result.reason, RequestReason::RestartRequired); + assert!(s.host_faulted()); + if release_early { + s.release_request(c, id).unwrap(); + } + Some(result) + } else { + None + }; assert_eq!(s.write(c, 0, 0, b"unsafe"), Err(Error::Busy)); - assert_eq!(s.commit(c, 0, 7, Instant::now(), &lane), Err(Error::Busy)); + assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::Busy)); rt.send(()).unwrap(); controller.stop().unwrap(); - s.poll(Instant::now()); - assert!(s.unsaved()); - assert!(s.host_faulted()); + s.poll(admitted + Duration::from_millis(if escalate { 1001 } else { 501 })); + if let Some(terminal) = terminal { + if release_early { + assert_eq!(s.query(c, id), Err(Error::InvalidRange)); + } else { + assert_eq!(s.query(c, id), Ok(terminal)); + assert_guest_query(&mut s, id, if publication { 3 } else { 2 }, 15); + s.release_request(c, id).unwrap(); + } + assert!(s.unsaved()); + assert!(s.host_faulted()); + assert_eq!(s.commit(c, 0, 7, admitted, &lane), Err(Error::ReadOnly)); + } else { + let expected = if fail_publication { + RequestState::Indeterminate + } else if publication { + RequestState::Success + } else { + RequestState::Failure + }; + assert_eq!(s.query(c, id).unwrap().state, expected); + assert_eq!(s.query(c, id).unwrap().reason, RequestReason::Timeout); + assert_guest_query( + &mut s, + id, + if fail_publication { + 3 + } else if publication { + 1 + } else { + 2 + }, + 15, + ); + assert_eq!( + s.request_storage_error(c, id), + Ok(if fail_publication { + Some(Error::Io(MemcardIoError::IoError)) + } else if publication { + None + } else { + Some(Error::Canceled) + }) + ); + assert_eq!(s.unsaved(), expected != RequestState::Success); + assert_eq!(s.removal_pending(), fail_publication); + assert!(!s.host_faulted()); + s.release_request(c, id).unwrap(); + } + let mut durable = reboot(&disk); + let mut bytes = [0; 7]; + let n = durable.read_slot(0, &mut bytes).unwrap(); + if publication && !fail_publication { + assert_eq!(&bytes[..n], b"pending"); + } else { + assert_eq!(n, 0); + } } } diff --git a/discussion/lessons/DSC-0061-memcard-fixed-slots-v1/LSN-0067-save-publication-and-resource-ownership-are-separate.md b/discussion/lessons/DSC-0061-memcard-fixed-slots-v1/LSN-0067-save-publication-and-resource-ownership-are-separate.md index d5c6ad3..88d8b8a 100644 --- a/discussion/lessons/DSC-0061-memcard-fixed-slots-v1/LSN-0067-save-publication-and-resource-ownership-are-separate.md +++ b/discussion/lessons/DSC-0061-memcard-fixed-slots-v1/LSN-0067-save-publication-and-resource-ownership-are-separate.md @@ -40,12 +40,33 @@ already completed result, which exposed an ownership leak during integration. Cancellation and entry into publication share an atomic control state. At the deadline, a canceled attempt that never entered publication can report failure. -Once publication may have started, the result remains indeterminate until -reconciliation. A blocked host requires restart escalation even when publication +Once publication may have started, cancellation alone cannot determine the outcome. +A worker that returns a known success before escalation still reports success; +an uncertain publication requires reconciliation. A blocked host requires restart escalation even when publication was provably prevented: storage and source ownership still have to remain valid. The runtime neither kills the thread nor starts a replacement worker over its resources. The watchdog runs outside that worker. +The watchdog cause is a third observation, independent of durable outcome and +resource lifetime. Keeping it only in the terminal result loses information: +a worker's generic cancellation error can overwrite the reason the cancellation +was requested. Conversely, setting a terminal timeout before the worker returns +can accidentally suppress its eventual success. The request now records the +first watchdog cancellation separately, leaves the result Pending, and preserves +the original worker error for native diagnostics. Completion before escalation +combines the actual durable state with guest reason 15, including Success/15. +Removal or termination cancellation that happened first retains its own reason. + +This correction follows accepted decision DEC-0058. Deterministic regression +fixtures hold queued work until after the simulated 500ms poll and deliver it +at 501ms. Channel barriers hold IO before or inside publication, exercising +Failure/15, Success/15, Indeterminate/15 and immutable restart results after 1s. +The tests invoke the actual guest query handler and assert all three ABI slots; +wall-clock receive limits only detect a hung fixture. They also verify deferred +and composite bootstrap saves, dirty snapshots, durable bytes after reopening, +result release and teardown isolation. These checks establish behavior under +controlled scheduling, not real-device latency. + ## A checksum does not make an old reference safe Two valid map pages can reference different generations of payload and public diff --git a/docs/specs/runtime/08-save-memory-and-memcard.md b/docs/specs/runtime/08-save-memory-and-memcard.md index 9e01577..c244d5f 100644 --- a/docs/specs/runtime/08-save-memory-and-memcard.md +++ b/docs/specs/runtime/08-save-memory-and-memcard.md @@ -236,6 +236,32 @@ not proof of rollback. If cessation is unconfirmed, isolate mount/resources, block new work and retain source/scratch. Late completion belongs to original reconciliation. Never kill threads unsafely, free buffers in use or create a replacement worker. Report restart requirement when host recovery is impossible. + +The first watchdog cancellation is recorded independently of the request result, +before signaling cancellation. Repeated polls preserve that cause. At 500ms the +request remains Pending with no terminal reason until the worker returns or the +1s escalation occurs. Removal or session termination that canceled first does +not become a watchdog timeout merely because a later poll crosses 500ms. + +On completion before escalation, the worker's durable outcome determines +Success, Failure or Indeterminate. A recorded watchdog cancellation supplies +Timeout (guest reason 15) independently of that state; native diagnostics retain +the original worker storage error. For example: + +- A save canceled while queued at 500ms, then completed at 501ms without entering + publication, reports Failure/15. The durable slot is unchanged. +- A save already inside publication at cancellation can finish successfully and + report Success/15. Its saved snapshot becomes clean; cancellation is not rollback. +- An uncertain publication reports Indeterminate/15 and requires reconciliation. +- At 1s without confirmed cessation, RestartRequired reports Failure/15 before + publication entry or Indeterminate/15 after entry. Late completion returns card + ownership for reconciliation but cannot rewrite this terminal guest result or + mark the snapshot clean. Releasing the result does not release in-use resources. + +A deferred bootstrap save that has not reached the worker expires as Failure/15 +at 500ms. Memory-only commits remain MemoryOnly with reason 0 and perform no IO. +The status/state/reason result slots and request identities do not change. + Fast-profile claims require measurements under asset load, including tail latency. ## 8. Player choice and lifecycle