fix(headless): wait for memcard bootstrap IO progress
Intrepid/Prometeu/Runtime/pipeline/head There was a failure building this commit
Intrepid/Prometeu/Runtime/pipeline/head There was a failure building this commit
Coordinate bounded native publication progress during frame advance while preserving watchdogs, guest budgets and resource ownership. Add controlled and packaged reopen regressions, clarify the runtime specs, and retain qualification evidence.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use prometeu_drivers::hardware::Hardware;
|
||||
use prometeu_drivers::{ControlledWallClock, MemoryBanks};
|
||||
@@ -20,6 +21,21 @@ pub use memcard::{
|
||||
|
||||
const BOOT_TICK_LIMIT: usize = 5000;
|
||||
const HOST_FRAME_ADVANCE_TICK_LIMIT: usize = 100_000;
|
||||
const BOOTSTRAP_IO_WAIT_QUANTUM: Duration = Duration::from_millis(5);
|
||||
const BOOTSTRAP_IO_WAIT_LIMIT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// One deadline for the entire advance, including repeated admission to the gate.
|
||||
#[derive(Default)]
|
||||
struct BootstrapIoWait {
|
||||
deadline: Option<Instant>,
|
||||
}
|
||||
impl BootstrapIoWait {
|
||||
fn next_wait(&mut self, now: Instant) -> Option<Duration> {
|
||||
let deadline = *self.deadline.get_or_insert(now + BOOTSTRAP_IO_WAIT_LIMIT);
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
(!remaining.is_zero()).then_some(remaining.min(BOOTSTRAP_IO_WAIT_QUANTUM))
|
||||
}
|
||||
}
|
||||
|
||||
/// One published logical frame plus the automation counter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -360,6 +376,7 @@ impl HeadlessHost {
|
||||
self.firmware.os.vm().set_paused(false);
|
||||
self.firmware.os.vm().request_debug_step();
|
||||
|
||||
let mut io_wait = BootstrapIoWait::default();
|
||||
for _ in 0..HOST_FRAME_ADVANCE_TICK_LIMIT {
|
||||
// A controlled lifecycle slice can consume the runtime step marker
|
||||
// without closing a logical frame. This operation still owns the
|
||||
@@ -398,6 +415,24 @@ impl HeadlessHost {
|
||||
frame_id: frame.frame_id,
|
||||
});
|
||||
}
|
||||
|
||||
loop {
|
||||
// Refresh before every observation/poll: another lane job may
|
||||
// have progressed without completing this publication.
|
||||
let progress = self.firmware.os.memcard().lane.progress_stamp();
|
||||
if !self.bootstrap_io_pending() {
|
||||
break;
|
||||
}
|
||||
// Native polling and waiting do not execute machine ticks.
|
||||
self.poll_memcard();
|
||||
if self.host_action_required() {
|
||||
self.finish_frame_advance();
|
||||
return Ok(AdvanceFrameOutcome::ActionRequired);
|
||||
}
|
||||
if self.bootstrap_io_pending() {
|
||||
self.wait_for_bootstrap_io(&mut io_wait, progress)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.finish_frame_advance();
|
||||
@@ -406,6 +441,32 @@ impl HeadlessHost {
|
||||
})
|
||||
}
|
||||
|
||||
fn bootstrap_io_pending(&mut self) -> bool {
|
||||
let FirmwareState::GameRunning(step) = &self.firmware.state else { return false };
|
||||
let task = step.task_id;
|
||||
let hook_complete = self
|
||||
.firmware
|
||||
.os
|
||||
.sessions()
|
||||
.vm_session_for_task(task)
|
||||
.is_some_and(|session| session.vm.memcard_boot_result() == Some(0));
|
||||
let service = self.firmware.os.memcard();
|
||||
hook_complete && service.session.boot_pending() && service.session.io_active()
|
||||
}
|
||||
|
||||
fn wait_for_bootstrap_io(
|
||||
&mut self,
|
||||
wait: &mut BootstrapIoWait,
|
||||
progress: u64,
|
||||
) -> Result<(), HostError> {
|
||||
let Some(timeout) = wait.next_wait(Instant::now()) else {
|
||||
self.finish_frame_advance();
|
||||
return Err(self.memcard_error("advance", "restart_required"));
|
||||
};
|
||||
self.firmware.os.memcard().lane.wait_for_progress(progress, timeout);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn advance_frames(&mut self, count: u64) -> Result<AdvanceFramesOutcome, HostError> {
|
||||
let mut completed = 0;
|
||||
for _ in 0..count {
|
||||
|
||||
@@ -3,6 +3,282 @@ use crate::{AdvanceFrameOutcome, HostExecutionStatus};
|
||||
use prometeu_test_support::memcard_fixture::{MemcardFixture, memcard_program};
|
||||
use std::{fs, sync::mpsc};
|
||||
|
||||
/// Holds the first native write after admission; the controller decides its outcome.
|
||||
struct BootstrapStore {
|
||||
file: FileMemcard,
|
||||
entered: mpsc::Sender<()>,
|
||||
continue_io: mpsc::Receiver<Result<(), MemcardIoError>>,
|
||||
held: bool,
|
||||
}
|
||||
impl MemcardStorage for BootstrapStore {
|
||||
fn len(&self) -> Result<u64, MemcardIoError> {
|
||||
self.file.len()
|
||||
}
|
||||
fn read_only(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn read_at(&mut self, offset: u64, bytes: &mut [u8]) -> Result<(), MemcardIoError> {
|
||||
self.file.read_at(offset, bytes)
|
||||
}
|
||||
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), MemcardIoError> {
|
||||
if !self.held {
|
||||
self.held = true;
|
||||
let _ = self.entered.send(());
|
||||
self.continue_io
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or(Err(MemcardIoError::IoError))?;
|
||||
}
|
||||
self.file.write_at(offset, bytes)
|
||||
}
|
||||
fn sync(&mut self) -> Result<(), MemcardIoError> {
|
||||
self.file.sync()
|
||||
}
|
||||
}
|
||||
|
||||
fn reopen_with_controlled_io(
|
||||
kind: MemcardFixture,
|
||||
) -> (Fixture, HeadlessHost, mpsc::Receiver<()>, mpsc::Sender<Result<(), MemcardIoError>>) {
|
||||
use prometeu_system::memcard::{
|
||||
image::{Game, Icon, crc32c},
|
||||
transaction::{DurableResult, PublicationControl},
|
||||
};
|
||||
let f = Fixture::new(kind);
|
||||
let file = FileMemcard::create(&f.image, IMAGE_BYTES, |io, id| {
|
||||
Card::format(io, id).map_err(|_| MemcardIoError::IoError)
|
||||
})
|
||||
.unwrap();
|
||||
let mut card = Card::mount(Box::new(file)).unwrap();
|
||||
let icon = [0; 560];
|
||||
card.register(
|
||||
Game {
|
||||
app_id: 42,
|
||||
title: "Existing record".into(),
|
||||
data_format_version: 1,
|
||||
created_at: 0,
|
||||
slots: vec![0],
|
||||
icon: Icon { offset: 0, frames: 1, duration: 1, crc32c: crc32c(&icon) },
|
||||
},
|
||||
&icon,
|
||||
&PublicationControl::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
card.commit(0, &[0, 127, 128, 255], 0, &PublicationControl::default()).durable,
|
||||
DurableResult::Success
|
||||
);
|
||||
drop(card);
|
||||
let (entered_tx, entered) = mpsc::channel();
|
||||
let (continue_io, release_rx) = mpsc::channel();
|
||||
let mut host = HeadlessHost::new();
|
||||
host.media.path = Some(f.image.clone());
|
||||
host.media.access = Some(MediaAccess::ReadWrite);
|
||||
host.install_memcard(Box::new(BootstrapStore {
|
||||
file: FileMemcard::open(&f.image, false).unwrap(),
|
||||
entered: entered_tx,
|
||||
continue_io: release_rx,
|
||||
held: false,
|
||||
}))
|
||||
.unwrap();
|
||||
host.boot_with_memcard_choice(f.cart.to_string_lossy(), Choice::Persistent).unwrap();
|
||||
assert_eq!(host.get_state().tick_index, 0);
|
||||
(f, host, entered, continue_io)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_reopen_frame_waits_for_controlled_presentation_without_replaying() {
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let (finished_tx, finished) = mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
entered.recv_timeout(Duration::from_secs(3)).unwrap();
|
||||
// Observe the owner while publication is provably held. This is a
|
||||
// channel deadline, not a sleep used to infer that IO has started.
|
||||
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
|
||||
let _ = continue_io.send(Ok(()));
|
||||
premature
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
let _ = finished_tx.send(());
|
||||
let premature = controller.join().unwrap();
|
||||
assert!(!premature, "advance returned while native publication was held: {result:?}");
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
|
||||
));
|
||||
let state = host.get_state();
|
||||
assert_eq!(state.frame_index, 1);
|
||||
assert_eq!(state.tick_index, 2);
|
||||
assert_eq!(state.status, HostExecutionStatus::Paused);
|
||||
assert_eq!(state.stack_top.last(), Some(&prometeu_bytecode::Value::Int64(201)));
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
assert!(!host.get_memcard_state().io.active);
|
||||
assert_eq!(host.capture_frame().unwrap().logical_frame_index, 1);
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_bootstrap_reaches_native_deadline_and_keeps_worker_lock() {
|
||||
let (f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let started = Instant::now();
|
||||
let result = host.advance_frame();
|
||||
entered.recv_timeout(Duration::from_secs(1)).unwrap();
|
||||
let state = host.get_memcard_state();
|
||||
let busy = matches!(FileMemcard::open(&f.image, false), Err(MemcardIoError::Busy));
|
||||
// Always let the worker return before asserting, including on the old host.
|
||||
continue_io.send(Ok(())).unwrap();
|
||||
settle(&mut host);
|
||||
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
|
||||
assert!(started.elapsed() < Duration::from_secs(3));
|
||||
assert!(state.recovery.restart_required);
|
||||
assert!(state.io.active && busy);
|
||||
assert_eq!(host.get_state().frame_index, 0);
|
||||
assert_eq!(host.get_state().tick_index, 1);
|
||||
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_gameplay_commit_does_not_wait_for_native_io() {
|
||||
let (_f, mut host, entered, continue_io) = reopen_with_controlled_io(MemcardFixture::Read);
|
||||
host.advance_frame().unwrap();
|
||||
let service = host.firmware.os.memcard();
|
||||
let context = service.session.context();
|
||||
service.session.write(context, 0, 0, &[0, 127, 128, 255]).unwrap();
|
||||
service.session.commit(context, 0, 0, Instant::now(), &service.lane).unwrap();
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
let result = host.advance_frame();
|
||||
let pending = host.get_memcard_state();
|
||||
continue_io.send(Ok(())).unwrap();
|
||||
settle(&mut host);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 2, .. })
|
||||
));
|
||||
assert!(pending.io.active);
|
||||
assert_eq!(pending.request.unwrap().operation, "commit");
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_wait_deadline_is_shared_across_wakes_and_reentry() {
|
||||
use super::super::{BOOTSTRAP_IO_WAIT_QUANTUM, BootstrapIoWait};
|
||||
let start = Instant::now();
|
||||
let mut wait = BootstrapIoWait::default();
|
||||
assert_eq!(wait.next_wait(start), Some(BOOTSTRAP_IO_WAIT_QUANTUM));
|
||||
assert_eq!(
|
||||
wait.next_wait(start + Duration::from_millis(1000)),
|
||||
Some(BOOTSTRAP_IO_WAIT_QUANTUM)
|
||||
);
|
||||
assert_eq!(wait.next_wait(start + Duration::from_millis(1999)), Some(Duration::from_millis(1)));
|
||||
assert_eq!(wait.next_wait(start + Duration::from_secs(2)), None);
|
||||
assert_eq!(wait.next_wait(start + Duration::from_secs(3)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_outer_timeout_preserves_native_snapshot_and_machine_state() {
|
||||
use super::super::BootstrapIoWait;
|
||||
let (_f, mut host, _entered, _release) = reopen_with_controlled_io(MemcardFixture::Read);
|
||||
let before = host.get_state();
|
||||
host.firmware.os.vm().set_paused(false);
|
||||
host.firmware.os.vm().request_debug_step();
|
||||
let mut wait = BootstrapIoWait { deadline: Some(Instant::now()) };
|
||||
let error = host.wait_for_bootstrap_io(&mut wait, 0).unwrap_err();
|
||||
let snapshot = host.get_memcard_state();
|
||||
let HostError::Memcard(error) = error else { panic!("expected a native memcard error") };
|
||||
assert_eq!(error.operation, "advance");
|
||||
assert_eq!(error.reason, "restart_required");
|
||||
assert_eq!(error.state, snapshot);
|
||||
let after = host.get_state();
|
||||
assert_eq!((after.pc, after.frame_index, after.tick_index), (before.pc, 0, 0));
|
||||
assert_eq!(after.stack_top.to_vec(), before.stack_top.to_vec());
|
||||
assert_eq!(after.status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_bootstrap_io_returns_native_media_action_without_guest_fault() {
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let controller = std::thread::spawn(move || {
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
continue_io.send(Err(MemcardIoError::Unavailable)).unwrap();
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
controller.join().unwrap();
|
||||
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
|
||||
let state = host.get_memcard_state();
|
||||
assert!(state.recovery.removal_pending);
|
||||
assert!(!state.io.active);
|
||||
assert_eq!(host.get_state().frame_index, 0);
|
||||
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_bootstrap_publication_can_progress_without_moving_guest_during_poll() {
|
||||
use prometeu_system::AsyncWorkJobOutcome;
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let (busy_tx, busy) = mpsc::channel();
|
||||
let (unblock, unblock_rx) = mpsc::channel();
|
||||
host.firmware
|
||||
.os
|
||||
.memcard()
|
||||
.lane
|
||||
.submit_memcard_commit_write(move |_| {
|
||||
busy_tx.send(()).unwrap();
|
||||
let _ = unblock_rx.recv_timeout(Duration::from_secs(3));
|
||||
AsyncWorkJobOutcome::Completed
|
||||
})
|
||||
.unwrap();
|
||||
busy.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
for _ in 0..10 {
|
||||
host.firmware.os.vm().set_paused(false);
|
||||
host.firmware.os.vm().request_debug_step();
|
||||
host.tick_once();
|
||||
if host.bootstrap_io_pending() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let pending = host.bootstrap_io_pending();
|
||||
let queued = host.get_memcard_state();
|
||||
let before = host.get_state();
|
||||
host.poll_memcard();
|
||||
let after = host.get_state();
|
||||
let (finished_tx, finished) = mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
|
||||
let _ = unblock.send(());
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
let _ = continue_io.send(Ok(()));
|
||||
premature
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
let _ = finished_tx.send(());
|
||||
let premature = controller.join().unwrap();
|
||||
assert!(pending);
|
||||
assert!(!premature, "advance returned while its publication was queued: {result:?}");
|
||||
assert_eq!(queued.io.phase, "queued");
|
||||
assert_eq!(
|
||||
(before.pc, before.frame_index, before.tick_index),
|
||||
(after.pc, after.frame_index, after.tick_index)
|
||||
);
|
||||
assert_eq!(before.stack_top.to_vec(), after.stack_top.to_vec());
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
|
||||
));
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
root: PathBuf,
|
||||
image: PathBuf,
|
||||
@@ -10,10 +286,12 @@ struct Fixture {
|
||||
}
|
||||
impl Fixture {
|
||||
fn new(kind: MemcardFixture) -> Self {
|
||||
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"native-memcard-{}-{}",
|
||||
"native-memcard-{}-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(),
|
||||
COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||
));
|
||||
let cart = root.join("cart");
|
||||
fs::create_dir_all(&cart).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user