fix(headless): wait for memcard bootstrap IO progress
Intrepid/Prometeu/Runtime/pipeline/head There was a failure building this commit
Intrepid/Prometeu/Runtime/pipeline/head There was a failure building this commit
Coordinate bounded native publication progress during frame advance while preserving watchdogs, guest budgets and resource ownership. Add controlled and packaged reopen regressions, clarify the runtime specs, and retain qualification evidence.
This commit is contained in:
@@ -3,6 +3,282 @@ use crate::{AdvanceFrameOutcome, HostExecutionStatus};
|
||||
use prometeu_test_support::memcard_fixture::{MemcardFixture, memcard_program};
|
||||
use std::{fs, sync::mpsc};
|
||||
|
||||
/// Holds the first native write after admission; the controller decides its outcome.
|
||||
struct BootstrapStore {
|
||||
file: FileMemcard,
|
||||
entered: mpsc::Sender<()>,
|
||||
continue_io: mpsc::Receiver<Result<(), MemcardIoError>>,
|
||||
held: bool,
|
||||
}
|
||||
impl MemcardStorage for BootstrapStore {
|
||||
fn len(&self) -> Result<u64, MemcardIoError> {
|
||||
self.file.len()
|
||||
}
|
||||
fn read_only(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn read_at(&mut self, offset: u64, bytes: &mut [u8]) -> Result<(), MemcardIoError> {
|
||||
self.file.read_at(offset, bytes)
|
||||
}
|
||||
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), MemcardIoError> {
|
||||
if !self.held {
|
||||
self.held = true;
|
||||
let _ = self.entered.send(());
|
||||
self.continue_io
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.unwrap_or(Err(MemcardIoError::IoError))?;
|
||||
}
|
||||
self.file.write_at(offset, bytes)
|
||||
}
|
||||
fn sync(&mut self) -> Result<(), MemcardIoError> {
|
||||
self.file.sync()
|
||||
}
|
||||
}
|
||||
|
||||
fn reopen_with_controlled_io(
|
||||
kind: MemcardFixture,
|
||||
) -> (Fixture, HeadlessHost, mpsc::Receiver<()>, mpsc::Sender<Result<(), MemcardIoError>>) {
|
||||
use prometeu_system::memcard::{
|
||||
image::{Game, Icon, crc32c},
|
||||
transaction::{DurableResult, PublicationControl},
|
||||
};
|
||||
let f = Fixture::new(kind);
|
||||
let file = FileMemcard::create(&f.image, IMAGE_BYTES, |io, id| {
|
||||
Card::format(io, id).map_err(|_| MemcardIoError::IoError)
|
||||
})
|
||||
.unwrap();
|
||||
let mut card = Card::mount(Box::new(file)).unwrap();
|
||||
let icon = [0; 560];
|
||||
card.register(
|
||||
Game {
|
||||
app_id: 42,
|
||||
title: "Existing record".into(),
|
||||
data_format_version: 1,
|
||||
created_at: 0,
|
||||
slots: vec![0],
|
||||
icon: Icon { offset: 0, frames: 1, duration: 1, crc32c: crc32c(&icon) },
|
||||
},
|
||||
&icon,
|
||||
&PublicationControl::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
card.commit(0, &[0, 127, 128, 255], 0, &PublicationControl::default()).durable,
|
||||
DurableResult::Success
|
||||
);
|
||||
drop(card);
|
||||
let (entered_tx, entered) = mpsc::channel();
|
||||
let (continue_io, release_rx) = mpsc::channel();
|
||||
let mut host = HeadlessHost::new();
|
||||
host.media.path = Some(f.image.clone());
|
||||
host.media.access = Some(MediaAccess::ReadWrite);
|
||||
host.install_memcard(Box::new(BootstrapStore {
|
||||
file: FileMemcard::open(&f.image, false).unwrap(),
|
||||
entered: entered_tx,
|
||||
continue_io: release_rx,
|
||||
held: false,
|
||||
}))
|
||||
.unwrap();
|
||||
host.boot_with_memcard_choice(f.cart.to_string_lossy(), Choice::Persistent).unwrap();
|
||||
assert_eq!(host.get_state().tick_index, 0);
|
||||
(f, host, entered, continue_io)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_reopen_frame_waits_for_controlled_presentation_without_replaying() {
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let (finished_tx, finished) = mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
entered.recv_timeout(Duration::from_secs(3)).unwrap();
|
||||
// Observe the owner while publication is provably held. This is a
|
||||
// channel deadline, not a sleep used to infer that IO has started.
|
||||
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
|
||||
let _ = continue_io.send(Ok(()));
|
||||
premature
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
let _ = finished_tx.send(());
|
||||
let premature = controller.join().unwrap();
|
||||
assert!(!premature, "advance returned while native publication was held: {result:?}");
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
|
||||
));
|
||||
let state = host.get_state();
|
||||
assert_eq!(state.frame_index, 1);
|
||||
assert_eq!(state.tick_index, 2);
|
||||
assert_eq!(state.status, HostExecutionStatus::Paused);
|
||||
assert_eq!(state.stack_top.last(), Some(&prometeu_bytecode::Value::Int64(201)));
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
assert!(!host.get_memcard_state().io.active);
|
||||
assert_eq!(host.capture_frame().unwrap().logical_frame_index, 1);
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_bootstrap_reaches_native_deadline_and_keeps_worker_lock() {
|
||||
let (f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let started = Instant::now();
|
||||
let result = host.advance_frame();
|
||||
entered.recv_timeout(Duration::from_secs(1)).unwrap();
|
||||
let state = host.get_memcard_state();
|
||||
let busy = matches!(FileMemcard::open(&f.image, false), Err(MemcardIoError::Busy));
|
||||
// Always let the worker return before asserting, including on the old host.
|
||||
continue_io.send(Ok(())).unwrap();
|
||||
settle(&mut host);
|
||||
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
|
||||
assert!(started.elapsed() < Duration::from_secs(3));
|
||||
assert!(state.recovery.restart_required);
|
||||
assert!(state.io.active && busy);
|
||||
assert_eq!(host.get_state().frame_index, 0);
|
||||
assert_eq!(host.get_state().tick_index, 1);
|
||||
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_gameplay_commit_does_not_wait_for_native_io() {
|
||||
let (_f, mut host, entered, continue_io) = reopen_with_controlled_io(MemcardFixture::Read);
|
||||
host.advance_frame().unwrap();
|
||||
let service = host.firmware.os.memcard();
|
||||
let context = service.session.context();
|
||||
service.session.write(context, 0, 0, &[0, 127, 128, 255]).unwrap();
|
||||
service.session.commit(context, 0, 0, Instant::now(), &service.lane).unwrap();
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
let result = host.advance_frame();
|
||||
let pending = host.get_memcard_state();
|
||||
continue_io.send(Ok(())).unwrap();
|
||||
settle(&mut host);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 2, .. })
|
||||
));
|
||||
assert!(pending.io.active);
|
||||
assert_eq!(pending.request.unwrap().operation, "commit");
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_wait_deadline_is_shared_across_wakes_and_reentry() {
|
||||
use super::super::{BOOTSTRAP_IO_WAIT_QUANTUM, BootstrapIoWait};
|
||||
let start = Instant::now();
|
||||
let mut wait = BootstrapIoWait::default();
|
||||
assert_eq!(wait.next_wait(start), Some(BOOTSTRAP_IO_WAIT_QUANTUM));
|
||||
assert_eq!(
|
||||
wait.next_wait(start + Duration::from_millis(1000)),
|
||||
Some(BOOTSTRAP_IO_WAIT_QUANTUM)
|
||||
);
|
||||
assert_eq!(wait.next_wait(start + Duration::from_millis(1999)), Some(Duration::from_millis(1)));
|
||||
assert_eq!(wait.next_wait(start + Duration::from_secs(2)), None);
|
||||
assert_eq!(wait.next_wait(start + Duration::from_secs(3)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_outer_timeout_preserves_native_snapshot_and_machine_state() {
|
||||
use super::super::BootstrapIoWait;
|
||||
let (_f, mut host, _entered, _release) = reopen_with_controlled_io(MemcardFixture::Read);
|
||||
let before = host.get_state();
|
||||
host.firmware.os.vm().set_paused(false);
|
||||
host.firmware.os.vm().request_debug_step();
|
||||
let mut wait = BootstrapIoWait { deadline: Some(Instant::now()) };
|
||||
let error = host.wait_for_bootstrap_io(&mut wait, 0).unwrap_err();
|
||||
let snapshot = host.get_memcard_state();
|
||||
let HostError::Memcard(error) = error else { panic!("expected a native memcard error") };
|
||||
assert_eq!(error.operation, "advance");
|
||||
assert_eq!(error.reason, "restart_required");
|
||||
assert_eq!(error.state, snapshot);
|
||||
let after = host.get_state();
|
||||
assert_eq!((after.pc, after.frame_index, after.tick_index), (before.pc, 0, 0));
|
||||
assert_eq!(after.stack_top.to_vec(), before.stack_top.to_vec());
|
||||
assert_eq!(after.status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_bootstrap_io_returns_native_media_action_without_guest_fault() {
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let controller = std::thread::spawn(move || {
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
continue_io.send(Err(MemcardIoError::Unavailable)).unwrap();
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
controller.join().unwrap();
|
||||
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
|
||||
let state = host.get_memcard_state();
|
||||
assert!(state.recovery.removal_pending);
|
||||
assert!(!state.io.active);
|
||||
assert_eq!(host.get_state().frame_index, 0);
|
||||
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
|
||||
assert!(!host.firmware.os.vm().debug_step_requested());
|
||||
assert!(host.get_fault().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_bootstrap_publication_can_progress_without_moving_guest_during_poll() {
|
||||
use prometeu_system::AsyncWorkJobOutcome;
|
||||
let (_f, mut host, entered, continue_io) =
|
||||
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
|
||||
let (busy_tx, busy) = mpsc::channel();
|
||||
let (unblock, unblock_rx) = mpsc::channel();
|
||||
host.firmware
|
||||
.os
|
||||
.memcard()
|
||||
.lane
|
||||
.submit_memcard_commit_write(move |_| {
|
||||
busy_tx.send(()).unwrap();
|
||||
let _ = unblock_rx.recv_timeout(Duration::from_secs(3));
|
||||
AsyncWorkJobOutcome::Completed
|
||||
})
|
||||
.unwrap();
|
||||
busy.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
for _ in 0..10 {
|
||||
host.firmware.os.vm().set_paused(false);
|
||||
host.firmware.os.vm().request_debug_step();
|
||||
host.tick_once();
|
||||
if host.bootstrap_io_pending() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let pending = host.bootstrap_io_pending();
|
||||
let queued = host.get_memcard_state();
|
||||
let before = host.get_state();
|
||||
host.poll_memcard();
|
||||
let after = host.get_state();
|
||||
let (finished_tx, finished) = mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
|
||||
let _ = unblock.send(());
|
||||
entered.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
let _ = continue_io.send(Ok(()));
|
||||
premature
|
||||
});
|
||||
let result = host.advance_frame();
|
||||
let _ = finished_tx.send(());
|
||||
let premature = controller.join().unwrap();
|
||||
assert!(pending);
|
||||
assert!(!premature, "advance returned while its publication was queued: {result:?}");
|
||||
assert_eq!(queued.io.phase, "queued");
|
||||
assert_eq!(
|
||||
(before.pc, before.frame_index, before.tick_index),
|
||||
(after.pc, after.frame_index, after.tick_index)
|
||||
);
|
||||
assert_eq!(before.stack_top.to_vec(), after.stack_top.to_vec());
|
||||
assert!(matches!(
|
||||
result,
|
||||
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
|
||||
));
|
||||
host.shutdown(TeardownPolicy::default()).unwrap();
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
root: PathBuf,
|
||||
image: PathBuf,
|
||||
@@ -10,10 +286,12 @@ struct Fixture {
|
||||
}
|
||||
impl Fixture {
|
||||
fn new(kind: MemcardFixture) -> Self {
|
||||
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"native-memcard-{}-{}",
|
||||
"native-memcard-{}-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(),
|
||||
COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||
));
|
||||
let cart = root.join("cart");
|
||||
fs::create_dir_all(&cart).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user