fix(headless): wait for memcard bootstrap IO progress
Intrepid/Prometeu/Runtime/pipeline/head There was a failure building this commit

Coordinate bounded native publication progress during frame advance while preserving watchdogs, guest budgets and resource ownership. Add controlled and packaged reopen regressions, clarify the runtime specs, and retain qualification evidence.
This commit is contained in:
2026-11-10 23:26:40 +00:00
parent 49d7fca0b7
commit c3589be1ad
34 changed files with 5904 additions and 5 deletions
@@ -11,6 +11,7 @@ use std::collections::BTreeMap;
pub enum MemcardFixture {
Write,
Read,
ReadWithPresentation,
Dirty,
}
#[derive(Default)]
@@ -47,12 +48,14 @@ impl Code {
/// 104 MEMORY_ONLY, 201 exact existing readback, 301 dirty/uncommitted RAM.
/// Read never writes or submits a commit; a missing/wrong payload traps.
pub fn memcard_program(kind: MemcardFixture) -> Vec<u8> {
let reads = matches!(kind, MemcardFixture::Read | MemcardFixture::ReadWithPresentation);
let mut main = Code::default();
main.emit("HOSTCALL 0");
main.check(0, true);
main.emit("PUSH_I64 4\nARRAY_NEW 9\nSET_GLOBAL 1");
if matches!(kind, MemcardFixture::Read) {
main.emit("GET_GLOBAL 0\nPUSH_I32 0\nPUSH_I32 0\nGET_GLOBAL 1\nPUSH_I32 0\nPUSH_I32 4\nHOSTCALL 1");
if reads {
let read_import = if matches!(kind, MemcardFixture::Read) { 1 } else { 2 };
main.emit(&format!("GET_GLOBAL 0\nPUSH_I32 0\nPUSH_I32 0\nGET_GLOBAL 1\nPUSH_I32 0\nPUSH_I32 4\nHOSTCALL {read_import}"));
main.check(4, false);
main.check(0, false);
for (i, value) in [0, 127, 128, 255].into_iter().enumerate() {
@@ -101,7 +104,9 @@ pub fn memcard_program(kind: MemcardFixture) -> Vec<u8> {
imports.push(("presentation", 8, 1));
}
match kind {
MemcardFixture::Read => imports.push(("slot_read", 6, 2)),
MemcardFixture::Read | MemcardFixture::ReadWithPresentation => {
imports.push(("slot_read", 6, 2));
}
MemcardFixture::Dirty => imports.push(("slot_write", 6, 2)),
MemcardFixture::Write => {
imports.extend([("slot_write", 6, 2), ("slot_commit", 2, 2), ("query", 2, 3)])
@@ -1,5 +1,6 @@
use std::fmt;
use std::sync::Arc;
use std::time::{Duration, Instant};
use prometeu_drivers::hardware::Hardware;
use prometeu_drivers::{ControlledWallClock, MemoryBanks};
@@ -20,6 +21,21 @@ pub use memcard::{
const BOOT_TICK_LIMIT: usize = 5000;
const HOST_FRAME_ADVANCE_TICK_LIMIT: usize = 100_000;
const BOOTSTRAP_IO_WAIT_QUANTUM: Duration = Duration::from_millis(5);
const BOOTSTRAP_IO_WAIT_LIMIT: Duration = Duration::from_secs(2);
/// One deadline for the entire advance, including repeated admission to the gate.
#[derive(Default)]
struct BootstrapIoWait {
deadline: Option<Instant>,
}
impl BootstrapIoWait {
fn next_wait(&mut self, now: Instant) -> Option<Duration> {
let deadline = *self.deadline.get_or_insert(now + BOOTSTRAP_IO_WAIT_LIMIT);
let remaining = deadline.saturating_duration_since(now);
(!remaining.is_zero()).then_some(remaining.min(BOOTSTRAP_IO_WAIT_QUANTUM))
}
}
/// One published logical frame plus the automation counter.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -360,6 +376,7 @@ impl HeadlessHost {
self.firmware.os.vm().set_paused(false);
self.firmware.os.vm().request_debug_step();
let mut io_wait = BootstrapIoWait::default();
for _ in 0..HOST_FRAME_ADVANCE_TICK_LIMIT {
// A controlled lifecycle slice can consume the runtime step marker
// without closing a logical frame. This operation still owns the
@@ -398,6 +415,24 @@ impl HeadlessHost {
frame_id: frame.frame_id,
});
}
loop {
// Refresh before every observation/poll: another lane job may
// have progressed without completing this publication.
let progress = self.firmware.os.memcard().lane.progress_stamp();
if !self.bootstrap_io_pending() {
break;
}
// Native polling and waiting do not execute machine ticks.
self.poll_memcard();
if self.host_action_required() {
self.finish_frame_advance();
return Ok(AdvanceFrameOutcome::ActionRequired);
}
if self.bootstrap_io_pending() {
self.wait_for_bootstrap_io(&mut io_wait, progress)?;
}
}
}
self.finish_frame_advance();
@@ -406,6 +441,32 @@ impl HeadlessHost {
})
}
fn bootstrap_io_pending(&mut self) -> bool {
let FirmwareState::GameRunning(step) = &self.firmware.state else { return false };
let task = step.task_id;
let hook_complete = self
.firmware
.os
.sessions()
.vm_session_for_task(task)
.is_some_and(|session| session.vm.memcard_boot_result() == Some(0));
let service = self.firmware.os.memcard();
hook_complete && service.session.boot_pending() && service.session.io_active()
}
fn wait_for_bootstrap_io(
&mut self,
wait: &mut BootstrapIoWait,
progress: u64,
) -> Result<(), HostError> {
let Some(timeout) = wait.next_wait(Instant::now()) else {
self.finish_frame_advance();
return Err(self.memcard_error("advance", "restart_required"));
};
self.firmware.os.memcard().lane.wait_for_progress(progress, timeout);
Ok(())
}
pub fn advance_frames(&mut self, count: u64) -> Result<AdvanceFramesOutcome, HostError> {
let mut completed = 0;
for _ in 0..count {
@@ -3,6 +3,282 @@ use crate::{AdvanceFrameOutcome, HostExecutionStatus};
use prometeu_test_support::memcard_fixture::{MemcardFixture, memcard_program};
use std::{fs, sync::mpsc};
/// Holds the first native write after admission; the controller decides its outcome.
struct BootstrapStore {
file: FileMemcard,
entered: mpsc::Sender<()>,
continue_io: mpsc::Receiver<Result<(), MemcardIoError>>,
held: bool,
}
impl MemcardStorage for BootstrapStore {
fn len(&self) -> Result<u64, MemcardIoError> {
self.file.len()
}
fn read_only(&self) -> bool {
false
}
fn read_at(&mut self, offset: u64, bytes: &mut [u8]) -> Result<(), MemcardIoError> {
self.file.read_at(offset, bytes)
}
fn write_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), MemcardIoError> {
if !self.held {
self.held = true;
let _ = self.entered.send(());
self.continue_io
.recv_timeout(Duration::from_secs(5))
.unwrap_or(Err(MemcardIoError::IoError))?;
}
self.file.write_at(offset, bytes)
}
fn sync(&mut self) -> Result<(), MemcardIoError> {
self.file.sync()
}
}
fn reopen_with_controlled_io(
kind: MemcardFixture,
) -> (Fixture, HeadlessHost, mpsc::Receiver<()>, mpsc::Sender<Result<(), MemcardIoError>>) {
use prometeu_system::memcard::{
image::{Game, Icon, crc32c},
transaction::{DurableResult, PublicationControl},
};
let f = Fixture::new(kind);
let file = FileMemcard::create(&f.image, IMAGE_BYTES, |io, id| {
Card::format(io, id).map_err(|_| MemcardIoError::IoError)
})
.unwrap();
let mut card = Card::mount(Box::new(file)).unwrap();
let icon = [0; 560];
card.register(
Game {
app_id: 42,
title: "Existing record".into(),
data_format_version: 1,
created_at: 0,
slots: vec![0],
icon: Icon { offset: 0, frames: 1, duration: 1, crc32c: crc32c(&icon) },
},
&icon,
&PublicationControl::default(),
)
.unwrap();
assert_eq!(
card.commit(0, &[0, 127, 128, 255], 0, &PublicationControl::default()).durable,
DurableResult::Success
);
drop(card);
let (entered_tx, entered) = mpsc::channel();
let (continue_io, release_rx) = mpsc::channel();
let mut host = HeadlessHost::new();
host.media.path = Some(f.image.clone());
host.media.access = Some(MediaAccess::ReadWrite);
host.install_memcard(Box::new(BootstrapStore {
file: FileMemcard::open(&f.image, false).unwrap(),
entered: entered_tx,
continue_io: release_rx,
held: false,
}))
.unwrap();
host.boot_with_memcard_choice(f.cart.to_string_lossy(), Choice::Persistent).unwrap();
assert_eq!(host.get_state().tick_index, 0);
(f, host, entered, continue_io)
}
#[test]
fn first_reopen_frame_waits_for_controlled_presentation_without_replaying() {
let (_f, mut host, entered, continue_io) =
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
let (finished_tx, finished) = mpsc::channel();
let controller = std::thread::spawn(move || {
entered.recv_timeout(Duration::from_secs(3)).unwrap();
// Observe the owner while publication is provably held. This is a
// channel deadline, not a sleep used to infer that IO has started.
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
let _ = continue_io.send(Ok(()));
premature
});
let result = host.advance_frame();
let _ = finished_tx.send(());
let premature = controller.join().unwrap();
assert!(!premature, "advance returned while native publication was held: {result:?}");
assert!(matches!(
result,
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
));
let state = host.get_state();
assert_eq!(state.frame_index, 1);
assert_eq!(state.tick_index, 2);
assert_eq!(state.status, HostExecutionStatus::Paused);
assert_eq!(state.stack_top.last(), Some(&prometeu_bytecode::Value::Int64(201)));
assert!(!host.firmware.os.vm().debug_step_requested());
assert!(host.get_fault().is_none());
assert!(!host.get_memcard_state().io.active);
assert_eq!(host.capture_frame().unwrap().logical_frame_index, 1);
host.shutdown(TeardownPolicy::default()).unwrap();
}
#[test]
fn blocked_bootstrap_reaches_native_deadline_and_keeps_worker_lock() {
let (f, mut host, entered, continue_io) =
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
let started = Instant::now();
let result = host.advance_frame();
entered.recv_timeout(Duration::from_secs(1)).unwrap();
let state = host.get_memcard_state();
let busy = matches!(FileMemcard::open(&f.image, false), Err(MemcardIoError::Busy));
// Always let the worker return before asserting, including on the old host.
continue_io.send(Ok(())).unwrap();
settle(&mut host);
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
assert!(started.elapsed() < Duration::from_secs(3));
assert!(state.recovery.restart_required);
assert!(state.io.active && busy);
assert_eq!(host.get_state().frame_index, 0);
assert_eq!(host.get_state().tick_index, 1);
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
assert!(!host.firmware.os.vm().debug_step_requested());
assert!(host.get_fault().is_none());
}
#[test]
fn ordinary_gameplay_commit_does_not_wait_for_native_io() {
let (_f, mut host, entered, continue_io) = reopen_with_controlled_io(MemcardFixture::Read);
host.advance_frame().unwrap();
let service = host.firmware.os.memcard();
let context = service.session.context();
service.session.write(context, 0, 0, &[0, 127, 128, 255]).unwrap();
service.session.commit(context, 0, 0, Instant::now(), &service.lane).unwrap();
entered.recv_timeout(Duration::from_secs(2)).unwrap();
let result = host.advance_frame();
let pending = host.get_memcard_state();
continue_io.send(Ok(())).unwrap();
settle(&mut host);
assert!(matches!(
result,
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 2, .. })
));
assert!(pending.io.active);
assert_eq!(pending.request.unwrap().operation, "commit");
assert!(!host.firmware.os.vm().debug_step_requested());
assert!(host.get_fault().is_none());
host.shutdown(TeardownPolicy::default()).unwrap();
}
#[test]
fn bootstrap_wait_deadline_is_shared_across_wakes_and_reentry() {
use super::super::{BOOTSTRAP_IO_WAIT_QUANTUM, BootstrapIoWait};
let start = Instant::now();
let mut wait = BootstrapIoWait::default();
assert_eq!(wait.next_wait(start), Some(BOOTSTRAP_IO_WAIT_QUANTUM));
assert_eq!(
wait.next_wait(start + Duration::from_millis(1000)),
Some(BOOTSTRAP_IO_WAIT_QUANTUM)
);
assert_eq!(wait.next_wait(start + Duration::from_millis(1999)), Some(Duration::from_millis(1)));
assert_eq!(wait.next_wait(start + Duration::from_secs(2)), None);
assert_eq!(wait.next_wait(start + Duration::from_secs(3)), None);
}
#[test]
fn bootstrap_outer_timeout_preserves_native_snapshot_and_machine_state() {
use super::super::BootstrapIoWait;
let (_f, mut host, _entered, _release) = reopen_with_controlled_io(MemcardFixture::Read);
let before = host.get_state();
host.firmware.os.vm().set_paused(false);
host.firmware.os.vm().request_debug_step();
let mut wait = BootstrapIoWait { deadline: Some(Instant::now()) };
let error = host.wait_for_bootstrap_io(&mut wait, 0).unwrap_err();
let snapshot = host.get_memcard_state();
let HostError::Memcard(error) = error else { panic!("expected a native memcard error") };
assert_eq!(error.operation, "advance");
assert_eq!(error.reason, "restart_required");
assert_eq!(error.state, snapshot);
let after = host.get_state();
assert_eq!((after.pc, after.frame_index, after.tick_index), (before.pc, 0, 0));
assert_eq!(after.stack_top.to_vec(), before.stack_top.to_vec());
assert_eq!(after.status, HostExecutionStatus::Paused);
assert!(!host.firmware.os.vm().debug_step_requested());
assert!(host.get_fault().is_none());
host.shutdown(TeardownPolicy::default()).unwrap();
}
#[test]
fn failed_bootstrap_io_returns_native_media_action_without_guest_fault() {
let (_f, mut host, entered, continue_io) =
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
let controller = std::thread::spawn(move || {
entered.recv_timeout(Duration::from_secs(2)).unwrap();
continue_io.send(Err(MemcardIoError::Unavailable)).unwrap();
});
let result = host.advance_frame();
controller.join().unwrap();
assert_eq!(result.unwrap(), AdvanceFrameOutcome::ActionRequired);
let state = host.get_memcard_state();
assert!(state.recovery.removal_pending);
assert!(!state.io.active);
assert_eq!(host.get_state().frame_index, 0);
assert_eq!(host.get_state().status, HostExecutionStatus::Paused);
assert!(!host.firmware.os.vm().debug_step_requested());
assert!(host.get_fault().is_none());
}
#[test]
fn queued_bootstrap_publication_can_progress_without_moving_guest_during_poll() {
use prometeu_system::AsyncWorkJobOutcome;
let (_f, mut host, entered, continue_io) =
reopen_with_controlled_io(MemcardFixture::ReadWithPresentation);
let (busy_tx, busy) = mpsc::channel();
let (unblock, unblock_rx) = mpsc::channel();
host.firmware
.os
.memcard()
.lane
.submit_memcard_commit_write(move |_| {
busy_tx.send(()).unwrap();
let _ = unblock_rx.recv_timeout(Duration::from_secs(3));
AsyncWorkJobOutcome::Completed
})
.unwrap();
busy.recv_timeout(Duration::from_secs(2)).unwrap();
for _ in 0..10 {
host.firmware.os.vm().set_paused(false);
host.firmware.os.vm().request_debug_step();
host.tick_once();
if host.bootstrap_io_pending() {
break;
}
}
let pending = host.bootstrap_io_pending();
let queued = host.get_memcard_state();
let before = host.get_state();
host.poll_memcard();
let after = host.get_state();
let (finished_tx, finished) = mpsc::channel();
let controller = std::thread::spawn(move || {
let premature = finished.recv_timeout(Duration::from_millis(300)).is_ok();
let _ = unblock.send(());
entered.recv_timeout(Duration::from_secs(2)).unwrap();
let _ = continue_io.send(Ok(()));
premature
});
let result = host.advance_frame();
let _ = finished_tx.send(());
let premature = controller.join().unwrap();
assert!(pending);
assert!(!premature, "advance returned while its publication was queued: {result:?}");
assert_eq!(queued.io.phase, "queued");
assert_eq!(
(before.pc, before.frame_index, before.tick_index),
(after.pc, after.frame_index, after.tick_index)
);
assert_eq!(before.stack_top.to_vec(), after.stack_top.to_vec());
assert!(matches!(
result,
Ok(AdvanceFrameOutcome::FrameCompleted { logical_frame_index: 1, .. })
));
host.shutdown(TeardownPolicy::default()).unwrap();
}
struct Fixture {
root: PathBuf,
image: PathBuf,
@@ -10,10 +286,12 @@ struct Fixture {
}
impl Fixture {
fn new(kind: MemcardFixture) -> Self {
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let root = std::env::temp_dir().join(format!(
"native-memcard-{}-{}",
"native-memcard-{}-{}-{}",
std::process::id(),
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(),
COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
let cart = root.join("cart");
fs::create_dir_all(&cart).unwrap();
@@ -844,6 +844,15 @@ fn packaged_installation_smoke() {
c.wait_exit();
}
#[test]
#[ignore = "requires PROMETEU_PROTOCOL_TEST_INSTALL pointing to an isolated installation"]
fn packaged_memcard_reopen_first_frame() {
let installation = PathBuf::from(
std::env::var_os("PROMETEU_PROTOCOL_TEST_INSTALL").expect("set isolated installation path"),
);
memcard_process::presentation_reopen(&installation.join("prometeu"));
}
#[test]
#[ignore = "requires PROMETEU_PROTOCOL_TEST_INSTALL pointing to an isolated installation"]
fn packaged_native_log_observation() {
@@ -102,6 +102,63 @@ fn persistence(binary: &Path) {
assert!(fs::read(&image).unwrap() == bytes);
}
pub(super) fn presentation_reopen(binary: &Path) {
let root = TempDir::new();
let image = root.0.join("presentation-reopen.pmc");
let write = card(MemcardFixture::Write, "1");
let mut writer = start_at(binary);
mount(&mut writer, &image, "create", "read_write");
launch(&mut writer, &write, "persistent");
oracle(&mut writer, 101);
assert_eq!(writer.ok("get_memcard_state", json!({}))["dirty"], false);
writer.close();
for version in ["1", "2"] {
let read = card(MemcardFixture::ReadWithPresentation, version);
let mut reader = start_at(binary);
mount(&mut reader, &image, "open", "read_write");
launch(&mut reader, &read, "persistent");
let booted = reader.ok("get_state", json!({}));
assert_eq!(booted["pc"], 0);
assert_eq!(booted["tickIndex"], 0);
assert_eq!(booted["logicalFrameIndex"], 0);
// Exactly one command owns the first frame; there is no oracle retry.
let advanced = reader.ok("advance", json!({"frames":1}));
assert_eq!(advanced, json!({"requested":1,"completed":1,"stop":"completed"}));
let state = reader.ok("get_state", json!({}));
assert_eq!(state["logicalFrameIndex"], 1);
assert_eq!(state["tickIndex"], 2);
assert_eq!(state["status"], "paused");
assert_eq!(state["appId"], 4294967295u32);
assert_eq!(state["stackTop"].as_array().unwrap().last().unwrap()["value"], "201");
assert_eq!(reader.ok("get_fault", json!({})), Value::Null);
let media = reader.ok("get_memcard_state", json!({}));
assert_eq!(
media["request"],
Value::Null,
"presentation must finish without a payload save"
);
assert_eq!(media["io"]["active"], false);
assert_eq!(media["dirty"], false);
assert_eq!(reader.ok("capture", json!({}))["logicalFrameIndex"], 1);
reader.close();
}
// Preserve the original reader's independent read-without-writing oracle.
let bytes = fs::read(&image).unwrap();
let read = card(MemcardFixture::Read, "2");
let mut reader = start_at(binary);
mount(&mut reader, &image, "open", "read_write");
launch(&mut reader, &read, "persistent");
oracle(&mut reader, 201);
reader.close();
assert_eq!(fs::read(&image).unwrap(), bytes);
}
#[test]
fn memcard_reopen_presentation_completes_exactly_one_first_frame() {
presentation_reopen(Path::new(env!("CARGO_BIN_EXE_prometeu")));
}
#[test]
fn memcard_public_distinct_process_save_readback_and_version_identity() {
persistence(Path::new(env!("CARGO_BIN_EXE_prometeu")));