feat(runtime): implement shared transactional memcard with async saves
Intrepid/Prometeu/Runtime/pipeline/pr-master This commit looks good

This commit is contained in:
2026-10-13 15:24:01 +01:00
parent 0af8b2b1fe
commit 6c2a07e579
65 changed files with 6222 additions and 985 deletions
@@ -172,7 +172,13 @@ impl AsyncWorkJobContext {
}
}
type AsyncWorkFn = Box<dyn FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static>;
/// A step must release the executor when yielding; it retains its own bounded progress.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AsyncWorkStep {
Yield,
Complete(AsyncWorkJobOutcome),
}
type AsyncWorkFn = Box<dyn FnMut(AsyncWorkJobContext) -> AsyncWorkStep + Send + 'static>;
struct PendingAsyncWorkJob {
id: AsyncWorkJobId,
@@ -180,6 +186,7 @@ struct PendingAsyncWorkJob {
priority: AsyncWorkPriority,
cancel_token: AsyncWorkCancelToken,
work: AsyncWorkFn,
started: bool,
}
struct RunningAsyncWorkJob {
@@ -188,11 +195,13 @@ struct RunningAsyncWorkJob {
priority: AsyncWorkPriority,
cancel_token: AsyncWorkCancelToken,
work: AsyncWorkFn,
started: bool,
}
#[derive(Default)]
struct AsyncWorkLaneState {
next_job_id: u64,
progress: u64,
pending: VecDeque<PendingAsyncWorkJob>,
active: Option<AsyncWorkActiveJob>,
active_cancel_token: Option<AsyncWorkCancelToken>,
@@ -204,9 +213,25 @@ struct AsyncWorkLaneState {
pub struct AsyncWorkLane {
state: Mutex<AsyncWorkLaneState>,
ready: Condvar,
progressed: Condvar,
}
impl AsyncWorkLane {
pub fn progress_stamp(&self) -> u64 {
self.state.lock().unwrap().progress
}
/// Used by synchronous headless boot admission, never by a guest/frame tick.
/// Wakes on actual IO progress; timeout lets the caller run its watchdog.
pub fn wait_for_progress(&self, stamp: u64, timeout: Duration) {
let state = self.state.lock().unwrap();
let _guard = self
.progressed
.wait_timeout_while(state, timeout, |s| {
s.progress == stamp && (!s.pending.is_empty() || s.active.is_some())
})
.unwrap();
}
pub fn submit(
&self,
kind: AsyncWorkJobKind,
@@ -220,16 +245,36 @@ impl AsyncWorkLane {
kind: AsyncWorkJobKind,
priority: AsyncWorkPriority,
work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static,
) -> Result<AsyncWorkJobId, AsyncWorkLaneError> {
let mut once = Some(work);
self.submit_steps(kind, priority, move |context| {
AsyncWorkStep::Complete(once.take().expect("one-shot job executed once")(context))
})
}
pub fn submit_steps(
&self,
kind: AsyncWorkJobKind,
priority: AsyncWorkPriority,
work: impl FnMut(AsyncWorkJobContext) -> AsyncWorkStep + Send + 'static,
) -> Result<AsyncWorkJobId, AsyncWorkLaneError> {
let mut state = self.state.lock().map_err(|_| AsyncWorkLaneError::InternalFailure)?;
if state.shutdown_requested {
return Err(AsyncWorkLaneError::Shutdown);
}
state.next_job_id += 1;
state.next_job_id =
state.next_job_id.checked_add(1).ok_or(AsyncWorkLaneError::InternalFailure)?;
let id = AsyncWorkJobId::new(state.next_job_id);
let cancel_token = AsyncWorkCancelToken::new();
let job = PendingAsyncWorkJob { id, kind, priority, cancel_token, work: Box::new(work) };
let job = PendingAsyncWorkJob {
id,
kind,
priority,
cancel_token,
work: Box::new(work),
started: false,
};
let insert_at = state
.pending
.iter()
@@ -340,13 +385,19 @@ impl AsyncWorkLane {
state.active = Some(active);
state.active_cancel_token = Some(pending.cancel_token.clone());
let pending_depth = state.pending.len();
state.telemetry.record_started(pending.id, pending_depth);
if !pending.started {
state.telemetry.record_started(pending.id, pending_depth);
} else {
state.telemetry.active_job_id = Some(pending.id);
state.telemetry.pending_depth = pending_depth;
}
return Some(RunningAsyncWorkJob {
id: pending.id,
kind: pending.kind,
priority: pending.priority,
cancel_token: pending.cancel_token,
work: pending.work,
started: true,
});
}
@@ -355,6 +406,33 @@ impl AsyncWorkLane {
}
}
fn yield_job(&self, job: RunningAsyncWorkJob) {
let mut state = self.state.lock().unwrap();
state.active = None;
state.active_cancel_token = None;
state.telemetry.active_job_id = None;
let at = state
.pending
.iter()
.position(|p| job.priority.rank() < p.priority.rank())
.unwrap_or(state.pending.len());
state.pending.insert(
at,
PendingAsyncWorkJob {
id: job.id,
kind: job.kind,
priority: job.priority,
cancel_token: job.cancel_token,
work: job.work,
started: job.started,
},
);
state.telemetry.pending_depth = state.pending.len();
state.progress = state.progress.saturating_add(1);
self.progressed.notify_all();
self.ready.notify_one();
}
fn record_finished(&self, job_id: AsyncWorkJobId, outcome: AsyncWorkJobOutcome) {
let mut state = self.state.lock().unwrap();
if state.active.is_some_and(|active| active.id == job_id) {
@@ -362,6 +440,8 @@ impl AsyncWorkLane {
state.active_cancel_token = None;
}
state.telemetry.record_closed(job_id, outcome);
state.progress = state.progress.saturating_add(1);
self.progressed.notify_all();
}
}
@@ -415,16 +495,19 @@ impl Drop for AsyncWorkLaneController {
}
fn run_async_work_loop(lane: Arc<AsyncWorkLane>) {
while let Some(job) = lane.wait_take() {
while let Some(mut job) = lane.wait_take() {
let context = AsyncWorkJobContext {
job_id: job.id,
kind: job.kind,
priority: job.priority,
cancel_token: job.cancel_token,
cancel_token: job.cancel_token.clone(),
};
let result = catch_unwind(AssertUnwindSafe(|| (job.work)(context)))
.unwrap_or(AsyncWorkJobOutcome::Failed);
lane.record_finished(job.id, result);
.unwrap_or(AsyncWorkStep::Complete(AsyncWorkJobOutcome::Failed));
match result {
AsyncWorkStep::Yield => lane.yield_job(job),
AsyncWorkStep::Complete(outcome) => lane.record_finished(job.id, outcome),
}
}
}
@@ -599,3 +682,63 @@ mod tests {
assert_eq!(lane.telemetry().shutdown_discards, 1);
}
}
#[cfg(test)]
mod step_tests {
use super::*;
#[test]
fn yielded_asset_resumes_same_job_after_priority_save() {
let lane = Arc::new(AsyncWorkLane::default());
let events = Arc::new(Mutex::new(Vec::new()));
let nested = lane.clone();
let log = events.clone();
let mut step = 0;
let mut identity = None;
lane.submit_steps(AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad, move |context| {
assert_eq!(*identity.get_or_insert(context.job_id), context.job_id);
if step == 0 {
log.lock().unwrap().push("asset first");
let log = log.clone();
nested
.submit_memcard_commit_write(move |_| {
log.lock().unwrap().push("save");
AsyncWorkJobOutcome::Completed
})
.unwrap();
step += 1;
AsyncWorkStep::Yield
} else {
log.lock().unwrap().push("asset resumes");
AsyncWorkStep::Complete(AsyncWorkJobOutcome::Completed)
}
})
.unwrap();
let mut controller =
AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), lane.clone());
let deadline = std::time::Instant::now() + Duration::from_secs(1);
while lane.telemetry().completed_jobs < 2 {
assert!(std::time::Instant::now() < deadline);
std::thread::yield_now();
}
controller.stop().unwrap();
assert_eq!(*events.lock().unwrap(), vec!["asset first", "save", "asset resumes"]);
}
}
impl prometeu_hal::background_work::BackgroundWork for AsyncWorkLane {
fn submit_asset(
&self,
mut work: prometeu_hal::background_work::AssetWork,
) -> Result<(), prometeu_hal::background_work::WorkUnavailable> {
use prometeu_hal::background_work::{WorkStep, WorkUnavailable};
self.submit_steps(AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad, move |context| {
match work(context.is_canceled()) {
WorkStep::Yield => AsyncWorkStep::Yield,
WorkStep::Complete => AsyncWorkStep::Complete(AsyncWorkJobOutcome::Completed),
WorkStep::Failed => AsyncWorkStep::Complete(AsyncWorkJobOutcome::Failed),
}
})
.map(|_| ())
.map_err(|_| WorkUnavailable)
}
}