feat(runtime): implement shared transactional memcard with async saves
Intrepid/Prometeu/Runtime/pipeline/pr-master This commit looks good
Intrepid/Prometeu/Runtime/pipeline/pr-master This commit looks good
This commit is contained in:
@@ -172,7 +172,13 @@ impl AsyncWorkJobContext {
|
||||
}
|
||||
}
|
||||
|
||||
type AsyncWorkFn = Box<dyn FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static>;
|
||||
/// A step must release the executor when yielding; it retains its own bounded progress.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AsyncWorkStep {
|
||||
Yield,
|
||||
Complete(AsyncWorkJobOutcome),
|
||||
}
|
||||
type AsyncWorkFn = Box<dyn FnMut(AsyncWorkJobContext) -> AsyncWorkStep + Send + 'static>;
|
||||
|
||||
struct PendingAsyncWorkJob {
|
||||
id: AsyncWorkJobId,
|
||||
@@ -180,6 +186,7 @@ struct PendingAsyncWorkJob {
|
||||
priority: AsyncWorkPriority,
|
||||
cancel_token: AsyncWorkCancelToken,
|
||||
work: AsyncWorkFn,
|
||||
started: bool,
|
||||
}
|
||||
|
||||
struct RunningAsyncWorkJob {
|
||||
@@ -188,11 +195,13 @@ struct RunningAsyncWorkJob {
|
||||
priority: AsyncWorkPriority,
|
||||
cancel_token: AsyncWorkCancelToken,
|
||||
work: AsyncWorkFn,
|
||||
started: bool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AsyncWorkLaneState {
|
||||
next_job_id: u64,
|
||||
progress: u64,
|
||||
pending: VecDeque<PendingAsyncWorkJob>,
|
||||
active: Option<AsyncWorkActiveJob>,
|
||||
active_cancel_token: Option<AsyncWorkCancelToken>,
|
||||
@@ -204,9 +213,25 @@ struct AsyncWorkLaneState {
|
||||
pub struct AsyncWorkLane {
|
||||
state: Mutex<AsyncWorkLaneState>,
|
||||
ready: Condvar,
|
||||
progressed: Condvar,
|
||||
}
|
||||
|
||||
impl AsyncWorkLane {
|
||||
pub fn progress_stamp(&self) -> u64 {
|
||||
self.state.lock().unwrap().progress
|
||||
}
|
||||
/// Used by synchronous headless boot admission, never by a guest/frame tick.
|
||||
/// Wakes on actual IO progress; timeout lets the caller run its watchdog.
|
||||
pub fn wait_for_progress(&self, stamp: u64, timeout: Duration) {
|
||||
let state = self.state.lock().unwrap();
|
||||
let _guard = self
|
||||
.progressed
|
||||
.wait_timeout_while(state, timeout, |s| {
|
||||
s.progress == stamp && (!s.pending.is_empty() || s.active.is_some())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn submit(
|
||||
&self,
|
||||
kind: AsyncWorkJobKind,
|
||||
@@ -220,16 +245,36 @@ impl AsyncWorkLane {
|
||||
kind: AsyncWorkJobKind,
|
||||
priority: AsyncWorkPriority,
|
||||
work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static,
|
||||
) -> Result<AsyncWorkJobId, AsyncWorkLaneError> {
|
||||
let mut once = Some(work);
|
||||
self.submit_steps(kind, priority, move |context| {
|
||||
AsyncWorkStep::Complete(once.take().expect("one-shot job executed once")(context))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn submit_steps(
|
||||
&self,
|
||||
kind: AsyncWorkJobKind,
|
||||
priority: AsyncWorkPriority,
|
||||
work: impl FnMut(AsyncWorkJobContext) -> AsyncWorkStep + Send + 'static,
|
||||
) -> Result<AsyncWorkJobId, AsyncWorkLaneError> {
|
||||
let mut state = self.state.lock().map_err(|_| AsyncWorkLaneError::InternalFailure)?;
|
||||
if state.shutdown_requested {
|
||||
return Err(AsyncWorkLaneError::Shutdown);
|
||||
}
|
||||
|
||||
state.next_job_id += 1;
|
||||
state.next_job_id =
|
||||
state.next_job_id.checked_add(1).ok_or(AsyncWorkLaneError::InternalFailure)?;
|
||||
let id = AsyncWorkJobId::new(state.next_job_id);
|
||||
let cancel_token = AsyncWorkCancelToken::new();
|
||||
let job = PendingAsyncWorkJob { id, kind, priority, cancel_token, work: Box::new(work) };
|
||||
let job = PendingAsyncWorkJob {
|
||||
id,
|
||||
kind,
|
||||
priority,
|
||||
cancel_token,
|
||||
work: Box::new(work),
|
||||
started: false,
|
||||
};
|
||||
let insert_at = state
|
||||
.pending
|
||||
.iter()
|
||||
@@ -340,13 +385,19 @@ impl AsyncWorkLane {
|
||||
state.active = Some(active);
|
||||
state.active_cancel_token = Some(pending.cancel_token.clone());
|
||||
let pending_depth = state.pending.len();
|
||||
state.telemetry.record_started(pending.id, pending_depth);
|
||||
if !pending.started {
|
||||
state.telemetry.record_started(pending.id, pending_depth);
|
||||
} else {
|
||||
state.telemetry.active_job_id = Some(pending.id);
|
||||
state.telemetry.pending_depth = pending_depth;
|
||||
}
|
||||
return Some(RunningAsyncWorkJob {
|
||||
id: pending.id,
|
||||
kind: pending.kind,
|
||||
priority: pending.priority,
|
||||
cancel_token: pending.cancel_token,
|
||||
work: pending.work,
|
||||
started: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -355,6 +406,33 @@ impl AsyncWorkLane {
|
||||
}
|
||||
}
|
||||
|
||||
fn yield_job(&self, job: RunningAsyncWorkJob) {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.active = None;
|
||||
state.active_cancel_token = None;
|
||||
state.telemetry.active_job_id = None;
|
||||
let at = state
|
||||
.pending
|
||||
.iter()
|
||||
.position(|p| job.priority.rank() < p.priority.rank())
|
||||
.unwrap_or(state.pending.len());
|
||||
state.pending.insert(
|
||||
at,
|
||||
PendingAsyncWorkJob {
|
||||
id: job.id,
|
||||
kind: job.kind,
|
||||
priority: job.priority,
|
||||
cancel_token: job.cancel_token,
|
||||
work: job.work,
|
||||
started: job.started,
|
||||
},
|
||||
);
|
||||
state.telemetry.pending_depth = state.pending.len();
|
||||
state.progress = state.progress.saturating_add(1);
|
||||
self.progressed.notify_all();
|
||||
self.ready.notify_one();
|
||||
}
|
||||
|
||||
fn record_finished(&self, job_id: AsyncWorkJobId, outcome: AsyncWorkJobOutcome) {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.active.is_some_and(|active| active.id == job_id) {
|
||||
@@ -362,6 +440,8 @@ impl AsyncWorkLane {
|
||||
state.active_cancel_token = None;
|
||||
}
|
||||
state.telemetry.record_closed(job_id, outcome);
|
||||
state.progress = state.progress.saturating_add(1);
|
||||
self.progressed.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,16 +495,19 @@ impl Drop for AsyncWorkLaneController {
|
||||
}
|
||||
|
||||
fn run_async_work_loop(lane: Arc<AsyncWorkLane>) {
|
||||
while let Some(job) = lane.wait_take() {
|
||||
while let Some(mut job) = lane.wait_take() {
|
||||
let context = AsyncWorkJobContext {
|
||||
job_id: job.id,
|
||||
kind: job.kind,
|
||||
priority: job.priority,
|
||||
cancel_token: job.cancel_token,
|
||||
cancel_token: job.cancel_token.clone(),
|
||||
};
|
||||
let result = catch_unwind(AssertUnwindSafe(|| (job.work)(context)))
|
||||
.unwrap_or(AsyncWorkJobOutcome::Failed);
|
||||
lane.record_finished(job.id, result);
|
||||
.unwrap_or(AsyncWorkStep::Complete(AsyncWorkJobOutcome::Failed));
|
||||
match result {
|
||||
AsyncWorkStep::Yield => lane.yield_job(job),
|
||||
AsyncWorkStep::Complete(outcome) => lane.record_finished(job.id, outcome),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -599,3 +682,63 @@ mod tests {
|
||||
assert_eq!(lane.telemetry().shutdown_discards, 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod step_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn yielded_asset_resumes_same_job_after_priority_save() {
|
||||
let lane = Arc::new(AsyncWorkLane::default());
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let nested = lane.clone();
|
||||
let log = events.clone();
|
||||
let mut step = 0;
|
||||
let mut identity = None;
|
||||
lane.submit_steps(AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad, move |context| {
|
||||
assert_eq!(*identity.get_or_insert(context.job_id), context.job_id);
|
||||
if step == 0 {
|
||||
log.lock().unwrap().push("asset first");
|
||||
let log = log.clone();
|
||||
nested
|
||||
.submit_memcard_commit_write(move |_| {
|
||||
log.lock().unwrap().push("save");
|
||||
AsyncWorkJobOutcome::Completed
|
||||
})
|
||||
.unwrap();
|
||||
step += 1;
|
||||
AsyncWorkStep::Yield
|
||||
} else {
|
||||
log.lock().unwrap().push("asset resumes");
|
||||
AsyncWorkStep::Complete(AsyncWorkJobOutcome::Completed)
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let mut controller =
|
||||
AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), lane.clone());
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(1);
|
||||
while lane.telemetry().completed_jobs < 2 {
|
||||
assert!(std::time::Instant::now() < deadline);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
controller.stop().unwrap();
|
||||
assert_eq!(*events.lock().unwrap(), vec!["asset first", "save", "asset resumes"]);
|
||||
}
|
||||
}
|
||||
|
||||
impl prometeu_hal::background_work::BackgroundWork for AsyncWorkLane {
|
||||
fn submit_asset(
|
||||
&self,
|
||||
mut work: prometeu_hal::background_work::AssetWork,
|
||||
) -> Result<(), prometeu_hal::background_work::WorkUnavailable> {
|
||||
use prometeu_hal::background_work::{WorkStep, WorkUnavailable};
|
||||
self.submit_steps(AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad, move |context| {
|
||||
match work(context.is_canceled()) {
|
||||
WorkStep::Yield => AsyncWorkStep::Yield,
|
||||
WorkStep::Complete => AsyncWorkStep::Complete(AsyncWorkJobOutcome::Completed),
|
||||
WorkStep::Failed => AsyncWorkStep::Complete(AsyncWorkJobOutcome::Failed),
|
||||
}
|
||||
})
|
||||
.map(|_| ())
|
||||
.map_err(|_| WorkUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user