Files
prometeu-runtime/crates/console/prometeu-drivers/src/asset/resumable.rs
T

573 lines
23 KiB
Rust

//! Bounded decoding steps retain progress while the shared lane services saves.
use super::*;
use prometeu_hal::background_work::{BackgroundWork, WorkStep};
use prometeu_hal::cartridge::AssetsPayloadReader;
#[derive(Clone)]
pub(super) struct Resources {
handles: Arc<RwLock<HandleTable>>,
generations: Arc<RwLock<TargetGenerationTable>>,
source: Arc<RwLock<AssetsPayloadSource>>,
telemetry: Arc<Mutex<AssetPipelineTelemetryState>>,
gfx: BankPolicy<GlyphBank>,
sound: BankPolicy<SoundBank>,
scene: BankPolicy<SceneBank>,
}
impl Resources {
pub(super) fn new(m: &AssetManager) -> Self {
Self {
handles: m.handles.clone(),
generations: m.target_generations.clone(),
source: m.assets_data.clone(),
telemetry: m.pipeline_telemetry.clone(),
gfx: m.gfx_policy.clone(),
sound: m.sound_policy.clone(),
scene: m.scene_policy.clone(),
}
}
fn current(&self, job: &AssetLoadJob) -> bool {
AssetManager::is_current_target_generation(
&self.generations,
job.slot,
job.request_generation,
)
}
}
pub(super) fn schedule(
queue: Arc<AssetLoadQueue>,
lane: Arc<dyn BackgroundWork>,
resources: Resources,
) {
{
let mut state = queue.state.lock().unwrap();
if state.scheduled || state.shutdown {
return;
}
state.scheduled = true;
}
let owner = queue.clone();
let failure_queue = queue.clone();
let failures = resources.clone();
let mut active: Option<(AssetLoadJob, Decoder, Instant)> = None;
if lane
.submit_asset(Box::new(move |canceled| {
if canceled || owner.state.lock().unwrap().shutdown {
let mut state = owner.state.lock().unwrap();
state.scheduled = false;
state.active = None;
return WorkStep::Failed;
}
if active.is_none() {
let job = {
let mut state = owner.state.lock().unwrap();
match state.pending.pop_front() {
Some(job) => {
state.active = Some(job.clone());
job
}
None => {
state.scheduled = false;
return WorkStep::Complete;
}
}
};
if !resources.current(&job) {
resources.telemetry.lock().unwrap().record_stale_discard();
owner.complete_active(job.handle_id, job.request_generation);
return WorkStep::Yield;
}
if let Some(handle) = resources.handles.write().unwrap().get_mut(&job.handle_id) {
handle.status = LoadStatus::LOADING;
handle.progress = ASSET_PROGRESS_LOADING;
}
let start = Instant::now();
resources.telemetry.lock().unwrap().record_started(0);
match Decoder::new(&job.entry, &resources.source) {
Ok(decoder) => active = Some((job, decoder, start)),
Err(_) => {
AssetManager::complete_load_job(
&resources.handles,
&job,
LoadStatus::ERROR,
);
AssetManager::record_load_job_finished(
&resources.telemetry,
&job,
start,
LoadStatus::ERROR,
);
owner.complete_active(job.handle_id, job.request_generation);
return WorkStep::Yield;
}
}
return WorkStep::Yield;
}
let (job, decoder, start) = active.as_mut().unwrap();
if !resources.current(job) {
resources.telemetry.lock().unwrap().record_stale_discard();
owner.complete_active(job.handle_id, job.request_generation);
active = None;
return WorkStep::Yield;
}
match decoder.step() {
Ok(None) => WorkStep::Yield,
outcome => {
let generations = resources.generations.read().unwrap();
let status = if let Ok(Some(bank)) = outcome {
if generations.get(&job.slot).copied().unwrap_or(0)
== job.request_generation
{
match bank {
Decoded::Glyph(bank) => {
let resident = resources.gfx.put_resident(
job.asset_id,
Arc::new(bank),
job.entry.decoded_size as usize,
);
resources.gfx.stage(
job.handle_id,
resident,
job.entry.decoded_size as usize,
);
}
Decoded::Sound(bank) => {
let resident = resources.sound.put_resident(
job.asset_id,
Arc::new(bank),
job.entry.decoded_size as usize,
);
resources.sound.stage(
job.handle_id,
resident,
job.entry.decoded_size as usize,
);
}
Decoded::Scene(bank) => {
let resident = resources.scene.put_resident(
job.asset_id,
Arc::new(bank),
job.entry.decoded_size as usize,
);
resources.scene.stage(
job.handle_id,
resident,
job.entry.decoded_size as usize,
);
}
}
LoadStatus::READY
} else {
LoadStatus::ERROR
}
} else {
LoadStatus::ERROR
};
drop(generations);
AssetManager::complete_load_job(&resources.handles, job, status);
AssetManager::record_load_job_finished(
&resources.telemetry,
job,
*start,
status,
);
owner.complete_active(job.handle_id, job.request_generation);
active = None;
WorkStep::Yield
}
}
}))
.is_err()
{
let mut state = failure_queue.state.lock().unwrap();
state.scheduled = false;
for job in state.pending.drain(..) {
AssetManager::complete_load_job(&failures.handles, &job, LoadStatus::ERROR);
}
}
}
enum Decoded {
Glyph(GlyphBank),
Sound(SoundBank),
Scene(SceneBank),
}
enum State {
Glyph {
bank: GlyphBank,
packed: usize,
position: usize,
palette_bytes: usize,
palette_position: usize,
},
Sound {
sample_rate: u32,
data: Vec<i16>,
remaining: usize,
},
Scene {
layers: [SceneLayer; SCENE_LAYER_COUNT_V1],
layer: usize,
tiles_left: usize,
header: bool,
decoded: usize,
},
Done,
}
struct Decoder {
reader: AssetsPayloadReader,
entry: AssetEntry,
state: State,
position: usize,
}
impl Decoder {
fn new(entry: &AssetEntry, source: &Arc<RwLock<AssetsPayloadSource>>) -> Result<Self, String> {
let reader = source
.read()
.unwrap()
.open_slice(entry.offset, entry.size)
.map_err(|e| e.to_string())?
.open_reader()
.map_err(|e| e.to_string())?;
let state = match entry.bank_type {
BankType::GLYPH => {
let (tile_size, width, height, packed, palette_count, palette_bytes) =
AssetManager::decode_glyph_bank_layout(entry)?;
State::Glyph {
bank: GlyphBank {
tile_size,
width,
height,
pixel_indices: Vec::with_capacity(width * height),
palettes: vec![[Color::BLACK; 16]; palette_count],
},
packed,
position: 0,
palette_bytes,
palette_position: 0,
}
}
BankType::SOUNDS => {
let sample_rate = decode::sound_rate(entry)?;
State::Sound {
sample_rate,
data: Vec::with_capacity(entry.size as usize / 2),
remaining: entry.size as usize,
}
}
BankType::SCENE => {
decode::scene_metadata(entry)?;
State::Scene {
layers: decode::empty_layers(),
layer: 0,
tiles_left: 0,
header: false,
decoded: 0,
}
}
};
Ok(Self { reader, entry: entry.clone(), state, position: 0 })
}
fn step(&mut self) -> Result<Option<Decoded>, String> {
let mut bytes = [0; 4096];
match &mut self.state {
State::Glyph { bank, packed, position, palette_bytes, palette_position } => {
if *position < *packed {
let n = (*packed - *position).min(bytes.len());
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
decode::append_pixels(
&mut bank.pixel_indices,
&bytes[..n],
bank.width * bank.height,
);
*position += n;
return Ok(None);
}
if *palette_position < *palette_bytes {
let n = (*palette_bytes - *palette_position).min(bytes.len());
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
decode::write_palettes(&mut bank.palettes, *palette_position, &bytes[..n]);
*palette_position += n;
return Ok(None);
}
let State::Glyph { bank, .. } = std::mem::replace(&mut self.state, State::Done)
else {
unreachable!()
};
Ok(Some(Decoded::Glyph(bank)))
}
State::Sound { sample_rate, data, remaining } => {
if *remaining > 0 {
let n = (*remaining).min(bytes.len());
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
decode::append_pcm(data, &bytes[..n]);
*remaining -= n;
return Ok(None);
}
let rate = *sample_rate;
let data = std::mem::take(data);
self.state = State::Done;
Ok(Some(Decoded::Sound(SoundBank::new(vec![Arc::new(Sample::new(rate, data))]))))
}
State::Scene { layers, layer, tiles_left, header, decoded } => {
if !*header {
self.reader
.read_exact(&mut bytes[..SCENE_HEADER_BYTES_V1])
.map_err(|e| e.to_string())?;
decode::scene_header(&bytes[..SCENE_HEADER_BYTES_V1])?;
self.position += SCENE_HEADER_BYTES_V1;
*header = true;
return Ok(None);
}
if *tiles_left > 0 {
let n = (*tiles_left).min(bytes.len() / SCENE_TILE_RECORD_BYTES_V1);
let count = n * SCENE_TILE_RECORD_BYTES_V1;
self.reader.read_exact(&mut bytes[..count]).map_err(|e| e.to_string())?;
decode::append_tiles(&mut layers[*layer - 1].tilemap.tiles, &bytes[..count]);
*tiles_left -= n;
self.position += count;
return Ok(None);
}
if *layer < layers.len() {
self.reader
.read_exact(&mut bytes[..SCENE_LAYER_HEADER_BYTES_V1])
.map_err(|e| e.to_string())?;
self.position += SCENE_LAYER_HEADER_BYTES_V1;
layers[*layer] = decode::scene_layer(
&bytes[..SCENE_LAYER_HEADER_BYTES_V1],
self.position,
self.entry.size,
decoded,
)?;
let count = layers[*layer].tilemap.width * layers[*layer].tilemap.height;
*layer += 1;
*tiles_left = count;
return Ok(None);
}
decode::scene_finish(&self.entry, self.position, *decoded)?;
let State::Scene { layers, .. } = std::mem::replace(&mut self.state, State::Done)
else {
unreachable!()
};
Ok(Some(Decoded::Scene(SceneBank { layers })))
}
State::Done => Err("Decoder already complete".into()),
}
}
}
// Test-only access to the production state machine; no alternate decode rules.
#[cfg(test)]
pub(super) fn conformance_result(
entry: &AssetEntry,
source: &Arc<RwLock<AssetsPayloadSource>>,
) -> Result<serde_json::Value, String> {
use std::io::Seek;
let mut decoder = Decoder::new(entry, source)?;
for _ in 0..100 {
let before = decoder.reader.stream_position().unwrap();
let result = decoder.step();
let after = decoder.reader.stream_position().unwrap();
assert!(after >= before && after - before <= 4096, "unbounded or rewound decode step");
if let Some(bank) = result? {
return Ok(match bank {
Decoded::Glyph(bank) => super::decode::tests::glyph_snapshot(&bank),
Decoded::Sound(bank) => super::decode::tests::sound_snapshot(&bank),
Decoded::Scene(bank) => super::decode::tests::scene_snapshot(&bank),
});
}
}
panic!("conformance fixture exceeded its step budget")
}
#[cfg(test)]
mod tests {
use super::*;
use prometeu_hal::background_work::{AssetWork, WorkUnavailable};
#[derive(Default)]
struct ManualLane(Mutex<VecDeque<AssetWork>>);
impl BackgroundWork for ManualLane {
fn submit_asset(&self, work: AssetWork) -> Result<(), WorkUnavailable> {
self.0.lock().unwrap().push_back(work);
Ok(())
}
}
impl ManualLane {
fn step(&self) -> bool {
let Some(mut work) = self.0.lock().unwrap().pop_front() else { return false };
if work(false) == WorkStep::Yield {
self.0.lock().unwrap().push_back(work);
}
true
}
}
#[test]
fn shared_decoder_yields_and_matches_existing_glyph_format() {
let mut data = vec![0x12; 32];
data.extend([255; 64]);
let entry = AssetEntry {
asset_id: 7,
asset_name: "glyph".into(),
bank_type: BankType::GLYPH,
codec: AssetCodec::None,
offset: 0,
size: data.len() as u64,
decoded_size: 128,
metadata: serde_json::json!({"tile_size":8,"width":8,"height":8,"palette_count":1,"palette_authored":1}),
};
let expected = AssetManager::decode_glyph_bank_from_buffer(&entry, &data).unwrap();
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
let mut manager = AssetManager::new(
vec![entry],
AssetsPayloadSource::from_bytes(data),
banks.clone(),
banks.clone(),
banks,
);
let lane = Arc::new(ManualLane::default());
assert!(manager.bind_background_work(lane.clone()));
let handle = manager.load(7, 0).unwrap();
let mut steps = 0;
while lane.step() {
steps += 1;
assert!(steps < 20);
}
assert!(steps >= 4);
assert_eq!(manager.status(handle), LoadStatus::READY);
let staging = manager.gfx_policy.staging.read().unwrap();
let (actual, _) = staging.get(&handle).unwrap();
assert_eq!(actual.pixel_indices, expected.pixel_indices);
assert_eq!(actual.palettes, expected.palettes);
}
fn decode_in_steps(entry: &AssetEntry, bytes: Vec<u8>) -> Decoded {
let source = Arc::new(RwLock::new(AssetsPayloadSource::from_bytes(bytes)));
let mut decoder = Decoder::new(entry, &source).unwrap();
for _ in 0..100 {
if let Some(result) = decoder.step().unwrap() {
return result;
}
}
panic!("decoder did not finish")
}
#[test]
fn resumable_sound_and_scene_preserve_existing_formats() {
let pcm: Vec<i16> = (0..5000).map(|n| n as i16 - 2500).collect();
let bytes: Vec<u8> = pcm.iter().flat_map(|n| n.to_le_bytes()).collect();
let entry = AssetEntry {
asset_id: 1,
asset_name: "pcm".into(),
bank_type: BankType::SOUNDS,
offset: 0,
size: bytes.len() as u64,
decoded_size: bytes.len() as u64,
codec: AssetCodec::None,
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
};
let Decoded::Sound(bank) = decode_in_steps(&entry, bytes) else { panic!() };
assert_eq!(bank.samples[0].data, pcm);
let scene = crate::asset::tests::test_scene();
let bytes = crate::asset::tests::encode_scene_payload(&scene);
let entry = crate::asset::tests::test_scene_asset_entry("scene", &bytes, &scene);
let Decoded::Scene(actual) = decode_in_steps(&entry, bytes) else { panic!() };
for (a, b) in actual.layers.iter().zip(scene.layers.iter()) {
assert_eq!(a.active, b.active);
assert_eq!(a.glyph_asset_id, b.glyph_asset_id);
assert_eq!(a.tile_size, b.tile_size);
assert_eq!(a.parallax_factor.x, b.parallax_factor.x);
assert_eq!(a.tilemap.width, b.tilemap.width);
assert_eq!(a.tilemap.height, b.tilemap.height);
assert_eq!(a.tilemap.tiles.len(), b.tilemap.tiles.len());
for (x, y) in a.tilemap.tiles.iter().zip(&b.tilemap.tiles) {
assert_eq!(
(x.active, x.glyph.glyph_id, x.glyph.palette_id, x.flip_x, x.flip_y),
(y.active, y.glyph.glyph_id, y.glyph.palette_id, y.flip_x, y.flip_y)
);
}
}
}
#[test]
fn shared_preload_waits_for_worker_and_frame_publication() {
let bytes = vec![0; 8192];
let entry = AssetEntry {
asset_id: 1,
asset_name: "pcm".into(),
bank_type: BankType::SOUNDS,
offset: 0,
size: bytes.len() as u64,
decoded_size: bytes.len() as u64,
codec: AssetCodec::None,
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
};
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
let mut manager = AssetManager::new(
vec![],
AssetsPayloadSource::empty(),
banks.clone(),
banks.clone(),
banks,
);
let lane = Arc::new(ManualLane::default());
assert!(manager.bind_background_work(lane.clone()));
manager.initialize_for_cartridge(
vec![entry],
vec![PreloadEntry { asset_id: 1, slot: 0 }],
AssetsPayloadSource::from_bytes(bytes),
);
assert_eq!(manager.preloads_ready(), Ok(false));
while lane.step() {}
assert_eq!(manager.preloads_ready(), Ok(false));
manager.apply_commits();
assert_eq!(manager.preloads_ready(), Ok(true));
assert_eq!(manager.slot_info(SlotRef::audio(0)).asset_id, Some(1));
}
#[test]
fn canceled_and_superseded_partial_decoders_never_publish() {
use crate::memory_banks::SoundBankPoolAccess;
let bytes = vec![0x12; 16384];
let entry = AssetEntry {
asset_id: 1,
asset_name: "pcm".into(),
bank_type: BankType::SOUNDS,
offset: 0,
size: bytes.len() as u64,
decoded_size: bytes.len() as u64,
codec: AssetCodec::None,
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
};
let mut newer = entry.clone();
newer.asset_id = 2;
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
let mut manager = AssetManager::new(
vec![entry, newer],
AssetsPayloadSource::from_bytes(bytes),
banks.clone(),
banks.clone(),
banks.clone(),
);
let lane = Arc::new(ManualLane::default());
assert!(manager.bind_background_work(lane.clone()));
let handle = manager.load(1, 0).unwrap();
assert!(lane.step());
assert!(lane.step());
assert_eq!(manager.status(handle), LoadStatus::LOADING);
assert_eq!(manager.cancel(handle), AssetOpStatus::Ok);
while lane.step() {}
assert_eq!(manager.status(handle), LoadStatus::CANCELED);
assert!(manager.sound_policy.staging.read().unwrap().is_empty());
manager.apply_commits();
assert!(banks.sound_bank_slot(0).is_none());
let old = manager.load(1, 0).unwrap();
assert!(lane.step());
assert!(lane.step());
let new = manager.load(2, 0).unwrap();
assert_eq!(old, new); // Stable slot handle, different request generation.
assert!(lane.step()); // Discard the in-flight old decoder.
assert!(manager.sound_policy.staging.read().unwrap().is_empty());
while lane.step() {}
assert_eq!(manager.status(new), LoadStatus::READY);
assert!(banks.sound_bank_slot(0).is_none());
assert_eq!(manager.commit(new), AssetOpStatus::Ok);
manager.apply_commits();
assert_eq!(manager.slot_info(SlotRef::audio(0)).asset_id, Some(2));
assert_eq!(banks.sound_bank_slot(0).unwrap().samples[0].data, vec![0x1212; 8192]);
}
}