573 lines
23 KiB
Rust
573 lines
23 KiB
Rust
//! Bounded decoding steps retain progress while the shared lane services saves.
|
|
use super::*;
|
|
use prometeu_hal::background_work::{BackgroundWork, WorkStep};
|
|
use prometeu_hal::cartridge::AssetsPayloadReader;
|
|
|
|
#[derive(Clone)]
|
|
pub(super) struct Resources {
|
|
handles: Arc<RwLock<HandleTable>>,
|
|
generations: Arc<RwLock<TargetGenerationTable>>,
|
|
source: Arc<RwLock<AssetsPayloadSource>>,
|
|
telemetry: Arc<Mutex<AssetPipelineTelemetryState>>,
|
|
gfx: BankPolicy<GlyphBank>,
|
|
sound: BankPolicy<SoundBank>,
|
|
scene: BankPolicy<SceneBank>,
|
|
}
|
|
impl Resources {
|
|
pub(super) fn new(m: &AssetManager) -> Self {
|
|
Self {
|
|
handles: m.handles.clone(),
|
|
generations: m.target_generations.clone(),
|
|
source: m.assets_data.clone(),
|
|
telemetry: m.pipeline_telemetry.clone(),
|
|
gfx: m.gfx_policy.clone(),
|
|
sound: m.sound_policy.clone(),
|
|
scene: m.scene_policy.clone(),
|
|
}
|
|
}
|
|
fn current(&self, job: &AssetLoadJob) -> bool {
|
|
AssetManager::is_current_target_generation(
|
|
&self.generations,
|
|
job.slot,
|
|
job.request_generation,
|
|
)
|
|
}
|
|
}
|
|
pub(super) fn schedule(
|
|
queue: Arc<AssetLoadQueue>,
|
|
lane: Arc<dyn BackgroundWork>,
|
|
resources: Resources,
|
|
) {
|
|
{
|
|
let mut state = queue.state.lock().unwrap();
|
|
if state.scheduled || state.shutdown {
|
|
return;
|
|
}
|
|
state.scheduled = true;
|
|
}
|
|
let owner = queue.clone();
|
|
let failure_queue = queue.clone();
|
|
let failures = resources.clone();
|
|
let mut active: Option<(AssetLoadJob, Decoder, Instant)> = None;
|
|
if lane
|
|
.submit_asset(Box::new(move |canceled| {
|
|
if canceled || owner.state.lock().unwrap().shutdown {
|
|
let mut state = owner.state.lock().unwrap();
|
|
state.scheduled = false;
|
|
state.active = None;
|
|
return WorkStep::Failed;
|
|
}
|
|
if active.is_none() {
|
|
let job = {
|
|
let mut state = owner.state.lock().unwrap();
|
|
match state.pending.pop_front() {
|
|
Some(job) => {
|
|
state.active = Some(job.clone());
|
|
job
|
|
}
|
|
None => {
|
|
state.scheduled = false;
|
|
return WorkStep::Complete;
|
|
}
|
|
}
|
|
};
|
|
if !resources.current(&job) {
|
|
resources.telemetry.lock().unwrap().record_stale_discard();
|
|
owner.complete_active(job.handle_id, job.request_generation);
|
|
return WorkStep::Yield;
|
|
}
|
|
if let Some(handle) = resources.handles.write().unwrap().get_mut(&job.handle_id) {
|
|
handle.status = LoadStatus::LOADING;
|
|
handle.progress = ASSET_PROGRESS_LOADING;
|
|
}
|
|
let start = Instant::now();
|
|
resources.telemetry.lock().unwrap().record_started(0);
|
|
match Decoder::new(&job.entry, &resources.source) {
|
|
Ok(decoder) => active = Some((job, decoder, start)),
|
|
Err(_) => {
|
|
AssetManager::complete_load_job(
|
|
&resources.handles,
|
|
&job,
|
|
LoadStatus::ERROR,
|
|
);
|
|
AssetManager::record_load_job_finished(
|
|
&resources.telemetry,
|
|
&job,
|
|
start,
|
|
LoadStatus::ERROR,
|
|
);
|
|
owner.complete_active(job.handle_id, job.request_generation);
|
|
return WorkStep::Yield;
|
|
}
|
|
}
|
|
return WorkStep::Yield;
|
|
}
|
|
let (job, decoder, start) = active.as_mut().unwrap();
|
|
if !resources.current(job) {
|
|
resources.telemetry.lock().unwrap().record_stale_discard();
|
|
owner.complete_active(job.handle_id, job.request_generation);
|
|
active = None;
|
|
return WorkStep::Yield;
|
|
}
|
|
match decoder.step() {
|
|
Ok(None) => WorkStep::Yield,
|
|
outcome => {
|
|
let generations = resources.generations.read().unwrap();
|
|
let status = if let Ok(Some(bank)) = outcome {
|
|
if generations.get(&job.slot).copied().unwrap_or(0)
|
|
== job.request_generation
|
|
{
|
|
match bank {
|
|
Decoded::Glyph(bank) => {
|
|
let resident = resources.gfx.put_resident(
|
|
job.asset_id,
|
|
Arc::new(bank),
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
resources.gfx.stage(
|
|
job.handle_id,
|
|
resident,
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
}
|
|
Decoded::Sound(bank) => {
|
|
let resident = resources.sound.put_resident(
|
|
job.asset_id,
|
|
Arc::new(bank),
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
resources.sound.stage(
|
|
job.handle_id,
|
|
resident,
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
}
|
|
Decoded::Scene(bank) => {
|
|
let resident = resources.scene.put_resident(
|
|
job.asset_id,
|
|
Arc::new(bank),
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
resources.scene.stage(
|
|
job.handle_id,
|
|
resident,
|
|
job.entry.decoded_size as usize,
|
|
);
|
|
}
|
|
}
|
|
LoadStatus::READY
|
|
} else {
|
|
LoadStatus::ERROR
|
|
}
|
|
} else {
|
|
LoadStatus::ERROR
|
|
};
|
|
drop(generations);
|
|
AssetManager::complete_load_job(&resources.handles, job, status);
|
|
AssetManager::record_load_job_finished(
|
|
&resources.telemetry,
|
|
job,
|
|
*start,
|
|
status,
|
|
);
|
|
owner.complete_active(job.handle_id, job.request_generation);
|
|
active = None;
|
|
WorkStep::Yield
|
|
}
|
|
}
|
|
}))
|
|
.is_err()
|
|
{
|
|
let mut state = failure_queue.state.lock().unwrap();
|
|
state.scheduled = false;
|
|
for job in state.pending.drain(..) {
|
|
AssetManager::complete_load_job(&failures.handles, &job, LoadStatus::ERROR);
|
|
}
|
|
}
|
|
}
|
|
enum Decoded {
|
|
Glyph(GlyphBank),
|
|
Sound(SoundBank),
|
|
Scene(SceneBank),
|
|
}
|
|
enum State {
|
|
Glyph {
|
|
bank: GlyphBank,
|
|
packed: usize,
|
|
position: usize,
|
|
palette_bytes: usize,
|
|
palette_position: usize,
|
|
},
|
|
Sound {
|
|
sample_rate: u32,
|
|
data: Vec<i16>,
|
|
remaining: usize,
|
|
},
|
|
Scene {
|
|
layers: [SceneLayer; SCENE_LAYER_COUNT_V1],
|
|
layer: usize,
|
|
tiles_left: usize,
|
|
header: bool,
|
|
decoded: usize,
|
|
},
|
|
Done,
|
|
}
|
|
struct Decoder {
|
|
reader: AssetsPayloadReader,
|
|
entry: AssetEntry,
|
|
state: State,
|
|
position: usize,
|
|
}
|
|
impl Decoder {
|
|
fn new(entry: &AssetEntry, source: &Arc<RwLock<AssetsPayloadSource>>) -> Result<Self, String> {
|
|
let reader = source
|
|
.read()
|
|
.unwrap()
|
|
.open_slice(entry.offset, entry.size)
|
|
.map_err(|e| e.to_string())?
|
|
.open_reader()
|
|
.map_err(|e| e.to_string())?;
|
|
let state = match entry.bank_type {
|
|
BankType::GLYPH => {
|
|
let (tile_size, width, height, packed, palette_count, palette_bytes) =
|
|
AssetManager::decode_glyph_bank_layout(entry)?;
|
|
State::Glyph {
|
|
bank: GlyphBank {
|
|
tile_size,
|
|
width,
|
|
height,
|
|
pixel_indices: Vec::with_capacity(width * height),
|
|
palettes: vec![[Color::BLACK; 16]; palette_count],
|
|
},
|
|
packed,
|
|
position: 0,
|
|
palette_bytes,
|
|
palette_position: 0,
|
|
}
|
|
}
|
|
BankType::SOUNDS => {
|
|
let sample_rate = decode::sound_rate(entry)?;
|
|
State::Sound {
|
|
sample_rate,
|
|
data: Vec::with_capacity(entry.size as usize / 2),
|
|
remaining: entry.size as usize,
|
|
}
|
|
}
|
|
BankType::SCENE => {
|
|
decode::scene_metadata(entry)?;
|
|
State::Scene {
|
|
layers: decode::empty_layers(),
|
|
layer: 0,
|
|
tiles_left: 0,
|
|
header: false,
|
|
decoded: 0,
|
|
}
|
|
}
|
|
};
|
|
Ok(Self { reader, entry: entry.clone(), state, position: 0 })
|
|
}
|
|
fn step(&mut self) -> Result<Option<Decoded>, String> {
|
|
let mut bytes = [0; 4096];
|
|
match &mut self.state {
|
|
State::Glyph { bank, packed, position, palette_bytes, palette_position } => {
|
|
if *position < *packed {
|
|
let n = (*packed - *position).min(bytes.len());
|
|
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
|
|
decode::append_pixels(
|
|
&mut bank.pixel_indices,
|
|
&bytes[..n],
|
|
bank.width * bank.height,
|
|
);
|
|
*position += n;
|
|
return Ok(None);
|
|
}
|
|
if *palette_position < *palette_bytes {
|
|
let n = (*palette_bytes - *palette_position).min(bytes.len());
|
|
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
|
|
decode::write_palettes(&mut bank.palettes, *palette_position, &bytes[..n]);
|
|
*palette_position += n;
|
|
return Ok(None);
|
|
}
|
|
let State::Glyph { bank, .. } = std::mem::replace(&mut self.state, State::Done)
|
|
else {
|
|
unreachable!()
|
|
};
|
|
Ok(Some(Decoded::Glyph(bank)))
|
|
}
|
|
State::Sound { sample_rate, data, remaining } => {
|
|
if *remaining > 0 {
|
|
let n = (*remaining).min(bytes.len());
|
|
self.reader.read_exact(&mut bytes[..n]).map_err(|e| e.to_string())?;
|
|
decode::append_pcm(data, &bytes[..n]);
|
|
*remaining -= n;
|
|
return Ok(None);
|
|
}
|
|
let rate = *sample_rate;
|
|
let data = std::mem::take(data);
|
|
self.state = State::Done;
|
|
Ok(Some(Decoded::Sound(SoundBank::new(vec![Arc::new(Sample::new(rate, data))]))))
|
|
}
|
|
State::Scene { layers, layer, tiles_left, header, decoded } => {
|
|
if !*header {
|
|
self.reader
|
|
.read_exact(&mut bytes[..SCENE_HEADER_BYTES_V1])
|
|
.map_err(|e| e.to_string())?;
|
|
decode::scene_header(&bytes[..SCENE_HEADER_BYTES_V1])?;
|
|
self.position += SCENE_HEADER_BYTES_V1;
|
|
*header = true;
|
|
return Ok(None);
|
|
}
|
|
if *tiles_left > 0 {
|
|
let n = (*tiles_left).min(bytes.len() / SCENE_TILE_RECORD_BYTES_V1);
|
|
let count = n * SCENE_TILE_RECORD_BYTES_V1;
|
|
self.reader.read_exact(&mut bytes[..count]).map_err(|e| e.to_string())?;
|
|
decode::append_tiles(&mut layers[*layer - 1].tilemap.tiles, &bytes[..count]);
|
|
*tiles_left -= n;
|
|
self.position += count;
|
|
return Ok(None);
|
|
}
|
|
if *layer < layers.len() {
|
|
self.reader
|
|
.read_exact(&mut bytes[..SCENE_LAYER_HEADER_BYTES_V1])
|
|
.map_err(|e| e.to_string())?;
|
|
self.position += SCENE_LAYER_HEADER_BYTES_V1;
|
|
layers[*layer] = decode::scene_layer(
|
|
&bytes[..SCENE_LAYER_HEADER_BYTES_V1],
|
|
self.position,
|
|
self.entry.size,
|
|
decoded,
|
|
)?;
|
|
let count = layers[*layer].tilemap.width * layers[*layer].tilemap.height;
|
|
*layer += 1;
|
|
*tiles_left = count;
|
|
return Ok(None);
|
|
}
|
|
decode::scene_finish(&self.entry, self.position, *decoded)?;
|
|
let State::Scene { layers, .. } = std::mem::replace(&mut self.state, State::Done)
|
|
else {
|
|
unreachable!()
|
|
};
|
|
Ok(Some(Decoded::Scene(SceneBank { layers })))
|
|
}
|
|
State::Done => Err("Decoder already complete".into()),
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test-only access to the production state machine; no alternate decode rules.
|
|
#[cfg(test)]
|
|
pub(super) fn conformance_result(
|
|
entry: &AssetEntry,
|
|
source: &Arc<RwLock<AssetsPayloadSource>>,
|
|
) -> Result<serde_json::Value, String> {
|
|
use std::io::Seek;
|
|
let mut decoder = Decoder::new(entry, source)?;
|
|
for _ in 0..100 {
|
|
let before = decoder.reader.stream_position().unwrap();
|
|
let result = decoder.step();
|
|
let after = decoder.reader.stream_position().unwrap();
|
|
assert!(after >= before && after - before <= 4096, "unbounded or rewound decode step");
|
|
if let Some(bank) = result? {
|
|
return Ok(match bank {
|
|
Decoded::Glyph(bank) => super::decode::tests::glyph_snapshot(&bank),
|
|
Decoded::Sound(bank) => super::decode::tests::sound_snapshot(&bank),
|
|
Decoded::Scene(bank) => super::decode::tests::scene_snapshot(&bank),
|
|
});
|
|
}
|
|
}
|
|
panic!("conformance fixture exceeded its step budget")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use prometeu_hal::background_work::{AssetWork, WorkUnavailable};
|
|
#[derive(Default)]
|
|
struct ManualLane(Mutex<VecDeque<AssetWork>>);
|
|
impl BackgroundWork for ManualLane {
|
|
fn submit_asset(&self, work: AssetWork) -> Result<(), WorkUnavailable> {
|
|
self.0.lock().unwrap().push_back(work);
|
|
Ok(())
|
|
}
|
|
}
|
|
impl ManualLane {
|
|
fn step(&self) -> bool {
|
|
let Some(mut work) = self.0.lock().unwrap().pop_front() else { return false };
|
|
if work(false) == WorkStep::Yield {
|
|
self.0.lock().unwrap().push_back(work);
|
|
}
|
|
true
|
|
}
|
|
}
|
|
#[test]
|
|
fn shared_decoder_yields_and_matches_existing_glyph_format() {
|
|
let mut data = vec![0x12; 32];
|
|
data.extend([255; 64]);
|
|
let entry = AssetEntry {
|
|
asset_id: 7,
|
|
asset_name: "glyph".into(),
|
|
bank_type: BankType::GLYPH,
|
|
codec: AssetCodec::None,
|
|
offset: 0,
|
|
size: data.len() as u64,
|
|
decoded_size: 128,
|
|
metadata: serde_json::json!({"tile_size":8,"width":8,"height":8,"palette_count":1,"palette_authored":1}),
|
|
};
|
|
let expected = AssetManager::decode_glyph_bank_from_buffer(&entry, &data).unwrap();
|
|
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
|
|
let mut manager = AssetManager::new(
|
|
vec![entry],
|
|
AssetsPayloadSource::from_bytes(data),
|
|
banks.clone(),
|
|
banks.clone(),
|
|
banks,
|
|
);
|
|
let lane = Arc::new(ManualLane::default());
|
|
assert!(manager.bind_background_work(lane.clone()));
|
|
let handle = manager.load(7, 0).unwrap();
|
|
let mut steps = 0;
|
|
while lane.step() {
|
|
steps += 1;
|
|
assert!(steps < 20);
|
|
}
|
|
assert!(steps >= 4);
|
|
assert_eq!(manager.status(handle), LoadStatus::READY);
|
|
let staging = manager.gfx_policy.staging.read().unwrap();
|
|
let (actual, _) = staging.get(&handle).unwrap();
|
|
assert_eq!(actual.pixel_indices, expected.pixel_indices);
|
|
assert_eq!(actual.palettes, expected.palettes);
|
|
}
|
|
fn decode_in_steps(entry: &AssetEntry, bytes: Vec<u8>) -> Decoded {
|
|
let source = Arc::new(RwLock::new(AssetsPayloadSource::from_bytes(bytes)));
|
|
let mut decoder = Decoder::new(entry, &source).unwrap();
|
|
for _ in 0..100 {
|
|
if let Some(result) = decoder.step().unwrap() {
|
|
return result;
|
|
}
|
|
}
|
|
panic!("decoder did not finish")
|
|
}
|
|
#[test]
|
|
fn resumable_sound_and_scene_preserve_existing_formats() {
|
|
let pcm: Vec<i16> = (0..5000).map(|n| n as i16 - 2500).collect();
|
|
let bytes: Vec<u8> = pcm.iter().flat_map(|n| n.to_le_bytes()).collect();
|
|
let entry = AssetEntry {
|
|
asset_id: 1,
|
|
asset_name: "pcm".into(),
|
|
bank_type: BankType::SOUNDS,
|
|
offset: 0,
|
|
size: bytes.len() as u64,
|
|
decoded_size: bytes.len() as u64,
|
|
codec: AssetCodec::None,
|
|
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
|
|
};
|
|
let Decoded::Sound(bank) = decode_in_steps(&entry, bytes) else { panic!() };
|
|
assert_eq!(bank.samples[0].data, pcm);
|
|
let scene = crate::asset::tests::test_scene();
|
|
let bytes = crate::asset::tests::encode_scene_payload(&scene);
|
|
let entry = crate::asset::tests::test_scene_asset_entry("scene", &bytes, &scene);
|
|
let Decoded::Scene(actual) = decode_in_steps(&entry, bytes) else { panic!() };
|
|
for (a, b) in actual.layers.iter().zip(scene.layers.iter()) {
|
|
assert_eq!(a.active, b.active);
|
|
assert_eq!(a.glyph_asset_id, b.glyph_asset_id);
|
|
assert_eq!(a.tile_size, b.tile_size);
|
|
assert_eq!(a.parallax_factor.x, b.parallax_factor.x);
|
|
assert_eq!(a.tilemap.width, b.tilemap.width);
|
|
assert_eq!(a.tilemap.height, b.tilemap.height);
|
|
assert_eq!(a.tilemap.tiles.len(), b.tilemap.tiles.len());
|
|
for (x, y) in a.tilemap.tiles.iter().zip(&b.tilemap.tiles) {
|
|
assert_eq!(
|
|
(x.active, x.glyph.glyph_id, x.glyph.palette_id, x.flip_x, x.flip_y),
|
|
(y.active, y.glyph.glyph_id, y.glyph.palette_id, y.flip_x, y.flip_y)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
#[test]
|
|
fn shared_preload_waits_for_worker_and_frame_publication() {
|
|
let bytes = vec![0; 8192];
|
|
let entry = AssetEntry {
|
|
asset_id: 1,
|
|
asset_name: "pcm".into(),
|
|
bank_type: BankType::SOUNDS,
|
|
offset: 0,
|
|
size: bytes.len() as u64,
|
|
decoded_size: bytes.len() as u64,
|
|
codec: AssetCodec::None,
|
|
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
|
|
};
|
|
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
|
|
let mut manager = AssetManager::new(
|
|
vec![],
|
|
AssetsPayloadSource::empty(),
|
|
banks.clone(),
|
|
banks.clone(),
|
|
banks,
|
|
);
|
|
let lane = Arc::new(ManualLane::default());
|
|
assert!(manager.bind_background_work(lane.clone()));
|
|
manager.initialize_for_cartridge(
|
|
vec![entry],
|
|
vec![PreloadEntry { asset_id: 1, slot: 0 }],
|
|
AssetsPayloadSource::from_bytes(bytes),
|
|
);
|
|
assert_eq!(manager.preloads_ready(), Ok(false));
|
|
while lane.step() {}
|
|
assert_eq!(manager.preloads_ready(), Ok(false));
|
|
manager.apply_commits();
|
|
assert_eq!(manager.preloads_ready(), Ok(true));
|
|
assert_eq!(manager.slot_info(SlotRef::audio(0)).asset_id, Some(1));
|
|
}
|
|
#[test]
|
|
fn canceled_and_superseded_partial_decoders_never_publish() {
|
|
use crate::memory_banks::SoundBankPoolAccess;
|
|
let bytes = vec![0x12; 16384];
|
|
let entry = AssetEntry {
|
|
asset_id: 1,
|
|
asset_name: "pcm".into(),
|
|
bank_type: BankType::SOUNDS,
|
|
offset: 0,
|
|
size: bytes.len() as u64,
|
|
decoded_size: bytes.len() as u64,
|
|
codec: AssetCodec::None,
|
|
metadata: serde_json::json!({"sample_rate":44100,"channels":1}),
|
|
};
|
|
let mut newer = entry.clone();
|
|
newer.asset_id = 2;
|
|
let banks = Arc::new(crate::memory_banks::MemoryBanks::new());
|
|
let mut manager = AssetManager::new(
|
|
vec![entry, newer],
|
|
AssetsPayloadSource::from_bytes(bytes),
|
|
banks.clone(),
|
|
banks.clone(),
|
|
banks.clone(),
|
|
);
|
|
let lane = Arc::new(ManualLane::default());
|
|
assert!(manager.bind_background_work(lane.clone()));
|
|
let handle = manager.load(1, 0).unwrap();
|
|
assert!(lane.step());
|
|
assert!(lane.step());
|
|
assert_eq!(manager.status(handle), LoadStatus::LOADING);
|
|
assert_eq!(manager.cancel(handle), AssetOpStatus::Ok);
|
|
while lane.step() {}
|
|
assert_eq!(manager.status(handle), LoadStatus::CANCELED);
|
|
assert!(manager.sound_policy.staging.read().unwrap().is_empty());
|
|
manager.apply_commits();
|
|
assert!(banks.sound_bank_slot(0).is_none());
|
|
let old = manager.load(1, 0).unwrap();
|
|
assert!(lane.step());
|
|
assert!(lane.step());
|
|
let new = manager.load(2, 0).unwrap();
|
|
assert_eq!(old, new); // Stable slot handle, different request generation.
|
|
assert!(lane.step()); // Discard the in-flight old decoder.
|
|
assert!(manager.sound_policy.staging.read().unwrap().is_empty());
|
|
while lane.step() {}
|
|
assert_eq!(manager.status(new), LoadStatus::READY);
|
|
assert!(banks.sound_bank_slot(0).is_none());
|
|
assert_eq!(manager.commit(new), AssetOpStatus::Ok);
|
|
manager.apply_commits();
|
|
assert_eq!(manager.slot_info(SlotRef::audio(0)).asset_id, Some(2));
|
|
assert_eq!(banks.sound_bank_slot(0).unwrap().samples[0].data, vec![0x1212; 8192]);
|
|
}
|
|
}
|