feat(audio): add 32-voice PCM mixing and native music sequencing

Add self-contained SoundBanks and MusicBanks with per-sample PCM16 and IMA ADPCM storage, materialized to resident PCM16. Integrate instrument envelopes, automation, shared DSP and host lifecycle controls while preserving staged real-time execution.
This commit is contained in:
2026-11-20 22:15:24 +00:00
parent a32f834b2b
commit 8510c5b00d
40 changed files with 4708 additions and 509 deletions
@@ -960,7 +960,7 @@ fn persistent_save_preempts_real_multichunk_asset_decoders() {
self.inner.sync()
}
}
for kind in [BankType::GLYPH, BankType::SOUNDS, BankType::SCENE] {
for kind in [BankType::GLYPH, BankType::SOUNDS, BankType::MUSIC, BankType::SCENE] {
let (card, disk) = formatted();
drop(card);
let events: Events = Arc::default();
@@ -985,8 +985,37 @@ fn persistent_save_preempts_real_multichunk_asset_decoders() {
)
}
BankType::SOUNDS => {
(vec![0x12; 16385], 16385, serde_json::json!({"sample_rate":44100,"channels":1}))
let mut bytes = vec![0u8; 88];
bytes[..4].copy_from_slice(b"SBNK");
bytes[4] = 1;
bytes[49] = 1;
bytes[80] = 255;
bytes[81] = 128;
for (at, value) in [
(8, 1u32),
(20, 960),
(24, 500000),
(32, u32::MAX),
(52, 44100),
(56, 8192),
(68, 88),
(72, 16384),
(84, 65536),
] {
bytes[at..at + 4].copy_from_slice(&value.to_le_bytes());
}
bytes.extend([0x12; 16384]);
(bytes, 16472, serde_json::json!({"version":1}))
}
BankType::MUSIC => (
include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../test-fixtures/audio/mixed-music.mbnk"
))
.to_vec(),
6395,
serde_json::json!({"version":1}),
),
BankType::SCENE => {
let mut bytes = vec![0; 140];
bytes[..8].copy_from_slice(&[b'S', b'C', b'N', b'E', 1, 0, 4, 0]);
@@ -1079,6 +1108,12 @@ fn persistent_save_preempts_real_multichunk_asset_decoders() {
assert_eq!(bank.samples[0].sample_rate, 44100);
assert_eq!(bank.samples[0].data, vec![0x1212; 8192]);
}
BankType::MUSIC => {
let bank = banks.music_bank_slot(0).unwrap();
assert_eq!(bank.events.len(), 4);
assert_eq!(bank.samples.len(), 2);
assert_eq!(bank.samples[1].channels, 2);
}
BankType::SCENE => {
let bank = banks.scene_bank_slot(0).unwrap();
assert_eq!(
@@ -4,6 +4,7 @@ use crate::services::memcard::MemcardService;
use prometeu_bytecode::{TRAP_INVALID_SYSCALL, TRAP_OOB, TRAP_TYPE, Value};
use prometeu_hal::app_mode::AppMode;
use prometeu_hal::asset::{AssetId, AssetOpStatus, BankType};
use prometeu_hal::audio_bridge::AudioControl;
use prometeu_hal::color::Color;
use prometeu_hal::glyph::Glyph;
use prometeu_hal::log::{LogLevel, LogSource};
@@ -465,7 +466,7 @@ impl NativeInterface for VmRuntimeHost<'_> {
let pan_raw = expect_int(args, 3)?;
let pitch = expect_number(args, 4, "pitch")?;
if !(0..16).contains(&voice_id_raw) {
if !(0..8).contains(&voice_id_raw) {
ret.push_int(AudioOpStatus::VoiceInvalid as i64);
return Ok(());
}
@@ -508,7 +509,7 @@ impl NativeInterface for VmRuntimeHost<'_> {
prometeu_hal::LoopMode::On
};
if !(0..16).contains(&voice_id_raw) {
if !(0..8).contains(&voice_id_raw) {
ret.push_int(AudioOpStatus::VoiceInvalid as i64);
return Ok(());
}
@@ -558,7 +559,7 @@ impl NativeInterface for VmRuntimeHost<'_> {
} else {
1.0
};
let status = if !(0..16).contains(&voice) {
let status = if !(0..8).contains(&voice) {
AudioOpStatus::VoiceInvalid
} else if !(0..=255).contains(&value) || !pitch.is_finite() || pitch <= 0.0 {
AudioOpStatus::ArgRangeInvalid
@@ -575,6 +576,113 @@ impl NativeInterface for VmRuntimeHost<'_> {
ret.push_int(status as i64);
Ok(())
}
Syscall::MusicPlay
| Syscall::MusicStop
| Syscall::MusicPause
| Syscall::MusicResume
| Syscall::MusicSetVolume
| Syscall::MusicFadeTo
| Syscall::MusicCrossfade
| Syscall::AudioSetBusVolume
| Syscall::AudioSetEffect
| Syscall::AudioSetSend
| Syscall::AudioUnloadBank
| Syscall::AudioSetGroupVolume => {
macro_rules! uint {
($i:expr, $max:expr) => {{
let n = expect_int(args, $i)?;
if !(0..=$max).contains(&n) {
ret.push_int(AudioOpStatus::ArgRangeInvalid as i64);
return Ok(());
}
n as u32
}};
}
let status = match syscall {
Syscall::MusicPlay => {
let group = uint!(0, 1);
let bank = uint!(1, 15);
let looping = uint!(2, 1);
platform.audio_mut().music_play(group as usize, bank as usize, looping != 0)
}
Syscall::AudioUnloadBank => {
let music = uint!(0, 1);
let slot = uint!(1, 15);
let result = platform.assets().unload_audio(music != 0, slot as usize);
ret.push_int(if result == AssetOpStatus::Ok {
AudioOpStatus::Ok
} else {
AudioOpStatus::BankInvalid
} as i64);
return Ok(());
}
_ => {
let control = match syscall {
Syscall::MusicStop => {
AudioControl::MusicStop { group: uint!(0, 1) as usize }
}
Syscall::MusicPause | Syscall::MusicResume => {
AudioControl::MusicPause {
group: uint!(0, 1) as usize,
paused: syscall == Syscall::MusicPause,
}
}
Syscall::MusicSetVolume => AudioControl::MusicVolume {
group: uint!(0, 1) as usize,
gain: uint!(1, 255) as u8,
},
Syscall::MusicFadeTo => {
let group = uint!(0, 1) as usize;
let gain = uint!(1, 255) as u8;
let ms = uint!(2, (u32::MAX / 48) as i64);
let curve = uint!(3, 1);
AudioControl::MusicFade {
group,
gain,
frames: ms * 48,
smooth: curve != 0,
}
}
Syscall::MusicCrossfade => {
let source = uint!(0, 1) as usize;
let target = uint!(1, 1) as usize;
let ms = uint!(2, (u32::MAX / 48) as i64);
let curve = uint!(3, 1);
AudioControl::Crossfade {
source,
target,
frames: ms * 48,
smooth: curve != 0,
}
}
Syscall::AudioSetGroupVolume => AudioControl::GroupVolume {
group: uint!(0, 2) as usize,
gain: uint!(1, 255) as u8,
},
Syscall::AudioSetBusVolume => AudioControl::BusVolume {
bus: uint!(0, 2) as usize,
gain: uint!(1, 255) as u8,
},
Syscall::AudioSetSend => AudioControl::SfxSend {
voice: uint!(0, 7) as usize,
gain: uint!(1, 255) as u8,
},
Syscall::AudioSetEffect => AudioControl::Effect {
group: uint!(0, 2) as usize,
cutoff_hz: uint!(1, 24_000),
delay_frames: uint!(2, 2000) * 48,
feedback: expect_number(args, 3, "feedback")? as f32,
wet: expect_number(args, 4, "wet")? as f32,
dry: expect_number(args, 5, "dry")? as f32,
},
_ => unreachable!(),
};
platform.audio_mut().control(control)
}
};
ret.push_int(status as i64);
Ok(())
}
Syscall::FsOpen => {
let path = expect_string(args, 0, "path")?;
if *self.fs_state != FsState::Mounted {
@@ -749,6 +857,7 @@ impl NativeInterface for VmRuntimeHost<'_> {
0 => BankType::GLYPH,
1 => BankType::SOUNDS,
2 => BankType::SCENE,
3 => BankType::MUSIC,
other => {
return Err(VmFault::Trap(
TRAP_TYPE,
@@ -770,6 +879,8 @@ impl NativeInterface for VmRuntimeHost<'_> {
let asset_type = match expect_int(args, 0)? as u32 {
0 => BankType::GLYPH,
1 => BankType::SOUNDS,
2 => BankType::SCENE,
3 => BankType::MUSIC,
_ => return Err(VmFault::Trap(TRAP_TYPE, "Invalid asset type".to_string())),
};
let telemetry = platform
@@ -14,13 +14,16 @@ struct Harness {
}
impl Harness {
fn new(code: &str, name: &str, args: u16) -> Self {
Self::new_domain(code, "audio", name, args)
}
fn new_domain(code: &str, module: &str, name: &str, args: u16) -> Self {
let mut runtime = VirtualMachineRuntime::new(None);
let mut vm = VirtualMachine::default();
let mut log = LogService::new(4096);
let program = serialized_single_function_module(
assemble(code).unwrap(),
vec![SyscallDecl {
module: "audio".into(),
module: module.into(),
name: name.into(),
version: 1,
arg_slots: args,
@@ -76,7 +79,7 @@ impl Harness {
fn guest_audio_status_matrix_and_type_traps() {
for (name, args, inputs, expected) in [
("stop", 1, "PUSH_I32 0", 0),
("stop", 1, "PUSH_I32 15", 0),
("stop", 1, "PUSH_I32 7", 0),
("stop", 1, "PUSH_I32 -1", 1),
("stop", 1, "PUSH_I32 16", 1),
("set_volume", 2, "PUSH_I32 0\nPUSH_I32 0", 0),
@@ -84,7 +87,7 @@ fn guest_audio_status_matrix_and_type_traps() {
("set_volume", 2, "PUSH_I32 0\nPUSH_I32 256", 3),
("set_pan", 2, "PUSH_I32 0\nPUSH_I32 -1", 3),
("set_pan", 2, "PUSH_I32 16\nPUSH_I32 -1", 1),
("set_pan", 2, "PUSH_I32 15\nPUSH_I32 255", 0),
("set_pan", 2, "PUSH_I32 7\nPUSH_I32 255", 0),
("set_pitch", 2, "PUSH_I32 0\nPUSH_I32 1", 0),
("set_pitch", 2, "PUSH_I32 0\nPUSH_I32 0", 3),
("set_pitch", 2, "PUSH_I32 0\nPUSH_I32 -1", 3),
@@ -216,3 +219,61 @@ fn normal_entry_return_is_a_successful_implicit_audio_frame() {
}
assert_eq!(h.transport().try_take().unwrap().frame.unwrap().len(), 1);
}
#[test]
fn music_and_bus_controls_resolve_with_audio_capability_and_validate_without_partial_commands() {
use prometeu_drivers::SoundBankPoolInstaller;
use prometeu_drivers::bank_format::{DecodedAudioBank, decode_audio_bank};
for (module, name, args, inputs, expected) in [
("music", "play", 3, "PUSH_I32 0\nPUSH_I32 0\nPUSH_I32 1", 0),
("music", "play", 3, "PUSH_I32 0\nPUSH_I32 16\nPUSH_I32 1", 3),
("music", "stop", 1, "PUSH_I32 1", 0),
("music", "pause", 1, "PUSH_I32 0", 0),
("music", "resume", 1, "PUSH_I32 0", 0),
("music", "set_volume", 2, "PUSH_I32 0\nPUSH_I32 255", 0),
("music", "fade_to", 4, "PUSH_I32 0\nPUSH_I32 0\nPUSH_I32 2000\nPUSH_I32 1", 0),
("music", "crossfade", 4, "PUSH_I32 0\nPUSH_I32 1\nPUSH_I32 2000\nPUSH_I32 0", 0),
("music", "crossfade", 4, "PUSH_I32 0\nPUSH_I32 0\nPUSH_I32 2000\nPUSH_I32 0", 3),
("audio", "set_group_volume", 2, "PUSH_I32 2\nPUSH_I32 255", 0),
("audio", "set_bus_volume", 2, "PUSH_I32 2\nPUSH_I32 255", 0),
("audio", "set_send", 2, "PUSH_I32 7\nPUSH_I32 128", 0),
(
"audio",
"set_effect",
6,
"PUSH_I32 2\nPUSH_I32 4000\nPUSH_I32 2000\nPUSH_F64 f64:0x3fe0000000000000\nPUSH_F64 f64:0x3fd3333333333333\nPUSH_F64 f64:0x3fe6666666666666",
0,
),
(
"audio",
"set_effect",
6,
"PUSH_I32 2\nPUSH_I32 4000\nPUSH_I32 2001\nPUSH_F64 f64:0x3fe0000000000000\nPUSH_F64 f64:0x3fd3333333333333\nPUSH_F64 f64:0x3fe6666666666666",
3,
),
("audio", "unload_bank", 2, "PUSH_I32 1\nPUSH_I32 0", 0),
] {
let mut h = Harness::new_domain(
&format!("{inputs}\nHOSTCALL 0\nFRAME_SYNC\nHALT"),
module,
name,
args,
);
let bytes = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../test-fixtures/audio/mixed-music.mbnk"
));
let DecodedAudioBank::Music(bank) = decode_audio_bank(bytes, true).unwrap() else {
panic!()
};
h.platform.local_hardware().memory_banks.install_music_bank(0, Arc::new(bank));
assert!(h.tick().is_none(), "{module}.{name}");
assert_eq!(h.vm.operand_stack_top(1), vec![Value::Int64(expected)], "{module}.{name}");
let batch = h.transport().try_take().unwrap().frame;
assert_eq!(
batch.map_or(0, |v| v.len()),
usize::from(expected == 0 && name != "unload_bank"),
"{module}.{name}"
);
}
}