feat(audio): add 32-voice PCM mixing and native music sequencing

Add self-contained SoundBanks and MusicBanks with per-sample PCM16 and IMA ADPCM storage, materialized to resident PCM16. Integrate instrument envelopes, automation, shared DSP and host lifecycle controls while preserving staged real-time execution.
This commit is contained in:
2026-11-20 22:15:24 +00:00
parent a32f834b2b
commit 8510c5b00d
40 changed files with 4708 additions and 509 deletions
+15 -11
View File
@@ -8,6 +8,7 @@ pub type AssetId = i32;
pub enum BankType {
GLYPH,
SOUNDS,
MUSIC,
SCENE,
// TILEMAPS,
// BLOBS,
@@ -41,9 +42,8 @@ pub struct GlyphBankMetadata {
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct SoundBankMetadata {
pub sample_rate: u32,
pub channels: u32,
pub struct AudioBankMetadata {
pub version: u16,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
@@ -73,15 +73,16 @@ impl AssetEntry {
.map_err(|e| format!("Invalid GLYPH metadata for asset {}: {}", self.asset_id, e))
}
pub fn metadata_as_sound_bank(&self) -> Result<SoundBankMetadata, String> {
if self.bank_type != BankType::SOUNDS {
return Err(format!(
"Asset {} is not a SOUNDS bank (type: {:?})",
self.asset_id, self.bank_type
));
pub fn metadata_as_audio_bank(&self) -> Result<AudioBankMetadata, String> {
if !matches!(self.bank_type, BankType::SOUNDS | BankType::MUSIC) {
return Err("asset is not an audio bank".into());
}
serde_json::from_value(self.metadata.clone())
.map_err(|e| format!("Invalid SOUNDS metadata for asset {}: {}", self.asset_id, e))
let metadata: AudioBankMetadata = serde_json::from_value(self.metadata.clone())
.map_err(|e| format!("Invalid audio metadata: {e}"))?;
if metadata.version != 1 {
return Err("unsupported audio metadata version".into());
}
Ok(metadata)
}
pub fn metadata_as_scene_bank(&self) -> Result<SceneBankMetadata, String> {
@@ -189,6 +190,9 @@ pub struct SlotRef {
}
impl SlotRef {
pub fn music(index: usize) -> Self {
Self { asset_type: BankType::MUSIC, index }
}
pub fn gfx(index: usize) -> Self {
Self { asset_type: BankType::GLYPH, index }
}
@@ -6,6 +6,9 @@ use crate::asset::{
use crate::cartridge::AssetsPayloadSource;
pub trait AssetBridge {
fn unload_audio(&self, _music: bool, _slot: usize) -> AssetOpStatus {
AssetOpStatus::InvalidState
}
/// Bind before cartridge preload/loading. Hosts supporting memcard must use
/// this same lane for assets; an unsupported bridge refuses the fast profile.
fn bind_background_work(
@@ -20,6 +20,20 @@ pub enum AudioOpStatus {
CommandLimit = 7,
}
/// High-level commands are staged with the ordinary frame/lifecycle batch.
#[derive(Debug, Clone, Copy)]
pub enum AudioControl {
GroupVolume { group: usize, gain: u8 },
MusicStop { group: usize },
MusicPause { group: usize, paused: bool },
MusicVolume { group: usize, gain: u8 },
MusicFade { group: usize, gain: u8, frames: u32, smooth: bool },
Crossfade { source: usize, target: usize, frames: u32, smooth: bool },
BusVolume { bus: usize, gain: u8 },
Effect { group: usize, cutoff_hz: u32, delay_frames: u32, feedback: f32, wet: f32, dry: f32 },
SfxSend { voice: usize, gain: u8 },
}
pub trait AudioBridge {
#[allow(clippy::too_many_arguments)]
fn play(
@@ -44,6 +58,8 @@ pub trait AudioBridge {
priority: u8,
loop_mode: LoopMode,
) -> AudioOpStatus;
fn music_play(&mut self, group: usize, bank: usize, looping: bool) -> AudioOpStatus;
fn control(&mut self, control: AudioControl) -> AudioOpStatus;
fn stop(&mut self, voice_id: usize) -> AudioOpStatus;
fn set_volume(&mut self, voice_id: usize, volume: u8) -> AudioOpStatus;
fn set_pan(&mut self, voice_id: usize, pan: u8) -> AudioOpStatus;
+1
View File
@@ -18,6 +18,7 @@ pub mod host_return;
pub mod input_signals;
pub mod log;
pub mod memcard;
pub mod music_bank;
pub mod native_helpers;
pub mod native_interface;
pub mod owned_frame;
@@ -0,0 +1,376 @@
//! Resident, self-contained music data. Construction/compilation is off the audio lane.
use crate::{sample::Sample, sound_bank::AudioBankMetrics};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
pub const AUDIO_PPQ: u32 = 960;
pub const AUDIO_RATE: u32 = 48_000;
pub const MAX_MUSIC_TRACKS: usize = 256;
pub const MAX_EVENTS_PER_FRAME: usize = 64;
#[derive(Debug, Clone, Copy)]
pub struct Instrument {
pub id: u16,
pub sample: u16,
pub low: u8,
pub high: u8,
pub root: u8,
pub looping: bool,
pub tuning_cents: i16,
pub pan: u8,
pub gain: u8,
pub attack_us: u32,
pub decay_us: u32,
pub sustain: u16,
pub release_us: u32,
pub vibrato_millihz: u32,
pub vibrato_cents: u16,
pub tremolo_depth: u16,
pub tremolo_millihz: u32,
pub send: u16,
pub curve: u8,
}
impl Default for Instrument {
fn default() -> Self {
Self {
id: 0,
sample: 0,
low: 0,
high: 127,
root: 60,
looping: false,
tuning_cents: 0,
pan: 128,
gain: 255,
attack_us: 0,
decay_us: 0,
sustain: u16::MAX,
release_us: 0,
vibrato_millihz: 0,
vibrato_cents: 0,
tremolo_depth: 0,
tremolo_millihz: 0,
send: 0,
curve: 1,
}
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct MusicEvent {
pub tick: u32,
pub kind: u8,
pub track: u8,
pub flags: u16,
pub a: u32,
pub b: u32,
pub c: i32,
pub d: i32,
pub duration_frames: u32,
pub curve: u32,
}
#[derive(Debug, Clone, Copy)]
pub struct TimedMusicEvent {
pub time: u128,
pub event: MusicEvent,
pub zone: usize,
}
#[derive(Debug)]
pub struct MusicBank {
pub name: String,
pub samples: Arc<Vec<Arc<Sample>>>,
pub instruments: Arc<Vec<Instrument>>,
pub events: Arc<Vec<TimedMusicEvent>>,
pub denominator: u128,
pub end_time: u128,
pub loop_time: Option<u128>,
pub loop_event: usize,
pub tracks: usize,
pub alive: Arc<AtomicBool>,
pub metrics: AudioBankMetrics,
}
impl MusicBank {
#[allow(clippy::too_many_arguments)]
pub fn compile(
name: String,
samples: Vec<Arc<Sample>>,
instruments: Vec<Instrument>,
events: Vec<MusicEvent>,
ppq: u32,
tempo: u32,
duration: u32,
loop_tick: Option<u32>,
tracks: usize,
) -> Result<Self, String> {
let mut compiler = MusicCompiler::new(
name,
samples,
instruments,
events,
ppq,
tempo,
duration,
loop_tick,
tracks,
)?;
loop {
if let Some(bank) = compiler.step()? {
return Ok(bank);
}
}
}
pub fn invalidate(&self) {
self.alive.store(false, Ordering::Release);
}
pub fn resident_copy(&self) -> Self {
Self {
name: self.name.clone(),
samples: self.samples.clone(),
instruments: self.instruments.clone(),
events: self.events.clone(),
denominator: self.denominator,
end_time: self.end_time,
loop_time: self.loop_time,
loop_event: self.loop_event,
tracks: self.tracks,
alive: Arc::new(AtomicBool::new(true)),
metrics: self.metrics,
}
}
}
/// Cooperative off-lane validation/clock compilation. At most 64 zones/events
/// per step; lookup is logarithmic and the loop-seam check is bounded by 65.
pub struct MusicCompiler {
name: Option<String>,
samples: Vec<Arc<Sample>>,
instruments: Vec<Instrument>,
source: std::vec::IntoIter<MusicEvent>,
compiled: Vec<TimedMusicEvent>,
zones: std::collections::BTreeMap<(u16, u8), usize>,
zone_cursor: usize,
ppq: u32,
tempo: u32,
duration: u32,
loop_tick: Option<u32>,
tracks: usize,
tick: u32,
time: u128,
loop_time: Option<u128>,
selected: [u32; MAX_MUSIC_TRACKS],
}
impl MusicCompiler {
#[allow(clippy::too_many_arguments)]
pub fn new(
name: String,
samples: Vec<Arc<Sample>>,
instruments: Vec<Instrument>,
events: Vec<MusicEvent>,
ppq: u32,
tempo: u32,
duration: u32,
loop_tick: Option<u32>,
tracks: usize,
) -> Result<Self, String> {
if ppq != AUDIO_PPQ
|| tempo == 0
|| duration == 0
|| tracks == 0
|| tracks > MAX_MUSIC_TRACKS
|| loop_tick.is_some_and(|t| t >= duration)
{
return Err("invalid music clock/tracks/loop".into());
}
let mut compiled = Vec::new();
compiled.try_reserve_exact(events.len()).map_err(|_| "music allocation failed")?;
Ok(Self {
name: Some(name),
samples,
instruments,
source: events.into_iter(),
compiled,
zones: std::collections::BTreeMap::new(),
zone_cursor: 0,
ppq,
tempo,
duration,
loop_tick,
tracks,
tick: 0,
time: 0,
loop_time: None,
selected: [u32::MAX; MAX_MUSIC_TRACKS],
})
}
pub fn step(&mut self) -> Result<Option<MusicBank>, String> {
let denominator = self.ppq as u128 * 1_000_000;
for _ in 0..64 {
if self.zone_cursor < self.instruments.len() {
let i = self.zone_cursor;
let inst = self.instruments[i];
if inst.low > inst.high
|| inst.high > 127
|| inst.root > 127
|| inst.curve > 1
|| inst.sample as usize >= self.samples.len()
|| inst.vibrato_millihz > 100_000
|| inst.tremolo_millihz > 100_000
|| inst.vibrato_cents > 2400
{
return Err("invalid instrument zone/modulation".into());
}
if inst.looping && self.samples[inst.sample as usize].loop_start.is_none() {
return Err("instrument requires a sample loop".into());
}
if self
.zones
.range((inst.id, 0)..=(inst.id, inst.high))
.next_back()
.is_some_and(|(_, j)| self.instruments[*j].high >= inst.low)
{
return Err("overlapping instrument zones".into());
}
self.zones.insert((inst.id, inst.low), i);
self.zone_cursor += 1;
continue;
}
let Some(event) = self.source.next() else {
return self.finish(denominator).map(Some);
};
if event.tick < self.tick
|| event.tick > self.duration
|| event.track as usize >= self.tracks
|| event.curve > 1
|| event.flags & !1 != 0
|| !(1..=10).contains(&event.kind)
{
return Err("invalid music event ordering/shape".into());
}
if let Some(t) = self.loop_tick.filter(|&t| t >= self.tick && t <= event.tick)
&& self.loop_time.is_none()
{
self.loop_time = Some(
self.time + (t - self.tick) as u128 * self.tempo as u128 * AUDIO_RATE as u128,
);
}
self.time += (event.tick - self.tick) as u128 * self.tempo as u128 * AUDIO_RATE as u128;
self.tick = event.tick;
let mut zone = 0;
match event.kind {
1 => {
if !(0..=127).contains(&event.c) || !(0..=255).contains(&event.d) {
return Err("invalid note/velocity".into());
}
let id = if event.b == u32::MAX {
self.selected[event.track as usize]
} else {
event.b
};
if id > u16::MAX as u32 {
return Err("note has no bank-local instrument".into());
}
zone = *self
.zones
.range((id as u16, 0)..=(id as u16, event.c as u8))
.next_back()
.map(|(_, i)| i)
.filter(|&&i| self.instruments[i].high as i32 >= event.c)
.ok_or("note has no bank-local instrument zone")?;
}
3 => {
if event.a == 0 {
return Err("zero tempo".into());
}
self.tempo = event.a;
}
4 | 9 => {
if !(0..=65535).contains(&event.c) {
return Err("invalid gain/send".into());
}
}
5 => {
if !(0..=255).contains(&event.c) {
return Err("invalid pan".into());
}
}
6 => {
if !(-9600..=9600).contains(&event.c) {
return Err("invalid pitch bend".into());
}
}
7 | 8 => {
if event.a > 100_000
|| event.c < 0
|| event.c > if event.kind == 7 { 2400 } else { 65535 }
{
return Err("invalid modulation".into());
}
}
10 => {
if event.a > u16::MAX as u32
|| self
.zones
.range((event.a as u16, 0)..=(event.a as u16, 127))
.next()
.is_none()
{
return Err("unknown instrument".into());
}
self.selected[event.track as usize] = event.a;
}
_ => {}
}
if self.compiled.len() >= MAX_EVENTS_PER_FRAME
&& self.time - self.compiled[self.compiled.len() - MAX_EVENTS_PER_FRAME].time
< denominator
{
return Err("music event density exceeds real-time bound".into());
}
self.compiled.push(TimedMusicEvent { time: self.time, event, zone });
}
Ok(None)
}
fn finish(&mut self, denominator: u128) -> Result<MusicBank, String> {
if let Some(t) = self.loop_tick.filter(|&t| t >= self.tick)
&& self.loop_time.is_none()
{
self.loop_time =
Some(self.time + (t - self.tick) as u128 * self.tempo as u128 * AUDIO_RATE as u128);
}
let end_time = self.time
+ (self.duration - self.tick) as u128 * self.tempo as u128 * AUDIO_RATE as u128;
if end_time < denominator || self.loop_time.is_some_and(|t| end_time - t < denominator) {
return Err("music/loop shorter than one output frame".into());
}
let loop_event =
self.loop_time.map_or(0, |t| self.compiled.partition_point(|e| e.time < t));
let length = end_time - self.loop_time.unwrap_or(0);
let slice = &self.compiled[loop_event..];
for i in 0..slice.len().min(MAX_EVENTS_PER_FRAME + 1) {
let tail =
slice.len().saturating_sub(MAX_EVENTS_PER_FRAME - i.min(MAX_EVENTS_PER_FRAME));
if tail < slice.len()
&& slice.len() - tail + i + 1 > MAX_EVENTS_PER_FRAME
&& slice[i].time + length - slice[tail].time < denominator
{
return Err("music loop seam event density".into());
}
}
let pcm_bytes = self.samples.iter().map(|s| s.data.len() * 2).sum();
Ok(MusicBank {
name: self.name.take().ok_or("compiler already complete")?,
samples: Arc::new(std::mem::take(&mut self.samples)),
instruments: Arc::new(std::mem::take(&mut self.instruments)),
events: Arc::new(std::mem::take(&mut self.compiled)),
denominator,
end_time,
loop_time: self.loop_time,
loop_event,
tracks: self.tracks,
alive: Arc::new(AtomicBool::new(true)),
metrics: AudioBankMetrics { pcm_bytes, ..Default::default() },
})
}
}
+24 -2
View File
@@ -1,13 +1,27 @@
#[derive(Debug)]
pub struct Sample {
pub sample_rate: u32,
pub channels: u8,
pub data: Vec<i16>,
pub loop_start: Option<u32>,
pub loop_end: Option<u32>,
pub default_volume: u8,
pub default_pan: u8,
pub default_pitch: f64,
}
impl Sample {
pub fn new(sample_rate: u32, data: Vec<i16>) -> Self {
Self { sample_rate, data, loop_start: None, loop_end: None }
Self {
sample_rate,
channels: 1,
data,
loop_start: None,
loop_end: None,
default_volume: 255,
default_pan: 128,
default_pitch: 1.0,
}
}
pub fn with_loop(mut self, start: u32, end: u32) -> Self {
@@ -17,6 +31,14 @@ impl Sample {
}
pub fn frames_len(&self) -> usize {
self.data.len()
self.data.len() / self.channels.max(1) as usize
}
pub fn with_channels(mut self, channels: u8) -> Result<Self, String> {
if !(1..=2).contains(&channels) || !self.data.len().is_multiple_of(channels as usize) {
return Err("audio sample channels/frame count".into());
}
self.channels = channels;
Ok(self)
}
}
+33 -2
View File
@@ -1,16 +1,47 @@
use crate::sample::Sample;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
#[derive(Debug, Clone, Copy, Default)]
pub struct AudioBankMetrics {
pub cartridge_bytes: usize,
pub pcm_bytes: usize,
pub canonical_decoded_bytes: usize,
pub materialization_ns: u64,
}
/// A container for audio assets.
///
/// A SoundBank stores multiple audio samples that can be played by the
/// audio subsystem.
#[derive(Debug)]
pub struct SoundBank {
pub samples: Vec<Arc<Sample>>,
pub samples: Arc<Vec<Arc<Sample>>>,
pub name: String,
pub alive: Arc<AtomicBool>,
pub metrics: AudioBankMetrics,
}
impl SoundBank {
pub fn new(samples: Vec<Arc<Sample>>) -> Self {
Self { samples }
let pcm_bytes = samples.iter().map(|s| s.data.len() * 2).sum();
Self {
samples: Arc::new(samples),
name: String::new(),
alive: Arc::new(AtomicBool::new(true)),
metrics: AudioBankMetrics { pcm_bytes, ..Default::default() },
}
}
pub fn invalidate(&self) {
self.alive.store(false, Ordering::Release);
}
pub fn resident_copy(&self) -> Self {
Self {
samples: self.samples.clone(),
name: self.name.clone(),
alive: Arc::new(AtomicBool::new(true)),
metrics: self.metrics,
}
}
}
@@ -59,6 +59,19 @@ pub enum Syscall {
AudioSetVolume = 0x3004,
AudioSetPan = 0x3005,
AudioSetPitch = 0x3006,
MusicPlay = 0x3010,
MusicStop = 0x3011,
MusicPause = 0x3012,
MusicResume = 0x3013,
MusicSetVolume = 0x3014,
MusicFadeTo = 0x3015,
MusicCrossfade = 0x3016,
AudioSetBusVolume = 0x3020,
AudioSetEffect = 0x3021,
AudioSetSend = 0x3022,
AudioUnloadBank = 0x3023,
AudioSetGroupVolume = 0x3024,
FsOpen = 0x4001,
FsRead = 0x4002,
FsWrite = 0x4003,
@@ -31,4 +31,64 @@ pub(crate) const ENTRIES: &[SyscallRegistryEntry] = &[
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicPlay, "music", "play")
.args(3)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicStop, "music", "stop")
.args(1)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicPause, "music", "pause")
.args(1)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicResume, "music", "resume")
.args(1)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicSetVolume, "music", "set_volume")
.args(2)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicFadeTo, "music", "fade_to")
.args(4)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::MusicCrossfade, "music", "crossfade")
.args(4)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::AudioSetBusVolume, "audio", "set_bus_volume")
.args(2)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::AudioSetEffect, "audio", "set_effect")
.args(6)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::AudioSetSend, "audio", "set_send")
.args(2)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::AudioUnloadBank, "audio", "unload_bank")
.args(2)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
SyscallRegistryEntry::builder(Syscall::AudioSetGroupVolume, "audio", "set_group_volume")
.args(2)
.rets(1)
.caps(caps::AUDIO)
.cost(5),
];
@@ -37,6 +37,19 @@ impl Syscall {
0x3004 => Some(Self::AudioSetVolume),
0x3005 => Some(Self::AudioSetPan),
0x3006 => Some(Self::AudioSetPitch),
0x3010 => Some(Self::MusicPlay),
0x3011 => Some(Self::MusicStop),
0x3012 => Some(Self::MusicPause),
0x3013 => Some(Self::MusicResume),
0x3014 => Some(Self::MusicSetVolume),
0x3015 => Some(Self::MusicFadeTo),
0x3016 => Some(Self::MusicCrossfade),
0x3020 => Some(Self::AudioSetBusVolume),
0x3021 => Some(Self::AudioSetEffect),
0x3022 => Some(Self::AudioSetSend),
0x3023 => Some(Self::AudioUnloadBank),
0x3024 => Some(Self::AudioSetGroupVolume),
0x4001 => Some(Self::FsOpen),
0x4002 => Some(Self::FsRead),
0x4003 => Some(Self::FsWrite),
@@ -110,6 +123,19 @@ impl Syscall {
Self::AudioSetVolume => "AudioSetVolume",
Self::AudioSetPan => "AudioSetPan",
Self::AudioSetPitch => "AudioSetPitch",
Self::MusicPlay => "MusicPlay",
Self::MusicStop => "MusicStop",
Self::MusicPause => "MusicPause",
Self::MusicResume => "MusicResume",
Self::MusicSetVolume => "MusicSetVolume",
Self::MusicFadeTo => "MusicFadeTo",
Self::MusicCrossfade => "MusicCrossfade",
Self::AudioSetBusVolume => "AudioSetBusVolume",
Self::AudioSetEffect => "AudioSetEffect",
Self::AudioSetSend => "AudioSetSend",
Self::AudioUnloadBank => "AudioUnloadBank",
Self::AudioSetGroupVolume => "AudioSetGroupVolume",
Self::FsOpen => "FsOpen",
Self::FsRead => "FsRead",
Self::FsWrite => "FsWrite",