feat(audio): add guest voice controls and reliable lifecycle delivery
Intrepid/Prometeu/Runtime/pipeline/pr-master This commit looks good

This commit is contained in:
2026-10-14 09:08:27 +01:00
parent 3899b7f12c
commit e0e01ed1de
23 changed files with 1442 additions and 482 deletions
@@ -1,12 +1,14 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use prometeu_drivers::{AudioCommand, Channel, MAX_CHANNELS, OUTPUT_SAMPLE_RATE};
use prometeu_drivers::{AudioDelivery, AudioTransport};
use prometeu_hal::LoopMode;
use ringbuf::HeapRb;
use ringbuf::traits::{Consumer, Producer, Split};
use std::error::Error;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
trait AudioStreamHandle {}
@@ -88,7 +90,11 @@ impl AudioRuntime for CpalAudioRuntime {
}
pub struct HostAudio {
pub producer: Option<ringbuf::wrap::CachingProd<Arc<HeapRb<AudioCommand>>>>,
transport: AudioTransport,
failed: Arc<AtomicBool>,
heartbeat: Arc<AtomicU64>,
last_heartbeat: u64,
last_progress: Instant,
pub perf_consumer: Option<ringbuf::wrap::CachingCons<Arc<HeapRb<u64>>>>,
_stream: Option<Box<dyn AudioStreamHandle>>,
}
@@ -101,15 +107,27 @@ impl Default for HostAudio {
impl HostAudio {
pub fn new() -> Self {
Self { producer: None, perf_consumer: None, _stream: None }
Self {
transport: AudioTransport::default(),
failed: Arc::new(AtomicBool::new(false)),
heartbeat: Arc::new(AtomicU64::new(0)),
last_heartbeat: 0,
last_progress: Instant::now(),
perf_consumer: None,
_stream: None,
}
}
pub fn init(&mut self) -> Result<(), HostAudioError> {
pub fn init(&mut self, transport: AudioTransport) -> Result<(), HostAudioError> {
self.transport = transport;
self.init_with_runtime(&CpalAudioRuntime)
}
fn init_with_runtime<R: AudioRuntime>(&mut self, runtime: &R) -> Result<(), HostAudioError> {
self.producer = None;
self.failed = Arc::new(AtomicBool::new(false));
self.heartbeat = Arc::new(AtomicU64::new(0));
self.last_heartbeat = 0;
self.last_progress = Instant::now();
self.perf_consumer = None;
self._stream = None;
@@ -121,8 +139,9 @@ impl HostAudio {
buffer_size: cpal::BufferSize::Default,
};
let rb = HeapRb::<AudioCommand>::new(1024);
let (prod, mut cons) = rb.split();
let transport = self.transport.clone();
let failed = self.failed.clone();
let heartbeat = self.heartbeat.clone();
let mut mixer = AudioMixer::new();
@@ -134,34 +153,45 @@ impl HostAudio {
&device,
&config,
move |data: &mut [f32]| {
// Consumes commands from ringbuffer
while let Some(cmd) = cons.try_pop() {
mixer.process_command(cmd);
heartbeat.fetch_add(1, Ordering::Relaxed);
if let Some(delivery) = transport.try_take() {
mixer.apply_delivery(delivery);
}
// Mixes audio
mixer.fill_buffer(data);
// Sends processing time in microseconds
let _ = perf_prod.try_push(mixer.last_processing_time.as_micros() as u64);
},
|err| eprintln!("audio stream error: {}", err),
move |err| {
failed.store(true, Ordering::Relaxed);
eprintln!("audio stream error: {err}; switching to silent audio");
},
)?;
runtime.play_stream(&stream)?;
self.producer = Some(prod);
self.perf_consumer = Some(perf_cons);
self._stream = Some(Box::new(stream));
Ok(())
}
pub fn send_commands(&mut self, commands: &mut Vec<AudioCommand>) {
if let Some(producer) = &mut self.producer {
for cmd in commands.drain(..) {
if producer.try_push(cmd).is_err() {
eprintln!("[HostAudio] Command ringbuffer full, dropping command.");
}
}
} else {
commands.clear();
/// Called even while guest execution is backpressured. A failed or stalled
/// device cannot retain the only publication slot indefinitely.
pub fn service(&mut self) {
let heartbeat = self.heartbeat.load(Ordering::Relaxed);
if heartbeat != self.last_heartbeat {
self.last_heartbeat = heartbeat;
self.last_progress = Instant::now();
}
if self._stream.is_some()
&& (self.failed.load(Ordering::Relaxed)
|| self.last_progress.elapsed() >= Duration::from_secs(2))
{
eprintln!("[HostAudio] Output unavailable/stalled; consuming audio silently.");
self._stream = None;
self.perf_consumer = None;
}
if self._stream.is_none() {
self.transport.consume_silently();
}
}
@@ -221,14 +251,10 @@ mod tests {
}
#[test]
fn test_send_commands_without_audio_drops_pending_commands() {
fn test_service_without_audio_is_safe() {
let mut audio = HostAudio::new();
let mut commands = vec![AudioCommand::MasterPause, AudioCommand::MasterResume];
audio.send_commands(&mut commands);
assert!(commands.is_empty());
assert!(audio.producer.is_none());
audio.service();
assert!(audio.transport.ready());
}
#[test]
@@ -250,7 +276,6 @@ mod tests {
let result = audio.init_with_runtime(&runtime);
assert_eq!(result, Err(HostAudioError::NoOutputDevice));
assert!(audio.producer.is_none());
assert!(audio.perf_consumer.is_none());
assert!(audio._stream.is_none());
}
@@ -267,7 +292,6 @@ mod tests {
let result = audio.init_with_runtime(&runtime);
assert_eq!(result, Err(HostAudioError::BuildStream("simulated build failure".to_string())));
assert!(audio.producer.is_none());
assert!(audio.perf_consumer.is_none());
assert!(audio._stream.is_none());
}
@@ -284,7 +308,6 @@ mod tests {
let result = audio.init_with_runtime(&runtime);
assert_eq!(result, Err(HostAudioError::PlayStream("simulated play failure".to_string())));
assert!(audio.producer.is_none());
assert!(audio.perf_consumer.is_none());
assert!(audio._stream.is_none());
}
@@ -296,7 +319,6 @@ mod tests {
audio.init_with_runtime(&runtime).unwrap();
assert!(audio.producer.is_some());
assert!(audio.perf_consumer.is_some());
assert!(audio._stream.is_some());
}
@@ -306,6 +328,7 @@ pub struct AudioMixer {
voices: [Channel; MAX_CHANNELS],
pub last_processing_time: Duration,
paused: bool,
epoch: u64,
}
impl Default for AudioMixer {
@@ -316,17 +339,35 @@ impl Default for AudioMixer {
impl AudioMixer {
pub fn new() -> Self {
Self { voices: Default::default(), last_processing_time: Duration::ZERO, paused: false }
Self {
voices: Default::default(),
last_processing_time: Duration::ZERO,
paused: false,
epoch: 0,
}
}
pub fn apply_delivery(&mut self, delivery: AudioDelivery) {
if delivery.epoch < self.epoch {
return;
}
if delivery.epoch != self.epoch {
self.voices = Default::default();
self.epoch = delivery.epoch;
}
// A frame pending before a lifecycle hook is always applied first.
for batch in [delivery.frame, delivery.lifecycle].into_iter().flatten() {
for command in batch {
self.process_command(command);
}
}
self.paused = delivery.paused;
}
pub fn process_command(&mut self, cmd: AudioCommand) {
match cmd {
AudioCommand::Play { sample, voice_id, volume, pan, pitch, priority, loop_mode } => {
if voice_id < MAX_CHANNELS {
println!(
"[AudioMixer] Playing voice {}: vol={}, pitch={}, loop={:?}",
voice_id, volume, pitch, loop_mode
);
self.voices[voice_id] = Channel {
sample: Some(sample),
active: true,
@@ -360,14 +401,6 @@ impl AudioMixer {
self.voices[voice_id].pitch = pitch;
}
}
AudioCommand::MasterPause => {
println!("[AudioMixer] Master Pause");
self.paused = true;
}
AudioCommand::MasterResume => {
println!("[AudioMixer] Master Resume");
self.paused = false;
}
}
}
@@ -384,59 +417,62 @@ impl AudioMixer {
}
for voice in self.voices.iter_mut() {
let sample_data = match &voice.sample {
Some(s) => s,
None => continue,
let Some(sample) = voice.sample.as_ref() else {
continue;
};
let pitch_ratio = sample_data.sample_rate as f64 / OUTPUT_SAMPLE_RATE as f64;
let step = voice.pitch * pitch_ratio;
let vol_f = voice.volume as f32 / 255.0;
let pan_f = voice.pan as f32 / 255.0;
let vol_l = vol_f * (1.0 - pan_f).sqrt();
let vol_r = vol_f * pan_f.sqrt();
let len = sample.data.len();
if len == 0 {
voice.active = false;
voice.sample = None;
continue;
}
let (loop_start, end) = match (sample.loop_start, sample.loop_end) {
(Some(start), Some(end)) if start < end && end as usize <= len => {
(start as f64, end as f64)
}
_ => (0.0, len as f64),
};
let ratio = sample.sample_rate as f64 / OUTPUT_SAMPLE_RATE as f64;
let volume = voice.volume as f32 / 255.0;
let pan = voice.pan as f32 / 255.0;
let left = volume * (1.0 - pan).sqrt();
let right = volume * pan.sqrt();
for frame in buffer.as_chunks_mut::<2>().0 {
let pos_int = voice.pos as usize;
let pos_fract = voice.pos - pos_int as f64;
if pos_int >= sample_data.data.len() {
let pos = voice.pos as usize;
if pos >= len {
voice.active = false;
voice.sample = None;
break;
}
// Linear Interpolation
let s1 = sample_data.data[pos_int] as f32 / 32768.0;
let s2 = if pos_int + 1 < sample_data.data.len() {
sample_data.data[pos_int + 1] as f32 / 32768.0
} else if voice.loop_mode == LoopMode::On {
let loop_start = sample_data.loop_start.unwrap_or(0) as usize;
sample_data.data[loop_start] as f32 / 32768.0
let s1 = sample.data[pos] as f32 / 32768.0;
let next = if voice.loop_mode == LoopMode::On && (pos + 1) as f64 >= end {
loop_start as usize
} else {
0.0
pos + 1
};
let sample_val = s1 + (s2 - s1) * pos_fract as f32;
frame[0] += sample_val * vol_l;
frame[1] += sample_val * vol_r;
voice.pos += step;
let end_pos =
sample_data.loop_end.map(|e| e as f64).unwrap_or(sample_data.data.len() as f64);
if voice.pos >= end_pos {
if voice.loop_mode == LoopMode::On {
let loop_start = sample_data.loop_start.unwrap_or(0) as f64;
voice.pos = loop_start + (voice.pos - end_pos);
} else {
let s2 = sample.data.get(next).copied().unwrap_or(0) as f32 / 32768.0;
let value = s1 + (s2 - s1) * (voice.pos - pos as f64) as f32;
frame[0] += value * left;
frame[1] += value * right;
if ratio == 0.0 {
continue;
}
if voice.pitch >= (end - voice.pos) / ratio {
if voice.loop_mode == LoopMode::Off {
voice.active = false;
voice.sample = None;
break;
}
let length = end - loop_start;
// Reduce before multiplication: every finite positive pitch,
// including f64::MAX, remains finite and wraps multiple loops.
let step = (voice.pitch % (length / ratio)) * ratio;
voice.pos = loop_start + (voice.pos - loop_start + step).rem_euclid(length);
if voice.pos >= end {
voice.pos = loop_start;
}
} else {
voice.pos += voice.pitch * ratio;
}
}
}
@@ -448,3 +484,241 @@ impl AudioMixer {
self.last_processing_time = start.elapsed();
}
}
#[cfg(test)]
mod control_tests {
use super::*;
use prometeu_drivers::{Audio, MemoryBanks};
use prometeu_hal::{AudioBridge, AudioOpStatus, sample::Sample};
fn pair() -> (Audio, AudioMixer) {
(Audio::new(Arc::new(MemoryBanks::new())), AudioMixer::new())
}
fn apply(audio: &Audio, mixer: &mut AudioMixer) {
mixer.apply_delivery(audio.transport().try_take().unwrap());
}
fn play(audio: &mut Audio, sample: Arc<Sample>, pitch: f64, looping: bool) {
assert_eq!(
audio.play_sample(
sample,
0,
255,
0,
pitch,
0,
if looping { LoopMode::On } else { LoopMode::Off }
),
AudioOpStatus::Ok
);
}
#[test]
fn pcm_controls_preserve_position_and_stop_is_idempotent() {
let (mut audio, mut mixer) = pair();
play(&mut audio, Arc::new(Sample::new(48_000, vec![16384; 32])), 1.0, true);
audio.finish_frame();
apply(&audio, &mut mixer);
let mut pcm = [0.0; 4];
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.5, 0.0, 0.5, 0.0]);
assert_eq!(mixer.voices[0].pos, 2.0);
audio.set_pan(0, 255);
audio.set_volume(0, 51);
audio.set_pitch(0, 2.0);
audio.finish_frame();
apply(&audio, &mut mixer);
assert_eq!(mixer.voices[0].pos, 2.0);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0, 0.1, 0.0, 0.1]);
assert_eq!(mixer.voices[0].pos, 6.0);
audio.set_volume(0, 0);
audio.finish_frame();
apply(&audio, &mut mixer);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 4]);
assert_eq!(mixer.voices[0].pos, 10.0);
audio.stop(0);
audio.stop(0);
audio.finish_frame();
apply(&audio, &mut mixer);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 4]);
assert!(mixer.voices[0].sample.is_none());
}
#[test]
fn fifo_reuse_and_inactive_setters_do_not_create_play_defaults() {
let (mut audio, mut mixer) = pair();
audio.set_volume(0, 0);
audio.set_pitch(0, 4.0);
play(&mut audio, Arc::new(Sample::new(48_000, vec![16384; 16])), 1.0, true);
audio.stop(0);
play(&mut audio, Arc::new(Sample::new(48_000, vec![8192; 16])), 1.0, true);
audio.finish_frame();
apply(&audio, &mut mixer);
let mut pcm = [0.0; 2];
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.25, 0.0]);
assert_eq!(mixer.voices[0].pos, 1.0);
}
#[test]
fn pause_and_session_reset_work_with_pending_batches() {
let (mut audio, mut mixer) = pair();
audio.set_session(Some(1));
play(&mut audio, Arc::new(Sample::new(48_000, vec![16384; 16])), 1.0, true);
audio.finish_frame();
audio.set_paused(true);
apply(&audio, &mut mixer);
let mut pcm = [1.0; 4];
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 4]);
assert_eq!(mixer.voices[0].pos, 0.0);
audio.begin_lifecycle();
audio.set_pan(0, 255);
audio.finish_lifecycle(true);
audio.set_paused(false);
apply(&audio, &mut mixer);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0, 0.5, 0.0, 0.5]);
assert_eq!(mixer.voices[0].pos, 2.0);
play(&mut audio, Arc::new(Sample::new(48_000, vec![16384; 16])), 1.0, true);
audio.finish_frame();
let stale = audio.transport().try_take().unwrap();
audio.reset();
apply(&audio, &mut mixer);
mixer.apply_delivery(stale);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 4]);
}
#[test]
fn play_before_home_cannot_restart_after_callback_stop_and_resume() {
let (mut audio, mut mixer) = pair();
play(&mut audio, Arc::new(Sample::new(48_000, vec![16384; 16])), 1.0, true);
// Home occurs before FRAME_SYNC; the old play must precede the hook stop.
audio.set_paused(true);
audio.begin_lifecycle();
audio.stop(0);
audio.finish_lifecycle(true);
apply(&audio, &mut mixer);
assert!(mixer.voices[0].sample.is_none());
audio.begin_lifecycle();
audio.finish_lifecycle(true);
audio.set_paused(false);
apply(&audio, &mut mixer);
let mut pcm = [1.0; 16];
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 16]);
audio.finish_frame();
apply(&audio, &mut mixer);
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm, [0.0; 16]);
assert!(mixer.voices[0].sample.is_none());
}
#[test]
fn short_loops_and_extreme_pitch_remain_finite_and_release_at_completion() {
for pitch in [0.125, 1.0, 4.0, 1000.0, f64::MAX] {
let (mut audio, mut mixer) = pair();
play(
&mut audio,
Arc::new(Sample::new(96_000, vec![16384; 3]).with_loop(1, 3)),
pitch,
true,
);
audio.finish_frame();
apply(&audio, &mut mixer);
let mut pcm = [0.0; 128];
mixer.fill_buffer(&mut pcm);
assert!(pcm.iter().all(|v| v.is_finite()));
assert!(pcm.chunks_exact(2).all(|frame| frame == [0.5, 0.0]));
assert!(mixer.voices[0].active);
assert!(mixer.voices[0].pos >= 1.0 && mixer.voices[0].pos < 3.0);
}
let (mut audio, mut mixer) = pair();
let sample = Arc::new(Sample::new(48_000, vec![16384]));
let weak = Arc::downgrade(&sample);
play(&mut audio, sample, 1.0, false);
audio.finish_frame();
apply(&audio, &mut mixer);
mixer.fill_buffer(&mut [0.0; 4]);
assert!(weak.upgrade().is_none());
}
#[test]
fn unloading_bank_keeps_playback_until_stop() {
use prometeu_drivers::SoundBankPoolInstaller;
use prometeu_hal::sound_bank::SoundBank;
let banks = Arc::new(MemoryBanks::new());
let sample = Arc::new(Sample::new(48_000, vec![16384; 4]));
let weak = Arc::downgrade(&sample);
banks.install_sound_bank(0, Arc::new(SoundBank::new(vec![sample])));
let mut audio = Audio::new(banks.clone());
let mut mixer = AudioMixer::new();
assert_eq!(audio.play(0, 0, 0, 255, 0, 1.0, 0, LoopMode::On), AudioOpStatus::Ok);
audio.finish_frame();
banks.clear_sound_banks();
apply(&audio, &mut mixer);
let mut pcm = [0.0; 8];
mixer.fill_buffer(&mut pcm);
assert_eq!(pcm[0], 0.5);
assert!(weak.upgrade().is_some());
assert_eq!(audio.play(0, 0, 0, 255, 0, 1.0, 0, LoopMode::On), AudioOpStatus::BankInvalid);
audio.stop(0);
audio.finish_frame();
apply(&audio, &mut mixer);
assert!(weak.upgrade().is_none());
}
#[test]
fn failed_or_stalled_output_releases_backpressure() {
for failed in [true, false] {
let (mut audio, _) = pair();
audio.stop(0);
audio.finish_frame();
let mut host = HostAudio::new();
host.transport = audio.transport();
host._stream = Some(Box::new(FakeSmokeStream));
host.failed.store(failed, Ordering::Relaxed);
if !failed {
host.last_progress = Instant::now() - Duration::from_secs(3);
}
host.service();
assert!(audio.ready());
assert!(host._stream.is_none());
}
}
struct FakeSmokeStream;
impl AudioStreamHandle for FakeSmokeStream {}
#[test]
#[ignore = "requires a real desktop audio device; emits a brief quiet tone"]
fn desktop_audio_device_smoke() {
let (mut audio, _) = pair();
let mut host = HostAudio::new();
host.init(audio.transport()).expect("physical output device");
let data = (0..4800)
.map(|i| ((i as f64 * std::f64::consts::TAU * 440.0 / 48_000.0).sin() * 1500.0) as i16)
.collect();
play(&mut audio, Arc::new(Sample::new(48_000, data)), 1.0, true);
audio.finish_frame();
let deadline = Instant::now() + Duration::from_secs(2);
while !audio.ready() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
assert!(audio.ready());
assert!(host.heartbeat.load(Ordering::Relaxed) > 0);
audio.set_volume(0, 127);
audio.set_pan(0, 255);
audio.set_pitch(0, 1.5);
audio.finish_frame();
std::thread::sleep(Duration::from_millis(100));
assert!(audio.ready());
audio.stop(0);
audio.finish_frame();
std::thread::sleep(Duration::from_millis(100));
assert!(audio.ready());
assert!(!host.failed.load(Ordering::Relaxed));
}
}
@@ -8,10 +8,10 @@ use crate::utilities::{draw_owned_rgba8888_frame_to_rgba8, draw_rgba8888_to_rgba
use pixels::wgpu::PresentMode;
use pixels::{Pixels, PixelsBuilder, SurfaceTexture};
use prometeu_drivers::hardware::Hardware;
use prometeu_drivers::{AudioCommand, LocalFramebufferRenderBackend, MemoryBanks};
use prometeu_drivers::{LocalFramebufferRenderBackend, MemoryBanks};
use prometeu_firmware::{BootTarget, Firmware, FirmwareState};
use prometeu_hal::telemetry::CertificationConfig;
use prometeu_hal::{OwnedRgba8888Frame, RenderOwnership, RuntimePlatform};
use prometeu_hal::{AudioBridge, OwnedRgba8888Frame, RenderOwnership, RuntimePlatform};
use prometeu_system::{LatestRenderFrameStore, RenderWorkerConfig, discover_games_root};
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -147,8 +147,6 @@ pub struct HostRunner {
/// The physical audio driver.
audio: HostAudio,
/// Last known pause state to sync with audio.
last_paused_state: bool,
/// Worker-published frame store used as the single observable publication source.
render_frame_store: LatestRenderFrameStore,
/// Tracks whether a new logical frame or host-only surface invalidation requires presentation.
@@ -238,7 +236,6 @@ impl HostRunner {
stats: HostStats::new(),
debugger: HostDebugger::new(),
audio: HostAudio::new(),
last_paused_state: false,
render_frame_store,
presentation: PresentationState::default(),
}
@@ -312,7 +309,7 @@ impl ApplicationHandler for HostRunner {
self.pixels = Some(pixels);
if let Err(err) = self.audio.init() {
if let Err(err) = self.audio.init(self.hardware.audio.transport()) {
eprintln!("[HostAudio] Disabled: {}", err);
}
@@ -442,22 +439,11 @@ impl ApplicationHandler for HostRunner {
self.stats.record_host_cpu_time(self.firmware.os.vm().last_frame_cpu_time_us());
}
// Sync pause state with audio.
// We do this AFTER firmware.tick to avoid MasterPause/Resume commands
// being cleared by the OS if a new logical frame starts in this tick.
let lifecycle_audio_paused = self.firmware.game_lifecycle_audio_paused();
let is_paused = lifecycle_audio_paused
let is_paused = self.firmware.game_lifecycle_audio_paused()
|| self.firmware.os.vm().paused()
|| self.debugger.waiting_for_start;
if is_paused != self.last_paused_state {
self.last_paused_state = is_paused;
let cmd =
if is_paused { AudioCommand::MasterPause } else { AudioCommand::MasterResume };
self.hardware.audio.commands.push(cmd);
}
// Sync virtual audio commands to the physical mixer.
self.audio.send_commands(&mut self.hardware.audio.commands);
self.hardware.audio.set_paused(is_paused);
self.audio.service();
self.accumulator -= self.frame_target_dt;
self.stats.record_frame();