diff --git a/core/common/src/avm_string/common.rs b/core/common/src/avm_string/common.rs index 84f08237c1d9..a12fb7943aa5 100644 --- a/core/common/src/avm_string/common.rs +++ b/core/common/src/avm_string/common.rs @@ -273,6 +273,7 @@ ruffle_macros::define_common_strings! { "rollOut", "rollOver", "rr", + "sampleData", "save", "Selection", "separatorBefore", diff --git a/core/src/avm2.rs b/core/src/avm2.rs index 979d331eafb3..edbde21c6f26 100644 --- a/core/src/avm2.rs +++ b/core/src/avm2.rs @@ -95,7 +95,7 @@ pub use crate::avm2::multiname::Multiname; pub use crate::avm2::namespace::{CommonNamespaces, Namespace}; pub use crate::avm2::object::{ ArrayObject, BitmapDataObject, ClassObject, EventObject, LoaderInfoObject, Object, - SharedObjectObject, SoundChannelObject, StageObject, TObject, + SharedObjectObject, SoundChannelObject, SoundObject, StageObject, TObject, }; pub use crate::avm2::qname::QName; pub use crate::avm2::value::Value; diff --git a/core/src/avm2/globals/flash/events/SampleDataEvent.as b/core/src/avm2/globals/flash/events/SampleDataEvent.as index ac95d57fa1f0..7e4047fd0c7a 100644 --- a/core/src/avm2/globals/flash/events/SampleDataEvent.as +++ b/core/src/avm2/globals/flash/events/SampleDataEvent.as @@ -5,6 +5,8 @@ package flash.events { public static const SAMPLE_DATA:String = "sampleData"; public var _position:Number; + + [Ruffle(NativeAccessible)] public var _data:ByteArray; public function SampleDataEvent( diff --git a/core/src/avm2/globals/flash/media/sound.rs b/core/src/avm2/globals/flash/media/sound.rs index c6b319a3ffec..3b0a48ee3d4b 100644 --- a/core/src/avm2/globals/flash/media/sound.rs +++ b/core/src/avm2/globals/flash/media/sound.rs @@ -174,6 +174,33 @@ pub fn play<'gc>( None }; + // If no load has been initiated yet, this is a generated (synthesized) sound. + // Register it with the audio backend so it can receive SampleDataEvent callbacks. + if sound_object.is_empty() { + let sound_channel = SoundChannelObject::empty(activation); + let handle = activation + .context + .audio_manager + .start_generated_sound(activation.context.audio, sound_object) + .expect("not too many sounds"); + sound_channel.set_sound_instance(activation.context, handle); + activation + .context + .audio_manager + .attach_avm2_sound_channel(handle, sound_channel); + // Transition state from Empty → Generated + sound_object.play( + QueuedPlay { + position, + sound_info, + sound_transform, + sound_channel, + }, + activation, + ); + return Ok(sound_channel.into()); + } + let sound_channel = SoundChannelObject::empty(activation); let queued_play = QueuedPlay { @@ -261,6 +288,7 @@ pub fn load<'gc>( Request::get(url.to_string()), ); activation.context.navigator.spawn_future(future); + this.load_called(activation.context); this.set_loading_state(SoundLoadingState::Loading); Ok(Value::Undefined) diff --git a/core/src/avm2/object/event_object.rs b/core/src/avm2/object/event_object.rs index 71e1ced39c0f..83a88c10b40d 100644 --- a/core/src/avm2/object/event_object.rs +++ b/core/src/avm2/object/event_object.rs @@ -2,9 +2,10 @@ use crate::avm2::Error; use crate::avm2::activation::Activation; +use crate::avm2::bytearray::ByteArrayStorage; use crate::avm2::events::Event; use crate::avm2::object::script_object::ScriptObjectData; -use crate::avm2::object::{ClassObject, Object, ScriptObject, TObject}; +use crate::avm2::object::{ByteArrayObject, ClassObject, Object, ScriptObject, TObject}; use crate::avm2::value::Value; use crate::context::UpdateContext; use crate::display_object::TDisplayObject; @@ -227,6 +228,30 @@ impl<'gc> EventObject<'gc> { ) } + pub fn sample_data_event( + activation: &mut Activation<'_, 'gc>, + position: u32, + ) -> EventObject<'gc> { + let storage = ByteArrayStorage::new(activation.context); + let data = ByteArrayObject::from_storage(activation.context, storage); + + let event_name = istr!("sampleData"); + let sample_data_event_cls = activation.avm2().classes().sampledataevent; + Self::from_class_and_args( + activation, + sample_data_event_cls, + &[ + event_name.into(), + //bubbles + false.into(), + //cancelable + false.into(), + position.into(), + data.into(), + ], + ) + } + pub fn net_status_event<'a>( activation: &mut Activation<'_, 'gc>, info: impl IntoIterator, diff --git a/core/src/avm2/object/sound_object.rs b/core/src/avm2/object/sound_object.rs index c9f50d31106b..4e2292f4bc23 100644 --- a/core/src/avm2/object/sound_object.rs +++ b/core/src/avm2/object/sound_object.rs @@ -37,9 +37,7 @@ pub fn sound_allocator<'gc>( SoundObjectData { base, loading_state: Cell::new(SoundLoadingState::New), - sound_data: RefLock::new(SoundData::NotLoaded { - queued_plays: Vec::new(), - }), + sound_data: RefLock::new(SoundData::Empty), id3: Lock::new(None), }, )) @@ -95,13 +93,17 @@ pub struct SoundObjectData<'gc> { #[derive(Collect)] #[collect(no_drop)] pub enum SoundData<'gc> { - NotLoaded { - queued_plays: Vec>, - }, + /// Initial state: no load or play called yet. + Empty, + /// `load()` was called; waiting for data. + Loading { queued_plays: Vec> }, Loaded { #[collect(require_static)] sound: SoundHandle, }, + /// `play()` was called on an empty sound (no load initiated); + /// audio data comes from `SampleDataEvent` dispatches. + Generated, } #[derive(Clone, Collect)] @@ -126,8 +128,8 @@ impl<'gc> SoundObject<'gc> { pub fn sound_handle(self) -> Option { let sound_data = self.0.sound_data.borrow(); match &*sound_data { - SoundData::NotLoaded { .. } => None, SoundData::Loaded { sound } => Some(*sound), + _ => None, } } @@ -148,7 +150,13 @@ impl<'gc> SoundObject<'gc> { ) .borrow_mut(); match &mut *sound_data { - SoundData::NotLoaded { queued_plays } => { + SoundData::Empty => { + // play() was called before load() — this becomes a generated sound. + *sound_data = SoundData::Generated; + // We don't know the length yet, so return the `SoundChannel` + true + } + SoundData::Loading { queued_plays } => { // Avoid to enqueue more unloaded sounds than the maximum allowed to be played if queued_plays.len() >= AudioManager::MAX_SOUNDS { tracing::warn!("Sound.play: too many unloaded sounds queued"); @@ -161,6 +169,38 @@ impl<'gc> SoundObject<'gc> { true } SoundData::Loaded { sound } => play_queued(queued, *sound, activation.context), + SoundData::Generated => { + // Already generated, return a channel + true + } + } + } + + /// Returns `true` if this sound is in the `Empty` state + /// (no `load()` call has been made yet). + pub fn is_empty(self) -> bool { + matches!(&*self.0.sound_data.borrow(), SoundData::Empty) + } + + /// Transitions from `Empty` → `Loading`. Called when `Sound.load()` is invoked. + pub fn load_called(self, context: &mut UpdateContext<'gc>) { + let mut sound_data = + unlock!(Gc::write(context.gc(), self.0), SoundObjectData, sound_data).borrow_mut(); + match &*sound_data { + SoundData::Empty => { + *sound_data = SoundData::Loading { + queued_plays: Vec::new(), + }; + } + SoundData::Loading { .. } => { + panic!("Tried to load sound that is already Loading"); + } + SoundData::Loaded { .. } => { + panic!("Tried to load sound that is already Loaded"); + } + SoundData::Generated => { + panic!("Tried to load sound that is already Generated"); + } } } @@ -169,7 +209,10 @@ impl<'gc> SoundObject<'gc> { unlock!(Gc::write(context.gc(), self.0), SoundObjectData, sound_data).borrow_mut(); match &mut *sound_data { - SoundData::NotLoaded { queued_plays } => { + SoundData::Empty => { + *sound_data = SoundData::Loaded { sound }; + } + SoundData::Loading { queued_plays } => { for queued in std::mem::take(queued_plays) { play_queued(queued, sound, context); } @@ -178,6 +221,9 @@ impl<'gc> SoundObject<'gc> { SoundData::Loaded { sound: old_sound } => { panic!("Tried to replace sound {old_sound:?} with {sound:?}") } + SoundData::Generated => { + panic!("Tried to replace generated sound with {sound:?}") + } } self.set_loading_state(SoundLoadingState::Loaded); } diff --git a/core/src/backend/audio.rs b/core/src/backend/audio.rs index 6bed987b7b3a..970e839fcca9 100644 --- a/core/src/backend/audio.rs +++ b/core/src/backend/audio.rs @@ -1,8 +1,16 @@ use std::any::Any; +use std::{ + collections::VecDeque, + sync::{Arc, RwLock}, +}; use crate::{ avm1::{NativeObject, Object as Avm1Object}, - avm2::{Avm2, EventObject as Avm2EventObject, SoundChannelObject}, + avm2::{ + Activation, Avm2, EventObject as Avm2EventObject, SoundChannelObject, + SoundObject as Avm2SoundObject, TObject, + globals::slots::flash_events_sample_data_event as sample_data_event_slots, + }, context::UpdateContext, display_object::{self, DisplayObject, MovieClip, TDisplayObject}, string::AvmString, @@ -133,6 +141,12 @@ pub trait AudioBackend: Any { stream_info: &SoundStreamInfo, ) -> Result; + /// Starts a generated (synthesized) sound stream. + /// + /// Audio data is provided externally via the shared `deque` on each frame + /// by dispatching `SampleDataEvent` to the associated ActionScript object. + fn start_generated_sound(&mut self, deque: Arc>>) -> SoundInstanceHandle; + /// Stops a playing sound instance. /// No-op if the sound is not playing. fn stop_sound(&mut self, sound: SoundInstanceHandle); @@ -283,6 +297,10 @@ impl AudioBackend for NullAudioBackend { Ok(SoundInstanceHandle::null()) } + fn start_generated_sound(&mut self, _deque: Arc>>) -> SoundInstanceHandle { + SoundInstanceHandle::null() + } + fn stop_sound(&mut self, _sound: SoundInstanceHandle) {} fn stop_all_sounds(&mut self) {} @@ -369,6 +387,25 @@ impl<'gc> AudioManager<'gc> { /// The player will adjust animation speed to stay within this many seconds of the audio track. pub const STREAM_DEFAULT_SYNC_THRESHOLD: f64 = 0.2; + /// Sample rate for generated sounds (Hz). + const GENERATED_SOUND_SAMPLE_RATE: f32 = 44100.0; + + /// Number of channels for generated sounds. + const GENERATED_SOUND_CHANNELS: f32 = 2.0; + + /// How many frames of audio to buffer ahead for generated sounds. + const GENERATED_SOUND_LOOKAHEAD_FRAMES: f32 = 3.0; + + /// Upper bound on the number of channel-samples buffered ahead, regardless of frame rate. + /// Prevents runaway pre-buffering (~100 ms) when the frame rate is very low or throttled. + /// 44100 Hz × 2 channels × 0.1 s = 8820 channel-samples. + const GENERATED_SOUND_MAX_LOOKAHEAD_SAMPLES: usize = 8820; + + /// Minimum number of *bytes* that must be present in the `SampleDataEvent` data ByteArray + /// per dispatch before buffering is considered complete for one iteration. + /// Flash typically provides 2048 stereo pairs per callback: 2048 pairs × 2 floats × 4 bytes = 16384 bytes. + const GENERATED_SOUND_MIN_EVENT_BYTES: usize = 2048 * 8; + pub fn new() -> Self { Self { sounds: Vec::with_capacity(Self::MAX_SOUNDS), @@ -400,11 +437,21 @@ impl<'gc> AudioManager<'gc> { } true } else { + // Generated sounds never stop on their own. + if matches!(sound.source_data, SoundInstanceSourceData::Generated(_)) { + return true; + } + // Sound ended. - let duration = sound - .sound - .and_then(|sound| context.audio.get_sound_duration(sound)) - .unwrap_or_default(); + let duration = + if let SoundInstanceSourceData::Event(sound_handle) = &sound.source_data { + context + .audio + .get_sound_duration(*sound_handle) + .unwrap_or_default() + } else { + Default::default() + }; if let Some(object) = sound.avm1_object { if let NativeObject::Sound(sound) = object.native() { sound.set_position(duration.as_millis().round() as u32); @@ -439,10 +486,81 @@ impl<'gc> AudioManager<'gc> { Avm2::dispatch_event(context, event, target.into()); } + // Pump audio data for all generated sounds. + let domain = context.avm2.stage_domain(); + let mut activation = Activation::from_domain(context, domain); + Self::update_generated_sounds(&mut activation); + // Update sound transforms, if dirty. context.audio_manager.update_sound_transforms(context.audio); } + fn update_generated_sounds(activation: &mut Activation<'_, 'gc>) { + let mut states = Vec::new(); + for sound in &activation.context.audio_manager.sounds { + if let SoundInstanceSourceData::Generated(state) = &sound.source_data { + states.push(state.clone()); + } + } + + for sound_state in &mut states { + let fps = activation.caller_movie_or_root().frame_rate().to_f32(); + + let mut pos = sound_state.position.write().unwrap(); + // Pre-buffer a few frames of audio to reduce stuttering, capped to + // avoid excessive pre-buffering at very low or throttled frame rates. + let lookahead_samples = ((Self::GENERATED_SOUND_SAMPLE_RATE + * Self::GENERATED_SOUND_CHANNELS + * Self::GENERATED_SOUND_LOOKAHEAD_FRAMES + / fps) + .ceil() as usize) + .min(Self::GENERATED_SOUND_MAX_LOOKAHEAD_SAMPLES); + loop { + // Check the queue length with a short-lived lock. The lock + // must NOT be held across the AS3 dispatch below: the audio + // callback thread also needs it, and blocking it causes glitches. + let current_len = sound_state.next_samples.read().unwrap().len(); + if current_len >= lookahead_samples { + break; + } + + let sample_data_evt = Avm2EventObject::sample_data_event(activation, *pos); + Avm2::dispatch_event( + activation.context, + sample_data_evt, + sound_state.sound_object.into(), + ); + + let data_value = sample_data_evt + .get_slot(sample_data_event_slots::_DATA) + .as_object() + .unwrap(); + let ba = data_value.as_bytearray().unwrap(); + + let ba_len = ba.len(); + ba.set_position(0); + + // Collect samples into a local buffer first, then push to the + // shared queue with a brief write lock. + let mut local_samples = Vec::with_capacity(ba_len / 4); + for _ in 0..ba_len / 4 { + local_samples.push(ba.read_float().expect("float can be read")); + } + + { + let mut ns = sound_state.next_samples.write().unwrap(); + ns.extend(local_samples); + } + + *pos += ba_len as u32 / 8; + + if ba_len < Self::GENERATED_SOUND_MIN_EVENT_BYTES { + break; + } + } + } + } + /// Starts a sound and optionally associates it with a Display Object. /// Sounds associated with DOs are an AVM1/Timeline concept and should not be called from AVM2 scripts. pub fn start_sound( @@ -460,7 +578,7 @@ impl<'gc> AudioManager<'gc> { .map_err(|e| tracing::warn!("Cannot start sound: {e}")) .ok()?; let mut instance = SoundInstance { - sound: Some(sound), + source_data: SoundInstanceSourceData::Event(sound), instance: handle, display_object, transform: display_object::SoundTransform::default(), @@ -510,12 +628,13 @@ impl<'gc> AudioManager<'gc> { pub fn stop_sounds_with_handle(&mut self, audio: &mut dyn AudioBackend, sound: SoundHandle) { self.sounds.retain(move |other| { - if other.sound == Some(sound) { + if let SoundInstanceSourceData::Event(other_sound) = other.source_data + && other_sound == sound + { audio.stop_sound(other.instance); - false - } else { - true + return false; } + true }); } @@ -576,7 +695,9 @@ impl<'gc> AudioManager<'gc> { } pub fn is_sound_playing_with_handle(&self, sound: SoundHandle) -> bool { - self.sounds.iter().any(|other| other.sound == Some(sound)) + self.sounds.iter().any( + |other| matches!(other.source_data, SoundInstanceSourceData::Event(s) if s == sound), + ) } pub fn start_stream( @@ -590,7 +711,7 @@ impl<'gc> AudioManager<'gc> { if self.sounds.len() < Self::MAX_SOUNDS { let handle = audio.start_stream(data, stream_info).ok()?; let instance = SoundInstance { - sound: None, + source_data: SoundInstanceSourceData::Stream, instance: handle, display_object: Some(movie_clip.into()), transform: display_object::SoundTransform::default(), @@ -616,7 +737,7 @@ impl<'gc> AudioManager<'gc> { if self.sounds.len() < Self::MAX_SOUNDS { let handle = audio.start_substream(stream_data, stream_info)?; let instance = SoundInstance { - sound: None, + source_data: SoundInstanceSourceData::Stream, instance: handle, display_object: Some(movie_clip.into()), transform: display_object::SoundTransform::default(), @@ -632,6 +753,35 @@ impl<'gc> AudioManager<'gc> { } } + /// Starts a generated (synthesized) sound stream. + pub fn start_generated_sound( + &mut self, + audio: &mut dyn AudioBackend, + sound_object: Avm2SoundObject<'gc>, + ) -> Result { + if self.sounds.len() < Self::MAX_SOUNDS { + let sample_queue = Arc::new(RwLock::new(VecDeque::new())); + let handle = audio.start_generated_sound(sample_queue.clone()); + let instance = SoundInstance { + source_data: SoundInstanceSourceData::Generated(GeneratedSoundState::new( + sound_object, + sample_queue, + )), + instance: handle, + display_object: None, + transform: display_object::SoundTransform::default(), + avm1_object: None, + avm2_object: None, + stream_start_frame: None, + }; + audio.set_sound_transform(handle, self.transform_for_sound(&instance)); + self.sounds.push(instance); + Ok(handle) + } else { + Err(DecodeError::TooManySounds) + } + } + /// Returns the difference in seconds between the primary audio stream's time and the player's time. pub fn audio_skew_time(&mut self, audio: &mut dyn AudioBackend, offset_ms: f64) -> f64 { // Consider the first playing "stream" sound to be the primary audio track. @@ -813,6 +963,45 @@ impl Default for AudioManager<'_> { } } +/// State for a sound synthesized by ActionScript via `SampleDataEvent`. +#[derive(Clone, Collect)] +#[collect(no_drop)] +pub struct GeneratedSoundState<'gc> { + /// The AS3 `Sound` object that dispatches `SampleDataEvent`s. + sound_object: Avm2SoundObject<'gc>, + /// Pre-filled audio samples shared with the audio backend mixer thread. + #[collect(require_static)] + next_samples: Arc>>, + /// Current playback position in samples (stereo pairs written so far). + #[collect(require_static)] + position: Arc>, +} + +impl<'gc> GeneratedSoundState<'gc> { + pub fn new( + sound_object: Avm2SoundObject<'gc>, + sample_queue: Arc>>, + ) -> Self { + Self { + sound_object, + next_samples: sample_queue, + position: Arc::new(RwLock::new(0)), + } + } +} + +/// Describes the source of a `SoundInstance`. +#[derive(Clone, Collect)] +#[collect(no_drop)] +pub enum SoundInstanceSourceData<'gc> { + /// A regular event sound loaded from a SWF or URL. + Event(#[collect(require_static)] SoundHandle), + /// A stream sound embedded in a SWF MovieClip or video container. + Stream, + /// A synthesized sound driven by `SampleDataEvent`. + Generated(GeneratedSoundState<'gc>), +} + #[derive(Clone, Collect)] #[collect(no_drop)] pub struct SoundInstance<'gc> { @@ -820,10 +1009,8 @@ pub struct SoundInstance<'gc> { #[collect(require_static)] instance: SoundInstanceHandle, - /// The handle to the sound definition in the audio backend. - /// This will be `None` for stream sounds. - #[collect(require_static)] - sound: Option, + /// Source of audio data for this sound. + source_data: SoundInstanceSourceData<'gc>, /// The display object that this sound is playing in, if any. /// Used for volume mixing and `Sound.stop()`. diff --git a/core/src/backend/audio/mixer.rs b/core/src/backend/audio/mixer.rs index b98eb3f767d4..df414f7f06b2 100644 --- a/core/src/backend/audio/mixer.rs +++ b/core/src/backend/audio/mixer.rs @@ -7,6 +7,7 @@ use crate::tag_utils::SwfSlice; use ruffle_common::buffer::Substream; use ruffle_common::duration::FloatDuration; use slotmap::SlotMap; +use std::collections::VecDeque; use std::io::Cursor; use std::sync::{Arc, Mutex, RwLock}; use swf::AudioCompression; @@ -575,6 +576,22 @@ impl AudioMixer { Ok(handle) } + /// Starts a generated (synthesized) sound stream. + /// + /// Audio samples are provided externally via the shared `deque`, + /// which is filled each frame by dispatching `SampleDataEvent`. + pub fn start_generated_sound( + &mut self, + deque: Arc>>, + ) -> SoundInstanceHandle { + let stream = GeneratedSoundStream::new(deque); + let mut sound_instances = self + .sound_instances + .lock() + .expect("Cannot be called reentrant"); + sound_instances.insert(SoundInstance::new_stream(Box::new(stream))) + } + /// Starts a sound. /// /// The sound must have been registered using `AudioMixer::register_sound`. @@ -879,6 +896,72 @@ impl Stream for EventSoundStream { } } +/// A stream for sounds synthesized in ActionScript via `SampleDataEvent`. +struct GeneratedSoundStream { + /// Position counter (in output sample frames). + position: u32, + /// Local buffer drained from `next_samples` to reduce lock contention. + playout_buffer: VecDeque, + /// Shared sample queue filled by the main thread each frame. + next_samples: Arc>>, +} + +impl GeneratedSoundStream { + /// Minimum local playout buffer size before pulling more samples from the shared queue. + /// 1024 channel-samples ≈ 11.6 ms at 44100 Hz, roughly one or two typical OS audio + /// callback sizes, keeping lock acquisitions to at most one or two per callback. + const REFILL_THRESHOLD: usize = 1024; + + fn new(stream: Arc>>) -> Self { + Self { + position: 0, + playout_buffer: VecDeque::new(), + next_samples: stream, + } + } +} + +impl dasp::signal::Signal for GeneratedSoundStream { + type Frame = [i16; 2]; + + #[inline] + fn next(&mut self) -> Self::Frame { + use dasp::Sample; + + // Refill local buffer in bulk to reduce RwLock contention. + if self.playout_buffer.len() < Self::REFILL_THRESHOLD { + let mut w = self.next_samples.write().unwrap(); + self.playout_buffer.append(&mut w); + } + + self.position += 1; + + if let Some(left) = self.playout_buffer.pop_front() + && let Some(right) = self.playout_buffer.pop_front() { + return [left.to_sample(), right.to_sample()]; + } + + Default::default() + } + + #[inline] + fn is_exhausted(&self) -> bool { + false + } +} + +impl Stream for GeneratedSoundStream { + #[inline] + fn source_position(&self) -> u32 { + self.position + } + + #[inline] + fn source_sample_rate(&self) -> u16 { + 44100 + } +} + /// A stream that converts a source stream to a different sample rate. struct ConverterStream(dasp::signal::interpolate::Converter) where @@ -1110,6 +1193,14 @@ macro_rules! impl_audio_mixer_backend { self.$mixer.start_substream(stream_data, stream_info) } + #[inline] + fn start_generated_sound( + &mut self, + deque: std::sync::Arc>>, + ) -> SoundInstanceHandle { + self.$mixer.start_generated_sound(deque) + } + #[inline] fn stop_sound(&mut self, sound: SoundInstanceHandle) { self.$mixer.stop_sound(sound)