diff --git a/.changeset/audioengine-platformaudio.md b/.changeset/audioengine-platformaudio.md new file mode 100644 index 000000000..47bc5261f --- /dev/null +++ b/.changeset/audioengine-platformaudio.md @@ -0,0 +1,10 @@ +--- +livekit: minor +webrtc-sys: minor +--- + +Use the Apple AudioEngine ADM for PlatformAudio on iOS and macOS. + +- The platform ADM on Apple platforms is now the AVAudioEngine based device with runtime switchable voice processing and device change handling. +- `prefer_hardware_processing` now defaults to `true` on macOS as well as iOS, so PlatformAudio uses Apple voice processing by default on both. Pass `prefer_hardware_processing: false` to keep WebRTC software processing. +- The ADM proxy forwards the platform voice processing interface (topology, path toggle, state) so WebRTC's audio processing resolution works through it when track audio options are applied. diff --git a/livekit/src/platform_audio/mod.rs b/livekit/src/platform_audio/mod.rs index 7af17dfc2..bf0771c59 100644 --- a/livekit/src/platform_audio/mod.rs +++ b/livekit/src/platform_audio/mod.rs @@ -95,9 +95,9 @@ //! //! # Platform-Specific Notes //! -//! - **iOS**: Creates a VPIO (Voice Processing IO) AudioUnit. Only one VPIO -//! can exist per process. Drop all `PlatformAudio` instances to release it. -//! - **macOS**: Uses CoreAudio. Full device enumeration and selection supported. +//! - **iOS**: Uses WebRTC's Apple AudioEngine ADM with platform voice processing. +//! Drop all `PlatformAudio` instances to release active audio I/O. +//! - **macOS**: Uses WebRTC's Apple AudioEngine ADM. Full device enumeration and selection supported. //! - **Windows**: Uses WASAPI. Full device enumeration and selection supported. //! - **Linux**: Uses PulseAudio or ALSA. Full device enumeration and selection supported. //! - **Android**: Uses Java AudioRecord/AudioTrack via WebRTC's `JavaAudioDeviceModule`. @@ -299,6 +299,9 @@ lazy_static! { /// When the last PlatformAudio is dropped, the Platform ADM is released. struct PlatformAdmHandle { runtime: Arc, + // Device level processing configuration last applied via + // configure_audio_processing, used by the active_*_type getters + processing_options: Mutex, } impl Drop for PlatformAdmHandle { @@ -399,9 +402,9 @@ impl Drop for PlatformAdmHandle { /// /// # Platform-Specific Notes /// -/// - **iOS**: Creates a VPIO AudioUnit (exclusive microphone access). +/// - **iOS**: Uses WebRTC's Apple AudioEngine ADM with platform voice processing. /// Drop all instances to allow other audio frameworks to use the mic. -/// - **macOS**: Uses CoreAudio for device management. +/// - **macOS**: Uses WebRTC's Apple AudioEngine ADM for device management. /// - **Windows**: Uses WASAPI for device management. /// - **Linux**: Uses PulseAudio or ALSA. #[derive(Clone)] @@ -478,15 +481,18 @@ impl PlatformAudio { playout_count ); - let handle = Arc::new(PlatformAdmHandle { runtime }); + let handle = Arc::new(PlatformAdmHandle { + runtime, + processing_options: Mutex::new(AudioProcessingOptions::default()), + }); *handle_ref = Arc::downgrade(&handle); let audio = Self { handle }; // Configure audio processing with platform-appropriate defaults: - // - iOS: prefer_hardware_processing=true (VPIO is excellent) + // - iOS/macOS: prefer_hardware_processing=true (Apple voice processing is preferred) // - Android: prefer_hardware_processing=false (hardware AEC unreliable across devices) - // - Desktop: prefer_hardware_processing=false (hardware not available anyway) + // - Windows/Linux: prefer_hardware_processing=false (hardware not available) if let Err(e) = audio.configure_audio_processing(AudioProcessingOptions::default()) { log::warn!("PlatformAudio: failed to configure audio processing: {}", e); } @@ -662,7 +668,7 @@ impl PlatformAudio { /// /// **Mobile (iOS/Android):** Device selection is a no-op. Both platforms handle /// microphone selection at the system level. This method will succeed but has no effect. - /// - iOS: VPIO AudioUnit handles input selection + /// - iOS: Apple AudioEngine handles input selection /// - Android: System selects best input source based on audio mode /// /// # Arguments @@ -987,7 +993,7 @@ impl PlatformAudio { /// /// # Platform Behavior /// - /// - **iOS**: Returns `true` (VPIO provides hardware AEC) + /// - **iOS**: Returns `true` when Apple voice processing can provide AEC /// - **Android**: Returns `true` on devices with hardware AEC support /// - **Desktop**: Returns `false` (hardware AEC not available) /// @@ -1007,7 +1013,7 @@ impl PlatformAudio { /// /// # Platform Behavior /// - /// - **iOS**: Returns `true` (VPIO provides hardware AGC) + /// - **iOS**: Returns `true` when Apple voice processing can provide AGC /// - **Android**: Returns `true` on devices with hardware AGC support /// - **Desktop**: Returns `false` (hardware AGC not available) pub fn is_hardware_agc_available(&self) -> bool { @@ -1018,7 +1024,7 @@ impl PlatformAudio { /// /// # Platform Behavior /// - /// - **iOS**: Returns `true` (VPIO provides hardware NS) + /// - **iOS**: Returns `true` when Apple voice processing can provide NS /// - **Android**: Returns `true` on devices with hardware NS support /// - **Desktop**: Returns `false` (hardware NS not available) pub fn is_hardware_ns_available(&self) -> bool { @@ -1044,7 +1050,10 @@ impl PlatformAudio { /// } /// ``` pub fn active_aec_type(&self) -> AudioProcessingType { - if self.is_hardware_aec_available() { + let options = *self.handle.processing_options.lock(); + if !options.echo_cancellation { + AudioProcessingType::None + } else if options.prefer_hardware_processing && self.is_hardware_aec_available() { AudioProcessingType::Hardware } else { AudioProcessingType::Software @@ -1053,7 +1062,10 @@ impl PlatformAudio { /// Gets the type of automatic gain control currently active. pub fn active_agc_type(&self) -> AudioProcessingType { - if self.is_hardware_agc_available() { + let options = *self.handle.processing_options.lock(); + if !options.auto_gain_control { + AudioProcessingType::None + } else if options.prefer_hardware_processing && self.is_hardware_agc_available() { AudioProcessingType::Hardware } else { AudioProcessingType::Software @@ -1062,7 +1074,10 @@ impl PlatformAudio { /// Gets the type of noise suppression currently active. pub fn active_ns_type(&self) -> AudioProcessingType { - if self.is_hardware_ns_available() { + let options = *self.handle.processing_options.lock(); + if !options.noise_suppression { + AudioProcessingType::None + } else if options.prefer_hardware_processing && self.is_hardware_ns_available() { AudioProcessingType::Hardware } else { AudioProcessingType::Software @@ -1076,10 +1091,11 @@ impl PlatformAudio { /// /// # Platform Behavior /// - /// - **iOS**: `prefer_hardware_processing` is ignored (always uses VPIO) + /// - **iOS/macOS**: `prefer_hardware_processing` uses Apple voice processing + /// when available (enabled by default) /// - **Android**: When `prefer_hardware_processing` is `false`, hardware /// effects are disabled and WebRTC's software APM is used instead - /// - **Desktop**: `prefer_hardware_processing` is ignored (hardware not available) + /// - **Windows/Linux**: `prefer_hardware_processing` is ignored (hardware not available) /// /// # Example /// @@ -1130,6 +1146,8 @@ impl PlatformAudio { } } + *self.handle.processing_options.lock() = options; + log::info!( "Audio processing configured: AEC={}, AGC={}, NS={}, prefer_hw={}", options.echo_cancellation, diff --git a/livekit/src/platform_audio/processing.rs b/livekit/src/platform_audio/processing.rs index 8b99ef90d..3af85f865 100644 --- a/livekit/src/platform_audio/processing.rs +++ b/livekit/src/platform_audio/processing.rs @@ -17,7 +17,7 @@ /// The type of audio processing being used. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AudioProcessingType { - /// Hardware audio processing (iOS VPIO, Android hardware effects). + /// Platform audio processing (Apple voice processing, Android hardware effects). Hardware, /// Software audio processing (WebRTC's built-in APM). Software, @@ -35,15 +35,15 @@ impl Default for AudioProcessingType { /// /// # Platform Behavior /// -/// - **iOS**: Hardware processing via VPIO is always used and provides excellent -/// AEC/AGC/NS. The `prefer_hardware_processing` default is `true` on iOS. +/// - **iOS/macOS**: Apple voice processing can provide platform AEC/AGC/NS. +/// The `prefer_hardware_processing` default is `true` on both. /// /// - **Android**: Hardware AEC quality varies significantly across manufacturers /// and device models. Many devices have broken or poorly-tuned hardware AEC. /// The default is `false` to use WebRTC's reliable software processing. /// See: /// -/// - **Desktop** (macOS, Windows, Linux): Hardware processing is not available. +/// - **Desktop** (Windows, Linux): Hardware processing is not available. /// WebRTC's software Audio Processing Module (APM) is always used. /// The `prefer_hardware_processing` setting is ignored. /// @@ -96,9 +96,9 @@ pub struct AudioProcessingOptions { /// /// # Platform Defaults /// - /// - **iOS**: `true` - VPIO hardware processing is excellent and always used. - /// Apple's Voice Processing IO unit provides reliable, low-latency AEC/AGC/NS - /// that is tightly integrated with the audio hardware. + /// - **iOS/macOS**: `true` - Apple voice processing provides reliable, + /// low-latency AEC/AGC/NS that is tightly integrated with audio I/O, + /// and is especially effective for built-in speaker/microphone echo. /// /// - **Android**: `false` - Hardware AEC is unreliable on many devices. /// Quality varies significantly across manufacturers (Samsung, Xiaomi, etc.) @@ -107,8 +107,8 @@ pub struct AudioProcessingOptions { /// Reference: Meta found hardware AEC "broken on many combinations of HW + OS" /// when supporting billions of users across thousands of device models. /// - /// - **Desktop**: `false` - Hardware processing is not available. - /// This setting is ignored; WebRTC software APM is always used. + /// - **Windows/Linux**: `false` - Hardware processing is not available. + /// WebRTC software APM is used. pub prefer_hardware_processing: bool, } @@ -118,12 +118,12 @@ impl Default for AudioProcessingOptions { echo_cancellation: true, noise_suppression: true, auto_gain_control: true, - // iOS: VPIO hardware processing is excellent and tightly integrated. + // iOS/macOS: Apple voice processing is preferred when available. // Android: Hardware AEC is unreliable across the fragmented device ecosystem. - // Desktop: Hardware processing not available, setting is ignored. - #[cfg(target_os = "ios")] + // Windows/Linux: Hardware processing not available. + #[cfg(any(target_os = "ios", target_os = "macos"))] prefer_hardware_processing: true, - #[cfg(not(target_os = "ios"))] + #[cfg(not(any(target_os = "ios", target_os = "macos")))] prefer_hardware_processing: false, } } diff --git a/webrtc-sys/include/livekit/adm_proxy.h b/webrtc-sys/include/livekit/adm_proxy.h index 31555103d..7a6149940 100644 --- a/webrtc-sys/include/livekit/adm_proxy.h +++ b/webrtc-sys/include/livekit/adm_proxy.h @@ -216,6 +216,14 @@ class AdmProxy : public webrtc::AudioDeviceModule { int32_t EnableBuiltInAGC(bool enable) override; int32_t EnableBuiltInNS(bool enable) override; + // Platform voice processing (Apple's coupled AEC+NS path) + webrtc::AudioDeviceModule::PlatformAudioProcessingTopology + GetPlatformAudioProcessingTopology() const override; + bool PlatformVoiceProcessingPathIsAvailable() const override; + int32_t EnablePlatformVoiceProcessingPath(bool enable) override; + webrtc::AudioDeviceModule::PlatformAudioProcessingState + GetPlatformAudioProcessingState() const override; + #if defined(WEBRTC_IOS) int GetPlayoutAudioParameters(webrtc::AudioParameters* params) const override; int GetRecordAudioParameters(webrtc::AudioParameters* params) const override; diff --git a/webrtc-sys/libwebrtc/patches/external_audio_source.patch b/webrtc-sys/libwebrtc/patches/external_audio_source.patch index 088be4044..970fad6d8 100644 --- a/webrtc-sys/libwebrtc/patches/external_audio_source.patch +++ b/webrtc-sys/libwebrtc/patches/external_audio_source.patch @@ -1,11 +1,11 @@ diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h -index fb1cc4e58e..85062ba60e 100644 +index 643b5f273b..7b5b568692 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h -@@ -267,6 +267,11 @@ class RTC_EXPORT AudioSourceInterface : public MediaSourceInterface { - // (for some of the settings this approach is broken, e.g. setting +@@ -270,6 +270,11 @@ class RTC_EXPORT AudioSourceInterface : public MediaSourceInterface { // audio network adaptation on the source is the wrong layer of abstraction). virtual const AudioOptions options() const; + virtual void SetOptions(const AudioOptions & /* options */) {} + + // Returns true if this source delivers audio externally (via AddSink), + // bypassing the ADM/AudioState audio distribution path. @@ -79,10 +79,10 @@ index 04a7d19dfa..9f513f3c75 100644 virtual ~AudioSource() {} }; diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc -index 762f9d584c..4ce07ddc9d 100644 +index 44ff51e439..65b32e2d74 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc -@@ -1017,6 +1017,14 @@ class WebRtcVoiceSendChannel::WebRtcAudioSendStream : public AudioSource::Sink { +@@ -942,6 +942,14 @@ class WebRtcVoiceSendChannel::WebRtcAudioSendStream : public AudioSource::Sink { RTC_DCHECK(source_ == source); return; } @@ -98,10 +98,10 @@ index 762f9d584c..4ce07ddc9d 100644 source_ = source; UpdateSendState(); diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc -index d5edbbf0ed..c14ddfe868 100644 +index bd3e4a34d8..6498a4eef8 100644 --- a/pc/rtp_sender.cc +++ b/pc/rtp_sender.cc -@@ -786,6 +786,13 @@ void AudioRtpSender::SetSend() { +@@ -792,6 +792,13 @@ void AudioRtpSender::SetSend() { RTC_DCHECK_RUN_ON(signaling_thread_); RTC_DCHECK(!stopped_); RTC_DCHECK(can_send_track()); @@ -116,7 +116,7 @@ index d5edbbf0ed..c14ddfe868 100644 RTC_LOG(LS_ERROR) << "SetAudioSend: No audio channel exists."; return; diff --git a/pc/rtp_sender.h b/pc/rtp_sender.h -index eaffd4ef0f..d0489df9e7 100644 +index 516ce7de05..c6216b6b7e 100644 --- a/pc/rtp_sender.h +++ b/pc/rtp_sender.h @@ -307,6 +307,12 @@ class LocalAudioSinkAdapter : public AudioTrackSinkInterface, diff --git a/webrtc-sys/src/adm_proxy.cpp b/webrtc-sys/src/adm_proxy.cpp index 16b8ff5cf..f27945357 100644 --- a/webrtc-sys/src/adm_proxy.cpp +++ b/webrtc-sys/src/adm_proxy.cpp @@ -57,9 +57,16 @@ AdmProxy::AdmProxy(const webrtc::Environment& env, webrtc::Thread* worker_thread // 3. Deferring creation ensures JNI is ready when we actually need the ADM #if defined(__ANDROID__) // platform_adm_ stays nullptr, will be created in EnsurePlatformAdmCreated() +#else +#if defined(WEBRTC_IOS) || defined(WEBRTC_MAC) + // Use the AVAudioEngine based ADM on Apple platforms. It supports runtime + // switchable voice processing and device change handling. + platform_adm_ = webrtc::CreateAudioDeviceModule( + env_, webrtc::AudioDeviceModule::kAppleAudioEngine); #else platform_adm_ = webrtc::CreateAudioDeviceModule( env_, webrtc::AudioDeviceModule::kPlatformDefaultAudio); +#endif if (!platform_adm_) { RTC_LOG(LS_ERROR) << "AdmProxy: CreateAudioDeviceModule returned nullptr"; @@ -836,6 +843,38 @@ int32_t AdmProxy::EnableBuiltInNS(bool enable) { }); } +webrtc::AudioDeviceModule::PlatformAudioProcessingTopology +AdmProxy::GetPlatformAudioProcessingTopology() const { + return WithPlatformAdm< + webrtc::AudioDeviceModule::PlatformAudioProcessingTopology>( + webrtc::AudioDeviceModule::PlatformAudioProcessingTopology::kIndependent, + [](webrtc::AudioDeviceModule& adm) { + return adm.GetPlatformAudioProcessingTopology(); + }); +} + +bool AdmProxy::PlatformVoiceProcessingPathIsAvailable() const { + return WithPlatformAdm(false, [](webrtc::AudioDeviceModule& adm) { + return adm.PlatformVoiceProcessingPathIsAvailable(); + }); +} + +int32_t AdmProxy::EnablePlatformVoiceProcessingPath(bool enable) { + return WithPlatformAdm(-1, [enable](webrtc::AudioDeviceModule& adm) { + return adm.EnablePlatformVoiceProcessingPath(enable); + }); +} + +webrtc::AudioDeviceModule::PlatformAudioProcessingState +AdmProxy::GetPlatformAudioProcessingState() const { + return WithPlatformAdm< + webrtc::AudioDeviceModule::PlatformAudioProcessingState>( + webrtc::AudioDeviceModule::PlatformAudioProcessingState(), + [](webrtc::AudioDeviceModule& adm) { + return adm.GetPlatformAudioProcessingState(); + }); +} + #if defined(WEBRTC_IOS) int AdmProxy::GetPlayoutAudioParameters(webrtc::AudioParameters* params) const { return WithPlatformAdm(-1, [params](webrtc::AudioDeviceModule& adm) {