From 622caa99b8628323613d3039e0d16a064d3fe7e1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:34:24 +0900 Subject: [PATCH 01/24] Tests --- Sources/LiveKit/Track/AudioManager.swift | 86 +++++++++++++---- Tests/LiveKitTests/AudioEngineTests.swift | 96 +++++++++++++++++++ .../Support/SinWaveSourceNode.swift | 58 +++++++++++ 3 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 Tests/LiveKitTests/AudioEngineTests.swift create mode 100644 Tests/LiveKitTests/Support/SinWaveSourceNode.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index de670b01f..b8a1c5499 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -68,6 +68,8 @@ public class AudioManager: Loggable { #endif public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void + public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void #if os(iOS) || os(visionOS) || os(tvOS) @@ -208,13 +210,56 @@ public class AudioManager: Loggable { public var onDeviceUpdate: DeviceUpdateFunc? { didSet { - RTC.audioDeviceModule.setDevicesUpdatedHandler { [weak self] in + RTC.audioDeviceModule.setDevicesDidUpdateCallback { [weak self] in guard let self else { return } self.onDeviceUpdate?(self) } } } + public var onEngineWillConnectInput: OnEngineWillConnectInput? { + didSet { + RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, inputMixerNode in + guard let self else { return } + self.onEngineWillConnectInput?(self, engine, inputMixerNode) + } + } + } + + public var isManualRenderingMode: Bool { + get { RTC.audioDeviceModule.isManualRenderingMode } + set { + let result = RTC.audioDeviceModule.setManualRenderingMode(newValue) + if !result { + log("Failed to set manual rendering mode", .error) + } + } + } + + // MARK: Testing + + public func startPlayout() { + RTC.audioDeviceModule.initPlayout() + RTC.audioDeviceModule.startPlayout() + } + + public func stopPlayout() { + RTC.audioDeviceModule.stopPlayout() + } + + public func initRecording() { + RTC.audioDeviceModule.initRecording() + } + + public func startRecording() { + RTC.audioDeviceModule.initRecording() + RTC.audioDeviceModule.startRecording() + } + + public func stopRecording() { + RTC.audioDeviceModule.stopRecording() + } + // MARK: - Internal enum `Type` { @@ -224,19 +269,34 @@ public class AudioManager: Loggable { let state = StateSync(State()) - // MARK: - Private + init() { + RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - private let _configureRunner = SerialRunnerActor() + #if os(iOS) || os(visionOS) || os(tvOS) + self.log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + let config = LKRTCAudioSessionConfiguration.webRTC() + + if isRecordingEnabled { + config.category = AVAudioSession.Category.playAndRecord.rawValue + config.mode = AVAudioSession.Mode.videoChat.rawValue + config.categoryOptions = [.defaultToSpeaker, .allowBluetooth] + } else { + config.category = AVAudioSession.Category.playback.rawValue + config.mode = AVAudioSession.Mode.spokenAudio.rawValue + config.categoryOptions = [.mixWithOthers] + } - #if os(iOS) || os(visionOS) || os(tvOS) - private func _asyncConfigure(newState: State, oldState: State) async throws { - try await _configureRunner.run { - self.log("\(oldState) -> \(newState)") - let configureFunc = newState.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc - configureFunc(newState, oldState) + session.lockForConfiguration() + try? session.setConfiguration(config) + session.unlockForConfiguration() + #endif } } - #endif + + // MARK: - Private func trackDidStart(_ type: Type) async throws { let (newState, oldState) = state.mutate { state in @@ -245,9 +305,6 @@ public class AudioManager: Loggable { if type == .remote { state.remoteTracksCount += 1 } return (state, oldState) } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif } func trackDidStop(_ type: Type) async throws { @@ -257,9 +314,6 @@ public class AudioManager: Loggable { if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } return (state, oldState) } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif } #if os(iOS) || os(visionOS) || os(tvOS) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift new file mode 100644 index 000000000..2005df332 --- /dev/null +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -0,0 +1,96 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation +@testable import LiveKit +import LiveKitWebRTC +import XCTest + +class AudioEngineTests: XCTestCase { + override class func setUp() { + LiveKitSDK.setLoggerStandardOutput() + RTCSetMinDebugLogLevel(.info) + } + + override func tearDown() async throws {} + + // Test if mic is authorized. Only works on device. + func testMicAuthorized() async { + let status = AVCaptureDevice.authorizationStatus(for: .audio) + if status != .authorized { + let result = await AVCaptureDevice.requestAccess(for: .audio) + XCTAssert(result) + } + + XCTAssert(status == .authorized) + } + + // Test start generating local audio buffer without joining to room. + func testPrejoinLocalAudioBuffer() async throws { + // Set up expectation... + let didReceiveAudioFrame = expectation(description: "Did receive audio frame") + didReceiveAudioFrame.assertForOverFulfill = false + + // Start watching for audio frame... + let audioFrameWatcher = AudioTrackWatcher(id: "notifier01") { _ in + didReceiveAudioFrame.fulfill() + } + + let localMicTrack = LocalAudioTrack.createTrack() + // Attach audio frame watcher... + localMicTrack.add(audioRenderer: audioFrameWatcher) + + Task.detached { + print("Starting audio track in 3 seconds...") + try? await Task.sleep(for: .seconds(3)) + AudioManager.shared.startRecording() + } + + // Wait for audio frame... + print("Waiting for first audio frame...") + await fulfillment(of: [didReceiveAudioFrame], timeout: 10) + + // Remove audio frame watcher... + localMicTrack.remove(audioRenderer: audioFrameWatcher) + } + + // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. + // In manual rendering, no device access will be initialized such as mic and speaker. + func testManualRenderingMode() async throws { + // Attach sin wave generator when engine requests input node... + // inputMixerNode will automatically convert to RTC's internal format (int16). + // AVAudioEngine.attach() retains the node. + AudioManager.shared.onEngineWillConnectInput = { _, engine, inputMixerNode in + let sin = SineWaveSourceNode() + engine.attach(sin) + engine.connect(sin, to: inputMixerNode, format: nil) + } + + // Set manual rendering mode... + AudioManager.shared.isManualRenderingMode = true + + // Check if manual rendering mode is set... + let isManualRenderingMode = AudioManager.shared.isManualRenderingMode + print("manualRenderingMode: \(isManualRenderingMode)") + XCTAssert(isManualRenderingMode) + + // Start rendering... + AudioManager.shared.startRecording() + + // Render for 10 seconds... + try? await Task.sleep(for: .seconds(10)) + } +} diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift new file mode 100644 index 000000000..b1fc632f7 --- /dev/null +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -0,0 +1,58 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +class SineWaveSourceNode: AVAudioSourceNode { + private let sampleRate: Double + private let frequency: Double + + init(frequency: Double = 440.0, sampleRate: Double = 48000.0) { + self.frequency = frequency + self.sampleRate = sampleRate + + let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1)! + + var currentPhase = 0.0 + let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate + + let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in + print("SineWaveSourceNode render block, frameCount: \(frameCount)") + + let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) + guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { + return kAudioUnitErr_InvalidParameter + } + + // Generate sine wave samples + for frame in 0 ..< Int(frameCount) { + ptr[frame] = Float(sin(currentPhase)) + + // Update the phase + currentPhase += phaseIncrement + + // Keep phase in [0, 2π] to prevent floating point errors + if currentPhase >= 2.0 * Double.pi { + currentPhase -= 2.0 * Double.pi + } + } + + return noErr + } + + super.init(format: format, renderBlock: renderBlock) + } +} From 9908d8eb94e3be6d90621230475536fea318643b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:10:37 +0900 Subject: [PATCH 02/24] Update render test --- Tests/LiveKitTests/AudioEngineTests.swift | 22 +++++- .../LiveKitTests/Support/AudioRecorder.swift | 68 +++++++++++++++++++ .../Support/SinWaveSourceNode.swift | 10 +-- 3 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 Tests/LiveKitTests/Support/AudioRecorder.swift diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 2005df332..c9fde51ba 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -87,10 +87,26 @@ class AudioEngineTests: XCTestCase { print("manualRenderingMode: \(isManualRenderingMode)") XCTAssert(isManualRenderingMode) - // Start rendering... + let recorder = try AudioRecorder() + + let track = LocalAudioTrack.createTrack() + track.add(audioRenderer: recorder) + + // Start engine... AudioManager.shared.startRecording() - // Render for 10 seconds... - try? await Task.sleep(for: .seconds(10)) + // Render for 5 seconds... + try? await Task.sleep(for: .seconds(5)) + + recorder.close() + print("Written to: \(recorder.filePath)") + + // Stop engine + AudioManager.shared.stopRecording() + + // Play the recorded file... + let player = try AVAudioPlayer(contentsOf: recorder.filePath) + player.play() + try? await Task.sleep(for: .seconds(5)) } } diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift new file mode 100644 index 000000000..641af9c25 --- /dev/null +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -0,0 +1,68 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio +@testable import LiveKit + +// Used to save audio data for inspecting the correct format, etc. +class AudioRecorder { + public let sampleRate: Double + public let audioFile: AVAudioFile + public let filePath: URL + + init(sampleRate: Double = 16000, channels: Int = 1) throws { + self.sampleRate = sampleRate + + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: sampleRate, + AVNumberOfChannelsKey: channels, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsNonInterleaved: false, + AVLinearPCMIsBigEndianKey: false, + ] + + let fileName = UUID().uuidString + ".wav" + let filePath = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(fileName) + self.filePath = filePath + + audioFile = try AVAudioFile(forWriting: filePath, + settings: settings, + commonFormat: .pcmFormatInt16, + interleaved: true) + } + + func write(pcmBuffer: AVAudioPCMBuffer) throws { + if #available(macOS 15.0, *) { + guard audioFile.isOpen else { return } + } + + try audioFile.write(from: pcmBuffer) + } + + func close() { + if #available(macOS 15.0, *) { + audioFile.close() + } + } +} + +extension AudioRecorder: AudioRenderer { + func render(pcmBuffer: AVAudioPCMBuffer) { + try? write(pcmBuffer: pcmBuffer) + } +} diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index b1fc632f7..1ab1ee53c 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -20,7 +20,7 @@ class SineWaveSourceNode: AVAudioSourceNode { private let sampleRate: Double private let frequency: Double - init(frequency: Double = 440.0, sampleRate: Double = 48000.0) { + init(frequency: Double = 400.0, sampleRate: Double = 48000.0) { self.frequency = frequency self.sampleRate = sampleRate @@ -30,8 +30,6 @@ class SineWaveSourceNode: AVAudioSourceNode { let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in - print("SineWaveSourceNode render block, frameCount: \(frameCount)") - let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { return kAudioUnitErr_InvalidParameter @@ -44,10 +42,8 @@ class SineWaveSourceNode: AVAudioSourceNode { // Update the phase currentPhase += phaseIncrement - // Keep phase in [0, 2π] to prevent floating point errors - if currentPhase >= 2.0 * Double.pi { - currentPhase -= 2.0 * Double.pi - } + // Keep phase within [0, 2π] range using fmod for stability + currentPhase = fmod(currentPhase, 2.0 * Double.pi) } return noErr From 3b9bbb3fc54ad350393a7913bffbb77ded678896 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:23:01 +0900 Subject: [PATCH 03/24] Backward compatible session config --- Sources/LiveKit/Track/AudioManager.swift | 54 ++++++++++--------- Tests/LiveKitTests/AudioEngineTests.swift | 20 ++++--- .../LiveKitTests/Support/AudioRecorder.swift | 4 +- .../Support/SinWaveSourceNode.swift | 1 + 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index b8a1c5499..c29e271f7 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -236,27 +236,43 @@ public class AudioManager: Loggable { } } - // MARK: Testing + // MARK: - Recording - public func startPlayout() { + /// Initialize recording (mic input) and pre-warm voice processing etc. + /// Mic permission is required and dialog will appear if not already granted. + public func prepareRecording() { + RTC.audioDeviceModule.initRecording() + } + + /// Starts mic input to the SDK even without any ``Room`` or a connection. + /// Audio buffers will flow into ``LocalAudioTrack/add(audioRenderer:)`` and ``capturePostProcessingDelegate``. + public func startLocalRecording() { + RTC.audioDeviceModule.initAndStartRecording() + } + + // MARK: Internal for testing + + func initPlayout() { RTC.audioDeviceModule.initPlayout() + } + + func startPlayout() { RTC.audioDeviceModule.startPlayout() } - public func stopPlayout() { + func stopPlayout() { RTC.audioDeviceModule.stopPlayout() } - public func initRecording() { + func initRecording() { RTC.audioDeviceModule.initRecording() } - public func startRecording() { - RTC.audioDeviceModule.initRecording() + func startRecording() { RTC.audioDeviceModule.startRecording() } - public func stopRecording() { + func stopRecording() { RTC.audioDeviceModule.stopRecording() } @@ -276,22 +292,10 @@ public class AudioManager: Loggable { #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - let config = LKRTCAudioSessionConfiguration.webRTC() - - if isRecordingEnabled { - config.category = AVAudioSession.Category.playAndRecord.rawValue - config.mode = AVAudioSession.Mode.videoChat.rawValue - config.categoryOptions = [.defaultToSpeaker, .allowBluetooth] - } else { - config.category = AVAudioSession.Category.playback.rawValue - config.mode = AVAudioSession.Mode.spokenAudio.rawValue - config.categoryOptions = [.mixWithOthers] - } - - session.lockForConfiguration() - try? session.setConfiguration(config) - session.unlockForConfiguration() + // Backward compatibility + let configureFunc = state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc + let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) + configureFunc(simulatedState, AudioManager.State()) #endif } } @@ -299,7 +303,7 @@ public class AudioManager: Loggable { // MARK: - Private func trackDidStart(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in + state.mutate { state in let oldState = state if type == .local { state.localTracksCount += 1 } if type == .remote { state.remoteTracksCount += 1 } @@ -308,7 +312,7 @@ public class AudioManager: Loggable { } func trackDidStop(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in + state.mutate { state in let oldState = state if type == .local { state.localTracksCount = max(state.localTracksCount - 1, 0) } if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index c9fde51ba..5b869c020 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -38,6 +38,14 @@ class AudioEngineTests: XCTestCase { XCTAssert(status == .authorized) } + // Test if state transitions pass internal checks. + func testStates() async { + let adm = AudioManager.shared + adm.initPlayout() + adm.startPlayout() + adm.stopPlayout() + } + // Test start generating local audio buffer without joining to room. func testPrejoinLocalAudioBuffer() async throws { // Set up expectation... @@ -55,8 +63,8 @@ class AudioEngineTests: XCTestCase { Task.detached { print("Starting audio track in 3 seconds...") - try? await Task.sleep(for: .seconds(3)) - AudioManager.shared.startRecording() + try? await Task.sleep(nanoseconds: 3 * 1_000_000_000) + AudioManager.shared.startLocalRecording() } // Wait for audio frame... @@ -93,20 +101,20 @@ class AudioEngineTests: XCTestCase { track.add(audioRenderer: recorder) // Start engine... - AudioManager.shared.startRecording() + AudioManager.shared.startLocalRecording() // Render for 5 seconds... - try? await Task.sleep(for: .seconds(5)) + try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) recorder.close() print("Written to: \(recorder.filePath)") - + // Stop engine AudioManager.shared.stopRecording() // Play the recorded file... let player = try AVAudioPlayer(contentsOf: recorder.filePath) player.play() - try? await Task.sleep(for: .seconds(5)) + try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) } } diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift index 641af9c25..de79a2038 100644 --- a/Tests/LiveKitTests/Support/AudioRecorder.swift +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -47,7 +47,7 @@ class AudioRecorder { } func write(pcmBuffer: AVAudioPCMBuffer) throws { - if #available(macOS 15.0, *) { + if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { guard audioFile.isOpen else { return } } @@ -55,7 +55,7 @@ class AudioRecorder { } func close() { - if #available(macOS 15.0, *) { + if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { audioFile.close() } } diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index 1ab1ee53c..fa031d9bf 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -30,6 +30,7 @@ class SineWaveSourceNode: AVAudioSourceNode { let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in + print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { return kAudioUnitErr_InvalidParameter From b1f3ae1d3ab1b8588c9a63eafabfdc32a26ca976 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:38:07 +0900 Subject: [PATCH 04/24] .mixWithOthers by default --- Sources/LiveKit/Types/AudioSessionConfiguration.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/LiveKit/Types/AudioSessionConfiguration.swift b/Sources/LiveKit/Types/AudioSessionConfiguration.swift index 171da686b..89f855ea7 100644 --- a/Sources/LiveKit/Types/AudioSessionConfiguration.swift +++ b/Sources/LiveKit/Types/AudioSessionConfiguration.swift @@ -37,11 +37,11 @@ public extension AudioSessionConfiguration { mode: .spokenAudio) static let playAndRecordSpeaker = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .videoChat) static let playAndRecordReceiver = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .voiceChat) } From b1871daf33fd5be57505e32a3028aff1ab503b72 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:36:32 +0900 Subject: [PATCH 05/24] Ducking config --- Sources/LiveKit/Track/AudioManager.swift | 11 +++++++++++ Tests/LiveKitTests/AudioEngineTests.swift | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index c29e271f7..eff4f08c0 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -236,6 +236,17 @@ public class AudioManager: Loggable { } } + public var isAdvancedDuckingEnabled: Bool { + get { RTC.audioDeviceModule.isAdvancedDuckingEnabled } + set { RTC.audioDeviceModule.isAdvancedDuckingEnabled = newValue } + } + + @available(iOS 17, macOS 14.0, visionOS 1.0, *) + public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { + get { RTC.audioDeviceModule.duckingLevel } + set { RTC.audioDeviceModule.duckingLevel = newValue } + } + // MARK: - Recording /// Initialize recording (mic input) and pre-warm voice processing etc. diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 5b869c020..dee6ff8d4 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -46,6 +46,28 @@ class AudioEngineTests: XCTestCase { adm.stopPlayout() } + func testConfigureDucking() async { + AudioManager.shared.isAdvancedDuckingEnabled = false + XCTAssert(!AudioManager.shared.isAdvancedDuckingEnabled) + + AudioManager.shared.isAdvancedDuckingEnabled = true + XCTAssert(AudioManager.shared.isAdvancedDuckingEnabled) + + if #available(iOS 17, macOS 14.0, visionOS 1.0, *) { + AudioManager.shared.duckingLevel = .default + XCTAssert(AudioManager.shared.duckingLevel == .default) + + AudioManager.shared.duckingLevel = .min + XCTAssert(AudioManager.shared.duckingLevel == .min) + + AudioManager.shared.duckingLevel = .max + XCTAssert(AudioManager.shared.duckingLevel == .max) + + AudioManager.shared.duckingLevel = .mid + XCTAssert(AudioManager.shared.duckingLevel == .mid) + } + } + // Test start generating local audio buffer without joining to room. func testPrejoinLocalAudioBuffer() async throws { // Set up expectation... From be593c138623d38a525685d541349c6976ff4cdf Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:33:51 +0900 Subject: [PATCH 06/24] Use 125.6422.12-exp.2 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index bf7183a9a..c7547e651 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 37e1c664d..82d48b771 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 89c084c443dcd5b0bd859f61454d4b771eacc7cc Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 9 Jan 2025 06:03:17 +0900 Subject: [PATCH 07/24] Muted speech activity --- Sources/LiveKit/Track/AudioManager.swift | 12 +++++++ .../LiveKit/Types/SpeechActivityEvent.swift | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 Sources/LiveKit/Types/SpeechActivityEvent.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index eff4f08c0..ca050aa76 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -71,6 +71,8 @@ public class AudioManager: Loggable { public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void + public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void + #if os(iOS) || os(visionOS) || os(tvOS) public typealias ConfigureAudioSessionFunc = @Sendable (_ newState: State, @@ -226,6 +228,16 @@ public class AudioManager: Loggable { } } + // Invoked on internal thread, do not block. + public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { + didSet { + RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in + guard let self else { return } + self.onMutedSpeechActivityEvent?(self, event.toLKType()) + } + } + } + public var isManualRenderingMode: Bool { get { RTC.audioDeviceModule.isManualRenderingMode } set { diff --git a/Sources/LiveKit/Types/SpeechActivityEvent.swift b/Sources/LiveKit/Types/SpeechActivityEvent.swift new file mode 100644 index 000000000..98d59917f --- /dev/null +++ b/Sources/LiveKit/Types/SpeechActivityEvent.swift @@ -0,0 +1,36 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public enum SpeechActivityEvent { + case started + case ended +} + +extension RTCSpeechActivityEvent { + func toLKType() -> SpeechActivityEvent { + switch self { + case .started: return .started + case .ended: return .ended + @unknown default: return .ended + } + } +} From 282cbc786046be18aafe042e57d930521b3d648f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:53:36 +0900 Subject: [PATCH 08/24] Update node config methods --- Sources/LiveKit/Track/AudioManager.swift | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index ca050aa76..4b314db23 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -69,7 +69,16 @@ public class AudioManager: Loggable { public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void - public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void + public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, + _ engine: AVAudioEngine, + _ src: AVAudioNode, + _ dst: AVAudioNode, + _ format: AVAudioFormat) -> Bool + public typealias OnEngineWillConnectOutput = (_ audioManager: AudioManager, + _ engine: AVAudioEngine, + _ src: AVAudioNode, + _ dst: AVAudioNode, + _ format: AVAudioFormat) -> Bool public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void @@ -221,9 +230,18 @@ public class AudioManager: Loggable { public var onEngineWillConnectInput: OnEngineWillConnectInput? { didSet { - RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, inputMixerNode in - guard let self else { return } - self.onEngineWillConnectInput?(self, engine, inputMixerNode) + RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in + guard let self else { return false } + return self.onEngineWillConnectInput?(self, engine, src, dst, format) ?? false + } + } + } + + public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { + didSet { + RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in + guard let self else { return false } + return self.onEngineWillConnectOutput?(self, engine, src, dst, format) ?? false } } } From 8f705401f4b9d099b0bd2c26e8d0bb82e7aac943 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:19:33 +0900 Subject: [PATCH 09/24] Move audio buffer --- Sources/LiveKit/Track/AudioManager.swift | 33 -------------- Sources/LiveKit/Types/AudioBuffer.swift | 56 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 33 deletions(-) create mode 100644 Sources/LiveKit/Types/AudioBuffer.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 4b314db23..3cddde7df 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -24,39 +24,6 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif -// Wrapper for LKRTCAudioBuffer -@objc -public class LKAudioBuffer: NSObject { - private let _audioBuffer: LKRTCAudioBuffer - - @objc - public var channels: Int { _audioBuffer.channels } - - @objc - public var frames: Int { _audioBuffer.frames } - - @objc - public var framesPerBand: Int { _audioBuffer.framesPerBand } - - @objc - public var bands: Int { _audioBuffer.bands } - - @objc - @available(*, deprecated, renamed: "rawBuffer(forChannel:)") - public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - @objc - public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - init(audioBuffer: LKRTCAudioBuffer) { - _audioBuffer = audioBuffer - } -} - // Audio Session Configuration related public class AudioManager: Loggable { // MARK: - Public diff --git a/Sources/LiveKit/Types/AudioBuffer.swift b/Sources/LiveKit/Types/AudioBuffer.swift new file mode 100644 index 000000000..63b39fd2f --- /dev/null +++ b/Sources/LiveKit/Types/AudioBuffer.swift @@ -0,0 +1,56 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +import Foundation + +// Wrapper for LKRTCAudioBuffer +@objc +public class LKAudioBuffer: NSObject { + private let _audioBuffer: LKRTCAudioBuffer + + @objc + public var channels: Int { _audioBuffer.channels } + + @objc + public var frames: Int { _audioBuffer.frames } + + @objc + public var framesPerBand: Int { _audioBuffer.framesPerBand } + + @objc + public var bands: Int { _audioBuffer.bands } + + @objc + @available(*, deprecated, renamed: "rawBuffer(forChannel:)") + public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + @objc + public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + init(audioBuffer: LKRTCAudioBuffer) { + _audioBuffer = audioBuffer + } +} From 92e34065f12951ddbc7166c6ca9d61b544064e93 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:33:00 +0900 Subject: [PATCH 10/24] Update AudioManager.swift Docs --- Sources/LiveKit/Track/AudioManager.swift | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 3cddde7df..6a59b545f 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -195,6 +195,9 @@ public class AudioManager: Loggable { } } + /// Provide custom implementation for internal AVAudioEngine's input configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. public var onEngineWillConnectInput: OnEngineWillConnectInput? { didSet { RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in @@ -204,6 +207,9 @@ public class AudioManager: Loggable { } } + /// Provide custom implementation for internal AVAudioEngine's output configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { didSet { RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in @@ -213,7 +219,9 @@ public class AudioManager: Loggable { } } - // Invoked on internal thread, do not block. + /// Detect voice activity even if the mic is muted. + /// Internal audio engine must be initialized by calling ``prepareRecording()`` or + /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { didSet { RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in @@ -233,11 +241,14 @@ public class AudioManager: Loggable { } } + /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. + /// Default: true. public var isAdvancedDuckingEnabled: Bool { get { RTC.audioDeviceModule.isAdvancedDuckingEnabled } set { RTC.audioDeviceModule.isAdvancedDuckingEnabled = newValue } } + /// The ducking(audio reducing) level of other audio. @available(iOS 17, macOS 14.0, visionOS 1.0, *) public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { get { RTC.audioDeviceModule.duckingLevel } From 4b77f849a90a642faf77b2995cfdc1f0eb915d24 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:46:37 +0900 Subject: [PATCH 11/24] Use 125.6422.12-exp.3 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index c7547e651..31977d747 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 82d48b771..0b652d4df 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 7cd4f29ae43e1863c1b185d2190eb8c6b03724c6 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:01:24 +0900 Subject: [PATCH 12/24] Fix tests --- Tests/LiveKitTests/AudioEngineTests.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index dee6ff8d4..5a33bd365 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -103,10 +103,11 @@ class AudioEngineTests: XCTestCase { // Attach sin wave generator when engine requests input node... // inputMixerNode will automatically convert to RTC's internal format (int16). // AVAudioEngine.attach() retains the node. - AudioManager.shared.onEngineWillConnectInput = { _, engine, inputMixerNode in + AudioManager.shared.onEngineWillConnectInput = { _, engine, _, dst, format in let sin = SineWaveSourceNode() engine.attach(sin) - engine.connect(sin, to: inputMixerNode, format: nil) + engine.connect(sin, to: dst, format: format) + return true } // Set manual rendering mode... From 130e1d219f75099eac3177981b0bdb8a9a87c99c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:15:20 +0900 Subject: [PATCH 13/24] Fix tests --- Tests/LiveKitTests/Support/AudioRecorder.swift | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift index de79a2038..dc0cffe99 100644 --- a/Tests/LiveKitTests/Support/AudioRecorder.swift +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -20,8 +20,8 @@ import AVFAudio // Used to save audio data for inspecting the correct format, etc. class AudioRecorder { public let sampleRate: Double - public let audioFile: AVAudioFile public let filePath: URL + private var audioFile: AVAudioFile? init(sampleRate: Double = 16000, channels: Int = 1) throws { self.sampleRate = sampleRate @@ -47,17 +47,12 @@ class AudioRecorder { } func write(pcmBuffer: AVAudioPCMBuffer) throws { - if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { - guard audioFile.isOpen else { return } - } - + guard let audioFile else { return } try audioFile.write(from: pcmBuffer) } func close() { - if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { - audioFile.close() - } + audioFile = nil } } From 874b3a4d37c20850d7885b2f6c0f67e179e2d7f0 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:51:29 +0900 Subject: [PATCH 14/24] AudioDuckingLevel type --- Sources/LiveKit/Track/AudioManager.swift | 6 ++--- Sources/LiveKit/Types/AudioDuckingLevel.swift | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 Sources/LiveKit/Types/AudioDuckingLevel.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 6a59b545f..e4232afc6 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -250,9 +250,9 @@ public class AudioManager: Loggable { /// The ducking(audio reducing) level of other audio. @available(iOS 17, macOS 14.0, visionOS 1.0, *) - public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { - get { RTC.audioDeviceModule.duckingLevel } - set { RTC.audioDeviceModule.duckingLevel = newValue } + public var duckingLevel: AudioDuckingLevel { + get { AudioDuckingLevel(rawValue: RTC.audioDeviceModule.duckingLevel) ?? .default } + set { RTC.audioDeviceModule.duckingLevel = newValue.rawValue } } // MARK: - Recording diff --git a/Sources/LiveKit/Types/AudioDuckingLevel.swift b/Sources/LiveKit/Types/AudioDuckingLevel.swift new file mode 100644 index 000000000..6caa49673 --- /dev/null +++ b/Sources/LiveKit/Types/AudioDuckingLevel.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public enum AudioDuckingLevel: Int { + case `default` = 0 + case min = 10 + case mid = 20 + case max = 30 +} From 49c91ef8634ae401fe792f0147242c376329ac5e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:34:51 +0900 Subject: [PATCH 15/24] Use 125.6422.12-exp.4 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 31977d747..753c73108 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 0b652d4df..2e8dddf83 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 4b84621d2ff4f41b63badf2fe17201eb575dfe64 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:53:31 +0900 Subject: [PATCH 16/24] Fix Xcode 14.2 --- Sources/LiveKit/Track/AudioManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index e4232afc6..9397b1549 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -312,7 +312,7 @@ public class AudioManager: Loggable { #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") // Backward compatibility - let configureFunc = state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc + let configureFunc = self.state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) configureFunc(simulatedState, AudioManager.State()) #endif From 5a585a3b6203ae70975802cf421076f78852c9f6 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:14:26 +0900 Subject: [PATCH 17/24] Change session config timing --- Sources/LiveKit/Track/AudioManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 9397b1549..6a84fc5b4 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -305,7 +305,7 @@ public class AudioManager: Loggable { let state = StateSync(State()) init() { - RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in guard let self else { return } self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") From a0103ad349ed465ed55051ea8cc32bb4f589b05c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:07:48 +0900 Subject: [PATCH 18/24] Update state tests --- Sources/LiveKit/Track/AudioManager.swift | 18 +++++++++++++++++- Tests/LiveKitTests/AudioEngineTests.swift | 19 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 6a84fc5b4..e2fbdcb63 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -269,7 +269,23 @@ public class AudioManager: Loggable { RTC.audioDeviceModule.initAndStartRecording() } - // MARK: Internal for testing + // MARK: - For testing + + var isPlayoutInitialized: Bool { + RTC.audioDeviceModule.isPlayoutInitialized + } + + var isPlaying: Bool { + RTC.audioDeviceModule.isPlaying + } + + var isRecordingInitialized: Bool { + RTC.audioDeviceModule.isRecordingInitialized + } + + var isRecording: Bool { + RTC.audioDeviceModule.isRecording + } func initPlayout() { RTC.audioDeviceModule.initPlayout() diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 5a33bd365..33e558baf 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -39,11 +39,28 @@ class AudioEngineTests: XCTestCase { } // Test if state transitions pass internal checks. - func testStates() async { + func testStateTransitions() async { let adm = AudioManager.shared + // Start Playout adm.initPlayout() + XCTAssert(adm.isPlayoutInitialized) adm.startPlayout() + XCTAssert(adm.isPlaying) + + // Start Recording + adm.initRecording() + XCTAssert(adm.isRecordingInitialized) + adm.startRecording() + XCTAssert(adm.isRecording) + + // Stop engine + adm.stopRecording() + XCTAssert(!adm.isRecording) + XCTAssert(!adm.isRecordingInitialized) + adm.stopPlayout() + XCTAssert(!adm.isPlaying) + XCTAssert(!adm.isPlayoutInitialized) } func testConfigureDucking() async { From 256b42aa1f0095b19752b619fd8079c945700fff Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:56:01 +0900 Subject: [PATCH 19/24] P1 --- .../Audio/AudioManager+EngineObserver.swift | 45 ++++++++++++ Sources/LiveKit/Track/AudioManager.swift | 71 ++++++++++++++++--- Tests/LiveKitTests/AudioEngineTests.swift | 18 +++++ 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioManager+EngineObserver.swift diff --git a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift new file mode 100644 index 000000000..662e30f0f --- /dev/null +++ b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift @@ -0,0 +1,45 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +/// Do not retain the engine object. +@objc +public protocol AudioEngineObserver: Chainable { + @objc optional + func engineDidCreate(_ engine: AVAudioEngine) + + @objc optional + func engineWillEnable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineWillStart(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineDidStop(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineDidDisable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineWillRelease(_ engine: AVAudioEngine) + + @objc optional + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) + + @objc optional + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index e2fbdcb63..ea4c1ef06 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -35,6 +35,8 @@ public class AudioManager: Loggable { #endif public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + + // Engine events public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, @@ -257,10 +259,12 @@ public class AudioManager: Loggable { // MARK: - Recording - /// Initialize recording (mic input) and pre-warm voice processing etc. + /// Keep recording initialized (mic input) and pre-warm voice processing etc. /// Mic permission is required and dialog will appear if not already granted. - public func prepareRecording() { - RTC.audioDeviceModule.initRecording() + /// This will per persisted accross Rooms and connections. + public var isRecordingAlwaysPrepared: Bool { + get { RTC.audioDeviceModule.isInitRecordingPersistentMode } + set { RTC.audioDeviceModule.isInitRecordingPersistentMode = newValue } } /// Starts mic input to the SDK even without any ``Room`` or a connection. @@ -320,19 +324,70 @@ public class AudioManager: Loggable { let state = StateSync(State()) + var isSessionActive: Bool = false + init() { RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in guard let self else { return } - self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + self.log("OnEngineWillEnable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") - // Backward compatibility - let configureFunc = self.state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc - let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) - configureFunc(simulatedState, AudioManager.State()) + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if isSessionActive { + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } else { + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + isSessionActive = true + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") #endif } + + RTC.audioDeviceModule.setOnEngineDidStopCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineDidDisable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + + #if os(iOS) || os(visionOS) || os(tvOS) + self.log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + isSessionActive = false + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + } + + RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + } } // MARK: - Private diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 33e558baf..4db9ac664 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -63,6 +63,24 @@ class AudioEngineTests: XCTestCase { XCTAssert(!adm.isPlayoutInitialized) } + func testRecordingAlwaysPreparedMode() async { + let adm = AudioManager.shared + + // Ensure initially not initialized. + XCTAssert(!adm.isRecordingInitialized) + + // Ensure recording is initialized after set to true. + adm.isRecordingAlwaysPrepared = true + XCTAssert(adm.isRecordingInitialized) + + adm.startRecording() + XCTAssert(adm.isRecordingInitialized) + + // Should be still initialized after stopRecording() is called. + adm.stopRecording() + XCTAssert(adm.isRecordingInitialized) + } + func testConfigureDucking() async { AudioManager.shared.isAdvancedDuckingEnabled = false XCTAssert(!AudioManager.shared.isAdvancedDuckingEnabled) From 3c9c0ddc7d5343b886f6efc03f0acd1d38444059 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:19:25 +0900 Subject: [PATCH 20/24] Chained engine observer --- .../AudioDeviceModuleDelegateAdapter.swift | 88 +++++++++++ .../LiveKit/Audio/AudioEngineObserver.swift | 63 ++++++++ .../Audio/AudioManager+EngineObserver.swift | 45 ------ .../Audio/DefaultAudioSessionObserver.swift | 89 ++++++++++++ Sources/LiveKit/Protocols/NextInvokable.swift | 22 +++ Sources/LiveKit/Track/AudioManager.swift | 137 +++--------------- 6 files changed, 283 insertions(+), 161 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift create mode 100644 Sources/LiveKit/Audio/AudioEngineObserver.swift delete mode 100644 Sources/LiveKit/Audio/AudioManager+EngineObserver.swift create mode 100644 Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift create mode 100644 Sources/LiveKit/Protocols/NextInvokable.swift diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift new file mode 100644 index 000000000..6f5a382cb --- /dev/null +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -0,0 +1,88 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +// Invoked on WebRTC's worker thread, do not block. +class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate { + weak var audioManager: AudioManager? + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent) { + guard let audioManager else { return } + audioManager.onMutedSpeechActivityEvent?(audioManager, speechActivityEvent.toLKType()) + } + + func audioDeviceModuleDidUpdateDevices(_: LKRTCAudioDeviceModule) { + guard let audioManager else { return } + audioManager.onDeviceUpdate?(audioManager) + } + + // Engine events + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidCreate(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillRelease(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager.state.engineObservers.buildChain() + return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager.state.engineObservers.buildChain() + return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false + } +} diff --git a/Sources/LiveKit/Audio/AudioEngineObserver.swift b/Sources/LiveKit/Audio/AudioEngineObserver.swift new file mode 100644 index 000000000..94c09b56c --- /dev/null +++ b/Sources/LiveKit/Audio/AudioEngineObserver.swift @@ -0,0 +1,63 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +/// Do not retain the engine object. +public protocol AudioEngineObserver: NextInvokable, Sendable { + func setNext(_ handler: any AudioEngineObserver) + + func engineDidCreate(_ engine: AVAudioEngine) + func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillStart(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillRelease(_ engine: AVAudioEngine) + + /// Provide custom implementation for internal AVAudioEngine's output configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool + /// Provide custom implementation for internal AVAudioEngine's input configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool +} + +/// Default implementation to make it optional. +public extension AudioEngineObserver { + func engineDidCreate(_: AVAudioEngine) {} + func engineWillEnable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillStart(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidStop(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillRelease(_: AVAudioEngine) {} + + func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } + func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } +} + +extension [any AudioEngineObserver] { + func buildChain() -> Element? { + guard let first else { return nil } + + for i in 0 ..< count - 1 { + self[i].setNext(self[i + 1]) + } + + return first + } +} diff --git a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift deleted file mode 100644 index 662e30f0f..000000000 --- a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2025 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import AVFAudio - -/// Do not retain the engine object. -@objc -public protocol AudioEngineObserver: Chainable { - @objc optional - func engineDidCreate(_ engine: AVAudioEngine) - - @objc optional - func engineWillEnable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineWillStart(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineDidStop(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineDidDisable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineWillRelease(_ engine: AVAudioEngine) - - @objc optional - func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) - - @objc optional - func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -} diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift new file mode 100644 index 000000000..7bde7be0d --- /dev/null +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -0,0 +1,89 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { + var next: (any AudioEngineObserver)? + var isSessionActive = false + + public func setNext(_ handler: any AudioEngineObserver) { + next = handler + } + + public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + #if os(iOS) || os(visionOS) || os(tvOS) + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if isSessionActive { + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } else { + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + isSessionActive = true + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + + // Call next last + next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + public func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + // Call next first + next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + + #if os(iOS) || os(visionOS) || os(tvOS) + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + isSessionActive = false + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + } +} diff --git a/Sources/LiveKit/Protocols/NextInvokable.swift b/Sources/LiveKit/Protocols/NextInvokable.swift new file mode 100644 index 000000000..4dc516155 --- /dev/null +++ b/Sources/LiveKit/Protocols/NextInvokable.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +public protocol NextInvokable { + associatedtype Next + func setNext(_ handler: Next) +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 00c2cd622..48fb3c774 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -36,19 +36,6 @@ public class AudioManager: Loggable { public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void - // Engine events - public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void - public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, - _ engine: AVAudioEngine, - _ src: AVAudioNode, - _ dst: AVAudioNode, - _ format: AVAudioFormat) -> Bool - public typealias OnEngineWillConnectOutput = (_ audioManager: AudioManager, - _ engine: AVAudioEngine, - _ src: AVAudioNode, - _ dst: AVAudioNode, - _ format: AVAudioFormat) -> Bool - public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void #if os(iOS) || os(visionOS) || os(tvOS) @@ -125,6 +112,8 @@ public class AudioManager: Loggable { public var sessionConfiguration: AudioSessionConfiguration? #endif + public var engineObservers = [any AudioEngineObserver]() + public var trackState: TrackState { switch (localTracksCount > 0, remoteTracksCount > 0) { case (true, false): return .localOnly @@ -195,50 +184,12 @@ public class AudioManager: Loggable { set { RTC.audioDeviceModule.inputDevice = newValue._ioDevice } } - public var onDeviceUpdate: DeviceUpdateFunc? { - didSet { - RTC.audioDeviceModule.setDevicesDidUpdateCallback { [weak self] in - guard let self else { return } - self.onDeviceUpdate?(self) - } - } - } - - /// Provide custom implementation for internal AVAudioEngine's input configuration. - /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. - /// Return true if custom implementation is provided, otherwise default implementation will be used. - public var onEngineWillConnectInput: OnEngineWillConnectInput? { - didSet { - RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in - guard let self else { return false } - return self.onEngineWillConnectInput?(self, engine, src, dst, format) ?? false - } - } - } - - /// Provide custom implementation for internal AVAudioEngine's output configuration. - /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. - /// Return true if custom implementation is provided, otherwise default implementation will be used. - public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { - didSet { - RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in - guard let self else { return false } - return self.onEngineWillConnectOutput?(self, engine, src, dst, format) ?? false - } - } - } + public var onDeviceUpdate: DeviceUpdateFunc? /// Detect voice activity even if the mic is muted. /// Internal audio engine must be initialized by calling ``prepareRecording()`` or /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. - public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { - didSet { - RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in - guard let self else { return } - self.onMutedSpeechActivityEvent?(self, event.toLKType()) - } - } - } + public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? public var isManualRenderingMode: Bool { get { RTC.audioDeviceModule.isManualRenderingMode } @@ -280,6 +231,17 @@ public class AudioManager: Loggable { RTC.audioDeviceModule.initAndStartRecording() } + /// Set a chain of ``AudioEngineObserver``s. + /// Defaults to having a single ``DefaultAudioSessionObserver`` initially. + /// + /// The first object will be invoked and is responsible for calling the next object. + /// See ``NextInvokable`` protocol for details. + /// + /// Objects set here will be retained. + public func set(engineObservers: [any AudioEngineObserver]) { + state.mutate { $0.engineObservers = engineObservers } + } + // MARK: - For testing var isPlayoutInitialized: Bool { @@ -329,72 +291,15 @@ public class AudioManager: Loggable { case remote } - let state = StateSync(State()) + let state: StateSync - var isSessionActive: Bool = false + let admDelegateAdapter: AudioDeviceModuleDelegateAdapter init() { - RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineWillEnable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - - #if os(iOS) || os(visionOS) || os(tvOS) - self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback - do { - if isSessionActive { - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } else { - log("AudioSession activating category to: \(config.category)") - try session.setConfiguration(config.toRTCType(), active: true) - isSessionActive = true - } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) - } - - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif - } - - RTC.audioDeviceModule.setOnEngineDidStopCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineDidDisable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - - #if os(iOS) || os(visionOS) || os(tvOS) - self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - do { - if isPlayoutEnabled, !isRecordingEnabled { - let config: AudioSessionConfiguration = .playback - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } - if !isPlayoutEnabled, !isRecordingEnabled { - log("AudioSession deactivating") - try session.setActive(false) - isSessionActive = false - } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) - } - - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif - } - - RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - } + state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) + admDelegateAdapter = AudioDeviceModuleDelegateAdapter() + admDelegateAdapter.audioManager = self + RTC.audioDeviceModule.observer = admDelegateAdapter } // MARK: - Private From 7e48b7b686a58a750cc2c9d5e88ec15225ff44d9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:20:26 +0900 Subject: [PATCH 21/24] lib 125.6422.12-exp.5 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 753c73108..fd2b06a40 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 2e8dddf83..c038dc0c4 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From d3deb72f6c92eeb11b6153fe652901c3eacb5f4f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:36:41 +0900 Subject: [PATCH 22/24] Update test --- .../AudioDeviceModuleDelegateAdapter.swift | 4 +-- Tests/LiveKitTests/AudioEngineTests.swift | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index 6f5a382cb..53652f525 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -74,13 +74,13 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate entryPoint?.engineWillRelease(engine) } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 4db9ac664..9cb6d54d0 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -135,19 +135,13 @@ class AudioEngineTests: XCTestCase { // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. // In manual rendering, no device access will be initialized such as mic and speaker. func testManualRenderingMode() async throws { - // Attach sin wave generator when engine requests input node... - // inputMixerNode will automatically convert to RTC's internal format (int16). - // AVAudioEngine.attach() retains the node. - AudioManager.shared.onEngineWillConnectInput = { _, engine, _, dst, format in - let sin = SineWaveSourceNode() - engine.attach(sin) - engine.connect(sin, to: dst, format: format) - return true - } - // Set manual rendering mode... AudioManager.shared.isManualRenderingMode = true + // Attach sine wave generator when engine requests input node. + // inputMixerNode will automatically convert to RTC's internal format (int16). + AudioManager.shared.set(engineObservers: [RewriteInputToSineWaveGenerator()]) + // Check if manual rendering mode is set... let isManualRenderingMode = AudioManager.shared.isManualRenderingMode print("manualRenderingMode: \(isManualRenderingMode)") @@ -176,3 +170,15 @@ class AudioEngineTests: XCTestCase { try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) } } + +final class RewriteInputToSineWaveGenerator: AudioEngineObserver { + func setNext(_: any LiveKit.AudioEngineObserver) {} + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + print("engineWillConnectInput") + let sin = SineWaveSourceNode() + // AVAudioEngine.attach() retains the node. + engine.attach(sin) + engine.connect(sin, to: dst, format: format) + return true + } +} From aa8977b10a102449a0a654cbad34549d2e343a80 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:40:27 +0900 Subject: [PATCH 23/24] Fix test --- Tests/LiveKitTests/AudioEngineTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 9cb6d54d0..70f586ebb 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -27,6 +27,7 @@ class AudioEngineTests: XCTestCase { override func tearDown() async throws {} + #if !targetEnvironment(simulator) // Test if mic is authorized. Only works on device. func testMicAuthorized() async { let status = AVCaptureDevice.authorizationStatus(for: .audio) @@ -37,6 +38,7 @@ class AudioEngineTests: XCTestCase { XCTAssert(status == .authorized) } + #endif // Test if state transitions pass internal checks. func testStateTransitions() async { From 82323b4bfc4f836133f654b9fb3303d1388ae7b9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 24 Jan 2025 00:37:22 +0900 Subject: [PATCH 24/24] Update manual render test --- Sources/LiveKit/Track/AudioManager.swift | 3 +- Tests/LiveKitTests/AudioEngineTests.swift | 7 +++-- .../Support/SinWaveSourceNode.swift | 29 ++++++++++++------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 48fb3c774..15212265e 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -293,11 +293,10 @@ public class AudioManager: Loggable { let state: StateSync - let admDelegateAdapter: AudioDeviceModuleDelegateAdapter + let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() init() { state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) - admDelegateAdapter = AudioDeviceModuleDelegateAdapter() admDelegateAdapter.audioManager = self RTC.audioDeviceModule.observer = admDelegateAdapter } diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 70f586ebb..62c0e0da8 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -139,7 +139,6 @@ class AudioEngineTests: XCTestCase { func testManualRenderingMode() async throws { // Set manual rendering mode... AudioManager.shared.isManualRenderingMode = true - // Attach sine wave generator when engine requests input node. // inputMixerNode will automatically convert to RTC's internal format (int16). AudioManager.shared.set(engineObservers: [RewriteInputToSineWaveGenerator()]) @@ -151,7 +150,10 @@ class AudioEngineTests: XCTestCase { let recorder = try AudioRecorder() - let track = LocalAudioTrack.createTrack() + // Note: AudioCaptureOptions will not be applied since track is not published. + let noProcessingOptions = AudioCaptureOptions(echoCancellation: false, noiseSuppression: false, autoGainControl: false, highpassFilter: false) + + let track = LocalAudioTrack.createTrack(options: noProcessingOptions) track.add(audioRenderer: recorder) // Start engine... @@ -178,7 +180,6 @@ final class RewriteInputToSineWaveGenerator: AudioEngineObserver { func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { print("engineWillConnectInput") let sin = SineWaveSourceNode() - // AVAudioEngine.attach() retains the node. engine.attach(sin) engine.connect(sin, to: dst, format: format) return true diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index fa031d9bf..8913ac0a3 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -26,25 +26,32 @@ class SineWaveSourceNode: AVAudioSourceNode { let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1)! - var currentPhase = 0.0 - let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate + let twoPi = 2 * Float.pi + let amplitude: Float = 0.5 + var currentPhase: Float = 0.0 + let phaseIncrement: Float = (twoPi / Float(sampleRate)) * Float(frequency) let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) - guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { - return kAudioUnitErr_InvalidParameter - } // Generate sine wave samples for frame in 0 ..< Int(frameCount) { - ptr[frame] = Float(sin(currentPhase)) - - // Update the phase + // Get the signal value for this frame at time. + let value = sin(currentPhase) * amplitude + // Advance the phase for the next frame. currentPhase += phaseIncrement - - // Keep phase within [0, 2π] range using fmod for stability - currentPhase = fmod(currentPhase, 2.0 * Double.pi) + if currentPhase >= twoPi { + currentPhase -= twoPi + } + if currentPhase < 0.0 { + currentPhase += twoPi + } + // Set the same value on all channels (due to the inputFormat, there's only one channel though). + for buffer in ablPointer { + let buf: UnsafeMutableBufferPointer = UnsafeMutableBufferPointer(buffer) + buf[frame] = value + } } return noErr