diff --git a/Examples/iOS/Screencast/SampleHandler.swift b/Examples/iOS/Screencast/SampleHandler.swift index 991e66eb4..582cd1b9d 100644 --- a/Examples/iOS/Screencast/SampleHandler.swift +++ b/Examples/iOS/Screencast/SampleHandler.swift @@ -15,7 +15,8 @@ open class SampleHandler: RPBroadcastSampleHandler { }() private lazy var rtmpStream: RTMPStream = { - RTMPStream(connection: rtmpConnection) + FeatureUtil.setEnabled(feature: .multiTrackMixing, isEnabled: true) + return RTMPStream(connection: rtmpConnection) }() private var isMirophoneOn = false diff --git a/HaishinKit.xcodeproj/project.pbxproj b/HaishinKit.xcodeproj/project.pbxproj index 50db726a9..0a8616d04 100644 --- a/HaishinKit.xcodeproj/project.pbxproj +++ b/HaishinKit.xcodeproj/project.pbxproj @@ -118,7 +118,7 @@ 2EC97B7327880FF400D8BE32 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6F27880FF400D8BE32 /* Views.swift */; }; 2EC97B7427880FF400D8BE32 /* MTHKSwiftUiView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */; }; B34239852B9FD3E30068C3FB /* AudioNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = B34239842B9FD3E30068C3FB /* AudioNode.swift */; }; - B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixer.swift */; }; + B3D687822B80302B00E6A28E /* IOAudioMixerConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */; }; BC0394562AA8A384006EDE38 /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; }; BC03945F2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */; }; BC04A2D42AD2D1D700C87A3E /* AVAudioTime+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */; }; @@ -161,7 +161,9 @@ BC3802192AB6AD79001AE399 /* IOAudioResamplerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3802182AB6AD79001AE399 /* IOAudioResamplerTests.swift */; }; BC3E384429C216BB007CD972 /* ADTSReaderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3E384329C216BB007CD972 /* ADTSReaderTests.swift */; }; BC4078C42AD5CC7E00BBB4FA /* IOMuxer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4078C32AD5CC7E00BBB4FA /* IOMuxer.swift */; }; - BC4231642BCA5F28003A80DC /* IOAUAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */; }; + BC4231642BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */; }; + BC42316A2BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */; }; + BC42316C2BCB7084003A80DC /* FeatureUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC42316B2BCB7084003A80DC /* FeatureUtil.swift */; }; BC4914A228DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */; }; BC4914A628DDD367009E2DF6 /* VTSessionOption.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A528DDD367009E2DF6 /* VTSessionOption.swift */; }; BC4914AE28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */; }; @@ -556,7 +558,7 @@ 2EC97B6F27880FF400D8BE32 /* Views.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Views.swift; sourceTree = ""; }; 2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MTHKSwiftUiView.swift; sourceTree = ""; }; B34239842B9FD3E30068C3FB /* AudioNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioNode.swift; sourceTree = ""; }; - B3D687812B80302B00E6A28E /* IOAudioMixer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOAudioMixer.swift; sourceTree = ""; }; + B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertible.swift; sourceTree = ""; }; BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ExpressibleByIntegerLiteral+ExtensionTests.swift"; sourceTree = ""; }; BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioTime+Extension.swift"; sourceTree = ""; }; BC04A2D52AD2D95500C87A3E /* CMTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CMTime+Extension.swift"; sourceTree = ""; }; @@ -595,7 +597,9 @@ BC3802182AB6AD79001AE399 /* IOAudioResamplerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioResamplerTests.swift; sourceTree = ""; }; BC3E384329C216BB007CD972 /* ADTSReaderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ADTSReaderTests.swift; sourceTree = ""; }; BC4078C32AD5CC7E00BBB4FA /* IOMuxer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOMuxer.swift; sourceTree = ""; }; - BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAUAudioMixer.swift; sourceTree = ""; }; + BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertibleByMultiTrack.swift; sourceTree = ""; }; + BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertibleBySingleTrack.swift; sourceTree = ""; }; + BC42316B2BCB7084003A80DC /* FeatureUtil.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeatureUtil.swift; sourceTree = ""; }; BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionConvertible.swift; sourceTree = ""; }; BC4914A528DDD367009E2DF6 /* VTSessionOption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionOption.swift; sourceTree = ""; }; BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VTDecompressionSession+Extension.swift"; sourceTree = ""; }; @@ -805,6 +809,7 @@ BC0D236C26331BAB001DDA0C /* DataBuffer.swift */, 29B876671CD70AB300FC07DA /* DataConvertible.swift */, 2976A4851D4903C300B53EF2 /* DeviceUtil.swift */, + BC42316B2BCB7084003A80DC /* FeatureUtil.swift */, BC32E88729C9971100051507 /* InstanceHolder.swift */, 2942424C1CF4C01300D65DCB /* MD5.swift */, 2942A4F721A9418A004E1BEE /* Running.swift */, @@ -1059,9 +1064,10 @@ B34239842B9FD3E30068C3FB /* AudioNode.swift */, BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */, BC959EEE296EE4190067BA97 /* ImageTransform.swift */, - BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */, BC3802132AB5E7CC001AE399 /* IOAudioCaptureUnit.swift */, - B3D687812B80302B00E6A28E /* IOAudioMixer.swift */, + B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */, + BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */, + BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */, BC31DBD12A653D1600C4DEA3 /* IOAudioMonitor.swift */, BCFC51FD2AAB420700014428 /* IOAudioResampler.swift */, BC5019C02A6D266B0046E02F /* IOAudioRingBuffer.swift */, @@ -1749,7 +1755,7 @@ 29B8765D1CD70A7900FC07DA /* VideoCodec.swift in Sources */, 2999C3752071138F00892E55 /* MTHKView.swift in Sources */, 29AF3FCF1D7C744C00E41212 /* IOStream.swift in Sources */, - B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */, + B3D687822B80302B00E6A28E /* IOAudioMixerConvertible.swift in Sources */, BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */, 29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */, 29B876841CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift in Sources */, @@ -1786,9 +1792,10 @@ BC0F1FDA2ACC4CC100C326FF /* IOCaptureVideoPreview.swift in Sources */, BC4914B228DDFE31009E2DF6 /* VTSessionOptionKey.swift in Sources */, BC7C56CD29A786AE00C41A9B /* ADTS.swift in Sources */, + BC42316A2BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift in Sources */, BC562DCB29576D220048D89A /* AVCaptureSession.Preset+Extension.swift in Sources */, 29B876AB1CD70B2800FC07DA /* AMF0Serializer.swift in Sources */, - BC4231642BCA5F28003A80DC /* IOAUAudioMixer.swift in Sources */, + BC4231642BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift in Sources */, 29B8765B1CD70A7900FC07DA /* AudioCodec.swift in Sources */, 29EA87D51E799F670043A5F8 /* Mirror+Extension.swift in Sources */, BCC4F4152AD6FC1100954EF5 /* IOTellyUnit.swift in Sources */, @@ -1798,6 +1805,7 @@ BC562DC7295767860048D89A /* AVCaptureDevice+Extension.swift in Sources */, BC0F1FDC2ACC630400C326FF /* NSView+Extension.swift in Sources */, 29EA87E21E79A1E90043A5F8 /* CMVideoFormatDescription+Extension.swift in Sources */, + BC42316C2BCB7084003A80DC /* FeatureUtil.swift in Sources */, BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */, BC1DC4A429F4F74F00E928ED /* AVCaptureSession+Extension.swift in Sources */, 29EA87D81E79A0090043A5F8 /* URL+Extension.swift in Sources */, diff --git a/Sources/IO/IOAudioMixer.swift b/Sources/IO/IOAudioMixerConvertible.swift similarity index 97% rename from Sources/IO/IOAudioMixer.swift rename to Sources/IO/IOAudioMixerConvertible.swift index 0945fea68..073833e98 100644 --- a/Sources/IO/IOAudioMixer.swift +++ b/Sources/IO/IOAudioMixerConvertible.swift @@ -48,6 +48,7 @@ struct IOAudioMixerSettings { } protocol IOAudioMixerConvertible: AnyObject { + var delegate: (any IOAudioMixerDelegate)? { get set } var inputFormat: AVAudioFormat? { get } var settings: IOAudioMixerSettings { get set } diff --git a/Sources/IO/IOAUAudioMixer.swift b/Sources/IO/IOAudioMixerConvertibleByMultiTrack.swift similarity index 89% rename from Sources/IO/IOAUAudioMixer.swift rename to Sources/IO/IOAudioMixerConvertibleByMultiTrack.swift index 8bc54d172..c57d9eb89 100644 --- a/Sources/IO/IOAUAudioMixer.swift +++ b/Sources/IO/IOAudioMixerConvertibleByMultiTrack.swift @@ -1,14 +1,14 @@ import AVFoundation import Foundation -final class IOAUAudioMixer: IOAudioMixerConvertible { +final class IOAudioMixerConvertibleByMultiTrack: IOAudioMixerConvertible { private static let defaultSampleTime: AVAudioFramePosition = 0 private class Track { - let resampler: IOAudioResampler + let resampler: IOAudioResampler var ringBuffer: IOAudioRingBuffer? - init(resampler: IOAudioResampler, format: AVAudioFormat? = nil) { + init(resampler: IOAudioResampler, format: AVAudioFormat? = nil) { self.resampler = resampler if let format { self.ringBuffer = .init(format) @@ -42,7 +42,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible { numberOfTracks > 1 } private var anchor: AVAudioTime? - private var sampleTime: AVAudioFramePosition = IOAUAudioMixer.defaultSampleTime + private var sampleTime: AVAudioFramePosition = IOAudioMixerConvertibleByMultiTrack.defaultSampleTime private var mixerNode: MixerNode? private var outputNode: OutputNode? private var defaultTrack: Track? { @@ -50,7 +50,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible { } private let inputRenderCallback: AURenderCallback = { (inRefCon: UnsafeMutableRawPointer, _: UnsafeMutablePointer, _: UnsafePointer, inBusNumber: UInt32, inNumberFrames: UInt32, ioData: UnsafeMutablePointer?) in - let audioMixer = Unmanaged.fromOpaque(inRefCon).takeUnretainedValue() + let audioMixer = Unmanaged.fromOpaque(inRefCon).takeUnretainedValue() let status = audioMixer.provideInput(inNumberFrames, channel: Int(inBusNumber), ioData: ioData) guard status == noErr else { audioMixer.delegate?.audioMixer(audioMixer, errorOccurred: .failedToMix(error: IOAudioMixerError.unableToProvideInputData)) @@ -90,7 +90,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible { } private func makeTrack(channel: Int) -> Track { - let resampler = IOAudioResampler() + let resampler = IOAudioResampler() resampler.channel = channel if channel == settings.mainTrack { resampler.settings = settings.defaultResamplerSettings @@ -201,7 +201,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible { } } - private func applySettings(resampler: IOAudioResampler, + private func applySettings(resampler: IOAudioResampler, defaultFormat: AVAudioFormat?, preferredSettings: IOAudioResamplerSettings?) { let preferredSettings = preferredSettings ?? .init() @@ -218,9 +218,9 @@ final class IOAUAudioMixer: IOAudioMixerConvertible { } } -extension IOAUAudioMixer: IOAudioResamplerDelegate { +extension IOAudioMixerConvertibleByMultiTrack: IOAudioResamplerDelegate { // MARK: IOAudioResamplerDelegate - func resampler(_ resampler: IOAudioResampler, didOutput audioFormat: AVAudioFormat) { + func resampler(_ resampler: IOAudioResampler, didOutput audioFormat: AVAudioFormat) { guard shouldMix else { delegate?.audioMixer(self, didOutput: audioFormat) return @@ -233,7 +233,7 @@ extension IOAUAudioMixer: IOAudioResamplerDelegate { track(channel: resampler.channel).ringBuffer = .init(audioFormat) } - func resampler(_ resampler: IOAudioResampler, didOutput audioBuffer: AVAudioPCMBuffer, when: AVAudioTime) { + func resampler(_ resampler: IOAudioResampler, didOutput audioBuffer: AVAudioPCMBuffer, when: AVAudioTime) { guard shouldMix else { delegate?.audioMixer(self, didOutput: audioBuffer, when: when) return @@ -255,7 +255,7 @@ extension IOAUAudioMixer: IOAudioResamplerDelegate { } } - func resampler(_ resampler: IOAudioResampler, errorOccurred error: IOAudioUnitError) { + func resampler(_ resampler: IOAudioResampler, errorOccurred error: IOAudioUnitError) { delegate?.audioMixer(self, errorOccurred: error) } } diff --git a/Sources/IO/IOAudioMixerConvertibleBySingleTrack.swift b/Sources/IO/IOAudioMixerConvertibleBySingleTrack.swift new file mode 100644 index 000000000..babb6c233 --- /dev/null +++ b/Sources/IO/IOAudioMixerConvertibleBySingleTrack.swift @@ -0,0 +1,43 @@ +import AVFoundation +import Foundation + +final class IOAudioMixerConvertibleBySingleTrack: IOAudioMixerConvertible { + var delegate: (any IOAudioMixerDelegate)? + var inputFormat: AVAudioFormat? + var settings: IOAudioMixerSettings = .init() + + private lazy var resampler: IOAudioResampler = { + var resampler = IOAudioResampler() + resampler.delegate = self + return resampler + }() + + func append(_ buffer: CMSampleBuffer, track: UInt8) { + guard settings.mainTrack == track else { + return + } + resampler.append(buffer) + } + + func append(_ buffer: AVAudioPCMBuffer, when: AVAudioTime, track: UInt8) { + guard settings.mainTrack == track else { + return + } + resampler.append(buffer, when: when) + } +} + +extension IOAudioMixerConvertibleBySingleTrack: IOAudioResamplerDelegate { + // MARK: IOAudioResamplerDelegate + func resampler(_ resampler: IOAudioResampler, didOutput audioFormat: AVAudioFormat) { + delegate?.audioMixer(self, didOutput: audioFormat) + } + + func resampler(_ resampler: IOAudioResampler, didOutput audioPCMBuffer: AVAudioPCMBuffer, when: AVAudioTime) { + delegate?.audioMixer(self, didOutput: audioPCMBuffer, when: when) + } + + func resampler(_ resampler: IOAudioResampler, errorOccurred error: IOAudioUnitError) { + delegate?.audioMixer(self, errorOccurred: error) + } +} diff --git a/Sources/IO/IOAudioUnit.swift b/Sources/IO/IOAudioUnit.swift index 804531e70..a6811247a 100644 --- a/Sources/IO/IOAudioUnit.swift +++ b/Sources/IO/IOAudioUnit.swift @@ -53,9 +53,15 @@ final class IOAudioUnit: NSObject, IOUnit { return codec }() private lazy var audioMixer: any IOAudioMixerConvertible = { - var audioMixer = IOAUAudioMixer() - audioMixer.delegate = self - return audioMixer + if FeatureUtil.isEnabled(feature: .multiTrackMixing) { + var audioMixer = IOAudioMixerConvertibleByMultiTrack() + audioMixer.delegate = self + return audioMixer + } else { + var audioMixer = IOAudioMixerConvertibleBySingleTrack() + audioMixer.delegate = self + return audioMixer + } }() private var monitor: IOAudioMonitor = .init() #if os(tvOS) diff --git a/Sources/Util/FeatureUtil.swift b/Sources/Util/FeatureUtil.swift new file mode 100644 index 000000000..f1b5cb96b --- /dev/null +++ b/Sources/Util/FeatureUtil.swift @@ -0,0 +1,43 @@ +import Foundation + +/// The util object to get feature flag info. +public enum FeatureUtil { + /// A structure that defines the name of a feature. + public struct Name: RawRepresentable, ExpressibleByStringLiteral { + // swiftlint:disable:next nesting + public typealias RawValue = String + // swiftlint:disable:next nesting + public typealias StringLiteralType = String + + /// This is a feature to mix multiple audio tracks. For example, it is possible to mix .appAudio and .micAudio from ReplayKit. + public static let multiTrackMixing: Name = "multiTrackMixing" + + /// The raw type value. + public let rawValue: String + + /// Create a feature name by rawValue. + public init(rawValue: String) { + self.rawValue = rawValue + } + + /// Create a feature name by stringLiteral. + public init(stringLiteral value: String) { + self.rawValue = value + } + } + + private static var flags: [String: Bool] = [:] + + /// Whether or not a flag is enabled. + public static func isEnabled(feature: Name) -> Bool { + return flags[feature.rawValue] ?? false + } + + /// Setter for a feature flag. + public static func setEnabled( + feature: Name, + isEnabled: Bool + ) { + flags[feature.rawValue] = isEnabled + } +}