Skip to content

Commit

Permalink
Merge pull request #1420 from shogo4405/feature/IOSingleAudioMixer
Browse files Browse the repository at this point in the history
Add an ON/OFF switch for the multi-track audio mixing feature.
  • Loading branch information
shogo4405 authored Apr 14, 2024
2 parents dac08bf + 1c8eeaf commit ead3c15
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 23 deletions.
3 changes: 2 additions & 1 deletion Examples/iOS/Screencast/SampleHandler.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ open class SampleHandler: RPBroadcastSampleHandler {
}()

private lazy var rtmpStream: RTMPStream = {
RTMPStream(connection: rtmpConnection)
FeatureUtil.setEnabled(feature: .multiTrackMixing, isEnabled: true)
return RTMPStream(connection: rtmpConnection)
}()

private var isMirophoneOn = false
Expand Down
24 changes: 16 additions & 8 deletions HaishinKit.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@
2EC97B7327880FF400D8BE32 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6F27880FF400D8BE32 /* Views.swift */; };
2EC97B7427880FF400D8BE32 /* MTHKSwiftUiView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */; };
B34239852B9FD3E30068C3FB /* AudioNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = B34239842B9FD3E30068C3FB /* AudioNode.swift */; };
B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixer.swift */; };
B3D687822B80302B00E6A28E /* IOAudioMixerConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */; };
BC0394562AA8A384006EDE38 /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; };
BC03945F2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */; };
BC04A2D42AD2D1D700C87A3E /* AVAudioTime+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */; };
Expand Down Expand Up @@ -161,7 +161,9 @@
BC3802192AB6AD79001AE399 /* IOAudioResamplerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3802182AB6AD79001AE399 /* IOAudioResamplerTests.swift */; };
BC3E384429C216BB007CD972 /* ADTSReaderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3E384329C216BB007CD972 /* ADTSReaderTests.swift */; };
BC4078C42AD5CC7E00BBB4FA /* IOMuxer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4078C32AD5CC7E00BBB4FA /* IOMuxer.swift */; };
BC4231642BCA5F28003A80DC /* IOAUAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */; };
BC4231642BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */; };
BC42316A2BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */; };
BC42316C2BCB7084003A80DC /* FeatureUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC42316B2BCB7084003A80DC /* FeatureUtil.swift */; };
BC4914A228DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */; };
BC4914A628DDD367009E2DF6 /* VTSessionOption.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A528DDD367009E2DF6 /* VTSessionOption.swift */; };
BC4914AE28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */; };
Expand Down Expand Up @@ -556,7 +558,7 @@
2EC97B6F27880FF400D8BE32 /* Views.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Views.swift; sourceTree = "<group>"; };
2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MTHKSwiftUiView.swift; sourceTree = "<group>"; };
B34239842B9FD3E30068C3FB /* AudioNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioNode.swift; sourceTree = "<group>"; };
B3D687812B80302B00E6A28E /* IOAudioMixer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOAudioMixer.swift; sourceTree = "<group>"; };
B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertible.swift; sourceTree = "<group>"; };
BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ExpressibleByIntegerLiteral+ExtensionTests.swift"; sourceTree = "<group>"; };
BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioTime+Extension.swift"; sourceTree = "<group>"; };
BC04A2D52AD2D95500C87A3E /* CMTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CMTime+Extension.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -595,7 +597,9 @@
BC3802182AB6AD79001AE399 /* IOAudioResamplerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioResamplerTests.swift; sourceTree = "<group>"; };
BC3E384329C216BB007CD972 /* ADTSReaderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ADTSReaderTests.swift; sourceTree = "<group>"; };
BC4078C32AD5CC7E00BBB4FA /* IOMuxer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOMuxer.swift; sourceTree = "<group>"; };
BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAUAudioMixer.swift; sourceTree = "<group>"; };
BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertibleByMultiTrack.swift; sourceTree = "<group>"; };
BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioMixerConvertibleBySingleTrack.swift; sourceTree = "<group>"; };
BC42316B2BCB7084003A80DC /* FeatureUtil.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeatureUtil.swift; sourceTree = "<group>"; };
BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionConvertible.swift; sourceTree = "<group>"; };
BC4914A528DDD367009E2DF6 /* VTSessionOption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionOption.swift; sourceTree = "<group>"; };
BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VTDecompressionSession+Extension.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -805,6 +809,7 @@
BC0D236C26331BAB001DDA0C /* DataBuffer.swift */,
29B876671CD70AB300FC07DA /* DataConvertible.swift */,
2976A4851D4903C300B53EF2 /* DeviceUtil.swift */,
BC42316B2BCB7084003A80DC /* FeatureUtil.swift */,
BC32E88729C9971100051507 /* InstanceHolder.swift */,
2942424C1CF4C01300D65DCB /* MD5.swift */,
2942A4F721A9418A004E1BEE /* Running.swift */,
Expand Down Expand Up @@ -1059,9 +1064,10 @@
B34239842B9FD3E30068C3FB /* AudioNode.swift */,
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */,
BC959EEE296EE4190067BA97 /* ImageTransform.swift */,
BC4231632BCA5F28003A80DC /* IOAUAudioMixer.swift */,
BC3802132AB5E7CC001AE399 /* IOAudioCaptureUnit.swift */,
B3D687812B80302B00E6A28E /* IOAudioMixer.swift */,
B3D687812B80302B00E6A28E /* IOAudioMixerConvertible.swift */,
BC4231632BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift */,
BC4231692BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift */,
BC31DBD12A653D1600C4DEA3 /* IOAudioMonitor.swift */,
BCFC51FD2AAB420700014428 /* IOAudioResampler.swift */,
BC5019C02A6D266B0046E02F /* IOAudioRingBuffer.swift */,
Expand Down Expand Up @@ -1749,7 +1755,7 @@
29B8765D1CD70A7900FC07DA /* VideoCodec.swift in Sources */,
2999C3752071138F00892E55 /* MTHKView.swift in Sources */,
29AF3FCF1D7C744C00E41212 /* IOStream.swift in Sources */,
B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */,
B3D687822B80302B00E6A28E /* IOAudioMixerConvertible.swift in Sources */,
BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */,
29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */,
29B876841CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift in Sources */,
Expand Down Expand Up @@ -1786,9 +1792,10 @@
BC0F1FDA2ACC4CC100C326FF /* IOCaptureVideoPreview.swift in Sources */,
BC4914B228DDFE31009E2DF6 /* VTSessionOptionKey.swift in Sources */,
BC7C56CD29A786AE00C41A9B /* ADTS.swift in Sources */,
BC42316A2BCA8BE5003A80DC /* IOAudioMixerConvertibleBySingleTrack.swift in Sources */,
BC562DCB29576D220048D89A /* AVCaptureSession.Preset+Extension.swift in Sources */,
29B876AB1CD70B2800FC07DA /* AMF0Serializer.swift in Sources */,
BC4231642BCA5F28003A80DC /* IOAUAudioMixer.swift in Sources */,
BC4231642BCA5F28003A80DC /* IOAudioMixerConvertibleByMultiTrack.swift in Sources */,
29B8765B1CD70A7900FC07DA /* AudioCodec.swift in Sources */,
29EA87D51E799F670043A5F8 /* Mirror+Extension.swift in Sources */,
BCC4F4152AD6FC1100954EF5 /* IOTellyUnit.swift in Sources */,
Expand All @@ -1798,6 +1805,7 @@
BC562DC7295767860048D89A /* AVCaptureDevice+Extension.swift in Sources */,
BC0F1FDC2ACC630400C326FF /* NSView+Extension.swift in Sources */,
29EA87E21E79A1E90043A5F8 /* CMVideoFormatDescription+Extension.swift in Sources */,
BC42316C2BCB7084003A80DC /* FeatureUtil.swift in Sources */,
BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */,
BC1DC4A429F4F74F00E928ED /* AVCaptureSession+Extension.swift in Sources */,
29EA87D81E79A0090043A5F8 /* URL+Extension.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ struct IOAudioMixerSettings {
}

protocol IOAudioMixerConvertible: AnyObject {
var delegate: (any IOAudioMixerDelegate)? { get set }
var inputFormat: AVAudioFormat? { get }
var settings: IOAudioMixerSettings { get set }

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import AVFoundation
import Foundation

final class IOAUAudioMixer: IOAudioMixerConvertible {
final class IOAudioMixerConvertibleByMultiTrack: IOAudioMixerConvertible {
private static let defaultSampleTime: AVAudioFramePosition = 0

private class Track {
let resampler: IOAudioResampler<IOAUAudioMixer>
let resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>
var ringBuffer: IOAudioRingBuffer?

init(resampler: IOAudioResampler<IOAUAudioMixer>, format: AVAudioFormat? = nil) {
init(resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>, format: AVAudioFormat? = nil) {
self.resampler = resampler
if let format {
self.ringBuffer = .init(format)
Expand Down Expand Up @@ -42,15 +42,15 @@ final class IOAUAudioMixer: IOAudioMixerConvertible {
numberOfTracks > 1
}
private var anchor: AVAudioTime?
private var sampleTime: AVAudioFramePosition = IOAUAudioMixer.defaultSampleTime
private var sampleTime: AVAudioFramePosition = IOAudioMixerConvertibleByMultiTrack.defaultSampleTime
private var mixerNode: MixerNode?
private var outputNode: OutputNode?
private var defaultTrack: Track? {
tracks[settings.mainTrack]
}

private let inputRenderCallback: AURenderCallback = { (inRefCon: UnsafeMutableRawPointer, _: UnsafeMutablePointer<AudioUnitRenderActionFlags>, _: UnsafePointer<AudioTimeStamp>, inBusNumber: UInt32, inNumberFrames: UInt32, ioData: UnsafeMutablePointer<AudioBufferList>?) in
let audioMixer = Unmanaged<IOAUAudioMixer>.fromOpaque(inRefCon).takeUnretainedValue()
let audioMixer = Unmanaged<IOAudioMixerConvertibleByMultiTrack>.fromOpaque(inRefCon).takeUnretainedValue()
let status = audioMixer.provideInput(inNumberFrames, channel: Int(inBusNumber), ioData: ioData)
guard status == noErr else {
audioMixer.delegate?.audioMixer(audioMixer, errorOccurred: .failedToMix(error: IOAudioMixerError.unableToProvideInputData))
Expand Down Expand Up @@ -90,7 +90,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible {
}

private func makeTrack(channel: Int) -> Track {
let resampler = IOAudioResampler<IOAUAudioMixer>()
let resampler = IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>()
resampler.channel = channel
if channel == settings.mainTrack {
resampler.settings = settings.defaultResamplerSettings
Expand Down Expand Up @@ -201,7 +201,7 @@ final class IOAUAudioMixer: IOAudioMixerConvertible {
}
}

private func applySettings(resampler: IOAudioResampler<IOAUAudioMixer>,
private func applySettings(resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>,
defaultFormat: AVAudioFormat?,
preferredSettings: IOAudioResamplerSettings?) {
let preferredSettings = preferredSettings ?? .init()
Expand All @@ -218,9 +218,9 @@ final class IOAUAudioMixer: IOAudioMixerConvertible {
}
}

extension IOAUAudioMixer: IOAudioResamplerDelegate {
extension IOAudioMixerConvertibleByMultiTrack: IOAudioResamplerDelegate {
// MARK: IOAudioResamplerDelegate
func resampler(_ resampler: IOAudioResampler<IOAUAudioMixer>, didOutput audioFormat: AVAudioFormat) {
func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>, didOutput audioFormat: AVAudioFormat) {
guard shouldMix else {
delegate?.audioMixer(self, didOutput: audioFormat)
return
Expand All @@ -233,7 +233,7 @@ extension IOAUAudioMixer: IOAudioResamplerDelegate {
track(channel: resampler.channel).ringBuffer = .init(audioFormat)
}

func resampler(_ resampler: IOAudioResampler<IOAUAudioMixer>, didOutput audioBuffer: AVAudioPCMBuffer, when: AVAudioTime) {
func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>, didOutput audioBuffer: AVAudioPCMBuffer, when: AVAudioTime) {
guard shouldMix else {
delegate?.audioMixer(self, didOutput: audioBuffer, when: when)
return
Expand All @@ -255,7 +255,7 @@ extension IOAUAudioMixer: IOAudioResamplerDelegate {
}
}

func resampler(_ resampler: IOAudioResampler<IOAUAudioMixer>, errorOccurred error: IOAudioUnitError) {
func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleByMultiTrack>, errorOccurred error: IOAudioUnitError) {
delegate?.audioMixer(self, errorOccurred: error)
}
}
43 changes: 43 additions & 0 deletions Sources/IO/IOAudioMixerConvertibleBySingleTrack.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import AVFoundation
import Foundation

final class IOAudioMixerConvertibleBySingleTrack: IOAudioMixerConvertible {
var delegate: (any IOAudioMixerDelegate)?
var inputFormat: AVAudioFormat?
var settings: IOAudioMixerSettings = .init()

private lazy var resampler: IOAudioResampler<IOAudioMixerConvertibleBySingleTrack> = {
var resampler = IOAudioResampler<IOAudioMixerConvertibleBySingleTrack>()
resampler.delegate = self
return resampler
}()

func append(_ buffer: CMSampleBuffer, track: UInt8) {
guard settings.mainTrack == track else {
return
}
resampler.append(buffer)
}

func append(_ buffer: AVAudioPCMBuffer, when: AVAudioTime, track: UInt8) {
guard settings.mainTrack == track else {
return
}
resampler.append(buffer, when: when)
}
}

extension IOAudioMixerConvertibleBySingleTrack: IOAudioResamplerDelegate {
// MARK: IOAudioResamplerDelegate
func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleBySingleTrack>, didOutput audioFormat: AVAudioFormat) {
delegate?.audioMixer(self, didOutput: audioFormat)
}

func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleBySingleTrack>, didOutput audioPCMBuffer: AVAudioPCMBuffer, when: AVAudioTime) {
delegate?.audioMixer(self, didOutput: audioPCMBuffer, when: when)
}

func resampler(_ resampler: IOAudioResampler<IOAudioMixerConvertibleBySingleTrack>, errorOccurred error: IOAudioUnitError) {
delegate?.audioMixer(self, errorOccurred: error)
}
}
12 changes: 9 additions & 3 deletions Sources/IO/IOAudioUnit.swift
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,15 @@ final class IOAudioUnit: NSObject, IOUnit {
return codec
}()
private lazy var audioMixer: any IOAudioMixerConvertible = {
var audioMixer = IOAUAudioMixer()
audioMixer.delegate = self
return audioMixer
if FeatureUtil.isEnabled(feature: .multiTrackMixing) {
var audioMixer = IOAudioMixerConvertibleByMultiTrack()
audioMixer.delegate = self
return audioMixer
} else {
var audioMixer = IOAudioMixerConvertibleBySingleTrack()
audioMixer.delegate = self
return audioMixer
}
}()
private var monitor: IOAudioMonitor = .init()
#if os(tvOS)
Expand Down
43 changes: 43 additions & 0 deletions Sources/Util/FeatureUtil.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import Foundation

/// The util object to get feature flag info.
public enum FeatureUtil {
/// A structure that defines the name of a feature.
public struct Name: RawRepresentable, ExpressibleByStringLiteral {
// swiftlint:disable:next nesting
public typealias RawValue = String
// swiftlint:disable:next nesting
public typealias StringLiteralType = String

/// This is a feature to mix multiple audio tracks. For example, it is possible to mix .appAudio and .micAudio from ReplayKit.
public static let multiTrackMixing: Name = "multiTrackMixing"

/// The raw type value.
public let rawValue: String

/// Create a feature name by rawValue.
public init(rawValue: String) {
self.rawValue = rawValue
}

/// Create a feature name by stringLiteral.
public init(stringLiteral value: String) {
self.rawValue = value
}
}

private static var flags: [String: Bool] = [:]

/// Whether or not a flag is enabled.
public static func isEnabled(feature: Name) -> Bool {
return flags[feature.rawValue] ?? false
}

/// Setter for a feature flag.
public static func setEnabled(
feature: Name,
isEnabled: Bool
) {
flags[feature.rawValue] = isEnabled
}
}

0 comments on commit ead3c15

Please sign in to comment.