Skip to content

Commit

Permalink
Add multichannel audio mixer
Browse files Browse the repository at this point in the history
  • Loading branch information
levs42 committed Mar 13, 2024
1 parent 17d67f5 commit ef87fd7
Show file tree
Hide file tree
Showing 11 changed files with 956 additions and 27 deletions.
7 changes: 3 additions & 4 deletions Examples/iOS/Screencast/SampleHandler.swift
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,12 @@ open class SampleHandler: RPBroadcastSampleHandler {
}
rtmpStream.append(sampleBuffer)
case .audioMic:
isMirophoneOn = true
if CMSampleBufferDataIsReady(sampleBuffer) {
rtmpStream.append(sampleBuffer)
rtmpStream.append(sampleBuffer, channel: 0)
}
case .audioApp:
if !isMirophoneOn && CMSampleBufferDataIsReady(sampleBuffer) {
rtmpStream.append(sampleBuffer)
if CMSampleBufferDataIsReady(sampleBuffer) {
rtmpStream.append(sampleBuffer, channel: 1)
}
@unknown default:
break
Expand Down
12 changes: 12 additions & 0 deletions HaishinKit.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@
2EC97B7227880FF400D8BE32 /* OnTapGestureView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6E27880FF400D8BE32 /* OnTapGestureView.swift */; };
2EC97B7327880FF400D8BE32 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6F27880FF400D8BE32 /* Views.swift */; };
2EC97B7427880FF400D8BE32 /* MTHKSwiftUiView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */; };
B3016D252B98FF9A0043DB39 /* AudioNode+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3016D242B98FF9A0043DB39 /* AudioNode+Extension.swift */; };
B34239852B9FD3E30068C3FB /* AudioNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = B34239842B9FD3E30068C3FB /* AudioNode.swift */; };
B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixer.swift */; };
BC0394562AA8A384006EDE38 /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; };
BC03945F2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */; };
BC04A2D42AD2D1D700C87A3E /* AVAudioTime+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */; };
Expand Down Expand Up @@ -577,6 +580,9 @@
2EC97B6E27880FF400D8BE32 /* OnTapGestureView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OnTapGestureView.swift; sourceTree = "<group>"; };
2EC97B6F27880FF400D8BE32 /* Views.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Views.swift; sourceTree = "<group>"; };
2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MTHKSwiftUiView.swift; sourceTree = "<group>"; };
B3016D242B98FF9A0043DB39 /* AudioNode+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AudioNode+Extension.swift"; sourceTree = "<group>"; };
B34239842B9FD3E30068C3FB /* AudioNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioNode.swift; sourceTree = "<group>"; };
B3D687812B80302B00E6A28E /* IOAudioMixer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOAudioMixer.swift; sourceTree = "<group>"; };
BC03945E2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ExpressibleByIntegerLiteral+ExtensionTests.swift"; sourceTree = "<group>"; };
BC04A2D32AD2D1D700C87A3E /* AVAudioTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioTime+Extension.swift"; sourceTree = "<group>"; };
BC04A2D52AD2D95500C87A3E /* CMTime+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CMTime+Extension.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1087,9 +1093,11 @@
29BDE0BD1C65BC2400D6A768 /* IO */ = {
isa = PBXGroup;
children = (
B34239842B9FD3E30068C3FB /* AudioNode.swift */,
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */,
BC959EEE296EE4190067BA97 /* ImageTransform.swift */,
BC3802132AB5E7CC001AE399 /* IOAudioCaptureUnit.swift */,
B3D687812B80302B00E6A28E /* IOAudioMixer.swift */,
BC31DBD12A653D1600C4DEA3 /* IOAudioMonitor.swift */,
BCFC51FD2AAB420700014428 /* IOAudioResampler.swift */,
BC5019C02A6D266B0046E02F /* IOAudioRingBuffer.swift */,
Expand Down Expand Up @@ -1160,6 +1168,7 @@
isa = PBXGroup;
children = (
BC4C9EAE23F2E736004A14F2 /* AudioStreamBasicDescription+Extension.swift */,
B3016D242B98FF9A0043DB39 /* AudioNode+Extension.swift */,
BC93792E2ADD76BE001097DB /* AVAudioCompressedBuffer+Extension.swift */,
1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */,
BC22EEF12AAF5D6300E3406D /* AVAudioPCMBuffer+Extension.swift */,
Expand Down Expand Up @@ -1776,6 +1785,7 @@
BCB9773F2621812800C9A649 /* AVCFormatStream.swift in Sources */,
BC83A4732403D83B006BDE06 /* VTCompressionSession+Extension.swift in Sources */,
BC4914A228DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */,
B3016D252B98FF9A0043DB39 /* AudioNode+Extension.swift in Sources */,
2915EC4D1D85BB8C00621092 /* RTMPTSocket.swift in Sources */,
BC11023E2917C35B00D48035 /* CVPixelBufferPool+Extension.swift in Sources */,
29C2631C1D0083B50098D4EF /* IOVideoUnit.swift in Sources */,
Expand All @@ -1791,6 +1801,7 @@
2999C3752071138F00892E55 /* MTHKView.swift in Sources */,
29AF3FCF1D7C744C00E41212 /* IOStream.swift in Sources */,
2958910E1EEB8D3C00CE51E1 /* FLVVideoCodec.swift in Sources */,
B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */,
BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */,
BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */,
BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */,
Expand Down Expand Up @@ -1867,6 +1878,7 @@
29B8766D1CD70AB300FC07DA /* DataConvertible.swift in Sources */,
BC570B4828E9ACC10098A12C /* IOUnit.swift in Sources */,
2976A4861D4903C300B53EF2 /* DeviceUtil.swift in Sources */,
B34239852B9FD3E30068C3FB /* AudioNode.swift in Sources */,
BC7C56BB299E595000C41A9B /* VideoCodecSettings.swift in Sources */,
29B876881CD70AE800FC07DA /* TSPacket.swift in Sources */,
BC22EEEE2AAF50F200E3406D /* Codec.swift in Sources */,
Expand Down
19 changes: 18 additions & 1 deletion Sources/Codec/AudioCodecSettings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ public struct AudioCodecSettings: Codable {
/// channelMap = [2, 3]
/// ```
public var channelMap: [Int]?
/// Specifies settings for alternative audio sources.
public var sourceSettings: [Int: AudioCodecSettings]?
/// Specifies the output format.
var format: AudioCodecSettings.Format = .aac

Expand All @@ -147,13 +149,15 @@ public struct AudioCodecSettings: Codable {
sampleRate: Float64 = 0,
channels: UInt32 = 0,
downmix: Bool = false,
channelMap: [Int]? = nil
channelMap: [Int]? = nil,
sourceSettings: [Int: AudioCodecSettings]? = nil
) {
self.bitRate = bitRate
self.sampleRate = sampleRate
self.channels = channels
self.downmix = downmix
self.channelMap = channelMap
self.sourceSettings = sourceSettings
}

func apply(_ converter: AVAudioConverter?, oldValue: AudioCodecSettings?) {
Expand All @@ -171,6 +175,19 @@ public struct AudioCodecSettings: Codable {
}
}

func makeAudioMixerSettings() -> IOAudioMixerSettings {
guard let sourceSettings else {
return IOAudioMixerSettings(defaultResamplerSettings: makeAudioResamplerSettings())
}
var resamplersSettings: [Int: IOAudioResamplerSettings] = [
0: makeAudioResamplerSettings()
]
for (source, codecSettings) in sourceSettings {
resamplersSettings[source] = codecSettings.makeAudioResamplerSettings()
}
return IOAudioMixerSettings(resamplersSettings: resamplersSettings)
}

func makeAudioResamplerSettings() -> IOAudioResamplerSettings {
return .init(
sampleRate: sampleRate,
Expand Down
161 changes: 161 additions & 0 deletions Sources/Extension/AudioNode+Extension.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
//
// AudioUnit+Extension.swift
// HaishinKit
//
// Created by Lev Sokolov on 2024-03-06.
//

import AudioUnit

extension AudioNode: CustomStringConvertible {
public var description: String {
var description: [String] = []

for scope in BusScope.allCases {
guard let busCount = try? busCount(scope: scope) else {
description.append("failed to get \(scope.rawValue) bus count")
continue
}
guard busCount > 0 else {
continue
}
var busDescription: [String] = []
for busIndex in 0..<busCount {
guard let asbd = try? format(bus: busIndex, scope: scope) else {
busDescription.append("failed to get \(scope.rawValue) bus format for bus \(busIndex)")
continue
}
if let mixerNode = self as? MixerNode, let volume = try? mixerNode.volume(bus: busIndex, of: scope) {
if scope != .input || scope == .input && (try? mixerNode.isEnabled(bus: busIndex, scope: scope)) ?? false {
busDescription.append("bus: \(busIndex), volume: \(volume), format: \(asbd)")
}
} else {
busDescription.append("bus: \(busIndex), format: \(asbd)")
}
}

description.append("\(scope.rawValue) \(busDescription.count)/\(busCount)")
description.append(busDescription.joined(separator: "; "))
}

let parametersList = (try? parameters) ?? []
if !parametersList.isEmpty {
description.append("parameters: ")
for parameter in parametersList {
description.append("\(parameter)")
}
}

return "AudioNode(\(description.joined(separator: "; ")))"
}

private var parameters: [AudioUnitParameter] {
get throws {
var result = [AudioUnitParameter]()
var status: OSStatus = noErr

var parameterListSize: UInt32 = 0
AudioUnitGetPropertyInfo(audioUnit,
kAudioUnitProperty_ParameterList,
kAudioUnitScope_Global,
0,
&parameterListSize,
nil)

let numberOfParameters = Int(parameterListSize) / MemoryLayout<AudioUnitParameterID>.size
let parameterIds = UnsafeMutablePointer<AudioUnitParameterID>.allocate(capacity: numberOfParameters)
defer { parameterIds.deallocate() }

if numberOfParameters > 0 {
status = AudioUnitGetProperty(audioUnit,
kAudioUnitProperty_ParameterList,
kAudioUnitScope_Global,
0,
parameterIds,
&parameterListSize)
guard status == noErr else {
throw AudioNodeError.unableToRetrieveValue(status)
}
}

var info = AudioUnitParameterInfo()
var infoSize = UInt32(MemoryLayout<AudioUnitParameterInfo>.size)

for i in 0..<numberOfParameters {
let id = parameterIds[i]
status = AudioUnitGetProperty(audioUnit,
kAudioUnitProperty_ParameterInfo,
kAudioUnitScope_Global,
id,
&info,
&infoSize)
guard status == noErr else {
throw AudioNodeError.unableToRetrieveValue(status)
}
result.append(AudioUnitParameter(info, id: id))
}

return result
}
}
}

private struct AudioUnitParameter: CustomStringConvertible {
var id: Int
var name: String = ""
var minValue: Float
var maxValue: Float
var defaultValue: Float
var unit: AudioUnitParameterUnit

init(_ info: AudioUnitParameterInfo, id: AudioUnitParameterID) {
self.id = Int(id)
if let cfName = info.cfNameString?.takeUnretainedValue() {
name = String(cfName)
}
minValue = info.minValue
maxValue = info.maxValue
defaultValue = info.defaultValue
unit = info.unit
}

var description: String {
return "\(name), id: \(id), min: \(minValue), max: \(maxValue), default: \(defaultValue), unit: \(unit) \(unitName)"
}

var unitName: String {
switch unit {
// swiftlint:disable switch_case_on_newline
case .generic: return "generic"
case .indexed: return "indexed"
case .boolean: return "boolean"
case .percent: return "percent"
case .seconds: return "seconds"
case .sampleFrames: return "sampleFrames"
case .phase: return "phase"
case .rate: return "rate"
case .hertz: return "hertz"
case .cents: return "cents"
case .relativeSemiTones: return "relativeSemiTones"
case .midiNoteNumber: return "midiNoteNumber"
case .midiController: return "midiController"
case .decibels: return "decibels"
case .linearGain: return "linearGain"
case .degrees: return "degrees"
case .equalPowerCrossfade: return "equalPowerCrossfade"
case .mixerFaderCurve1: return "mixerFaderCurve1"
case .pan: return "pan"
case .meters: return "meters"
case .absoluteCents: return "absoluteCents"
case .octaves: return "octaves"
case .BPM: return "BPM"
case .beats: return "beats"
case .milliseconds: return "milliseconds"
case .ratio: return "ratio"
case .customUnit: return "customUnit"
case .midi2Controller: return "midi2Controller"
default: return "unknown_\(unit)"
// swiftlint:enable switch_case_on_newline
}
}
}
Loading

0 comments on commit ef87fd7

Please sign in to comment.