diff --git a/Examples/iOS/LiveViewController.swift b/Examples/iOS/LiveViewController.swift index fbe73178b..e79dcfd9b 100644 --- a/Examples/iOS/LiveViewController.swift +++ b/Examples/iOS/LiveViewController.swift @@ -27,20 +27,19 @@ final class LiveViewController: UIViewController { private var currentEffect: VideoEffect? private var currentPosition: AVCaptureDevice.Position = .back private var retryCount: Int = 0 - private var videoBitRate = VideoCodecSettings.default.bitRate private var preferedStereo = false override func viewDidLoad() { super.viewDidLoad() - rtmpConnection.delegate = self - pipIntentView.layer.borderWidth = 1.0 pipIntentView.layer.borderColor = UIColor.white.cgColor pipIntentView.bounds = MultiCamCaptureSettings.default.regionOfInterest pipIntentView.isUserInteractionEnabled = true view.addSubview(pipIntentView) + rtmpConnection.delegate = self + rtmpStream = RTMPStream(connection: rtmpConnection) if let orientation = DeviceUtil.videoOrientation(by: UIApplication.shared.statusBarOrientation) { rtmpStream.videoOrientation = orientation @@ -62,7 +61,7 @@ final class LiveViewController: UIViewController { allowFrameReordering: nil, isHardwareEncoderEnabled: true ) - + rtmpStream.bitrateStrategy = VideoAdaptiveNetBitRateStrategy(mamimumVideoBitrate: VideoCodecSettings.default.bitRate) rtmpStream.mixer.recorder.delegate = self videoBitrateSlider?.value = Float(VideoCodecSettings.default.bitRate) / 1000 audioBitrateSlider?.value = Float(AudioCodecSettings.default.bitRate) / 1000 @@ -163,7 +162,7 @@ final class LiveViewController: UIViewController { } if slider == videoBitrateSlider { videoBitrateLabel?.text = "video \(Int(slider.value))/kbps" - rtmpStream.videoSettings.bitRate = UInt32(slider.value * 1000) + rtmpStream.bitrateStrategy = VideoAdaptiveNetBitRateStrategy(mamimumVideoBitrate: Int(slider.value * 1000)) } if slider == zoomSlider { let zoomFactor = CGFloat(slider.value) @@ -348,21 +347,15 @@ final class LiveViewController: UIViewController { extension LiveViewController: RTMPConnectionDelegate { func connection(_ connection: RTMPConnection, publishInsufficientBWOccured stream: RTMPStream) { - // Adaptive bitrate streaming exsample. Please feedback me your good algorithm. :D - videoBitRate -= 32 * 1000 - stream.videoSettings.bitRate = max(videoBitRate, 64 * 1000) } func connection(_ connection: RTMPConnection, publishSufficientBWOccured stream: RTMPStream) { - videoBitRate += 32 * 1000 - stream.videoSettings.bitRate = min(videoBitRate, VideoCodecSettings.default.bitRate) } func connection(_ connection: RTMPConnection, updateStats stream: RTMPStream) { } func connection(_ connection: RTMPConnection, didClear stream: RTMPStream) { - videoBitRate = VideoCodecSettings.default.bitRate } } diff --git a/Examples/iOS/PlaybackViewController.swift b/Examples/iOS/PlaybackViewController.swift index 2a6d009e3..0909d1802 100644 --- a/Examples/iOS/PlaybackViewController.swift +++ b/Examples/iOS/PlaybackViewController.swift @@ -141,10 +141,6 @@ extension PlaybackViewController: NetStreamDelegate { func stream(_ stream: NetStream, audioCodecErrorOccurred error: HaishinKit.AudioCodec.Error) { } - func streamWillDropFrame(_ stream: NetStream) -> Bool { - return false - } - func streamDidOpen(_ stream: NetStream) { } } diff --git a/Examples/iOS/VideoAdaptiveNetBitRateStrategy.swift b/Examples/iOS/VideoAdaptiveNetBitRateStrategy.swift new file mode 100644 index 000000000..421582e88 --- /dev/null +++ b/Examples/iOS/VideoAdaptiveNetBitRateStrategy.swift @@ -0,0 +1,39 @@ +import Foundation + +public final class VideoAdaptiveNetBitRateStrategy: NetBitRateStrategyConvertible { + public weak var stream: NetStream? + public let mamimumVideoBitRate: Int + public let mamimumAudioBitRate: Int = 0 + private var zeroBytesOutPerSecondCounts: Int = 0 + + public init(mamimumVideoBitrate: Int) { + self.mamimumVideoBitRate = mamimumVideoBitrate + } + + public func setUp() { + zeroBytesOutPerSecondCounts = 0 + stream?.videoSettings.bitRate = mamimumVideoBitRate + } + + public func sufficientBWOccured(_ stats: NetBitRateStats) { + logger.info(stats) + guard let stream else { + return + } + stream.videoSettings.bitRate = min(stream.videoSettings.bitRate + 64 * 1000, mamimumVideoBitRate) + } + + public func insufficientBWOccured(_ stats: NetBitRateStats) { + logger.info(stats) + guard let stream, 0 < stats.currentBytesOutPerSecond else { + return + } + if 0 < stats.currentBytesOutPerSecond { + let bitRate = Int(stats.currentBytesOutPerSecond * 8) / (zeroBytesOutPerSecondCounts + 1) + stream.videoSettings.bitRate = max(bitRate - stream.audioSettings.bitRate, 64 * 1000) + zeroBytesOutPerSecondCounts = 0 + } else { + zeroBytesOutPerSecondCounts += 1 + } + } +} diff --git a/HaishinKit.xcodeproj/project.pbxproj b/HaishinKit.xcodeproj/project.pbxproj index a1c7aa420..828b26710 100644 --- a/HaishinKit.xcodeproj/project.pbxproj +++ b/HaishinKit.xcodeproj/project.pbxproj @@ -145,6 +145,7 @@ BC11024A2925147300D48035 /* IOCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1102492925147300D48035 /* IOCaptureUnit.swift */; }; BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; }; BC110257292E661E00D48035 /* MultiCamCaptureSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSettings.swift */; }; + BC1BC9042AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */; }; BC1DC4A429F4F74F00E928ED /* AVCaptureSession+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC4A329F4F74F00E928ED /* AVCaptureSession+Extension.swift */; }; BC1DC4FB2A02868900E928ED /* FLVVideoFourCC.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC4FA2A02868900E928ED /* FLVVideoFourCC.swift */; }; BC1DC5042A02894D00E928ED /* FLVVideoFourCCTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC5032A02894D00E928ED /* FLVVideoFourCCTests.swift */; }; @@ -189,6 +190,7 @@ BC562DCB29576D220048D89A /* AVCaptureSession.Preset+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC562DCA29576D220048D89A /* AVCaptureSession.Preset+Extension.swift */; }; BC566F6E25D2ECC500573C4C /* HLSService.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC566F6D25D2ECC500573C4C /* HLSService.swift */; }; BC570B4828E9ACC10098A12C /* IOUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC570B4728E9ACC10098A12C /* IOUnit.swift */; }; + BC6692F32AC2F717009EC058 /* NetBitRateStrategyConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */; }; BC6FC91E29609A6800A746EE /* ShapeFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6FC91D29609A6800A746EE /* ShapeFactory.swift */; }; BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6FC9212961B3D800A746EE /* vImage_CGImageFormat+Extension.swift */; }; BC701F322AAC676C00C4BEFE /* AVAudioFormatFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC701F312AAC676C00C4BEFE /* AVAudioFormatFactory.swift */; }; @@ -556,6 +558,7 @@ BC1102492925147300D48035 /* IOCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOCaptureUnit.swift; sourceTree = ""; }; BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_Buffer+Extension.swift"; sourceTree = ""; }; BC110256292E661E00D48035 /* MultiCamCaptureSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultiCamCaptureSettings.swift; sourceTree = ""; }; + BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoAdaptiveNetBitRateStrategy.swift; sourceTree = ""; }; BC1DC4A329F4F74F00E928ED /* AVCaptureSession+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+Extension.swift"; sourceTree = ""; }; BC1DC4FA2A02868900E928ED /* FLVVideoFourCC.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FLVVideoFourCC.swift; sourceTree = ""; }; BC1DC5032A02894D00E928ED /* FLVVideoFourCCTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FLVVideoFourCCTests.swift; sourceTree = ""; }; @@ -596,6 +599,7 @@ BC562DCA29576D220048D89A /* AVCaptureSession.Preset+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession.Preset+Extension.swift"; sourceTree = ""; }; BC566F6D25D2ECC500573C4C /* HLSService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HLSService.swift; sourceTree = ""; }; BC570B4728E9ACC10098A12C /* IOUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOUnit.swift; sourceTree = ""; }; + BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetBitRateStrategyConvertible.swift; sourceTree = ""; }; BC6FC91D29609A6800A746EE /* ShapeFactory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShapeFactory.swift; sourceTree = ""; }; BC6FC9212961B3D800A746EE /* vImage_CGImageFormat+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_CGImageFormat+Extension.swift"; sourceTree = ""; }; BC701F312AAC676C00C4BEFE /* AVAudioFormatFactory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVAudioFormatFactory.swift; sourceTree = ""; }; @@ -965,7 +969,6 @@ 2968973F1CDB01AD0074D5F0 /* iOS */ = { isa = PBXGroup; children = ( - 29A39C801D85BEFA007C27E9 /* Screencast */, 296897411CDB01D20074D5F0 /* AppDelegate.swift */, 296897421CDB01D20074D5F0 /* Assets.xcassets */, 291F4E361CF206E200F59C51 /* Icon.png */, @@ -977,6 +980,8 @@ BCFB355324FA275600DC5108 /* PlaybackViewController.swift */, 291468161E581C7D00E619BA /* Preference.swift */, 2950742E1E4620B7007F15A4 /* PreferenceViewController.swift */, + 29A39C801D85BEFA007C27E9 /* Screencast */, + BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */, 296897461CDB01D20074D5F0 /* VisualEffect.swift */, ); path = iOS; @@ -1005,6 +1010,7 @@ isa = PBXGroup; children = ( 29B876971CD70B1100FC07DA /* MIME.swift */, + BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */, 29B876981CD70B1100FC07DA /* NetClient.swift */, 29B876991CD70B1100FC07DA /* NetService.swift */, 29B8769A1CD70B1100FC07DA /* NetSocket.swift */, @@ -1708,6 +1714,7 @@ BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */, BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */, 299B13271D3B751400A1E8F5 /* HKView.swift in Sources */, + BC1BC9042AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift in Sources */, BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */, 29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */, 29B876841CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift in Sources */, @@ -1789,6 +1796,7 @@ 295891261EEB8EF300CE51E1 /* FLVAACPacket.swift in Sources */, 29B876791CD70ACE00FC07DA /* HTTPStream.swift in Sources */, BC1DC50A2A039B4400E928ED /* HEVCDecoderConfigurationRecord.swift in Sources */, + BC6692F32AC2F717009EC058 /* NetBitRateStrategyConvertible.swift in Sources */, BC6FC91E29609A6800A746EE /* ShapeFactory.swift in Sources */, BC32E88829C9971100051507 /* InstanceHolder.swift in Sources */, BC7C56B7299E579F00C41A9B /* AudioCodecSettings.swift in Sources */, diff --git a/README.md b/README.md index 828a09b96..5090ff799 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ Project name |Notes |License - [x] Authentication - [x] Publish and Recording - [x] _Playback (Beta)_ -- [x] Adaptive bitrate streaming - - [x] Handling (see also [#1153](/../../issues/1153)) +- [x] Adaptive bitrate streaming (see also [#1308](/../../issues/1308)) - [ ] Action Message Format - [x] AMF0 - [ ] AMF3 diff --git a/Sources/Codec/VTSessionMode.swift b/Sources/Codec/VTSessionMode.swift index 72d21aa62..9693e13b8 100644 --- a/Sources/Codec/VTSessionMode.swift +++ b/Sources/Codec/VTSessionMode.swift @@ -35,6 +35,7 @@ enum VTSessionMode { videoCodec.delegate?.videoCodec(videoCodec, errorOccurred: .failedToPrepare(status: status)) return nil } + videoCodec.frameInterval = videoCodec.settings.frameInterval return session case .decompression: guard let formatDescription = videoCodec.outputFormat else { diff --git a/Sources/Codec/VideoCodec.swift b/Sources/Codec/VideoCodec.swift index 51cf2ad69..f5034f708 100644 --- a/Sources/Codec/VideoCodec.swift +++ b/Sources/Codec/VideoCodec.swift @@ -1,8 +1,7 @@ import AVFoundation import CoreFoundation import VideoToolbox - -#if os(iOS) +#if canImport(UIKit) import UIKit #endif @@ -16,15 +15,15 @@ public protocol VideoCodecDelegate: AnyObject { func videoCodec(_ codec: VideoCodec, didOutput sampleBuffer: CMSampleBuffer) /// Tells the receiver to occured an error. func videoCodec(_ codec: VideoCodec, errorOccurred error: VideoCodec.Error) - /// Tells the receiver to drop frame. - func videoCodecWillDropFame(_ codec: VideoCodec) -> Bool } // MARK: - /** * The VideoCodec class provides methods for encode or decode for video. */ -public class VideoCodec { +public final class VideoCodec { + static let defaultFrameInterval = 0.0 + /** * The VideoCodec error domain codes. */ @@ -61,15 +60,6 @@ public class VideoCodec { public private(set) var isRunning: Atomic = .init(false) var lockQueue = DispatchQueue(label: "com.haishinkit.HaishinKit.VideoCodec.lock") - var expectedFrameRate = IOMixer.defaultFrameRate - private(set) var outputFormat: CMFormatDescription? { - didSet { - guard !CMFormatDescriptionEqual(outputFormat, otherFormatDescription: oldValue) else { - return - } - delegate?.videoCodec(self, didOutput: outputFormat) - } - } var needsSync: Atomic = .init(true) var attributes: [NSString: AnyObject]? { guard VideoCodec.defaultAttributes != nil else { @@ -83,7 +73,17 @@ public class VideoCodec { attributes[kCVPixelBufferHeightKey] = NSNumber(value: settings.videoSize.height) return attributes } + var frameInterval = VideoCodec.defaultFrameInterval + var expectedFrameRate = IOMixer.defaultFrameRate weak var delegate: (any VideoCodecDelegate)? + private(set) var outputFormat: CMFormatDescription? { + didSet { + guard !CMFormatDescriptionEqual(outputFormat, otherFormatDescription: oldValue) else { + return + } + delegate?.videoCodec(self, didOutput: outputFormat) + } + } private(set) var session: (any VTSessionConvertible)? { didSet { oldValue?.invalidate() @@ -91,9 +91,10 @@ public class VideoCodec { } } private var invalidateSession = true + private var presentationTimeStamp: CMTime = .invalid func appendImageBuffer(_ imageBuffer: CVImageBuffer, presentationTimeStamp: CMTime, duration: CMTime) { - guard isRunning.value, !(delegate?.videoCodecWillDropFame(self) ?? false) else { + guard isRunning.value, !willDropFrame(presentationTimeStamp) else { return } if invalidateSession { @@ -108,6 +109,7 @@ public class VideoCodec { delegate?.videoCodec(self, errorOccurred: .failedToFlame(status: status)) return } + self.presentationTimeStamp = sampleBuffer.presentationTimeStamp outputFormat = sampleBuffer.formatDescription delegate?.videoCodec(self, didOutput: sampleBuffer) } @@ -163,7 +165,15 @@ public class VideoCodec { } } - #if os(iOS) + private func willDropFrame(_ presentationTimeStamp: CMTime) -> Bool { + guard Self.defaultFrameInterval < frameInterval else { + return false + } + print(presentationTimeStamp.seconds - self.presentationTimeStamp.seconds <= frameInterval) + return presentationTimeStamp.seconds - self.presentationTimeStamp.seconds <= frameInterval + } + + #if os(iOS) || os(tvOS) || os(visionOS) @objc private func applicationWillEnterForeground(_ notification: Notification) { invalidateSession = true @@ -192,7 +202,7 @@ extension VideoCodec: Running { public func startRunning() { lockQueue.async { self.isRunning.mutate { $0 = true } - #if os(iOS) + #if os(iOS) || os(tvOS) || os(visionOS) NotificationCenter.default.addObserver( self, selector: #selector(self.didAudioSessionInterruption), @@ -215,7 +225,8 @@ extension VideoCodec: Running { self.invalidateSession = true self.needsSync.mutate { $0 = true } self.outputFormat = nil - #if os(iOS) + self.presentationTimeStamp = .invalid + #if os(iOS) || os(tvOS) || os(visionOS) NotificationCenter.default.removeObserver(self, name: AVAudioSession.interruptionNotification, object: nil) NotificationCenter.default.removeObserver(self, name: UIApplication.willEnterForegroundNotification, object: nil) #endif diff --git a/Sources/Codec/VideoCodecSettings.swift b/Sources/Codec/VideoCodecSettings.swift index 21489619d..605908044 100644 --- a/Sources/Codec/VideoCodecSettings.swift +++ b/Sources/Codec/VideoCodecSettings.swift @@ -3,6 +3,11 @@ import VideoToolbox /// The VideoCodecSettings class specifying video compression settings. public struct VideoCodecSettings: Codable { + public static let frameInterval30 = (1 / 30) - 0.001 + public static let frameInterval10 = (1 / 10) - 0.001 + public static let frameInterval05 = (1 / 05) - 0.001 + public static let frameInterval01 = (1 / 01) - 0.001 + /// The defulat value. public static let `default` = VideoCodecSettings() @@ -76,7 +81,9 @@ public struct VideoCodecSettings: Codable { /// Specifies the video size of encoding video. public var videoSize: CGSize /// Specifies the bitrate. - public var bitRate: UInt32 + public var bitRate: Int + /// Specifies the video frame interval. + public var frameInterval: Double /// Specifies the keyframeInterval. public var maxKeyFrameIntervalDuration: Int32 /// Specifies the scalingMode. @@ -104,7 +111,8 @@ public struct VideoCodecSettings: Codable { public init( videoSize: CGSize = .init(width: 854, height: 480), profileLevel: String = kVTProfileLevel_H264_Baseline_3_1 as String, - bitRate: UInt32 = 640 * 1000, + bitRate: Int = 640 * 1000, + frameInterval: Double = 0.0, maxKeyFrameIntervalDuration: Int32 = 2, scalingMode: ScalingMode = .trim, bitRateMode: BitRateMode = .average, @@ -114,6 +122,7 @@ public struct VideoCodecSettings: Codable { self.videoSize = videoSize self.profileLevel = profileLevel self.bitRate = bitRate + self.frameInterval = frameInterval self.maxKeyFrameIntervalDuration = maxKeyFrameIntervalDuration self.scalingMode = scalingMode self.bitRateMode = bitRateMode @@ -142,6 +151,9 @@ public struct VideoCodecSettings: Codable { codec.delegate?.videoCodec(codec, errorOccurred: .failedToSetOption(status: status, option: option)) } } + if frameInterval != rhs.frameInterval { + codec.frameInterval = frameInterval + } } func options(_ codec: VideoCodec) -> Set { diff --git a/Sources/MPEG/TSWriter.swift b/Sources/MPEG/TSWriter.swift index 6ca309608..42e84616c 100644 --- a/Sources/MPEG/TSWriter.swift +++ b/Sources/MPEG/TSWriter.swift @@ -289,10 +289,6 @@ extension TSWriter: VideoCodecDelegate { public func videoCodec(_ codec: VideoCodec, errorOccurred error: VideoCodec.Error) { } - - public func videoCodecWillDropFame(_ codec: VideoCodec) -> Bool { - return false - } } class TSFileWriter: TSWriter { diff --git a/Sources/Media/IOAudioUnit.swift b/Sources/Media/IOAudioUnit.swift index 9bc7adf97..83704a53f 100644 --- a/Sources/Media/IOAudioUnit.swift +++ b/Sources/Media/IOAudioUnit.swift @@ -102,7 +102,7 @@ final class IOAudioUnit: NSObject, IOUnit { guard var audioStreamBasicDescription else { return } - let status = CMAudioFormatDescriptionCreate( + CMAudioFormatDescriptionCreate( allocator: kCFAllocatorDefault, asbd: &audioStreamBasicDescription, layoutSize: 0, diff --git a/Sources/Net/NetBitRateStrategyConvertible.swift b/Sources/Net/NetBitRateStrategyConvertible.swift new file mode 100644 index 000000000..bdf67ef2e --- /dev/null +++ b/Sources/Net/NetBitRateStrategyConvertible.swift @@ -0,0 +1,37 @@ +import Foundation + +/// A structure that represents a NetStream's bitRate statics. +public struct NetBitRateStats { + public let currentQueueBytesOut: Int64 + public let currentBytesInPerSecond: Int32 + public let currentBytesOutPerSecond: Int32 +} + +/// A type with a NetStream's bitrate strategy representation. +public protocol NetBitRateStrategyConvertible: AnyObject { + var stream: NetStream? { get set } + var mamimumVideoBitRate: Int { get } + var mamimumAudioBitRate: Int { get } + + func setUp() + func sufficientBWOccured(_ stats: NetBitRateStats) + func insufficientBWOccured(_ stats: NetBitRateStats) +} + +/// The NetBitRateStrategy class provides a no operative bitrate storategy. +public final class NetBitRateStrategy: NetBitRateStrategyConvertible { + public static let shared = NetBitRateStrategy() + + public weak var stream: NetStream? + public let mamimumVideoBitRate: Int = 0 + public let mamimumAudioBitRate: Int = 0 + + public func setUp() { + } + + public func sufficientBWOccured(_ stats: NetBitRateStats) { + } + + public func insufficientBWOccured(_ stats: NetBitRateStats) { + } +} diff --git a/Sources/Net/NetStream.swift b/Sources/Net/NetStream.swift index c5516a9d7..3e051e609 100644 --- a/Sources/Net/NetStream.swift +++ b/Sources/Net/NetStream.swift @@ -24,8 +24,6 @@ public protocol NetStreamDelegate: AnyObject { func stream(_ stream: NetStream, videoCodecErrorOccurred error: VideoCodec.Error) /// Tells the receiver to audio codec error occured. func stream(_ stream: NetStream, audioCodecErrorOccurred error: AudioCodec.Error) - /// Tells the receiver to will drop video frame. - func streamWillDropFrame(_ stream: NetStream) -> Bool /// Tells the receiver to the stream opened. func streamDidOpen(_ stream: NetStream) } @@ -42,7 +40,15 @@ open class NetStream: NSObject { return mixer }() - /// Specifies the delegate of the NetStream. + /// Specifies the adaptibe bitrate strategy. + public var bitrateStrategy: any NetBitRateStrategyConvertible = NetBitRateStrategy.shared { + didSet { + bitrateStrategy.stream = self + bitrateStrategy.setUp() + } + } + + /// Specifies the delegate.. public weak var delegate: (any NetStreamDelegate)? /// Specifies the audio monitoring enabled or not. @@ -116,8 +122,8 @@ open class NetStream: NSObject { } #endif - /// Specifies the video orientation for stream. #if os(iOS) || os(macOS) + /// Specifies the video orientation for stream. public var videoOrientation: AVCaptureVideoOrientation { get { mixer.videoIO.videoOrientation diff --git a/Sources/RTMP/RTMPConnection.swift b/Sources/RTMP/RTMPConnection.swift index 9472dbdf0..5a086d292 100644 --- a/Sources/RTMP/RTMPConnection.swift +++ b/Sources/RTMP/RTMPConnection.swift @@ -450,11 +450,12 @@ public class RTMPConnection: EventDispatcher { private func on(timer: Timer) { let totalBytesIn = self.totalBytesIn let totalBytesOut = self.totalBytesOut + let queueBytesOut = self.socket.queueBytesOut.value currentBytesInPerSecond = Int32(totalBytesIn - previousTotalBytesIn) currentBytesOutPerSecond = Int32(totalBytesOut - previousTotalBytesOut) previousTotalBytesIn = totalBytesIn previousTotalBytesOut = totalBytesOut - previousQueueBytesOut.append(socket.queueBytesOut.value) + previousQueueBytesOut.append(queueBytesOut) for stream in streams { stream.on(timer: timer) } @@ -465,10 +466,20 @@ public class RTMPConnection: EventDispatcher { } if total == measureInterval - 1 { for stream in streams { + stream.bitrateStrategy.insufficientBWOccured(NetBitRateStats( + currentQueueBytesOut: queueBytesOut, + currentBytesInPerSecond: currentBytesInPerSecond, + currentBytesOutPerSecond: currentBytesOutPerSecond + )) delegate?.connection(self, publishInsufficientBWOccured: stream) } } else if total == 0 { for stream in streams { + stream.bitrateStrategy.sufficientBWOccured(NetBitRateStats( + currentQueueBytesOut: queueBytesOut, + currentBytesInPerSecond: currentBytesInPerSecond, + currentBytesOutPerSecond: currentBytesOutPerSecond + )) delegate?.connection(self, publishSufficientBWOccured: stream) } } diff --git a/Sources/RTMP/RTMPMuxer.swift b/Sources/RTMP/RTMPMuxer.swift index 9054b553e..3253fcade 100644 --- a/Sources/RTMP/RTMPMuxer.swift +++ b/Sources/RTMP/RTMPMuxer.swift @@ -5,7 +5,6 @@ protocol RTMPMuxerDelegate: AnyObject { func muxer(_ muxer: RTMPMuxer, didOutputVideo buffer: Data, withTimestamp: Double) func muxer(_ muxer: RTMPMuxer, audioCodecErrorOccurred error: AudioCodec.Error) func muxer(_ muxer: RTMPMuxer, videoCodecErrorOccurred error: VideoCodec.Error) - func muxerWillDropFrame(_ muxer: RTMPMuxer) -> Bool } // MARK: - @@ -108,10 +107,6 @@ extension RTMPMuxer: VideoCodecDelegate { videoTimeStamp = decodeTimeStamp } - func videoCodecWillDropFame(_ codec: VideoCodec) -> Bool { - return delegate?.muxerWillDropFrame(self) ?? false - } - private func getCompositionTime(_ sampleBuffer: CMSampleBuffer) -> Int32 { guard sampleBuffer.decodeTimeStamp.isValid, sampleBuffer.decodeTimeStamp != sampleBuffer.presentationTimeStamp else { return 0 diff --git a/Sources/RTMP/RTMPStream.swift b/Sources/RTMP/RTMPStream.swift index c7129385f..8f1984a89 100644 --- a/Sources/RTMP/RTMPStream.swift +++ b/Sources/RTMP/RTMPStream.swift @@ -512,6 +512,7 @@ open class RTMPStream: NetStream { mixer.delegate = self mixer.startDecoding() case .publish: + bitrateStrategy.setUp() startedAt = .init() muxer.dispose() muxer.delegate = self @@ -636,8 +637,4 @@ extension RTMPStream: RTMPMuxerDelegate { func muxer(_ muxer: RTMPMuxer, audioCodecErrorOccurred error: AudioCodec.Error) { delegate?.stream(self, audioCodecErrorOccurred: error) } - - func muxerWillDropFrame(_ muxer: RTMPMuxer) -> Bool { - return delegate?.streamWillDropFrame(self) ?? false - } } diff --git a/Tests/Media/IOMixerTests.swift b/Tests/Media/IOMixerTests.swift index 7c8418f91..2d5e12e95 100644 --- a/Tests/Media/IOMixerTests.swift +++ b/Tests/Media/IOMixerTests.swift @@ -8,8 +8,8 @@ final class IOMixerTests: XCTestCase { weak var weakIOMixer: IOMixer? _ = { let mixer = IOMixer() - mixer.audioIO.codec.settings.bitRate = 100000 - mixer.videoIO.codec.settings.bitRate = 100000 + mixer.audioIO.settings.bitRate = 100000 + mixer.videoIO.settings.bitRate = 100000 weakIOMixer = mixer }() XCTAssertNil(weakIOMixer)