Skip to content

Commit

Permalink
Bugfixes with multicodec simulcast (#349)
Browse files Browse the repository at this point in the history 
* Attempt to simplify example app

* * Fix crashes when backupCodec isn't set
* Avoid using default settings, instead use the track's publication options
* Correctly set SVC codec layers

* fix import_sorter.

* fix flutter analyzer.

---------

Co-authored-by: cloudwebrtc <[email protected]>
  • Loading branch information
@davidzhao @cloudwebrtc
davidzhao and cloudwebrtc authored Sep 2, 2023
1 parent 0a1e881 commit 9339f92
Show file tree
Hide file tree
Showing 10 changed files with 122 additions and 124 deletions.
2 changes: 1 addition & 1 deletion example/android/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,6 @@ subprojects {
project.evaluationDependsOn(':app')
}

task clean(type: Delete) {
tasks.register("clean", Delete) {
delete rootProject.buildDir
}
1 change: 1 addition & 0 deletions example/ios/Runner.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@
files = (
);
inputPaths = (
"${TARGET_BUILD_DIR}/${INFOPLIST_PATH}",
);
name = "Thin Binary";
outputPaths = (
Expand Down
73 changes: 27 additions & 46 deletions example/lib/pages/connect.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,6 @@ class _ConnectPageState extends State<ConnectPage> {
print('Connecting with url: ${_uriCtrl.text}, '
'token: ${_tokenCtrl.text}...');

//create new room
final room = Room();

// Create a Listener before connecting
final listener = room.createListener();
E2EEOptions? e2eeOptions;
if (_e2ee) {
final keyProvider = await BaseKeyProvider.create();
Expand All @@ -143,55 +138,41 @@ class _ConnectPageState extends State<ConnectPage> {
await keyProvider.setKey(sharedKey);
}

BackupVideoCodec? backupVideoCodec;
String preferredCodec = 'H264';
if (_multiCodec && _preferredCodec != 'Preferred Codec') {
if (['av1', 'vp9'].contains(_preferredCodec.toLowerCase())) {
backupVideoCodec = BackupVideoCodec(
simulcast: true,
codec: _backupCodec,
encoding: const VideoEncoding(
maxBitrate: 2 * 1000 * 1000,
maxFramerate: 30,
));
}

String preferredCodec = 'VP8';
if (_preferredCodec != 'Preferred Codec') {
preferredCodec = _preferredCodec;
}

// create new room
final room = Room(
roomOptions: RoomOptions(
adaptiveStream: _adaptiveStream,
dynacast: _dynacast,
defaultAudioPublishOptions: const AudioPublishOptions(
dtx: true,
),
defaultVideoPublishOptions: VideoPublishOptions(
simulcast: _simulcast,
videoCodec: preferredCodec,
),
defaultScreenShareCaptureOptions: const ScreenShareCaptureOptions(
useiOSBroadcastExtension: true,
params: VideoParametersPresets.screenShareH1080FPS30),
e2eeOptions: e2eeOptions,
defaultCameraCaptureOptions: const CameraCaptureOptions(
maxFrameRate: 30,
params: VideoParametersPresets.h720_169,
),
));

// Create a Listener before connecting
final listener = room.createListener();

// Try to connect to the room
// This will throw an Exception if it fails for any reason.
await room.connect(
_uriCtrl.text,
_tokenCtrl.text,
roomOptions: RoomOptions(
adaptiveStream: _adaptiveStream,
dynacast: _dynacast,
defaultAudioPublishOptions:
const AudioPublishOptions(name: 'custom_audio_track_name'),
defaultVideoPublishOptions: VideoPublishOptions(
simulcast: _simulcast,
videoCodec: preferredCodec,
backupCodec: backupVideoCodec,
),
defaultScreenShareCaptureOptions: const ScreenShareCaptureOptions(
useiOSBroadcastExtension: true,
params: VideoParameters(
dimensions: VideoDimensionsPresets.h1080_169,
encoding: VideoEncoding(
maxBitrate: 3 * 1000 * 1000,
maxFramerate: 15,
))),
e2eeOptions: e2eeOptions,
defaultCameraCaptureOptions: const CameraCaptureOptions(
maxFrameRate: 30,
params: VideoParameters(
dimensions: VideoDimensionsPresets.h720_169,
encoding: VideoEncoding(
maxBitrate: 2 * 1000 * 1000,
maxFramerate: 30,
))),
),
fastConnectOptions: _fastConnect
? FastConnectOptions(
microphone: const TrackOption(enabled: true),
Expand Down
6 changes: 3 additions & 3 deletions lib/src/core/room.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,13 +284,13 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
}
var videoTrack = publication.track as LocalVideoTrack;
final newCodecs = await videoTrack.setPublishingCodecs(
event.subscribedCodecs, publication);
event.subscribedCodecs, videoTrack);
for (var codec in newCodecs) {
if (isBackupCodec(codec)) {
logger.info(
'publishing backup codec ${codec} for ${publication.track?.sid}');
await localParticipant?.publishAdditionalCodecForTrack(
videoTrack, codec, roomOptions.defaultVideoPublishOptions);
await localParticipant?.publishAdditionalCodecForPublication(
publication, codec);
}
}
} else if (event.subscribedQualities.isNotEmpty) {
Expand Down
7 changes: 4 additions & 3 deletions lib/src/options.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,13 @@ class RoomOptions {

class BackupVideoCodec {
BackupVideoCodec({
required this.codec,
required this.encoding,
this.codec = 'vp8',
this.encoding,
this.simulcast = true,
});
String codec;
VideoEncoding encoding;
// optional, when unset, it'll be computed based on dimensions and codec
VideoEncoding? encoding;
bool simulcast;
}

Expand Down
87 changes: 45 additions & 42 deletions lib/src/participant/local.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,24 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
);
}

// handle SVC publishing
final isSVC = isSVCCodec(publishOptions.videoCodec);
if (isSVC) {
if (!room.roomOptions.dynacast) {
room.engine.roomOptions = room.roomOptions.copyWith(dynacast: true);
}
if (publishOptions.backupCodec == null) {
publishOptions = publishOptions.copyWith(
backupCodec: BackupVideoCodec(),
);
}
if (publishOptions.scalabilityMode == null) {
publishOptions = publishOptions.copyWith(
scalabilityMode: 'L3T3_KEY',
);
}
}

// use constraints passed to getUserMedia by default
VideoDimensions dimensions = track.currentOptions.params.dimensions;

Expand All @@ -159,14 +177,6 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
logger.fine(
'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}');

if (isSVCCodec(publishOptions.videoCodec) &&
publishOptions.scalabilityMode == null) {
// set scalabilityMode to 'L3T3_KEY' by default
publishOptions = publishOptions.copyWith(
scalabilityMode: 'L3T3_KEY',
);
}

// Video encodings and simulcasts
final encodings = Utils.computeVideoEncodings(
isScreenShare: track.source == TrackSource.screenShareVideo,
Expand All @@ -177,17 +187,17 @@ class LocalParticipant extends Participant<LocalTrackPublication> {

logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');

final layers = Utils.computeVideoLayers(dimensions, encodings);
final layers = Utils.computeVideoLayers(
dimensions,
encodings,
isSVC,
);

logger.fine('Video layers: ${layers.map((e) => e)}');
var simulcastCodecs = <lk_rtc.SimulcastCodec>[];

if (publishOptions.backupCodec != null &&
publishOptions.backupCodec!.codec != publishOptions.videoCodec) {
if (!room.roomOptions.dynacast) {
room.engine.roomOptions = room.roomOptions.copyWith(dynacast: true);
}

simulcastCodecs = <lk_rtc.SimulcastCodec>[
lk_rtc.SimulcastCodec(
codec: publishOptions.videoCodec.toLowerCase(),
Expand Down Expand Up @@ -263,6 +273,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
track: track,
);
addTrackPublication(pub);
pub.backupVideoCodec = publishOptions.backupCodec;

// did publish
await track.onPublish();
Expand Down Expand Up @@ -545,44 +556,35 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
return oldValue;
}

Future<void> publishAdditionalCodecForTrack(
LocalVideoTrack track,
Future<void> publishAdditionalCodecForPublication(
LocalTrackPublication publication,
String backupCodec,
VideoPublishOptions? options,
) async {
// is it not published? if so skip
LocalTrackPublication? existingPublication;
for (var publication in videoTracks) {
if (publication.track == null) {
continue;
}
if (publication.track == track) {
existingPublication = publication;
}
}
if (existingPublication == null) {
throw Exception('track is not published');
if (publication.track is! LocalVideoTrack) {
throw Exception('multi-codec simulcast is supported only for video');
}
var track = publication.track as LocalVideoTrack;

options ??= room.roomOptions.defaultVideoPublishOptions;
final backupCodecOpts = publication.backupVideoCodec;
if (backupCodecOpts == null) {
throw Exception('backupCodec settings not specified');
}

options = options.copyWith(simulcast: options.backupCodec!.simulcast);
var options = room.roomOptions.defaultVideoPublishOptions;
options = options.copyWith(simulcast: backupCodecOpts.simulcast);

if (options.backupCodec == null ||
options.backupCodec?.codec.toLowerCase() ==
options.videoCodec.toLowerCase()) {
// backup codec publishing is disabled
if (backupCodec.toLowerCase() == publication.track?.codec?.toLowerCase()) {
// not needed, same codec already published
return;
}

if (backupCodec != options.backupCodec?.codec.toLowerCase()) {
if (backupCodec != backupCodecOpts.codec.toLowerCase()) {
logger.warning(
'requested a different codec than specified as backup serverRequested: ${backupCodec}, backup: ${options.backupCodec?.codec}',
'requested a different codec than specified as backup serverRequested: ${backupCodec}, backup: ${backupCodecOpts.codec}',
);
}

var encodings = Utils.computeTrackBackupEncodings(track, options);

var encodings = Utils.computeTrackBackupEncodings(track, backupCodecOpts);
if (encodings == null) {
logger.fine(
'backup codec has been disabled, ignoring request to add additional codec for track');
Expand All @@ -592,13 +594,14 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
var simulcastTrack = track.addSimulcastTrack(backupCodec, encodings);
var dimensions = track.currentOptions.params.dimensions;

var layers = Utils.computeVideoLayers(dimensions, encodings);
var layers = Utils.computeVideoLayers(
dimensions, encodings, isSVCCodec(backupCodec));

simulcastTrack.sender = await room.engine.createSimulcastTransceiverSender(
track,
simulcastTrack,
encodings,
existingPublication,
publication,
backupCodec,
);

Expand All @@ -614,12 +617,12 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
source: track.source.toPBType(),
dimensions: dimensions,
videoLayers: layers,
sid: existingPublication.sid,
sid: publication.sid,
simulcastCodecs: <lk_rtc.SimulcastCodec>[
lk_rtc.SimulcastCodec(
codec: backupCodec.toLowerCase(),
cid: cid,
enableSimulcastLayers: options.backupCodec!.simulcast),
enableSimulcastLayers: backupCodecOpts.simulcast),
]);

await room.engine.negotiate();
Expand Down
3 changes: 3 additions & 0 deletions lib/src/publication/local.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import '../options.dart';
import '../participant/local.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../proto/livekit_rtc.pb.dart' as lk_rtc;
Expand All @@ -24,6 +25,8 @@ class LocalTrackPublication<T extends LocalTrack> extends TrackPublication<T> {
@override
final LocalParticipant participant;

BackupVideoCodec? backupVideoCodec;

LocalTrackPublication({
required this.participant,
required lk_models.TrackInfo info,
Expand Down
9 changes: 4 additions & 5 deletions lib/src/track/local/video.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import '../../events.dart';
import '../../logger.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../proto/livekit_rtc.pb.dart' as lk_rtc;
import '../../publication/local.dart';
import '../../support/platform.dart';
import '../../types/other.dart';
import '../options.dart';
Expand Down Expand Up @@ -277,13 +276,13 @@ extension LocalVideoTrackExt on LocalVideoTrack {
});
}

Future<List<String>> setPublishingCodecs(List<lk_rtc.SubscribedCodec> codecs,
LocalTrackPublication publication) async {
Future<List<String>> setPublishingCodecs(
List<lk_rtc.SubscribedCodec> codecs, LocalTrack track) async {
logger.fine('setPublishingCodecs $codecs');

// only enable simulcast codec for preference codec setted
if (codec == null && codecs.isNotEmpty) {
await updatePublishingLayers(publication.track, codecs[0].qualities);
await updatePublishingLayers(track, codecs[0].qualities);
return [];
}

Expand All @@ -293,7 +292,7 @@ extension LocalVideoTrackExt on LocalVideoTrack {

for (var codec in codecs) {
if (this.codec?.toLowerCase() == codec.codec.toLowerCase()) {
await updatePublishingLayers(publication.track, codec.qualities);
await updatePublishingLayers(track, codec.qualities);
} else {
final simulcastCodecInfo = simulcastCodecs[codec.codec];
logger.fine('setPublishingCodecs $codecs');
Expand Down
Loading

0 comments on commit 9339f92

Please sign in to comment.