From 00ec2f08579d609619f0e96a18d278be647d9331 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 6 Jul 2023 10:57:07 +0200 Subject: [PATCH 01/22] First foundation of BWE context, using TWCC feedback --- fuzzers/rtcp_fuzzer.c | 4 +- src/Makefile.am | 4 +- src/bwe.c | 93 +++++++++++++++++++++++++++++++++++ src/bwe.h | 86 ++++++++++++++++++++++++++++++++ src/ice.c | 32 ++++++++---- src/ice.h | 5 ++ src/janus.c | 1 + src/plugins/janus_nosip.c | 2 +- src/plugins/janus_sip.c | 6 +-- src/plugins/janus_streaming.c | 5 +- src/plugins/janus_videoroom.c | 4 +- src/plugins/plugin.h | 2 +- src/rtcp.c | 92 ++++++++++++++-------------------- src/rtcp.h | 14 +++--- src/sdp.c | 3 ++ 15 files changed, 271 insertions(+), 82 deletions(-) create mode 100644 src/bwe.c create mode 100644 src/bwe.h diff --git a/fuzzers/rtcp_fuzzer.c b/fuzzers/rtcp_fuzzer.c index c742f1db93..dc9273714f 100644 --- a/fuzzers/rtcp_fuzzer.c +++ b/fuzzers/rtcp_fuzzer.c @@ -61,8 +61,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { janus_rtcp_cap_remb((char *)copy_data[idx++], size, 256000); janus_rtcp_swap_report_blocks((char *)copy_data[idx++], size, 2); janus_rtcp_fix_report_data((char *)copy_data[idx++], size, 2000, 1000, 2, 2, 2, TRUE); - janus_rtcp_fix_ssrc(&ctx0, (char *)copy_data[idx++], size, 1, 2, 2); - janus_rtcp_parse(&ctx1, (char *)copy_data[idx++], size); + janus_rtcp_fix_ssrc(&ctx0, NULL, (char *)copy_data[idx++], size, 1, 2, 2); + janus_rtcp_parse(&ctx1, NULL, (char *)copy_data[idx++], size); janus_rtcp_remove_nacks((char *)copy_data[idx++], size); /* Functions that allocate new memory */ char *output_data = janus_rtcp_filter((char *)data, size, &newlen); diff --git a/src/Makefile.am b/src/Makefile.am index 9b467a7f51..64e5ff9e33 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,7 @@ CLEANFILES = $(NULL) bin_PROGRAMS = janus headerdir = $(includedir)/janus -header_HEADERS = apierror.h config.h log.h debug.h mutex.h record.h \ +header_HEADERS = apierror.h config.h log.h debug.h mutex.h record.h bwe.h \ rtcp.h rtp.h rtpsrtp.h sdp-utils.h ip-utils.h utils.h refcount.h text2pcap.h pluginsheaderdir = $(includedir)/janus/plugins @@ -72,6 +72,8 @@ janus_SOURCES = \ apierror.h \ auth.c \ auth.h \ + bwe.c \ + bwe.h \ config.c \ config.h \ debug.h \ diff --git a/src/bwe.c b/src/bwe.c new file mode 100644 index 0000000000..3157b28c57 --- /dev/null +++ b/src/bwe.c @@ -0,0 +1,93 @@ +/*! \file bwe.h + * \author Lorenzo Miniero + * \copyright GNU General Public License v3 + * \brief Bandwidth estimation tools + * \details Implementation of a basic bandwidth estimator for outgoing + * RTP flows, based on Transport Wide CC and a few other utilities. + * + * \ingroup protocols + * \ref protocols + */ + +#include +#include + +#include "bwe.h" +#include "debug.h" +#include "utils.h" + +const char *janus_bwe_twcc_status_description(janus_bwe_twcc_status status) { + switch(status) { + case janus_bwe_twcc_status_notreceived: + return "notreceived"; + case janus_bwe_twcc_status_smalldelta: + return "smalldelta"; + case janus_bwe_twcc_status_largeornegativedelta: + return "largeornegativedelta"; + case janus_bwe_twcc_status_reserved: + return "reserved"; + default: break; + } + return NULL; +} + +static void janus_bwe_twcc_inflight_destroy(janus_bwe_twcc_inflight *stat) { + g_free(stat); +} + +janus_bwe_context *janus_bwe_context_create(void) { + janus_bwe_context *bwe = g_malloc0(sizeof(janus_bwe_context)); + /* TODO */ + bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); + return bwe; +} + +void janus_bwe_context_destroy(janus_bwe_context *bwe) { + if(bwe) { + /* TODO clean everything up */ + g_hash_table_destroy(bwe->packets); + g_free(bwe); + } +} + +gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, + uint16_t seq, int64_t sent, janus_bwe_packet_type type, int size) { + if(bwe == NULL) + return FALSE; + janus_bwe_twcc_inflight *stat = g_malloc(sizeof(janus_bwe_twcc_inflight)); + stat->seq = seq; + stat->delta_us = bwe->sent_ts ? (sent - bwe->sent_ts) : 0; + bwe->sent_ts = sent; + stat->type = type; + stat->size = size; + g_hash_table_insert(bwe->packets, GUINT_TO_POINTER(seq), stat); + return TRUE; +} + +void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, + uint16_t seq, janus_bwe_twcc_status status, uint32_t delta_us) { + if(bwe == NULL) + return; + /* Find the inflight information we stored when sending this packet */ + janus_bwe_twcc_inflight *p = g_hash_table_lookup(bwe->packets, GUINT_TO_POINTER(seq)); + if(p == NULL) + return; + bwe->sent_bytes += p->size; + if(status != janus_bwe_twcc_status_notreceived) + bwe->received_bytes += p->size; + /* Print summary */ + JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNu32"us) (send: %"SCNi64"us)\n", seq, + janus_bwe_twcc_status_description(status), delta_us, p ? ((p->delta_us/250)*250) : 0); + /* Check if it's time to compute the bitrates */ + int64_t now = janus_get_monotonic_time(); + if(bwe->bitrate_ts == 0) + bwe->bitrate_ts = now; + if(now - bwe->bitrate_ts >= G_USEC_PER_SEC) { + /* It is, show the outgoing and (acked) incoming bitrate */ + JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps\n", + (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8); + bwe->bitrate_ts += G_USEC_PER_SEC; + bwe->sent_bytes = 0; + bwe->received_bytes = 0; + } +} diff --git a/src/bwe.h b/src/bwe.h new file mode 100644 index 0000000000..ce04229d2a --- /dev/null +++ b/src/bwe.h @@ -0,0 +1,86 @@ +/*! \file bwe.h + * \author Lorenzo Miniero + * \copyright GNU General Public License v3 + * \brief Bandwidth estimation tools (headers) + * \details Implementation of a basic bandwidth estimator for outgoing + * RTP flows, based on Transport Wide CC and a few other utilities. + * + * \ingroup protocols + * \ref protocols + */ + +#ifndef JANUS_BWE_H +#define JANUS_BWE_H + +#include + + +/*! \brief Transport Wide CC statuses */ +typedef enum janus_bwe_twcc_status { + janus_bwe_twcc_status_notreceived = 0, + janus_bwe_twcc_status_smalldelta = 1, + janus_bwe_twcc_status_largeornegativedelta = 2, + janus_bwe_twcc_status_reserved = 3 +} janus_bwe_twcc_status; +/*! \brief Helper to return a string description of a TWCC status + * @param[in] status The janus_bwe_twcc_status status + * @returns A string description */ +const char *janus_bwe_twcc_status_description(janus_bwe_twcc_status status); + +/*! \brief Type of in-flight packet */ +typedef enum janus_bwe_packet_type { + /*! \brief Regular RTP packet */ + janus_bwe_packet_type_regular = 0, + /*! \brief RTC packet */ + janus_bwe_packet_type_rtx, + /* TODO other types? e.g., padding? */ +} janus_bwe_packet_type; + +/*! \brief Tracking info for in-flight packet we're waiting TWCC feedback for */ +typedef struct janus_bwe_twcc_inflight { + /*! \brief The TWCC sequence number */ + uint16_t seq; + /*! \brief Delta (in us) since the delivery of the previous packet */ + int64_t delta_us; + /*! \brief Type of packet (e.g., regular or rtx) */ + janus_bwe_packet_type type; + /*! \brief Size of the sent packet */ + int size; +} janus_bwe_twcc_inflight; + +/*! \brief Bandwidth estimation context */ +typedef struct janus_bwe_context { + /*! \brief Monotonic timestamp of the last sent packet */ + int64_t sent_ts; + /*! \brief Map of in-flight packets */ + GHashTable *packets; + /*! \brief Monotonic timestamp of when we last computed the bitrates */ + int64_t bitrate_ts; + /*! \brief Amount of bytes we've sent and the ones we've had feedback were received */ + uint32_t sent_bytes, received_bytes; +} janus_bwe_context; +/*! \brief Helper to create a new bandwidth estimation context + * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ +janus_bwe_context *janus_bwe_context_create(void); +/*! \brief Helper to destroy an existing bandwidth estimation context + * @param[in] bew The janus_bwe_context instance to destroy */ +void janus_bwe_context_destroy(janus_bwe_context *bwe); + +/*! \brief Helper method to quickly add a new inflight packet to a BWE instance + * @param[in] bwe The janus_bwe_context instance to update + * @param[in] seq The TWCC sequence number of the new inflight packet + * @param[in] sent The timestamp of the packet delivery + * @param[in] type The type of this packet + * @param[in] size The size of this packet + * @returns TRUE, if successful, or FALSE otherwise */ +gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, + uint16_t seq, int64_t sent, janus_bwe_packet_type type, int size); +/*! \brief Handle feedback on an inflight packet + * @param[in] bwe The janus_bwe_context instance to update + * @param[in] seq The TWCC sequence number of the inflight packet we have feedback for + * @param[in] status Feedback status for the packet + * @param[in] delta_us If the packet was received, the delta that was provided */ +void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, + uint16_t seq, janus_bwe_twcc_status status, uint32_t delta_us); + +#endif diff --git a/src/ice.c b/src/ice.c index 5ed7310c19..4b175390c9 100644 --- a/src/ice.c +++ b/src/ice.c @@ -462,6 +462,7 @@ typedef struct janus_ice_queued_packet { gboolean control; gboolean retransmission; gboolean encrypted; + guint16 twcc_seq; gint64 added; } janus_ice_queued_packet; /* A few static, fake, messages we use as a trigger: e.g., to start a @@ -1844,6 +1845,7 @@ static void janus_ice_peerconnection_free(const janus_refcount *pc_ref) { if(pc->rtx_payload_types_rev != NULL) g_hash_table_destroy(pc->rtx_payload_types_rev); pc->rtx_payload_types_rev = NULL; + janus_bwe_context_destroy(pc->bwe); g_free(pc); pc = NULL; } @@ -3029,7 +3031,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp int res = janus_rtcp_nacks(nackbuf, sizeof(nackbuf), nacks); if(res > 0) { /* Set the right local and remote SSRC in the RTCP packet */ - janus_rtcp_fix_ssrc(NULL, nackbuf, res, 1, + janus_rtcp_fix_ssrc(NULL, NULL, nackbuf, res, 1, medium->ssrc, medium->ssrc_peer[vindex]); janus_plugin_rtcp rtcp = { .mindex = medium->mindex, .video = video, .buffer = nackbuf, .length = res }; janus_ice_relay_rtcp_internal(handle, medium, &rtcp, FALSE); @@ -3108,7 +3110,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp /* Let's process this RTCP (compound?) packet, and update the RTCP context for this stream in case */ rtcp_context *rtcp_ctx = medium->rtcp_ctx[vindex]; uint32_t rtt = rtcp_ctx ? rtcp_ctx->rtt : 0; - if(janus_rtcp_parse(rtcp_ctx, buf, buflen) < 0) { + if(janus_rtcp_parse(rtcp_ctx, pc->bwe, buf, buflen) < 0) { /* Drop the packet if the parsing function returns with an error */ return; } @@ -3176,6 +3178,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp pkt->retransmission = TRUE; pkt->label = NULL; pkt->protocol = NULL; + pkt->twcc_seq = 0; pkt->added = janus_get_monotonic_time(); /* What to send and how depends on whether we're doing RFC4588 or not */ if(!video || !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_RFC4588_RTX)) { @@ -3730,6 +3733,7 @@ int janus_ice_setup_local(janus_ice_handle *handle, gboolean offer, gboolean tri pc->stream_id = handle->stream_id; pc->handle = handle; pc->dtls_role = dtls_role; + pc->bwe = janus_bwe_context_create(); janus_mutex_init(&pc->mutex); if(!have_turnrest_credentials) { /* No TURN REST API server and credentials, any static ones? */ @@ -3883,7 +3887,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p /* Add core and plugin extensions, if any */ gboolean video = (packet->type == JANUS_ICE_PACKET_VIDEO); if(handle->pc->mid_ext_id > 0 || (video && handle->pc->abs_send_time_ext_id > 0) || - (video && handle->pc->transport_wide_cc_ext_id > 0) || + (medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) || (!video && packet->extensions.audio_level > -1 && handle->pc->audiolevel_ext_id > 0) || (video && packet->extensions.video_rotation > -1 && handle->pc->videoorientation_ext_id > 0) || (video && packet->extensions.min_delay > -1 && packet->extensions.max_delay > -1 && handle->pc->playoutdelay_ext_id > 0) || @@ -3920,9 +3924,10 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p } } /* Check if we need to add the transport-wide CC extension */ - if(video && handle->pc->transport_wide_cc_ext_id > 0) { + if(medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) { handle->pc->transport_wide_cc_out_seq_num++; - uint16_t transSeqNum = htons(handle->pc->transport_wide_cc_out_seq_num); + packet->twcc_seq = handle->pc->transport_wide_cc_out_seq_num; + uint16_t transSeqNum = htons(packet->twcc_seq); if(!use_2byte) { *index = (handle->pc->transport_wide_cc_ext_id << 4) + 1; memcpy(index+1, &transSeqNum, 2); @@ -4096,7 +4101,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p } static gint rtcp_transport_wide_cc_stats_comparator(gconstpointer item1, gconstpointer item2) { - return ((rtcp_transport_wide_cc_stats*)item1)->transport_seq_num - ((rtcp_transport_wide_cc_stats*)item2)->transport_seq_num; + return ((janus_rtcp_transport_wide_cc_stats *)item1)->transport_seq_num - ((janus_rtcp_transport_wide_cc_stats *)item2)->transport_seq_num; } static gboolean janus_ice_outgoing_transport_wide_cc_feedback(gpointer user_data) { janus_ice_handle *handle = (janus_ice_handle *)user_data; @@ -4838,6 +4843,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu handle->handle_id, janus_srtp_error_str(res), pkt->length, protected, timestamp, seq); janus_ice_free_rtp_packet(p); } else { + /* Keep track of the packet if we're using TWCC */ + if(medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) { + janus_bwe_context_add_inflight(pc->bwe, pkt->twcc_seq, janus_get_monotonic_time(), + (pkt->retransmission ? janus_bwe_packet_type_rtx : janus_bwe_packet_type_regular), protected); + } /* Shoot! */ int sent = nice_agent_send(handle->agent, pc->stream_id, pc->component_id, protected, pkt->data); if(sent < protected) { @@ -4991,6 +5001,7 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { pkt->retransmission = FALSE; pkt->label = NULL; pkt->protocol = NULL; + pkt->twcc_seq = 0; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); } @@ -5017,7 +5028,7 @@ void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_ice_peerconne * ones created by the core already have the right SSRCs in the right place */ JANUS_LOG(LOG_HUGE, "[%"SCNu64"] Fixing SSRCs (local %u, peer %u)\n", handle->handle_id, medium->ssrc, medium->ssrc_peer[0]); - janus_rtcp_fix_ssrc(NULL, rtcp_buf, rtcp_len, 1, + janus_rtcp_fix_ssrc(NULL, NULL, rtcp_buf, rtcp_len, 1, medium->ssrc, medium->ssrc_peer[0]); } /* Queue this packet */ @@ -5033,6 +5044,7 @@ void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_ice_peerconne pkt->retransmission = FALSE; pkt->label = NULL; pkt->protocol = NULL; + pkt->twcc_seq = 0; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); if(rtcp_buf != packet->buffer) { @@ -5067,7 +5079,7 @@ void janus_ice_relay_rtcp(janus_ice_handle *handle, janus_plugin_rtcp *packet) { char plibuf[12]; memset(plibuf, 0, 12); janus_rtcp_pli((char *)&plibuf, 12); - janus_rtcp_fix_ssrc(NULL, plibuf, sizeof(plibuf), 1, + janus_rtcp_fix_ssrc(NULL, NULL, plibuf, sizeof(plibuf), 1, medium->ssrc, medium->ssrc_peer[1]); janus_plugin_rtcp rtcp = { .mindex = medium->mindex, .video = TRUE, .buffer = plibuf, .length = sizeof(plibuf) }; janus_ice_relay_rtcp_internal(handle, medium, &rtcp, FALSE); @@ -5076,7 +5088,7 @@ void janus_ice_relay_rtcp(janus_ice_handle *handle, janus_plugin_rtcp *packet) { char plibuf[12]; memset(plibuf, 0, 12); janus_rtcp_pli((char *)&plibuf, 12); - janus_rtcp_fix_ssrc(NULL, plibuf, sizeof(plibuf), 1, + janus_rtcp_fix_ssrc(NULL, NULL, plibuf, sizeof(plibuf), 1, medium->ssrc, medium->ssrc_peer[2]); janus_plugin_rtcp rtcp = { .mindex = medium->mindex, .video = TRUE, .buffer = plibuf, .length = sizeof(plibuf) }; janus_ice_relay_rtcp_internal(handle, medium, &rtcp, FALSE); @@ -5132,6 +5144,7 @@ void janus_ice_relay_data(janus_ice_handle *handle, janus_plugin_data *packet) { pkt->retransmission = FALSE; pkt->label = packet->label ? g_strdup(packet->label) : NULL; pkt->protocol = packet->protocol ? g_strdup(packet->protocol) : NULL; + pkt->twcc_seq = 0; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); } @@ -5159,6 +5172,7 @@ void janus_ice_relay_sctp(janus_ice_handle *handle, char *buffer, int length) { pkt->retransmission = FALSE; pkt->label = NULL; pkt->protocol = NULL; + pkt->twcc_seq = 0; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); #endif diff --git a/src/ice.h b/src/ice.h index 2c28d1b5c3..48c19c9b38 100644 --- a/src/ice.h +++ b/src/ice.h @@ -28,6 +28,7 @@ #include "dtls.h" #include "sctp.h" #include "rtcp.h" +#include "bwe.h" #include "text2pcap.h" #include "utils.h" #include "ip-utils.h" @@ -523,6 +524,8 @@ struct janus_ice_peerconnection { GHashTable *rtx_payload_types_rev; /*! \brief Helper flag to avoid flooding the console with the same error all over again */ gboolean noerrorlog; + /*! Bandwidth estimation context */ + janus_bwe_context *bwe; /*! \brief Mutex to lock/unlock this stream */ janus_mutex mutex; /*! \brief Atomic flag to check if this instance has been destroyed */ @@ -596,6 +599,8 @@ struct janus_ice_peerconnection_medium { guint32 last_rtp_ts; /*! \brief Whether we should do NACKs (in or out) for this medium */ gboolean do_nacks; + /*! \brief Whether we should do Transport Wide CC for this medium */ + gboolean do_twcc; /*! \brief List of previously sent janus_rtp_packet RTP packets, in case we receive NACKs */ GQueue *retransmit_buffer; /*! \brief HashTable of retransmittable sequence numbers, in case we receive NACKs */ diff --git a/src/janus.c b/src/janus.c index 80be2646c0..156dac46e4 100644 --- a/src/janus.c +++ b/src/janus.c @@ -3316,6 +3316,7 @@ json_t *janus_admin_peerconnection_medium_summary(janus_ice_peerconnection_mediu if(medium->type != JANUS_MEDIA_DATA) { json_object_set_new(m, "do_nacks", medium->do_nacks ? json_true() : json_false()); json_object_set_new(m, "nack-queue-ms", json_integer(medium->nack_queue_ms)); + json_object_set_new(m, "do_twcc", medium->do_twcc ? json_true() : json_false()); } if(medium->type != JANUS_MEDIA_DATA) { json_t *ms = json_object(); diff --git a/src/plugins/janus_nosip.c b/src/plugins/janus_nosip.c index 2499aaab12..e6156a9f86 100644 --- a/src/plugins/janus_nosip.c +++ b/src/plugins/janus_nosip.c @@ -1162,7 +1162,7 @@ void janus_nosip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp * session, video ? "video" : "audio", (video ? session->media.video_ssrc : session->media.audio_ssrc), (video ? session->media.video_ssrc_peer : session->media.audio_ssrc_peer)); - janus_rtcp_fix_ssrc(NULL, (char *)buf, len, video, + janus_rtcp_fix_ssrc(NULL, NULL, (char *)buf, len, video, (video ? session->media.video_ssrc : session->media.audio_ssrc), (video ? session->media.video_ssrc_peer : session->media.audio_ssrc_peer)); /* Is SRTP involved? */ diff --git a/src/plugins/janus_sip.c b/src/plugins/janus_sip.c index e8b7366538..5379a656cc 100644 --- a/src/plugins/janus_sip.c +++ b/src/plugins/janus_sip.c @@ -2568,7 +2568,7 @@ void janus_sip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *pa /* Fix SSRCs as the Janus core does */ JANUS_LOG(LOG_HUGE, "[SIP] Fixing SSRCs (local %u, peer %u)\n", session->media.video_ssrc, session->media.video_ssrc_peer); - janus_rtcp_fix_ssrc(NULL, (char *)buf, len, 1, session->media.video_ssrc, session->media.video_ssrc_peer); + janus_rtcp_fix_ssrc(NULL, NULL, (char *)buf, len, 1, session->media.video_ssrc, session->media.video_ssrc_peer); /* Is SRTP involved? */ if(session->media.has_srtp_local_video) { char sbuf[2048]; @@ -2598,7 +2598,7 @@ void janus_sip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *pa /* Fix SSRCs as the Janus core does */ JANUS_LOG(LOG_HUGE, "[SIP] Fixing SSRCs (local %u, peer %u)\n", session->media.audio_ssrc, session->media.audio_ssrc_peer); - janus_rtcp_fix_ssrc(NULL, (char *)buf, len, 1, session->media.audio_ssrc, session->media.audio_ssrc_peer); + janus_rtcp_fix_ssrc(NULL, NULL, (char *)buf, len, 1, session->media.audio_ssrc, session->media.audio_ssrc_peer); /* Is SRTP involved? */ if(session->media.has_srtp_local_audio) { char sbuf[2048]; @@ -7409,7 +7409,7 @@ static void janus_sip_rtcp_pli_send(janus_sip_session *session) { /* Fix SSRCs as the Janus core does */ JANUS_LOG(LOG_HUGE, "[SIP] Fixing SSRCs (local %u, peer %u)\n", session->media.video_ssrc, session->media.video_ssrc_peer); - janus_rtcp_fix_ssrc(NULL, (char *)rtcp_buf, rtcp_len, 1, session->media.video_ssrc, session->media.video_ssrc_peer); + janus_rtcp_fix_ssrc(NULL, NULL, (char *)rtcp_buf, rtcp_len, 1, session->media.video_ssrc, session->media.video_ssrc_peer); /* Is SRTP involved? */ if(session->media.has_srtp_local_video) { char sbuf[50]; diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index 5d740cdfcb..acdcc7ee24 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -1820,7 +1820,7 @@ static void janus_streaming_rtcp_pli_send(janus_streaming_rtp_source_stream *str char rtcp_buf[12]; int rtcp_len = 12; janus_rtcp_pli((char *)&rtcp_buf, rtcp_len); - janus_rtcp_fix_ssrc(NULL, rtcp_buf, rtcp_len, 1, 1, stream->ssrc); + janus_rtcp_fix_ssrc(NULL, NULL, rtcp_buf, rtcp_len, 1, 1, stream->ssrc); /* Send the packet */ socklen_t addrlen = stream->rtcp_addr.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); int sent = 0; @@ -1843,7 +1843,7 @@ static void janus_streaming_rtcp_remb_send(janus_streaming_rtp_source *source, j char rtcp_buf[24]; int rtcp_len = 24; janus_rtcp_remb((char *)(&rtcp_buf), rtcp_len, source->lowest_bitrate); - janus_rtcp_fix_ssrc(NULL, rtcp_buf, rtcp_len, 1, 1, stream->ssrc); + janus_rtcp_fix_ssrc(NULL, NULL, rtcp_buf, rtcp_len, 1, 1, stream->ssrc); JANUS_LOG(LOG_HUGE, "Sending REMB: %"SCNu32"\n", source->lowest_bitrate); /* Reset the lowest bitrate */ source->lowest_bitrate = 0; @@ -6110,6 +6110,7 @@ static void *janus_streaming_handler(void *data) { JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_ABS_SEND_TIME, janus_rtp_extension_id(JANUS_RTP_EXTMAP_ABS_SEND_TIME), JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, (session->playoutdelay_ext ? janus_rtp_extension_id(JANUS_RTP_EXTMAP_PLAYOUT_DELAY) : 0), + JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, janus_rtp_extension_id(JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC), JANUS_SDP_OA_DONE); } else { /* Iterate on all media streams */ diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index a7a7c2b550..ca018ee6ae 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -2576,7 +2576,7 @@ static void janus_videoroom_rtcp_pli_send(janus_videoroom_publisher_stream *ps) int rtcp_len = 12; janus_rtcp_pli((char *)&rtcp_buf, rtcp_len); uint32_t ssrc = REMOTE_PUBLISHER_BASE_SSRC + (ps->mindex*REMOTE_PUBLISHER_SSRC_STEP); - janus_rtcp_fix_ssrc(NULL, rtcp_buf, rtcp_len, 1, 1, ssrc); + janus_rtcp_fix_ssrc(NULL, NULL, rtcp_buf, rtcp_len, 1, 1, ssrc); /* Send the packet */ socklen_t addrlen = publisher->rtcp_addr.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); int sent = 0; @@ -3178,7 +3178,7 @@ static json_t *janus_videoroom_subscriber_offer(janus_videoroom_subscriber *subs JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, (stream->type == JANUS_VIDEOROOM_MEDIA_VIDEO && (ps && ps->playout_delay_extmap_id > 0)) ? janus_rtp_extension_id(JANUS_RTP_EXTMAP_PLAYOUT_DELAY) : 0, JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, - (stream->type == JANUS_VIDEOROOM_MEDIA_VIDEO && subscriber->room->transport_wide_cc_ext) ? janus_rtp_extension_id(JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC) : 0, + (subscriber->room->transport_wide_cc_ext) ? janus_rtp_extension_id(JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC) : 0, JANUS_SDP_OA_EXTENSION, JANUS_RTP_EXTMAP_ABS_SEND_TIME, (stream->type == JANUS_VIDEOROOM_MEDIA_VIDEO) ? janus_rtp_extension_id(JANUS_RTP_EXTMAP_ABS_SEND_TIME) : 0, /* TODO Add other properties from original SDP */ diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 2c3c174024..0814a541ab 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -171,7 +171,7 @@ janus_plugin *create(void) { * Janus instance or it will crash. * */ -#define JANUS_PLUGIN_API_VERSION 102 +#define JANUS_PLUGIN_API_VERSION 103 /*! \brief Initialization of all plugin properties to NULL * diff --git a/src/rtcp.c b/src/rtcp.c index 637c65cf8e..858fded47b 100644 --- a/src/rtcp.c +++ b/src/rtcp.c @@ -23,25 +23,6 @@ #include "utils.h" -/* Transport CC statuses */ -typedef enum janus_rtp_packet_status { - janus_rtp_packet_status_notreceived = 0, - janus_rtp_packet_status_smalldelta = 1, - janus_rtp_packet_status_largeornegativedelta = 2, - janus_rtp_packet_status_reserved = 3 -} janus_rtp_packet_status; -static const char *janus_rtp_packet_status_description(janus_rtp_packet_status status) { - switch(status) { - case janus_rtp_packet_status_notreceived: return "notreceived"; - case janus_rtp_packet_status_smalldelta: return "smalldelta"; - case janus_rtp_packet_status_largeornegativedelta: return "largeornegativedelta"; - case janus_rtp_packet_status_reserved: return "reserved"; - default: break; - } - return NULL; -} - - gboolean janus_is_rtcp(char *buf, guint len) { if (len < 8) return FALSE; @@ -49,8 +30,8 @@ gboolean janus_is_rtcp(char *buf, guint len) { return ((header->type >= 64) && (header->type < 96)); } -int janus_rtcp_parse(janus_rtcp_context *ctx, char *packet, int len) { - return janus_rtcp_fix_ssrc(ctx, packet, len, 0, 0, 0); +int janus_rtcp_parse(janus_rtcp_context *ctx, janus_bwe_context *bwe, char *packet, int len) { + return janus_rtcp_fix_ssrc(ctx, bwe, packet, len, 0, 0, 0); } guint32 janus_rtcp_get_sender_ssrc(char *packet, int len) { @@ -226,9 +207,9 @@ static void janus_rtcp_incoming_sr(janus_rtcp_context *ctx, janus_rtcp_sr *sr) { ctx->lsr = (ntp >> 16); } -/* Helper to handle an incoming transport-cc feedback: triggered by a call to janus_rtcp_fix_ssrc a valid context pointer */ -static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp_fb *twcc, int total) { - if(ctx == NULL || twcc == NULL || total < 20) +/* Helper to handle an incoming transport-cc feedback: triggered by a call to janus_rtcp_fix_ssrc */ +static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_bwe_context *bwe, janus_rtcp_fb *twcc, int total) { + if(ctx == NULL || bwe == NULL || twcc == NULL || total < 20) return; if(!janus_rtcp_check_fci((janus_rtcp_header *)twcc, total, 4)) return; @@ -266,7 +247,7 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp s = (chunk & 0x6000) >> 13; length = (chunk & 0x1FFF); JANUS_LOG(LOG_HUGE, " [%"SCNu16"] t=run-length, s=%s, l=%"SCNu8"\n", num, - janus_rtp_packet_status_description(s), length); + janus_bwe_twcc_status_description(s), length); while(length > 0 && psc > 0) { list = g_list_prepend(list, GUINT_TO_POINTER(s)); length--; @@ -280,7 +261,7 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp ss ? "2-bit" : "bit", length); while(length > 0 && psc > 0) { if(!ss) - s = (chunk & (1 << (length-1))) ? janus_rtp_packet_status_smalldelta : janus_rtp_packet_status_notreceived; + s = (chunk & (1 << (length-1))) ? janus_bwe_twcc_status_smalldelta : janus_bwe_twcc_status_notreceived; else s = (chunk & (3 << (2*length-2))) >> (2*length-2); list = g_list_prepend(list, GUINT_TO_POINTER(s)); @@ -300,19 +281,21 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp /* Iterate on all recv deltas */ JANUS_LOG(LOG_HUGE, "[TWCC] Recv Deltas (%d/%"SCNu16"):\n", g_list_length(list), status_count); num = 0; - uint16_t delta = 0; + uint16_t delta = 0, seq = 0; uint32_t delta_us = 0; GList *iter = list; while(iter != NULL && total > 0) { num++; delta = 0; + seq = (uint16_t)(base_seq+num-1); + /* Get the delta */ s = GPOINTER_TO_UINT(iter->data); - if(s == janus_rtp_packet_status_smalldelta) { + if(s == janus_bwe_twcc_status_smalldelta) { /* Small delta = 1 byte */ delta = *data; total--; data++; - } else if(s == janus_rtp_packet_status_largeornegativedelta) { + } else if(s == janus_bwe_twcc_status_largeornegativedelta) { /* Large or negative delta = 2 bytes */ if(total < 2) break; @@ -322,12 +305,11 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp data += 2; } delta_us = delta*250; - /* Print summary */ - JANUS_LOG(LOG_HUGE, " [%02"SCNu16"][%"SCNu16"] %s (%"SCNu32"us)\n", num, (uint16_t)(base_seq+num-1), - janus_rtp_packet_status_description(s), delta_us); + /* Pass the feedback to the bandwidth estimation context */ + janus_bwe_context_handle_feedback(bwe, seq, s, delta_us); + /* Move to the next feedback */ iter = iter->next; } - /* TODO Update the context with the feedback we got */ g_list_free(list); } @@ -520,7 +502,7 @@ gboolean janus_rtcp_check_remb(janus_rtcp_header *rtcp, int len) { return TRUE; } -int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, char *packet, int len, int fixssrc, uint32_t newssrcl, uint32_t newssrcr) { +int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, janus_bwe_context *bwe, char *packet, int len, int fixssrc, uint32_t newssrcl, uint32_t newssrcr) { if(packet == NULL || len <= 0) return -1; janus_rtcp_header *rtcp = (janus_rtcp_header *)packet; @@ -648,7 +630,7 @@ int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, char *packet, int len, int fixs } } else if(fmt == 15) { /* transport-cc */ /* If an RTCP context was provided, parse this transport-cc feedback */ - janus_rtcp_incoming_transport_cc(ctx, rtcpfb, total); + janus_rtcp_incoming_transport_cc(ctx, bwe, rtcpfb, total); } else { JANUS_LOG(LOG_HUGE, " #%d ??? -- RTPFB (205, fmt=%d)\n", pno, fmt); } @@ -1640,13 +1622,13 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* Store delta array */ GQueue *deltas = g_queue_new(); GQueue *statuses = g_queue_new(); - janus_rtp_packet_status last_status = janus_rtp_packet_status_reserved; - janus_rtp_packet_status max_status = janus_rtp_packet_status_notreceived; + janus_bwe_twcc_status last_status = janus_bwe_twcc_status_reserved; + janus_bwe_twcc_status max_status = janus_bwe_twcc_status_notreceived; gboolean all_same = TRUE; /* For each packet */ while (stat != NULL) { - janus_rtp_packet_status status = janus_rtp_packet_status_notreceived; + janus_bwe_twcc_status status = janus_bwe_twcc_status_notreceived; /* If got packet */ if (stat->timestamp) { @@ -1672,10 +1654,10 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* If it is negative or too big */ if (delta<0 || delta> 255) { /* Big one */ - status = janus_rtp_packet_status_largeornegativedelta; + status = janus_bwe_twcc_status_largeornegativedelta; } else { /* Small */ - status = janus_rtp_packet_status_smalldelta; + status = janus_bwe_twcc_status_smalldelta; } /* Store delta */ /* Overflows are possible here */ @@ -1685,7 +1667,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr } /* Check if all previoues ones were equal and this one the first different */ - if (all_same && last_status!=janus_rtp_packet_status_reserved && status!=last_status) { + if (all_same && last_status != janus_bwe_twcc_status_reserved && status != last_status) { /* How big was the same run */ if (g_queue_get_length(statuses)>7) { guint32 word = 0; @@ -1707,8 +1689,8 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* Remove all statuses */ g_queue_clear(statuses); /* Reset status */ - last_status = janus_rtp_packet_status_reserved; - max_status = janus_rtp_packet_status_notreceived; + last_status = janus_bwe_twcc_status_reserved; + max_status = janus_bwe_twcc_status_notreceived; all_same = TRUE; } else { /* Not same */ @@ -1730,7 +1712,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* Check if we can still be enqueuing for a run */ if (!all_same) { /* Check */ - if (!all_same && max_status==janus_rtp_packet_status_largeornegativedelta && g_queue_get_length(statuses)>6) { + if (!all_same && max_status == janus_bwe_twcc_status_largeornegativedelta && g_queue_get_length(statuses)>6) { guint32 word = 0; /* 0 1 @@ -1747,7 +1729,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr size_t i = 0; for (i=0;i<7;++i) { /* Get status */ - janus_rtp_packet_status status = (janus_rtp_packet_status) GPOINTER_TO_UINT(g_queue_pop_head (statuses)); + janus_bwe_twcc_status status = (janus_bwe_twcc_status) GPOINTER_TO_UINT(g_queue_pop_head (statuses)); /* Write */ word = janus_push_bits(word, 2, (guint8)status); } @@ -1755,21 +1737,21 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr janus_set2(data, len, word); len += 2; /* Reset */ - last_status = janus_rtp_packet_status_reserved; - max_status = janus_rtp_packet_status_notreceived; + last_status = janus_bwe_twcc_status_reserved; + max_status = janus_bwe_twcc_status_notreceived; all_same = TRUE; /* We need to restore the values, as there may be more elements on the buffer */ for (i=0; imax_status) { /* Store it */ max_status = status; } //Check if it is the same */ - if (all_same && last_status!=janus_rtp_packet_status_reserved && status!=last_status) { + if (all_same && last_status != janus_bwe_twcc_status_reserved && status!=last_status) { /* Not the same */ all_same = FALSE; } @@ -1793,7 +1775,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr guint32 i = 0; for (i=0;i<14;++i) { /* Get status */ - janus_rtp_packet_status status = (janus_rtp_packet_status) GPOINTER_TO_UINT(g_queue_pop_head (statuses)); + janus_bwe_twcc_status status = (janus_bwe_twcc_status) GPOINTER_TO_UINT(g_queue_pop_head (statuses)); /* Write */ word = janus_push_bits(word, 1, (guint8)status); } @@ -1801,8 +1783,8 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr janus_set2(data, len, word); len += 2; /* Reset */ - last_status = janus_rtp_packet_status_reserved; - max_status = janus_rtp_packet_status_notreceived; + last_status = janus_bwe_twcc_status_reserved; + max_status = janus_bwe_twcc_status_notreceived; all_same = TRUE; } } @@ -1828,7 +1810,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* Write word */ janus_set2(data, len, word); len += 2; - } else if (max_status == janus_rtp_packet_status_largeornegativedelta) { + } else if (max_status == janus_bwe_twcc_status_largeornegativedelta) { guint32 word = 0; /* Write chunk */ word = janus_push_bits(word, 1, 1); @@ -1837,7 +1819,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr unsigned int i = 0; for (i=0;i #include +#include "bwe.h" + /*! \brief RTCP Packet Types (http://www.networksorcery.com/enp/protocol/rtcp.htm) */ typedef enum { RTCP_FIR = 192, @@ -287,14 +289,12 @@ typedef struct rtcp_context typedef rtcp_context janus_rtcp_context; /*! \brief Stores transport wide packet reception statistics */ -typedef struct rtcp_transport_wide_cc_stats -{ +typedef struct janus_rtcp_transport_wide_cc_stats { /*! \brief Transwport wide sequence number */ guint32 transport_seq_num; /*! \brief Reception time */ guint64 timestamp; -} rtcp_transport_wide_cc_stats; -typedef rtcp_transport_wide_cc_stats janus_rtcp_transport_wide_cc_stats; +} janus_rtcp_transport_wide_cc_stats; /*! \brief Method to retrieve the estimated round-trip time from an existing RTCP context * @param[in] ctx The RTCP context to query @@ -379,10 +379,11 @@ gboolean janus_is_rtcp(char *buf, guint len); /*! \brief Method to parse/validate an RTCP message * @param[in] ctx RTCP context to update, if needed (optional) + * @param[in] bwe Bandwidth estimation context to update, if needed (optional) * @param[in] packet The message data * @param[in] len The message data length in bytes * @returns 0 in case of success, -1 on errors */ -int janus_rtcp_parse(janus_rtcp_context *ctx, char *packet, int len); +int janus_rtcp_parse(janus_rtcp_context *ctx, janus_bwe_context *bwe, char *packet, int len); /*! \brief Method to fix incoming RTCP SR and RR data * @param[in] packet The message data @@ -398,13 +399,14 @@ int janus_rtcp_fix_report_data(char *packet, int len, uint32_t base_ts, uint32_t /*! \brief Method to fix an RTCP message (http://tools.ietf.org/html/draft-ietf-straw-b2bua-rtcp-00) * @param[in] ctx RTCP context to update, if needed (optional) + * @param[in] bwe Bandwidth estimation context to update, if needed (optional) * @param[in] packet The message data * @param[in] len The message data length in bytes * @param[in] fixssrc Whether the method needs to fix the message or just parse it * @param[in] newssrcl The SSRC of the sender to put in the message * @param[in] newssrcr The SSRC of the receiver to put in the message * @returns 0 in case of success, -1 on errors */ -int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, char *packet, int len, int fixssrc, uint32_t newssrcl, uint32_t newssrcr); +int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, janus_bwe_context *bwe, char *packet, int len, int fixssrc, uint32_t newssrcl, uint32_t newssrcr); /*! \brief Method to filter an outgoing RTCP message (http://tools.ietf.org/html/draft-ietf-straw-b2bua-rtcp-00) * @param[in] packet The message data diff --git a/src/sdp.c b/src/sdp.c index 4136877814..b5ed350313 100644 --- a/src/sdp.c +++ b/src/sdp.c @@ -575,6 +575,9 @@ int janus_sdp_process_remote(void *ice_handle, janus_sdp *remote_sdp, gboolean r if(a->value && strstr(a->value, "nack") && medium) { /* Enable NACKs */ medium->do_nacks = TRUE; + } else if(a->value && strstr(a->value, "transport-cc") && medium) { + /* Enable TWCC */ + medium->do_twcc = TRUE; } } else if(!strcasecmp(a->name, "fmtp")) { if(a->value && strstr(a->value, "apt=")) { From 38003df2d787e5500a3c11062f604f1f8d21fec6 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 6 Jul 2023 15:01:29 +0200 Subject: [PATCH 02/22] Also keep track of monotonic time of delivery of each inflight packet --- src/bwe.c | 5 +++-- src/bwe.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 3157b28c57..654cb063c1 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -56,8 +56,9 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, return FALSE; janus_bwe_twcc_inflight *stat = g_malloc(sizeof(janus_bwe_twcc_inflight)); stat->seq = seq; - stat->delta_us = bwe->sent_ts ? (sent - bwe->sent_ts) : 0; - bwe->sent_ts = sent; + stat->sent_ts = sent; + stat->delta_us = bwe->last_sent_ts ? (sent - bwe->last_sent_ts) : 0; + bwe->last_sent_ts = sent; stat->type = type; stat->size = size; g_hash_table_insert(bwe->packets, GUINT_TO_POINTER(seq), stat); diff --git a/src/bwe.h b/src/bwe.h index ce04229d2a..18123cd477 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -40,7 +40,9 @@ typedef enum janus_bwe_packet_type { typedef struct janus_bwe_twcc_inflight { /*! \brief The TWCC sequence number */ uint16_t seq; - /*! \brief Delta (in us) since the delivery of the previous packet */ + /*! \brief Monotonic time this packet was delivered */ + int64_t sent_ts; + /*! \brief Delta (in us) since the delivery of the previous packet */ int64_t delta_us; /*! \brief Type of packet (e.g., regular or rtx) */ janus_bwe_packet_type type; @@ -51,7 +53,7 @@ typedef struct janus_bwe_twcc_inflight { /*! \brief Bandwidth estimation context */ typedef struct janus_bwe_context { /*! \brief Monotonic timestamp of the last sent packet */ - int64_t sent_ts; + int64_t last_sent_ts; /*! \brief Map of in-flight packets */ GHashTable *packets; /*! \brief Monotonic timestamp of when we last computed the bitrates */ From 503646ff8c14f34e4999dbc4023cdae3d1352379 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 1 Aug 2023 13:13:41 +0200 Subject: [PATCH 03/22] Added first draft of basic bitrate tracker for BWE purposes --- src/bwe.c | 63 ++++++++++++++++++++++++++++++++++-- src/bwe.h | 35 ++++++++++++++++++++ src/plugins/janus_echotest.c | 27 ++++++++++++++++ src/rtp.c | 28 ++++++++++++---- src/rtp.h | 4 +++ 5 files changed, 147 insertions(+), 10 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 654cb063c1..8f9793a307 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -61,6 +61,7 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, bwe->last_sent_ts = sent; stat->type = type; stat->size = size; + bwe->sent_bytes += size; g_hash_table_insert(bwe->packets, GUINT_TO_POINTER(seq), stat); return TRUE; } @@ -73,9 +74,9 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, janus_bwe_twcc_inflight *p = g_hash_table_lookup(bwe->packets, GUINT_TO_POINTER(seq)); if(p == NULL) return; - bwe->sent_bytes += p->size; if(status != janus_bwe_twcc_status_notreceived) bwe->received_bytes += p->size; + bwe->delay += (delta_us && p) ? (delta_us - p->delta_us) : 0; /* Print summary */ JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNu32"us) (send: %"SCNi64"us)\n", seq, janus_bwe_twcc_status_description(status), delta_us, p ? ((p->delta_us/250)*250) : 0); @@ -85,10 +86,66 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, bwe->bitrate_ts = now; if(now - bwe->bitrate_ts >= G_USEC_PER_SEC) { /* It is, show the outgoing and (acked) incoming bitrate */ - JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps\n", - (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8); + JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, delay=%"SCNi64"\n", + (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, bwe->delay); bwe->bitrate_ts += G_USEC_PER_SEC; bwe->sent_bytes = 0; bwe->received_bytes = 0; + bwe->delay = 0; } } + +janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void) { + janus_bwe_stream_bitrate *bwe_sb = g_malloc0(sizeof(janus_bwe_stream_bitrate)); + janus_mutex_init(&bwe_sb->mutex); + return bwe_sb; +} + +void janus_bwe_stream_bitrate_update(janus_bwe_stream_bitrate *bwe_sb, int64_t when, int sl, int tl, int size) { + if(bwe_sb == NULL || sl < 0 || sl > 2 || tl > 2) + return; + if(tl < 0) + tl = 0; + int i = 0; + int64_t cleanup_ts = when - G_USEC_PER_SEC; + janus_mutex_lock(&bwe_sb->mutex); + for(i=tl; i<3; i++) { + if(i <= tl && bwe_sb->packets[sl*3 + i] == NULL) + bwe_sb->packets[sl*3 + i] = g_queue_new(); + if(bwe_sb->packets[sl*3 + i] == NULL) + continue; + /* Check if we need to get rid of some old packets */ + janus_bwe_stream_packet *sp = g_queue_peek_head(bwe_sb->packets[sl*3 + i]); + while(sp && sp->sent_ts < cleanup_ts) { + sp = g_queue_pop_head(bwe_sb->packets[sl*3 + i]); + if(bwe_sb->bitrate[sl*3 + i] >= sp->size) + bwe_sb->bitrate[sl*3 + i] -= sp->size; + g_free(sp); + sp = g_queue_peek_head(bwe_sb->packets[sl*3 + i]); + } + /* Check if there's anything new we need to add now */ + if(size > 0) { + sp = g_malloc(sizeof(janus_bwe_stream_packet)); + sp->sent_ts = when; + sp->size = size*8; + bwe_sb->bitrate[sl*3 + i] += sp->size; + g_queue_push_tail(bwe_sb->packets[sl*3 + i], sp); + } + } + janus_mutex_unlock(&bwe_sb->mutex); +} + +void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb) { + if(bwe_sb == NULL) + return; + int i = 0; + janus_mutex_lock(&bwe_sb->mutex); + for(i=0; i<9; i++) { + if(bwe_sb->packets[i] != NULL) + g_queue_free_full(bwe_sb->packets[i], (GDestroyNotify)g_free); + } + bwe_sb->packets[i] = NULL; + janus_mutex_unlock(&bwe_sb->mutex); + janus_mutex_destroy(&bwe_sb->mutex); + g_free(bwe_sb); +} diff --git a/src/bwe.h b/src/bwe.h index 18123cd477..3a920206ae 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -14,6 +14,7 @@ #include +#include "mutex.h" /*! \brief Transport Wide CC statuses */ typedef enum janus_bwe_twcc_status { @@ -60,6 +61,8 @@ typedef struct janus_bwe_context { int64_t bitrate_ts; /*! \brief Amount of bytes we've sent and the ones we've had feedback were received */ uint32_t sent_bytes, received_bytes; + /*! \brief How much delay has been accumulated (may be negative) */ + int64_t delay; } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ @@ -85,4 +88,36 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, uint16_t seq, janus_bwe_twcc_status status, uint32_t delta_us); +/*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ +typedef struct janus_bwe_stream_bitrate { + /*! \brief Time based queue of packet sizes */ + GQueue *packets[9]; + /*! \brief Current bitrate */ + uint32_t bitrate[9]; + /*! \brief Mutex to lock this instance */ + janus_mutex mutex; +} janus_bwe_stream_bitrate; +/*! \brief Helper method to create a new janus_bwe_stream_bitrate instance + * @returns A janus_bwe_stream_bitrate instance, if successful, or NULL otherwise */ +janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void); +/*! \brief Helper method to update an existing janus_bwe_stream_bitrate instance with new data + * \note Passing \c -1 or \c 0 as size just updates the queue to get rid of older values + * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to update + * @param[in] when Timestamp of the packet + * @param[in] sl Substream or spatial layer of the packet (can be 0 for audio) + * @param[in] sl Temporal layer of the packet (can be 0 for audio) + * @param[in] size Size of the packet */ +void janus_bwe_stream_bitrate_update(janus_bwe_stream_bitrate *bwe_sb, int64_t when, int sl, int tl, int size); +/*! \brief Helper method to destroy an existing janus_bwe_stream_bitrate instance + * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to destroy */ +void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb); + +/*! \brief Packet size and time */ +typedef struct janus_bwe_stream_packet { + /*! \brief Timestamp */ + int64_t sent_ts; + /*! \brief Size of packet */ + uint16_t size; +} janus_bwe_stream_packet; + #endif diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 4ef6907014..17cb009e47 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -125,6 +125,7 @@ #include "../rtcp.h" #include "../sdp-utils.h" #include "../utils.h" +#include "../bwe.h" /* Plugin information */ @@ -253,6 +254,7 @@ typedef struct janus_echotest_session { janus_vp8_simulcast_context vp8_context; gboolean svc; janus_rtp_svc_context svc_context; + janus_bwe_stream_bitrate *ab, *vb; janus_recorder *arc; /* The Janus recorder instance for this user's audio, if enabled */ janus_recorder *vrc; /* The Janus recorder instance for this user's video, if enabled */ janus_recorder *drc; /* The Janus recorder instance for this user's data, if enabled */ @@ -525,6 +527,18 @@ json_t *janus_echotest_query_session(janus_plugin_session *handle) { json_object_set_new(info, "temporal-layer", json_integer(session->svc_context.temporal)); json_object_set_new(info, "temporal-layer-target", json_integer(session->svc_context.temporal_target)); } + if(session->ab && session->ab->packets[0]) + json_object_set_new(info, "audio-bitrate", json_integer(session->ab->bitrate[0])); + if(session->vb) { + int i=0; + for(i=0; i<9; i++) { + if(session->vb->packets[i]) { + char name[20]; + g_snprintf(name, sizeof(name), "video-bitrate-%d", i); + json_object_set_new(info, name, json_integer(session->vb->bitrate[i])); + } + } + } if(session->arc || session->vrc || session->drc) { json_t *recording = json_object(); if(session->arc && session->arc->filename) @@ -588,6 +602,8 @@ void janus_echotest_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&sessions_mutex); return; } + session->ab = janus_bwe_stream_bitrate_create(); + session->vb = janus_bwe_stream_bitrate_create(); g_atomic_int_set(&session->hangingup, 0); janus_mutex_unlock(&sessions_mutex); /* We really don't care, as we only send RTP/RTCP we get in the first place back anyway */ @@ -626,6 +642,8 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context, buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex); + janus_bwe_stream_bitrate_update(session->vb, janus_get_monotonic_time(), + session->sim_context.substream_last, session->sim_context.temporal_last, len); } else { /* Process this SVC packet: don't relay if it's not the layer we wanted to handle */ relay = janus_rtp_svc_context_process_rtp(&session->svc_context, @@ -695,6 +713,11 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp if((!video && session->audio_active) || (video && session->video_active)) { /* Save the frame if we're recording */ janus_recorder_save_frame(video ? session->vrc : session->arc, buf, len); + /* Update the bitrate counter */ + if(video) + janus_bwe_stream_bitrate_update(session->vb, janus_get_monotonic_time(), 0, 0, len); + else + janus_bwe_stream_bitrate_update(session->ab, janus_get_monotonic_time(), 0, 0, len); /* Send the frame back */ gateway->relay_rtp(handle, packet); } @@ -903,6 +926,10 @@ static void janus_echotest_hangup_media_internal(janus_plugin_session *handle) { janus_rtp_switching_context_reset(&session->context); janus_rtp_simulcasting_context_reset(&session->sim_context); janus_vp8_simulcast_context_reset(&session->vp8_context); + janus_bwe_stream_bitrate_destroy(session->ab); + session->ab = NULL; + janus_bwe_stream_bitrate_destroy(session->vb); + session->vb = NULL; session->min_delay = -1; session->max_delay = -1; g_atomic_int_set(&session->hangingup, 0); diff --git a/src/rtp.c b/src/rtp.c index 086ff19054..81d84ed323 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1088,6 +1088,8 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex) { if(!context || !buf || len < 1) return FALSE; + context->substream_last = -1; + context->temporal_last = -1; janus_rtp_header *header = (janus_rtp_header *)buf; uint32_t ssrc = ntohl(header->ssrc); int substream = -1; @@ -1126,6 +1128,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte return FALSE; } } + context->substream_last = substream; /* Reset the flags */ context->changed_substream = FALSE; context->changed_temporal = FALSE; @@ -1146,6 +1149,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte } int target = (context->substream_target_temp == -1) ? context->substream_target : context->substream_target_temp; /* Check what we need to do with the packet */ + gboolean relay = TRUE; if(context->substream == -1) { if((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) || (vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) || @@ -1158,7 +1162,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte context->last_relayed = now; } else { /* Don't relay anything until we get a keyframe */ - return FALSE; + relay = FALSE; } } else if(context->substream != target) { /* We're not on the substream we'd like: let's wait for a keyframe on the target */ @@ -1178,10 +1182,10 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte } } /* If we haven't received our desired substream yet, let's drop temporarily */ - if(context->last_relayed == 0) { + if(relay && context->last_relayed == 0) { /* Let's start slow */ context->last_relayed = now; - } else if(context->substream > 0) { + } else if(relay && context->substream > 0) { /* Check if too much time went by with no packet relayed */ if((now - context->last_relayed) > (context->drop_trigger ? context->drop_trigger : 250000)) { context->last_relayed = now; @@ -1207,12 +1211,13 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte /* Do we need to drop this? */ if(context->substream < 0) return FALSE; - if(substream != context->substream) { + if(relay && substream != context->substream) { JANUS_LOG(LOG_HUGE, "Dropping packet (it's from SSRC %"SCNu32", but we're only relaying SSRC %"SCNu32" now\n", ssrc, *(ssrcs + context->substream)); - return FALSE; + relay = FALSE; } - context->last_relayed = janus_get_monotonic_time(); + if(relay) + context->last_relayed = janus_get_monotonic_time(); /* Temporal layers are only easily available for some codecs */ if(vcodec == JANUS_VIDEOCODEC_VP8) { /* Check if there's any temporal scalability to take into account */ @@ -1224,6 +1229,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte uint8_t keyidx = 0; if(janus_vp8_parse_descriptor(payload, plen, &m, &picid, &tlzi, &tid, &ybit, &keyidx) == 0) { //~ JANUS_LOG(LOG_WARN, "%"SCNu16", %u, %u, %u, %u\n", picid, tlzi, tid, ybit, keyidx); + context->temporal_last = tid; + if(!relay) + return FALSE; if(context->templayer != context->templayer_target && tid == context->templayer_target) { /* FIXME We should be smarter in deciding when to switch */ context->templayer = context->templayer_target; @@ -1244,6 +1252,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte gboolean found = FALSE; janus_vp9_svc_info svc_info = { 0 }; if(janus_vp9_parse_svc(payload, plen, &found, &svc_info) == 0 && found) { + context->temporal_last = svc_info.temporal_layer; + if(!relay) + return FALSE; int temporal_layer = context->templayer; if(context->templayer_target > context->templayer) { /* We need to upscale */ @@ -1280,6 +1291,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(janus_av1_svc_context_process_dd(av1ctx, dd_content, dd_len, &template, NULL)) { janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template)); if(t) { + context->temporal_last = t->temporal; + if(!relay) + return FALSE; int temporal_layer = context->templayer; if(context->templayer_target > context->templayer) { /* We need to upscale */ @@ -1308,7 +1322,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte } } /* If we got here, the packet can be relayed */ - return TRUE; + return relay; } /* VP9 SVC */ diff --git a/src/rtp.h b/src/rtp.h index cebb8b3c86..43c76fe9d3 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -356,10 +356,14 @@ typedef struct janus_rtp_simulcasting_context { int substream; /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ int substream_target, substream_target_temp; + /*! \brief Last substream that was identified by janus_rtp_simulcasting_context_process_rtp */ + int substream_last; /*! \brief Which simulcast temporal layer we should forward back */ int templayer; /*! \brief As above, but to handle transitions (e.g., wait for keyframe) */ int templayer_target; + /*! \brief Last temporal layer that was identified by janus_rtp_simulcasting_context_process_rtp */ + int temporal_last; /*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */ guint32 drop_trigger; /*! \brief When we relayed the last packet (used to detect when substreams become unavailable) */ From 9668640cd01eadb5a1f0899502693ceec5e8f616 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 3 Aug 2023 12:01:13 +0200 Subject: [PATCH 04/22] WIP: fixes to parsing and stats evaluation - parse recv delta as signed int from feedbacks - detect first recv delta in a chunk - fix cleanup of pkt queues - evaluate average delay in a time window of 1 second --- src/bwe.c | 31 +++++++++++++++++++------------ src/bwe.h | 7 +++++-- src/rtcp.c | 20 +++++++++++++------- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 8f9793a307..d52055d3db 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -67,31 +67,38 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, } void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, - uint16_t seq, janus_bwe_twcc_status status, uint32_t delta_us) { + uint16_t seq, janus_bwe_twcc_status status, int64_t delta_us, gboolean first) { if(bwe == NULL) return; /* Find the inflight information we stored when sending this packet */ janus_bwe_twcc_inflight *p = g_hash_table_lookup(bwe->packets, GUINT_TO_POINTER(seq)); if(p == NULL) return; - if(status != janus_bwe_twcc_status_notreceived) + if(status != janus_bwe_twcc_status_notreceived) { bwe->received_bytes += p->size; - bwe->delay += (delta_us && p) ? (delta_us - p->delta_us) : 0; - /* Print summary */ - JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNu32"us) (send: %"SCNi64"us)\n", seq, - janus_bwe_twcc_status_description(status), delta_us, p ? ((p->delta_us/250)*250) : 0); + bwe->received_pkts++; + } + /* The first recv delta is relative to the reference time, not to the previous packet */ + if(!first) { + int64_t rounded_delta_us = (p->delta_us / 250) * 250; + int64_t diff_us = delta_us - rounded_delta_us; + bwe->delay += diff_us; + JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNi64"us) (send: %"SCNi64"us) diff_us=%"SCNi64"\n", seq, + janus_bwe_twcc_status_description(status), delta_us, rounded_delta_us, diff_us); + } /* Check if it's time to compute the bitrates */ int64_t now = janus_get_monotonic_time(); if(bwe->bitrate_ts == 0) bwe->bitrate_ts = now; if(now - bwe->bitrate_ts >= G_USEC_PER_SEC) { /* It is, show the outgoing and (acked) incoming bitrate */ - JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, delay=%"SCNi64"\n", - (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, bwe->delay); + JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, avg_delay=%.2fms\n", + (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, ((double)bwe->delay / (double)bwe->received_pkts) / 1000); bwe->bitrate_ts += G_USEC_PER_SEC; bwe->sent_bytes = 0; bwe->received_bytes = 0; bwe->delay = 0; + bwe->received_pkts = 0; } } @@ -138,13 +145,13 @@ void janus_bwe_stream_bitrate_update(janus_bwe_stream_bitrate *bwe_sb, int64_t w void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb) { if(bwe_sb == NULL) return; - int i = 0; janus_mutex_lock(&bwe_sb->mutex); - for(i=0; i<9; i++) { - if(bwe_sb->packets[i] != NULL) + for(int i=0; i<9; i++) { + if(bwe_sb->packets[i] != NULL) { g_queue_free_full(bwe_sb->packets[i], (GDestroyNotify)g_free); + bwe_sb->packets[i] = NULL; + } } - bwe_sb->packets[i] = NULL; janus_mutex_unlock(&bwe_sb->mutex); janus_mutex_destroy(&bwe_sb->mutex); g_free(bwe_sb); diff --git a/src/bwe.h b/src/bwe.h index 3a920206ae..8245cac427 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -63,6 +63,8 @@ typedef struct janus_bwe_context { uint32_t sent_bytes, received_bytes; /*! \brief How much delay has been accumulated (may be negative) */ int64_t delay; + /*! \brief Number of packets with a received status */ + uint16_t received_pkts; } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ @@ -84,9 +86,10 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, * @param[in] bwe The janus_bwe_context instance to update * @param[in] seq The TWCC sequence number of the inflight packet we have feedback for * @param[in] status Feedback status for the packet - * @param[in] delta_us If the packet was received, the delta that was provided */ + * @param[in] delta_us If the packet was received, the delta that was provided + * @param[in] first True if this is the first received packet in a twcc feedback */ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, - uint16_t seq, janus_bwe_twcc_status status, uint32_t delta_us); + uint16_t seq, janus_bwe_twcc_status status, int64_t delta_us, gboolean first); /*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ typedef struct janus_bwe_stream_bitrate { diff --git a/src/rtcp.c b/src/rtcp.c index 858fded47b..28dfa49173 100644 --- a/src/rtcp.c +++ b/src/rtcp.c @@ -281,9 +281,10 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_bwe_ /* Iterate on all recv deltas */ JANUS_LOG(LOG_HUGE, "[TWCC] Recv Deltas (%d/%"SCNu16"):\n", g_list_length(list), status_count); num = 0; - uint16_t delta = 0, seq = 0; - uint32_t delta_us = 0; + int16_t delta = 0, seq = 0; + int64_t delta_us = 0; GList *iter = list; + gboolean first_recv_found = FALSE; while(iter != NULL && total > 0) { num++; delta = 0; @@ -291,22 +292,27 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_bwe_ /* Get the delta */ s = GPOINTER_TO_UINT(iter->data); if(s == janus_bwe_twcc_status_smalldelta) { - /* Small delta = 1 byte */ - delta = *data; + /* Small delta = unsigned 1 byte */ + delta = (int16_t)*data; total--; data++; } else if(s == janus_bwe_twcc_status_largeornegativedelta) { - /* Large or negative delta = 2 bytes */ + /* Large or negative delta = signed 2 bytes */ if(total < 2) break; - memcpy(&delta, data, sizeof(uint16_t)); + memcpy(&delta, data, sizeof(int16_t)); delta = ntohs(delta); total -= 2; data += 2; } delta_us = delta*250; + gboolean first = FALSE; + if(!first_recv_found && s != janus_bwe_twcc_status_notreceived) { + first_recv_found = TRUE; + first = TRUE; + } /* Pass the feedback to the bandwidth estimation context */ - janus_bwe_context_handle_feedback(bwe, seq, s, delta_us); + janus_bwe_context_handle_feedback(bwe, seq, s, delta_us, first); /* Move to the next feedback */ iter = iter->next; } From c46d7c806ee871573bd36c62dcadb256e4915936 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 3 Aug 2023 13:13:51 +0200 Subject: [PATCH 05/22] Move trigger to update BWE stats to handle loop source, and added placeholder plugin callback --- src/bwe.c | 30 ++++++++++++++++++++---------- src/bwe.h | 7 ++++++- src/ice.c | 30 ++++++++++++++++++++++++++++++ src/ice.h | 2 +- src/plugins/janus_echotest.c | 24 ++++++++++++++++++++++++ src/plugins/plugin.h | 6 ++++++ 6 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index d52055d3db..7f82059e82 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -86,20 +86,30 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNi64"us) (send: %"SCNi64"us) diff_us=%"SCNi64"\n", seq, janus_bwe_twcc_status_description(status), delta_us, rounded_delta_us, diff_us); } - /* Check if it's time to compute the bitrates */ +} + +void janus_bwe_context_update(janus_bwe_context *bwe) { + if(bwe == NULL) + return; int64_t now = janus_get_monotonic_time(); if(bwe->bitrate_ts == 0) bwe->bitrate_ts = now; - if(now - bwe->bitrate_ts >= G_USEC_PER_SEC) { - /* It is, show the outgoing and (acked) incoming bitrate */ - JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, avg_delay=%.2fms\n", - (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, ((double)bwe->delay / (double)bwe->received_pkts) / 1000); - bwe->bitrate_ts += G_USEC_PER_SEC; - bwe->sent_bytes = 0; - bwe->received_bytes = 0; - bwe->delay = 0; - bwe->received_pkts = 0; + /* Reset the outgoing and (acked) incoming bitrate, and estimate the bitrate */ + if(now > bwe->bitrate_ts) { + /* TODO Actually estimate the bitrate: now we're just checking how + * much the peer received out of what we sent, which is not enough */ + int64_t diff = now - bwe->bitrate_ts; + float ratio = (float)G_USEC_PER_SEC / (float)diff; + float estimate = ratio * bwe->received_bytes * 8; + bwe->estimate = estimate; + bwe->bitrate_ts = now; } + JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, avg_delay=%.2fms\n", + (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, ((double)bwe->delay / (double)bwe->received_pkts) / 1000); + bwe->sent_bytes = 0; + bwe->received_bytes = 0; + bwe->delay = 0; + bwe->received_pkts = 0; } janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void) { diff --git a/src/bwe.h b/src/bwe.h index 8245cac427..79ea2a87ca 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -65,6 +65,8 @@ typedef struct janus_bwe_context { int64_t delay; /*! \brief Number of packets with a received status */ uint16_t received_pkts; + /*! \brief Latest estimated bitrate */ + uint32_t estimate; } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ @@ -87,9 +89,12 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, * @param[in] seq The TWCC sequence number of the inflight packet we have feedback for * @param[in] status Feedback status for the packet * @param[in] delta_us If the packet was received, the delta that was provided - * @param[in] first True if this is the first received packet in a twcc feedback */ + * @param[in] first True if this is the first received packet in a TWCC feedback */ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, uint16_t seq, janus_bwe_twcc_status status, int64_t delta_us, gboolean first); +/*! \brief Update the internal BWE context state with a new tick + * @param[in] bwe The janus_bwe_context instance to update */ +void janus_bwe_context_update(janus_bwe_context *bwe); /*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ typedef struct janus_bwe_stream_bitrate { diff --git a/src/ice.c b/src/ice.c index 4b175390c9..95311e5025 100644 --- a/src/ice.c +++ b/src/ice.c @@ -527,6 +527,7 @@ typedef struct janus_ice_outgoing_traffic { } janus_ice_outgoing_traffic; static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data); static gboolean janus_ice_outgoing_stats_handle(gpointer user_data); +static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data); static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janus_ice_queued_packet *pkt); static gboolean janus_ice_outgoing_traffic_prepare(GSource *source, gint *timeout) { janus_ice_outgoing_traffic *t = (janus_ice_outgoing_traffic *)source; @@ -4459,6 +4460,25 @@ static gboolean janus_ice_outgoing_stats_handle(gpointer user_data) { return G_SOURCE_CONTINUE; } +static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { + janus_ice_handle *handle = (janus_ice_handle *)user_data; + janus_ice_peerconnection *pc = handle->pc; + if(pc == NULL) + return G_SOURCE_CONTINUE; + /* This callback is for updating the state of the bandwidth estimator */ + if(pc->bwe != NULL) { + janus_bwe_context_update(pc->bwe); + if(pc->bwe->estimate > 0) { + /* Notify the plugin about the current value */ + janus_plugin *plugin = (janus_plugin *)handle->app; + if(plugin && plugin->estimated_bandwidth && janus_plugin_session_is_alive(handle->app_handle) && + !g_atomic_int_get(&handle->destroyed)) + plugin->estimated_bandwidth(handle->app_handle, pc->bwe->estimate); + } + } + return G_SOURCE_CONTINUE; +} + static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janus_ice_queued_packet *pkt) { janus_session *session = (janus_session *)handle->session; janus_ice_peerconnection *pc = handle->pc; @@ -4556,6 +4576,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu g_source_unref(handle->twcc_source); handle->twcc_source = NULL; } + if(handle->bwe_source) { + g_source_destroy(handle->bwe_source); + g_source_unref(handle->bwe_source); + handle->bwe_source = NULL; + } if(handle->stats_source) { g_source_destroy(handle->stats_source); g_source_unref(handle->stats_source); @@ -5235,6 +5260,11 @@ void janus_ice_dtls_handshake_done(janus_ice_handle *handle) { g_source_set_callback(handle->stats_source, janus_ice_outgoing_stats_handle, handle, NULL); g_source_set_priority(handle->stats_source, G_PRIORITY_DEFAULT); g_source_attach(handle->stats_source, handle->mainctx); + /* FIXME Finally, let's create a source for BWE */ + handle->bwe_source = g_timeout_source_new(1000); + g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); + g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); + g_source_attach(handle->bwe_source, handle->mainctx); janus_mutex_unlock(&handle->mutex); JANUS_LOG(LOG_INFO, "[%"SCNu64"] The DTLS handshake has been completed\n", handle->handle_id); /* Notify the plugin that the WebRTC PeerConnection is ready to be used */ diff --git a/src/ice.h b/src/ice.h index 48c19c9b38..202a95f08a 100644 --- a/src/ice.h +++ b/src/ice.h @@ -374,7 +374,7 @@ struct janus_ice_handle { /*! \brief GLib thread for the handle and libnice */ GThread *thread; /*! \brief GLib sources for outgoing traffic, recurring RTCP, and stats (and optionally TWCC) */ - GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source; + GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source, *bwe_source; /*! \brief libnice ICE agent */ NiceAgent *agent; /*! \brief Monotonic time of when the ICE agent has been created */ diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 17cb009e47..d35bc407b6 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -156,6 +156,7 @@ void janus_echotest_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtc void janus_echotest_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_echotest_data_ready(janus_plugin_session *handle); void janus_echotest_slow_link(janus_plugin_session *handle, int mindex, gboolean video, gboolean uplink); +void janus_echotest_estimated_bandwidth(janus_plugin_session *handle, uint32_t estimate); void janus_echotest_hangup_media(janus_plugin_session *handle); void janus_echotest_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_echotest_query_session(janus_plugin_session *handle); @@ -183,6 +184,7 @@ static janus_plugin janus_echotest_plugin = .incoming_data = janus_echotest_incoming_data, .data_ready = janus_echotest_data_ready, .slow_link = janus_echotest_slow_link, + .estimated_bandwidth = janus_echotest_estimated_bandwidth, .hangup_media = janus_echotest_hangup_media, .destroy_session = janus_echotest_destroy_session, .query_session = janus_echotest_query_session, @@ -854,6 +856,28 @@ void janus_echotest_slow_link(janus_plugin_session *handle, int mindex, gboolean janus_refcount_decrease(&session->ref); } +void janus_echotest_estimated_bandwidth(janus_plugin_session *handle, uint32_t estimate) { + /* The core is informing us that our peer got or sent too many NACKs, are we pushing media too hard? */ + if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) + return; + janus_mutex_lock(&sessions_mutex); + janus_echotest_session *session = janus_echotest_lookup_session(handle); + if(!session) { + janus_mutex_unlock(&sessions_mutex); + JANUS_LOG(LOG_ERR, "No session associated with this handle...\n"); + return; + } + if(g_atomic_int_get(&session->destroyed)) { + janus_mutex_unlock(&sessions_mutex); + return; + } + janus_refcount_increase(&session->ref); + janus_mutex_unlock(&sessions_mutex); + /* TODO Actually use this estimate for something */ + JANUS_LOG(LOG_WARN, "[echotest] BWE=%"SCNu32"\n", estimate); + janus_refcount_decrease(&session->ref); +} + static void janus_echotest_recorder_close(janus_echotest_session *session) { if(session->arc) { janus_recorder *rc = session->arc; diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 0814a541ab..9e592c099b 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -208,6 +208,7 @@ static janus_plugin janus_echotest_plugin = .incoming_data = NULL, \ .data_ready = NULL, \ .slow_link = NULL, \ + .estimated_bandwidth = NULL, \ .hangup_media = NULL, \ .destroy_session = NULL, \ .query_session = NULL, \ @@ -335,6 +336,11 @@ struct janus_plugin { * @param[in] uplink Whether this is related to the uplink (Janus to peer) * or downlink (peer to Janus) */ void (* const slow_link)(janus_plugin_session *handle, int mindex, gboolean video, gboolean uplink); + /*! \brief Callback to be notified about the estimated outgoing bandwidth + * on this PeerConnection, e.g., for simulcast/SVC automated switches + * @param[in] handle The plugin/gateway session used for this peer + * @param[in] estimate The estimated bandwidth in bps */ + void (* const estimated_bandwidth)(janus_plugin_session *handle, uint32_t estimate); /*! \brief Callback to be notified about DTLS alerts from a peer (i.e., the PeerConnection is not valid any more) * @param[in] handle The plugin/gateway session used for this peer */ void (* const hangup_media)(janus_plugin_session *handle); From 74b6256b264c87103f4b2df43a5d216f62111559 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 3 Aug 2023 16:43:08 +0200 Subject: [PATCH 06/22] Use double for bw estimate. Try to address recv delta for not consecutive packets. --- src/bwe.c | 30 ++++++++++++++++++++++-------- src/bwe.h | 2 ++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 7f82059e82..5b8e65ad42 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -72,20 +72,34 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, return; /* Find the inflight information we stored when sending this packet */ janus_bwe_twcc_inflight *p = g_hash_table_lookup(bwe->packets, GUINT_TO_POINTER(seq)); - if(p == NULL) + if(p == NULL) { + JANUS_LOG(LOG_WARN, "[BWE] [%"SCNu16"] not found in inflight packets table\n", seq); return; - if(status != janus_bwe_twcc_status_notreceived) { - bwe->received_bytes += p->size; - bwe->received_pkts++; } /* The first recv delta is relative to the reference time, not to the previous packet */ if(!first) { - int64_t rounded_delta_us = (p->delta_us / 250) * 250; + int64_t send_delta_us; + if(seq == bwe->last_recv_seq + 1) { + send_delta_us = p->delta_us; + } else { + janus_bwe_twcc_inflight *prev_p = g_hash_table_lookup(bwe->packets, GUINT_TO_POINTER(bwe->last_recv_seq)); + if(prev_p != NULL) { + send_delta_us = p->sent_ts - prev_p->sent_ts; + } else { + JANUS_LOG(LOG_WARN, "[BWE] [%"SCNu16"] not found in inflight packets table\n", bwe->last_recv_seq); + } + } + int64_t rounded_delta_us = (send_delta_us / 250) * 250; int64_t diff_us = delta_us - rounded_delta_us; bwe->delay += diff_us; JANUS_LOG(LOG_HUGE, "[BWE] [%"SCNu16"] %s (%"SCNi64"us) (send: %"SCNi64"us) diff_us=%"SCNi64"\n", seq, janus_bwe_twcc_status_description(status), delta_us, rounded_delta_us, diff_us); } + if(status != janus_bwe_twcc_status_notreceived) { + bwe->received_bytes += p->size; + bwe->received_pkts++; + bwe->last_recv_seq = seq; + } } void janus_bwe_context_update(janus_bwe_context *bwe) { @@ -99,9 +113,9 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { /* TODO Actually estimate the bitrate: now we're just checking how * much the peer received out of what we sent, which is not enough */ int64_t diff = now - bwe->bitrate_ts; - float ratio = (float)G_USEC_PER_SEC / (float)diff; - float estimate = ratio * bwe->received_bytes * 8; - bwe->estimate = estimate; + double ratio = (double)G_USEC_PER_SEC / (double)diff; + double estimate_bytes = ratio * bwe->received_bytes; + bwe->estimate = 8 * estimate_bytes; bwe->bitrate_ts = now; } JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, avg_delay=%.2fms\n", diff --git a/src/bwe.h b/src/bwe.h index 79ea2a87ca..8c23acaa08 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -55,6 +55,8 @@ typedef struct janus_bwe_twcc_inflight { typedef struct janus_bwe_context { /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; + /*! \brief Last twcc seq number of a received packet */ + uint16_t last_recv_seq; /*! \brief Map of in-flight packets */ GHashTable *packets; /*! \brief Monotonic timestamp of when we last computed the bitrates */ From 728b6fe55cdaf5cb5eb35f8e629cc997d1b81ae6 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 3 Aug 2023 16:46:17 +0200 Subject: [PATCH 07/22] Initialize int var to 0 --- src/bwe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bwe.c b/src/bwe.c index 5b8e65ad42..66caf3639f 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -78,7 +78,7 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, } /* The first recv delta is relative to the reference time, not to the previous packet */ if(!first) { - int64_t send_delta_us; + int64_t send_delta_us = 0; if(seq == bwe->last_recv_seq + 1) { send_delta_us = p->delta_us; } else { From 2301f8f742bacdcc48b2f4ee28818d2c3f57927c Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Fri, 4 Aug 2023 13:42:49 +0200 Subject: [PATCH 08/22] Track inflight packets after nice_agent_send --- src/ice.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ice.c b/src/ice.c index 95311e5025..2a18f5b50f 100644 --- a/src/ice.c +++ b/src/ice.c @@ -4868,16 +4868,16 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu handle->handle_id, janus_srtp_error_str(res), pkt->length, protected, timestamp, seq); janus_ice_free_rtp_packet(p); } else { - /* Keep track of the packet if we're using TWCC */ - if(medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) { - janus_bwe_context_add_inflight(pc->bwe, pkt->twcc_seq, janus_get_monotonic_time(), - (pkt->retransmission ? janus_bwe_packet_type_rtx : janus_bwe_packet_type_regular), protected); - } /* Shoot! */ int sent = nice_agent_send(handle->agent, pc->stream_id, pc->component_id, protected, pkt->data); if(sent < protected) { JANUS_LOG(LOG_ERR, "[%"SCNu64"] ... only sent %d bytes? (was %d)\n", handle->handle_id, sent, protected); } + /* Keep track of the packet if we're using TWCC */ + if(medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) { + janus_bwe_context_add_inflight(pc->bwe, pkt->twcc_seq, janus_get_monotonic_time(), + (pkt->retransmission ? janus_bwe_packet_type_rtx : janus_bwe_packet_type_regular), protected); + } /* Update stats */ if(sent > 0) { /* Update the RTCP context as well */ From 784c21775fd5599030e2119fac0832a5fa37ee73 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 7 Sep 2023 11:55:27 +0200 Subject: [PATCH 09/22] Further experiments, and basic integration in VideoRoom --- src/bwe.c | 55 +++++++++++-- src/bwe.h | 30 ++++++- src/ice.c | 57 ++++++++++++-- src/ice.h | 6 ++ src/janus.c | 20 +++++ src/plugins/janus_videoroom.c | 144 ++++++++++++++++++++++++++++++++++ src/plugins/plugin.h | 9 +++ 7 files changed, 308 insertions(+), 13 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 66caf3639f..1a7a083bb5 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -31,20 +31,37 @@ const char *janus_bwe_twcc_status_description(janus_bwe_twcc_status status) { return NULL; } +const char *janus_bwe_status_description(janus_bwe_status status) { + switch(status) { + case janus_bwe_status_start: + return "start"; + case janus_bwe_status_regular: + return "regular"; + case janus_bwe_status_lossy: + return "lossy"; + case janus_bwe_status_congested: + return "congested"; + case janus_bwe_status_recovering: + return "recovering"; + default: break; + } + return NULL; +} + static void janus_bwe_twcc_inflight_destroy(janus_bwe_twcc_inflight *stat) { g_free(stat); } janus_bwe_context *janus_bwe_context_create(void) { janus_bwe_context *bwe = g_malloc0(sizeof(janus_bwe_context)); - /* TODO */ + /* FIXME */ bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); return bwe; } void janus_bwe_context_destroy(janus_bwe_context *bwe) { if(bwe) { - /* TODO clean everything up */ + /* FIXME clean everything up */ g_hash_table_destroy(bwe->packets); g_free(bwe); } @@ -54,6 +71,13 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, uint16_t seq, int64_t sent, janus_bwe_packet_type type, int size) { if(bwe == NULL) return FALSE; + int64_t now = janus_get_monotonic_time(); + if(bwe->started == 0) + bwe->started = now; + if(bwe->status == janus_bwe_status_start && now - bwe->started >= G_USEC_PER_SEC) { + /* Let's move from the starting phase to the regular stage */ + bwe->status = janus_bwe_status_regular; + } janus_bwe_twcc_inflight *stat = g_malloc(sizeof(janus_bwe_twcc_inflight)); stat->seq = seq; stat->sent_ts = sent; @@ -99,6 +123,8 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, bwe->received_bytes += p->size; bwe->received_pkts++; bwe->last_recv_seq = seq; + } else { + bwe->lost_pkts++; } } @@ -115,15 +141,34 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { int64_t diff = now - bwe->bitrate_ts; double ratio = (double)G_USEC_PER_SEC / (double)diff; double estimate_bytes = ratio * bwe->received_bytes; - bwe->estimate = 8 * estimate_bytes; + uint32_t estimate = 8 * estimate_bytes; + uint16_t tot = bwe->received_pkts + bwe->lost_pkts; + if(tot > 0) + bwe->loss_ratio = (double)bwe->lost_pkts / (double)tot; + /* Depending on the severity of the loss, we change the estimate */ + if(bwe->loss_ratio > 0.05 || estimate > bwe->estimate) { + /* FIXME Lossy network? */ + bwe->status = janus_bwe_status_lossy; + bwe->estimate = estimate; + } + /* Let's also check of the average delay is increasing */ + double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; + if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.5) { + /* FIXME Delay is increasing */ + bwe->status = janus_bwe_status_congested; + bwe->estimate = estimate; + } + bwe->avg_delay = avg_delay; bwe->bitrate_ts = now; } - JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, avg_delay=%.2fms\n", - (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, ((double)bwe->delay / (double)bwe->received_pkts) / 1000); + JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, loss=%.2f%%, avg_delay=%.2fms\n", + (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, + bwe->loss_ratio, bwe->avg_delay); bwe->sent_bytes = 0; bwe->received_bytes = 0; bwe->delay = 0; bwe->received_pkts = 0; + bwe->lost_pkts = 0; } janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void) { diff --git a/src/bwe.h b/src/bwe.h index 8c23acaa08..6d2174bab3 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -51,8 +51,30 @@ typedef struct janus_bwe_twcc_inflight { int size; } janus_bwe_twcc_inflight; +/*! \brief Current status of the bandwidth estimator */ +typedef enum janus_bwe_status { + /* BWE just started */ + janus_bwe_status_start = 0, + /* BWE in the regular/increasing stage */ + janus_bwe_status_regular, + /* BWE detected too many losses */ + janus_bwe_status_lossy, + /* BWE detected congestion */ + janus_bwe_status_congested, + /* BWE recovering from losses/congestion */ + janus_bwe_status_recovering +} janus_bwe_status; +/*! \brief Helper to return a string description of a BWE status + * @param[in] status The janus_bwe_status status + * @returns A string description */ +const char *janus_bwe_status_description(janus_bwe_status status); + /*! \brief Bandwidth estimation context */ typedef struct janus_bwe_context { + /*! \brief Current status of the context */ + janus_bwe_status status; + /*! \brief Monotonic timestamp of when the BWE work started */ + int64_t started; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ @@ -65,8 +87,12 @@ typedef struct janus_bwe_context { uint32_t sent_bytes, received_bytes; /*! \brief How much delay has been accumulated (may be negative) */ int64_t delay; - /*! \brief Number of packets with a received status */ - uint16_t received_pkts; + /*! \brief Number of packets with a received status, and number of lost ones */ + uint16_t received_pkts, lost_pkts; + /*! \brief Latest average delay */ + double avg_delay; + /*! \brief Latest loss ratio */ + double loss_ratio; /*! \brief Latest estimated bitrate */ uint32_t estimate; } janus_bwe_context; diff --git a/src/ice.c b/src/ice.c index 2a18f5b50f..45cc80510a 100644 --- a/src/ice.c +++ b/src/ice.c @@ -472,6 +472,8 @@ static janus_ice_queued_packet janus_ice_add_candidates, janus_ice_dtls_handshake, janus_ice_media_stopped, + janus_ice_enable_bwe, + janus_ice_disable_bwe, janus_ice_hangup_peerconnection, janus_ice_detach_handle, janus_ice_data_ready; @@ -650,6 +652,8 @@ static void janus_ice_free_queued_packet(janus_ice_queued_packet *pkt) { if(pkt == NULL || pkt == &janus_ice_start_gathering || pkt == &janus_ice_add_candidates || pkt == &janus_ice_dtls_handshake || + pkt == &janus_ice_enable_bwe || + pkt == &janus_ice_disable_bwe || pkt == &janus_ice_media_stopped || pkt == &janus_ice_hangup_peerconnection || pkt == &janus_ice_detach_handle || @@ -3734,7 +3738,6 @@ int janus_ice_setup_local(janus_ice_handle *handle, gboolean offer, gboolean tri pc->stream_id = handle->stream_id; pc->handle = handle; pc->dtls_role = dtls_role; - pc->bwe = janus_bwe_context_create(); janus_mutex_init(&pc->mutex); if(!have_turnrest_credentials) { /* No TURN REST API server and credentials, any static ones? */ @@ -3870,6 +3873,30 @@ void janus_ice_resend_trickles(janus_ice_handle *handle) { janus_ice_notify_trickle(handle, NULL); } +void janus_ice_handle_enable_bwe(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Enabling bandwidth estimation\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_enable_bwe); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_enable_bwe); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + +void janus_ice_handle_disable_bwe(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_disable_bwe); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_disable_bwe); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_peerconnection_medium *medium, janus_ice_queued_packet *packet) { if(handle == NULL || handle->pc == NULL || medium == NULL || packet == NULL || packet->data == NULL) return; @@ -4532,6 +4559,29 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu guint id = g_source_attach(pc->dtlsrt_source, handle->mainctx); JANUS_LOG(LOG_VERB, "[%"SCNu64"] Creating retransmission timer with ID %u\n", handle->handle_id, id); return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_enable_bwe) { + /* Create a new bandwidth estimation context for this handle */ + if(pc == NULL || pc->bwe != NULL) + return G_SOURCE_CONTINUE; + pc->bwe = janus_bwe_context_create(); + /* Let's create a source for BWE */ + handle->bwe_source = g_timeout_source_new(1000); + g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); + g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); + g_source_attach(handle->bwe_source, handle->mainctx); + return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_disable_bwe) { + /* We need to get rif of the bandwidth estimator */ + if(pc == NULL || pc->bwe == NULL) + return G_SOURCE_CONTINUE; + if(handle->bwe_source) { + g_source_destroy(handle->bwe_source); + g_source_unref(handle->bwe_source); + handle->bwe_source = NULL; + } + janus_bwe_context_destroy(pc->bwe); + pc->bwe = NULL; + return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_media_stopped) { /* Some media has been disabled on the way in, so use the callback to notify the peer */ if(pc == NULL) @@ -5260,11 +5310,6 @@ void janus_ice_dtls_handshake_done(janus_ice_handle *handle) { g_source_set_callback(handle->stats_source, janus_ice_outgoing_stats_handle, handle, NULL); g_source_set_priority(handle->stats_source, G_PRIORITY_DEFAULT); g_source_attach(handle->stats_source, handle->mainctx); - /* FIXME Finally, let's create a source for BWE */ - handle->bwe_source = g_timeout_source_new(1000); - g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); - g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); - g_source_attach(handle->bwe_source, handle->mainctx); janus_mutex_unlock(&handle->mutex); JANUS_LOG(LOG_INFO, "[%"SCNu64"] The DTLS handshake has been completed\n", handle->handle_id); /* Notify the plugin that the WebRTC PeerConnection is ready to be used */ diff --git a/src/ice.h b/src/ice.h index 202a95f08a..6cbc309ad6 100644 --- a/src/ice.h +++ b/src/ice.h @@ -785,6 +785,12 @@ void janus_ice_restart(janus_ice_handle *handle); /*! \brief Method to resend all the existing candidates via trickle (e.g., after an ICE restart) * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_resend_trickles(janus_ice_handle *handle); +/*! \brief Method to dynamically enable bandwidth estimation for a handle + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_enable_bwe(janus_ice_handle *handle); +/*! \brief Method to dynamically disable bandwidth estimation for a handle + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_disable_bwe(janus_ice_handle *handle); ///@} diff --git a/src/janus.c b/src/janus.c index 156dac46e4..a41ab0ccc3 100644 --- a/src/janus.c +++ b/src/janus.c @@ -611,6 +611,8 @@ void janus_plugin_relay_data(janus_plugin_session *plugin_session, janus_plugin_ void janus_plugin_send_pli(janus_plugin_session *plugin_session); void janus_plugin_send_pli_stream(janus_plugin_session *plugin_session, int mindex); void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitrate); +void janus_plugin_enable_bwe(janus_plugin_session *plugin_session); +void janus_plugin_disable_bwe(janus_plugin_session *plugin_session); void janus_plugin_close_pc(janus_plugin_session *plugin_session); void janus_plugin_end_session(janus_plugin_session *plugin_session); void janus_plugin_notify_event(janus_plugin *plugin, janus_plugin_session *plugin_session, json_t *event); @@ -626,6 +628,8 @@ static janus_callbacks janus_handler_plugin = .send_pli = janus_plugin_send_pli, .send_pli_stream = janus_plugin_send_pli_stream, .send_remb = janus_plugin_send_remb, + .enable_bwe = janus_plugin_enable_bwe, + .disable_bwe = janus_plugin_disable_bwe, .close_pc = janus_plugin_close_pc, .end_session = janus_plugin_end_session, .events_is_enabled = janus_events_is_enabled, @@ -4254,6 +4258,22 @@ void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitra janus_ice_send_remb(handle, bitrate); } +void janus_plugin_enable_bwe(janus_plugin_session *plugin_session) { + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_handle_enable_bwe(handle); +} + +void janus_plugin_disable_bwe(janus_plugin_session *plugin_session) { + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_handle_disable_bwe(handle); +} + static gboolean janus_plugin_close_pc_internal(gpointer user_data) { /* We actually enforce the close_pc here */ janus_plugin_session *plugin_session = (janus_plugin_session *) user_data; diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index ca018ee6ae..3f1b1fe10a 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -1550,6 +1550,7 @@ void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rt void janus_videoroom_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_videoroom_data_ready(janus_plugin_session *handle); void janus_videoroom_slow_link(janus_plugin_session *handle, int mindex, gboolean video, gboolean uplink); +void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t estimate); void janus_videoroom_hangup_media(janus_plugin_session *handle); void janus_videoroom_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_videoroom_query_session(janus_plugin_session *handle); @@ -1577,6 +1578,7 @@ static janus_plugin janus_videoroom_plugin = .incoming_data = janus_videoroom_incoming_data, .data_ready = janus_videoroom_data_ready, .slow_link = janus_videoroom_slow_link, + .estimated_bandwidth = janus_videoroom_estimated_bandwidth, .hangup_media = janus_videoroom_hangup_media, .destroy_session = janus_videoroom_destroy_session, .query_session = janus_videoroom_query_session, @@ -2134,6 +2136,8 @@ typedef struct janus_videoroom_publisher_stream { char *rid[3]; /* Only needed if simulcasting is rid-based */ int rid_extmap_id; /* rid extmap ID */ janus_mutex rid_mutex; /* Mutex to protect access to the rid array and the extmap ID */ + /* Tracker of the bitrate(s) of this stream, for BWE purposes */ + janus_bwe_stream_bitrate *bitrates; /* RTP extensions, if negotiated */ guint8 audio_level_extmap_id; /* Audio level extmap ID */ guint8 video_orient_extmap_id; /* Video orientation extmap ID */ @@ -2351,6 +2355,7 @@ static void janus_videoroom_publisher_stream_free(const janus_refcount *ps_ref) g_free(ps->h264_profile); g_free(ps->vp9_profile); janus_recorder_destroy(ps->rc); + janus_bwe_stream_bitrate_destroy(ps->bitrates); g_hash_table_destroy(ps->rtp_forwarders); ps->rtp_forwarders = NULL; janus_mutex_destroy(&ps->rtp_forwarders_mutex); @@ -4090,6 +4095,21 @@ json_t *janus_videoroom_query_session(janus_plugin_session *handle) { json_object_set_new(m, "audio-level-dBov", json_integer(ps->audio_dBov_level)); json_object_set_new(m, "talking", ps->talking ? json_true() : json_false()); } + if(ps->bitrates) { + int i=0; + json_t *bitrates = NULL; + for(i=0; i<9; i++) { + if(ps->bitrates->packets[i]) { + if(bitrates == NULL) + bitrates = json_object(); + char name[2]; + g_snprintf(name, sizeof(name), "%d", i); + json_object_set_new(bitrates, name, json_integer(ps->bitrates->bitrate[i])); + } + } + if(bitrates != NULL) + json_object_set_new(m, "bitrates", bitrates); + } janus_mutex_lock(&ps->subscribers_mutex); json_object_set_new(m, "subscribers", json_integer(g_slist_length(ps->subscribers))); janus_mutex_unlock(&ps->subscribers_mutex); @@ -7192,6 +7212,7 @@ static json_t *janus_videoroom_process_synchronous_request(janus_videoroom_sessi ps->vcodec = JANUS_VIDEOCODEC_NONE; ps->min_delay = -1; ps->max_delay = -1; + ps->bitrates = janus_bwe_stream_bitrate_create(); if(mtype == JANUS_VIDEOROOM_MEDIA_AUDIO) { ps->acodec = janus_audiocodec_from_name(codec); ps->pt = janus_audiocodec_pt(ps->acodec); @@ -7468,6 +7489,7 @@ static json_t *janus_videoroom_process_synchronous_request(janus_videoroom_sessi ps->vcodec = JANUS_VIDEOCODEC_NONE; ps->min_delay = -1; ps->max_delay = -1; + ps->bitrates = janus_bwe_stream_bitrate_create(); if(mtype == JANUS_VIDEOROOM_MEDIA_AUDIO) { ps->acodec = janus_audiocodec_from_name(codec); ps->pt = janus_audiocodec_pt(ps->acodec); @@ -7838,6 +7860,8 @@ void janus_videoroom_setup_media(janus_plugin_session *handle) { } else if(session->participant_type == janus_videoroom_p_type_subscriber) { janus_videoroom_subscriber *s = janus_videoroom_session_get_subscriber(session); if(s && s->streams) { + /* FIXME Enable bandwidth estimation for this session */ + gateway->enable_bwe(session->handle); /* Send a PLI for all the video streams we subscribed to */ GList *temp = s->streams; while(temp) { @@ -8001,11 +8025,14 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi /* Save the frame if we're recording */ if(!video || !ps->simulcast) { janus_recorder_save_frame(ps->rc, buf, len); + janus_bwe_stream_bitrate_update(ps->bitrates, janus_get_monotonic_time(), 0, 0, len); } else { /* We're simulcasting, save the best video quality */ gboolean save = janus_rtp_simulcasting_context_process_rtp(&ps->rec_simctx, buf, len, pkt->extensions.dd_content, pkt->extensions.dd_len, ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex); + janus_bwe_stream_bitrate_update(ps->bitrates, janus_get_monotonic_time(), + ps->rec_simctx.substream_last, ps->rec_simctx.temporal_last, len); if(save) { uint32_t seq_number = ntohs(rtp->seq_number); uint32_t timestamp = ntohl(rtp->timestamp); @@ -8351,6 +8378,122 @@ void janus_videoroom_slow_link(janus_plugin_session *handle, int mindex, gboolea janus_refcount_decrease(&session->ref); } +void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t estimate) { + /* The core is informing us that our peer got or sent too many NACKs, are we pushing media too hard? */ + if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) + return; + janus_mutex_lock(&sessions_mutex); + janus_videoroom_session *session = janus_videoroom_lookup_session(handle); + if(!session || g_atomic_int_get(&session->destroyed) || !session->participant) { + janus_mutex_unlock(&sessions_mutex); + return; + } + janus_refcount_increase(&session->ref); + janus_mutex_unlock(&sessions_mutex); + /* This will only make sense for subscribers, publishers have their own BWE */ + if(session->participant_type == janus_videoroom_p_type_subscriber) { + janus_videoroom_subscriber *subscriber = janus_videoroom_session_get_subscriber(session); + if(subscriber == NULL) { + janus_refcount_decrease(&session->ref); + return; + } + if(g_atomic_int_get(&subscriber->destroyed)) { + janus_refcount_decrease(&subscriber->ref); + janus_refcount_decrease(&session->ref); + return; + } + JANUS_LOG(LOG_WARN, "[videoroom] BWE=%"SCNu32"\n", estimate); + janus_mutex_lock(&subscriber->streams_mutex); + GList *temp = subscriber->streams; + while(temp) { + janus_videoroom_subscriber_stream *s = (janus_videoroom_subscriber_stream *)temp->data; + if(s->type == JANUS_VIDEOROOM_MEDIA_DATA) { + temp = temp->next; + continue; + } + GSList *list = s->publisher_streams; + if(list) { + janus_videoroom_publisher_stream *ps = list->data; + if(ps && ps->publisher != NULL && ps->bitrates) { + /* FIXME Check the bandwidth usage for this stream */ + if(s->type == JANUS_VIDEOROOM_MEDIA_AUDIO && ps->bitrates->packets[0]) { + if(estimate < ps->bitrates->bitrate[0]) { + estimate = 0; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for stream %d (audio): %"SCNu32" < %"SCNu32"\n", + s->mindex, estimate, ps->bitrates->bitrate[0]); + } else { + //~ estimate -= ps->bitrates->bitrate[0]; + } + } else if(s->type == JANUS_VIDEOROOM_MEDIA_VIDEO) { + if(ps->simulcast) { + /* FIXME Simulcast, check the current targets, and retarget if needed */ + int substream = s->sim_context.substream; + if(substream < 0) + substream = 0; + int templayer = s->sim_context.templayer; + if(templayer < 0) + templayer = 0; + int target = 3*substream + templayer; + if(target > 8) + target = 8; + if(ps->bitrates->packets[target]) { + if(estimate < ps->bitrates->bitrate[target]) { + /* We don't have room for these layers, find one below that fits */ + if(target == 0) { + estimate = 0; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); + } else { + gboolean found = FALSE; + int new_substream = 0, new_templayer = 0; + while(target > 0) { + target--; + new_substream = target/3; + new_templayer = target%3; + if(ps->bitrates->packets[target] && estimate >= ps->bitrates->bitrate[target]) { + found = TRUE; + break; + } + } + if(!found) { + estimate = 0; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); + } else { + estimate -= ps->bitrates->bitrate[target]; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (video), switching to %d/%d\n", + substream, templayer, s->mindex); + /* FIXME */ + s->sim_context.substream_target = new_substream; + s->sim_context.templayer_target = new_templayer; + } + } + } else { + estimate -= ps->bitrates->bitrate[target]; + } + } + } else if(ps->svc) { + /* TODO SVC */ + } else { + /* No simulcast, no SVC */ + if(ps->bitrates->packets[0]) { + if(estimate < ps->bitrates->bitrate[0]) { + estimate = 0; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for stream %d (video)\n", s->mindex); + } else { + estimate -= ps->bitrates->bitrate[0]; + } + } + } + } + } + } + temp = temp->next; + } + janus_mutex_unlock(&subscriber->streams_mutex); + janus_refcount_decrease(&subscriber->ref); + } + janus_refcount_decrease(&session->ref); +} + static void janus_videoroom_recorder_create(janus_videoroom_publisher_stream *ps) { char filename[255]; janus_recorder *rc = NULL; @@ -11658,6 +11801,7 @@ static void *janus_videoroom_handler(void *data) { ps->pt = -1; ps->min_delay = -1; ps->max_delay = -1; + ps->bitrates = janus_bwe_stream_bitrate_create(); g_atomic_int_set(&ps->destroyed, 0); janus_refcount_init(&ps->ref, janus_videoroom_publisher_stream_free); janus_refcount_increase(&ps->ref); /* This is for the mid-indexed hashtable */ diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 9e592c099b..9bc473e7c6 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -405,6 +405,15 @@ struct janus_callbacks { * @param[in] bitrate The bitrate value to send in the REMB message */ void (* const send_remb)(janus_plugin_session *handle, guint32 bitrate); + /*! \brief Create a new bandwidth estimation context for this session + * \note A call to this method will result in the core invoking the + * estimated_bandwidth callback on a regular basis for this session + * @param[in] handle The plugin/gateway session to enable BWE for */ + void (* const enable_bwe)(janus_plugin_session *handle); + /*! \brief Get rid of the bandwidth estimation context for this session + * @param[in] handle The plugin/gateway session to disnable BWE for */ + void (* const disable_bwe)(janus_plugin_session *handle); + /*! \brief Callback to ask the core to close a WebRTC PeerConnection * \note A call to this method will result in the core invoking the hangup_media * callback on this plugin when done From f505e3e0db9d1d915fb11852c5d40e45a724f7fd Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 11 Sep 2023 15:50:18 +0200 Subject: [PATCH 10/22] First draft of probing mechanism --- src/bwe.c | 61 +++++++++++++--- src/bwe.h | 9 ++- src/ice.c | 134 +++++++++++++++++++++++++++++++--- src/ice.h | 6 +- src/janus.c | 4 + src/plugins/janus_videoroom.c | 21 ++++-- src/rtp.c | 35 +++++---- src/rtp.h | 4 +- 8 files changed, 225 insertions(+), 49 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 1a7a083bb5..2b51445146 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -56,6 +56,7 @@ janus_bwe_context *janus_bwe_context_create(void) { janus_bwe_context *bwe = g_malloc0(sizeof(janus_bwe_context)); /* FIXME */ bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); + bwe->probing_mindex = -1; return bwe; } @@ -77,6 +78,7 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, if(bwe->status == janus_bwe_status_start && now - bwe->started >= G_USEC_PER_SEC) { /* Let's move from the starting phase to the regular stage */ bwe->status = janus_bwe_status_regular; + bwe->status_changed = now; } janus_bwe_twcc_inflight *stat = g_malloc(sizeof(janus_bwe_twcc_inflight)); stat->seq = seq; @@ -86,6 +88,8 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, stat->type = type; stat->size = size; bwe->sent_bytes += size; + if(type == janus_bwe_packet_type_probing) + bwe->sent_bytes_probing += size; g_hash_table_insert(bwe->packets, GUINT_TO_POINTER(seq), stat); return TRUE; } @@ -121,6 +125,8 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, } if(status != janus_bwe_twcc_status_notreceived) { bwe->received_bytes += p->size; + if(p->type == janus_bwe_packet_type_probing) + bwe->received_bytes_probing += p->size; bwe->received_pkts++; bwe->last_recv_seq = seq; } else { @@ -145,27 +151,60 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { uint16_t tot = bwe->received_pkts + bwe->lost_pkts; if(tot > 0) bwe->loss_ratio = (double)bwe->lost_pkts / (double)tot; - /* Depending on the severity of the loss, we change the estimate */ - if(bwe->loss_ratio > 0.05 || estimate > bwe->estimate) { - /* FIXME Lossy network? */ + double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; + /* Check if there's packet loss or congestion */ + if(bwe->loss_ratio > 0.05) { + /* FIXME Lossy network? Set the estimate to the acknowledged bitrate */ bwe->status = janus_bwe_status_lossy; + bwe->status_changed = now; bwe->estimate = estimate; - } - /* Let's also check of the average delay is increasing */ - double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; - if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.5) { - /* FIXME Delay is increasing */ + } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.5) { + /* FIXME Delay is increasing, converge to acknowledged bitrate */ bwe->status = janus_bwe_status_congested; - bwe->estimate = estimate; + bwe->status_changed = now; + //~ if(estimate > bwe->estimate) + bwe->estimate = estimate; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + } else { + /* FIXME All is fine? Check what state we're in */ + if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { + bwe->status = janus_bwe_status_recovering; + bwe->status_changed = now; + } + if(bwe->status == janus_bwe_status_recovering) { + /* FIXME Still recovering */ + if(now - bwe->status_changed >= 5*G_USEC_PER_SEC) { + bwe->status = janus_bwe_status_regular; + bwe->status_changed = now; + } else { + /* FIXME Keep converging to the estimate */ + if(estimate > bwe->estimate) + bwe->estimate = estimate; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + } + } + if(bwe->status == janus_bwe_status_regular) { + /* FIXME Slowly increase */ + if(estimate > bwe->estimate) + bwe->estimate = estimate; + else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) + bwe->estimate = ((double)bwe->estimate * 1.02); + } } bwe->avg_delay = avg_delay; bwe->bitrate_ts = now; } - JANUS_LOG(LOG_WARN, "[BWE] sent=%"SCNu32"kbps, received=%"SCNu32"kbps, loss=%.2f%%, avg_delay=%.2fms\n", - (bwe->sent_bytes / 1000) * 8, (bwe->received_bytes / 1000) * 8, + JANUS_LOG(LOG_WARN, "[BWE][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), received=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms\n", + janus_bwe_status_description(bwe->status), + (bwe->sent_bytes / 1000) * 8, (bwe->sent_bytes_probing / 1000) * 8, + (bwe->received_bytes / 1000) * 8, (bwe->received_bytes_probing / 1000) * 8, bwe->loss_ratio, bwe->avg_delay); bwe->sent_bytes = 0; bwe->received_bytes = 0; + bwe->sent_bytes_probing = 0; + bwe->received_bytes_probing = 0; bwe->delay = 0; bwe->received_pkts = 0; bwe->lost_pkts = 0; diff --git a/src/bwe.h b/src/bwe.h index 6d2174bab3..64e7664e31 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -34,7 +34,8 @@ typedef enum janus_bwe_packet_type { janus_bwe_packet_type_regular = 0, /*! \brief RTC packet */ janus_bwe_packet_type_rtx, - /* TODO other types? e.g., padding? */ + /*! \brief Probing */ + janus_bwe_packet_type_probing } janus_bwe_packet_type; /*! \brief Tracking info for in-flight packet we're waiting TWCC feedback for */ @@ -75,6 +76,10 @@ typedef struct janus_bwe_context { janus_bwe_status status; /*! \brief Monotonic timestamp of when the BWE work started */ int64_t started; + /*! \brief Monotonic timestamp of when the BWE status last changed */ + int64_t status_changed; + /*! \brief Index of the m-line we're using for probing */ + int probing_mindex; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ @@ -85,6 +90,8 @@ typedef struct janus_bwe_context { int64_t bitrate_ts; /*! \brief Amount of bytes we've sent and the ones we've had feedback were received */ uint32_t sent_bytes, received_bytes; + /*! \brief As above, but subset we've specifically done for probing */ + uint32_t sent_bytes_probing, received_bytes_probing; /*! \brief How much delay has been accumulated (may be negative) */ int64_t delay; /*! \brief Number of packets with a received status, and number of lost ones */ diff --git a/src/ice.c b/src/ice.c index 45cc80510a..2ce2e20a5f 100644 --- a/src/ice.c +++ b/src/ice.c @@ -461,6 +461,7 @@ typedef struct janus_ice_queued_packet { gint type; gboolean control; gboolean retransmission; + gboolean probing; gboolean encrypted; guint16 twcc_seq; gint64 added; @@ -530,6 +531,7 @@ typedef struct janus_ice_outgoing_traffic { static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data); static gboolean janus_ice_outgoing_stats_handle(gpointer user_data); static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data); +static gboolean janus_ice_outgoing_probing_handle(gpointer user_data); static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janus_ice_queued_packet *pkt); static gboolean janus_ice_outgoing_traffic_prepare(GSource *source, gint *timeout) { janus_ice_outgoing_traffic *t = (janus_ice_outgoing_traffic *)source; @@ -1882,6 +1884,7 @@ janus_ice_peerconnection_medium *janus_ice_peerconnection_medium_create(janus_ic medium->rtcp_ctx[0]->in_media_link_quality = 100; medium->rtcp_ctx[0]->out_link_quality = 100; medium->rtcp_ctx[0]->out_media_link_quality = 100; + medium->nack_queue_ms = min_nack_queue; /* We can address media by SSRC */ g_hash_table_insert(pc->media_byssrc, GINT_TO_POINTER(medium->ssrc), medium); janus_refcount_increase(&medium->ref); @@ -3181,6 +3184,7 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp pkt->extensions = p->extensions; pkt->control = FALSE; pkt->retransmission = TRUE; + pkt->probing = TRUE; pkt->label = NULL; pkt->protocol = NULL; pkt->twcc_seq = 0; @@ -4490,19 +4494,99 @@ static gboolean janus_ice_outgoing_stats_handle(gpointer user_data) { static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { janus_ice_handle *handle = (janus_ice_handle *)user_data; janus_ice_peerconnection *pc = handle->pc; - if(pc == NULL) + if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; /* This callback is for updating the state of the bandwidth estimator */ - if(pc->bwe != NULL) { - janus_bwe_context_update(pc->bwe); - if(pc->bwe->estimate > 0) { - /* Notify the plugin about the current value */ - janus_plugin *plugin = (janus_plugin *)handle->app; - if(plugin && plugin->estimated_bandwidth && janus_plugin_session_is_alive(handle->app_handle) && - !g_atomic_int_get(&handle->destroyed)) - plugin->estimated_bandwidth(handle->app_handle, pc->bwe->estimate); + janus_bwe_context_update(pc->bwe); + if(pc->bwe->estimate > 0) { + /* Notify the plugin about the current value */ + janus_plugin *plugin = (janus_plugin *)handle->app; + if(plugin && plugin->estimated_bandwidth && janus_plugin_session_is_alive(handle->app_handle) && + !g_atomic_int_get(&handle->destroyed)) + plugin->estimated_bandwidth(handle->app_handle, pc->bwe->estimate); + } + return G_SOURCE_CONTINUE; +} + +static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { + janus_ice_handle *handle = (janus_ice_handle *)user_data; + janus_ice_peerconnection *pc = handle->pc; + janus_ice_peerconnection_medium *medium = NULL; + if(pc == NULL || pc->bwe == NULL) + return G_SOURCE_CONTINUE; + /* This callback is for regularly sending probing for the bandwidth estimator */ + if(pc->bwe->status != janus_bwe_status_start && pc->bwe->status != janus_bwe_status_regular) { + /* The BWE status may be lossy, congested, or recovering: don't probe for now */ + return G_SOURCE_CONTINUE; + } + /* Get the medium instance we'll use for probing */ + if(pc->bwe->probing_mindex != -1) + medium = g_hash_table_lookup(pc->media, GINT_TO_POINTER(pc->bwe->probing_mindex)); + if(medium == NULL || !medium->send || medium->rtx_payload_type == -1 || g_queue_is_empty(medium->retransmit_buffer)) { + /* We don't have a video medium we can use for probing (anymore?) */ + pc->bwe->probing_mindex = -1; + return G_SOURCE_CONTINUE; + } + int attempts = 10; + uint16_t seqnr = medium->last_rtp_seqnum; + janus_rtp_packet *p = NULL; + while(attempts > 0) { + p = g_hash_table_lookup(medium->retransmit_seqs, GUINT_TO_POINTER(seqnr)); + if(p == NULL) { + attempts--; + seqnr--; + continue; + } + /* FIXME Should we check if the packet is large enough? */ + break; + } + if(p == NULL) { + /* We don't have a probing packet to send */ + return G_SOURCE_CONTINUE; + } + /* FIXME We have a packet we can retransmit for probing purposes, enqueue it N times */ + JANUS_LOG(LOG_WARN, "[%"SCNu64"] Scheduling %u for retransmission (probing)\n", handle->handle_id, seqnr); + int i = 0, n = 20; + uint32_t enqueued = 0; + gint64 now = janus_get_monotonic_time(); + for(i=0; ilast_retransmit = now; + /* Enqueue it */ + janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); + pkt->mindex = medium->mindex; + pkt->data = g_malloc(p->length+SRTP_MAX_TAG_LEN); + memcpy(pkt->data, p->data, p->length); + pkt->length = p->length; + pkt->type = JANUS_ICE_PACKET_VIDEO; + pkt->extensions = p->extensions; + pkt->control = FALSE; + pkt->retransmission = TRUE; + pkt->probing = TRUE; + pkt->label = NULL; + pkt->protocol = NULL; + pkt->twcc_seq = 0; + pkt->added = janus_get_monotonic_time(); + pkt->encrypted = FALSE; + janus_rtp_header *header = (janus_rtp_header *)pkt->data; + header->type = medium->rtx_payload_type; + header->ssrc = htonl(medium->ssrc_rtx); + medium->rtx_seq_number++; + header->seq_number = htons(medium->rtx_seq_number); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, pkt); +#else + g_async_queue_push(handle->queued_packets, pkt); +#endif + g_main_context_wakeup(handle->mainctx); + } else { + janus_ice_free_queued_packet(pkt); } + enqueued += p->length+SRTP_MAX_TAG_LEN; } + JANUS_LOG(LOG_WARN, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps)\n", + handle->handle_id, n, enqueued, (enqueued/1000)*8); + /* TODO */ return G_SOURCE_CONTINUE; } @@ -4564,11 +4648,15 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu if(pc == NULL || pc->bwe != NULL) return G_SOURCE_CONTINUE; pc->bwe = janus_bwe_context_create(); - /* Let's create a source for BWE */ + /* Let's create a source for BWE and one for probing */ handle->bwe_source = g_timeout_source_new(1000); g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); g_source_attach(handle->bwe_source, handle->mainctx); + handle->probing_source = g_timeout_source_new(500); + g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); + g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); + g_source_attach(handle->probing_source, handle->mainctx); return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_bwe) { /* We need to get rif of the bandwidth estimator */ @@ -4579,6 +4667,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu g_source_unref(handle->bwe_source); handle->bwe_source = NULL; } + if(handle->probing_source) { + g_source_destroy(handle->probing_source); + g_source_unref(handle->probing_source); + handle->probing_source = NULL; + } janus_bwe_context_destroy(pc->bwe); pc->bwe = NULL; return G_SOURCE_CONTINUE; @@ -4631,6 +4724,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu g_source_unref(handle->bwe_source); handle->bwe_source = NULL; } + if(handle->probing_source) { + g_source_destroy(handle->probing_source); + g_source_unref(handle->probing_source); + handle->probing_source = NULL; + } if(handle->stats_source) { g_source_destroy(handle->stats_source); g_source_unref(handle->stats_source); @@ -4901,6 +4999,9 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu p->extensions = pkt->extensions; /* Copy the payload */ memcpy(p->data+hsize+2, payload, pkt->length - hsize); + /* Check if there's a BWE context and we need a medium for probing */ + if(pc->bwe && pc->bwe->probing_mindex == -1) + pc->bwe->probing_mindex = pkt->mindex; } /* Encrypt SRTP */ int protected = pkt->length; @@ -4925,8 +5026,12 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu } /* Keep track of the packet if we're using TWCC */ if(medium->do_twcc && handle->pc->transport_wide_cc_ext_id > 0) { - janus_bwe_context_add_inflight(pc->bwe, pkt->twcc_seq, janus_get_monotonic_time(), - (pkt->retransmission ? janus_bwe_packet_type_rtx : janus_bwe_packet_type_regular), protected); + janus_bwe_packet_type type = janus_bwe_packet_type_regular; + if(pkt->probing) + type = janus_bwe_packet_type_probing; + else if(pkt->retransmission) + type = janus_bwe_packet_type_rtx; + janus_bwe_context_add_inflight(pc->bwe, pkt->twcc_seq, janus_get_monotonic_time(), type, protected); } /* Update stats */ if(sent > 0) { @@ -4951,6 +5056,7 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu if(medium->last_ntp_ts == 0 || (gint32)(timestamp - medium->last_rtp_ts) > 0) { medium->last_ntp_ts = (gint64)tv.tv_sec*G_USEC_PER_SEC + tv.tv_usec; medium->last_rtp_ts = timestamp; + medium->last_rtp_seqnum = ntohs(header->seq_number); } if(medium->first_ntp_ts[0] == 0) { medium->first_ntp_ts[0] = (gint64)tv.tv_sec*G_USEC_PER_SEC + tv.tv_usec; @@ -5074,6 +5180,7 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { pkt->control = FALSE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; + pkt->probing = FALSE; pkt->label = NULL; pkt->protocol = NULL; pkt->twcc_seq = 0; @@ -5117,6 +5224,7 @@ void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_ice_peerconne pkt->control = TRUE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; + pkt->probing = FALSE; pkt->label = NULL; pkt->protocol = NULL; pkt->twcc_seq = 0; @@ -5217,6 +5325,7 @@ void janus_ice_relay_data(janus_ice_handle *handle, janus_plugin_data *packet) { pkt->control = FALSE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; + pkt->probing = FALSE; pkt->label = packet->label ? g_strdup(packet->label) : NULL; pkt->protocol = packet->protocol ? g_strdup(packet->protocol) : NULL; pkt->twcc_seq = 0; @@ -5245,6 +5354,7 @@ void janus_ice_relay_sctp(janus_ice_handle *handle, char *buffer, int length) { pkt->control = FALSE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; + pkt->probing = FALSE; pkt->label = NULL; pkt->protocol = NULL; pkt->twcc_seq = 0; diff --git a/src/ice.h b/src/ice.h index 6cbc309ad6..bbdd63e14d 100644 --- a/src/ice.h +++ b/src/ice.h @@ -373,8 +373,8 @@ struct janus_ice_handle { void *static_event_loop; /*! \brief GLib thread for the handle and libnice */ GThread *thread; - /*! \brief GLib sources for outgoing traffic, recurring RTCP, and stats (and optionally TWCC) */ - GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source, *bwe_source; + /*! \brief GLib sources for outgoing traffic, recurring RTCP, and stats (and optionally TWCC/BWE/probing) */ + GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source, *bwe_source, *probing_source; /*! \brief libnice ICE agent */ NiceAgent *agent; /*! \brief Monotonic time of when the ICE agent has been created */ @@ -597,6 +597,8 @@ struct janus_ice_peerconnection_medium { gint64 last_ntp_ts; /*! \brief Last sent RTP timestamp */ guint32 last_rtp_ts; + /*! \brief Last sent RTP sequence number */ + guint16 last_rtp_seqnum; /*! \brief Whether we should do NACKs (in or out) for this medium */ gboolean do_nacks; /*! \brief Whether we should do Transport Wide CC for this medium */ diff --git a/src/janus.c b/src/janus.c index a41ab0ccc3..174b0d01da 100644 --- a/src/janus.c +++ b/src/janus.c @@ -3275,6 +3275,10 @@ json_t *janus_admin_peerconnection_summary(janus_ice_peerconnection *pc) { json_object_set_new(bwe, "twcc", pc->do_transport_wide_cc ? json_true() : json_false()); if(pc->transport_wide_cc_ext_id >= 0) json_object_set_new(bwe, "twcc-ext-id", json_integer(pc->transport_wide_cc_ext_id)); + if(pc->bwe) { + json_object_set_new(bwe, "estimate", json_integer(pc->bwe->estimate)); + json_object_set_new(bwe, "status", json_string(janus_bwe_status_description(pc->bwe->status))); + } json_object_set_new(w, "bwe", bwe); json_t *media = json_object(); /* Iterate on all media */ diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index ee0d177298..6f9822cce5 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8436,11 +8436,15 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t } } else if(s->type == JANUS_VIDEOROOM_MEDIA_VIDEO) { if(ps->simulcast) { + JANUS_LOG(LOG_WARN, "current=%d/%d, target=%d/%d, bwe=%d/%d\n", + s->sim_context.substream, s->sim_context.templayer, + s->sim_context.substream_target, s->sim_context.templayer_target, + s->sim_context.substream_target_bwe, s->sim_context.templayer_target_bwe); /* FIXME Simulcast, check the current targets, and retarget if needed */ - int substream = s->sim_context.substream; + int substream = s->sim_context.substream_target; if(substream < 0) substream = 0; - int templayer = s->sim_context.templayer; + int templayer = s->sim_context.templayer_target; if(templayer < 0) templayer = 0; int target = 3*substream + templayer; @@ -8469,11 +8473,14 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); } else { estimate -= ps->bitrates->bitrate[target]; - JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (video), switching to %d/%d\n", - substream, templayer, s->mindex); - /* FIXME */ - s->sim_context.substream_target = new_substream; - s->sim_context.templayer_target = new_templayer; + if(s->sim_context.substream_target_bwe == -1 || s->sim_context.substream_target_bwe > new_substream || + s->sim_context.templayer_target_bwe == -1 || s->sim_context.templayer_target_bwe > new_templayer) { + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (video), switching to %d/%d\n", + substream, templayer, s->mindex, new_substream, new_templayer); + /* FIXME */ + } + s->sim_context.substream_target_bwe = new_substream; + s->sim_context.templayer_target_bwe = new_templayer; } } } else { diff --git a/src/rtp.c b/src/rtp.c index 81d84ed323..f2c53d96fc 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1030,7 +1030,9 @@ void janus_rtp_simulcasting_context_reset(janus_rtp_simulcasting_context *contex context->rid_ext_id = -1; context->substream = -1; context->substream_target_temp = -1; + context->substream_target_bwe = -1; context->templayer = -1; + context->templayer_target_bwe = -1; } void janus_rtp_simulcasting_prepare(json_t *simulcast, int *rid_ext_id, uint32_t *ssrcs, char **rids) { @@ -1148,6 +1150,8 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte context->substream_target_temp = -1; } int target = (context->substream_target_temp == -1) ? context->substream_target : context->substream_target_temp; + if(context->substream_target_bwe != -1 && context->substream_target_bwe < target) + target = context->substream_target_bwe; /* Check what we need to do with the packet */ gboolean relay = TRUE; if(context->substream == -1) { @@ -1190,10 +1194,10 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if((now - context->last_relayed) > (context->drop_trigger ? context->drop_trigger : 250000)) { context->last_relayed = now; if(context->substream != substream && context->substream_target_temp != 0) { - if(context->substream_target > substream) { + if(target > substream) { int prev_target = context->substream_target_temp; if(context->substream_target_temp == -1) - context->substream_target_temp = context->substream_target - 1; + context->substream_target_temp = target - 1; else context->substream_target_temp--; if(context->substream_target_temp < 0) @@ -1219,6 +1223,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(relay) context->last_relayed = janus_get_monotonic_time(); /* Temporal layers are only easily available for some codecs */ + target = context->templayer_target; + if(context->templayer_target_bwe != -1 && context->templayer_target_bwe < target) + target = context->templayer_target_bwe; if(vcodec == JANUS_VIDEOCODEC_VP8) { /* Check if there's any temporal scalability to take into account */ gboolean m = FALSE; @@ -1232,9 +1239,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte context->temporal_last = tid; if(!relay) return FALSE; - if(context->templayer != context->templayer_target && tid == context->templayer_target) { + if(context->templayer != target && tid == target) { /* FIXME We should be smarter in deciding when to switch */ - context->templayer = context->templayer_target; + context->templayer = target; /* Notify the caller that the temporal layer changed */ context->changed_temporal = TRUE; } @@ -1256,19 +1263,19 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(!relay) return FALSE; int temporal_layer = context->templayer; - if(context->templayer_target > context->templayer) { + if(target > context->templayer) { /* We need to upscale */ if(svc_info.ubit && svc_info.bbit && svc_info.temporal_layer > context->templayer && - svc_info.temporal_layer <= context->templayer_target) { + svc_info.temporal_layer <= target) { context->templayer = svc_info.temporal_layer; temporal_layer = context->templayer; context->changed_temporal = TRUE; } - } else if(context->templayer_target < context->templayer) { + } else if(target < context->templayer) { /* We need to downscale */ - if(svc_info.ebit && svc_info.temporal_layer == context->templayer_target) { - context->templayer = context->templayer_target; + if(svc_info.ebit && svc_info.temporal_layer == target) { + context->templayer = target; context->changed_temporal = TRUE; } } @@ -1295,17 +1302,17 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(!relay) return FALSE; int temporal_layer = context->templayer; - if(context->templayer_target > context->templayer) { + if(target > context->templayer) { /* We need to upscale */ - if(t->temporal > context->templayer && t->temporal <= context->templayer_target) { + if(t->temporal > context->templayer && t->temporal <= target) { context->templayer = t->temporal; temporal_layer = context->templayer; context->changed_temporal = TRUE; } - } else if(context->templayer_target < context->templayer) { + } else if(target < context->templayer) { /* We need to downscale */ - if(t->temporal == context->templayer_target) { - context->templayer = context->templayer_target; + if(t->temporal == target) { + context->templayer = target; context->changed_temporal = TRUE; } } diff --git a/src/rtp.h b/src/rtp.h index 43c76fe9d3..dfd18c6cf2 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -355,13 +355,13 @@ typedef struct janus_rtp_simulcasting_context { /*! \brief Which simulcast substream we should forward back */ int substream; /*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */ - int substream_target, substream_target_temp; + int substream_target, substream_target_temp, substream_target_bwe; /*! \brief Last substream that was identified by janus_rtp_simulcasting_context_process_rtp */ int substream_last; /*! \brief Which simulcast temporal layer we should forward back */ int templayer; /*! \brief As above, but to handle transitions (e.g., wait for keyframe) */ - int templayer_target; + int templayer_target, templayer_target_bwe; /*! \brief Last temporal layer that was identified by janus_rtp_simulcasting_context_process_rtp */ int temporal_last; /*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */ From b5d4edc62d7cda5c3bcae5430d490224de624596 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 12 Sep 2023 17:57:41 +0200 Subject: [PATCH 11/22] Added APIs to dynamically enable/disable probing from plugins --- src/bwe.c | 6 +++ src/bwe.h | 2 + src/ice.c | 69 +++++++++++++++++++++++++++++------ src/ice.h | 6 +++ src/janus.c | 20 ++++++++++ src/plugins/janus_videoroom.c | 16 +++++++- src/plugins/plugin.h | 9 +++++ src/rtp.c | 6 +++ 8 files changed, 122 insertions(+), 12 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 2b51445146..9d9d21a3f8 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -57,6 +57,7 @@ janus_bwe_context *janus_bwe_context_create(void) { /* FIXME */ bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); bwe->probing_mindex = -1; + bwe->probing_duplicates = 20; /* FIXME */ return bwe; } @@ -175,8 +176,11 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { if(bwe->status == janus_bwe_status_recovering) { /* FIXME Still recovering */ if(now - bwe->status_changed >= 5*G_USEC_PER_SEC) { + /* FIXME Recovery ended, let's assume everything is fine now */ bwe->status = janus_bwe_status_regular; bwe->status_changed = now; + /* Restore probing but incrementally */ + bwe->probing_duplicates = 1; } else { /* FIXME Keep converging to the estimate */ if(estimate > bwe->estimate) @@ -191,6 +195,8 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->estimate = estimate; else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) bwe->estimate = ((double)bwe->estimate * 1.02); + if(bwe->probing_duplicates < 20) /* FIXME */ + bwe->probing_duplicates++; } } bwe->avg_delay = avg_delay; diff --git a/src/bwe.h b/src/bwe.h index 64e7664e31..68b8e413d2 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -80,6 +80,8 @@ typedef struct janus_bwe_context { int64_t status_changed; /*! \brief Index of the m-line we're using for probing */ int probing_mindex; + /*! \brief Number of packets to send when probing */ + uint8_t probing_duplicates; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ diff --git a/src/ice.c b/src/ice.c index 2ce2e20a5f..5897c211b3 100644 --- a/src/ice.c +++ b/src/ice.c @@ -475,6 +475,8 @@ static janus_ice_queued_packet janus_ice_media_stopped, janus_ice_enable_bwe, janus_ice_disable_bwe, + janus_ice_enable_probing, + janus_ice_disable_probing, janus_ice_hangup_peerconnection, janus_ice_detach_handle, janus_ice_data_ready; @@ -654,8 +656,8 @@ static void janus_ice_free_queued_packet(janus_ice_queued_packet *pkt) { if(pkt == NULL || pkt == &janus_ice_start_gathering || pkt == &janus_ice_add_candidates || pkt == &janus_ice_dtls_handshake || - pkt == &janus_ice_enable_bwe || - pkt == &janus_ice_disable_bwe || + pkt == &janus_ice_enable_bwe || pkt == &janus_ice_disable_bwe || + pkt == &janus_ice_enable_probing || pkt == &janus_ice_disable_probing || pkt == &janus_ice_media_stopped || pkt == &janus_ice_hangup_peerconnection || pkt == &janus_ice_detach_handle || @@ -3901,6 +3903,30 @@ void janus_ice_handle_disable_bwe(janus_ice_handle *handle) { } } +void janus_ice_handle_enable_probing(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_enable_probing); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_enable_probing); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + +void janus_ice_handle_disable_probing(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_disable_probing); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_disable_probing); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_peerconnection_medium *medium, janus_ice_queued_packet *packet) { if(handle == NULL || handle->pc == NULL || medium == NULL || packet == NULL || packet->data == NULL) return; @@ -4512,7 +4538,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { janus_ice_handle *handle = (janus_ice_handle *)user_data; janus_ice_peerconnection *pc = handle->pc; janus_ice_peerconnection_medium *medium = NULL; - if(pc == NULL || pc->bwe == NULL) + if(pc == NULL || pc->bwe == NULL || pc->bwe->probing_duplicates == 0) return G_SOURCE_CONTINUE; /* This callback is for regularly sending probing for the bandwidth estimator */ if(pc->bwe->status != janus_bwe_status_start && pc->bwe->status != janus_bwe_status_regular) { @@ -4546,10 +4572,10 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { } /* FIXME We have a packet we can retransmit for probing purposes, enqueue it N times */ JANUS_LOG(LOG_WARN, "[%"SCNu64"] Scheduling %u for retransmission (probing)\n", handle->handle_id, seqnr); - int i = 0, n = 20; + int i = 0; uint32_t enqueued = 0; gint64 now = janus_get_monotonic_time(); - for(i=0; ibwe->probing_duplicates; i++) { p->last_retransmit = now; /* Enqueue it */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); @@ -4585,7 +4611,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { enqueued += p->length+SRTP_MAX_TAG_LEN; } JANUS_LOG(LOG_WARN, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps)\n", - handle->handle_id, n, enqueued, (enqueued/1000)*8); + handle->handle_id, pc->bwe->probing_duplicates, enqueued, (enqueued/1000)*8); /* TODO */ return G_SOURCE_CONTINUE; } @@ -4647,21 +4673,19 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu /* Create a new bandwidth estimation context for this handle */ if(pc == NULL || pc->bwe != NULL) return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth estimation\n", handle->handle_id); pc->bwe = janus_bwe_context_create(); - /* Let's create a source for BWE and one for probing */ + /* Let's create a source for BWE */ handle->bwe_source = g_timeout_source_new(1000); g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); g_source_attach(handle->bwe_source, handle->mainctx); - handle->probing_source = g_timeout_source_new(500); - g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); - g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); - g_source_attach(handle->probing_source, handle->mainctx); return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_bwe) { /* We need to get rif of the bandwidth estimator */ if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); if(handle->bwe_source) { g_source_destroy(handle->bwe_source); g_source_unref(handle->bwe_source); @@ -4675,6 +4699,29 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu janus_bwe_context_destroy(pc->bwe); pc->bwe = NULL; return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_enable_probing) { + /* Enable probing for the BWE context (assuming BWE is active) */ + if(pc == NULL || pc->bwe == NULL || handle->probing_source != NULL) + return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); + /* Let's create a source for probing */ + pc->bwe->probing_duplicates = 20; /* FIXME */ + handle->probing_source = g_timeout_source_new(500); + g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); + g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); + g_source_attach(handle->probing_source, handle->mainctx); + return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_disable_probing) { + /* We need to get rif of the bandwidth probing */ + if(pc == NULL || pc->bwe == NULL) + return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); + if(handle->probing_source) { + g_source_destroy(handle->probing_source); + g_source_unref(handle->probing_source); + handle->probing_source = NULL; + } + return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_media_stopped) { /* Some media has been disabled on the way in, so use the callback to notify the peer */ if(pc == NULL) diff --git a/src/ice.h b/src/ice.h index bbdd63e14d..087b81d8fe 100644 --- a/src/ice.h +++ b/src/ice.h @@ -793,6 +793,12 @@ void janus_ice_handle_enable_bwe(janus_ice_handle *handle); /*! \brief Method to dynamically disable bandwidth estimation for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_disable_bwe(janus_ice_handle *handle); +/*! \brief Method to dynamically enable bandwidth probing for a handle + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_enable_probing(janus_ice_handle *handle); +/*! \brief Method to dynamically disable bandwidth probing for a handle + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_disable_probing(janus_ice_handle *handle); ///@} diff --git a/src/janus.c b/src/janus.c index 174b0d01da..5690c438e7 100644 --- a/src/janus.c +++ b/src/janus.c @@ -613,6 +613,8 @@ void janus_plugin_send_pli_stream(janus_plugin_session *plugin_session, int mind void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitrate); void janus_plugin_enable_bwe(janus_plugin_session *plugin_session); void janus_plugin_disable_bwe(janus_plugin_session *plugin_session); +void janus_plugin_enable_probing(janus_plugin_session *plugin_session); +void janus_plugin_disable_probing(janus_plugin_session *plugin_session); void janus_plugin_close_pc(janus_plugin_session *plugin_session); void janus_plugin_end_session(janus_plugin_session *plugin_session); void janus_plugin_notify_event(janus_plugin *plugin, janus_plugin_session *plugin_session, json_t *event); @@ -630,6 +632,8 @@ static janus_callbacks janus_handler_plugin = .send_remb = janus_plugin_send_remb, .enable_bwe = janus_plugin_enable_bwe, .disable_bwe = janus_plugin_disable_bwe, + .enable_probing = janus_plugin_enable_probing, + .disable_probing = janus_plugin_disable_probing, .close_pc = janus_plugin_close_pc, .end_session = janus_plugin_end_session, .events_is_enabled = janus_events_is_enabled, @@ -4278,6 +4282,22 @@ void janus_plugin_disable_bwe(janus_plugin_session *plugin_session) { janus_ice_handle_disable_bwe(handle); } +void janus_plugin_enable_probing(janus_plugin_session *plugin_session) { + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_handle_enable_probing(handle); +} + +void janus_plugin_disable_probing(janus_plugin_session *plugin_session) { + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_handle_disable_probing(handle); +} + static gboolean janus_plugin_close_pc_internal(gpointer user_data) { /* We actually enforce the close_pc here */ janus_plugin_session *plugin_session = (janus_plugin_session *) user_data; diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 6f9822cce5..3b767e7426 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8441,7 +8441,8 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t s->sim_context.substream_target, s->sim_context.templayer_target, s->sim_context.substream_target_bwe, s->sim_context.templayer_target_bwe); /* FIXME Simulcast, check the current targets, and retarget if needed */ - int substream = s->sim_context.substream_target; + int substream = (s->sim_context.substream_target_temp == -1) ? + s->sim_context.substream_target : s->sim_context.substream_target_temp; if(substream < 0) substream = 0; int templayer = s->sim_context.templayer_target; @@ -8485,6 +8486,8 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t } } else { estimate -= ps->bitrates->bitrate[target]; + s->sim_context.substream_target_bwe = -1; + s->sim_context.templayer_target_bwe = -1; } } } else if(ps->svc) { @@ -12450,6 +12453,17 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) gateway->push_event(subscriber->session->handle, &janus_videoroom_plugin, NULL, event, NULL); json_decref(event); } + if(stream->sim_context.changed_substream || stream->sim_context.changed_temporal) { + if(stream->sim_context.substream_target_bwe == -1 && stream->sim_context.substream_target_temp == -1 && + stream->sim_context.substream == stream->sim_context.substream_target && + stream->sim_context.templayer == stream->sim_context.templayer_target) { + /* We're receiving what we wanted, stop probing (if it was active) */ + gateway->disable_probing(subscriber->session->handle); + } else { + /* We're not receiving what we need, start probing (if we weren't already) */ + gateway->enable_probing(subscriber->session->handle); + } + } /* If we got here, update the RTP header and send the packet */ janus_rtp_header_update(packet->data, &stream->context, TRUE, 0); char vp8pd[6]; diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 9bc473e7c6..cb853b0ae2 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -413,6 +413,15 @@ struct janus_callbacks { /*! \brief Get rid of the bandwidth estimation context for this session * @param[in] handle The plugin/gateway session to disnable BWE for */ void (* const disable_bwe)(janus_plugin_session *handle); + /*! \brief Enable bandwidth probing for this session, for bandwidth estimation purposes + * \note The request will be ignored if no BWE context is enabled for this session. + * Also notice that probing may be paused at any time by the core, whether it + * was enabled or not, e.g., in case congestion or losses are detected + * @param[in] handle The plugin/gateway session to enable BWE probing for */ + void (* const enable_probing)(janus_plugin_session *handle); + /*! \brief Disable bandwidth probing for this session + * @param[in] handle The plugin/gateway session to disnable BWE probing for */ + void (* const disable_probing)(janus_plugin_session *handle); /*! \brief Callback to ask the core to close a WebRTC PeerConnection * \note A call to this method will result in the core invoking the hangup_media diff --git a/src/rtp.c b/src/rtp.c index f2c53d96fc..c41eb7c982 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1183,6 +1183,12 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte /* Notify the caller that the substream changed */ context->changed_substream = TRUE; context->last_relayed = now; + } else if(context->substream_target_bwe != -1 && context->substream > target && substream < context->substream) { + /* We need to go down because of BWE, don't wait for a keyframe */ + context->substream = substream; + /* Notify the caller that the substream changed */ + context->changed_substream = TRUE; + context->last_relayed = now; } } /* If we haven't received our desired substream yet, let's drop temporarily */ From 1ee053a62f9b458c300a126eb0e7506bc0e39a90 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 19 Sep 2023 15:16:30 +0200 Subject: [PATCH 12/22] Fire probing related events more often --- src/bwe.c | 9 +++++---- src/bwe.h | 10 ++++++++-- src/ice.c | 51 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 9d9d21a3f8..0c33f936c0 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -57,7 +57,6 @@ janus_bwe_context *janus_bwe_context_create(void) { /* FIXME */ bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); bwe->probing_mindex = -1; - bwe->probing_duplicates = 20; /* FIXME */ return bwe; } @@ -180,7 +179,9 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->status = janus_bwe_status_regular; bwe->status_changed = now; /* Restore probing but incrementally */ - bwe->probing_duplicates = 1; + bwe->probing_sent = 0; + bwe->probing_portion = 0.0; + bwe->probing_part = 20; /* FIXME */ } else { /* FIXME Keep converging to the estimate */ if(estimate > bwe->estimate) @@ -195,8 +196,8 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->estimate = estimate; else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) bwe->estimate = ((double)bwe->estimate * 1.02); - if(bwe->probing_duplicates < 20) /* FIXME */ - bwe->probing_duplicates++; + if(bwe->probing_part > 0) /* FIXME */ + bwe->probing_part--; } } bwe->avg_delay = avg_delay; diff --git a/src/bwe.h b/src/bwe.h index 68b8e413d2..1b482713be 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -80,8 +80,14 @@ typedef struct janus_bwe_context { int64_t status_changed; /*! \brief Index of the m-line we're using for probing */ int probing_mindex; - /*! \brief Number of packets to send when probing */ - uint8_t probing_duplicates; + /*! \brief How much we should aim for with out probing (and how much we sent in a second) */ + uint32_t probing_target, probing_sent; + /*! \brief How many times we went through probing in a second */ + uint8_t probing_count; + /*! \brief Portion of probing we didn't manage to send the previous round */ + double probing_portion; + /*! \brief Whether probing should grow incrementally (e.g., after recovery) */ + uint8_t probing_part; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ diff --git a/src/ice.c b/src/ice.c index 5897c211b3..5983e910d6 100644 --- a/src/ice.c +++ b/src/ice.c @@ -4538,7 +4538,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { janus_ice_handle *handle = (janus_ice_handle *)user_data; janus_ice_peerconnection *pc = handle->pc; janus_ice_peerconnection_medium *medium = NULL; - if(pc == NULL || pc->bwe == NULL || pc->bwe->probing_duplicates == 0) + if(pc == NULL || pc->bwe == NULL || pc->bwe->probing_target == 0) return G_SOURCE_CONTINUE; /* This callback is for regularly sending probing for the bandwidth estimator */ if(pc->bwe->status != janus_bwe_status_start && pc->bwe->status != janus_bwe_status_regular) { @@ -4570,12 +4570,43 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { /* We don't have a probing packet to send */ return G_SOURCE_CONTINUE; } + /* Check how many duplicates we have to send of this packet */ + if(pc->bwe->probing_count == 20) { + pc->bwe->probing_count = 0; + pc->bwe->probing_sent = 0; + pc->bwe->probing_portion = 0.0; + } + pc->bwe->probing_count++; + uint32_t required = pc->bwe->probing_target; + if(pc->bwe->probing_part > 0) + required = required / pc->bwe->probing_part; + if(pc->bwe->probing_sent >= required) { + /* We sent enough for this round */ + return G_SOURCE_CONTINUE; + } + uint32_t required_now = required / (8 * 20); + //~ if(pc->bwe->probing_count == 20) + //~ required_now = (pc->bwe->probing_target - pc->bwe->probing_sent) / 8; + double prev_portion = pc->bwe->probing_portion; + double portion = (double)required_now / (double)(p->length+SRTP_MAX_TAG_LEN); + double new_portion = prev_portion + portion; + int duplicates = (int)(new_portion); + if(new_portion - (double)duplicates > 0.5) + duplicates++; + if(duplicates == 0) { + /* Skip this round, we'll send something later */ + pc->bwe->probing_portion = new_portion; + return G_SOURCE_CONTINUE; + } + pc->bwe->probing_portion = ((double)duplicates < new_portion) ? + (new_portion - (double)duplicates) : 0; /* FIXME We have a packet we can retransmit for probing purposes, enqueue it N times */ - JANUS_LOG(LOG_WARN, "[%"SCNu64"] Scheduling %u for retransmission (probing)\n", handle->handle_id, seqnr); + JANUS_LOG(LOG_WARN, "[%"SCNu64"] #%d: Scheduling %u for retransmission (probing, pkt_size=%d)\n", + handle->handle_id, (pc->bwe->probing_count - 1), seqnr, p->length+SRTP_MAX_TAG_LEN); int i = 0; uint32_t enqueued = 0; gint64 now = janus_get_monotonic_time(); - for(i=0; i < pc->bwe->probing_duplicates; i++) { + for(i=0; i < duplicates; i++) { p->last_retransmit = now; /* Enqueue it */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); @@ -4610,9 +4641,10 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { } enqueued += p->length+SRTP_MAX_TAG_LEN; } - JANUS_LOG(LOG_WARN, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps)\n", - handle->handle_id, pc->bwe->probing_duplicates, enqueued, (enqueued/1000)*8); - /* TODO */ + pc->bwe->probing_sent += (enqueued * 8); + JANUS_LOG(LOG_WARN, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps; sent=%"SCNu32"/%"SCNu32")\n", + handle->handle_id, duplicates, enqueued, (enqueued/1000)*8, pc->bwe->probing_sent, required); + /* Done */ return G_SOURCE_CONTINUE; } @@ -4705,8 +4737,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu return G_SOURCE_CONTINUE; JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); /* Let's create a source for probing */ - pc->bwe->probing_duplicates = 20; /* FIXME */ - handle->probing_source = g_timeout_source_new(500); + pc->bwe->probing_target = 200000; /* FIXME */ + pc->bwe->probing_count = 0; /* FIXME */ + pc->bwe->probing_sent = 0; /* FIXME */ + pc->bwe->probing_part = 0; /* FIXME */ + handle->probing_source = g_timeout_source_new(50); g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); g_source_attach(handle->probing_source, handle->mainctx); From 8f7cb4b4a1a55fe90ebfb80c999c543e60028544 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 21 Sep 2023 15:41:55 +0200 Subject: [PATCH 13/22] Refactored calculation of current bitrates, and changed estimate trigger --- src/bwe.c | 154 ++++++++++++++++++---------------- src/bwe.h | 74 ++++++++-------- src/ice.c | 13 ++- src/plugins/janus_videoroom.c | 6 +- src/rtcp.c | 2 + 5 files changed, 133 insertions(+), 116 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 0c33f936c0..499e36b052 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -56,6 +56,8 @@ janus_bwe_context *janus_bwe_context_create(void) { janus_bwe_context *bwe = g_malloc0(sizeof(janus_bwe_context)); /* FIXME */ bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); + bwe->sent = janus_bwe_stream_bitrate_create(); + bwe->acked = janus_bwe_stream_bitrate_create(); bwe->probing_mindex = -1; return bwe; } @@ -64,6 +66,8 @@ void janus_bwe_context_destroy(janus_bwe_context *bwe) { if(bwe) { /* FIXME clean everything up */ g_hash_table_destroy(bwe->packets); + janus_bwe_stream_bitrate_destroy(bwe->sent); + janus_bwe_stream_bitrate_destroy(bwe->acked); g_free(bwe); } } @@ -79,6 +83,7 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, /* Let's move from the starting phase to the regular stage */ bwe->status = janus_bwe_status_regular; bwe->status_changed = now; + bwe->last_notified = janus_get_monotonic_time(); } janus_bwe_twcc_inflight *stat = g_malloc(sizeof(janus_bwe_twcc_inflight)); stat->seq = seq; @@ -87,9 +92,7 @@ gboolean janus_bwe_context_add_inflight(janus_bwe_context *bwe, bwe->last_sent_ts = sent; stat->type = type; stat->size = size; - bwe->sent_bytes += size; - if(type == janus_bwe_packet_type_probing) - bwe->sent_bytes_probing += size; + janus_bwe_stream_bitrate_update(bwe->sent, now, type, 0, size); g_hash_table_insert(bwe->packets, GUINT_TO_POINTER(seq), stat); return TRUE; } @@ -124,9 +127,7 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, janus_bwe_twcc_status_description(status), delta_us, rounded_delta_us, diff_us); } if(status != janus_bwe_twcc_status_notreceived) { - bwe->received_bytes += p->size; - if(p->type == janus_bwe_packet_type_probing) - bwe->received_bytes_probing += p->size; + janus_bwe_stream_bitrate_update(bwe->acked, janus_get_monotonic_time(), p->type, 0, p->size); bwe->received_pkts++; bwe->last_recv_seq = seq; } else { @@ -137,84 +138,97 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, void janus_bwe_context_update(janus_bwe_context *bwe) { if(bwe == NULL) return; + /* Reset the outgoing and (acked) incoming bitrate, and estimate the bitrate */ int64_t now = janus_get_monotonic_time(); if(bwe->bitrate_ts == 0) bwe->bitrate_ts = now; - /* Reset the outgoing and (acked) incoming bitrate, and estimate the bitrate */ - if(now > bwe->bitrate_ts) { - /* TODO Actually estimate the bitrate: now we're just checking how - * much the peer received out of what we sent, which is not enough */ - int64_t diff = now - bwe->bitrate_ts; - double ratio = (double)G_USEC_PER_SEC / (double)diff; - double estimate_bytes = ratio * bwe->received_bytes; - uint32_t estimate = 8 * estimate_bytes; - uint16_t tot = bwe->received_pkts + bwe->lost_pkts; - if(tot > 0) - bwe->loss_ratio = (double)bwe->lost_pkts / (double)tot; - double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; - /* Check if there's packet loss or congestion */ - if(bwe->loss_ratio > 0.05) { - /* FIXME Lossy network? Set the estimate to the acknowledged bitrate */ - bwe->status = janus_bwe_status_lossy; - bwe->status_changed = now; + gboolean notify_plugin = FALSE; + /* Clean up old bitrate values, and get the current bitrates */ + janus_bwe_stream_bitrate_update(bwe->sent, now, janus_bwe_packet_type_regular, 0, 0); + janus_bwe_stream_bitrate_update(bwe->sent, now, janus_bwe_packet_type_rtx, 0, 0); + janus_bwe_stream_bitrate_update(bwe->sent, now, janus_bwe_packet_type_probing, 0, 0); + janus_bwe_stream_bitrate_update(bwe->acked, now, janus_bwe_packet_type_regular, 0, 0); + janus_bwe_stream_bitrate_update(bwe->acked, now, janus_bwe_packet_type_rtx, 0, 0); + janus_bwe_stream_bitrate_update(bwe->acked, now, janus_bwe_packet_type_probing, 0, 0); + uint32_t rtx_out = bwe->sent->packets[janus_bwe_packet_type_rtx*3] ? bwe->sent->bitrate[janus_bwe_packet_type_rtx*3] : 0; + uint32_t probing_out = bwe->sent->packets[janus_bwe_packet_type_probing*3] ? bwe->sent->bitrate[janus_bwe_packet_type_probing*3] : 0; + uint32_t bitrate_out = rtx_out + probing_out + (bwe->sent->packets[0] ? bwe->sent->bitrate[0] : 0); + uint32_t rtx_in = bwe->acked->packets[janus_bwe_packet_type_rtx*3] ? bwe->acked->bitrate[janus_bwe_packet_type_rtx*3] : 0; + uint32_t probing_in = bwe->acked->packets[janus_bwe_packet_type_probing*3] ? bwe->acked->bitrate[janus_bwe_packet_type_probing*3] : 0; + uint32_t bitrate_in = rtx_in + probing_in + (bwe->acked->packets[0] ? bwe->acked->bitrate[0] : 0); + /* TODO Actually estimate the bitrate: now we're just checking how + * much the peer received out of what we sent, which is not enough */ + uint32_t estimate = bitrate_in; + uint16_t tot = bwe->received_pkts + bwe->lost_pkts; + if(tot > 0) + bwe->loss_ratio = (double)bwe->lost_pkts / (double)tot; + double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; + /* Check if there's packet loss or congestion */ + if(bwe->loss_ratio > 0.05) { + /* FIXME Lossy network? Set the estimate to the acknowledged bitrate */ + if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) + notify_plugin = TRUE; + bwe->status = janus_bwe_status_lossy; + bwe->status_changed = now; + bwe->estimate = estimate; + } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 1.0) { + /* FIXME Delay is increasing, converge to acknowledged bitrate */ + if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) + notify_plugin = TRUE; + bwe->status = janus_bwe_status_congested; + bwe->status_changed = now; + //~ if(estimate > bwe->estimate) bwe->estimate = estimate; - } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.5) { - /* FIXME Delay is increasing, converge to acknowledged bitrate */ - bwe->status = janus_bwe_status_congested; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + } else { + /* FIXME All is fine? Check what state we're in */ + if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { + bwe->status = janus_bwe_status_recovering; bwe->status_changed = now; - //~ if(estimate > bwe->estimate) - bwe->estimate = estimate; - //~ else - //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); - } else { - /* FIXME All is fine? Check what state we're in */ - if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { - bwe->status = janus_bwe_status_recovering; + } + if(bwe->status == janus_bwe_status_recovering) { + /* FIXME Still recovering */ + if(now - bwe->status_changed >= 5*G_USEC_PER_SEC) { + /* FIXME Recovery ended, let's assume everything is fine now */ + bwe->status = janus_bwe_status_regular; bwe->status_changed = now; - } - if(bwe->status == janus_bwe_status_recovering) { - /* FIXME Still recovering */ - if(now - bwe->status_changed >= 5*G_USEC_PER_SEC) { - /* FIXME Recovery ended, let's assume everything is fine now */ - bwe->status = janus_bwe_status_regular; - bwe->status_changed = now; - /* Restore probing but incrementally */ - bwe->probing_sent = 0; - bwe->probing_portion = 0.0; - bwe->probing_part = 20; /* FIXME */ - } else { - /* FIXME Keep converging to the estimate */ - if(estimate > bwe->estimate) - bwe->estimate = estimate; - //~ else - //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); - } - } - if(bwe->status == janus_bwe_status_regular) { - /* FIXME Slowly increase */ + /* Restore probing but incrementally */ + bwe->probing_sent = 0; + bwe->probing_portion = 0.0; + bwe->probing_part = 20; /* FIXME */ + } else { + /* FIXME Keep converging to the estimate */ if(estimate > bwe->estimate) bwe->estimate = estimate; - else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) - bwe->estimate = ((double)bwe->estimate * 1.02); - if(bwe->probing_part > 0) /* FIXME */ - bwe->probing_part--; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); } } - bwe->avg_delay = avg_delay; - bwe->bitrate_ts = now; + if(bwe->status == janus_bwe_status_regular) { + /* FIXME Slowly increase */ + if(estimate > bwe->estimate) + bwe->estimate = estimate; + //~ else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) + //~ bwe->estimate = ((double)bwe->estimate * 1.02); + if(bwe->probing_part > 0) /* FIXME */ + bwe->probing_part--; + } } - JANUS_LOG(LOG_WARN, "[BWE][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), received=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms\n", - janus_bwe_status_description(bwe->status), - (bwe->sent_bytes / 1000) * 8, (bwe->sent_bytes_probing / 1000) * 8, - (bwe->received_bytes / 1000) * 8, (bwe->received_bytes_probing / 1000) * 8, - bwe->loss_ratio, bwe->avg_delay); - bwe->sent_bytes = 0; - bwe->received_bytes = 0; - bwe->sent_bytes_probing = 0; - bwe->received_bytes_probing = 0; + bwe->avg_delay = avg_delay; + bwe->bitrate_ts = now; + JANUS_LOG(LOG_WARN, "[BWE][%"SCNi64"][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), acked=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms, estimate=%"SCNu32"\n", + now, janus_bwe_status_description(bwe->status), + bitrate_out / 1024, probing_out / 1024, bitrate_in / 1024, probing_in / 1024, + bwe->loss_ratio, bwe->avg_delay, bwe->estimate); bwe->delay = 0; bwe->received_pkts = 0; bwe->lost_pkts = 0; + /* Check if we should notify the plugin about the estimate */ + if(notify_plugin || (now - bwe->last_notified) >= G_USEC_PER_SEC) { + bwe->notify_plugin = TRUE; + bwe->last_notified = now; + } } janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void) { diff --git a/src/bwe.h b/src/bwe.h index 1b482713be..cd59ad2f61 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -16,6 +16,38 @@ #include "mutex.h" +/*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ +typedef struct janus_bwe_stream_bitrate { + /*! \brief Time based queue of packet sizes */ + GQueue *packets[9]; + /*! \brief Current bitrate */ + uint32_t bitrate[9]; + /*! \brief Mutex to lock this instance */ + janus_mutex mutex; +} janus_bwe_stream_bitrate; +/*! \brief Helper method to create a new janus_bwe_stream_bitrate instance + * @returns A janus_bwe_stream_bitrate instance, if successful, or NULL otherwise */ +janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void); +/*! \brief Helper method to update an existing janus_bwe_stream_bitrate instance with new data + * \note Passing \c -1 or \c 0 as size just updates the queue to get rid of older values + * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to update + * @param[in] when Timestamp of the packet + * @param[in] sl Substream or spatial layer of the packet (can be 0 for audio) + * @param[in] sl Temporal layer of the packet (can be 0 for audio) + * @param[in] size Size of the packet */ +void janus_bwe_stream_bitrate_update(janus_bwe_stream_bitrate *bwe_sb, int64_t when, int sl, int tl, int size); +/*! \brief Helper method to destroy an existing janus_bwe_stream_bitrate instance + * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to destroy */ +void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb); + +/*! \brief Packet size and time */ +typedef struct janus_bwe_stream_packet { + /*! \brief Timestamp */ + int64_t sent_ts; + /*! \brief Size of packet */ + uint16_t size; +} janus_bwe_stream_packet; + /*! \brief Transport Wide CC statuses */ typedef enum janus_bwe_twcc_status { janus_bwe_twcc_status_notreceived = 0, @@ -96,10 +128,8 @@ typedef struct janus_bwe_context { GHashTable *packets; /*! \brief Monotonic timestamp of when we last computed the bitrates */ int64_t bitrate_ts; - /*! \brief Amount of bytes we've sent and the ones we've had feedback were received */ - uint32_t sent_bytes, received_bytes; - /*! \brief As above, but subset we've specifically done for probing */ - uint32_t sent_bytes_probing, received_bytes_probing; + /*! \brief Bitrate tracker for sent and acked packets */ + janus_bwe_stream_bitrate *sent, *acked; /*! \brief How much delay has been accumulated (may be negative) */ int64_t delay; /*! \brief Number of packets with a received status, and number of lost ones */ @@ -110,6 +140,10 @@ typedef struct janus_bwe_context { double loss_ratio; /*! \brief Latest estimated bitrate */ uint32_t estimate; + /*! \brief Whether we can notify the plugin about the estimate */ + gboolean notify_plugin; + /*! \brief When we last notified the plugin */ + int64_t last_notified; } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ @@ -139,36 +173,4 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, * @param[in] bwe The janus_bwe_context instance to update */ void janus_bwe_context_update(janus_bwe_context *bwe); -/*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ -typedef struct janus_bwe_stream_bitrate { - /*! \brief Time based queue of packet sizes */ - GQueue *packets[9]; - /*! \brief Current bitrate */ - uint32_t bitrate[9]; - /*! \brief Mutex to lock this instance */ - janus_mutex mutex; -} janus_bwe_stream_bitrate; -/*! \brief Helper method to create a new janus_bwe_stream_bitrate instance - * @returns A janus_bwe_stream_bitrate instance, if successful, or NULL otherwise */ -janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void); -/*! \brief Helper method to update an existing janus_bwe_stream_bitrate instance with new data - * \note Passing \c -1 or \c 0 as size just updates the queue to get rid of older values - * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to update - * @param[in] when Timestamp of the packet - * @param[in] sl Substream or spatial layer of the packet (can be 0 for audio) - * @param[in] sl Temporal layer of the packet (can be 0 for audio) - * @param[in] size Size of the packet */ -void janus_bwe_stream_bitrate_update(janus_bwe_stream_bitrate *bwe_sb, int64_t when, int sl, int tl, int size); -/*! \brief Helper method to destroy an existing janus_bwe_stream_bitrate instance - * @param[in] bwe_sb The janus_bwe_stream_bitrate instance to destroy */ -void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb); - -/*! \brief Packet size and time */ -typedef struct janus_bwe_stream_packet { - /*! \brief Timestamp */ - int64_t sent_ts; - /*! \brief Size of packet */ - uint16_t size; -} janus_bwe_stream_packet; - #endif diff --git a/src/ice.c b/src/ice.c index 5983e910d6..6511267fa7 100644 --- a/src/ice.c +++ b/src/ice.c @@ -4522,10 +4522,9 @@ static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { janus_ice_peerconnection *pc = handle->pc; if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; - /* This callback is for updating the state of the bandwidth estimator */ - janus_bwe_context_update(pc->bwe); - if(pc->bwe->estimate > 0) { + if(pc->bwe->notify_plugin) { /* Notify the plugin about the current value */ + pc->bwe->notify_plugin = FALSE; janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->estimated_bandwidth && janus_plugin_session_is_alive(handle->app_handle) && !g_atomic_int_get(&handle->destroyed)) @@ -4601,7 +4600,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { pc->bwe->probing_portion = ((double)duplicates < new_portion) ? (new_portion - (double)duplicates) : 0; /* FIXME We have a packet we can retransmit for probing purposes, enqueue it N times */ - JANUS_LOG(LOG_WARN, "[%"SCNu64"] #%d: Scheduling %u for retransmission (probing, pkt_size=%d)\n", + JANUS_LOG(LOG_DBG, "[%"SCNu64"] #%d: Scheduling %u for retransmission (probing, pkt_size=%d)\n", handle->handle_id, (pc->bwe->probing_count - 1), seqnr, p->length+SRTP_MAX_TAG_LEN); int i = 0; uint32_t enqueued = 0; @@ -4642,7 +4641,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { enqueued += p->length+SRTP_MAX_TAG_LEN; } pc->bwe->probing_sent += (enqueued * 8); - JANUS_LOG(LOG_WARN, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps; sent=%"SCNu32"/%"SCNu32")\n", + JANUS_LOG(LOG_DBG, "[%"SCNu64"] -- Enqueued %d duplicates (%"SCNu32" bytes, %"SCNu32" kbps; sent=%"SCNu32"/%"SCNu32")\n", handle->handle_id, duplicates, enqueued, (enqueued/1000)*8, pc->bwe->probing_sent, required); /* Done */ return G_SOURCE_CONTINUE; @@ -4707,8 +4706,8 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu return G_SOURCE_CONTINUE; JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth estimation\n", handle->handle_id); pc->bwe = janus_bwe_context_create(); - /* Let's create a source for BWE */ - handle->bwe_source = g_timeout_source_new(1000); + /* Let's create a source for BWE, just used to figure out when to notify the plugin */ + handle->bwe_source = g_timeout_source_new(50); /* FIXME */ g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); g_source_attach(handle->bwe_source, handle->mainctx); diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 3b767e7426..239d44b8c8 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8473,13 +8473,13 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t estimate = 0; JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); } else { - estimate -= ps->bitrates->bitrate[target]; if(s->sim_context.substream_target_bwe == -1 || s->sim_context.substream_target_bwe > new_substream || s->sim_context.templayer_target_bwe == -1 || s->sim_context.templayer_target_bwe > new_templayer) { - JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (video), switching to %d/%d\n", - substream, templayer, s->mindex, new_substream, new_templayer); + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (%"SCNu32" < %"SCNu32"), switching to %d/%d\n", + substream, templayer, s->mindex, estimate, ps->bitrates->bitrate[target], new_substream, new_templayer); /* FIXME */ } + estimate -= ps->bitrates->bitrate[target]; s->sim_context.substream_target_bwe = new_substream; s->sim_context.templayer_target_bwe = new_templayer; } diff --git a/src/rtcp.c b/src/rtcp.c index 28dfa49173..fa20c1acad 100644 --- a/src/rtcp.c +++ b/src/rtcp.c @@ -317,6 +317,8 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_bwe_ iter = iter->next; } g_list_free(list); + /* This callback is for updating the state of the bandwidth estimator */ + janus_bwe_context_update(bwe); } /* Link quality estimate filter coefficient */ From 8ad151a43c93f16dde97b030597280a2c1927aa9 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 27 Sep 2023 10:46:39 +0200 Subject: [PATCH 14/22] Fixed missing temporal layer change notification at startup --- src/rtp.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rtp.c b/src/rtp.c index c41eb7c982..1881b767f7 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1245,9 +1245,11 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte context->temporal_last = tid; if(!relay) return FALSE; - if(context->templayer != target && tid == target) { + if(context->templayer != target && (tid == target || + (target > tid && context->templayer < tid) || + (target < tid && context->templayer > tid))) { /* FIXME We should be smarter in deciding when to switch */ - context->templayer = target; + context->templayer = tid; /* Notify the caller that the temporal layer changed */ context->changed_temporal = TRUE; } From 6c333a73b2ea584b623ec551545f084b51f27e4c Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 27 Sep 2023 10:47:31 +0200 Subject: [PATCH 15/22] Changed delay tracking to moving average, and added CSV debugging --- src/bwe.c | 98 ++++++++++++++++++++++++++++++----- src/bwe.h | 43 ++++++++++++++- src/plugins/janus_videoroom.c | 62 +++++++++++----------- 3 files changed, 157 insertions(+), 46 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 499e36b052..8f299a9d30 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -58,7 +58,16 @@ janus_bwe_context *janus_bwe_context_create(void) { bwe->packets = g_hash_table_new_full(NULL, NULL, NULL, (GDestroyNotify)janus_bwe_twcc_inflight_destroy); bwe->sent = janus_bwe_stream_bitrate_create(); bwe->acked = janus_bwe_stream_bitrate_create(); + bwe->delays = janus_bwe_delay_tracker_create(0); bwe->probing_mindex = -1; +#ifdef BWE_DEBUGGING + char filename[256]; + g_snprintf(filename, sizeof(filename), "/tmp/bwe-janus-%"SCNi64, janus_get_real_time()); + bwe->csv = fopen(filename, "wt"); + char line[2048]; + g_snprintf(line, sizeof(line), "time,status,estimate,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_fb\n"); + fwrite(line, sizeof(char), strlen(line), bwe->csv); +#endif return bwe; } @@ -68,6 +77,10 @@ void janus_bwe_context_destroy(janus_bwe_context *bwe) { g_hash_table_destroy(bwe->packets); janus_bwe_stream_bitrate_destroy(bwe->sent); janus_bwe_stream_bitrate_destroy(bwe->acked); + janus_bwe_delay_tracker_destroy(bwe->delays); +#ifdef BWE_DEBUGGING + fclose(bwe->csv); +#endif g_free(bwe); } } @@ -156,13 +169,19 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { uint32_t rtx_in = bwe->acked->packets[janus_bwe_packet_type_rtx*3] ? bwe->acked->bitrate[janus_bwe_packet_type_rtx*3] : 0; uint32_t probing_in = bwe->acked->packets[janus_bwe_packet_type_probing*3] ? bwe->acked->bitrate[janus_bwe_packet_type_probing*3] : 0; uint32_t bitrate_in = rtx_in + probing_in + (bwe->acked->packets[0] ? bwe->acked->bitrate[0] : 0); - /* TODO Actually estimate the bitrate: now we're just checking how - * much the peer received out of what we sent, which is not enough */ + /* Get the average delay */ + double avg_delay_latest = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; + janus_bwe_delay_tracker_update(bwe->delays, now, avg_delay_latest); + int dts = bwe->delays->queue ? g_queue_get_length(bwe->delays->queue) : 0; + if(dts == 0) + dts = 1; + double avg_delay = bwe->delays->sum / (double)dts; + JANUS_LOG(LOG_WARN, "%.2f / %d = %.2f (%.2f)\n", bwe->delays->sum, dts, avg_delay, avg_delay_latest); + /* FIXME Estimate the bandwidth */ uint32_t estimate = bitrate_in; uint16_t tot = bwe->received_pkts + bwe->lost_pkts; if(tot > 0) bwe->loss_ratio = (double)bwe->lost_pkts / (double)tot; - double avg_delay = ((double)bwe->delay / (double)bwe->received_pkts) / 1000; /* Check if there's packet loss or congestion */ if(bwe->loss_ratio > 0.05) { /* FIXME Lossy network? Set the estimate to the acknowledged bitrate */ @@ -171,21 +190,30 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->status = janus_bwe_status_lossy; bwe->status_changed = now; bwe->estimate = estimate; - } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 1.0) { - /* FIXME Delay is increasing, converge to acknowledged bitrate */ - if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) - notify_plugin = TRUE; - bwe->status = janus_bwe_status_congested; - bwe->status_changed = now; - //~ if(estimate > bwe->estimate) - bwe->estimate = estimate; - //~ else - //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + } else if(avg_delay < 0) { + /* FIXME Average delay is negative, let's reset the delay increase counter */ + bwe->delay_increases = 0; + } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.2) { + /* FIXME Delay is increasing */ + bwe->delay_increases++; + if(bwe->delay_increases >= 4) { + bwe->delay_increases = 0; + /* FIXME Converge to acknowledged bitrate */ + if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) + notify_plugin = TRUE; + bwe->status = janus_bwe_status_congested; + bwe->status_changed = now; + //~ if(estimate > bwe->estimate) + bwe->estimate = estimate; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + } } else { /* FIXME All is fine? Check what state we're in */ if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { bwe->status = janus_bwe_status_recovering; bwe->status_changed = now; + bwe->delay_increases = 0; } if(bwe->status == janus_bwe_status_recovering) { /* FIXME Still recovering */ @@ -221,6 +249,16 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { now, janus_bwe_status_description(bwe->status), bitrate_out / 1024, probing_out / 1024, bitrate_in / 1024, probing_in / 1024, bwe->loss_ratio, bwe->avg_delay, bwe->estimate); +#ifdef BWE_DEBUGGING + /* Save the details to CSV */ + char line[2048]; + g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f\n", + now - bwe->started, bwe->status, + bwe->estimate, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, + bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, bwe->avg_delay, avg_delay_latest); + fwrite(line, sizeof(char), strlen(line), bwe->csv); +#endif + /* Reset values */ bwe->delay = 0; bwe->received_pkts = 0; bwe->lost_pkts = 0; @@ -285,3 +323,37 @@ void janus_bwe_stream_bitrate_destroy(janus_bwe_stream_bitrate *bwe_sb) { janus_mutex_destroy(&bwe_sb->mutex); g_free(bwe_sb); } + +janus_bwe_delay_tracker *janus_bwe_delay_tracker_create(int64_t keep_ts) { + janus_bwe_delay_tracker *dt = g_malloc0(sizeof(janus_bwe_delay_tracker)); + dt->keep_ts = (keep_ts > 0 ? keep_ts : G_USEC_PER_SEC); + return dt; +} + +void janus_bwe_delay_tracker_update(janus_bwe_delay_tracker *dt, int64_t when, double avg_delay) { + if(dt == NULL) + return; + if(dt->queue == NULL) + dt->queue = g_queue_new(); + /* Check if we need to get rid of some old feedback */ + int64_t cleanup_ts = when - dt->keep_ts; + janus_bwe_delay_fb *fb = g_queue_peek_head(dt->queue); + while(fb && fb->sent_ts < cleanup_ts) { + fb = g_queue_pop_head(dt->queue); + dt->sum -= fb->avg_delay; + g_free(fb); + fb = g_queue_peek_head(dt->queue); + } + /* Check if there's anything new we need to add now */ + fb = g_malloc(sizeof(janus_bwe_delay_fb)); + fb->sent_ts = when; + fb->avg_delay = avg_delay; + dt->sum += avg_delay; + g_queue_push_tail(dt->queue, fb); +} + +void janus_bwe_delay_tracker_destroy(janus_bwe_delay_tracker *dt) { + if(dt && dt->queue) + g_queue_free_full(dt->queue, (GDestroyNotify)g_free); + g_free(dt); +} diff --git a/src/bwe.h b/src/bwe.h index cd59ad2f61..3f9a4ab857 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -16,6 +16,8 @@ #include "mutex.h" +#define BWE_DEBUGGING + /*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ typedef struct janus_bwe_stream_bitrate { /*! \brief Time based queue of packet sizes */ @@ -48,6 +50,37 @@ typedef struct janus_bwe_stream_packet { uint16_t size; } janus_bwe_stream_packet; +/*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ +typedef struct janus_bwe_delay_tracker { + /*! \brief Time based queue of delays */ + GQueue *queue; + /*! \brief Current sum of average delays */ + double sum; + /*! \brief How long to keep items in queue (1s by default) */ + int64_t keep_ts; +} janus_bwe_delay_tracker; +/*! \brief Helper method to create a new janus_bwe_delay_tracker instance + * @note Passing 0 or a negative value for keep_ts will assume 1 second (G_USEC_PER_SEC) + * @param[im] keep_ts How long to keep items in queue + * @returns A janus_bwe_delay_tracker instance, if successful, or NULL otherwise */ +janus_bwe_delay_tracker *janus_bwe_delay_tracker_create(int64_t keep_ts); +/*! \brief Helper method to update an existing janus_bwe_delay_tracker instance with new data + * @param[in] dt The janus_bwe_delay_tracker instance to update + * @param[in] when Timestamp of the average delay + * @param[in] avg_delay Average delay */ +void janus_bwe_delay_tracker_update(janus_bwe_delay_tracker *dt, int64_t when, double avg_delay); +/*! \brief Helper method to destroy an existing janus_bwe_delay_tracker instance + * @param[in] dt The janus_bwe_delay_tracker instance to destroy */ +void janus_bwe_delay_tracker_destroy(janus_bwe_delay_tracker *dt); + +/*! \brief Instance of accumulated delay, from TWCC feedback */ +typedef struct janus_bwe_delay_fb { + /*! \brief Timestamp */ + int64_t sent_ts; + /*! \brief Average delay */ + double avg_delay; +} janus_bwe_delay_fb; + /*! \brief Transport Wide CC statuses */ typedef enum janus_bwe_twcc_status { janus_bwe_twcc_status_notreceived = 0, @@ -130,8 +163,12 @@ typedef struct janus_bwe_context { int64_t bitrate_ts; /*! \brief Bitrate tracker for sent and acked packets */ janus_bwe_stream_bitrate *sent, *acked; - /*! \brief How much delay has been accumulated (may be negative) */ + /*! \brief How much delay has been accumulated in the last feedback (may be negative) */ int64_t delay; + /*! \brief Accumulated delay over time */ + janus_bwe_delay_tracker *delays; + /*! \brief Number of consecutive times delay increased */ + uint8_t delay_increases; /*! \brief Number of packets with a received status, and number of lost ones */ uint16_t received_pkts, lost_pkts; /*! \brief Latest average delay */ @@ -144,6 +181,10 @@ typedef struct janus_bwe_context { gboolean notify_plugin; /*! \brief When we last notified the plugin */ int64_t last_notified; +#ifdef BWE_DEBUGGING + /*! \brief CSV where we save the the debugging information */ + FILE *csv; +#endif } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 239d44b8c8..4f1ed3f40c 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8451,44 +8451,42 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t int target = 3*substream + templayer; if(target > 8) target = 8; - if(ps->bitrates->packets[target]) { - if(estimate < ps->bitrates->bitrate[target]) { - /* We don't have room for these layers, find one below that fits */ - if(target == 0) { + if(ps->bitrates->packets[target] == NULL || estimate < ps->bitrates->bitrate[target]) { + /* Unavailable layer, or we don't have room for these layers, find one below that fits */ + if(target == 0) { + estimate = 0; + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); + } else { + gboolean found = FALSE; + int new_substream = 0, new_templayer = 0; + while(target > 0) { + target--; + new_substream = target/3; + new_templayer = target%3; + if(ps->bitrates->packets[target] && estimate >= ps->bitrates->bitrate[target]) { + found = TRUE; + break; + } + } + if(!found) { estimate = 0; JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); } else { - gboolean found = FALSE; - int new_substream = 0, new_templayer = 0; - while(target > 0) { - target--; - new_substream = target/3; - new_templayer = target%3; - if(ps->bitrates->packets[target] && estimate >= ps->bitrates->bitrate[target]) { - found = TRUE; - break; - } - } - if(!found) { - estimate = 0; - JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); - } else { - if(s->sim_context.substream_target_bwe == -1 || s->sim_context.substream_target_bwe > new_substream || - s->sim_context.templayer_target_bwe == -1 || s->sim_context.templayer_target_bwe > new_templayer) { - JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (%"SCNu32" < %"SCNu32"), switching to %d/%d\n", - substream, templayer, s->mindex, estimate, ps->bitrates->bitrate[target], new_substream, new_templayer); - /* FIXME */ - } - estimate -= ps->bitrates->bitrate[target]; - s->sim_context.substream_target_bwe = new_substream; - s->sim_context.templayer_target_bwe = new_templayer; + if(s->sim_context.substream_target_bwe == -1 || s->sim_context.substream_target_bwe > new_substream || + s->sim_context.templayer_target_bwe == -1 || s->sim_context.templayer_target_bwe > new_templayer) { + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (%"SCNu32" < %"SCNu32"), switching to %d/%d\n", + substream, templayer, s->mindex, estimate, ps->bitrates->bitrate[target], new_substream, new_templayer); + /* FIXME */ } + estimate -= ps->bitrates->bitrate[target]; + s->sim_context.substream_target_bwe = new_substream; + s->sim_context.templayer_target_bwe = new_templayer; } - } else { - estimate -= ps->bitrates->bitrate[target]; - s->sim_context.substream_target_bwe = -1; - s->sim_context.templayer_target_bwe = -1; } + } else { + estimate -= ps->bitrates->bitrate[target]; + s->sim_context.substream_target_bwe = -1; + s->sim_context.templayer_target_bwe = -1; } } else if(ps->svc) { /* TODO SVC */ From 8718e909e51e2454eaa64e79a3b5d6e15dd2947d Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 28 Sep 2023 11:48:33 +0200 Subject: [PATCH 16/22] Smoothen temporary layer transitions when driven by BWE --- src/rtp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rtp.c b/src/rtp.c index 1881b767f7..621852e400 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1229,8 +1229,10 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte if(relay) context->last_relayed = janus_get_monotonic_time(); /* Temporal layers are only easily available for some codecs */ + int substream_target = target; target = context->templayer_target; - if(context->templayer_target_bwe != -1 && context->templayer_target_bwe < target) + if(context->templayer_target_bwe != -1 && context->templayer_target_bwe < target && + substream_target <= context->substream) target = context->templayer_target_bwe; if(vcodec == JANUS_VIDEOCODEC_VP8) { /* Check if there's any temporal scalability to take into account */ From 8f3eb5c542901996e0493ab60434d7c25211490f Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 28 Sep 2023 11:53:16 +0200 Subject: [PATCH 17/22] Weighted average for delays, and experiments with smoother probing --- src/bwe.c | 41 +++++++++++++------------------ src/bwe.h | 6 ++--- src/ice.c | 45 ++++++++++++++++++++++++++++++----- src/ice.h | 3 +++ src/janus.c | 10 ++++++++ src/plugins/janus_videoroom.c | 1 + src/plugins/plugin.h | 5 +++- 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 8f299a9d30..96be1c944e 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -65,7 +65,7 @@ janus_bwe_context *janus_bwe_context_create(void) { g_snprintf(filename, sizeof(filename), "/tmp/bwe-janus-%"SCNi64, janus_get_real_time()); bwe->csv = fopen(filename, "wt"); char line[2048]; - g_snprintf(line, sizeof(line), "time,status,estimate,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_fb\n"); + g_snprintf(line, sizeof(line), "time,status,estimate,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); fwrite(line, sizeof(char), strlen(line), bwe->csv); #endif return bwe; @@ -176,7 +176,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { if(dts == 0) dts = 1; double avg_delay = bwe->delays->sum / (double)dts; - JANUS_LOG(LOG_WARN, "%.2f / %d = %.2f (%.2f)\n", bwe->delays->sum, dts, avg_delay, avg_delay_latest); + double avg_delay_weighted = (bwe->bitrate_ts == now ? avg_delay : ((bwe->avg_delay * 0.9) + (avg_delay * 0.1))); /* FIXME Estimate the bandwidth */ uint32_t estimate = bitrate_in; uint16_t tot = bwe->received_pkts + bwe->lost_pkts; @@ -190,30 +190,21 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->status = janus_bwe_status_lossy; bwe->status_changed = now; bwe->estimate = estimate; - } else if(avg_delay < 0) { - /* FIXME Average delay is negative, let's reset the delay increase counter */ - bwe->delay_increases = 0; - } else if(bwe->avg_delay > 0 && avg_delay - bwe->avg_delay > 0.2) { + } else if(avg_delay_weighted > 1.5 && avg_delay_weighted - bwe->avg_delay > 0.1) { /* FIXME Delay is increasing */ - bwe->delay_increases++; - if(bwe->delay_increases >= 4) { - bwe->delay_increases = 0; - /* FIXME Converge to acknowledged bitrate */ - if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) - notify_plugin = TRUE; - bwe->status = janus_bwe_status_congested; - bwe->status_changed = now; - //~ if(estimate > bwe->estimate) - bwe->estimate = estimate; - //~ else - //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); - } + if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) + notify_plugin = TRUE; + bwe->status = janus_bwe_status_congested; + bwe->status_changed = now; + //~ if(estimate > bwe->estimate) + bwe->estimate = estimate; + //~ else + //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); } else { /* FIXME All is fine? Check what state we're in */ if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { bwe->status = janus_bwe_status_recovering; bwe->status_changed = now; - bwe->delay_increases = 0; } if(bwe->status == janus_bwe_status_recovering) { /* FIXME Still recovering */ @@ -224,7 +215,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { /* Restore probing but incrementally */ bwe->probing_sent = 0; bwe->probing_portion = 0.0; - bwe->probing_part = 20; /* FIXME */ + bwe->probing_part = 200; /* FIXME */ } else { /* FIXME Keep converging to the estimate */ if(estimate > bwe->estimate) @@ -243,7 +234,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->probing_part--; } } - bwe->avg_delay = avg_delay; + bwe->avg_delay = avg_delay_weighted; bwe->bitrate_ts = now; JANUS_LOG(LOG_WARN, "[BWE][%"SCNi64"][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), acked=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms, estimate=%"SCNu32"\n", now, janus_bwe_status_description(bwe->status), @@ -252,10 +243,10 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { #ifdef BWE_DEBUGGING /* Save the details to CSV */ char line[2048]; - g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f\n", + g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f,%.2f\n", now - bwe->started, bwe->status, bwe->estimate, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, - bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, bwe->avg_delay, avg_delay_latest); + bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, avg_delay, avg_delay_weighted, avg_delay_latest); fwrite(line, sizeof(char), strlen(line), bwe->csv); #endif /* Reset values */ @@ -263,7 +254,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->received_pkts = 0; bwe->lost_pkts = 0; /* Check if we should notify the plugin about the estimate */ - if(notify_plugin || (now - bwe->last_notified) >= G_USEC_PER_SEC) { + if(notify_plugin || (now - bwe->last_notified) >= 250000) { bwe->notify_plugin = TRUE; bwe->last_notified = now; } diff --git a/src/bwe.h b/src/bwe.h index 3f9a4ab857..57476352bd 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -151,8 +151,10 @@ typedef struct janus_bwe_context { uint8_t probing_count; /*! \brief Portion of probing we didn't manage to send the previous round */ double probing_portion; + /*! \brief In case probing was deferred, when it shoult restart */ + int64_t probing_deferred; /*! \brief Whether probing should grow incrementally (e.g., after recovery) */ - uint8_t probing_part; + uint16_t probing_part; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ @@ -167,8 +169,6 @@ typedef struct janus_bwe_context { int64_t delay; /*! \brief Accumulated delay over time */ janus_bwe_delay_tracker *delays; - /*! \brief Number of consecutive times delay increased */ - uint8_t delay_increases; /*! \brief Number of packets with a received status, and number of lost ones */ uint16_t received_pkts, lost_pkts; /*! \brief Latest average delay */ diff --git a/src/ice.c b/src/ice.c index 6511267fa7..9b94c44855 100644 --- a/src/ice.c +++ b/src/ice.c @@ -476,6 +476,7 @@ static janus_ice_queued_packet janus_ice_enable_bwe, janus_ice_disable_bwe, janus_ice_enable_probing, + janus_ice_defer_probing, janus_ice_disable_probing, janus_ice_hangup_peerconnection, janus_ice_detach_handle, @@ -657,7 +658,7 @@ static void janus_ice_free_queued_packet(janus_ice_queued_packet *pkt) { pkt == &janus_ice_add_candidates || pkt == &janus_ice_dtls_handshake || pkt == &janus_ice_enable_bwe || pkt == &janus_ice_disable_bwe || - pkt == &janus_ice_enable_probing || pkt == &janus_ice_disable_probing || + pkt == &janus_ice_enable_probing || pkt == &janus_ice_defer_probing || pkt == &janus_ice_disable_probing || pkt == &janus_ice_media_stopped || pkt == &janus_ice_hangup_peerconnection || pkt == &janus_ice_detach_handle || @@ -3915,6 +3916,18 @@ void janus_ice_handle_enable_probing(janus_ice_handle *handle) { } } +void janus_ice_handle_defer_probing(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Deferring bandwidth probing\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_defer_probing); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_defer_probing); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + void janus_ice_handle_disable_probing(janus_ice_handle *handle) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); if(handle->queued_packets != NULL) { @@ -4544,6 +4557,14 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { /* The BWE status may be lossy, congested, or recovering: don't probe for now */ return G_SOURCE_CONTINUE; } + gint64 now = janus_get_monotonic_time(); + if(pc->bwe->probing_deferred > 0) { + if(now < pc->bwe->probing_deferred) { + /* The probing has been deferred for a few seconds */ + return G_SOURCE_CONTINUE; + } + pc->bwe->probing_deferred = 0; + } /* Get the medium instance we'll use for probing */ if(pc->bwe->probing_mindex != -1) medium = g_hash_table_lookup(pc->media, GINT_TO_POINTER(pc->bwe->probing_mindex)); @@ -4604,7 +4625,6 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { handle->handle_id, (pc->bwe->probing_count - 1), seqnr, p->length+SRTP_MAX_TAG_LEN); int i = 0; uint32_t enqueued = 0; - gint64 now = janus_get_monotonic_time(); for(i=0; i < duplicates; i++) { p->last_retransmit = now; /* Enqueue it */ @@ -4737,16 +4757,29 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); /* Let's create a source for probing */ pc->bwe->probing_target = 200000; /* FIXME */ - pc->bwe->probing_count = 0; /* FIXME */ - pc->bwe->probing_sent = 0; /* FIXME */ - pc->bwe->probing_part = 0; /* FIXME */ + pc->bwe->probing_count = 0; + pc->bwe->probing_sent = 0; + pc->bwe->probing_portion = 0.0; + pc->bwe->probing_part = 100; /* FIXME */ handle->probing_source = g_timeout_source_new(50); g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); g_source_attach(handle->probing_source, handle->mainctx); return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_defer_probing) { + /* We need to temporarily pause bandwidth probing */ + if(pc == NULL || pc->bwe == NULL || handle->probing_source == NULL) + return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Deferring bandwidth probing\n", handle->handle_id); + gint64 deferred = janus_get_monotonic_time() + G_USEC_PER_SEC; + if(deferred > pc->bwe->probing_deferred) + pc->bwe->probing_deferred = deferred; + pc->bwe->probing_sent = 0; + pc->bwe->probing_portion = 0.0; + pc->bwe->probing_part = 100; /* FIXME */ + return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_probing) { - /* We need to get rif of the bandwidth probing */ + /* We need to get rid of the bandwidth probing */ if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); diff --git a/src/ice.h b/src/ice.h index 087b81d8fe..07ac82b4c0 100644 --- a/src/ice.h +++ b/src/ice.h @@ -796,6 +796,9 @@ void janus_ice_handle_disable_bwe(janus_ice_handle *handle); /*! \brief Method to dynamically enable bandwidth probing for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_enable_probing(janus_ice_handle *handle); +/*! \brief Method to dynamically defer bandwidth probing for a handle for a few seconds + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_defer_probing(janus_ice_handle *handle); /*! \brief Method to dynamically disable bandwidth probing for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_disable_probing(janus_ice_handle *handle); diff --git a/src/janus.c b/src/janus.c index 5690c438e7..5aa82fbef4 100644 --- a/src/janus.c +++ b/src/janus.c @@ -614,6 +614,7 @@ void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitra void janus_plugin_enable_bwe(janus_plugin_session *plugin_session); void janus_plugin_disable_bwe(janus_plugin_session *plugin_session); void janus_plugin_enable_probing(janus_plugin_session *plugin_session); +void janus_plugin_defer_probing(janus_plugin_session *plugin_session); void janus_plugin_disable_probing(janus_plugin_session *plugin_session); void janus_plugin_close_pc(janus_plugin_session *plugin_session); void janus_plugin_end_session(janus_plugin_session *plugin_session); @@ -633,6 +634,7 @@ static janus_callbacks janus_handler_plugin = .enable_bwe = janus_plugin_enable_bwe, .disable_bwe = janus_plugin_disable_bwe, .enable_probing = janus_plugin_enable_probing, + .defer_probing = janus_plugin_defer_probing, .disable_probing = janus_plugin_disable_probing, .close_pc = janus_plugin_close_pc, .end_session = janus_plugin_end_session, @@ -4290,6 +4292,14 @@ void janus_plugin_enable_probing(janus_plugin_session *plugin_session) { janus_ice_handle_enable_probing(handle); } +void janus_plugin_defer_probing(janus_plugin_session *plugin_session) { + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_handle_defer_probing(handle); +} + void janus_plugin_disable_probing(janus_plugin_session *plugin_session) { janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 4f1ed3f40c..eab504cfa6 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -12460,6 +12460,7 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) } else { /* We're not receiving what we need, start probing (if we weren't already) */ gateway->enable_probing(subscriber->session->handle); + gateway->defer_probing(subscriber->session->handle); } } /* If we got here, update the RTP header and send the packet */ diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index cb853b0ae2..7bf54328e2 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -420,7 +420,10 @@ struct janus_callbacks { * @param[in] handle The plugin/gateway session to enable BWE probing for */ void (* const enable_probing)(janus_plugin_session *handle); /*! \brief Disable bandwidth probing for this session - * @param[in] handle The plugin/gateway session to disnable BWE probing for */ + * @param[in] handle The plugin/gateway session to disable BWE probing for */ + void (* const defer_probing)(janus_plugin_session *handle); + /*! \brief Defers bandwidth probing for this session for a few seconds + * @param[in] handle The plugin/gateway session to defer BWE probing for */ void (* const disable_probing)(janus_plugin_session *handle); /*! \brief Callback to ask the core to close a WebRTC PeerConnection From 317657b09f14b3fbff66caa2dbfa86b426412722 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 4 Oct 2023 12:00:29 +0200 Subject: [PATCH 18/22] Refactored probing API to chase a bitrate target --- src/bwe.c | 19 ++-- src/bwe.h | 8 +- src/ice.c | 165 +++++++++++++--------------------- src/ice.h | 19 ++-- src/janus.c | 32 ++----- src/plugins/janus_videoroom.c | 70 ++++++++++++--- src/plugins/plugin.h | 20 ++--- src/rtp.c | 5 +- 8 files changed, 158 insertions(+), 180 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 96be1c944e..f34999f5b7 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -65,7 +65,7 @@ janus_bwe_context *janus_bwe_context_create(void) { g_snprintf(filename, sizeof(filename), "/tmp/bwe-janus-%"SCNi64, janus_get_real_time()); bwe->csv = fopen(filename, "wt"); char line[2048]; - g_snprintf(line, sizeof(line), "time,status,estimate,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); + g_snprintf(line, sizeof(line), "time,status,estimate,probing_target,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); fwrite(line, sizeof(char), strlen(line), bwe->csv); #endif return bwe; @@ -190,7 +190,9 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->status = janus_bwe_status_lossy; bwe->status_changed = now; bwe->estimate = estimate; - } else if(avg_delay_weighted > 1.5 && avg_delay_weighted - bwe->avg_delay > 0.1) { + } else if(avg_delay_weighted > 1.2 && avg_delay_weighted - bwe->avg_delay > 0.1) { + JANUS_LOG(LOG_WARN, "[BWE] Congested (delay=%.2f, increase=%.2f)\n", + avg_delay_weighted, avg_delay_weighted - bwe->avg_delay); /* FIXME Delay is increasing */ if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) notify_plugin = TRUE; @@ -215,7 +217,8 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { /* Restore probing but incrementally */ bwe->probing_sent = 0; bwe->probing_portion = 0.0; - bwe->probing_part = 200; /* FIXME */ + bwe->probing_buildup = 0; + bwe->probing_buildup_timer = now; } else { /* FIXME Keep converging to the estimate */ if(estimate > bwe->estimate) @@ -230,22 +233,20 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->estimate = estimate; //~ else if(now - bwe->status_changed < 10*G_USEC_PER_SEC) //~ bwe->estimate = ((double)bwe->estimate * 1.02); - if(bwe->probing_part > 0) /* FIXME */ - bwe->probing_part--; } } bwe->avg_delay = avg_delay_weighted; bwe->bitrate_ts = now; - JANUS_LOG(LOG_WARN, "[BWE][%"SCNi64"][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), acked=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms, estimate=%"SCNu32"\n", + JANUS_LOG(LOG_DBG, "[BWE][%"SCNi64"][%s] sent=%"SCNu32"kbps (probing=%"SCNu32"kbps), acked=%"SCNu32"kbps (probing=%"SCNu32"kbps), loss=%.2f%%, avg_delay=%.2fms, estimate=%"SCNu32"\n", now, janus_bwe_status_description(bwe->status), bitrate_out / 1024, probing_out / 1024, bitrate_in / 1024, probing_in / 1024, bwe->loss_ratio, bwe->avg_delay, bwe->estimate); #ifdef BWE_DEBUGGING /* Save the details to CSV */ char line[2048]; - g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f,%.2f\n", + g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f,%.2f\n", now - bwe->started, bwe->status, - bwe->estimate, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, + bwe->estimate, bwe->probing_target, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, avg_delay, avg_delay_weighted, avg_delay_latest); fwrite(line, sizeof(char), strlen(line), bwe->csv); #endif @@ -254,7 +255,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->received_pkts = 0; bwe->lost_pkts = 0; /* Check if we should notify the plugin about the estimate */ - if(notify_plugin || (now - bwe->last_notified) >= 250000) { + if(bwe->status != janus_bwe_status_start && (notify_plugin || (now - bwe->last_notified) >= 250000)) { bwe->notify_plugin = TRUE; bwe->last_notified = now; } diff --git a/src/bwe.h b/src/bwe.h index 57476352bd..93b32c8444 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -145,16 +145,16 @@ typedef struct janus_bwe_context { int64_t status_changed; /*! \brief Index of the m-line we're using for probing */ int probing_mindex; - /*! \brief How much we should aim for with out probing (and how much we sent in a second) */ - uint32_t probing_target, probing_sent; + /*! \brief How much we should aim for with out probing (and how much to increase, plus much we sent in a second) */ + uint32_t probing_target, probing_buildup, probing_sent; /*! \brief How many times we went through probing in a second */ uint8_t probing_count; /*! \brief Portion of probing we didn't manage to send the previous round */ double probing_portion; /*! \brief In case probing was deferred, when it shoult restart */ int64_t probing_deferred; - /*! \brief Whether probing should grow incrementally (e.g., after recovery) */ - uint16_t probing_part; + /*! \brief Timer for building up probing */ + int64_t probing_buildup_timer; /*! \brief Monotonic timestamp of the last sent packet */ int64_t last_sent_ts; /*! \brief Last twcc seq number of a received packet */ diff --git a/src/ice.c b/src/ice.c index 9b94c44855..764ac49eda 100644 --- a/src/ice.c +++ b/src/ice.c @@ -474,10 +474,8 @@ static janus_ice_queued_packet janus_ice_dtls_handshake, janus_ice_media_stopped, janus_ice_enable_bwe, + janus_ice_set_bwe_target, janus_ice_disable_bwe, - janus_ice_enable_probing, - janus_ice_defer_probing, - janus_ice_disable_probing, janus_ice_hangup_peerconnection, janus_ice_detach_handle, janus_ice_data_ready; @@ -534,7 +532,6 @@ typedef struct janus_ice_outgoing_traffic { static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data); static gboolean janus_ice_outgoing_stats_handle(gpointer user_data); static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data); -static gboolean janus_ice_outgoing_probing_handle(gpointer user_data); static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janus_ice_queued_packet *pkt); static gboolean janus_ice_outgoing_traffic_prepare(GSource *source, gint *timeout) { janus_ice_outgoing_traffic *t = (janus_ice_outgoing_traffic *)source; @@ -657,8 +654,7 @@ static void janus_ice_free_queued_packet(janus_ice_queued_packet *pkt) { if(pkt == NULL || pkt == &janus_ice_start_gathering || pkt == &janus_ice_add_candidates || pkt == &janus_ice_dtls_handshake || - pkt == &janus_ice_enable_bwe || pkt == &janus_ice_disable_bwe || - pkt == &janus_ice_enable_probing || pkt == &janus_ice_defer_probing || pkt == &janus_ice_disable_probing || + pkt == &janus_ice_enable_bwe || pkt == &janus_ice_set_bwe_target || pkt == &janus_ice_disable_bwe || pkt == &janus_ice_media_stopped || pkt == &janus_ice_hangup_peerconnection || pkt == &janus_ice_detach_handle || @@ -3892,49 +3888,29 @@ void janus_ice_handle_enable_bwe(janus_ice_handle *handle) { } } -void janus_ice_handle_disable_bwe(janus_ice_handle *handle) { - JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); - if(handle->queued_packets != NULL) { -#if GLIB_CHECK_VERSION(2, 46, 0) - g_async_queue_push_front(handle->queued_packets, &janus_ice_disable_bwe); -#else - g_async_queue_push(handle->queued_packets, &janus_ice_disable_bwe); -#endif - g_main_context_wakeup(handle->mainctx); - } -} - -void janus_ice_handle_enable_probing(janus_ice_handle *handle) { - JANUS_LOG(LOG_VERB, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); - if(handle->queued_packets != NULL) { -#if GLIB_CHECK_VERSION(2, 46, 0) - g_async_queue_push_front(handle->queued_packets, &janus_ice_enable_probing); -#else - g_async_queue_push(handle->queued_packets, &janus_ice_enable_probing); -#endif - g_main_context_wakeup(handle->mainctx); - } -} - -void janus_ice_handle_defer_probing(janus_ice_handle *handle) { - JANUS_LOG(LOG_VERB, "[%"SCNu64"] Deferring bandwidth probing\n", handle->handle_id); +/* FIXME */ +void janus_ice_handle_set_bwe_target(janus_ice_handle *handle, uint32_t bitrate) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Configuring bandwidth estimation target: %"SCNu32"\n", handle->handle_id, bitrate); + janus_mutex_lock(&handle->mutex); + handle->bwe_target = bitrate; + janus_mutex_unlock(&handle->mutex); if(handle->queued_packets != NULL) { #if GLIB_CHECK_VERSION(2, 46, 0) - g_async_queue_push_front(handle->queued_packets, &janus_ice_defer_probing); + g_async_queue_push_front(handle->queued_packets, &janus_ice_set_bwe_target); #else - g_async_queue_push(handle->queued_packets, &janus_ice_defer_probing); + g_async_queue_push(handle->queued_packets, &janus_ice_set_bwe_target); #endif g_main_context_wakeup(handle->mainctx); } } -void janus_ice_handle_disable_probing(janus_ice_handle *handle) { - JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); +void janus_ice_handle_disable_bwe(janus_ice_handle *handle) { + JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); if(handle->queued_packets != NULL) { #if GLIB_CHECK_VERSION(2, 46, 0) - g_async_queue_push_front(handle->queued_packets, &janus_ice_disable_probing); + g_async_queue_push_front(handle->queued_packets, &janus_ice_disable_bwe); #else - g_async_queue_push(handle->queued_packets, &janus_ice_disable_probing); + g_async_queue_push(handle->queued_packets, &janus_ice_disable_bwe); #endif g_main_context_wakeup(handle->mainctx); } @@ -4535,24 +4511,20 @@ static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { janus_ice_peerconnection *pc = handle->pc; if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; + /* First of all, let's check if we have to notify the plugin */ if(pc->bwe->notify_plugin) { - /* Notify the plugin about the current value */ + /* Notify the plugin about the current estimate */ pc->bwe->notify_plugin = FALSE; janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->estimated_bandwidth && janus_plugin_session_is_alive(handle->app_handle) && !g_atomic_int_get(&handle->destroyed)) plugin->estimated_bandwidth(handle->app_handle, pc->bwe->estimate); } - return G_SOURCE_CONTINUE; -} - -static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { - janus_ice_handle *handle = (janus_ice_handle *)user_data; - janus_ice_peerconnection *pc = handle->pc; - janus_ice_peerconnection_medium *medium = NULL; - if(pc == NULL || pc->bwe == NULL || pc->bwe->probing_target == 0) + /* Now, let's check if there's probing we need to perform */ + uint32_t bitrate_out = (pc->bwe->sent->packets[janus_bwe_packet_type_regular*3] ? pc->bwe->sent->bitrate[janus_bwe_packet_type_regular*3] : 0) + + (pc->bwe->sent->packets[janus_bwe_packet_type_rtx*3] ? pc->bwe->sent->bitrate[janus_bwe_packet_type_rtx*3] : 0); + if(pc->bwe->probing_target == 0 || pc->bwe->probing_target <= bitrate_out) return G_SOURCE_CONTINUE; - /* This callback is for regularly sending probing for the bandwidth estimator */ if(pc->bwe->status != janus_bwe_status_start && pc->bwe->status != janus_bwe_status_regular) { /* The BWE status may be lossy, congested, or recovering: don't probe for now */ return G_SOURCE_CONTINUE; @@ -4566,6 +4538,7 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { pc->bwe->probing_deferred = 0; } /* Get the medium instance we'll use for probing */ + janus_ice_peerconnection_medium *medium = NULL; if(pc->bwe->probing_mindex != -1) medium = g_hash_table_lookup(pc->media, GINT_TO_POINTER(pc->bwe->probing_mindex)); if(medium == NULL || !medium->send || medium->rtx_payload_type == -1 || g_queue_is_empty(medium->retransmit_buffer)) { @@ -4597,16 +4570,24 @@ static gboolean janus_ice_outgoing_probing_handle(gpointer user_data) { pc->bwe->probing_portion = 0.0; } pc->bwe->probing_count++; - uint32_t required = pc->bwe->probing_target; - if(pc->bwe->probing_part > 0) - required = required / pc->bwe->probing_part; + /* Check if we need to build up probing */ + uint32_t required = pc->bwe->probing_target - bitrate_out; + if(pc->bwe->probing_buildup_timer > 0) { + if(pc->bwe->probing_buildup == 0) + pc->bwe->probing_buildup = 25000; + if(now - pc->bwe->probing_buildup_timer >= 500000) { + pc->bwe->probing_buildup += 25000; + pc->bwe->probing_buildup_timer = now; + } + if(pc->bwe->probing_buildup >= required) + pc->bwe->probing_buildup = required; + required = pc->bwe->probing_buildup; + } if(pc->bwe->probing_sent >= required) { /* We sent enough for this round */ return G_SOURCE_CONTINUE; } uint32_t required_now = required / (8 * 20); - //~ if(pc->bwe->probing_count == 20) - //~ required_now = (pc->bwe->probing_target - pc->bwe->probing_sent) / 8; double prev_portion = pc->bwe->probing_portion; double portion = (double)required_now / (double)(p->length+SRTP_MAX_TAG_LEN); double new_portion = prev_portion + portion; @@ -4726,12 +4707,36 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu return G_SOURCE_CONTINUE; JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth estimation\n", handle->handle_id); pc->bwe = janus_bwe_context_create(); - /* Let's create a source for BWE, just used to figure out when to notify the plugin */ + janus_mutex_lock(&handle->mutex); + handle->bwe_target = 0; + janus_mutex_unlock(&handle->mutex); + /* Let's create a source for BWE, for plugin notifications and probing */ handle->bwe_source = g_timeout_source_new(50); /* FIXME */ g_source_set_priority(handle->bwe_source, G_PRIORITY_DEFAULT); g_source_set_callback(handle->bwe_source, janus_ice_outgoing_bwe_handle, handle, NULL); g_source_attach(handle->bwe_source, handle->mainctx); return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_set_bwe_target) { + /* Enable probing for the BWE context (assuming BWE is active) */ + if(pc == NULL || pc->bwe == NULL) + return G_SOURCE_CONTINUE; + janus_mutex_lock(&handle->mutex); + uint32_t target = handle->bwe_target; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Configuring bandwidth estimation target: %"SCNu32"\n", handle->handle_id, target); + if(target == pc->bwe->probing_target) { + /* Nothing to do */ + janus_mutex_unlock(&handle->mutex); + return G_SOURCE_CONTINUE; + } + janus_mutex_unlock(&handle->mutex); + /* Let's configure probing accordingly */ + pc->bwe->probing_target = target; + pc->bwe->probing_count = 0; + pc->bwe->probing_sent = 0; + pc->bwe->probing_portion = 0.0; + pc->bwe->probing_buildup = 0; + pc->bwe->probing_buildup_timer = janus_get_monotonic_time(); + return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_bwe) { /* We need to get rif of the bandwidth estimator */ if(pc == NULL || pc->bwe == NULL) @@ -4742,52 +4747,11 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu g_source_unref(handle->bwe_source); handle->bwe_source = NULL; } - if(handle->probing_source) { - g_source_destroy(handle->probing_source); - g_source_unref(handle->probing_source); - handle->probing_source = NULL; - } janus_bwe_context_destroy(pc->bwe); pc->bwe = NULL; - return G_SOURCE_CONTINUE; - } else if(pkt == &janus_ice_enable_probing) { - /* Enable probing for the BWE context (assuming BWE is active) */ - if(pc == NULL || pc->bwe == NULL || handle->probing_source != NULL) - return G_SOURCE_CONTINUE; - JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling bandwidth probing\n", handle->handle_id); - /* Let's create a source for probing */ - pc->bwe->probing_target = 200000; /* FIXME */ - pc->bwe->probing_count = 0; - pc->bwe->probing_sent = 0; - pc->bwe->probing_portion = 0.0; - pc->bwe->probing_part = 100; /* FIXME */ - handle->probing_source = g_timeout_source_new(50); - g_source_set_priority(handle->probing_source, G_PRIORITY_DEFAULT); - g_source_set_callback(handle->probing_source, janus_ice_outgoing_probing_handle, handle, NULL); - g_source_attach(handle->probing_source, handle->mainctx); - return G_SOURCE_CONTINUE; - } else if(pkt == &janus_ice_defer_probing) { - /* We need to temporarily pause bandwidth probing */ - if(pc == NULL || pc->bwe == NULL || handle->probing_source == NULL) - return G_SOURCE_CONTINUE; - JANUS_LOG(LOG_INFO, "[%"SCNu64"] Deferring bandwidth probing\n", handle->handle_id); - gint64 deferred = janus_get_monotonic_time() + G_USEC_PER_SEC; - if(deferred > pc->bwe->probing_deferred) - pc->bwe->probing_deferred = deferred; - pc->bwe->probing_sent = 0; - pc->bwe->probing_portion = 0.0; - pc->bwe->probing_part = 100; /* FIXME */ - return G_SOURCE_CONTINUE; - } else if(pkt == &janus_ice_disable_probing) { - /* We need to get rid of the bandwidth probing */ - if(pc == NULL || pc->bwe == NULL) - return G_SOURCE_CONTINUE; - JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling bandwidth probing\n", handle->handle_id); - if(handle->probing_source) { - g_source_destroy(handle->probing_source); - g_source_unref(handle->probing_source); - handle->probing_source = NULL; - } + janus_mutex_lock(&handle->mutex); + handle->bwe_target = 0; + janus_mutex_unlock(&handle->mutex); return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_media_stopped) { /* Some media has been disabled on the way in, so use the callback to notify the peer */ @@ -4838,11 +4802,6 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu g_source_unref(handle->bwe_source); handle->bwe_source = NULL; } - if(handle->probing_source) { - g_source_destroy(handle->probing_source); - g_source_unref(handle->probing_source); - handle->probing_source = NULL; - } if(handle->stats_source) { g_source_destroy(handle->stats_source); g_source_unref(handle->stats_source); diff --git a/src/ice.h b/src/ice.h index 07ac82b4c0..41414c59f7 100644 --- a/src/ice.h +++ b/src/ice.h @@ -374,7 +374,7 @@ struct janus_ice_handle { /*! \brief GLib thread for the handle and libnice */ GThread *thread; /*! \brief GLib sources for outgoing traffic, recurring RTCP, and stats (and optionally TWCC/BWE/probing) */ - GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source, *bwe_source, *probing_source; + GSource *rtp_source, *rtcp_source, *stats_source, *twcc_source, *bwe_source; /*! \brief libnice ICE agent */ NiceAgent *agent; /*! \brief Monotonic time of when the ICE agent has been created */ @@ -409,6 +409,8 @@ struct janus_ice_handle { gint last_srtp_error, last_srtp_summary; /*! \brief Count of how many seconds passed since the last stats passed to event handlers */ gint last_event_stats; + /*! \brief Bandwidth estimation target, if any, as asked by the plugin */ + uint32_t bwe_target; /*! \brief Flag to decide whether or not packets need to be dumped to a text2pcap file */ volatile gint dump_packets; /*! \brief In case this session must be saved to text2pcap, the instance to dump packets to */ @@ -524,7 +526,7 @@ struct janus_ice_peerconnection { GHashTable *rtx_payload_types_rev; /*! \brief Helper flag to avoid flooding the console with the same error all over again */ gboolean noerrorlog; - /*! Bandwidth estimation context */ + /*! \brief Bandwidth estimation context */ janus_bwe_context *bwe; /*! \brief Mutex to lock/unlock this stream */ janus_mutex mutex; @@ -790,18 +792,13 @@ void janus_ice_resend_trickles(janus_ice_handle *handle); /*! \brief Method to dynamically enable bandwidth estimation for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_enable_bwe(janus_ice_handle *handle); +/*! \brief Method to dynamically tweak the bandwidth estimation target for a handle (for probing) + * @param[in] handle The Janus ICE handle this method refers to + * @param[in] bitrate The bitrate to target (will be used to generate probing) */ +void janus_ice_handle_set_bwe_target(janus_ice_handle *handle, uint32_t bitrate); /*! \brief Method to dynamically disable bandwidth estimation for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_disable_bwe(janus_ice_handle *handle); -/*! \brief Method to dynamically enable bandwidth probing for a handle - * @param[in] handle The Janus ICE handle this method refers to */ -void janus_ice_handle_enable_probing(janus_ice_handle *handle); -/*! \brief Method to dynamically defer bandwidth probing for a handle for a few seconds - * @param[in] handle The Janus ICE handle this method refers to */ -void janus_ice_handle_defer_probing(janus_ice_handle *handle); -/*! \brief Method to dynamically disable bandwidth probing for a handle - * @param[in] handle The Janus ICE handle this method refers to */ -void janus_ice_handle_disable_probing(janus_ice_handle *handle); ///@} diff --git a/src/janus.c b/src/janus.c index 5aa82fbef4..ca4351ac53 100644 --- a/src/janus.c +++ b/src/janus.c @@ -612,10 +612,8 @@ void janus_plugin_send_pli(janus_plugin_session *plugin_session); void janus_plugin_send_pli_stream(janus_plugin_session *plugin_session, int mindex); void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitrate); void janus_plugin_enable_bwe(janus_plugin_session *plugin_session); +void janus_plugin_set_bwe_target(janus_plugin_session *plugin_session, uint32_t bitrate); void janus_plugin_disable_bwe(janus_plugin_session *plugin_session); -void janus_plugin_enable_probing(janus_plugin_session *plugin_session); -void janus_plugin_defer_probing(janus_plugin_session *plugin_session); -void janus_plugin_disable_probing(janus_plugin_session *plugin_session); void janus_plugin_close_pc(janus_plugin_session *plugin_session); void janus_plugin_end_session(janus_plugin_session *plugin_session); void janus_plugin_notify_event(janus_plugin *plugin, janus_plugin_session *plugin_session, json_t *event); @@ -632,10 +630,8 @@ static janus_callbacks janus_handler_plugin = .send_pli_stream = janus_plugin_send_pli_stream, .send_remb = janus_plugin_send_remb, .enable_bwe = janus_plugin_enable_bwe, + .set_bwe_target = janus_plugin_set_bwe_target, .disable_bwe = janus_plugin_disable_bwe, - .enable_probing = janus_plugin_enable_probing, - .defer_probing = janus_plugin_defer_probing, - .disable_probing = janus_plugin_disable_probing, .close_pc = janus_plugin_close_pc, .end_session = janus_plugin_end_session, .events_is_enabled = janus_events_is_enabled, @@ -4276,36 +4272,20 @@ void janus_plugin_enable_bwe(janus_plugin_session *plugin_session) { janus_ice_handle_enable_bwe(handle); } -void janus_plugin_disable_bwe(janus_plugin_session *plugin_session) { - janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; - if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) - || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) - return; - janus_ice_handle_disable_bwe(handle); -} - -void janus_plugin_enable_probing(janus_plugin_session *plugin_session) { +void janus_plugin_set_bwe_target(janus_plugin_session *plugin_session, uint32_t bitrate) { janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) return; - janus_ice_handle_enable_probing(handle); + janus_ice_handle_set_bwe_target(handle, bitrate); } -void janus_plugin_defer_probing(janus_plugin_session *plugin_session) { - janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; - if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) - || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) - return; - janus_ice_handle_defer_probing(handle); -} - -void janus_plugin_disable_probing(janus_plugin_session *plugin_session) { +void janus_plugin_disable_bwe(janus_plugin_session *plugin_session) { janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) return; - janus_ice_handle_disable_probing(handle); + janus_ice_handle_disable_bwe(handle); } static gboolean janus_plugin_close_pc_internal(gpointer user_data) { diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index eab504cfa6..f567873564 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -2237,6 +2237,8 @@ typedef struct janus_videoroom_subscriber_stream { janus_vp8_simulcast_context vp8_context; /* SVC context */ janus_rtp_svc_context svc_context; + /* When we last dropped a layer because of BWE (to implement a cooldown period) */ + gint64 last_bwe_drop; /* Playout delays to enforce when relaying this stream, if the extension has been negotiated */ int16_t min_delay, max_delay; volatile gint ready, destroyed; @@ -8415,12 +8417,15 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t JANUS_LOG(LOG_WARN, "[videoroom] BWE=%"SCNu32"\n", estimate); janus_mutex_lock(&subscriber->streams_mutex); GList *temp = subscriber->streams; + gint64 now = janus_get_monotonic_time(); while(temp) { janus_videoroom_subscriber_stream *s = (janus_videoroom_subscriber_stream *)temp->data; if(s->type == JANUS_VIDEOROOM_MEDIA_DATA) { temp = temp->next; continue; } + if(s->last_bwe_drop > 0 && s->last_bwe_drop < now) + s->last_bwe_drop = 0; GSList *list = s->publisher_streams; if(list) { janus_videoroom_publisher_stream *ps = list->data; @@ -8436,7 +8441,7 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t } } else if(s->type == JANUS_VIDEOROOM_MEDIA_VIDEO) { if(ps->simulcast) { - JANUS_LOG(LOG_WARN, "current=%d/%d, target=%d/%d, bwe=%d/%d\n", + JANUS_LOG(LOG_DBG, "current=%d/%d, target=%d/%d, bwe=%d/%d\n", s->sim_context.substream, s->sim_context.templayer, s->sim_context.substream_target, s->sim_context.templayer_target, s->sim_context.substream_target_bwe, s->sim_context.templayer_target_bwe); @@ -8451,19 +8456,40 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t int target = 3*substream + templayer; if(target > 8) target = 8; - if(ps->bitrates->packets[target] == NULL || estimate < ps->bitrates->bitrate[target]) { + /* Check if the target is actually available */ + while(ps->bitrates->packets[target] == NULL) { + templayer--; + if(templayer < 0) { + substream--; + if(substream < 0) { + substream = 0; + } else { + templayer = 2; + } + } + target = 3*substream + templayer; + } + if(estimate < ps->bitrates->bitrate[target]) { /* Unavailable layer, or we don't have room for these layers, find one below that fits */ if(target == 0) { estimate = 0; JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast stream %d (video)\n", s->mindex); } else { gboolean found = FALSE; + int old_target = target; int new_substream = 0, new_templayer = 0; while(target > 0) { target--; new_substream = target/3; new_templayer = target%3; if(ps->bitrates->packets[target] && estimate >= ps->bitrates->bitrate[target]) { + /* If we're going up, make sure we're not in the cooldown period */ + if(s->sim_context.substream_target_bwe != -1 && + new_substream > s->sim_context.substream_target_bwe && + now < s->last_bwe_drop) { + /* We are, ignore this layer */ + continue; + } found = TRUE; break; } @@ -8474,13 +8500,20 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t } else { if(s->sim_context.substream_target_bwe == -1 || s->sim_context.substream_target_bwe > new_substream || s->sim_context.templayer_target_bwe == -1 || s->sim_context.templayer_target_bwe > new_templayer) { - JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (%"SCNu32" < %"SCNu32"), switching to %d/%d\n", - substream, templayer, s->mindex, estimate, ps->bitrates->bitrate[target], new_substream, new_templayer); - /* FIXME */ + JANUS_LOG(LOG_WARN, "Insufficient bandwidth for simulcast %d/%d of stream %d (%"SCNu32" < %"SCNu32"), switching to %d/%d (%"SCNu32")\n", + substream, templayer, s->mindex, estimate, ps->bitrates->bitrate[old_target], + new_substream, new_templayer, ps->bitrates->bitrate[target]); + /* If BWE made us go down, implement a cooldown period of about 5 seconds */ + if(s->sim_context.substream_target_bwe == -1 || + new_substream < s->sim_context.substream_target_bwe) + s->last_bwe_drop = now + 5*G_USEC_PER_SEC; } estimate -= ps->bitrates->bitrate[target]; s->sim_context.substream_target_bwe = new_substream; s->sim_context.templayer_target_bwe = new_templayer; + /* Request a PLI if needed */ + if(s->sim_context.substream_target_bwe != new_substream) + janus_videoroom_reqpli(ps, "Simulcasting substream change (BWE)"); } } } else { @@ -12452,15 +12485,24 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) json_decref(event); } if(stream->sim_context.changed_substream || stream->sim_context.changed_temporal) { - if(stream->sim_context.substream_target_bwe == -1 && stream->sim_context.substream_target_temp == -1 && - stream->sim_context.substream == stream->sim_context.substream_target && - stream->sim_context.templayer == stream->sim_context.templayer_target) { - /* We're receiving what we wanted, stop probing (if it was active) */ - gateway->disable_probing(subscriber->session->handle); - } else { - /* We're not receiving what we need, start probing (if we weren't already) */ - gateway->enable_probing(subscriber->session->handle); - gateway->defer_probing(subscriber->session->handle); + /* FIXME We changed substream, check if we need to reconfigure the bitrate target for probing */ + if(stream->sim_context.substream < stream->sim_context.substream_target || + (stream->sim_context.substream == stream->sim_context.substream_target && + stream->sim_context.templayer < stream->sim_context.templayer_target)) { + /* We're not receiving what we need, aim for a higher layer */ + uint8_t current_layer = stream->sim_context.substream*3 + stream->sim_context.templayer; + uint8_t target_layer = stream->sim_context.substream_target*3 + stream->sim_context.templayer_target; + uint32_t target = 0; + uint8_t i = 0; + JANUS_LOG(LOG_WARN, "[videoroom] current=%d, target=%d\n", current_layer, target_layer); + for(i = current_layer+1; i<=target_layer; i++) { + JANUS_LOG(LOG_WARN, "[videoroom] -- %d --> %"SCNu32"\n", i, (ps->bitrates->packets[i] ? ps->bitrates->bitrate[i] : 0)); + if(ps->bitrates->packets[i]) + target = ps->bitrates->bitrate[i]; + if(target > 0) + break; + } + gateway->set_bwe_target(subscriber->session->handle, target); } } /* If we got here, update the RTP header and send the packet */ diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 7bf54328e2..db00f51ff1 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -410,21 +410,17 @@ struct janus_callbacks { * estimated_bandwidth callback on a regular basis for this session * @param[in] handle The plugin/gateway session to enable BWE for */ void (* const enable_bwe)(janus_plugin_session *handle); - /*! \brief Get rid of the bandwidth estimation context for this session - * @param[in] handle The plugin/gateway session to disnable BWE for */ - void (* const disable_bwe)(janus_plugin_session *handle); - /*! \brief Enable bandwidth probing for this session, for bandwidth estimation purposes + /*! \brief Configure the target bitrate in this session, to generate + * probing for bandwidth estimation purposes (0 disables probing) * \note The request will be ignored if no BWE context is enabled for this session. * Also notice that probing may be paused at any time by the core, whether it * was enabled or not, e.g., in case congestion or losses are detected - * @param[in] handle The plugin/gateway session to enable BWE probing for */ - void (* const enable_probing)(janus_plugin_session *handle); - /*! \brief Disable bandwidth probing for this session - * @param[in] handle The plugin/gateway session to disable BWE probing for */ - void (* const defer_probing)(janus_plugin_session *handle); - /*! \brief Defers bandwidth probing for this session for a few seconds - * @param[in] handle The plugin/gateway session to defer BWE probing for */ - void (* const disable_probing)(janus_plugin_session *handle); + * @param[in] handle The plugin/gateway session to enable BWE probing for + * @param[in] target The bitrate to target (e.g., next simulcast layer) */ + void (* const set_bwe_target)(janus_plugin_session *handle, uint32_t bitrate); + /*! \brief Get rid of the bandwidth estimation context for this session + * @param[in] handle The plugin/gateway session to disnable BWE for */ + void (* const disable_bwe)(janus_plugin_session *handle); /*! \brief Callback to ask the core to close a WebRTC PeerConnection * \note A call to this method will result in the core invoking the hangup_media diff --git a/src/rtp.c b/src/rtp.c index 621852e400..373e522a72 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -1184,11 +1184,14 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte context->changed_substream = TRUE; context->last_relayed = now; } else if(context->substream_target_bwe != -1 && context->substream > target && substream < context->substream) { - /* We need to go down because of BWE, don't wait for a keyframe */ + /* We need to go down because of BWE, don't wait for a keyframe (hopefully one will follow) */ context->substream = substream; /* Notify the caller that the substream changed */ context->changed_substream = TRUE; context->last_relayed = now; + //~ } else if(context->substream_target_bwe != -1 && context->substream > target) { + //~ /* We need to go down because of BWE but don't have a keyframe yet, don't relay anything */ + //~ relay = FALSE; } } /* If we haven't received our desired substream yet, let's drop temporarily */ From c1d5a872383457e7436369847e59904e86ed39e2 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 9 Oct 2023 07:59:56 +0200 Subject: [PATCH 19/22] Experiment with live debugging of stats, and more probing tweaks --- src/bwe.c | 51 +++++++++++++++++++++++++++++++---- src/bwe.h | 6 +++-- src/ice.c | 14 +++++++--- src/plugins/janus_videoroom.c | 4 +-- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index f34999f5b7..35156924a4 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -16,6 +16,12 @@ #include "debug.h" #include "utils.h" +#ifdef BWE_DEBUGGING +#include +#include +#include +#endif + const char *janus_bwe_twcc_status_description(janus_bwe_twcc_status status) { switch(status) { case janus_bwe_twcc_status_notreceived: @@ -67,6 +73,28 @@ janus_bwe_context *janus_bwe_context_create(void) { char line[2048]; g_snprintf(line, sizeof(line), "time,status,estimate,probing_target,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); fwrite(line, sizeof(char), strlen(line), bwe->csv); + bwe->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if(bwe->fd == -1) { + JANUS_LOG(LOG_ERR, "Error creating socket: %s\n", g_strerror(errno)); + } else { + struct sockaddr_in address = { 0 }; + address.sin_family = AF_INET; + address.sin_port = htons(0); + address.sin_addr.s_addr = INADDR_ANY; + if(bind(bwe->fd, (struct sockaddr *)&address, sizeof(address)) < 0) { + JANUS_LOG(LOG_ERR, "Error binding socket: %s\n", g_strerror(errno)); + close(bwe->fd); + bwe->fd = -1; + } else { + address.sin_port = htons(8878); + address.sin_addr.s_addr = inet_addr("127.0.0.1"); + if(connect(bwe->fd, (struct sockaddr *)&address, sizeof(address)) < 0) { + JANUS_LOG(LOG_ERR, "Error connecting socket: %s\n", g_strerror(errno)); + close(bwe->fd); + bwe->fd = -1; + } + } + } #endif return bwe; } @@ -80,6 +108,8 @@ void janus_bwe_context_destroy(janus_bwe_context *bwe) { janus_bwe_delay_tracker_destroy(bwe->delays); #ifdef BWE_DEBUGGING fclose(bwe->csv); + if(bwe->fd > -1) + close(bwe->fd); #endif g_free(bwe); } @@ -190,18 +220,26 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->status = janus_bwe_status_lossy; bwe->status_changed = now; bwe->estimate = estimate; - } else if(avg_delay_weighted > 1.2 && avg_delay_weighted - bwe->avg_delay > 0.1) { - JANUS_LOG(LOG_WARN, "[BWE] Congested (delay=%.2f, increase=%.2f)\n", - avg_delay_weighted, avg_delay_weighted - bwe->avg_delay); + } else if(avg_delay_weighted >= 1.0 && (avg_delay_weighted - bwe->avg_delay) >= 0.05) { + JANUS_LOG(LOG_WARN, "[BWE][%"SCNi64"] Congested (delay=%.2f, increase=%.2f)\n", + now, avg_delay_weighted, avg_delay_weighted - bwe->avg_delay); /* FIXME Delay is increasing */ if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) notify_plugin = TRUE; bwe->status = janus_bwe_status_congested; bwe->status_changed = now; - //~ if(estimate > bwe->estimate) - bwe->estimate = estimate; + bwe->estimate = estimate; //~ else //~ bwe->estimate = ((double)bwe->estimate * 0.8) + ((double)estimate * 0.2); + //~ } else if(bitrate_out > bitrate_in && (bitrate_out - bitrate_in > 50000)) { + //~ /* FIXME We sent much more than what was acked, another indicator of possible congestion? */ + //~ JANUS_LOG(LOG_WARN, "[BWE][%"SCNi64"] Congested (too much diff with acked rate, %"SCNu32")\n", + //~ now, bitrate_out - bitrate_in); + //~ if(bwe->status != janus_bwe_status_lossy && bwe->status != janus_bwe_status_congested) + //~ notify_plugin = TRUE; + //~ bwe->status = janus_bwe_status_congested; + //~ bwe->status_changed = now; + //~ bwe->estimate = estimate; } else { /* FIXME All is fine? Check what state we're in */ if(bwe->status == janus_bwe_status_lossy || bwe->status == janus_bwe_status_congested) { @@ -218,6 +256,7 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->probing_sent = 0; bwe->probing_portion = 0.0; bwe->probing_buildup = 0; + bwe->probing_buildup_step = 1000; bwe->probing_buildup_timer = now; } else { /* FIXME Keep converging to the estimate */ @@ -249,6 +288,8 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { bwe->estimate, bwe->probing_target, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, avg_delay, avg_delay_weighted, avg_delay_latest); fwrite(line, sizeof(char), strlen(line), bwe->csv); + if(bwe->fd > -1) + send(bwe->fd, line, strlen(line), 0); #endif /* Reset values */ bwe->delay = 0; diff --git a/src/bwe.h b/src/bwe.h index 93b32c8444..bf02921ab5 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -146,7 +146,7 @@ typedef struct janus_bwe_context { /*! \brief Index of the m-line we're using for probing */ int probing_mindex; /*! \brief How much we should aim for with out probing (and how much to increase, plus much we sent in a second) */ - uint32_t probing_target, probing_buildup, probing_sent; + uint32_t probing_target, probing_buildup, probing_buildup_step, probing_sent; /*! \brief How many times we went through probing in a second */ uint8_t probing_count; /*! \brief Portion of probing we didn't manage to send the previous round */ @@ -182,8 +182,10 @@ typedef struct janus_bwe_context { /*! \brief When we last notified the plugin */ int64_t last_notified; #ifdef BWE_DEBUGGING - /*! \brief CSV where we save the the debugging information */ + /*! \brief CSV where we save the debugging information */ FILE *csv; + /*! \brief UDP socket where to send the debugging information */ + int fd; #endif } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context diff --git a/src/ice.c b/src/ice.c index 764ac49eda..96f298abf5 100644 --- a/src/ice.c +++ b/src/ice.c @@ -4536,6 +4536,7 @@ static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { return G_SOURCE_CONTINUE; } pc->bwe->probing_deferred = 0; + pc->bwe->probing_buildup_timer = now; } /* Get the medium instance we'll use for probing */ janus_ice_peerconnection_medium *medium = NULL; @@ -4574,9 +4575,13 @@ static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { uint32_t required = pc->bwe->probing_target - bitrate_out; if(pc->bwe->probing_buildup_timer > 0) { if(pc->bwe->probing_buildup == 0) - pc->bwe->probing_buildup = 25000; - if(now - pc->bwe->probing_buildup_timer >= 500000) { - pc->bwe->probing_buildup += 25000; + pc->bwe->probing_buildup = pc->bwe->probing_buildup_step; + uint32_t gap = (pc->bwe->probing_buildup_step >= 10000 ? 500000 : 200000); + if(now - pc->bwe->probing_buildup_timer >= gap) { + pc->bwe->probing_buildup_step += 1000; + if(pc->bwe->probing_buildup_step > 20000) + pc->bwe->probing_buildup_step = 20000; + pc->bwe->probing_buildup += pc->bwe->probing_buildup_step; pc->bwe->probing_buildup_timer = now; } if(pc->bwe->probing_buildup >= required) @@ -4735,7 +4740,8 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu pc->bwe->probing_sent = 0; pc->bwe->probing_portion = 0.0; pc->bwe->probing_buildup = 0; - pc->bwe->probing_buildup_timer = janus_get_monotonic_time(); + //~ pc->bwe->probing_buildup_timer = janus_get_monotonic_time(); + pc->bwe->probing_deferred = janus_get_monotonic_time() + G_USEC_PER_SEC; return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_bwe) { /* We need to get rif of the bandwidth estimator */ diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index f567873564..dce73a38f5 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8469,7 +8469,7 @@ void janus_videoroom_estimated_bandwidth(janus_plugin_session *handle, uint32_t } target = 3*substream + templayer; } - if(estimate < ps->bitrates->bitrate[target]) { + if(estimate < (ps->bitrates->bitrate[target] + 20000)) { /* Unavailable layer, or we don't have room for these layers, find one below that fits */ if(target == 0) { estimate = 0; @@ -12502,7 +12502,7 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) if(target > 0) break; } - gateway->set_bwe_target(subscriber->session->handle, target); + gateway->set_bwe_target(subscriber->session->handle, target ? (target + 50000) : 0); } } /* If we got here, update the RTP header and send the packet */ From 63a65a7390ba55d59a256414c790c91b94b3163e Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 9 Oct 2023 12:06:59 +0200 Subject: [PATCH 20/22] Added Admin API methods to tweak BWE debugging --- src/bwe.c | 162 ++++++++++++++++++++++++++++++++++++---------------- src/bwe.h | 22 +++++-- src/ice.c | 51 ++++++++++++++++- src/ice.h | 9 +++ src/janus.c | 89 ++++++++++++++++++++++++++++- 5 files changed, 277 insertions(+), 56 deletions(-) diff --git a/src/bwe.c b/src/bwe.c index 35156924a4..4c95c8fbd8 100644 --- a/src/bwe.c +++ b/src/bwe.c @@ -11,16 +11,15 @@ #include #include +#include +#include +#include #include "bwe.h" #include "debug.h" #include "utils.h" +#include "ip-utils.h" -#ifdef BWE_DEBUGGING -#include -#include -#include -#endif const char *janus_bwe_twcc_status_description(janus_bwe_twcc_status status) { switch(status) { @@ -66,36 +65,7 @@ janus_bwe_context *janus_bwe_context_create(void) { bwe->acked = janus_bwe_stream_bitrate_create(); bwe->delays = janus_bwe_delay_tracker_create(0); bwe->probing_mindex = -1; -#ifdef BWE_DEBUGGING - char filename[256]; - g_snprintf(filename, sizeof(filename), "/tmp/bwe-janus-%"SCNi64, janus_get_real_time()); - bwe->csv = fopen(filename, "wt"); - char line[2048]; - g_snprintf(line, sizeof(line), "time,status,estimate,probing_target,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); - fwrite(line, sizeof(char), strlen(line), bwe->csv); - bwe->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if(bwe->fd == -1) { - JANUS_LOG(LOG_ERR, "Error creating socket: %s\n", g_strerror(errno)); - } else { - struct sockaddr_in address = { 0 }; - address.sin_family = AF_INET; - address.sin_port = htons(0); - address.sin_addr.s_addr = INADDR_ANY; - if(bind(bwe->fd, (struct sockaddr *)&address, sizeof(address)) < 0) { - JANUS_LOG(LOG_ERR, "Error binding socket: %s\n", g_strerror(errno)); - close(bwe->fd); - bwe->fd = -1; - } else { - address.sin_port = htons(8878); - address.sin_addr.s_addr = inet_addr("127.0.0.1"); - if(connect(bwe->fd, (struct sockaddr *)&address, sizeof(address)) < 0) { - JANUS_LOG(LOG_ERR, "Error connecting socket: %s\n", g_strerror(errno)); - close(bwe->fd); - bwe->fd = -1; - } - } - } -#endif + bwe->fd = -1; return bwe; } @@ -106,11 +76,10 @@ void janus_bwe_context_destroy(janus_bwe_context *bwe) { janus_bwe_stream_bitrate_destroy(bwe->sent); janus_bwe_stream_bitrate_destroy(bwe->acked); janus_bwe_delay_tracker_destroy(bwe->delays); -#ifdef BWE_DEBUGGING - fclose(bwe->csv); + if(bwe->csv != NULL) + fclose(bwe->csv); if(bwe->fd > -1) close(bwe->fd); -#endif g_free(bwe); } } @@ -280,17 +249,20 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { now, janus_bwe_status_description(bwe->status), bitrate_out / 1024, probing_out / 1024, bitrate_in / 1024, probing_in / 1024, bwe->loss_ratio, bwe->avg_delay, bwe->estimate); -#ifdef BWE_DEBUGGING - /* Save the details to CSV */ - char line[2048]; - g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f,%.2f\n", - now - bwe->started, bwe->status, - bwe->estimate, bwe->probing_target, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, - bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, avg_delay, avg_delay_weighted, avg_delay_latest); - fwrite(line, sizeof(char), strlen(line), bwe->csv); - if(bwe->fd > -1) - send(bwe->fd, line, strlen(line), 0); -#endif + + /* Save the details to CSV and/or send them externally via UDP, if enabled */ + if(bwe->csv != NULL || bwe->fd > -1) { + char line[2048]; + g_snprintf(line, sizeof(line), "%"SCNi64",%d,%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu32",%"SCNu16",%"SCNu16",%.2f,%.2f,%.2f,%.2f\n", + now - bwe->started, bwe->status, + bwe->estimate, bwe->probing_target, bitrate_out, rtx_out, probing_out, bitrate_in, rtx_in, probing_in, + bwe->received_pkts, bwe->lost_pkts, bwe->loss_ratio, avg_delay, avg_delay_weighted, avg_delay_latest); + if(bwe->csv != NULL) + fwrite(line, sizeof(char), strlen(line), bwe->csv); + if(bwe->fd > -1) + send(bwe->fd, line, strlen(line), 0); + } + /* Reset values */ bwe->delay = 0; bwe->received_pkts = 0; @@ -302,6 +274,98 @@ void janus_bwe_context_update(janus_bwe_context *bwe) { } } +gboolean janus_bwe_save_csv(janus_bwe_context *bwe, const char *path) { + if(bwe == NULL || path == NULL || bwe->csv != NULL) + return FALSE; + /* Open the CSV file */ + bwe->csv = fopen(path, "wt"); + if(bwe->csv == NULL) { + JANUS_LOG(LOG_ERR, "Couldn't open CSV file for BWE stats: %s\n", g_strerror(errno)); + return FALSE; + } + /* Write a header line with the names of the fields we'll save */ + char line[2048]; + g_snprintf(line, sizeof(line), "time,status,estimate,probing_target,bitrate_out,rtx_out,probing_out,bitrate_in,rtx_in,probing_in,acked,lost,loss_ratio,avg_delay,avg_delay_weighted,avg_delay_fb\n"); + fwrite(line, sizeof(char), strlen(line), bwe->csv); + fflush(bwe->csv); + /* Done */ + return TRUE; +} + +void janus_bwe_close_csv(janus_bwe_context *bwe) { + if(bwe == NULL || bwe->csv == NULL) + return; + fclose(bwe->csv); + bwe->csv = NULL; +} + +gboolean janus_bwe_save_live(janus_bwe_context *bwe, const char *host, uint16_t port) { + if(bwe == NULL || host == NULL || port == 0 || bwe->fd > -1) + return FALSE; + /* Check if we need to resolve this host address */ + struct addrinfo *res = NULL, *start = NULL; + janus_network_address addr; + janus_network_address_string_buffer addr_buf; + const char *resolved_host = NULL; + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + if(getaddrinfo(host, NULL, NULL, &res) == 0) { + start = res; + while(res != NULL) { + if(janus_network_address_from_sockaddr(res->ai_addr, &addr) == 0 && + janus_network_address_to_string_buffer(&addr, &addr_buf) == 0) { + /* Resolved */ + resolved_host = janus_network_address_string_from_buffer(&addr_buf); + freeaddrinfo(start); + start = NULL; + break; + } + res = res->ai_next; + } + } + if(resolved_host == NULL) { + if(start) + freeaddrinfo(start); + JANUS_LOG(LOG_ERR, "Could not resolve address (%s) for BWE stats...\n", host); + return FALSE; + } + host = resolved_host; + /* Create the socket */ + bwe->fd = socket(addr.family, SOCK_DGRAM, IPPROTO_UDP); + if(bwe->fd == -1) { + JANUS_LOG(LOG_ERR, "Error creating socket for BWE stats: %s\n", g_strerror(errno)); + return FALSE; + } + struct sockaddr_in serv_addr = { 0 }; + struct sockaddr_in6 serv_addr6 = { 0 }; + if(addr.family == AF_INET6) { + serv_addr6.sin6_family = AF_INET6; + inet_pton(AF_INET6, host, &serv_addr6.sin6_addr); + serv_addr6.sin6_port = htons(port); + } else { + serv_addr.sin_family = AF_INET; + inet_pton(AF_INET, host, &serv_addr.sin_addr); + serv_addr.sin_port = htons(port); + } + /* Connect the socket to the provided address */ + struct sockaddr *address = (addr.family == AF_INET ? (struct sockaddr *)&serv_addr : (struct sockaddr *)&serv_addr6); + size_t addrlen = (addr.family == AF_INET ? sizeof(serv_addr) : sizeof(serv_addr6)); + if(connect(bwe->fd, (struct sockaddr *)address, addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Error connecting socket for BWE stats: %s\n", g_strerror(errno)); + close(bwe->fd); + bwe->fd = -1; + } + /* Done */ + return TRUE; +} + +void janus_bwe_close_live(janus_bwe_context *bwe) { + if(bwe == NULL || bwe->fd == -1) + return; + close(bwe->fd); + bwe->fd = -1; +} + janus_bwe_stream_bitrate *janus_bwe_stream_bitrate_create(void) { janus_bwe_stream_bitrate *bwe_sb = g_malloc0(sizeof(janus_bwe_stream_bitrate)); janus_mutex_init(&bwe_sb->mutex); diff --git a/src/bwe.h b/src/bwe.h index bf02921ab5..e77fa7f74e 100644 --- a/src/bwe.h +++ b/src/bwe.h @@ -16,8 +16,6 @@ #include "mutex.h" -#define BWE_DEBUGGING - /*! \brief Tracker for a stream bitrate (whether it's simulcast/SVC or not) */ typedef struct janus_bwe_stream_bitrate { /*! \brief Time based queue of packet sizes */ @@ -181,12 +179,10 @@ typedef struct janus_bwe_context { gboolean notify_plugin; /*! \brief When we last notified the plugin */ int64_t last_notified; -#ifdef BWE_DEBUGGING /*! \brief CSV where we save the debugging information */ FILE *csv; /*! \brief UDP socket where to send the debugging information */ int fd; -#endif } janus_bwe_context; /*! \brief Helper to create a new bandwidth estimation context * @returns a new janus_bwe_context instance, if successful, or NULL otherwise */ @@ -216,4 +212,22 @@ void janus_bwe_context_handle_feedback(janus_bwe_context *bwe, * @param[in] bwe The janus_bwe_context instance to update */ void janus_bwe_context_update(janus_bwe_context *bwe); +/*! \brief Helper method to start saving the stats related to the BWE processing to a CSV file + * @param[in] bwe The janus_bwe_context instance to save + * @param[in] path Path where to save the file to + * @returns TRUE, if successful, or FALSE otherwise */ +gboolean janus_bwe_save_csv(janus_bwe_context *bwe, const char *path); +/*! \brief Helper method to stop saving the stats related to the BWE processing to a CSV file + * @param[in] bwe The janus_bwe_context instance to update */ +void janus_bwe_close_csv(janus_bwe_context *bwe); +/*! \brief Helper method to relay stats related to the BWE processing to an external UDP address + * @param[in] bwe The janus_bwe_context instance to save + * @param[in] host The address to send stats to + * @param[in] port The port to send stats to + * @returns TRUE, if successful, or FALSE otherwise */ +gboolean janus_bwe_save_live(janus_bwe_context *bwe, const char *host, uint16_t port); +/*! \brief Helper method to stop relaying the stats related to the BWE processing + * @param[in] bwe The janus_bwe_context instance to update */ +void janus_bwe_close_live(janus_bwe_context *bwe); + #endif diff --git a/src/ice.c b/src/ice.c index 96f298abf5..a506556313 100644 --- a/src/ice.c +++ b/src/ice.c @@ -475,6 +475,7 @@ static janus_ice_queued_packet janus_ice_media_stopped, janus_ice_enable_bwe, janus_ice_set_bwe_target, + janus_ice_debug_bwe, janus_ice_disable_bwe, janus_ice_hangup_peerconnection, janus_ice_detach_handle, @@ -654,7 +655,8 @@ static void janus_ice_free_queued_packet(janus_ice_queued_packet *pkt) { if(pkt == NULL || pkt == &janus_ice_start_gathering || pkt == &janus_ice_add_candidates || pkt == &janus_ice_dtls_handshake || - pkt == &janus_ice_enable_bwe || pkt == &janus_ice_set_bwe_target || pkt == &janus_ice_disable_bwe || + pkt == &janus_ice_enable_bwe || pkt == &janus_ice_set_bwe_target || + pkt == &janus_ice_debug_bwe || pkt == &janus_ice_disable_bwe || pkt == &janus_ice_media_stopped || pkt == &janus_ice_hangup_peerconnection || pkt == &janus_ice_detach_handle || @@ -1617,6 +1619,8 @@ static void janus_ice_handle_free(const janus_refcount *handle_ref) { } g_free(handle->opaque_id); g_free(handle->token); + g_free(handle->bwe_csv); + g_free(handle->bwe_host); g_free(handle); } @@ -3904,6 +3908,18 @@ void janus_ice_handle_set_bwe_target(janus_ice_handle *handle, uint32_t bitrate) } } +void janus_ice_handle_debug_bwe(janus_ice_handle *handle) { + JANUS_LOG(LOG_WARN, "[%"SCNu64"] Tweaking debugging of bandwidth estimation\n", handle->handle_id); + if(handle->queued_packets != NULL) { +#if GLIB_CHECK_VERSION(2, 46, 0) + g_async_queue_push_front(handle->queued_packets, &janus_ice_debug_bwe); +#else + g_async_queue_push(handle->queued_packets, &janus_ice_debug_bwe); +#endif + g_main_context_wakeup(handle->mainctx); + } +} + void janus_ice_handle_disable_bwe(janus_ice_handle *handle) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); if(handle->queued_packets != NULL) { @@ -4714,6 +4730,15 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu pc->bwe = janus_bwe_context_create(); janus_mutex_lock(&handle->mutex); handle->bwe_target = 0; + /* Check if we need debugging */ + if(handle->bwe_csv) { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling CSV debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_save_csv(pc->bwe, handle->bwe_csv); + } + if(handle->bwe_host && handle->bwe_port > 0) { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling live debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_save_live(pc->bwe, handle->bwe_host, handle->bwe_port); + } janus_mutex_unlock(&handle->mutex); /* Let's create a source for BWE, for plugin notifications and probing */ handle->bwe_source = g_timeout_source_new(50); /* FIXME */ @@ -4743,8 +4768,30 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu //~ pc->bwe->probing_buildup_timer = janus_get_monotonic_time(); pc->bwe->probing_deferred = janus_get_monotonic_time() + G_USEC_PER_SEC; return G_SOURCE_CONTINUE; + } else if(pkt == &janus_ice_debug_bwe) { + /* Enable or disable debugging for the bandwidth estimator */ + if(pc == NULL || pc->bwe == NULL) + return G_SOURCE_CONTINUE; + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Tweaking debugging for bandwidth estimation\n", handle->handle_id); + janus_mutex_lock(&handle->mutex); + if(handle->bwe_csv) { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling CSV debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_save_csv(pc->bwe, handle->bwe_csv); + } else { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling CSV debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_close_csv(pc->bwe); + } + if(handle->bwe_host && handle->bwe_port > 0) { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Enabling live debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_save_live(pc->bwe, handle->bwe_host, handle->bwe_port); + } else { + JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling live debugging of bandwidth estimation\n", handle->handle_id); + janus_bwe_close_live(pc->bwe); + } + janus_mutex_unlock(&handle->mutex); + return G_SOURCE_CONTINUE; } else if(pkt == &janus_ice_disable_bwe) { - /* We need to get rif of the bandwidth estimator */ + /* We need to get rid of the bandwidth estimator */ if(pc == NULL || pc->bwe == NULL) return G_SOURCE_CONTINUE; JANUS_LOG(LOG_INFO, "[%"SCNu64"] Disabling bandwidth estimation\n", handle->handle_id); diff --git a/src/ice.h b/src/ice.h index 41414c59f7..a696de5721 100644 --- a/src/ice.h +++ b/src/ice.h @@ -411,6 +411,12 @@ struct janus_ice_handle { gint last_event_stats; /*! \brief Bandwidth estimation target, if any, as asked by the plugin */ uint32_t bwe_target; + /*! \brief In case offline BWE debugging is enabled, the CSV file to save to */ + char *bwe_csv; + /*! \brief In case live BWE debugging is enabled, the host to send stats to */ + char *bwe_host; + /*! \brief In case live BWE debugging is enabled, the port to send stats to */ + uint16_t bwe_port; /*! \brief Flag to decide whether or not packets need to be dumped to a text2pcap file */ volatile gint dump_packets; /*! \brief In case this session must be saved to text2pcap, the instance to dump packets to */ @@ -796,6 +802,9 @@ void janus_ice_handle_enable_bwe(janus_ice_handle *handle); * @param[in] handle The Janus ICE handle this method refers to * @param[in] bitrate The bitrate to target (will be used to generate probing) */ void janus_ice_handle_set_bwe_target(janus_ice_handle *handle, uint32_t bitrate); +/*! \brief Method to dynamically tweak the bandwidth estimation debugging for a handle + * @param[in] handle The Janus ICE handle this method refers to */ +void janus_ice_handle_debug_bwe(janus_ice_handle *handle); /*! \brief Method to dynamically disable bandwidth estimation for a handle * @param[in] handle The Janus ICE handle this method refers to */ void janus_ice_handle_disable_bwe(janus_ice_handle *handle); diff --git a/src/janus.c b/src/janus.c index ca4351ac53..a5f02ca8d9 100644 --- a/src/janus.c +++ b/src/janus.c @@ -189,6 +189,13 @@ static struct janus_json_parameter text2pcap_parameters[] = { {"filename", JSON_STRING, 0}, {"truncate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE} }; +static struct janus_json_parameter debug_bwe_parameters[] = { + {"csv", JANUS_JSON_BOOL, 0}, + {"path", JSON_STRING, 0}, + {"live", JANUS_JSON_BOOL, 0}, + {"host", JSON_STRING, 0}, + {"port", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE} +}; static struct janus_json_parameter handleinfo_parameters[] = { {"plugin_only", JANUS_JSON_BOOL, 0} }; @@ -3002,8 +3009,88 @@ int janus_process_incoming_admin_request(janus_request *request) { /* Send the success reply */ ret = janus_process_success(request, reply); goto jsondone; + } else if(!strcasecmp(message_text, "debug_bwe")) { + /* Enable or disable BWE debugging (to CSV and/or external UDP address) */ + JANUS_VALIDATE_JSON_OBJECT(root, debug_bwe_parameters, + error_code, error_cause, FALSE, + JANUS_ERROR_MISSING_MANDATORY_ELEMENT, JANUS_ERROR_INVALID_ELEMENT_TYPE); + json_t *csv = json_object_get(root, "csv"); + json_t *live = json_object_get(root, "live"); + gboolean changes = FALSE; + janus_mutex_lock(&handle->mutex); + if(json_is_true(csv)) { + /* Enable CSV offline debugging */ + if(handle->bwe_csv) { + janus_mutex_unlock(&handle->mutex); + ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_UNKNOWN, + "CSV debugging already configured"); + goto jsondone; + } + json_t *path = json_object_get(root, "path"); + const char *path_value = json_string_value(path); + if(path_value == NULL) { + janus_mutex_unlock(&handle->mutex); + ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_INVALID_ELEMENT_TYPE, + "Missing or invalid element (path)"); + goto jsondone; + } + changes = TRUE; + handle->bwe_csv = g_strdup(path_value); + } else { + /* Disable CSV offline debugging */ + if(handle->bwe_csv) + changes = TRUE; + g_free(handle->bwe_csv); + handle->bwe_csv = NULL; + } + if(json_is_true(live)) { + /* Enable live debugging */ + if(handle->bwe_host) { + janus_mutex_unlock(&handle->mutex); + ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_UNKNOWN, + "Live debugging already configured"); + goto jsondone; + } + json_t *host = json_object_get(root, "host"); + json_t *port = json_object_get(root, "port"); + const char *host_value = json_string_value(host); + uint16_t port_value = json_integer_value(port); + if(host_value == NULL) { + janus_mutex_unlock(&handle->mutex); + ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_INVALID_ELEMENT_TYPE, + "Missing or invalid element (host)"); + goto jsondone; + } + if(port_value == 0) { + janus_mutex_unlock(&handle->mutex); + ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_INVALID_ELEMENT_TYPE, + "Missing or invalid element (port)"); + goto jsondone; + } + changes = TRUE; + handle->bwe_host = g_strdup(host_value); + handle->bwe_port = port_value; + } else { + /* Disable live debugging */ + if(handle->bwe_host) + changes = TRUE; + g_free(handle->bwe_host); + handle->bwe_host = NULL; + handle->bwe_port = 0; + } + janus_mutex_unlock(&handle->mutex); + /* Apply the changes, if needed */ + if(changes) + janus_ice_handle_debug_bwe(handle); + /* Prepare JSON reply */ + json_t *reply = json_object(); + json_object_set_new(reply, "janus", json_string("success")); + json_object_set_new(reply, "transaction", json_string(transaction_text)); + /* Send the success reply */ + ret = janus_process_success(request, reply); + goto jsondone; } - /* If this is not a request to start/stop debugging to text2pcap, it must be a handle_info */ + /* If we git here, it must be a handle_info */ if(strcasecmp(message_text, "handle_info")) { ret = janus_process_error(request, session_id, transaction_text, JANUS_ERROR_INVALID_REQUEST_PATH, "Unhandled request '%s' at this path", message_text); goto jsondone; From 11449cc9aaafb2ae5e9d127e789156d91a9133f8 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Fri, 13 Oct 2023 13:05:28 +0200 Subject: [PATCH 21/22] Add missing objects for fuzzing targets --- fuzzers/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/config.sh b/fuzzers/config.sh index a362261956..a9d7841779 100755 --- a/fuzzers/config.sh +++ b/fuzzers/config.sh @@ -23,7 +23,7 @@ COVERAGE_LDFLAGS="-O1 -fno-omit-frame-pointer -g -ggdb3 -fprofile-instr-generate JANUS_CONF_FLAGS="--disable-docs --disable-post-processing --disable-turn-rest-api --disable-all-transports --disable-all-plugins --disable-all-handlers --disable-data-channels" # Janus objects needed for fuzzing -JANUS_OBJECTS="janus-log.o janus-utils.o janus-rtcp.o janus-rtp.o janus-sdp-utils.o" +JANUS_OBJECTS="janus-log.o janus-utils.o janus-rtcp.o janus-rtp.o janus-sdp-utils.o janus-bwe.o janus-ip-utils.o" # CFLAGS for fuzzer dependencies DEPS_CFLAGS="$(pkg-config --cflags glib-2.0)" From 8c16c7acf8fdb0983c65baf76b717105d433de01 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 21 Nov 2023 15:52:37 +0100 Subject: [PATCH 22/22] Reduced buildup step --- src/ice.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ice.c b/src/ice.c index a506556313..c76d3c8ff0 100644 --- a/src/ice.c +++ b/src/ice.c @@ -4595,8 +4595,8 @@ static gboolean janus_ice_outgoing_bwe_handle(gpointer user_data) { uint32_t gap = (pc->bwe->probing_buildup_step >= 10000 ? 500000 : 200000); if(now - pc->bwe->probing_buildup_timer >= gap) { pc->bwe->probing_buildup_step += 1000; - if(pc->bwe->probing_buildup_step > 20000) - pc->bwe->probing_buildup_step = 20000; + if(pc->bwe->probing_buildup_step > 10000) + pc->bwe->probing_buildup_step = 10000; pc->bwe->probing_buildup += pc->bwe->probing_buildup_step; pc->bwe->probing_buildup_timer = now; }