svcenc: RC parameters verified before RC init

RC uses int32_t to store bitrates and other parameters internally. For specific magnitudes of bitrate, framerate, and GOP period, this can result in signed integer overflow. This is now detected before calls to RC init. Note that calls to the 'ISVCE_CMD_CTL_SET_BITRATE' API can also result in this behaviour but it will be appropriately handled by 'isvce_svc_frame_params_validate'. Bug = ossfuzz:63053 Test: svc_enc_fuzzer
ittiam-systems · Oct 9, 2023 · b5a8615 · b5a8615
1 parent 0ab5c42
commit b5a8615
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 5 deletions.
diff --git a/encoder/svc/isvce_encode.c b/encoder/svc/isvce_encode.c
@@ -243,14 +243,17 @@ WORD32 isvce_encode(iv_obj_t *ps_codec_obj, void *pv_api_ip, void *pv_api_op)
     /* initialize codec ctxt with default params for the first encode api call */
     if(ps_codec->i4_encode_api_call_cnt == 0)
     {
-        isvce_codec_init(ps_codec);
+        error_status = isvce_codec_init(ps_codec);
+
+        SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR,
+                            ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
     }
 
     error_status =
         isvce_svc_frame_params_validate(ps_codec->s_rate_control.apps_rate_control_api,
                                         ps_codec->s_cfg.s_svc_params.u1_num_spatial_layers);
-    SET_ERROR_ON_RETURN(error_status, IVE_UNSUPPORTEDPARAM,
-                        ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
+    SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR, ps_video_encode_op->s_ive_op.u4_error_code,
+                        IV_FAIL);
 
     /* parse configuration params */
     for(i = 0; i < MAX_ACTIVE_CONFIG_PARAMS; i++)

diff --git a/encoder/svc/isvce_rate_control.c b/encoder/svc/isvce_rate_control.c
@@ -159,11 +159,9 @@ void isvce_rc_init(void *pv_rc_api, void *pv_frame_time, void *pv_time_stamp, vo
                    UWORD8 *pu1_init_qp, WORD32 i4_max_inter_frm_int, UWORD8 *pu1_min_max_qp,
                    UWORD8 u1_profile_level)
 {
-    //    UWORD8  u1_is_mb_level_rc_on = 0;
     UWORD32 au4_peak_bit_rate[2] = {0, 0};
     UWORD32 u4_min_bit_rate = 0;
     WORD32 i4_is_gop_closed = 1;
-    //    WORD32  i4_use_est_intra_sad = 1;
     UWORD32 u4_src_ticks = 0;
     UWORD32 u4_tgt_ticks = 0;
     UWORD8 u1_level_idx = ih264e_get_lvl_idx(u1_profile_level);

diff --git a/encoder/svc/isvce_utils.c b/encoder/svc/isvce_utils.c
@@ -515,6 +515,36 @@ WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t
     return u4_error_code;
 }
 
+/**
+*******************************************************************************
+*
+* @brief
+*  Validates SVC RC params
+*
+* @param[in] ps_cfg
+*  Cfg parameters
+*
+* @returns  error code in conformance with 'IH264E_ERROR_T'
+*
+*******************************************************************************
+*/
+WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg)
+{
+    WORD32 i;
+
+    /* RC requires total bits in a second to fit int32_t */
+    for(i = 0; i < ps_cfg->s_svc_params.u1_num_spatial_layers; i++)
+    {
+        if((((((UWORD64) ps_cfg->au4_target_bitrate[i]) * 1000llu) / ps_cfg->u4_tgt_frame_rate) *
+            ps_cfg->u4_idr_frm_interval) > ((UWORD64) INT32_MAX))
+        {
+            return IH264E_BITRATE_NOT_SUPPORTED;
+        }
+    }
+
+    return IH264E_SUCCESS;
+}
+
 /**
 *******************************************************************************
 *
@@ -3545,6 +3575,15 @@ IH264E_ERROR_T isvce_codec_init(isvce_codec_t *ps_codec)
         ps_codec->i4_air_pic_cnt = -1;
     }
 
+    {
+        WORD32 i4_err_code = isvce_svc_rc_params_validate(&ps_codec->s_cfg);
+
+        if(IH264E_SUCCESS != i4_err_code)
+        {
+            return i4_err_code;
+        }
+    }
+
     /****************************************************/
     /*           INITIALIZE RATE CONTROL                */
     /****************************************************/

diff --git a/encoder/svc/isvce_utils.h b/encoder/svc/isvce_utils.h
@@ -180,6 +180,8 @@ extern WORD32 isvce_svc_au_props_validate(svc_inp_params_t *ps_svc_inp_params, U
 
 extern WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t *ps_cfg);
 
+extern WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg);
+
 extern WORD32 isvce_svc_frame_params_validate(
     rate_control_api_t *aps_rate_control_api[MAX_NUM_SPATIAL_LAYERS], UWORD8 u1_num_spatial_layers);