From b5a861581ec1e47f11abb39ce3f9b49bb4c29082 Mon Sep 17 00:00:00 2001
From: Ashwin Natesan <ashwin.natesan@ittiam.com>
Date: Mon, 9 Oct 2023 12:09:51 +0530
Subject: [PATCH] svcenc: RC parameters verified before RC init

RC uses int32_t to store bitrates and other
parameters internally. For specific magnitudes of
bitrate, framerate, and GOP period, this can
result in signed integer overflow. This is now
detected before calls to RC init.

Note that calls to the 'ISVCE_CMD_CTL_SET_BITRATE' API can also
result in this behaviour but it will be appropriately handled
by 'isvce_svc_frame_params_validate'.

Bug = ossfuzz:63053
Test: svc_enc_fuzzer
---
 encoder/svc/isvce_encode.c       |  9 +++++---
 encoder/svc/isvce_rate_control.c |  2 --
 encoder/svc/isvce_utils.c        | 39 ++++++++++++++++++++++++++++++++
 encoder/svc/isvce_utils.h        |  2 ++
 4 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/encoder/svc/isvce_encode.c b/encoder/svc/isvce_encode.c
index d3f61980..8c6aa114 100644
--- a/encoder/svc/isvce_encode.c
+++ b/encoder/svc/isvce_encode.c
@@ -243,14 +243,17 @@ WORD32 isvce_encode(iv_obj_t *ps_codec_obj, void *pv_api_ip, void *pv_api_op)
     /* initialize codec ctxt with default params for the first encode api call */
     if(ps_codec->i4_encode_api_call_cnt == 0)
     {
-        isvce_codec_init(ps_codec);
+        error_status = isvce_codec_init(ps_codec);
+
+        SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR,
+                            ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
     }
 
     error_status =
         isvce_svc_frame_params_validate(ps_codec->s_rate_control.apps_rate_control_api,
                                         ps_codec->s_cfg.s_svc_params.u1_num_spatial_layers);
-    SET_ERROR_ON_RETURN(error_status, IVE_UNSUPPORTEDPARAM,
-                        ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
+    SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR, ps_video_encode_op->s_ive_op.u4_error_code,
+                        IV_FAIL);
 
     /* parse configuration params */
     for(i = 0; i < MAX_ACTIVE_CONFIG_PARAMS; i++)
diff --git a/encoder/svc/isvce_rate_control.c b/encoder/svc/isvce_rate_control.c
index cc3c41f6..e4a85bb3 100644
--- a/encoder/svc/isvce_rate_control.c
+++ b/encoder/svc/isvce_rate_control.c
@@ -159,11 +159,9 @@ void isvce_rc_init(void *pv_rc_api, void *pv_frame_time, void *pv_time_stamp, vo
                    UWORD8 *pu1_init_qp, WORD32 i4_max_inter_frm_int, UWORD8 *pu1_min_max_qp,
                    UWORD8 u1_profile_level)
 {
-    //    UWORD8  u1_is_mb_level_rc_on = 0;
     UWORD32 au4_peak_bit_rate[2] = {0, 0};
     UWORD32 u4_min_bit_rate = 0;
     WORD32 i4_is_gop_closed = 1;
-    //    WORD32  i4_use_est_intra_sad = 1;
     UWORD32 u4_src_ticks = 0;
     UWORD32 u4_tgt_ticks = 0;
     UWORD8 u1_level_idx = ih264e_get_lvl_idx(u1_profile_level);
diff --git a/encoder/svc/isvce_utils.c b/encoder/svc/isvce_utils.c
index 23fe5697..485a52e3 100644
--- a/encoder/svc/isvce_utils.c
+++ b/encoder/svc/isvce_utils.c
@@ -515,6 +515,36 @@ WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t
     return u4_error_code;
 }
 
+/**
+*******************************************************************************
+*
+* @brief
+*  Validates SVC RC params
+*
+* @param[in] ps_cfg
+*  Cfg parameters
+*
+* @returns  error code in conformance with 'IH264E_ERROR_T'
+*
+*******************************************************************************
+*/
+WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg)
+{
+    WORD32 i;
+
+    /* RC requires total bits in a second to fit int32_t */
+    for(i = 0; i < ps_cfg->s_svc_params.u1_num_spatial_layers; i++)
+    {
+        if((((((UWORD64) ps_cfg->au4_target_bitrate[i]) * 1000llu) / ps_cfg->u4_tgt_frame_rate) *
+            ps_cfg->u4_idr_frm_interval) > ((UWORD64) INT32_MAX))
+        {
+            return IH264E_BITRATE_NOT_SUPPORTED;
+        }
+    }
+
+    return IH264E_SUCCESS;
+}
+
 /**
 *******************************************************************************
 *
@@ -3545,6 +3575,15 @@ IH264E_ERROR_T isvce_codec_init(isvce_codec_t *ps_codec)
         ps_codec->i4_air_pic_cnt = -1;
     }
 
+    {
+        WORD32 i4_err_code = isvce_svc_rc_params_validate(&ps_codec->s_cfg);
+
+        if(IH264E_SUCCESS != i4_err_code)
+        {
+            return i4_err_code;
+        }
+    }
+
     /****************************************************/
     /*           INITIALIZE RATE CONTROL                */
     /****************************************************/
diff --git a/encoder/svc/isvce_utils.h b/encoder/svc/isvce_utils.h
index ad144469..497809e5 100644
--- a/encoder/svc/isvce_utils.h
+++ b/encoder/svc/isvce_utils.h
@@ -180,6 +180,8 @@ extern WORD32 isvce_svc_au_props_validate(svc_inp_params_t *ps_svc_inp_params, U
 
 extern WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t *ps_cfg);
 
+extern WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg);
+
 extern WORD32 isvce_svc_frame_params_validate(
     rate_control_api_t *aps_rate_control_api[MAX_NUM_SPATIAL_LAYERS], UWORD8 u1_num_spatial_layers);