From 8b8c1539913462fa8dbb71d40f5fc10baaeea9b1 Mon Sep 17 00:00:00 2001 From: Andy Ross Date: Tue, 12 Dec 2023 09:54:12 -0800 Subject: [PATCH] google_rtc_audio_processing: Correct floating point representation Lionel corrected me: there was an off-by-one error in the math here. The desired analysis space for S{16,32}_LE formats maps -1.0 to the minimum (2's complement signed) integer value, and 1.0 to "one more than the maximum integer value" (so -1.0 is exactly representable, but 1.0 is not). This requires a clamping step be added to the code, but that's probably a good idea anyway given that the floating point output data (we're not longer using the wrapper inside the google library) may not itself be fully saturated. Signed-off-by: Andy Ross --- .../google/google_rtc_audio_processing.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/audio/google/google_rtc_audio_processing.c b/src/audio/google/google_rtc_audio_processing.c index ff3792cb160b..2c41c2714370 100644 --- a/src/audio/google/google_rtc_audio_processing.c +++ b/src/audio/google/google_rtc_audio_processing.c @@ -97,33 +97,44 @@ struct google_rtc_audio_processing_comp_data { #if CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_MIC_BITS == 16 typedef int16_t mic_sample_t; -#define MIC_SCALE ((float)SHRT_MAX) +#define MIC_SCALE ((float)-SHRT_MIN) #else typedef int32_t mic_sample_t; -#define MIC_SCALE ((float)INT_MAX) +#define MIC_SCALE ((float)-INT_MIN) #endif #if CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_REF_BITS == 16 typedef int16_t ref_sample_t; -#define REF_SCALE ((float)SHRT_MAX) +#define REF_SCALE ((float)-SHRT_MIN) #else typedef int32_t ref_sample_t; -#define REF_SCALE ((float)INT_MAX) +#define REF_SCALE ((float)-INT_MIN) #endif +static inline float clamp(float x, float scale) +{ + float min = -1.0f; + float max = 1.0f - 1.0f / scale; + + return x < min ? min : (x > max ? max : x); +} + static inline float mic_to_float(mic_sample_t x) { - return (1.0f / MIC_SCALE) * (float)x; + return (1.0f / MIC_SCALE) * x; } static inline mic_sample_t float_to_mic(float x) { - return (mic_sample_t)(MIC_SCALE * x); + float min = -1.0f; + float max = 1.0f - 1.0f / MIC_SCALE; + + return (mic_sample_t)(MIC_SCALE * (x < min ? min : (x > max ? max : x))); } static inline float ref_to_float(ref_sample_t x) { - return (1.0f / REF_SCALE) * (float)x; + return (1.0f / REF_SCALE) * x; } void *GoogleRtcMalloc(size_t size)