google_rtc_audio_processing: Correct floating point representation

Lionel corrected me: there was an off-by-one error in the math here.
The desired analysis space for S{16,32}_LE formats maps -1.0 to the
minimum (2's complement signed) integer value, and 1.0 to "one more
than the maximum integer value" (so -1.0 is exactly representable, but
1.0 is not).

This requires a clamping step be added to the code, but that's
probably a good idea anyway given that the floating point output data
(we're not longer using the wrapper inside the google library) may not
itself be fully saturated.

Signed-off-by: Andy Ross <[email protected]>

src/audio/google/google_rtc_audio_processing.c

@@ -97,33 +97,44 @@ struct google_rtc_audio_processing_comp_data {
 
 #if CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_MIC_BITS == 16
 typedef int16_t mic_sample_t;
-#define MIC_SCALE ((float)SHRT_MAX)
+#define MIC_SCALE ((float)-SHRT_MIN)
 #else
 typedef int32_t mic_sample_t;
-#define MIC_SCALE ((float)INT_MAX)
+#define MIC_SCALE ((float)-INT_MIN)
 #endif
 
 #if CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_REF_BITS == 16
 typedef int16_t ref_sample_t;
-#define REF_SCALE ((float)SHRT_MAX)
+#define REF_SCALE ((float)-SHRT_MIN)
 #else
 typedef int32_t ref_sample_t;
-#define REF_SCALE ((float)INT_MAX)
+#define REF_SCALE ((float)-INT_MIN)
 #endif
 
+static inline float clamp(float x, float scale)
+{
+	float min = -1.0f;
+	float max = 1.0f - 1.0f / scale;
+
+	return x < min ? min : (x > max ? max : x);
+}
+
 static inline float mic_to_float(mic_sample_t x)
 {
-	return (1.0f / MIC_SCALE) * (float)x;
+	return (1.0f / MIC_SCALE) * x;
 }
 
 static inline mic_sample_t float_to_mic(float x)
 {
-	return (mic_sample_t)(MIC_SCALE * x);
+	float min = -1.0f;
+	float max = 1.0f - 1.0f / MIC_SCALE;
+
+	return (mic_sample_t)(MIC_SCALE * (x < min ? min : (x > max ? max : x)));
 }
 
 static inline float ref_to_float(ref_sample_t x)
 {
-	return (1.0f / REF_SCALE) * (float)x;
+	return (1.0f / REF_SCALE) * x;
 }
 
 void *GoogleRtcMalloc(size_t size)