Fix quantizer range in codec_avm (AOMediaCodec#2502)

It was done on rc_max/max_quantizer but not on AOME_SET_QP.
libsdl-org · Nov 8, 2024 · 5766fd1 · 5766fd1
1 parent 1cdeff7
commit 5766fd1
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 12 deletions.
diff --git a/src/codec_avm.c b/src/codec_avm.c
@@ -8,6 +8,7 @@
 #include "aom/aomcx.h"
 #include "aom/aomdx.h"
 
+#include <assert.h>
 #include <limits.h>
 #include <stdlib.h>
 #include <string.h>
@@ -520,9 +521,19 @@ static avifBool avifFindAOMScalingMode(const avifFraction * avifMode, AOM_SCALIN
     return AVIF_FALSE;
 }
 
-// Scales from aom's [0:63] to avm's [0:255]. TODO(yguyon): Accept [0:255] directly in avifEncoder.
-static int avmScaleQuantizer(int quantizer)
+// Scales from aom's [0:63] to avm's [M:255], where M=0/-48/-96 for 8/10/12 bit.
+// See --min-qp help in
+// https://gitlab.com/AOMediaCodec/avm/-/blob/main/apps/aomenc.c
+// TODO(yguyon): Accept [M:255] directly in avifEncoder.
+static int avmScaleQuantizer(int quantizer, uint32_t depth)
 {
+    if (depth == 10) {
+        return AVIF_CLAMP((quantizer * (255 + 48) + 31) / 63 - 48, -48, 255);
+    }
+    if (depth == 12) {
+        return AVIF_CLAMP((quantizer * (255 + 96) + 31) / 63 - 96, -96, 255);
+    }
+    assert(depth == 8);
     return AVIF_CLAMP((quantizer * 255 + 31) / 63, 0, 255);
 }
 
@@ -665,8 +676,9 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
             maxQuantizer = encoder->maxQuantizer;
         }
         // Scale from aom's [0:63] to avm's [0:255]. TODO(yguyon): Accept [0:255] directly in avifEncoder.
-        minQuantizer = avmScaleQuantizer(minQuantizer);
-        maxQuantizer = avmScaleQuantizer(maxQuantizer);
+        quantizer = avmScaleQuantizer(quantizer, image->depth);
+        minQuantizer = avmScaleQuantizer(minQuantizer, image->depth);
+        maxQuantizer = avmScaleQuantizer(maxQuantizer, image->depth);
         if ((cfg->rc_end_usage == AOM_VBR) || (cfg->rc_end_usage == AOM_CBR)) {
             // cq-level is ignored in these two end-usage modes, so adjust minQuantizer and
             // maxQuantizer to the target quantizer.
@@ -757,16 +769,17 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
             // We are not ready for dimension change for now.
             return AVIF_RESULT_NOT_IMPLEMENTED;
         }
+        quantizer = avmScaleQuantizer(quantizer, image->depth);
         if (alpha) {
             if (encoderChanges & (AVIF_ENCODER_CHANGE_MIN_QUANTIZER_ALPHA | AVIF_ENCODER_CHANGE_MAX_QUANTIZER_ALPHA)) {
-                cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizerAlpha);
-                cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizerAlpha);
+                cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizerAlpha, image->depth);
+                cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizerAlpha, image->depth);
                 quantizerUpdated = AVIF_TRUE;
             }
         } else {
             if (encoderChanges & (AVIF_ENCODER_CHANGE_MIN_QUANTIZER | AVIF_ENCODER_CHANGE_MAX_QUANTIZER)) {
-                cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizer);
-                cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizer);
+                cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizer, image->depth);
+                cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizer, image->depth);
                 quantizerUpdated = AVIF_TRUE;
             }
         }
@@ -788,8 +801,8 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
                         minQuantizer = encoder->minQuantizer;
                         maxQuantizer = encoder->maxQuantizer;
                     }
-                    minQuantizer = avmScaleQuantizer(minQuantizer);
-                    maxQuantizer = avmScaleQuantizer(maxQuantizer);
+                    minQuantizer = avmScaleQuantizer(minQuantizer, image->depth);
+                    maxQuantizer = avmScaleQuantizer(maxQuantizer, image->depth);
                     cfg->rc_min_quantizer = AVIF_MAX(quantizer - 4, minQuantizer);
                     cfg->rc_max_quantizer = AVIF_MIN(quantizer + 4, maxQuantizer);
                 }

diff --git a/tests/gtest/avifavmtest.cc b/tests/gtest/avifavmtest.cc
@@ -44,7 +44,7 @@ TEST_P(AvmTest, EncodeDecode) {
             AVIF_RESULT_OK);
 
   // Verify that the input and decoded images are close.
-  EXPECT_GT(testutil::GetPsnr(*image, *decoded), 40.0);
+  EXPECT_GT(testutil::GetPsnr(*image, *decoded), 39.0);
 
   // Forcing an AV1 decoding codec should fail.
   for (avifCodecChoice av1_codec :
@@ -71,10 +71,19 @@ INSTANTIATE_TEST_SUITE_P(Basic, AvmTest,
 
 INSTANTIATE_TEST_SUITE_P(Tiny, AvmTest,
                          Combine(/*width=*/Values(1), /*height=*/Values(1),
-                                 /*depth=*/Values(8),
+                                 /*depth=*/Values(8, 10, 12),
                                  Values(AVIF_PIXEL_FORMAT_YUV444),
                                  /*alpha=*/Values(false)));
 
+INSTANTIATE_TEST_SUITE_P(HighBitDepthAndEvenDimensions, AvmTest,
+                         Combine(/*width=*/Values(2), /*height=*/Values(34),
+                                 /*depth=*/Values(10, 12),
+                                 Values(AVIF_PIXEL_FORMAT_YUV400,
+                                        AVIF_PIXEL_FORMAT_YUV420,
+                                        AVIF_PIXEL_FORMAT_YUV444),
+                                 /*alpha=*/Values(true)));
+
+// Triggers "Fatal: Unsupported image conversion" in aom/src/aom_image.c.
 // TODO(yguyon): Implement or fix in avm then test the following combinations.
 INSTANTIATE_TEST_SUITE_P(DISABLED_Broken, AvmTest,
                          Combine(/*width=*/Values(1), /*height=*/Values(34),