From 5766fd13c3a6e177754005381cf8460fb0601b04 Mon Sep 17 00:00:00 2001 From: Yannis Guyon Date: Fri, 8 Nov 2024 10:32:01 +0000 Subject: [PATCH] Fix quantizer range in codec_avm (#2502) It was done on rc_max/max_quantizer but not on AOME_SET_QP. --- src/codec_avm.c | 33 +++++++++++++++++++++++---------- tests/gtest/avifavmtest.cc | 13 +++++++++++-- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/codec_avm.c b/src/codec_avm.c index c7a29a0bf2..698b062cca 100644 --- a/src/codec_avm.c +++ b/src/codec_avm.c @@ -8,6 +8,7 @@ #include "aom/aomcx.h" #include "aom/aomdx.h" +#include #include #include #include @@ -520,9 +521,19 @@ static avifBool avifFindAOMScalingMode(const avifFraction * avifMode, AOM_SCALIN return AVIF_FALSE; } -// Scales from aom's [0:63] to avm's [0:255]. TODO(yguyon): Accept [0:255] directly in avifEncoder. -static int avmScaleQuantizer(int quantizer) +// Scales from aom's [0:63] to avm's [M:255], where M=0/-48/-96 for 8/10/12 bit. +// See --min-qp help in +// https://gitlab.com/AOMediaCodec/avm/-/blob/main/apps/aomenc.c +// TODO(yguyon): Accept [M:255] directly in avifEncoder. +static int avmScaleQuantizer(int quantizer, uint32_t depth) { + if (depth == 10) { + return AVIF_CLAMP((quantizer * (255 + 48) + 31) / 63 - 48, -48, 255); + } + if (depth == 12) { + return AVIF_CLAMP((quantizer * (255 + 96) + 31) / 63 - 96, -96, 255); + } + assert(depth == 8); return AVIF_CLAMP((quantizer * 255 + 31) / 63, 0, 255); } @@ -665,8 +676,9 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, maxQuantizer = encoder->maxQuantizer; } // Scale from aom's [0:63] to avm's [0:255]. TODO(yguyon): Accept [0:255] directly in avifEncoder. - minQuantizer = avmScaleQuantizer(minQuantizer); - maxQuantizer = avmScaleQuantizer(maxQuantizer); + quantizer = avmScaleQuantizer(quantizer, image->depth); + minQuantizer = avmScaleQuantizer(minQuantizer, image->depth); + maxQuantizer = avmScaleQuantizer(maxQuantizer, image->depth); if ((cfg->rc_end_usage == AOM_VBR) || (cfg->rc_end_usage == AOM_CBR)) { // cq-level is ignored in these two end-usage modes, so adjust minQuantizer and // maxQuantizer to the target quantizer. @@ -757,16 +769,17 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, // We are not ready for dimension change for now. return AVIF_RESULT_NOT_IMPLEMENTED; } + quantizer = avmScaleQuantizer(quantizer, image->depth); if (alpha) { if (encoderChanges & (AVIF_ENCODER_CHANGE_MIN_QUANTIZER_ALPHA | AVIF_ENCODER_CHANGE_MAX_QUANTIZER_ALPHA)) { - cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizerAlpha); - cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizerAlpha); + cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizerAlpha, image->depth); + cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizerAlpha, image->depth); quantizerUpdated = AVIF_TRUE; } } else { if (encoderChanges & (AVIF_ENCODER_CHANGE_MIN_QUANTIZER | AVIF_ENCODER_CHANGE_MAX_QUANTIZER)) { - cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizer); - cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizer); + cfg->rc_min_quantizer = avmScaleQuantizer(encoder->minQuantizer, image->depth); + cfg->rc_max_quantizer = avmScaleQuantizer(encoder->maxQuantizer, image->depth); quantizerUpdated = AVIF_TRUE; } } @@ -788,8 +801,8 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, minQuantizer = encoder->minQuantizer; maxQuantizer = encoder->maxQuantizer; } - minQuantizer = avmScaleQuantizer(minQuantizer); - maxQuantizer = avmScaleQuantizer(maxQuantizer); + minQuantizer = avmScaleQuantizer(minQuantizer, image->depth); + maxQuantizer = avmScaleQuantizer(maxQuantizer, image->depth); cfg->rc_min_quantizer = AVIF_MAX(quantizer - 4, minQuantizer); cfg->rc_max_quantizer = AVIF_MIN(quantizer + 4, maxQuantizer); } diff --git a/tests/gtest/avifavmtest.cc b/tests/gtest/avifavmtest.cc index 6597e33d1f..7d9d78c14a 100644 --- a/tests/gtest/avifavmtest.cc +++ b/tests/gtest/avifavmtest.cc @@ -44,7 +44,7 @@ TEST_P(AvmTest, EncodeDecode) { AVIF_RESULT_OK); // Verify that the input and decoded images are close. - EXPECT_GT(testutil::GetPsnr(*image, *decoded), 40.0); + EXPECT_GT(testutil::GetPsnr(*image, *decoded), 39.0); // Forcing an AV1 decoding codec should fail. for (avifCodecChoice av1_codec : @@ -71,10 +71,19 @@ INSTANTIATE_TEST_SUITE_P(Basic, AvmTest, INSTANTIATE_TEST_SUITE_P(Tiny, AvmTest, Combine(/*width=*/Values(1), /*height=*/Values(1), - /*depth=*/Values(8), + /*depth=*/Values(8, 10, 12), Values(AVIF_PIXEL_FORMAT_YUV444), /*alpha=*/Values(false))); +INSTANTIATE_TEST_SUITE_P(HighBitDepthAndEvenDimensions, AvmTest, + Combine(/*width=*/Values(2), /*height=*/Values(34), + /*depth=*/Values(10, 12), + Values(AVIF_PIXEL_FORMAT_YUV400, + AVIF_PIXEL_FORMAT_YUV420, + AVIF_PIXEL_FORMAT_YUV444), + /*alpha=*/Values(true))); + +// Triggers "Fatal: Unsupported image conversion" in aom/src/aom_image.c. // TODO(yguyon): Implement or fix in avm then test the following combinations. INSTANTIATE_TEST_SUITE_P(DISABLED_Broken, AvmTest, Combine(/*width=*/Values(1), /*height=*/Values(34),