From 62a8b55b4bf0e9d110f609a8a682b7e48d39af98 Mon Sep 17 00:00:00 2001 From: Yannis Guyon Date: Tue, 15 Nov 2022 16:53:19 +0100 Subject: [PATCH] Experimental AVIF Sample Transform bit depth ext Allow encoding and decoding images of more than 12 bits, the limit of the AV1 format. This commit experiments 16-bit AVIF images, made of one primary lossless 8-bit image and one hidden lossy/lossless bit depth extension 8-bit image. The two are combined thanks to a Sample Transform derived image item. Grids and alpha are supported. Warning: This feature is experimental and not covered by the current AVIF specification. --- .github/workflows/ci-unix-static.yml | 1 + .github/workflows/ci-windows.yml | 4 +- CHANGELOG.md | 2 + CMakeLists.txt | 8 + include/avif/avif.h | 53 +++ include/avif/internal.h | 61 ++++ src/avif.c | 4 + src/read.c | 520 ++++++++++++++++++++++++--- src/sampletransform.c | 358 ++++++++++++++++++ src/write.c | 460 ++++++++++++++++++++++-- tests/CMakeLists.txt | 7 + tests/gtest/avif16bittest.cc | 137 +++++++ 12 files changed, 1517 insertions(+), 98 deletions(-) create mode 100644 src/sampletransform.c create mode 100644 tests/gtest/avif16bittest.cc diff --git a/.github/workflows/ci-unix-static.yml b/.github/workflows/ci-unix-static.yml index 1ab32f0a1a..529e6b3055 100644 --- a/.github/workflows/ci-unix-static.yml +++ b/.github/workflows/ci-unix-static.yml @@ -109,6 +109,7 @@ jobs: -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON + -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON -DAVIF_ENABLE_WERROR=ON - name: Build libavif (ninja) working-directory: ./build diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index a220aba6d6..eed77eb1e3 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -125,7 +125,9 @@ jobs: -DAVIF_BUILD_TESTS=ON -DAVIF_ENABLE_GTEST=ON -DAVIF_LOCAL_GTEST=ON -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON - -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON -DAVIF_ENABLE_WERROR=ON + -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON + -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON + -DAVIF_ENABLE_WERROR=ON - name: Build libavif (ninja) working-directory: ./build run: ninja diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c0c4c9bc..9a53a46908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,8 @@ The changes are relative to the previous release, unless the baseline is specifi * Require libyuv by default (but it can still be disabled with -DAVIF_LIBYUV=OFF). * Add avifdec --icc flag to override the output color profile. +* Add experimental API for reading and writing 16-bit AVIF files behind the + compilation flag AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM. ### Changed since 1.0.0 * Update aom.cmd: v3.8.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 30644e8a6b..67fe674cc0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ option(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP "Enable experimental gain map code (for HDR images that look good both on HDR and SDR displays)" OFF ) option(AVIF_ENABLE_EXPERIMENTAL_AVIR "Enable experimental reduced header" OFF) +option(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM "Enable experimental sample transform code" OFF) set(AVIF_PKG_CONFIG_EXTRA_LIBS_PRIVATE "") set(AVIF_PKG_CONFIG_EXTRA_REQUIRES_PRIVATE "") @@ -328,6 +329,10 @@ if(AVIF_ENABLE_EXPERIMENTAL_AVIR) add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_AVIR) endif() +if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +endif() + set(AVIF_SRCS src/alpha.c src/avif.c @@ -351,6 +356,9 @@ set(AVIF_SRCS if(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) list(APPEND AVIF_SRCS src/gainmap.c) endif() +if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + list(APPEND AVIF_SRCS src/sampletransform.c) +endif() if(AVIF_ENABLE_COMPLIANCE_WARDEN) if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/ext/ComplianceWarden") diff --git a/include/avif/avif.h b/include/avif/avif.h index 63d7077474..e0f7df1055 100644 --- a/include/avif/avif.h +++ b/include/avif/avif.h @@ -192,6 +192,10 @@ typedef enum AVIF_NODISCARD avifResult AVIF_RESULT_DECODE_GAIN_MAP_FAILED = 31, AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE = 32, #endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED = 33, + AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED = 34, +#endif // Kept for backward compatibility; please use the symbols above instead. AVIF_RESULT_NO_AV1_ITEMS_FOUND = AVIF_RESULT_MISSING_IMAGE_ITEM @@ -706,6 +710,43 @@ AVIF_NODISCARD AVIF_API avifBool avifGainMapMetadataFractionsToDouble(avifGainMa #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +// --------------------------------------------------------------------------- + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +// Sample Transforms are a HIGHLY EXPERIMENTAL FEATURE. The format might still +// change and images containing a sample transform item encoded with the current +// version of libavif might not decode with a future version of libavif. +// Use are your own risk. +// This is based on a proposal from the Alliance for Open Media. + +typedef enum avifSampleTransformRecipe +{ + AVIF_SAMPLE_TRANSFORM_NONE, + // Encode the 8 most significant bits of each input image sample losslessly + // into one base image. The remaining least 8 significant bits are encoded + // in a separate hidden image item. The two are combined at decoding into + // one image with the same bit depth as the original image. + // It is backward compatible in the sense that only the base image may be + // decoded (ignoring the hidden image item), leading to a valid image but + // with precision loss (16-bit samples truncated to the 8 most significant + // bits). + // Same as AVIF_SAMPLE_TRANSFORM_NONE on input images with a depth of at + // most 12 bits. Only available for 16-bit input images. + AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B, + // Encode the 12 most significant bits of each input image sample losslessly + // into one base image. The remaining least 4 significant bits are encoded + // in a separate hidden image item. The two are combined at decoding into + // one image with the same bit depth as the original image. + // It is backward compatible in the sense that only the base image may be + // decoded (ignoring the hidden image item), leading to a valid image but + // with precision loss (16-bit samples truncated to the 12 most significant + // bits). + // Same as AVIF_SAMPLE_TRANSFORM_NONE on input images with a depth of at + // most 12 bits. Only available for 16-bit input images. + AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B +} avifSampleTransformRecipe; +#endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP + // --------------------------------------------------------------------------- // avifImage @@ -1274,6 +1315,14 @@ typedef struct avifDecoder // Can be useful to decode the gain map image only. avifBool ignoreColorAndAlpha; #endif + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + // Input + // Optional sample transforms used as bit depth extensions will be ignored if the base image has + // a bit depth of at least preferredBitDepth. Otherwise, the output image will be the + // combination of the base image and the sample transform item, if any. Default is 0. + uint32_t preferredBitDepth; // TODO(yguyon): Implement +#endif } avifDecoder; // Returns NULL in case of memory allocation failure. @@ -1458,6 +1507,10 @@ typedef struct avifEncoder #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) int qualityGainMap; // changeable encoder setting #endif + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + avifSampleTransformRecipe sampleTransformRecipe; // See the avifSampleTransformRecipe description. +#endif } avifEncoder; // avifEncoderCreate() returns NULL if a memory allocation failed. diff --git a/include/avif/internal.h b/include/avif/internal.h index 3cf3aa016b..6b53bfab82 100644 --- a/include/avif/internal.h +++ b/include/avif/internal.h @@ -156,6 +156,62 @@ void avifImageCopyNoAlloc(avifImage * dstImage, const avifImage * srcImage); // Ignores the gainMap field (which exists only if AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP is defined). void avifImageCopySamples(avifImage * dstImage, const avifImage * srcImage, avifPlanesFlags planes); +// --------------------------------------------------------------------------- + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +// Mapping used in the coding of Sample Transform metadata. +typedef enum avifSampleTransformIntermediateBitDepth +{ + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_8 = 0, // Signed 8-bit. + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_16 = 1, // Signed 16-bit. + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32 = 2, // Signed 32-bit. + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_64 = 3 // Signed 64-bit. +} avifSampleTransformIntermediateBitDepth; + +// Meaning of an operation in Sample Transform metadata. +// For a constant_operator, L is the constant (left operand) and R is the sample (right operand). +// For an operator, L is the intermediate overall result (left operand) +// and R is the last sub-expression result (right operand). +typedef enum avifSampleTransformOperation +{ + AVIF_SAMPLE_TRANSFORM_SUM = 0, // S = L + R + AVIF_SAMPLE_TRANSFORM_PRODUCT = 1, // S = L * R + AVIF_SAMPLE_TRANSFORM_DIVIDE = 2, // S = floor(L / R) + AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED = 3, // S = floor(R / L) + AVIF_SAMPLE_TRANSFORM_POW = 4, // S = pow(L, R) + AVIF_SAMPLE_TRANSFORM_POW_REVERSED = 5, // S = pow(R, L) + AVIF_SAMPLE_TRANSFORM_LOG = 6, // S = log(L, R) + AVIF_SAMPLE_TRANSFORM_LOG_REVERSED = 7, // S = log(R, L) + AVIF_SAMPLE_TRANSFORM_AND = 8, // S = L & R + AVIF_SAMPLE_TRANSFORM_OR = 9, // S = L | R + AVIF_SAMPLE_TRANSFORM_XOR = 10, // S = L ^ R + AVIF_SAMPLE_TRANSFORM_NOR = 11, // S = !(L | R) + AVIF_SAMPLE_TRANSFORM_MIN = 12, // S = min(L, R) + AVIF_SAMPLE_TRANSFORM_MAX = 13 // S = max(L, R) +} avifSampleTransformOperation; + +// Performs the following for each sample in the selected planes of the result image: +// result = leftOperand (operation) rightOperand +// result, leftOperand and rightOperand must be allocated and have the same planes and dimensions. +// result can be point to leftOperand and/or rightOperand (in-place allowed). +avifResult avifImageTransformImageAndImageSamples(avifImage * result, + avifSampleTransformIntermediateBitDepth intermediateBitDepth, + const avifImage * leftOperand, + avifSampleTransformOperation operation, + const avifImage * rightOperand, + avifPlanesFlags planes); +// Same but leftOperand is a constant instead of a sample. +avifResult avifImageTransformConstantAndImageSamples(avifImage * result, + avifSampleTransformIntermediateBitDepth intermediateBitDepth, + int32_t leftOperand, + avifSampleTransformOperation operation, + const avifImage * rightOperand, + avifPlanesFlags planes); +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + +// --------------------------------------------------------------------------- +// Alpha + typedef struct avifAlphaParams { uint32_t width; @@ -317,6 +373,11 @@ typedef enum avifItemCategory AVIF_ITEM_ALPHA, #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) AVIF_ITEM_GAIN_MAP, +#endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + AVIF_ITEM_SAMPLE_TRANSFORM, // Sample Transform derived image item 'sato'. + AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR, // Second input image item of AVIF_ITEM_SAMPLE_TRANSFORM. First is AVIF_ITEM_COLOR. + AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA, // Auxiliary image item of AVIF_ITEM_SAMPLE_TRANSFORM_COLOR. #endif AVIF_ITEM_CATEGORY_COUNT } avifItemCategory; diff --git a/src/avif.c b/src/avif.c index 264d004572..d218456848 100644 --- a/src/avif.c +++ b/src/avif.c @@ -108,6 +108,10 @@ const char * avifResultToString(avifResult result) case AVIF_RESULT_ENCODE_GAIN_MAP_FAILED: return "Encoding of gain map planes failed"; case AVIF_RESULT_DECODE_GAIN_MAP_FAILED: return "Decoding of gain map planes failed"; case AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE: return "Invalid tone mapped image item"; +#endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + case AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED: return "Encoding of sample transformed image failed"; + case AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED: return "Decoding of sample transformed image failed"; #endif case AVIF_RESULT_UNKNOWN_ERROR: default: diff --git a/src/read.c b/src/read.c index e1a652bc9d..91d34b64a0 100644 --- a/src/read.c +++ b/src/read.c @@ -68,6 +68,19 @@ static const char * avifGetConfigurationPropertyName(avifCodecType codecType) } } +static avifBool avifIsAlpha(avifItemCategory itemCategory) +{ + if (itemCategory == AVIF_ITEM_ALPHA) { + return AVIF_TRUE; + } +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) { + return AVIF_TRUE; + } +#endif + return AVIF_FALSE; +} + // --------------------------------------------------------------------------- // Box data structures @@ -863,6 +876,11 @@ typedef struct avifDecoderData // The colour information property takes precedence over any colour information // in the image bitstream, i.e. if the property is present, colour information in // the bitstream shall be ignored. + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + // Interpreted from Sample Transform metadata if present, otherwise AVIF_SAMPLE_TRANSFORM_NONE. + avifSampleTransformRecipe sampleTransformRecipe; +#endif } avifDecoderData; static void avifDecoderDataDestroy(avifDecoderData * data); @@ -1241,6 +1259,57 @@ static avifResult avifDecoderItemValidateProperties(const avifDecoderItem * item } } } + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (!memcmp(item->type, "sato", 4)) { + for (uint32_t i = 0; i < item->meta->items.count; ++i) { + avifDecoderItem * satoInput = item->meta->items.item[i]; + if (satoInput->dimgForID != item->id) { + continue; + } + + if (avifGetCodecType(satoInput->type) == AVIF_CODEC_TYPE_UNKNOWN) { + avifDiagnosticsPrintf(diag, + "Item ID %u of unsupported type '%.4s' is used as input to a Sample Transform item", + satoInput->id, + (const char *)satoInput->type); + return AVIF_RESULT_BMFF_PARSE_FAILED; + } + + // The following is not part of the HEIF specification proposal but should be added to MIAF: + // All input images of a Sample Transform image item shall use the same chroma sampling format + // and the same decoder configuration, except for the bit depth which may differ. + // (similar to the constraints on grid items) + + // The chroma sampling format is part of the decoder configuration. + const avifProperty * satoInputConfigProp = avifPropertyArrayFind(&satoInput->properties, configPropName); + if (!satoInputConfigProp) { + avifDiagnosticsPrintf(diag, + "Sample Transform input item ID %u of type '%.4s' is missing mandatory %s property", + satoInput->id, + (const char *)satoInput->type, + configPropName); + return AVIF_RESULT_BMFF_PARSE_FAILED; + } + // configProp was copied from the first input item to the derived item. Comparing satoInputConfigProp with it + // is equivalent to comparing satoInputConfigProp with the configPropName from the first input item. + if ((satoInputConfigProp->u.av1C.seqProfile != configProp->u.av1C.seqProfile) || + (satoInputConfigProp->u.av1C.seqLevelIdx0 != configProp->u.av1C.seqLevelIdx0) || + (satoInputConfigProp->u.av1C.seqTier0 != configProp->u.av1C.seqTier0) || + (satoInputConfigProp->u.av1C.monochrome != configProp->u.av1C.monochrome) || + (satoInputConfigProp->u.av1C.chromaSubsamplingX != configProp->u.av1C.chromaSubsamplingX) || + (satoInputConfigProp->u.av1C.chromaSubsamplingY != configProp->u.av1C.chromaSubsamplingY) || + (satoInputConfigProp->u.av1C.chromaSamplePosition != configProp->u.av1C.chromaSamplePosition)) { + avifDiagnosticsPrintf(diag, + "The fields of the %s property of Sample Transform input item ID %u of type '%.4s' differs from other input items", + configPropName, + satoInput->id, + (const char *)satoInput->type); + return AVIF_RESULT_BMFF_PARSE_FAILED; + } + } + } +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM return AVIF_RESULT_OK; } @@ -1500,41 +1569,67 @@ static avifResult avifDecoderGenerateImageGridTiles(avifDecoder * decoder, avifI return AVIF_RESULT_OK; } -// Allocates the dstImage based on the grid image requirements. Also verifies some spec compliance rules for grids. -static avifResult avifDecoderDataAllocateGridImagePlanes(avifDecoderData * data, const avifTileInfo * info, avifImage * dstImage) +// Allocates the dstImage. Also verifies some spec compliance rules for grids, if relevant. +static avifResult avifDecoderDataAllocateImagePlanes(avifDecoderData * data, const avifTileInfo * info, avifImage * dstImage) { - const avifImageGrid * grid = &info->grid; const avifTile * tile = &data->tiles.tile[info->firstTileIndex]; - - // Validate grid image size and tile size. - // - // HEIF (ISO/IEC 23008-12:2017), Section 6.6.2.3.1: - // The tiled input images shall completely "cover" the reconstructed image grid canvas, ... - if (((tile->image->width * grid->columns) < grid->outputWidth) || ((tile->image->height * grid->rows) < grid->outputHeight)) { - avifDiagnosticsPrintf(data->diag, - "Grid image tiles do not completely cover the image (HEIF (ISO/IEC 23008-12:2017), Section 6.6.2.3.1)"); - return AVIF_RESULT_INVALID_IMAGE_GRID; - } - // Tiles in the rightmost column and bottommost row must overlap the reconstructed image grid canvas. See MIAF (ISO/IEC 23000-22:2019), Section 7.3.11.4.2, Figure 2. - if (((tile->image->width * (grid->columns - 1)) >= grid->outputWidth) || - ((tile->image->height * (grid->rows - 1)) >= grid->outputHeight)) { - avifDiagnosticsPrintf(data->diag, - "Grid image tiles in the rightmost column and bottommost row do not overlap the reconstructed image grid canvas. See MIAF (ISO/IEC 23000-22:2019), Section 7.3.11.4.2, Figure 2"); - return AVIF_RESULT_INVALID_IMAGE_GRID; + uint32_t dstWidth; + uint32_t dstHeight; + + if (info->grid.columns > 0 && info->grid.rows > 0) { + const avifImageGrid * grid = &info->grid; + // Validate grid image size and tile size. + // + // HEIF (ISO/IEC 23008-12:2017), Section 6.6.2.3.1: + // The tiled input images shall completely "cover" the reconstructed image grid canvas, ... + if (((tile->image->width * grid->columns) < grid->outputWidth) || ((tile->image->height * grid->rows) < grid->outputHeight)) { + avifDiagnosticsPrintf(data->diag, + "Grid image tiles do not completely cover the image (HEIF (ISO/IEC 23008-12:2017), Section 6.6.2.3.1)"); + return AVIF_RESULT_INVALID_IMAGE_GRID; + } + // Tiles in the rightmost column and bottommost row must overlap the reconstructed image grid canvas. See MIAF (ISO/IEC 23000-22:2019), Section 7.3.11.4.2, Figure 2. + if (((tile->image->width * (grid->columns - 1)) >= grid->outputWidth) || + ((tile->image->height * (grid->rows - 1)) >= grid->outputHeight)) { + avifDiagnosticsPrintf(data->diag, + "Grid image tiles in the rightmost column and bottommost row do not overlap the reconstructed image grid canvas. See MIAF (ISO/IEC 23000-22:2019), Section 7.3.11.4.2, Figure 2"); + return AVIF_RESULT_INVALID_IMAGE_GRID; + } + if (!avifAreGridDimensionsValid(tile->image->yuvFormat, + grid->outputWidth, + grid->outputHeight, + tile->image->width, + tile->image->height, + data->diag)) { + return AVIF_RESULT_INVALID_IMAGE_GRID; + } + dstWidth = grid->outputWidth; + dstHeight = grid->outputHeight; + } else { + // Only one tile. Width and height are inherited from the 'ispe' property of the corresponding avifDecoderItem. + dstWidth = data->tiles.tile[data->tileInfos[AVIF_ITEM_COLOR].firstTileIndex].width; + dstHeight = data->tiles.tile[data->tileInfos[AVIF_ITEM_COLOR].firstTileIndex].height; } - avifBool alpha = (tile->input->itemCategory == AVIF_ITEM_ALPHA); + const avifBool alpha = avifIsAlpha(tile->input->itemCategory); if (alpha) { // An alpha tile does not contain any YUV pixels. AVIF_ASSERT_OR_RETURN(tile->image->yuvFormat == AVIF_PIXEL_FORMAT_NONE); } - if (!avifAreGridDimensionsValid(tile->image->yuvFormat, grid->outputWidth, grid->outputHeight, tile->image->width, tile->image->height, data->diag)) { - return AVIF_RESULT_INVALID_IMAGE_GRID; + + uint32_t dstDepth = tile->image->depth; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B) { + dstDepth = 16; // 8+8 + } else if (data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B) { + dstDepth = 16; // 12+4 + } else { + assert(data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_NONE); } +#endif // Lazily populate dstImage with the new frame's properties. - const avifBool dimsOrDepthIsDifferent = (dstImage->width != grid->outputWidth) || (dstImage->height != grid->outputHeight) || - (dstImage->depth != tile->image->depth); + const avifBool dimsOrDepthIsDifferent = (dstImage->width != dstWidth) || (dstImage->height != dstHeight) || + (dstImage->depth != dstDepth); const avifBool yuvFormatIsDifferent = !alpha && (dstImage->yuvFormat != tile->image->yuvFormat); if (dimsOrDepthIsDifferent || yuvFormatIsDifferent) { if (alpha) { @@ -1545,9 +1640,9 @@ static avifResult avifDecoderDataAllocateGridImagePlanes(avifDecoderData * data, if (dimsOrDepthIsDifferent) { avifImageFreePlanes(dstImage, AVIF_PLANES_ALL); - dstImage->width = grid->outputWidth; - dstImage->height = grid->outputHeight; - dstImage->depth = tile->image->depth; + dstImage->width = dstWidth; + dstImage->height = dstHeight; + dstImage->depth = dstDepth; } if (yuvFormatIsDifferent) { avifImageFreePlanes(dstImage, AVIF_PLANES_YUV); @@ -1571,15 +1666,14 @@ static avifResult avifDecoderDataAllocateGridImagePlanes(avifDecoderData * data, return AVIF_RESULT_OK; } -// After verifying that the relevant properties of the tile match those of the first tile, copies over the pixels from the tile -// into dstImage. +// Copies over the pixels from the tile into dstImage. +// Verifies that the relevant properties of the tile match those of the first tile in case of a grid. static avifResult avifDecoderDataCopyTileToImage(avifDecoderData * data, const avifTileInfo * info, avifImage * dstImage, const avifTile * tile, unsigned int tileIndex) { - const avifImageGrid * grid = &info->grid; const avifTile * firstTile = &data->tiles.tile[info->firstTileIndex]; if (tile != firstTile) { // Check for tile consistency. All tiles in a grid image should match the first tile in the properties checked below. @@ -1593,20 +1687,22 @@ static avifResult avifDecoderDataCopyTileToImage(avifDecoderData * data, } } - unsigned int rowIndex = tileIndex / info->grid.columns; - unsigned int colIndex = tileIndex % info->grid.columns; avifImage srcView; avifImageSetDefaults(&srcView); avifImage dstView; avifImageSetDefaults(&dstView); - avifCropRect dstViewRect = { - firstTile->image->width * colIndex, firstTile->image->height * rowIndex, firstTile->image->width, firstTile->image->height - }; - if (dstViewRect.x + dstViewRect.width > grid->outputWidth) { - dstViewRect.width = grid->outputWidth - dstViewRect.x; - } - if (dstViewRect.y + dstViewRect.height > grid->outputHeight) { - dstViewRect.height = grid->outputHeight - dstViewRect.y; + avifCropRect dstViewRect = { 0, 0, firstTile->image->width, firstTile->image->height }; + if (info->grid.columns > 0 && info->grid.rows > 0) { + unsigned int rowIndex = tileIndex / info->grid.columns; + unsigned int colIndex = tileIndex % info->grid.columns; + dstViewRect.x = firstTile->image->width * colIndex; + dstViewRect.y = firstTile->image->height * rowIndex; + if (dstViewRect.x + dstViewRect.width > info->grid.outputWidth) { + dstViewRect.width = info->grid.outputWidth - dstViewRect.x; + } + if (dstViewRect.y + dstViewRect.height > info->grid.outputHeight) { + dstViewRect.height = info->grid.outputHeight - dstViewRect.y; + } } const avifCropRect srcViewRect = { 0, 0, dstViewRect.width, dstViewRect.height }; avifImage * dst = dstImage; @@ -1618,8 +1714,84 @@ static avifResult avifDecoderDataCopyTileToImage(avifDecoderData * data, #endif AVIF_ASSERT_OR_RETURN(avifImageSetViewRect(&dstView, dst, &dstViewRect) == AVIF_RESULT_OK && avifImageSetViewRect(&srcView, tile->image, &srcViewRect) == AVIF_RESULT_OK); - avifImageCopySamples(&dstView, &srcView, (tile->input->itemCategory == AVIF_ITEM_ALPHA) ? AVIF_PLANES_A : AVIF_PLANES_YUV); + const avifPlanesFlag planes = avifIsAlpha(tile->input->itemCategory) ? AVIF_PLANES_A : AVIF_PLANES_YUV; + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B) { + assert(dstView.depth == 16); + if (tile->input->itemCategory == AVIF_ITEM_COLOR || tile->input->itemCategory == AVIF_ITEM_ALPHA) { + AVIF_CHECKERR(srcView.depth == 8, AVIF_RESULT_UNKNOWN_ERROR); // TODO(yguyon): Check this earlier and make it an assertion here. + + // Copy the 8 most significant bits, and set the 8 least significant bits to zero. + // avifDecoderDecodeTiles() makes sure AVIF_ITEM_COLOR (respectively AVIF_ITEM_ALPHA) is decoded before + // AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR (respectively AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) so this is fine. + // dstView = srcView << 8 + AVIF_CHECKRES(avifImageTransformConstantAndImageSamples(&dstView, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/1 << 8, + AVIF_SAMPLE_TRANSFORM_PRODUCT, + /*rightOperand=*/&srcView, + planes)); + } else { + assert(tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA); + AVIF_CHECKERR(srcView.depth == 8, AVIF_RESULT_UNKNOWN_ERROR); // TODO(yguyon): Check this earlier and make it an assertion here. + + // Copy the 8 least significant bits. + // dstView = dstView | srcView + AVIF_CHECKRES(avifImageTransformImageAndImageSamples(&dstView, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/&dstView, + AVIF_SAMPLE_TRANSFORM_OR, + /*rightOperand=*/&srcView, + planes)); + } + + return AVIF_RESULT_OK; + } else if (data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B) { + assert(dstView.depth == 16); + if (tile->input->itemCategory == AVIF_ITEM_COLOR || tile->input->itemCategory == AVIF_ITEM_ALPHA) { + AVIF_CHECKERR(srcView.depth == 12, AVIF_RESULT_UNKNOWN_ERROR); // TODO(yguyon): Check this earlier and make it an assertion here. + + // dstView = srcView << 4 + AVIF_CHECKRES(avifImageTransformConstantAndImageSamples(&dstView, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/1 << 4, + AVIF_SAMPLE_TRANSFORM_PRODUCT, + /*rightOperand=*/&srcView, + planes)); + } else { + assert(tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA); + AVIF_CHECKERR(srcView.depth == 8, AVIF_RESULT_UNKNOWN_ERROR); // TODO(yguyon): Check this earlier and make it an assertion here. + + avifImage * shiftedSrc = avifImageCreate(srcView.width, srcView.height, srcView.depth, srcView.yuvFormat); + AVIF_CHECKERR(shiftedSrc != NULL, AVIF_RESULT_OUT_OF_MEMORY); + AVIF_CHECKRES(avifImageAllocatePlanes(shiftedSrc, planes)); + // shiftedSrc = srcView >> 4 + AVIF_CHECKRES(avifImageTransformConstantAndImageSamples(shiftedSrc, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/1 << 4, + AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED, + /*rightOperand=*/&srcView, + planes)); + + // dstView = dstView | shiftedSrc + AVIF_CHECKRES(avifImageTransformImageAndImageSamples(&dstView, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/&dstView, + AVIF_SAMPLE_TRANSFORM_OR, + /*rightOperand=*/shiftedSrc, + planes)); + } + return AVIF_RESULT_OK; + } else { + assert(data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_NONE); + } +#endif + + avifImageCopySamples(&dstView, &srcView, planes); return AVIF_RESULT_OK; } @@ -1988,6 +2160,62 @@ static avifBool avifParseToneMappedImageBox(avifGainMapMetadata * metadata, cons } #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +// Parses the raw bitstream of the 'sato' Sample Transform derived image item +// and extracts the recipe if the pattern is recognized. +static avifResult avifParseSampleTransformImageBox(const uint8_t * raw, + size_t rawLen, + uint32_t numInputItems, + avifSampleTransformRecipe * recipe, + avifDiagnostics * diag) +{ + BEGIN_STREAM(s, raw, rawLen, diag, "Box[sato]"); + // See avifEncoderWriteSampleTransformPayload() for the syntax. + + uint8_t version; + AVIF_CHECK(avifROStreamRead(&s, &version, 1)); // unsigned int(8) version = 0; + AVIF_CHECKERR(version == 0, AVIF_RESULT_NOT_IMPLEMENTED); + + uint8_t intermediateBitDepth, reserved; + AVIF_CHECK(avifROStreamReadBits8(&s, &intermediateBitDepth, /*bitCount=*/2)); // unsigned int(2) intermediate_bit_depth; + AVIF_CHECK(avifROStreamReadBits8(&s, &reserved, /*bitCount=*/6)); // unsigned int(6) reserved = 0; + AVIF_CHECKERR(intermediateBitDepth == AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, AVIF_RESULT_NOT_IMPLEMENTED); + AVIF_CHECKERR(reserved == 0, AVIF_RESULT_BMFF_PARSE_FAILED); + + // reference_count is the number of items input to the derived item. + AVIF_CHECKERR(numInputItems == 2, AVIF_RESULT_NOT_IMPLEMENTED); + uint32_t constants[2]; + uint8_t constantOperators[2]; + uint8_t operators[1]; + for (uint32_t i = 0; i < numInputItems; ++i) { + AVIF_CHECK(avifROStreamReadU32(&s, &constants[i])); // signed int(32) constant; + // constant is signed but libavif only supports unsigned sample operations, so just check that + // a negative value was not encoded (most significant bit, meaning sign bit, set to 0) rather + // than implementing avifROStreamReadS32() just for that purpose. + AVIF_CHECKERR(constants[i] <= INT32_MAX, AVIF_RESULT_NOT_IMPLEMENTED); + AVIF_CHECK(avifROStreamRead(&s, &constantOperators[i], 1)); // unsigned int(8) constant_operator; + if (i + 1 < numInputItems) { + AVIF_CHECK(avifROStreamRead(&s, &operators[i], 1)); // unsigned int(8) constant_operator; + } + } + + AVIF_CHECKERR(avifROStreamRemainingBytes(&s) == 0, AVIF_RESULT_BMFF_PARSE_FAILED); + + // Compare with known patterns. + if (constants[0] == (1u << 8) && constantOperators[0] == AVIF_SAMPLE_TRANSFORM_PRODUCT && + operators[0] == AVIF_SAMPLE_TRANSFORM_OR && constants[1] == 1 && constantOperators[1] == AVIF_SAMPLE_TRANSFORM_PRODUCT) { + *recipe = AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B; + return AVIF_RESULT_OK; + } + if (constants[0] == (1u << 4) && constantOperators[0] == AVIF_SAMPLE_TRANSFORM_PRODUCT && operators[0] == AVIF_SAMPLE_TRANSFORM_OR && + constants[1] == (1u << 4) && constantOperators[1] == AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED) { + *recipe = AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B; + return AVIF_RESULT_OK; + } + return AVIF_RESULT_NOT_IMPLEMENTED; +} +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + // Extracts the codecType from the item type or from its children. // Also parses and outputs grid information if the item is a grid. // isItemInInput must be false if the item is a made-up structure @@ -1998,6 +2226,9 @@ static avifResult avifDecoderItemReadAndParse(const avifDecoder * decoder, avifImageGrid * grid, avifCodecType * codecType) { + if (!item) { + return AVIF_RESULT_OK; + } if (!memcmp(item->type, "grid", 4)) { if (isItemInInput) { avifROData readData; @@ -2030,6 +2261,11 @@ static avifResult avifDecoderItemReadAndParse(const avifDecoder * decoder, *codecType = avifGetCodecType(item->type); AVIF_ASSERT_OR_RETURN(*codecType != AVIF_CODEC_TYPE_UNKNOWN); } + // Note: If AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM is defined, backward-incompatible files + // with a primary 'sato' Sample Transform derived image item could be handled here + // (compared to backward-compatible files with a 'sato' item in the same 'altr' group + // as the primary regular color item which are handled in + // avifDecoderDataFindSampleTransformImageItem() below). return AVIF_RESULT_OK; } @@ -4240,6 +4476,18 @@ static avifBool avifTilesCanBeDecodedWithSameCodecInstance(avifDecoderData * dat if (data->tileInfos[c].tileCount > 0) { ++numImageBuffers; } +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + // The sample operations require multiple buffers for compositing so no plane is stolen + // when there is a 'sato' Sample Transform derived image item. + if (data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR].tileCount > 0 && + (c == AVIF_ITEM_COLOR || c == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR)) { + continue; + } + if (data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA].tileCount > 0 && + (c == AVIF_ITEM_ALPHA || c == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA)) { + continue; + } +#endif if (data->tileInfos[c].tileCount == 1) { ++numStolenImageBuffers; } @@ -4644,6 +4892,60 @@ static avifResult avifDecoderFindGainMapItem(const avifDecoder * decoder, } #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +// Finds a 'sato' Sample Transform derived image item box associated with the given 'colorItem'. +// If found, fills 'sampleTransformItem' and sets 'bitDepthExtensionItemID' to the id of the +// second item input of the Sample Transform item. Otherwise, sets 'sampleTransformItem' to NULL. +// Returns AVIF_RESULT_OK on success (whether or not a 'sato' box was found). +// Assumes that there is a single 'sato' item per item category, and not, e.g., a grid of 'sato' items. +// Assumes that the 'sato' item is not the primary item and that both 'colorItem' and 'sato' are in +// the same 'altr' group. +// TODO(yguyon): Check instead of assuming. +static avifResult avifDecoderDataFindSampleTransformImageItem(avifDecoderData * data, + avifDecoderItem * colorItem, + avifDecoderItem ** sampleTransformItem, + uint32_t * bitDepthExtensionItemID) +{ + for (uint32_t itemIndex = 0; itemIndex < data->meta->items.count; ++itemIndex) { + avifDecoderItem * item = data->meta->items.item[itemIndex]; + if (!item->size || item->hasUnsupportedEssentialProperty || item->thumbnailForID != 0) { + continue; + } + if (!memcmp(item->type, "sato", 4)) { + // The 'sato' box should be associated (via 'iref'->'dimg') to two items: + // the first one is the base image, the second one is the bit depth extension. + uint32_t dimgItemIDs[2] = { 0, 0 }; + uint32_t numDimgItemIDs = 0; + for (uint32_t otherItemIndex = 0; otherItemIndex < data->meta->items.count; ++otherItemIndex) { + avifDecoderItem * otherItem = data->meta->items.item[otherItemIndex]; + if (otherItem->dimgForID != item->id) { + continue; + } + if (otherItem->dimgIdx < 2) { + assert(dimgItemIDs[otherItem->dimgIdx] == 0); + dimgItemIDs[otherItem->dimgIdx] = otherItem->id; + } + numDimgItemIDs++; + } + if (numDimgItemIDs != 2) { + avifDiagnosticsPrintf(data->diag, "Expected box[sato] to have 2 items associated with 'dimg': found [%d] instead", numDimgItemIDs); + return AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED; + } + if (dimgItemIDs[0] != colorItem->id) { + continue; + } + + *sampleTransformItem = item; + *bitDepthExtensionItemID = dimgItemIDs[1]; + return AVIF_RESULT_OK; + } + } + *sampleTransformItem = NULL; + *bitDepthExtensionItemID = 0; + return AVIF_RESULT_OK; +} +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + static avifResult avifDecoderGenerateImageTiles(avifDecoder * decoder, avifTileInfo * info, avifDecoderItem * item, avifItemCategory itemCategory) { const uint32_t previousTileCount = decoder->data->tiles.count; @@ -4900,13 +5202,11 @@ avifResult avifDecoderReset(avifDecoder * decoder) &mainItems[AVIF_ITEM_ALPHA], &data->tileInfos[AVIF_ITEM_ALPHA], &isAlphaItemInInput)); - if (mainItems[AVIF_ITEM_ALPHA]) { - AVIF_CHECKRES(avifDecoderItemReadAndParse(decoder, - mainItems[AVIF_ITEM_ALPHA], - isAlphaItemInInput, - &data->tileInfos[AVIF_ITEM_ALPHA].grid, - &codecType[AVIF_ITEM_ALPHA])); - } + AVIF_CHECKRES(avifDecoderItemReadAndParse(decoder, + mainItems[AVIF_ITEM_ALPHA], + isAlphaItemInInput, + &data->tileInfos[AVIF_ITEM_ALPHA].grid, + &codecType[AVIF_ITEM_ALPHA])); #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) avifDecoderItem * toneMappedImageItem; @@ -4943,6 +5243,69 @@ avifResult avifDecoderReset(avifDecoder * decoder) } #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + uint32_t bitDepthExtensionItemID; + avifBool isBitDepthExtensionAlphaItemInInput = AVIF_FALSE; // Initialized to avoid C4701 warning on Windows. + AVIF_CHECKRES(avifDecoderDataFindSampleTransformImageItem(data, + mainItems[AVIF_ITEM_COLOR], + &mainItems[AVIF_ITEM_SAMPLE_TRANSFORM], + &bitDepthExtensionItemID)); + if (mainItems[AVIF_ITEM_SAMPLE_TRANSFORM]) { + avifROData satoData; + AVIF_CHECKRES(avifDecoderItemRead(mainItems[AVIF_ITEM_SAMPLE_TRANSFORM], decoder->io, &satoData, 0, 0, data->diag)); + const avifResult result = avifParseSampleTransformImageBox(satoData.data, + satoData.size, + /*numInputItems=*/2, + &data->sampleTransformRecipe, + data->diag); + if (result == AVIF_RESULT_NOT_IMPLEMENTED) { + // Ignore the unsupported item and move along. + mainItems[AVIF_ITEM_SAMPLE_TRANSFORM] = NULL; + } else { + AVIF_CHECKRES(result); + assert(data->sampleTransformRecipe != AVIF_SAMPLE_TRANSFORM_NONE); + + AVIF_CHECKRES( + avifMetaFindOrCreateItem(data->meta, bitDepthExtensionItemID, &mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR])); + if (avifDecoderItemShouldBeSkipped(mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR])) { + avifDiagnosticsPrintf(data->diag, "Box[sato] input item %d is not a supported image type", bitDepthExtensionItemID); + return AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED; + } + + AVIF_CHECKRES(avifDecoderItemReadAndParse(decoder, + mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR], + /*isItemInInput=*/AVIF_TRUE, + &data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR].grid, + &codecType[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR])); + + // Optional alpha auxiliary item + AVIF_CHECKRES(avifMetaFindAlphaItem(data->meta, + mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR], + &data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR], + &mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA], + &data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA], + &isBitDepthExtensionAlphaItemInInput)); + if (!mainItems[AVIF_ITEM_ALPHA] != !mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA]) { + avifDiagnosticsPrintf(data->diag, "Box[sato] input items do not all consistently have alpha layers"); + return AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED; + } + if (mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA]) { + if ((mainItems[AVIF_ITEM_ALPHA]->premByID == mainItems[AVIF_ITEM_COLOR]->id) != + (mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA]->premByID == + mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR]->id)) { + avifDiagnosticsPrintf(data->diag, "Box[sato] input items do not all consistently have alpha premultiplication"); + return AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED; + } + AVIF_CHECKRES(avifDecoderItemReadAndParse(decoder, + mainItems[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA], + isBitDepthExtensionAlphaItemInInput, + &data->tileInfos[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA].grid, + &codecType[AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA])); + } + } + } +#endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP + // Find Exif and/or XMP metadata, if any AVIF_CHECKRES(avifDecoderFindMetadata(decoder, data->meta, decoder->image, mainItems[AVIF_ITEM_COLOR]->id)); @@ -4975,10 +5338,21 @@ avifResult avifDecoderReset(avifDecoder * decoder) mainItems[c]->height = mainItems[AVIF_ITEM_COLOR]->height; } + const avifBool isCodedImageItemOrGrid = avifGetCodecType(mainItems[c]->type) != AVIF_CODEC_TYPE_UNKNOWN || + !memcmp(mainItems[c]->type, "grid", 4); + if (!isCodedImageItemOrGrid) { + continue; + } AVIF_CHECKRES(avifDecoderGenerateImageTiles(decoder, &data->tileInfos[c], mainItems[c], c)); + avifBool isAlphaItemNotInInput = c == AVIF_ITEM_ALPHA && !isAlphaItemInInput; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + isAlphaItemNotInInput = isAlphaItemNotInInput || + (c == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA && !isBitDepthExtensionAlphaItemInInput); +#endif + avifStrictFlags strictFlags = decoder->strictFlags; - if (c == AVIF_ITEM_ALPHA && !isAlphaItemInInput) { + if (isAlphaItemNotInInput) { // In this case, the made up grid item will not have an associated pixi property. So validate everything else // but the pixi property. strictFlags &= ~AVIF_STRICT_PIXI_REQUIRED; @@ -4989,7 +5363,7 @@ avifResult avifDecoderReset(avifDecoder * decoder) if (mainItems[AVIF_ITEM_COLOR]->progressive) { decoder->progressiveState = AVIF_PROGRESSIVE_STATE_AVAILABLE; - // data->color.firstTileIndex is not yet defined but will be set to 0 a few lines below. + // data->tileInfos[AVIF_ITEM_COLOR].firstTileIndex is not yet defined but will be set to 0 a few lines below. const avifTile * colorTile = &data->tiles.tile[0]; if (colorTile->input->samples.count > 1) { decoder->progressiveState = AVIF_PROGRESSIVE_STATE_ACTIVE; @@ -5034,10 +5408,10 @@ avifResult avifDecoderReset(avifDecoder * decoder) return AVIF_RESULT_BMFF_PARSE_FAILED; } - if (tile->input->itemCategory == AVIF_ITEM_COLOR) { - decoder->ioStats.colorOBUSize += sample->size; - } else if (tile->input->itemCategory == AVIF_ITEM_ALPHA) { + if (avifIsAlpha(tile->input->itemCategory)) { decoder->ioStats.alphaOBUSize += sample->size; + } else { + decoder->ioStats.colorOBUSize += sample->size; } } } @@ -5183,6 +5557,12 @@ static avifResult avifGetErrorForItemCategory(avifItemCategory itemCategory) if (itemCategory == AVIF_ITEM_GAIN_MAP) { return AVIF_RESULT_DECODE_GAIN_MAP_FAILED; } +#endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (itemCategory == AVIF_ITEM_SAMPLE_TRANSFORM || itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) { + return AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED; + } #endif return (itemCategory == AVIF_ITEM_ALPHA) ? AVIF_RESULT_DECODE_ALPHA_FAILED : AVIF_RESULT_DECODE_COLOR_FAILED; } @@ -5200,8 +5580,23 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma return AVIF_RESULT_OK; } +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + tile->input->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) { + // AVIF_ITEM_COLOR and AVIF_ITEM_ALPHA samples are copied to the result buffer for simplicity. + // AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR and AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA samples are + // added to the result buffer afterwards. Make sure it happens in this order. + const avifTileInfo * mostSignificantBitsInfo = + &decoder->data->tileInfos[avifIsAlpha(tile->input->itemCategory) ? AVIF_ITEM_ALPHA : AVIF_ITEM_COLOR]; + if (mostSignificantBitsInfo->decodedTileCount < mostSignificantBitsInfo->tileCount) { + assert(decoder->allowIncremental); + return AVIF_RESULT_OK; + } + } +#endif + avifBool isLimitedRangeAlpha = AVIF_FALSE; - if (!tile->codec->getNextImage(tile->codec, decoder, sample, tile->input->itemCategory == AVIF_ITEM_ALPHA, &isLimitedRangeAlpha, tile->image)) { + if (!tile->codec->getNextImage(tile->codec, decoder, sample, avifIsAlpha(tile->input->itemCategory), &isLimitedRangeAlpha, tile->image)) { avifDiagnosticsPrintf(&decoder->diag, "tile->codec->getNextImage() failed"); return avifGetErrorForItemCategory(tile->input->itemCategory); } @@ -5229,7 +5624,7 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma // of the specification. However, it was allowed in version 1.0.0 of the // specification. To allow such files, simply convert the alpha plane to // full range. - if ((tile->input->itemCategory == AVIF_ITEM_ALPHA) && isLimitedRangeAlpha) { + if (avifIsAlpha(tile->input->itemCategory) && isLimitedRangeAlpha) { avifResult result = avifImageLimitedToFullAlpha(tile->image); if (result != AVIF_RESULT_OK) { avifDiagnosticsPrintf(&decoder->diag, "avifImageLimitedToFullAlpha failed"); @@ -5251,7 +5646,13 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma ++info->decodedTileCount; - if ((info->grid.rows > 0) && (info->grid.columns > 0)) { + const avifBool isGrid = (info->grid.rows > 0) && (info->grid.columns > 0); + avifBool stealPlanes = !isGrid; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + stealPlanes = stealPlanes && decoder->data->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_NONE; +#endif + + if (!stealPlanes) { if (tileIndex == 0) { avifImage * dstImage = decoder->image; #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) @@ -5260,7 +5661,7 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma dstImage = dstImage->gainMap->image; } #endif - AVIF_CHECKRES(avifDecoderDataAllocateGridImagePlanes(decoder->data, info, dstImage)); + AVIF_CHECKRES(avifDecoderDataAllocateImagePlanes(decoder->data, info, dstImage)); } AVIF_CHECKRES(avifDecoderDataCopyTileToImage(decoder->data, info, decoder->image, tile, tileIndex)); } else { @@ -5281,7 +5682,7 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma default: if ((decoder->image->width != src->width) || (decoder->image->height != src->height) || (decoder->image->depth != src->depth)) { - if (tile->input->itemCategory == AVIF_ITEM_ALPHA) { + if (avifIsAlpha(tile->input->itemCategory)) { avifDiagnosticsPrintf(&decoder->diag, "The color image item does not match the alpha image item in width, height, or bit depth"); return AVIF_RESULT_DECODE_ALPHA_FAILED; @@ -5295,7 +5696,7 @@ static avifResult avifDecoderDecodeTiles(avifDecoder * decoder, uint32_t nextIma break; } - if (tile->input->itemCategory == AVIF_ITEM_ALPHA) { + if (avifIsAlpha(tile->input->itemCategory)) { avifImageStealPlanes(decoder->image, src, AVIF_PLANES_A); #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) } else if (tile->input->itemCategory == AVIF_ITEM_GAIN_MAP) { @@ -5589,6 +5990,7 @@ avifResult avifDecoderRead(avifDecoder * decoder, avifImage * image) if (result != AVIF_RESULT_OK) { return result; } + // TODO(yguyon): If image owns its planes, transfer their ownership instead of copying? return avifImageCopy(image, decoder->image, AVIF_PLANES_ALL); } diff --git a/src/sampletransform.c b/src/sampletransform.c new file mode 100644 index 0000000000..8feac35612 --- /dev/null +++ b/src/sampletransform.c @@ -0,0 +1,358 @@ +// Copyright 2023 Google LLC +// SPDX-License-Identifier: BSD-2-Clause + +#include "avif/internal.h" + +#include +#include +#include + +avifResult avifImageTransformImageAndImageSamples(avifImage * result, + avifSampleTransformIntermediateBitDepth intermediateBitDepth, + const avifImage * leftOperand, + avifSampleTransformOperation operation, + const avifImage * rightOperand, + avifPlanesFlags planes) +{ + if (!(planes & AVIF_PLANES_YUV) && !(planes & AVIF_PLANES_A)) { + // Early exit. + return AVIF_RESULT_OK; + } + + // TODO(yguyon): Implement other avifSampleTransformIntermediateBitDepth values. + AVIF_CHECKERR(intermediateBitDepth == AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, AVIF_RESULT_NOT_IMPLEMENTED); + AVIF_CHECKERR(avifImageUsesU16(leftOperand), AVIF_RESULT_NOT_IMPLEMENTED); // libavif only uses 16-bit leftOperand. + AVIF_CHECKERR(avifImageUsesU16(result), AVIF_RESULT_NOT_IMPLEMENTED); // libavif only uses 16-bit result. + + const avifBool skipColor = !(planes & AVIF_PLANES_YUV); + const avifBool skipAlpha = !(planes & AVIF_PLANES_A); + for (int c = AVIF_CHAN_Y; c <= AVIF_CHAN_A; ++c) { + const avifBool alpha = c == AVIF_CHAN_A; + if ((skipColor && !alpha) || (skipAlpha && alpha)) { + continue; + } + + const uint32_t planeWidth = avifImagePlaneWidth(leftOperand, c); + const uint32_t planeHeight = avifImagePlaneHeight(leftOperand, c); + const uint8_t * leftOperandRow = avifImagePlane(leftOperand, c); + const uint8_t * rightOperandRow = avifImagePlane(rightOperand, c); + uint8_t * resultRow = avifImagePlane(result, c); + const uint32_t leftOperandRowBytes = avifImagePlaneRowBytes(leftOperand, c); + const uint32_t rightOperandRowBytes = avifImagePlaneRowBytes(rightOperand, c); + const uint32_t resultRowBytes = avifImagePlaneRowBytes(result, c); + AVIF_CHECKERR(!leftOperandRow == !resultRow, AVIF_RESULT_INVALID_ARGUMENT); + AVIF_CHECKERR(!rightOperandRow == !resultRow, AVIF_RESULT_INVALID_ARGUMENT); + if (!leftOperandRow) { + continue; + } + AVIF_CHECKERR(planeWidth == avifImagePlaneWidth(rightOperand, c), AVIF_RESULT_INVALID_ARGUMENT); + AVIF_CHECKERR(planeHeight == avifImagePlaneHeight(rightOperand, c), AVIF_RESULT_INVALID_ARGUMENT); + AVIF_CHECKERR(planeWidth == avifImagePlaneWidth(result, c), AVIF_RESULT_INVALID_ARGUMENT); + AVIF_CHECKERR(planeHeight == avifImagePlaneHeight(result, c), AVIF_RESULT_INVALID_ARGUMENT); + + if (operation == AVIF_SAMPLE_TRANSFORM_OR) { + // Reminder of the above checks. + assert(avifImageUsesU16(leftOperand)); + assert(avifImageUsesU16(result)); + + if (avifImageUsesU16(rightOperand)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * leftOperandRow16 = (const uint16_t *)leftOperandRow; + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperandRow16[x] | (uint16_t)rightOperandRow16[x]); + } + leftOperandRow += leftOperandRowBytes; + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * leftOperandRow16 = (const uint16_t *)leftOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperandRow16[x] | (uint16_t)rightOperandRow[x]); + } + leftOperandRow += leftOperandRowBytes; + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + continue; + } + + // The remaining operations are not used in libavif for now. + return AVIF_RESULT_NOT_IMPLEMENTED; + } + return AVIF_RESULT_OK; +} + +avifResult avifImageTransformConstantAndImageSamples(avifImage * result, + avifSampleTransformIntermediateBitDepth intermediateBitDepth, + int32_t leftOperand, + avifSampleTransformOperation operation, + const avifImage * rightOperand, + avifPlanesFlags planes) +{ + if (!(planes & AVIF_PLANES_YUV) && !(planes & AVIF_PLANES_A)) { + // Early exit. + return AVIF_RESULT_OK; + } + + // Is calling this function with leftOperand and operation equivalent to copying rightOperand samples to result? + const avifBool noop = (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_SUM) || + (leftOperand == 1 && operation == AVIF_SAMPLE_TRANSFORM_PRODUCT) || + (operation == AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED && leftOperand == 1) || + (operation == AVIF_SAMPLE_TRANSFORM_POW_REVERSED && leftOperand == 1) || + (operation == AVIF_SAMPLE_TRANSFORM_LOG_REVERSED && leftOperand == 1) || + (leftOperand == INT32_MAX && operation == AVIF_SAMPLE_TRANSFORM_AND) || // rightOperand is positive + (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_OR) || + (leftOperand == INT32_MAX && operation == AVIF_SAMPLE_TRANSFORM_MIN) || + (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_MAX); + if (noop && result == rightOperand) { + // Early exit. + return AVIF_RESULT_OK; + } + const avifBool leftOperandIsPowerOfTwo = (leftOperand > 0) && ((leftOperand & (leftOperand - 1)) == 0); + uint32_t leftOperandLog2 = 0; + if (leftOperandIsPowerOfTwo) { + while ((1 << leftOperandLog2) < leftOperand) { + ++leftOperandLog2; + } + } + + // Is calling this function with leftOperand and operation equivalent to setting all result samples to zero? + const avifBool clear = (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_PRODUCT) || + (operation == AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED && leftOperand > (1 << rightOperand->depth)) || + (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_AND) || + (leftOperand == 0 && operation == AVIF_SAMPLE_TRANSFORM_MIN); // rightOperand is positive + + // TODO(yguyon): Implement other avifSampleTransformIntermediateBitDepth values. + AVIF_CHECKERR(intermediateBitDepth == AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, AVIF_RESULT_NOT_IMPLEMENTED); + + const avifBool skipColor = !(planes & AVIF_PLANES_YUV); + const avifBool skipAlpha = !(planes & AVIF_PLANES_A); + for (int c = AVIF_CHAN_Y; c <= AVIF_CHAN_A; ++c) { + const avifBool alpha = c == AVIF_CHAN_A; + if ((skipColor && !alpha) || (skipAlpha && alpha)) { + continue; + } + + const uint32_t planeWidth = avifImagePlaneWidth(rightOperand, c); + const uint32_t planeHeight = avifImagePlaneHeight(rightOperand, c); + const uint8_t * rightOperandRow = avifImagePlane(rightOperand, c); + uint8_t * resultRow = avifImagePlane(result, c); + const uint32_t rightOperandRowBytes = avifImagePlaneRowBytes(rightOperand, c); + const uint32_t resultRowBytes = avifImagePlaneRowBytes(result, c); + AVIF_CHECKERR(!rightOperandRow == !resultRow, AVIF_RESULT_INVALID_ARGUMENT); + if (!rightOperandRow) { + continue; + } + AVIF_CHECKERR(planeWidth == avifImagePlaneWidth(result, c), AVIF_RESULT_INVALID_ARGUMENT); + AVIF_CHECKERR(planeHeight == avifImagePlaneHeight(result, c), AVIF_RESULT_INVALID_ARGUMENT); + + if (noop) { + // Just copy the rightOperand samples to result. + if (avifImageUsesU16(rightOperand) == avifImageUsesU16(result)) { + // The raw bytes can be copied. + const size_t planeWidthBytes = planeWidth * (avifImageUsesU16(rightOperand) ? 2 : 1); + for (uint32_t y = 0; y < planeHeight; ++y) { + memcpy(resultRow, rightOperandRow, planeWidthBytes); + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + AVIF_CHECKERR(avifImageUsesU16(result), AVIF_RESULT_INVALID_ARGUMENT); // Cannot fit 16-bit samples into 8 bits. + for (uint32_t y = 0; y < planeHeight; ++y) { + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = rightOperandRow[x]; // 8-bit to 16-bit + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + continue; + } + + if (clear) { + // Just set the result samples to zero. + const size_t planeWidthBytes = planeWidth * (avifImageUsesU16(result) ? 2 : 1); + for (uint32_t y = 0; y < planeHeight; ++y) { + memset(resultRow, 0, planeWidthBytes); + resultRow += resultRowBytes; + } + continue; + } + + if (operation == AVIF_SAMPLE_TRANSFORM_SUM) { + if (avifImageUsesU16(rightOperand)) { + AVIF_CHECKERR(avifImageUsesU16(result), AVIF_RESULT_INVALID_ARGUMENT); // Cannot fit 16-bit samples into 8 bits. + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperand + rightOperandRow16[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperand + rightOperandRow[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(leftOperand + rightOperandRow[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } + continue; + } + + if (operation == AVIF_SAMPLE_TRANSFORM_PRODUCT && leftOperandIsPowerOfTwo) { + if (avifImageUsesU16(rightOperand)) { + AVIF_CHECKERR(avifImageUsesU16(result), AVIF_RESULT_INVALID_ARGUMENT); // Cannot fit 16-bit samples into 8 bits. + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(rightOperandRow16[x] << leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(rightOperandRow[x] << leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(rightOperandRow[x] << leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } + continue; + } + + if (operation == AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED && leftOperandIsPowerOfTwo) { + if (avifImageUsesU16(rightOperand)) { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(rightOperandRow16[x] >> leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(rightOperandRow16[x] >> leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } else { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(rightOperandRow[x] >> leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(rightOperandRow[x] >> leftOperandLog2); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } + continue; + } + + if (operation == AVIF_SAMPLE_TRANSFORM_AND) { + // Instead of caring about signed bitwise AND, just make sure it does not happen. Unused in libavif anyway. + assert(leftOperand >= 0); + + if (avifImageUsesU16(rightOperand)) { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperand & rightOperandRow16[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + // Cannot fit 16-bit samples into 8 bits, so make sure the mask guarantees 8-bit samples. + AVIF_CHECKERR(leftOperand < (1 << 8), AVIF_RESULT_INVALID_ARGUMENT); + for (uint32_t y = 0; y < planeHeight; ++y) { + const uint16_t * rightOperandRow16 = (const uint16_t *)rightOperandRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(leftOperand & rightOperandRow16[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } else { + if (avifImageUsesU16(result)) { + for (uint32_t y = 0; y < planeHeight; ++y) { + uint16_t * resultRow16 = (uint16_t *)resultRow; + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow16[x] = (uint16_t)(leftOperand & rightOperandRow[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } else { + for (uint32_t y = 0; y < planeHeight; ++y) { + for (uint32_t x = 0; x < planeWidth; ++x) { + resultRow[x] = (uint8_t)(leftOperand & rightOperandRow[x]); + } + rightOperandRow += rightOperandRowBytes; + resultRow += resultRowBytes; + } + } + } + continue; + } + + // The remaining operations are not used in libavif for now. + return AVIF_RESULT_NOT_IMPLEMENTED; + } + return AVIF_RESULT_OK; +} diff --git a/src/write.c b/src/write.c index 9cbe1f8d0b..ea0ff01f54 100644 --- a/src/write.c +++ b/src/write.c @@ -485,6 +485,9 @@ avifEncoder * avifEncoderCreate(void) return NULL; } encoder->headerFormat = AVIF_HEADER_FULL; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + encoder->sampleTransformRecipe = AVIF_SAMPLE_TRANSFORM_NONE; +#endif return encoder; } @@ -983,6 +986,106 @@ static avifResult avifImageCopyAltImageMetadata(avifImage * altImageMetadata, co } #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +static avifResult avifEncoderWriteSampleTransformPayload(avifEncoder * encoder, avifRWData * data) +{ + // aligned(8) class SampleTransform { + // unsigned int(8) version = 0; + // unsigned int(2) intermediate_bit_depth; + // unsigned int(6) reserved = 0; + // for (i=1; i<=reference_count; i++) { + // signed int(1<<(intermediate_bit_depth+3)) constant; // 8, 16, 32 or 64-bit + // unsigned int(8) constant_operator; + // if (isampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B) { + // reference_count is two: two 8-bit input images. + // (base_sample << 8) | hidden_sample + // is expressed as + // (256 * base_sample) | (1 * hidden_sample) + // Note: base_sample is encoded losslessly. hidden_sample is encoded lossily or losslessly. + + // First sub-expression. + { + // The base image represents the 8 most significant bits of the reconstructed, bit-depth-extended output image. + // Left shift the base image (which is also the primary item, or the auxiliary alpha item of the primary item) + // by 8 bits. This is equivalent to multiplying by 2^8 and the latter is simpler to check for out-of-bounds issues + // at decoding than AVIF_SAMPLE_TRANSFORM_POW which could accept arbitrary high exponents. + // Note: avifRWStreamWriteU32() works as avifRWStreamWrite32() for non-negative values. + AVIF_CHECKRES(avifRWStreamWriteU32(&s, 1u << 8)); // signed int(32) constant; + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_PRODUCT)); // unsigned int(8) constant_operator; + } + + // Combine the two sub-expressions. + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_OR)); // unsigned int(8) operator; + + // Second sub-expression. + { + // The second image represents the 8 least significant bits of the reconstructed, bit-depth-extended output image. + // The constant and its operator are unused here. Keep AVIF_SAMPLE_TRANSFORM_PRODUCT for maximum compatibility. + AVIF_CHECKRES(avifRWStreamWriteU32(&s, 1)); // signed int(32) constant; + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_PRODUCT)); // unsigned int(8) constant_operator; + } + } else if (encoder->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B) { + // reference_count is two: one 12-bit input image and one 8-bit input image (because AV1 does not support 4-bit samples). + // (base_sample << 4) | (hidden_sample >> 4) + // is expressed as + // (16 * base_sample) | (16 \ hidden_sample) + // Note: base_sample is encoded losslessly. hidden_sample is encoded lossily or losslessly. + + // First sub-expression. + { + // The base image represents the 12 most significant bits of the reconstructed, bit-depth-extended output image. + // Left shift the base image (which is also the primary item, or the auxiliary alpha item of the primary item) + // by 4 bits. This is equivalent to multiplying by 2^4 and the latter is simpler to check for out-of-bounds issues + // at decoding than AVIF_SAMPLE_TRANSFORM_POW which could accept arbitrary high exponents. + AVIF_CHECKRES(avifRWStreamWriteU32(&s, 1u << 4)); // signed int(32) constant; + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_PRODUCT)); // unsigned int(8) constant_operator; + } + + // Combine the two sub-expressions. + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_OR)); // unsigned int(8) operator; + + // Second sub-expression. + { + // The second image represents the 4 least significant bits of the reconstructed, bit-depth-extended output image. + AVIF_CHECKRES(avifRWStreamWriteU32(&s, 1u << 4)); // signed int(32) constant; + AVIF_CHECKRES(avifRWStreamWriteU8(&s, AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED)); // unsigned int(8) constant_operator; + } + + // Note: If hidden_sample is encoded lossily, it is expected to be offset by +8 before encoding, + // so that right shifting it by 4 bits at decoding rounds the result, for better precision + // than plain truncation. + // + // Otherwise, better precision than truncation could alternatively be done only at decoding with + // (base_sample << 4) | ((hidden_sample * 0xF + 0x7F) / 0xFF) + // or + // ((8 + hidden_sample) / (16 + hidden_sample_zero)) | (16 * base_sample) + // which is then clamped to [0:65535] according to the specification proposal: + // "The output reconstructed image is made of the resulting samples, whose values are each clamped to fit in the + // number of bits per sample as defined by the Pixel Information property of the reconstructed image item." + // but this is complex so the former solution is kept, even though better precision kould be achieved. + } else { + avifDiagnosticsPrintf(&encoder->diag, "Failed to write sample transform metadata for recipe %d", encoder->sampleTransformRecipe); + return AVIF_RESULT_NOT_IMPLEMENTED; + } + + avifRWStreamFinishWrite(&s); + return AVIF_RESULT_OK; +} +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + static avifResult avifEncoderDataCreateExifItem(avifEncoderData * data, const avifRWData * exif) { size_t exifTiffHeaderOffset; @@ -1101,6 +1204,9 @@ static const char infeNameAlpha[] = "Alpha"; #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) static const char infeNameGainMap[] = "GMap"; #endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +static const char infeNameSampleTransform[] = "SampleTransform"; +#endif // Adds the items for a single cell or a grid of cells. Outputs the topLevelItemID which is // the only item if there is exactly one cell, or the grid item for multiple cells. @@ -1119,7 +1225,11 @@ static avifResult avifEncoderAddImageItems(avifEncoder * encoder, #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) : (itemCategory == AVIF_ITEM_GAIN_MAP) ? infeNameGainMap #endif - : infeNameColor; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + : (itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) + ? infeNameSampleTransform +#endif + : infeNameColor; const size_t infeNameSize = strlen(infeName) + 1; if (cellCount > 1) { @@ -1153,6 +1263,177 @@ static avifResult avifEncoderAddImageItems(avifEncoder * encoder, return AVIF_RESULT_OK; } +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) +static avifResult avifEncoderCreateSampleTransformItems(avifEncoder * encoder, + uint32_t gridCols, + uint32_t gridRows, + uint32_t gridWidth, + uint32_t gridHeight, + uint16_t colorItemID) +{ + // There are multiple possible ISOBMFF box hierarchies for translucent images, + // using 'sato' (Sample Transform) derived image items: + // - a primary 'sato' item uses a main color coded item and a hidden color coded item; each color coded + // item has an auxiliary alpha coded item; the main color coded item and the 'sato' item are in + // an 'altr' group (backward-compatible, implemented) + // - a primary 'sato' item uses a main color coded item and a hidden color coded item; the primary + // 'sato' item has an auxiliary alpha 'sato' item using two alpha coded items (backward-incompatible) + // Likewise, there are multiple possible ISOBMFF box hierarchies for bit-depth-extended grids, + // using 'sato' (Sample Transform) derived image items: + // - a primary color 'grid', an auxiliary alpha 'grid', a hidden color 'grid', a hidden auxiliary alpha 'grid' + // and a 'sato' using the two color 'grid's as input items in this order; the primary color item + // and the 'sato' item being in an 'altr' group (backward-compatible, implemented) + // - a primary 'grid' of 'sato' cells and an auxiliary alpha 'grid' of 'sato' cells (backward-incompatible) + avifEncoderItem * sampleTransformItem = avifEncoderDataCreateItem(encoder->data, + "sato", + infeNameSampleTransform, + /*infeNameSize=*/strlen(infeNameSampleTransform) + 1, + /*cellIndex=*/0); + AVIF_CHECKRES(avifEncoderWriteSampleTransformPayload(encoder, &sampleTransformItem->metadataPayload)); + sampleTransformItem->itemCategory = AVIF_ITEM_SAMPLE_TRANSFORM; + uint16_t sampleTransformItemID = sampleTransformItem->id; + // 'altr' group + *(uint16_t *)avifArrayPush(&encoder->data->alternativeItemIDs) = sampleTransformItem->id; + *(uint16_t *)avifArrayPush(&encoder->data->alternativeItemIDs) = colorItemID; + + uint16_t bitDepthExtensionColorItemId; + AVIF_CHECKRES( + avifEncoderAddImageItems(encoder, gridCols, gridRows, gridWidth, gridHeight, AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR, &bitDepthExtensionColorItemId)); + avifEncoderItem * bitDepthExtensionColorItem = avifEncoderDataFindItemByID(encoder->data, bitDepthExtensionColorItemId); + assert(bitDepthExtensionColorItem); + bitDepthExtensionColorItem->hiddenImage = AVIF_TRUE; + + // Set the color and bit depth extension items' dimgFromID value to point to the sample transform item. + // The color item shall be first, and the bit depth extension item second. avifEncoderFinish() writes the + // dimg item references in item id order, so as long as colorItemID < bitDepthExtensionColorItemId, the order + // will be correct. + assert(colorItemID < bitDepthExtensionColorItemId); + avifEncoderItem * colorItem = avifEncoderDataFindItemByID(encoder->data, colorItemID); + assert(colorItem); + assert(colorItem->dimgFromID == 0); // Our internal API only allows one dimg value per item. + colorItem->dimgFromID = sampleTransformItemID; + bitDepthExtensionColorItem->dimgFromID = sampleTransformItemID; + + if (encoder->data->alphaPresent) { + uint16_t bitDepthExtensionAlphaItemId; + AVIF_CHECKRES( + avifEncoderAddImageItems(encoder, gridCols, gridRows, gridWidth, gridHeight, AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA, &bitDepthExtensionAlphaItemId)); + avifEncoderItem * bitDepthExtensionAlphaItem = avifEncoderDataFindItemByID(encoder->data, bitDepthExtensionAlphaItemId); + assert(bitDepthExtensionAlphaItem); + bitDepthExtensionAlphaItem->irefType = "auxl"; + bitDepthExtensionAlphaItem->irefToID = bitDepthExtensionColorItemId; + if (encoder->data->imageMetadata->alphaPremultiplied) { + // The reference may have changed; fetch it again. + bitDepthExtensionColorItem = avifEncoderDataFindItemByID(encoder->data, bitDepthExtensionColorItemId); + assert(bitDepthExtensionColorItem); + bitDepthExtensionColorItem->irefType = "prem"; + bitDepthExtensionColorItem->irefToID = bitDepthExtensionAlphaItemId; + } + } + return AVIF_RESULT_OK; +} + +static avifResult avifEncoderCreateSampleTransformInputImage(const avifEncoder * encoder, + const avifEncoderItem * item, + avifBool itemWillBeEncodedLosslessly, + const avifImage * image, + avifImage ** sampleTransformedImage) +{ + // The bit depth of the first image item used as input to the 'sato' Sample Transform derived image item. + uint32_t numMostSignificantBits; + // The bit depth of the current image item used as input to the 'sato' Sample Transform derived image item. + uint32_t sampleTransformInputImageDepth; + if (encoder->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B) { + numMostSignificantBits = 8; + sampleTransformInputImageDepth = 8; + } else { + assert(encoder->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B); + numMostSignificantBits = 12; + if (item->itemCategory == AVIF_ITEM_COLOR || item->itemCategory == AVIF_ITEM_ALPHA) { + sampleTransformInputImageDepth = numMostSignificantBits; + } else { + assert(item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA); + sampleTransformInputImageDepth = 8; // Will be shifted to 4-bit samples at decoding. + } + } + assert(image->depth == 16); // Other bit depths could be supported but for now it is 16-bit only. + const uint32_t numLeastSignificantBits = image->depth - numMostSignificantBits; + + *sampleTransformedImage = avifImageCreate(image->width, image->height, sampleTransformInputImageDepth, image->yuvFormat); + AVIF_CHECKERR(*sampleTransformedImage, AVIF_RESULT_OUT_OF_MEMORY); + const avifPlanesFlag planes = (item->itemCategory == AVIF_ITEM_ALPHA || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) + ? AVIF_PLANES_A + : AVIF_PLANES_YUV; + const avifResult allocationResult = avifImageAllocatePlanes(*sampleTransformedImage, planes); + if (allocationResult != AVIF_RESULT_OK) { + avifImageDestroy(*sampleTransformedImage); + return allocationResult; + } + + if (item->itemCategory == AVIF_ITEM_COLOR || item->itemCategory == AVIF_ITEM_ALPHA) { + // 16-bit to sampleTransformInputImageDepth-bit so shift right by numLeastSignificantBits bits. + // Equivalent to dividing by 1<itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA); + // Keep the numLeastSignificantBits from the 16-bit image. Use a bit mask. + if (avifImageTransformConstantAndImageSamples(*sampleTransformedImage, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/(1 << numLeastSignificantBits) - 1, + AVIF_SAMPLE_TRANSFORM_AND, + image, + planes) != AVIF_RESULT_OK) { + assert(AVIF_FALSE); + } + + if ((*sampleTransformedImage)->depth != numLeastSignificantBits) { + assert((*sampleTransformedImage)->depth > numLeastSignificantBits); + // AVIF only supports 8, 10 or 12-bit image items. Scale the samples to fit the range. + // Note: The samples could be encoded as is without being shifted left before encoding, + // but they would not be shifted right after decoding either. Right shifting after + // decoding provides a guarantee on the range of values and on the lack of integer + // overflow, so it is safer to do these extra steps. + // It also makes more sense from a compression point-of-view to use the full range. + // Transform in-place. + const uint32_t numShiftedBits = (*sampleTransformedImage)->depth - numLeastSignificantBits; + if (avifImageTransformConstantAndImageSamples(/*result=*/*sampleTransformedImage, + AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/1 << numShiftedBits, + /*operation=*/AVIF_SAMPLE_TRANSFORM_PRODUCT, + /*rightOperand=*/*sampleTransformedImage, + planes) != AVIF_RESULT_OK) { + assert(AVIF_FALSE); + } + + if (!itemWillBeEncodedLosslessly) { + // Small loss at encoding could be amplified by the truncation caused by the right + // shift after decoding. Offset sample values now, before encoding, to round rather + // than floor the samples shifted after decoding. + // Note: Samples were just left shifted by numShiftedBits, so adding less than + // (1<> 1, + /*operation=*/AVIF_SAMPLE_TRANSFORM_SUM, + /*rightOperand=*/*sampleTransformedImage, + planes) != AVIF_RESULT_OK) { + assert(AVIF_FALSE); + } + } + } + } + return AVIF_RESULT_OK; +} +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + static avifCodecType avifEncoderGetCodecType(const avifEncoder * encoder) { // TODO(yguyon): Rework when AVIF_CODEC_CHOICE_AUTO can be AVM @@ -1192,20 +1473,18 @@ static avifBool avifEncoderDataShouldForceKeyframeForAlpha(const avifEncoderData static avifResult avifGetErrorForItemCategory(avifItemCategory itemCategory) { - return (itemCategory == AVIF_ITEM_ALPHA) ? AVIF_RESULT_ENCODE_ALPHA_FAILED : AVIF_RESULT_ENCODE_COLOR_FAILED; -} - -static avifResult avifValidateImageBasicProperties(const avifImage * avifImage) -{ - if ((avifImage->depth != 8) && (avifImage->depth != 10) && (avifImage->depth != 12)) { - return AVIF_RESULT_UNSUPPORTED_DEPTH; +#if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) + if (itemCategory == AVIF_ITEM_GAIN_MAP) { + return AVIF_RESULT_ENCODE_GAIN_MAP_FAILED; } - - if (avifImage->yuvFormat == AVIF_PIXEL_FORMAT_NONE) { - return AVIF_RESULT_NO_YUV_FORMAT_SELECTED; +#endif +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (itemCategory == AVIF_ITEM_SAMPLE_TRANSFORM || itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA) { + return AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED; } - - return AVIF_RESULT_OK; +#endif + return (itemCategory == AVIF_ITEM_ALPHA) ? AVIF_RESULT_ENCODE_ALPHA_FAILED : AVIF_RESULT_ENCODE_COLOR_FAILED; } static uint32_t avifGridWidth(uint32_t gridCols, const avifImage * firstCell, const avifImage * bottomRightCell) @@ -1326,7 +1605,14 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, const avifImage * firstCell = cellImages[0]; const avifImage * bottomRightCell = cellImages[cellCount - 1]; - AVIF_CHECKRES(avifValidateImageBasicProperties(firstCell)); +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + AVIF_CHECKERR(firstCell->depth == 8 || firstCell->depth == 10 || firstCell->depth == 12 || + (firstCell->depth == 16 && encoder->sampleTransformRecipe != AVIF_SAMPLE_TRANSFORM_NONE), + AVIF_RESULT_UNSUPPORTED_DEPTH); +#else + AVIF_CHECKERR(firstCell->depth == 8 || firstCell->depth == 10 || firstCell->depth == 12, AVIF_RESULT_UNSUPPORTED_DEPTH); +#endif + AVIF_CHECKERR(firstCell->yuvFormat != AVIF_PIXEL_FORMAT_NONE, AVIF_RESULT_NO_YUV_FORMAT_SELECTED); if (!firstCell->width || !firstCell->height || !bottomRightCell->width || !bottomRightCell->height) { return AVIF_RESULT_NO_CONTENT; } @@ -1388,7 +1674,12 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, } if (hasGainMap) { - AVIF_CHECKRES(avifValidateImageBasicProperties(firstCell->gainMap->image)); + // AVIF supports 16-bit images through sample transforms used as bit depth extensions, + // but this is not implemented for gain maps for now. Stick to at most 12 bits. + AVIF_CHECKERR(firstCell->gainMap->image->depth == 8 || firstCell->gainMap->image->depth == 10 || + firstCell->gainMap->image->depth == 12, + AVIF_RESULT_UNSUPPORTED_DEPTH); + AVIF_CHECKERR(firstCell->gainMap->image->yuvFormat != AVIF_PIXEL_FORMAT_NONE, AVIF_RESULT_NO_YUV_FORMAT_SELECTED); AVIF_CHECKRES(avifValidateGrid(gridCols, gridRows, cellImages, /*validateGainMap=*/AVIF_TRUE, &encoder->diag)); if (firstCell->gainMap->image->colorPrimaries != AVIF_COLOR_PRIMARIES_UNSPECIFIED || firstCell->gainMap->image->transferCharacteristics != AVIF_TRANSFER_CHARACTERISTICS_UNSPECIFIED) { @@ -1576,6 +1867,17 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, } #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (firstCell->depth > 12 && encoder->sampleTransformRecipe != AVIF_SAMPLE_TRANSFORM_NONE) { + // For now, only 16-bit depth is supported. + assert(firstCell->depth == 16); +#if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) + AVIF_CHECKERR(!firstCell->gainMap, AVIF_RESULT_NOT_IMPLEMENTED); // TODO(yguyon): Implement 16-bit HDR +#endif + AVIF_CHECKRES(avifEncoderCreateSampleTransformItems(encoder, gridCols, gridRows, gridWidth, gridHeight, colorItemID)); + } +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + // ----------------------------------------------------------------------- // Create metadata items (Exif, XMP) @@ -1642,7 +1944,9 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, avifEncoderItem * item = &encoder->data->items.item[itemIndex]; if (item->codec) { const avifImage * cellImage = cellImages[item->cellIndex]; + avifImage * cellImagePlaceholder = NULL; // May be used as a temporary, modified cellImage. Left as NULL otherwise. const avifImage * firstCellImage = firstCell; + #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) if (item->itemCategory == AVIF_ITEM_GAIN_MAP) { AVIF_ASSERT_OR_RETURN(cellImage->gainMap && cellImage->gainMap->image); @@ -1651,29 +1955,74 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, firstCellImage = firstCell->gainMap->image; } #endif - avifImage * paddedCellImage = NULL; + if ((cellImage->width != firstCellImage->width) || (cellImage->height != firstCellImage->height)) { - paddedCellImage = avifImageCreateEmpty(); - AVIF_CHECKERR(paddedCellImage, AVIF_RESULT_OUT_OF_MEMORY); - const avifResult result = avifImageCopyAndPad(paddedCellImage, cellImage, firstCellImage->width, firstCellImage->height); + cellImagePlaceholder = avifImageCreateEmpty(); + AVIF_CHECKERR(cellImagePlaceholder, AVIF_RESULT_OUT_OF_MEMORY); + const avifResult result = + avifImageCopyAndPad(cellImagePlaceholder, cellImage, firstCellImage->width, firstCellImage->height); if (result != AVIF_RESULT_OK) { - avifImageDestroy(paddedCellImage); + avifImageDestroy(cellImagePlaceholder); return result; } - cellImage = paddedCellImage; + cellImage = cellImagePlaceholder; } - const int quantizer = (item->itemCategory == AVIF_ITEM_ALPHA) ? encoder->data->quantizerAlpha + + avifBool isAlpha = item->itemCategory == AVIF_ITEM_ALPHA; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + isAlpha = isAlpha || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA; +#endif + int quantizer = isAlpha ? encoder->data->quantizerAlpha #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) - : (item->itemCategory == AVIF_ITEM_GAIN_MAP) ? encoder->data->quantizerGainMap + : (item->itemCategory == AVIF_ITEM_GAIN_MAP) ? encoder->data->quantizerGainMap #endif - : encoder->data->quantizer; + : encoder->data->quantizer; + +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + // Remember original quantizer values in case they change, to reset them afterwards. + int * encoderMinQuantizer = isAlpha ? &encoder->minQuantizerAlpha : &encoder->minQuantizer; + int * encoderMaxQuantizer = isAlpha ? &encoder->maxQuantizerAlpha : &encoder->maxQuantizer; + const int originalMinQuantizer = *encoderMinQuantizer; + const int originalMaxQuantizer = *encoderMaxQuantizer; + + if (encoder->sampleTransformRecipe != AVIF_SAMPLE_TRANSFORM_NONE) { + if (item->itemCategory == AVIF_ITEM_COLOR || item->itemCategory == AVIF_ITEM_ALPHA) { + // Encoding the least significant bits of a sample does not make any sense if the + // other bits are lossily compressed. Encode the most significant bits losslessly. + quantizer = AVIF_QUANTIZER_LOSSLESS; + *encoderMinQuantizer = AVIF_QUANTIZER_LOSSLESS; + *encoderMaxQuantizer = AVIF_QUANTIZER_LOSSLESS; + if (!avifEncoderDetectChanges(encoder, &encoderChanges)) { + assert(AVIF_FALSE); + } + } + + // Replace cellImage by the first or second input to the AVIF_ITEM_SAMPLE_TRANSFORM derived image item. + const avifBool itemWillBeEncodedLosslessly = (quantizer == AVIF_QUANTIZER_LOSSLESS); + avifImage * sampleTransformedImage = NULL; + const avifResult result = + avifEncoderCreateSampleTransformInputImage(encoder, item, itemWillBeEncodedLosslessly, cellImage, &sampleTransformedImage); + if (cellImagePlaceholder) { + avifImageDestroy(cellImagePlaceholder); // Replaced by sampleTransformedImage. + } + if (result != AVIF_RESULT_OK) { + if (sampleTransformedImage) { + avifImageDestroy(sampleTransformedImage); + } + return result; + } + cellImagePlaceholder = sampleTransformedImage; // Transfer ownership. + cellImage = cellImagePlaceholder; + } +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + // If alpha channel is present, set disableLaggedOutput to AVIF_TRUE. If the encoder supports it, this enables // avifEncoderDataShouldForceKeyframeForAlpha to force a keyframe in the alpha channel whenever a keyframe has been // encoded in the color channel for animated images. avifResult encodeResult = item->codec->encodeImage(item->codec, encoder, cellImage, - item->itemCategory == AVIF_ITEM_ALPHA, + isAlpha, encoder->data->tileRowsLog2, encoder->data->tileColsLog2, quantizer, @@ -1681,8 +2030,16 @@ static avifResult avifEncoderAddImageInternal(avifEncoder * encoder, /*disableLaggedOutput=*/encoder->data->alphaPresent, addImageFlags, item->encodeOutput); - if (paddedCellImage) { - avifImageDestroy(paddedCellImage); +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + // Revert quality settings if they changed. + if (*encoderMinQuantizer != originalMinQuantizer || *encoderMaxQuantizer != originalMaxQuantizer) { + avifEncoderBackupSettings(encoder); // Remember last encoding settings for next avifEncoderDetectChanges(). + *encoderMinQuantizer = originalMinQuantizer; + *encoderMaxQuantizer = originalMaxQuantizer; + } +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM + if (cellImagePlaceholder) { + avifImageDestroy(cellImagePlaceholder); } if (encodeResult == AVIF_RESULT_UNKNOWN_ERROR) { encodeResult = avifGetErrorForItemCategory(item->itemCategory); @@ -1780,7 +2137,11 @@ static avifResult avifEncoderWriteMediaDataBox(avifEncoder * encoder, // only process metadata (XMP/Exif) payloads when metadataPass is true continue; } - avifBool isAlphaOrGainMap = item->itemCategory == AVIF_ITEM_ALPHA; + avifBool isAlpha = item->itemCategory == AVIF_ITEM_ALPHA; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + isAlpha = isAlpha || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA; +#endif + avifBool isAlphaOrGainMap = isAlpha; #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP) isAlphaOrGainMap = isAlphaOrGainMap || item->itemCategory == AVIF_ITEM_GAIN_MAP; #endif @@ -1794,8 +2155,7 @@ static avifResult avifEncoderWriteMediaDataBox(avifEncoder * encoder, // We always interleave all AV1 items for layered images. AVIF_ASSERT_OR_RETURN(item->encodeOutput->samples.count == item->mdatFixups.count); - avifEncoderItemReference * ref = (item->itemCategory == AVIF_ITEM_ALPHA) ? avifArrayPush(layeredAlphaItems) - : avifArrayPush(layeredColorItems); + avifEncoderItemReference * ref = isAlpha ? avifArrayPush(layeredAlphaItems) : avifArrayPush(layeredColorItems); AVIF_CHECKERR(ref != NULL, AVIF_RESULT_OUT_OF_MEMORY); *ref = item; continue; @@ -1820,7 +2180,7 @@ static avifResult avifEncoderWriteMediaDataBox(avifEncoder * encoder, avifEncodeSample * sample = &item->encodeOutput->samples.sample[sampleIndex]; AVIF_CHECKRES(avifRWStreamWrite(s, sample->data.data, sample->data.size)); - if (item->itemCategory == AVIF_ITEM_ALPHA) { + if (isAlpha) { encoder->ioStats.alphaOBUSize += sample->data.size; } else if (item->itemCategory == AVIF_ITEM_COLOR) { encoder->ioStats.colorOBUSize += sample->data.size; @@ -2205,9 +2565,10 @@ static avifResult avifRWStreamWriteProperties(avifItemPropertyDedup * const dedu for (uint32_t itemIndex = 0; itemIndex < encoder->data->items.count; ++itemIndex) { avifEncoderItem * item = &encoder->data->items.item[itemIndex]; const avifBool isGrid = (item->gridCols > 0); - const avifBool isToneMappedImage = !memcmp(item->type, "tmap", 4); + const avifBool isToneMappedImage = !memcmp(item->type, "tmap", 4); // Derived image item, not a coded item. + const avifBool isSampleTransformImage = !memcmp(item->type, "sato", 4); // Derived image item, not a coded item. memset(&item->ipma, 0, sizeof(item->ipma)); - if (!item->codec && !isGrid && !isToneMappedImage) { + if (!item->codec && !isGrid && !isToneMappedImage && !isSampleTransformImage) { // No ipma to write for this item continue; } @@ -2254,7 +2615,7 @@ static avifResult avifRWStreamWriteProperties(avifItemPropertyDedup * const dedu imageHeight = item->gridHeight; } - // Properties all image items need + // Properties all image items need (coded and derived) // ispe = image spatial extent (width, height) avifItemPropertyDedupStart(dedup); avifBoxMarker ispe; @@ -2272,15 +2633,38 @@ static avifResult avifRWStreamWriteProperties(avifItemPropertyDedup * const dedu hasPixi = AVIF_FALSE; } #endif + avifBool isAlpha = item->itemCategory == AVIF_ITEM_ALPHA; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + isAlpha = isAlpha || item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA; +#endif + uint8_t depth = (uint8_t)itemMetadata->depth; +#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + if (encoder->sampleTransformRecipe != AVIF_SAMPLE_TRANSFORM_NONE) { + if (item->itemCategory == AVIF_ITEM_SAMPLE_TRANSFORM) { + assert(depth == 16); + } else if (encoder->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B) { + depth = 8; + } else { + assert(encoder->sampleTransformRecipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B); + if (item->itemCategory == AVIF_ITEM_COLOR || item->itemCategory == AVIF_ITEM_ALPHA) { + depth = 12; + } else { + assert(item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR || + item->itemCategory == AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA); + depth = 8; // Will be shifted to 4-bit samples at decoding. + } + } + } + assert(isSampleTransformImage == (item->itemCategory == AVIF_ITEM_SAMPLE_TRANSFORM)); +#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM if (hasPixi) { avifItemPropertyDedupStart(dedup); - uint8_t channelCount = - (item->itemCategory == AVIF_ITEM_ALPHA || (itemMetadata->yuvFormat == AVIF_PIXEL_FORMAT_YUV400)) ? 1 : 3; + uint8_t channelCount = (isAlpha || (itemMetadata->yuvFormat == AVIF_PIXEL_FORMAT_YUV400)) ? 1 : 3; avifBoxMarker pixi; AVIF_CHECKRES(avifRWStreamWriteFullBox(&dedup->s, "pixi", AVIF_BOX_SIZE_TBD, 0, 0, &pixi)); AVIF_CHECKRES(avifRWStreamWriteU8(&dedup->s, channelCount)); // unsigned int (8) num_channels; for (uint8_t chan = 0; chan < channelCount; ++chan) { - AVIF_CHECKRES(avifRWStreamWriteU8(&dedup->s, (uint8_t)itemMetadata->depth)); // unsigned int (8) bits_per_channel; + AVIF_CHECKRES(avifRWStreamWriteU8(&dedup->s, depth)); // unsigned int (8) bits_per_channel; } avifRWStreamFinishBox(&dedup->s, pixi); AVIF_CHECKRES(avifItemPropertyDedupFinish(dedup, s, &item->ipma, AVIF_FALSE)); @@ -2293,7 +2677,7 @@ static avifResult avifRWStreamWriteProperties(avifItemPropertyDedup * const dedu AVIF_CHECKRES(avifItemPropertyDedupFinish(dedup, s, &item->ipma, AVIF_TRUE)); } - if (item->itemCategory == AVIF_ITEM_ALPHA) { + if (isAlpha) { // Alpha specific properties avifItemPropertyDedupStart(dedup); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4ec1593530..8d55ee311f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -100,6 +100,10 @@ if(AVIF_ENABLE_GTEST) endif() if(AVIF_ENABLE_GTEST) + if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + add_avif_gtest_with_data(avif16bittest) + endif() + add_avif_gtest(avifallocationtest) add_avif_gtest_with_data(avifalphanoispetest) add_avif_gtest(avifalphapremtest) @@ -363,6 +367,9 @@ if(AVIF_CODEC_AVM_ENABLED) set_tests_properties(avifjpeggainmaptest PROPERTIES DISABLED True) endif() endif() + if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM) + set_tests_properties(avif16bittest PROPERTIES DISABLED True) + endif() endif() if(AVIF_BUILD_APPS) diff --git a/tests/gtest/avif16bittest.cc b/tests/gtest/avif16bittest.cc new file mode 100644 index 0000000000..57e37b4279 --- /dev/null +++ b/tests/gtest/avif16bittest.cc @@ -0,0 +1,137 @@ +// Copyright 2022 Google LLC +// SPDX-License-Identifier: BSD-2-Clause + +#include + +#include "avif/avif.h" +#include "avif/avif_cxx.h" +#include "avif/internal.h" +#include "aviftest_helpers.h" +#include "gtest/gtest.h" + +namespace avif { +namespace { + +//------------------------------------------------------------------------------ + +// Used to pass the data folder path to the GoogleTest suites. +const char* data_path = nullptr; + +class SampleTransformTest + : public testing::TestWithParam< + std::tuple> {}; + +//------------------------------------------------------------------------------ + +TEST_P(SampleTransformTest, Avif16bit) { + const avifSampleTransformRecipe recipe = std::get<0>(GetParam()); + const avifPixelFormat yuv_format = std::get<1>(GetParam()); + const bool create_alpha = std::get<2>(GetParam()); + const int quality = std::get<3>(GetParam()); + + const ImagePtr image = testutil::ReadImage( + data_path, "weld_16bit.png", yuv_format, /*requested_depth=*/16); + ASSERT_NE(image, nullptr); + if (create_alpha && !image->alphaPlane) { + // Simulate alpha plane with a view on luma. + image->alphaPlane = image->yuvPlanes[AVIF_CHAN_Y]; + image->alphaRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; + image->imageOwnsAlphaPlane = false; + } + + EncoderPtr encoder(avifEncoderCreate()); + ASSERT_NE(encoder, nullptr); + encoder->speed = AVIF_SPEED_FASTEST; + encoder->quality = quality; + encoder->qualityAlpha = quality; + encoder->sampleTransformRecipe = recipe; + testutil::AvifRwData encoded; + ASSERT_EQ(avifEncoderWrite(encoder.get(), image.get(), &encoded), + AVIF_RESULT_OK); + const ImagePtr decoded = testutil::Decode(encoded.data, encoded.size); + ASSERT_NE(decoded, nullptr); + + ASSERT_EQ(image->depth, decoded->depth); + ASSERT_EQ(image->width, decoded->width); + ASSERT_EQ(image->height, decoded->height); + + EXPECT_GE(testutil::GetPsnr(*image, *decoded), + (quality == AVIF_QUALITY_LOSSLESS) ? 99.0 : 15.0); + + // Replace all 'sato' box types by "zzzz" garbage. This simulates an old + // decoder that does not recognize the Sample Transform feature. + for (size_t i = 0; i + 4 <= encoded.size; ++i) { + if (!std::memcmp(&encoded.data[i], "sato", 4)) { + std::memcpy(&encoded.data[i], "zzzz", 4); + } + } + const ImagePtr decoded_no_sato = testutil::Decode(encoded.data, encoded.size); + ASSERT_NE(decoded_no_sato, nullptr); + // Only the most significant bits of each sample can be retrieved. + // They should be encoded losslessly no matter the quantizer settings. + ImagePtr image_no_sato = testutil::CreateImage( + static_cast(image->width), static_cast(image->height), + static_cast(decoded_no_sato->depth), image->yuvFormat, + image->alphaPlane ? AVIF_PLANES_ALL : AVIF_PLANES_YUV, image->yuvRange); + ASSERT_NE(image_no_sato, nullptr); + + const uint32_t shift = + recipe == AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B ? 8 : 4; + // image_8msb = image >> shift + ASSERT_EQ( + avifImageTransformConstantAndImageSamples( + image_no_sato.get(), AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32, + /*leftOperand=*/1 << shift, AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED, + /*rightOperand=*/image.get(), AVIF_PLANES_ALL), + AVIF_RESULT_OK); + EXPECT_TRUE(testutil::AreImagesEqual(*image_no_sato, *decoded_no_sato)); +} + +//------------------------------------------------------------------------------ + +INSTANTIATE_TEST_SUITE_P( + Formats, SampleTransformTest, + testing::Combine( + testing::Values(AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B), + testing::Values(AVIF_PIXEL_FORMAT_YUV444, AVIF_PIXEL_FORMAT_YUV420, + AVIF_PIXEL_FORMAT_YUV400), + /*create_alpha=*/testing::Values(false), + /*quality=*/ + testing::Values(AVIF_QUALITY_DEFAULT))); + +INSTANTIATE_TEST_SUITE_P( + BitDepthExtensions, SampleTransformTest, + testing::Combine( + testing::Values(AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B, + AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B), + testing::Values(AVIF_PIXEL_FORMAT_YUV444), + /*create_alpha=*/testing::Values(false), + /*quality=*/ + testing::Values(AVIF_QUALITY_LOSSLESS))); + +INSTANTIATE_TEST_SUITE_P( + Alpha, SampleTransformTest, + testing::Combine( + testing::Values(AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B), + testing::Values(AVIF_PIXEL_FORMAT_YUV444), + /*create_alpha=*/testing::Values(true), + /*quality=*/ + testing::Values(AVIF_QUALITY_LOSSLESS))); + +// TODO(yguyon): Test grids with bit depth extensions. + +} // namespace +} // namespace avif + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc != 2) { + std::cerr << "There must be exactly one argument containing the path to " + "the test data folder" + << std::endl; + return 1; + } + avif::data_path = argv[1]; + return RUN_ALL_TESTS(); +}