Experimental AVIF Sample Transform bit depth ext

Allow encoding and decoding images of more than 12 bits, the limit of
the AV1 format. This commit experiments 16-bit AVIF images, made of
one primary lossless 8-bit image and one hidden lossy/lossless bit
depth extension 8-bit image. The two are combined thanks to a Sample
Transform derived image item. Grids and alpha are supported.

Warning: This feature is experimental and not covered by the current
         AVIF specification.

.github/workflows/ci-unix-static.yml

@@ -109,6 +109,7 @@ jobs:
         -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON
         -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON
         -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON
         -DAVIF_ENABLE_WERROR=ON
     - name: Build libavif (ninja)
       working-directory: ./build

.github/workflows/ci-windows.yml

@@ -125,7 +125,9 @@ jobs:
         -DAVIF_BUILD_TESTS=ON -DAVIF_ENABLE_GTEST=ON -DAVIF_LOCAL_GTEST=ON
         -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON
         -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON
-        -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON -DAVIF_ENABLE_WERROR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON
+        -DAVIF_ENABLE_WERROR=ON
     - name: Build libavif (ninja)
       working-directory: ./build
       run: ninja

CHANGELOG.md

@@ -51,6 +51,8 @@ The changes are relative to the previous release, unless the baseline is specifi
 * Require libyuv by default (but it can still be disabled with
   -DAVIF_LIBYUV=OFF).
 * Add avifdec --icc flag to override the output color profile.
+* Add experimental API for reading and writing 16-bit AVIF files behind the
+  compilation flag AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM.
 
 ### Changed since 1.0.0
 * Update aom.cmd: v3.8.1

CMakeLists.txt

@@ -42,6 +42,7 @@ option(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP
        "Enable experimental gain map code (for HDR images that look good both on HDR and SDR displays)" OFF
 )
 option(AVIF_ENABLE_EXPERIMENTAL_AVIR "Enable experimental reduced header" OFF)
+option(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM "Enable experimental sample transform code" OFF)
 
 set(AVIF_PKG_CONFIG_EXTRA_LIBS_PRIVATE "")
 set(AVIF_PKG_CONFIG_EXTRA_REQUIRES_PRIVATE "")
@@ -328,6 +329,10 @@ if(AVIF_ENABLE_EXPERIMENTAL_AVIR)
     add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_AVIR)
 endif()
 
+if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+endif()
+
 set(AVIF_SRCS
     src/alpha.c
     src/avif.c
@@ -351,6 +356,9 @@ set(AVIF_SRCS
 if(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     list(APPEND AVIF_SRCS src/gainmap.c)
 endif()
+if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    list(APPEND AVIF_SRCS src/sampletransform.c)
+endif()
 
 if(AVIF_ENABLE_COMPLIANCE_WARDEN)
     if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/ext/ComplianceWarden")

include/avif/avif.h

@@ -192,6 +192,10 @@ typedef enum AVIF_NODISCARD avifResult
     AVIF_RESULT_DECODE_GAIN_MAP_FAILED = 31,
     AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE = 32,
 #endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED = 33,
+    AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED = 34,
+#endif
 
     // Kept for backward compatibility; please use the symbols above instead.
     AVIF_RESULT_NO_AV1_ITEMS_FOUND = AVIF_RESULT_MISSING_IMAGE_ITEM
@@ -706,6 +710,43 @@ AVIF_NODISCARD AVIF_API avifBool avifGainMapMetadataFractionsToDouble(avifGainMa
 
 #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP
 
+// ---------------------------------------------------------------------------
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+// Sample Transforms are a HIGHLY EXPERIMENTAL FEATURE. The format might still
+// change and images containing a sample transform item encoded with the current
+// version of libavif might not decode with a future version of libavif.
+// Use are your own risk.
+// This is based on a proposal from the Alliance for Open Media.
+
+typedef enum avifSampleTransformRecipe
+{
+    AVIF_SAMPLE_TRANSFORM_NONE,
+    // Encode the 8 most significant bits of each input image sample losslessly
+    // into one base image. The remaining least 8 significant bits are encoded
+    // in a separate hidden image item. The two are combined at decoding into
+    // one image with the same bit depth as the original image.
+    // It is backward compatible in the sense that only the base image may be
+    // decoded (ignoring the hidden image item), leading to a valid image but
+    // with precision loss (16-bit samples truncated to the 8 most significant
+    // bits).
+    // Same as AVIF_SAMPLE_TRANSFORM_NONE on input images with a depth of at
+    // most 12 bits. Only available for 16-bit input images.
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B,
+    // Encode the 12 most significant bits of each input image sample losslessly
+    // into one base image. The remaining least 4 significant bits are encoded
+    // in a separate hidden image item. The two are combined at decoding into
+    // one image with the same bit depth as the original image.
+    // It is backward compatible in the sense that only the base image may be
+    // decoded (ignoring the hidden image item), leading to a valid image but
+    // with precision loss (16-bit samples truncated to the 12 most significant
+    // bits).
+    // Same as AVIF_SAMPLE_TRANSFORM_NONE on input images with a depth of at
+    // most 12 bits. Only available for 16-bit input images.
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B
+} avifSampleTransformRecipe;
+#endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP
+
 // ---------------------------------------------------------------------------
 // avifImage
 
@@ -1274,6 +1315,14 @@ typedef struct avifDecoder
     // Can be useful to decode the gain map image only.
     avifBool ignoreColorAndAlpha;
 #endif
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    // Input
+    // Optional sample transforms used as bit depth extensions will be ignored if the base image has
+    // a bit depth of at least preferredBitDepth. Otherwise, the output image will be the
+    // combination of the base image and the sample transform item, if any. Default is 0.
+    uint32_t preferredBitDepth; // TODO(yguyon): Implement
+#endif
 } avifDecoder;
 
 // Returns NULL in case of memory allocation failure.
@@ -1458,6 +1507,10 @@ typedef struct avifEncoder
 #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     int qualityGainMap; // changeable encoder setting
 #endif
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    avifSampleTransformRecipe sampleTransformRecipe; // See the avifSampleTransformRecipe description.
+#endif
 } avifEncoder;
 
 // avifEncoderCreate() returns NULL if a memory allocation failed.

include/avif/internal.h

@@ -156,6 +156,62 @@ void avifImageCopyNoAlloc(avifImage * dstImage, const avifImage * srcImage);
 // Ignores the gainMap field (which exists only if AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP is defined).
 void avifImageCopySamples(avifImage * dstImage, const avifImage * srcImage, avifPlanesFlags planes);
 
+// ---------------------------------------------------------------------------
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+// Mapping used in the coding of Sample Transform metadata.
+typedef enum avifSampleTransformIntermediateBitDepth
+{
+    AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_8 = 0,  // Signed 8-bit.
+    AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_16 = 1, // Signed 16-bit.
+    AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_32 = 2, // Signed 32-bit.
+    AVIF_SAMPLE_TRANSFORM_INTERMEDIATE_BIT_DEPTH_64 = 3  // Signed 64-bit.
+} avifSampleTransformIntermediateBitDepth;
+
+// Meaning of an operation in Sample Transform metadata.
+// For a constant_operator, L is the constant (left operand) and R is the sample (right operand).
+// For an operator, L is the intermediate overall result (left operand)
+// and R is the last sub-expression result (right operand).
+typedef enum avifSampleTransformOperation
+{
+    AVIF_SAMPLE_TRANSFORM_SUM = 0,             // S = L + R
+    AVIF_SAMPLE_TRANSFORM_PRODUCT = 1,         // S = L * R
+    AVIF_SAMPLE_TRANSFORM_DIVIDE = 2,          // S = floor(L / R)
+    AVIF_SAMPLE_TRANSFORM_DIVIDE_REVERSED = 3, // S = floor(R / L)
+    AVIF_SAMPLE_TRANSFORM_POW = 4,             // S = pow(L, R)
+    AVIF_SAMPLE_TRANSFORM_POW_REVERSED = 5,    // S = pow(R, L)
+    AVIF_SAMPLE_TRANSFORM_LOG = 6,             // S = log(L, R)
+    AVIF_SAMPLE_TRANSFORM_LOG_REVERSED = 7,    // S = log(R, L)
+    AVIF_SAMPLE_TRANSFORM_AND = 8,             // S = L & R
+    AVIF_SAMPLE_TRANSFORM_OR = 9,              // S = L | R
+    AVIF_SAMPLE_TRANSFORM_XOR = 10,            // S = L ^ R
+    AVIF_SAMPLE_TRANSFORM_NOR = 11,            // S = !(L | R)
+    AVIF_SAMPLE_TRANSFORM_MIN = 12,            // S = min(L, R)
+    AVIF_SAMPLE_TRANSFORM_MAX = 13             // S = max(L, R)
+} avifSampleTransformOperation;
+
+// Performs the following for each sample in the selected planes of the result image:
+//   result = leftOperand (operation) rightOperand
+// result, leftOperand and rightOperand must be allocated and have the same planes and dimensions.
+// result can be point to leftOperand and/or rightOperand (in-place allowed).
+avifResult avifImageTransformImageAndImageSamples(avifImage * result,
+                                                  avifSampleTransformIntermediateBitDepth intermediateBitDepth,
+                                                  const avifImage * leftOperand,
+                                                  avifSampleTransformOperation operation,
+                                                  const avifImage * rightOperand,
+                                                  avifPlanesFlags planes);
+// Same but leftOperand is a constant instead of a sample.
+avifResult avifImageTransformConstantAndImageSamples(avifImage * result,
+                                                     avifSampleTransformIntermediateBitDepth intermediateBitDepth,
+                                                     int32_t leftOperand,
+                                                     avifSampleTransformOperation operation,
+                                                     const avifImage * rightOperand,
+                                                     avifPlanesFlags planes);
+#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM
+
+// ---------------------------------------------------------------------------
+// Alpha
+
 typedef struct avifAlphaParams
 {
     uint32_t width;
@@ -317,6 +373,11 @@ typedef enum avifItemCategory
     AVIF_ITEM_ALPHA,
 #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     AVIF_ITEM_GAIN_MAP,
+#endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    AVIF_ITEM_SAMPLE_TRANSFORM,          // Sample Transform derived image item 'sato'.
+    AVIF_ITEM_BIT_DEPTH_EXTENSION_COLOR, // Second input image item of AVIF_ITEM_SAMPLE_TRANSFORM. First is AVIF_ITEM_COLOR.
+    AVIF_ITEM_BIT_DEPTH_EXTENSION_ALPHA, // Auxiliary image item of AVIF_ITEM_SAMPLE_TRANSFORM_COLOR.
 #endif
     AVIF_ITEM_CATEGORY_COUNT
 } avifItemCategory;

src/avif.c

@@ -108,6 +108,10 @@ const char * avifResultToString(avifResult result)
         case AVIF_RESULT_ENCODE_GAIN_MAP_FAILED:        return "Encoding of gain map planes failed";
         case AVIF_RESULT_DECODE_GAIN_MAP_FAILED:        return "Decoding of gain map planes failed";
         case AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE:     return "Invalid tone mapped image item";
+#endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+        case AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED: return "Encoding of sample transformed image failed";
+        case AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED: return "Decoding of sample transformed image failed";
 #endif
         case AVIF_RESULT_UNKNOWN_ERROR:
         default: