Skip to content

Commit

Permalink
alpha.c: Fix overflows in multiply with rowBytes
Browse files Browse the repository at this point in the history 
Replace multiplication with rgb.rowBytes with addition of rgb.rowBytes
to a row pointer in a loop.

Related to AOMediaCodec#2271.
  • Loading branch information
@wantehchang
wantehchang committed Aug 1, 2024
1 parent 622ec93 commit 39cf485
Showing 1 changed file with 90 additions and 48 deletions.
138 changes: 90 additions & 48 deletions src/alpha.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,26 @@ void avifFillAlpha(const avifAlphaParams * params)
{
if (params->dstDepth > 8) {
const uint16_t maxChannel = (uint16_t)((1 << params->dstDepth) - 1);
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
*((uint16_t *)dstRow) = maxChannel;
dstRow += params->dstPixelBytes;
*((uint16_t *)dstPixel) = maxChannel;
dstPixel += params->dstPixelBytes;
}
dstRow += params->dstRowBytes;
}
} else {
// In this case, (1 << params->dstDepth) - 1 is always equal to 255.
const uint8_t maxChannel = 255;
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
*dstRow = maxChannel;
dstRow += params->dstPixelBytes;
*dstPixel = maxChannel;
dstPixel += params->dstPixelBytes;
}
dstRow += params->dstRowBytes;
}
}
}
Expand All @@ -43,22 +47,34 @@ void avifReformatAlpha(const avifAlphaParams * params)
if (params->srcDepth > 8) {
// no depth rescale, uint16_t -> uint16_t

const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
const uint8_t * srcPixel = srcRow;
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
*((uint16_t *)&dstRow[i * params->dstPixelBytes]) = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]);
*((uint16_t *)dstPixel) = *((const uint16_t *)srcPixel);
srcPixel += params->srcPixelBytes;
dstPixel += params->dstPixelBytes;
}
srcRow += params->srcRowBytes;
dstRow += params->dstRowBytes;
}
} else {
// no depth rescale, uint8_t -> uint8_t

const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
const uint8_t * srcPixel = srcRow;
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
dstRow[i * params->dstPixelBytes] = srcRow[i * params->srcPixelBytes];
*dstPixel = *srcPixel;
srcPixel += params->srcPixelBytes;
dstPixel += params->dstPixelBytes;
}
srcRow += params->srcRowBytes;
dstRow += params->dstRowBytes;
}
}
} else {
Expand All @@ -68,47 +84,65 @@ void avifReformatAlpha(const avifAlphaParams * params)
if (params->dstDepth > 8) {
// depth rescale, uint16_t -> uint16_t

const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
const uint8_t * srcPixel = srcRow;
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
int srcAlpha = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]);
int srcAlpha = *((const uint16_t *)srcPixel);
float alphaF = (float)srcAlpha / srcMaxChannelF;
int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF));
dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel);
*((uint16_t *)&dstRow[i * params->dstPixelBytes]) = (uint16_t)dstAlpha;
*((uint16_t *)dstPixel) = (uint16_t)dstAlpha;
srcPixel += params->srcPixelBytes;
dstPixel += params->dstPixelBytes;
}
srcRow += params->srcRowBytes;
dstRow += params->dstRowBytes;
}
} else {
// depth rescale, uint16_t -> uint8_t

const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
const uint8_t * srcPixel = srcRow;
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
int srcAlpha = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]);
int srcAlpha = *((const uint16_t *)srcPixel);
float alphaF = (float)srcAlpha / srcMaxChannelF;
int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF));
dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel);
dstRow[i * params->dstPixelBytes] = (uint8_t)dstAlpha;
*dstPixel = (uint8_t)dstAlpha;
srcPixel += params->srcPixelBytes;
dstPixel += params->dstPixelBytes;
}
srcRow += params->srcRowBytes;
dstRow += params->dstRowBytes;
}
}
} else {
// If (srcDepth == 8), dstDepth must be >8 otherwise we'd be in the (params->srcDepth == params->dstDepth) block above.
assert(params->dstDepth > 8);

// depth rescale, uint8_t -> uint16_t
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes];
for (uint32_t j = 0; j < params->height; ++j) {
const uint8_t * srcRow = &params->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)];
uint8_t * dstRow = &params->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)];
const uint8_t * srcPixel = srcRow;
uint8_t * dstPixel = dstRow;
for (uint32_t i = 0; i < params->width; ++i) {
int srcAlpha = srcRow[i * params->srcPixelBytes];
int srcAlpha = *srcPixel;
float alphaF = (float)srcAlpha / srcMaxChannelF;
int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF));
dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel);
*((uint16_t *)&dstRow[i * params->dstPixelBytes]) = (uint16_t)dstAlpha;
*((uint16_t *)dstPixel) = (uint16_t)dstAlpha;
srcPixel += params->srcPixelBytes;
dstPixel += params->dstPixelBytes;
}
srcRow += params->srcRowBytes;
dstRow += params->dstRowBytes;
}
}
}
Expand Down Expand Up @@ -138,14 +172,13 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb)

if (rgb->depth > 8) {
if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint16_t * pixel = (uint16_t *)row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint16_t * pixel = (uint16_t *)&row[i * 8];
uint16_t a = pixel[3];
if (a >= max) {
// opaque is no-op
continue;
} else if (a == 0) {
// result must be zero
pixel[0] = 0;
Expand All @@ -157,16 +190,17 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb)
pixel[1] = (uint16_t)avifRoundf((float)pixel[1] * (float)a / maxF);
pixel[2] = (uint16_t)avifRoundf((float)pixel[2] * (float)a / maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
} else {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint16_t * pixel = (uint16_t *)row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint16_t * pixel = (uint16_t *)&row[i * 8];
uint16_t a = pixel[0];
if (a >= max) {
continue;
} else if (a == 0) {
pixel[1] = 0;
pixel[2] = 0;
Expand All @@ -176,19 +210,20 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb)
pixel[2] = (uint16_t)avifRoundf((float)pixel[2] * (float)a / maxF);
pixel[3] = (uint16_t)avifRoundf((float)pixel[3] * (float)a / maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
}
} else {
if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint8_t * pixel = row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint8_t * pixel = &row[i * 4];
uint8_t a = pixel[3];
// uint8_t can't exceed 255
if (a == max) {
continue;
} else if (a == 0) {
pixel[0] = 0;
pixel[1] = 0;
Expand All @@ -198,16 +233,17 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb)
pixel[1] = (uint8_t)avifRoundf((float)pixel[1] * (float)a / maxF);
pixel[2] = (uint8_t)avifRoundf((float)pixel[2] * (float)a / maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
} else {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint8_t * pixel = row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint8_t * pixel = &row[i * 4];
uint8_t a = pixel[0];
if (a == max) {
continue;
} else if (a == 0) {
pixel[1] = 0;
pixel[2] = 0;
Expand All @@ -217,7 +253,9 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb)
pixel[2] = (uint8_t)avifRoundf((float)pixel[2] * (float)a / maxF);
pixel[3] = (uint8_t)avifRoundf((float)pixel[3] * (float)a / maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
}
}
Expand Down Expand Up @@ -249,14 +287,13 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb)

if (rgb->depth > 8) {
if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint16_t * pixel = (uint16_t *)row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint16_t * pixel = (uint16_t *)&row[i * 8];
uint16_t a = pixel[3];
if (a >= max) {
// opaque is no-op
continue;
} else if (a == 0) {
// prevent division by zero
pixel[0] = 0;
Expand All @@ -270,16 +307,17 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb)
pixel[1] = (uint16_t)AVIF_MIN(c2, maxF);
pixel[2] = (uint16_t)AVIF_MIN(c3, maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
} else {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint16_t * pixel = (uint16_t *)row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint16_t * pixel = (uint16_t *)&row[i * 8];
uint16_t a = pixel[0];
if (a >= max) {
continue;
} else if (a == 0) {
pixel[1] = 0;
pixel[2] = 0;
Expand All @@ -292,18 +330,19 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb)
pixel[2] = (uint16_t)AVIF_MIN(c2, maxF);
pixel[3] = (uint16_t)AVIF_MIN(c3, maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
}
} else {
if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint8_t * pixel = row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint8_t * pixel = &row[i * 4];
uint8_t a = pixel[3];
if (a == max) {
continue;
} else if (a == 0) {
pixel[0] = 0;
pixel[1] = 0;
Expand All @@ -316,16 +355,17 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb)
pixel[1] = (uint8_t)AVIF_MIN(c2, maxF);
pixel[2] = (uint8_t)AVIF_MIN(c3, maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
} else {
uint8_t * row = rgb->pixels;
for (uint32_t j = 0; j < rgb->height; ++j) {
uint8_t * row = &rgb->pixels[j * rgb->rowBytes];
uint8_t * pixel = row;
for (uint32_t i = 0; i < rgb->width; ++i) {
uint8_t * pixel = &row[i * 4];
uint8_t a = pixel[0];
if (a == max) {
continue;
} else if (a == 0) {
pixel[1] = 0;
pixel[2] = 0;
Expand All @@ -338,7 +378,9 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb)
pixel[2] = (uint8_t)AVIF_MIN(c2, maxF);
pixel[3] = (uint8_t)AVIF_MIN(c3, maxF);
}
pixel += 4;
}
row += rgb->rowBytes;
}
}
}
Expand Down

0 comments on commit 39cf485

Please sign in to comment.