Validate the |border| parameter earlier.

vpx_realloc_frame_buffer() should validate the |border| parameter
earlier, before it allocates the buffer and preferrably before it uses
|border|.

This backports libaom commit 2860b3ae8b764bdfa2b8c7a06df2673e907b993f:
https://aomedia-review.googlesource.com/c/aom/+/74324

Change-Id: Ib9d59d74e27430ccb1e83c6ad5424aff9672c989

diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index 287a5a0..c66d885 100644 (file)
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -154,6 +154,13 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
   if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;
 #endif
 
+  /* Only support allocating buffers that have a border that's a multiple
+   * of 32. The border restriction is required to get 16-byte alignment of
+   * the start of the chroma rows without introducing an arbitrary gap
+   * between planes, which would break the semantics of things like
+   * vpx_img_set_rect(). */
+  if (border & 0x1f) return -3;
+
   if (ybf) {
     const int vp9_byte_align = (byte_alignment == 0) ? 1 : byte_alignment;
     const int aligned_width = (width + 7) & ~7;
@@ -223,13 +230,6 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
       memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz);
     }
 
-    /* Only support allocating buffers that have a border that's a multiple
-     * of 32. The border restriction is required to get 16-byte alignment of
-     * the start of the chroma rows without introducing an arbitrary gap
-     * between planes, which would break the semantics of things like
-     * vpx_img_set_rect(). */
-    if (border & 0x1f) return -3;
-
     ybf->y_crop_width = width;
     ybf->y_crop_height = height;
     ybf->y_width = aligned_width;