[CVE-2017-16803] smacker: add sanity check for length in smacker_decode_tree()

Bug-Id: 1098
Cc: libav-stable@libav.org
Signed-off-by: Sean McGovern <gseanmcg@gmail.com>
Change-Id: I2e2236de12a0f6dead47f671907920b4193ff9a0

diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index e3e54752a747cb038da4f369d8cc5ad635573f21..6a327af4a7a1b012988ff9abc401150169a954d7 100644 (file)
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -42,7 +42,7 @@
 
 #define SMKTREE_BITS 9
 #define SMK_NODE 0x80000000
-
+#define SMKTREE_DECODE_MAX_RECURSION 32
 
 typedef struct SmackVContext {
     AVCodecContext *avctx;
@@ -95,6 +95,11 @@ enum SmkBlockTypes {
  */
 static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t prefix, int length)
 {
+    if (length > SMKTREE_DECODE_MAX_RECURSION) {
+      av_log(NULL, AV_LOG_ERROR, "Maximum tree recursion level exceeded.\n");
+      return AVERROR_INVALIDDATA;
+    }
+
     if(!get_bits1(gb)){ //Leaf
         if(hc->current >= 256){
             av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n");