From: James Zern <jzern@google.com>
Date: Tue, 5 Mar 2013 20:02:28 +0000 (-0800)
Subject: vp8: clamp probability values
X-Git-Tag: v1.3.0~1168^2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=215084b31c3e5683c75a558911172d8f34661333;p=platform%2Fupstream%2Flibvpx.git

vp8: clamp probability values

fixes out of bounds reads on vp8_prob_cost table.
present since:
217591f Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591f

Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd
---

diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index e666b6c..8f94171 100644
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -980,6 +980,12 @@ void vp8_calc_ref_frame_costs(int *ref_frame_cost,
                               int prob_garf
                              )
 {
+    assert(prob_intra >= 0);
+    assert(prob_intra <= 255);
+    assert(prob_last >= 0);
+    assert(prob_last <= 255);
+    assert(prob_garf >= 0);
+    assert(prob_garf <= 255);
     ref_frame_cost[INTRA_FRAME]   = vp8_cost_zero(prob_intra);
     ref_frame_cost[LAST_FRAME]    = vp8_cost_one(prob_intra)
                                     + vp8_cost_zero(prob_last);
diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 92f9818..555a9e4 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -2816,6 +2816,8 @@ static void update_rd_ref_frame_probs(VP8_COMP *cpi)
         if (cpi->common.refresh_alt_ref_frame)
         {
             cpi->prob_intra_coded += 40;
+            if (cpi->prob_intra_coded > 255)
+                cpi->prob_intra_coded = 255;
             cpi->prob_last_coded = 200;
             cpi->prob_gf_coded = 1;
         }