vp8: fix per-segment deblocking filter level in relative mode.

Fix possible bug when a per-segment deblocking filter level value
needs to be set in non-absolute mode, i.e. when the loop filter update
value is negative in delta mode.

Also clamp the resulting filter level value to 0..63 range.

diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c b/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c
index 7b82854..c7d2285 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_vp8.c
@@ -315,12 +315,14 @@ fill_picture (GstVaapiDecoderVp8 * decoder, GstVaapiPicture * picture)
     pic_param->mb_segment_tree_probs[i] = seg->segment_prob[i];
 
   for (i = 0; i < 4; i++) {
+    gint8 level;
     if (seg->segmentation_enabled) {
-      pic_param->loop_filter_level[i] = seg->lf_update_value[i];
-      if (!seg->segment_feature_mode)
-        pic_param->loop_filter_level[i] += frame_hdr->loop_filter_level;
+      level = seg->lf_update_value[i];
+      if (!seg->segment_feature_mode)   // 0 means delta update
+        level += frame_hdr->loop_filter_level;
     } else
-      pic_param->loop_filter_level[i] = frame_hdr->loop_filter_level;
+      level = frame_hdr->loop_filter_level;
+    pic_param->loop_filter_level[i] = CLAMP (level, 0, 63);
 
     pic_param->loop_filter_deltas_ref_frame[i] =
         parser->mb_lf_adjust.ref_frame_delta[i];