changed loop filter for MBs using 8x8 transform

This commit added a set of loop filter functions for macroblocks
using 8x8 transform. First we turned off the regular loop filtering
on 4x4 block boundaries that do not exist in macroblocks using 8x8
transform. Second, we change to use the same loop filter(mask and
7 tap filter) that used for macroblock edge filtering.

Change-Id: I3a00460b7674ced116917d86812ffc32578c1d3a

diff --git a/vp8/common/alloccommon.c b/vp8/common/alloccommon.c
index 4457c88..e643eff 100644 (file)
--- a/vp8/common/alloccommon.c
+++ b/vp8/common/alloccommon.c
@@ -153,10 +153,10 @@ void vp8_setup_version(VP8_COMMON *cm)
         if (!CONFIG_EXPERIMENTAL)
             vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
                                "Bitstream was created by an experimental "
-                               "encoder");        
+                               "encoder");
         cm->experimental = 1;
     }
-    
+
     switch (cm->version & 0x3)
     {
     case 0:

diff --git a/vp8/common/loopfilter.c b/vp8/common/loopfilter.c
index 052d00e..677f7c6 100644 (file)
--- a/vp8/common/loopfilter.c
+++ b/vp8/common/loopfilter.c
@@ -22,6 +22,8 @@ typedef unsigned char uc;
 
 prototype_loopfilter(vp8_loop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_loop_filter_vertical_edge_c);
+
+
 prototype_loopfilter(vp8_mbloop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_mbloop_filter_vertical_edge_c);
 
@@ -72,6 +74,16 @@ void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_horizontal_edge_c(v_ptr + 4 * uv_stride, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bh8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_horizontal_edge_c(
+        y_ptr + 8 * y_stride, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+#endif
+
 void vp8_loop_filter_bhs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -96,6 +108,17 @@ void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_vertical_edge_c(v_ptr + 4, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bv8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_vertical_edge_c(
+        y_ptr + 8, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+
+#endif
+
 void vp8_loop_filter_bvs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -328,7 +351,9 @@ void vp8_loop_filter_frame
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
-
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
             if (filter_level)
@@ -350,8 +375,17 @@ void vp8_loop_filter_frame
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -362,9 +396,18 @@ void vp8_loop_filter_frame
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
+
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    }
                 }
                 else
                 {
@@ -448,6 +491,9 @@ void vp8_loop_filter_frame_yonly
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
 
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
@@ -469,10 +515,17 @@ void vp8_loop_filter_frame_yonly
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
-
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -484,8 +537,16 @@ void vp8_loop_filter_frame_yonly
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
                 }
                 else
                 {

diff --git a/vp8/common/loopfilter_filters.c b/vp8/common/loopfilter_filters.c
index 72ba31e..afd0fde 100644 (file)
--- a/vp8/common/loopfilter_filters.c
+++ b/vp8/common/loopfilter_filters.c
@@ -10,6 +10,7 @@
 
 
 #include <stdlib.h>
+#include "vpx_config.h"
 #include "loopfilter.h"
 #include "onyxc_int.h"