[vpx highbd lpf NEON 5/6] horizontal 16

BUG=webm:1300

Change-Id: I21da32d6cfb8a1a6f58bc9756d17f48f13a59a12

diff --git a/test/lpf_test.cc b/test/lpf_test.cc
index ed57843..b976446 100644 (file)
--- a/test/lpf_test.cc
+++ b/test/lpf_test.cc
@@ -528,6 +528,18 @@ INSTANTIATE_TEST_CASE_P(
                                  &vpx_highbd_lpf_horizontal_8_c, 10),
                       make_tuple(&vpx_highbd_lpf_horizontal_8_neon,
                                  &vpx_highbd_lpf_horizontal_8_c, 12),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_neon,
+                                 &vpx_highbd_lpf_horizontal_16_c, 8),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_neon,
+                                 &vpx_highbd_lpf_horizontal_16_c, 10),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_neon,
+                                 &vpx_highbd_lpf_horizontal_16_c, 12),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_dual_neon,
+                                 &vpx_highbd_lpf_horizontal_16_dual_c, 8),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_dual_neon,
+                                 &vpx_highbd_lpf_horizontal_16_dual_c, 10),
+                      make_tuple(&vpx_highbd_lpf_horizontal_16_dual_neon,
+                                 &vpx_highbd_lpf_horizontal_16_dual_c, 12),
                       make_tuple(&vpx_highbd_lpf_vertical_4_neon,
                                  &vpx_highbd_lpf_vertical_4_c, 8),
                       make_tuple(&vpx_highbd_lpf_vertical_4_neon,

diff --git a/vpx_dsp/arm/highbd_loopfilter_neon.c b/vpx_dsp/arm/highbd_loopfilter_neon.c
index 65c49cd..4c3bb76 100644 (file)
--- a/vpx_dsp/arm/highbd_loopfilter_neon.c
+++ b/vpx_dsp/arm/highbd_loopfilter_neon.c
@@ -86,6 +86,28 @@ static INLINE uint16x8_t filter_flat_hev_mask(
   return mask;
 }
 
+static INLINE uint16x8_t flat_mask5(const uint16x8_t p4, const uint16x8_t p3,
+                                    const uint16x8_t p2, const uint16x8_t p1,
+                                    const uint16x8_t p0, const uint16x8_t q0,
+                                    const uint16x8_t q1, const uint16x8_t q2,
+                                    const uint16x8_t q3, const uint16x8_t q4,
+                                    const uint16x8_t flat,
+                                    uint32_t *flat2_status, const int bd) {
+  uint16x8_t flat2 = vabdq_u16(p4, p0);
+  flat2 = vmaxq_u16(flat2, vabdq_u16(p3, p0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(p2, p0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(p1, p0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(q1, q0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(q2, q0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(q3, q0));
+  flat2 = vmaxq_u16(flat2, vabdq_u16(q4, q0));
+  flat2 = vcleq_u16(flat2, vdupq_n_u16(1 << (bd - 8)));
+  flat2 = vandq_u16(flat2, flat);
+  *flat2_status = calc_flat_status(flat2);
+
+  return flat2;
+}
+
 static INLINE int16x8_t flip_sign(const uint16x8_t v, const int bd) {
   const uint16x8_t offset = vdupq_n_u16(0x80 << (bd - 8));
   return vreinterpretq_s16_u16(vsubq_u16(v, offset));
@@ -114,6 +136,14 @@ static INLINE uint16x8_t calc_7_tap_filter_kernel(const uint16x8_t sub0,
   return vrshrq_n_u16(*sum, 3);
 }
 
+static INLINE uint16x8_t apply_15_tap_filter_kernel(
+    const uint16x8_t flat, const uint16x8_t sub0, const uint16x8_t sub1,
+    const uint16x8_t add0, const uint16x8_t add1, const uint16x8_t in,
+    uint16x8_t *sum) {
+  filter_update(sub0, sub1, add0, add1, sum);
+  return vbslq_u16(flat, vrshrq_n_u16(*sum, 4), in);
+}
+
 // 7-tap filter [1, 1, 1, 2, 1, 1, 1]
 static INLINE void calc_7_tap_filter(const uint16x8_t p3, const uint16x8_t p2,
                                      const uint16x8_t p1, const uint16x8_t p0,
@@ -157,6 +187,45 @@ static INLINE void apply_7_tap_filter(const uint16x8_t flat,
   *oq2 = vbslq_u16(flat, *oq2, q2);
 }
 
+// 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1]
+static INLINE void apply_15_tap_filter(
+    const uint16x8_t flat2, const uint16x8_t p7, const uint16x8_t p6,
+    const uint16x8_t p5, const uint16x8_t p4, const uint16x8_t p3,
+    const uint16x8_t p2, const uint16x8_t p1, const uint16x8_t p0,
+    const uint16x8_t q0, const uint16x8_t q1, const uint16x8_t q2,
+    const uint16x8_t q3, const uint16x8_t q4, const uint16x8_t q5,
+    const uint16x8_t q6, const uint16x8_t q7, uint16x8_t *op6, uint16x8_t *op5,
+    uint16x8_t *op4, uint16x8_t *op3, uint16x8_t *op2, uint16x8_t *op1,
+    uint16x8_t *op0, uint16x8_t *oq0, uint16x8_t *oq1, uint16x8_t *oq2,
+    uint16x8_t *oq3, uint16x8_t *oq4, uint16x8_t *oq5, uint16x8_t *oq6) {
+  uint16x8_t sum;
+  sum = vshlq_n_u16(p7, 3);  // 8*p7
+  sum = vsubq_u16(sum, p7);  // 7*p7
+  sum = vaddq_u16(sum, p6);  // 7*p7+p6
+  sum = vaddq_u16(sum, p6);  // 7*p7+2*p6
+  sum = vaddq_u16(sum, p5);  // 7*p7+2*p6+p5
+  sum = vaddq_u16(sum, p4);  // 7*p7+2*p6+p5+p4
+  sum = vaddq_u16(sum, p3);  // 7*p7+2*p6+p5+p4+p3
+  sum = vaddq_u16(sum, p2);  // 7*p7+2*p6+p5+p4+p3+p2
+  sum = vaddq_u16(sum, p1);  // 7*p7+2*p6+p5+p4+p3+p2+p1
+  sum = vaddq_u16(sum, p0);  // 7*p7+2*p6+p5+p4+p3+p2+p1+p0
+  sum = vaddq_u16(sum, q0);  // 7*p7+2*p6+p5+p4+p3+p2+p1+p0+q0
+  *op6 = vbslq_u16(flat2, vrshrq_n_u16(sum, 4), p6);
+  *op5 = apply_15_tap_filter_kernel(flat2, p7, p6, p5, q1, p5, &sum);
+  *op4 = apply_15_tap_filter_kernel(flat2, p7, p5, p4, q2, p4, &sum);
+  *op3 = apply_15_tap_filter_kernel(flat2, p7, p4, p3, q3, p3, &sum);
+  *op2 = apply_15_tap_filter_kernel(flat2, p7, p3, p2, q4, *op2, &sum);
+  *op1 = apply_15_tap_filter_kernel(flat2, p7, p2, p1, q5, *op1, &sum);
+  *op0 = apply_15_tap_filter_kernel(flat2, p7, p1, p0, q6, *op0, &sum);
+  *oq0 = apply_15_tap_filter_kernel(flat2, p7, p0, q0, q7, *oq0, &sum);
+  *oq1 = apply_15_tap_filter_kernel(flat2, p6, q0, q1, q7, *oq1, &sum);
+  *oq2 = apply_15_tap_filter_kernel(flat2, p5, q1, q2, q7, *oq2, &sum);
+  *oq3 = apply_15_tap_filter_kernel(flat2, p4, q2, q3, q7, q3, &sum);
+  *oq4 = apply_15_tap_filter_kernel(flat2, p3, q3, q4, q7, q4, &sum);
+  *oq5 = apply_15_tap_filter_kernel(flat2, p2, q4, q5, q7, q5, &sum);
+  *oq6 = apply_15_tap_filter_kernel(flat2, p1, q5, q6, q7, q6, &sum);
+}
+
 static INLINE void filter4(const uint16x8_t mask, const uint16x8_t hev,
                            const uint16x8_t p1, const uint16x8_t p0,
                            const uint16x8_t q0, const uint16x8_t q1,
@@ -241,6 +310,37 @@ static INLINE void filter8(const uint16x8_t mask, const uint16x8_t flat,
   }
 }
 
+static INLINE void filter16(
+    const uint16x8_t mask, const uint16x8_t flat, const uint32_t flat_status,
+    const uint16x8_t flat2, const uint32_t flat2_status, const uint16x8_t hev,
+    const uint16x8_t p7, const uint16x8_t p6, const uint16x8_t p5,
+    const uint16x8_t p4, const uint16x8_t p3, const uint16x8_t p2,
+    const uint16x8_t p1, const uint16x8_t p0, const uint16x8_t q0,
+    const uint16x8_t q1, const uint16x8_t q2, const uint16x8_t q3,
+    const uint16x8_t q4, const uint16x8_t q5, const uint16x8_t q6,
+    const uint16x8_t q7, uint16x8_t *op6, uint16x8_t *op5, uint16x8_t *op4,
+    uint16x8_t *op3, uint16x8_t *op2, uint16x8_t *op1, uint16x8_t *op0,
+    uint16x8_t *oq0, uint16x8_t *oq1, uint16x8_t *oq2, uint16x8_t *oq3,
+    uint16x8_t *oq4, uint16x8_t *oq5, uint16x8_t *oq6, const int bd) {
+  if (flat_status != (uint32_t)-4) {
+    filter4(mask, hev, p1, p0, q0, q1, op1, op0, oq0, oq1, bd);
+  }
+
+  if (flat_status) {
+    *op2 = p2;
+    *oq2 = q2;
+    if (flat2_status != (uint32_t)-4) {
+      apply_7_tap_filter(flat, p3, p2, p1, p0, q0, q1, q2, q3, op2, op1, op0,
+                         oq0, oq1, oq2);
+    }
+    if (flat2_status) {
+      apply_15_tap_filter(flat2, p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3,
+                          q4, q5, q6, q7, op6, op5, op4, op3, op2, op1, op0,
+                          oq0, oq1, oq2, oq3, oq4, oq5, oq6);
+    }
+  }
+}
+
 static INLINE void load_8x8(const uint16_t *s, const int p, uint16x8_t *p3,
                             uint16x8_t *p2, uint16x8_t *p1, uint16x8_t *p0,
                             uint16x8_t *q0, uint16x8_t *q1, uint16x8_t *q2,
@@ -262,6 +362,46 @@ static INLINE void load_8x8(const uint16_t *s, const int p, uint16x8_t *p3,
   *q3 = vld1q_u16(s);
 }
 
+static INLINE void load_8x16(const uint16_t *s, const int p, uint16x8_t *s0,
+                             uint16x8_t *s1, uint16x8_t *s2, uint16x8_t *s3,
+                             uint16x8_t *s4, uint16x8_t *s5, uint16x8_t *s6,
+                             uint16x8_t *s7, uint16x8_t *s8, uint16x8_t *s9,
+                             uint16x8_t *s10, uint16x8_t *s11, uint16x8_t *s12,
+                             uint16x8_t *s13, uint16x8_t *s14,
+                             uint16x8_t *s15) {
+  *s0 = vld1q_u16(s);
+  s += p;
+  *s1 = vld1q_u16(s);
+  s += p;
+  *s2 = vld1q_u16(s);
+  s += p;
+  *s3 = vld1q_u16(s);
+  s += p;
+  *s4 = vld1q_u16(s);
+  s += p;
+  *s5 = vld1q_u16(s);
+  s += p;
+  *s6 = vld1q_u16(s);
+  s += p;
+  *s7 = vld1q_u16(s);
+  s += p;
+  *s8 = vld1q_u16(s);
+  s += p;
+  *s9 = vld1q_u16(s);
+  s += p;
+  *s10 = vld1q_u16(s);
+  s += p;
+  *s11 = vld1q_u16(s);
+  s += p;
+  *s12 = vld1q_u16(s);
+  s += p;
+  *s13 = vld1q_u16(s);
+  s += p;
+  *s14 = vld1q_u16(s);
+  s += p;
+  *s15 = vld1q_u16(s);
+}
+
 static INLINE void store_8x4(uint16_t *s, const int p, const uint16x8_t s0,
                              const uint16x8_t s1, const uint16x8_t s2,
                              const uint16x8_t s3) {
@@ -354,6 +494,35 @@ static INLINE void store_6x8(uint16_t *s, const int p, const uint16x8_t s0,
   vst3q_lane_u16(s + 0, o1, 7);
 }
 
+static INLINE void store_8x14(uint16_t *s, const int p, const uint16x8_t p6,
+                              const uint16x8_t p5, const uint16x8_t p4,
+                              const uint16x8_t p3, const uint16x8_t p2,
+                              const uint16x8_t p1, const uint16x8_t p0,
+                              const uint16x8_t q0, const uint16x8_t q1,
+                              const uint16x8_t q2, const uint16x8_t q3,
+                              const uint16x8_t q4, const uint16x8_t q5,
+                              const uint16x8_t q6, const uint32_t flat_status,
+                              const uint32_t flat2_status) {
+  if (flat_status) {
+    if (flat2_status) {
+      vst1q_u16(s - 7 * p, p6);
+      vst1q_u16(s - 6 * p, p5);
+      vst1q_u16(s - 5 * p, p4);
+      vst1q_u16(s - 4 * p, p3);
+      vst1q_u16(s + 3 * p, q3);
+      vst1q_u16(s + 4 * p, q4);
+      vst1q_u16(s + 5 * p, q5);
+      vst1q_u16(s + 6 * p, q6);
+    }
+    vst1q_u16(s - 3 * p, p2);
+    vst1q_u16(s + 2 * p, q2);
+  }
+  vst1q_u16(s - 2 * p, p1);
+  vst1q_u16(s - 1 * p, p0);
+  vst1q_u16(s + 0 * p, q0);
+  vst1q_u16(s + 1 * p, q1);
+}
+
 void vpx_highbd_lpf_horizontal_4_neon(uint16_t *s, int p, const uint8_t *blimit,
                                       const uint8_t *limit,
                                       const uint8_t *thresh, int bd) {
@@ -452,3 +621,46 @@ void vpx_highbd_lpf_vertical_8_dual_neon(
   vpx_highbd_lpf_vertical_8_neon(s, p, blimit0, limit0, thresh0, bd);
   vpx_highbd_lpf_vertical_8_neon(s + 8 * p, p, blimit1, limit1, thresh1, bd);
 }
+
+static void lpf_horizontal_16_kernel(uint16_t *s, int p,
+                                     const uint16x8_t blimit_vec,
+                                     const uint16x8_t limit_vec,
+                                     const uint16x8_t thresh_vec,
+                                     const int bd) {
+  uint16x8_t mask, flat, flat2, hev, p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2,
+      q3, q4, q5, q6, q7, op6, op5, op4, op3, op2, op1, op0, oq0, oq1, oq2, oq3,
+      oq4, oq5, oq6;
+  uint32_t flat_status, flat2_status;
+
+  load_8x16(s - 8 * p, p, &p7, &p6, &p5, &p4, &p3, &p2, &p1, &p0, &q0, &q1, &q2,
+            &q3, &q4, &q5, &q6, &q7);
+  mask = filter_flat_hev_mask(limit_vec, blimit_vec, thresh_vec, p3, p2, p1, p0,
+                              q0, q1, q2, q3, &flat, &flat_status, &hev, bd);
+  flat2 = flat_mask5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat,
+                     &flat2_status, bd);
+  filter16(mask, flat, flat_status, flat2, flat2_status, hev, p7, p6, p5, p4,
+           p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7, &op6, &op5, &op4,
+           &op3, &op2, &op1, &op0, &oq0, &oq1, &oq2, &oq3, &oq4, &oq5, &oq6,
+           bd);
+  store_8x14(s, p, op6, op5, op4, op3, op2, op1, op0, oq0, oq1, oq2, oq3, oq4,
+             oq5, oq6, flat_status, flat2_status);
+}
+
+void vpx_highbd_lpf_horizontal_16_neon(uint16_t *s, int p,
+                                       const uint8_t *blimit,
+                                       const uint8_t *limit,
+                                       const uint8_t *thresh, int bd) {
+  uint16x8_t blimit_vec, limit_vec, thresh_vec;
+  load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
+  lpf_horizontal_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
+}
+
+void vpx_highbd_lpf_horizontal_16_dual_neon(uint16_t *s, int p,
+                                            const uint8_t *blimit,
+                                            const uint8_t *limit,
+                                            const uint8_t *thresh, int bd) {
+  uint16x8_t blimit_vec, limit_vec, thresh_vec;
+  load_thresh(blimit, limit, thresh, &blimit_vec, &limit_vec, &thresh_vec, bd);
+  lpf_horizontal_16_kernel(s, p, blimit_vec, limit_vec, thresh_vec, bd);
+  lpf_horizontal_16_kernel(s + 8, p, blimit_vec, limit_vec, thresh_vec, bd);
+}

diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl
index 8108074..d4d46dd 100644 (file)
--- a/vpx_dsp/vpx_dsp_rtcd_defs.pl
+++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -475,10 +475,10 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
   specialize qw/vpx_highbd_lpf_vertical_4_dual sse2 neon/;
 
   add_proto qw/void vpx_highbd_lpf_horizontal_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd";
-  specialize qw/vpx_highbd_lpf_horizontal_16 sse2/;
+  specialize qw/vpx_highbd_lpf_horizontal_16 sse2 neon/;
 
   add_proto qw/void vpx_highbd_lpf_horizontal_16_dual/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd";
-  specialize qw/vpx_highbd_lpf_horizontal_16_dual sse2/;
+  specialize qw/vpx_highbd_lpf_horizontal_16_dual sse2 neon/;
 
   add_proto qw/void vpx_highbd_lpf_horizontal_8/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd";
   specialize qw/vpx_highbd_lpf_horizontal_8 sse2 neon/;