src/third_party/libvpx/source/libvpx/vp9/encoder/arm/neon/vp9_sad_neon.c

   1 /*
   2  *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include <arm_neon.h>
  12 #include "./vp9_rtcd.h"
  13 #include "./vpx_config.h"
  14
  15 #include "vpx/vpx_integer.h"
  16
  17 static INLINE unsigned int horizontal_long_add_16x8(const uint16x8_t vec_lo,
  18                                                     const uint16x8_t vec_hi) {
  19   const uint32x4_t vec_l_lo = vaddl_u16(vget_low_u16(vec_lo),
  20                                         vget_high_u16(vec_lo));
  21   const uint32x4_t vec_l_hi = vaddl_u16(vget_low_u16(vec_hi),
  22                                         vget_high_u16(vec_hi));
  23   const uint32x4_t a = vaddq_u32(vec_l_lo, vec_l_hi);
  24   const uint64x2_t b = vpaddlq_u32(a);
  25   const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
  26                                 vreinterpret_u32_u64(vget_high_u64(b)));
  27   return vget_lane_u32(c, 0);
  28 }
  29 static INLINE unsigned int horizontal_add_16x8(const uint16x8_t vec_lo,
  30                                                const uint16x8_t vec_hi) {
  31   const uint32x4_t a = vpaddlq_u16(vaddq_u16(vec_lo, vec_hi));
  32   const uint64x2_t b = vpaddlq_u32(a);
  33   const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
  34                                 vreinterpret_u32_u64(vget_high_u64(b)));
  35   return vget_lane_u32(c, 0);
  36 }
  37
  38 unsigned int vp9_sad64x64_neon(const uint8_t *src, int src_stride,
  39                                const uint8_t *ref, int ref_stride) {
  40   int i;
  41   uint16x8_t vec_accum_lo = vdupq_n_u16(0);
  42   uint16x8_t vec_accum_hi = vdupq_n_u16(0);
  43   for (i = 0; i < 64; ++i) {
  44     const uint8x16_t vec_src_00 = vld1q_u8(src);
  45     const uint8x16_t vec_src_16 = vld1q_u8(src + 16);
  46     const uint8x16_t vec_src_32 = vld1q_u8(src + 32);
  47     const uint8x16_t vec_src_48 = vld1q_u8(src + 48);
  48     const uint8x16_t vec_ref_00 = vld1q_u8(ref);
  49     const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16);
  50     const uint8x16_t vec_ref_32 = vld1q_u8(ref + 32);
  51     const uint8x16_t vec_ref_48 = vld1q_u8(ref + 48);
  52     src += src_stride;
  53     ref += ref_stride;
  54     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_00),
  55                             vget_low_u8(vec_ref_00));
  56     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_00),
  57                             vget_high_u8(vec_ref_00));
  58     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_16),
  59                             vget_low_u8(vec_ref_16));
  60     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_16),
  61                             vget_high_u8(vec_ref_16));
  62     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_32),
  63                             vget_low_u8(vec_ref_32));
  64     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_32),
  65                             vget_high_u8(vec_ref_32));
  66     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_48),
  67                             vget_low_u8(vec_ref_48));
  68     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_48),
  69                             vget_high_u8(vec_ref_48));
  70   }
  71   return horizontal_long_add_16x8(vec_accum_lo, vec_accum_hi);
  72 }
  73
  74 unsigned int vp9_sad32x32_neon(const uint8_t *src, int src_stride,
  75                                const uint8_t *ref, int ref_stride) {
  76   int i;
  77   uint16x8_t vec_accum_lo = vdupq_n_u16(0);
  78   uint16x8_t vec_accum_hi = vdupq_n_u16(0);
  79
  80   for (i = 0; i < 32; ++i) {
  81     const uint8x16_t vec_src_00 = vld1q_u8(src);
  82     const uint8x16_t vec_src_16 = vld1q_u8(src + 16);
  83     const uint8x16_t vec_ref_00 = vld1q_u8(ref);
  84     const uint8x16_t vec_ref_16 = vld1q_u8(ref + 16);
  85     src += src_stride;
  86     ref += ref_stride;
  87     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_00),
  88                             vget_low_u8(vec_ref_00));
  89     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_00),
  90                             vget_high_u8(vec_ref_00));
  91     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src_16),
  92                             vget_low_u8(vec_ref_16));
  93     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src_16),
  94                             vget_high_u8(vec_ref_16));
  95   }
  96   return horizontal_add_16x8(vec_accum_lo, vec_accum_hi);
  97 }
  98
  99 unsigned int vp9_sad16x16_neon(const uint8_t *src, int src_stride,
 100                                const uint8_t *ref, int ref_stride) {
 101   int i;
 102   uint16x8_t vec_accum_lo = vdupq_n_u16(0);
 103   uint16x8_t vec_accum_hi = vdupq_n_u16(0);
 104
 105   for (i = 0; i < 16; ++i) {
 106     const uint8x16_t vec_src = vld1q_u8(src);
 107     const uint8x16_t vec_ref = vld1q_u8(ref);
 108     src += src_stride;
 109     ref += ref_stride;
 110     vec_accum_lo = vabal_u8(vec_accum_lo, vget_low_u8(vec_src),
 111                             vget_low_u8(vec_ref));
 112     vec_accum_hi = vabal_u8(vec_accum_hi, vget_high_u8(vec_src),
 113                             vget_high_u8(vec_ref));
 114   }
 115   return horizontal_add_16x8(vec_accum_lo, vec_accum_hi);
 116 }