2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
12 #include "./vp9_rtcd.h"
13 #include "./vpx_config.h"
15 #include "vpx_ports/mem.h"
16 #include "vpx/vpx_integer.h"
18 #include "vp9/common/vp9_common.h"
19 #include "vp9/common/vp9_filter.h"
21 #include "vp9/encoder/vp9_variance.h"
24 enum { kHeight8 = 8 };
25 enum { kHeight8PlusOne = 9 };
26 enum { kWidth16 = 16 };
27 enum { kHeight16 = 16 };
28 enum { kHeight16PlusOne = 17 };
29 enum { kWidth32 = 32 };
30 enum { kHeight32 = 32 };
31 enum { kHeight32PlusOne = 33 };
32 enum { kWidth64 = 64 };
33 enum { kHeight64 = 64 };
34 enum { kHeight64PlusOne = 65 };
35 enum { kPixelStepOne = 1 };
36 enum { kAlign16 = 16 };
38 static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) {
39 const int32x4_t a = vpaddlq_s16(v_16x8);
40 const int64x2_t b = vpaddlq_s32(a);
41 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
42 vreinterpret_s32_s64(vget_high_s64(b)));
43 return vget_lane_s32(c, 0);
46 static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) {
47 const int64x2_t b = vpaddlq_s32(v_32x4);
48 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
49 vreinterpret_s32_s64(vget_high_s64(b)));
50 return vget_lane_s32(c, 0);
53 // w * h must be less than 2048 or local variable v_sum may overflow.
54 static void variance_neon_w8(const uint8_t *a, int a_stride,
55 const uint8_t *b, int b_stride,
56 int w, int h, uint32_t *sse, int *sum) {
58 int16x8_t v_sum = vdupq_n_s16(0);
59 int32x4_t v_sse_lo = vdupq_n_s32(0);
60 int32x4_t v_sse_hi = vdupq_n_s32(0);
62 for (i = 0; i < h; ++i) {
63 for (j = 0; j < w; j += 8) {
64 const uint8x8_t v_a = vld1_u8(&a[j]);
65 const uint8x8_t v_b = vld1_u8(&b[j]);
66 const uint16x8_t v_diff = vsubl_u8(v_a, v_b);
67 const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff);
68 v_sum = vaddq_s16(v_sum, sv_diff);
69 v_sse_lo = vmlal_s16(v_sse_lo,
70 vget_low_s16(sv_diff),
71 vget_low_s16(sv_diff));
72 v_sse_hi = vmlal_s16(v_sse_hi,
73 vget_high_s16(sv_diff),
74 vget_high_s16(sv_diff));
80 *sum = horizontal_add_s16x8(v_sum);
81 *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi));
84 void vp9_get8x8var_neon(const uint8_t *src_ptr, int source_stride,
85 const uint8_t *ref_ptr, int ref_stride,
86 unsigned int *sse, int *sum) {
87 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth8,
91 unsigned int vp9_variance8x8_neon(const uint8_t *a, int a_stride,
92 const uint8_t *b, int b_stride,
95 variance_neon_w8(a, a_stride, b, b_stride, kWidth8, kHeight8, sse, &sum);
96 return *sse - (((int64_t)sum * sum) >> 6); // >> 6 = / 8 * 8
99 void vp9_get16x16var_neon(const uint8_t *src_ptr, int source_stride,
100 const uint8_t *ref_ptr, int ref_stride,
101 unsigned int *sse, int *sum) {
102 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth16,
103 kHeight16, sse, sum);
106 unsigned int vp9_variance16x16_neon(const uint8_t *a, int a_stride,
107 const uint8_t *b, int b_stride,
110 variance_neon_w8(a, a_stride, b, b_stride, kWidth16, kHeight16, sse, &sum);
111 return *sse - (((int64_t)sum * sum) >> 8); // >> 8 = / 16 * 16
114 static void var_filter_block2d_bil_w8(const uint8_t *src_ptr,
116 unsigned int src_pixels_per_line,
118 unsigned int output_height,
119 unsigned int output_width,
120 const int16_t *vp9_filter) {
121 const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
122 const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
124 for (i = 0; i < output_height; ++i) {
125 const uint8x8_t src_0 = vld1_u8(&src_ptr[0]);
126 const uint8x8_t src_1 = vld1_u8(&src_ptr[pixel_step]);
127 const uint16x8_t a = vmull_u8(src_0, f0);
128 const uint16x8_t b = vmlal_u8(a, src_1, f1);
129 const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS);
130 vst1_u8(&output_ptr[0], out);
132 src_ptr += src_pixels_per_line;
133 output_ptr += output_width;
137 static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
139 unsigned int src_pixels_per_line,
141 unsigned int output_height,
142 unsigned int output_width,
143 const int16_t *vp9_filter) {
144 const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
145 const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
147 for (i = 0; i < output_height; ++i) {
148 for (j = 0; j < output_width; j += 16) {
149 const uint8x16_t src_0 = vld1q_u8(&src_ptr[j]);
150 const uint8x16_t src_1 = vld1q_u8(&src_ptr[j + pixel_step]);
151 const uint16x8_t a = vmull_u8(vget_low_u8(src_0), f0);
152 const uint16x8_t b = vmlal_u8(a, vget_low_u8(src_1), f1);
153 const uint8x8_t out_lo = vrshrn_n_u16(b, FILTER_BITS);
154 const uint16x8_t c = vmull_u8(vget_high_u8(src_0), f0);
155 const uint16x8_t d = vmlal_u8(c, vget_high_u8(src_1), f1);
156 const uint8x8_t out_hi = vrshrn_n_u16(d, FILTER_BITS);
157 vst1q_u8(&output_ptr[j], vcombine_u8(out_lo, out_hi));
160 src_ptr += src_pixels_per_line;
161 output_ptr += output_width;
165 unsigned int vp9_sub_pixel_variance8x8_neon(const uint8_t *src,
172 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight8 * kWidth8);
173 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight8PlusOne * kWidth8);
175 var_filter_block2d_bil_w8(src, fdata3, src_stride, kPixelStepOne,
176 kHeight8PlusOne, kWidth8,
177 BILINEAR_FILTERS_2TAP(xoffset));
178 var_filter_block2d_bil_w8(fdata3, temp2, kWidth8, kWidth8, kHeight8,
179 kWidth8, BILINEAR_FILTERS_2TAP(yoffset));
180 return vp9_variance8x8_neon(temp2, kWidth8, dst, dst_stride, sse);
183 unsigned int vp9_sub_pixel_variance16x16_neon(const uint8_t *src,
190 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight16 * kWidth16);
191 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight16PlusOne * kWidth16);
193 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
194 kHeight16PlusOne, kWidth16,
195 BILINEAR_FILTERS_2TAP(xoffset));
196 var_filter_block2d_bil_w16(fdata3, temp2, kWidth16, kWidth16, kHeight16,
197 kWidth16, BILINEAR_FILTERS_2TAP(yoffset));
198 return vp9_variance16x16_neon(temp2, kWidth16, dst, dst_stride, sse);
201 void vp9_get32x32var_neon(const uint8_t *src_ptr, int source_stride,
202 const uint8_t *ref_ptr, int ref_stride,
203 unsigned int *sse, int *sum) {
204 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth32,
205 kHeight32, sse, sum);
208 unsigned int vp9_variance32x32_neon(const uint8_t *a, int a_stride,
209 const uint8_t *b, int b_stride,
212 variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, sse, &sum);
213 return *sse - (((int64_t)sum * sum) >> 10); // >> 10 = / 32 * 32
216 unsigned int vp9_variance32x64_neon(const uint8_t *a, int a_stride,
217 const uint8_t *b, int b_stride,
221 variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, &sse1, &sum1);
222 variance_neon_w8(a + (kHeight32 * a_stride), a_stride,
223 b + (kHeight32 * b_stride), b_stride, kWidth32, kHeight32,
227 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64
230 unsigned int vp9_variance64x32_neon(const uint8_t *a, int a_stride,
231 const uint8_t *b, int b_stride,
235 variance_neon_w8(a, a_stride, b, b_stride, kWidth64, kHeight16, &sse1, &sum1);
236 variance_neon_w8(a + (kHeight16 * a_stride), a_stride,
237 b + (kHeight16 * b_stride), b_stride, kWidth64, kHeight16,
241 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64
244 unsigned int vp9_variance64x64_neon(const uint8_t *a, int a_stride,
245 const uint8_t *b, int b_stride,
250 variance_neon_w8(a, a_stride, b, b_stride, kWidth64, kHeight16, &sse1, &sum1);
251 variance_neon_w8(a + (kHeight16 * a_stride), a_stride,
252 b + (kHeight16 * b_stride), b_stride, kWidth64, kHeight16,
257 variance_neon_w8(a + (kHeight16 * 2 * a_stride), a_stride,
258 b + (kHeight16 * 2 * b_stride), b_stride,
259 kWidth64, kHeight16, &sse2, &sum2);
263 variance_neon_w8(a + (kHeight16 * 3 * a_stride), a_stride,
264 b + (kHeight16 * 3 * b_stride), b_stride,
265 kWidth64, kHeight16, &sse2, &sum2);
268 return *sse - (((int64_t)sum1 * sum1) >> 12); // >> 12 = / 64 * 64
271 unsigned int vp9_sub_pixel_variance32x32_neon(const uint8_t *src,
278 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight32 * kWidth32);
279 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight32PlusOne * kWidth32);
281 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
282 kHeight32PlusOne, kWidth32,
283 BILINEAR_FILTERS_2TAP(xoffset));
284 var_filter_block2d_bil_w16(fdata3, temp2, kWidth32, kWidth32, kHeight32,
285 kWidth32, BILINEAR_FILTERS_2TAP(yoffset));
286 return vp9_variance32x32_neon(temp2, kWidth32, dst, dst_stride, sse);
289 unsigned int vp9_sub_pixel_variance64x64_neon(const uint8_t *src,
296 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight64 * kWidth64);
297 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight64PlusOne * kWidth64);
299 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
300 kHeight64PlusOne, kWidth64,
301 BILINEAR_FILTERS_2TAP(xoffset));
302 var_filter_block2d_bil_w16(fdata3, temp2, kWidth64, kWidth64, kHeight64,
303 kWidth64, BILINEAR_FILTERS_2TAP(yoffset));
304 return vp9_variance64x64_neon(temp2, kWidth64, dst, dst_stride, sse);