}
class VpxMbPostProcAcrossIpTest
- : public ::testing::TestWithParam<VpxMbPostProcAcrossIpFunc> {
+ : public AbstractBench,
+ public ::testing::TestWithParam<VpxMbPostProcAcrossIpFunc> {
public:
+ VpxMbPostProcAcrossIpTest()
+ : rows(16), cols(16), mbPostProcAcrossIp(GetParam()),
+ src(Buffer<uint8_t>(rows, cols, 8, 8, 17, 8)) {}
virtual void TearDown() { libvpx_test::ClearSystemState(); }
protected:
+ const int rows;
+ const int cols;
+ const VpxMbPostProcAcrossIpFunc mbPostProcAcrossIp;
+ Buffer<uint8_t> src;
+ void run();
+
void SetCols(unsigned char *s, int rows, int cols, int src_width) {
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
}
};
-TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) {
- const int rows = 16;
- const int cols = 16;
+void VpxMbPostProcAcrossIpTest::run() {
+ mbPostProcAcrossIp(src.TopLeftPixel(), src.stride(), rows, cols, q2mbl(0));
+}
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
+TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) {
ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckMediumFilterOutput) {
- const int rows = 16;
- const int cols = 16;
-
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
- static const unsigned char kExpectedOutput[cols] = {
+ static const unsigned char kExpectedOutput[] = {
2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 13
};
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckHighFilterOutput) {
- const int rows = 16;
- const int cols = 16;
-
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
- static const unsigned char kExpectedOutput[cols] = {
+ static const unsigned char kExpectedOutput[] = {
2, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13
};
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckCvsAssembly) {
- const int rows = 16;
- const int cols = 16;
-
Buffer<uint8_t> c_mem = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(c_mem.Init());
Buffer<uint8_t> asm_mem = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
}
}
+TEST_P(VpxMbPostProcAcrossIpTest, DISABLED_Speed) {
+ ASSERT_TRUE(src.Init());
+ src.SetPadding(10);
+
+ SetCols(src.TopLeftPixel(), rows, cols, src.stride());
+
+ runNTimes(100000);
+ printMedian("16x16");
+}
+
class VpxMbPostProcDownTest
: public AbstractBench,
public ::testing::TestWithParam<VpxMbPostProcDownFunc> {
#endif // HAVE_MSA
#if HAVE_VSX
+
+INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcAcrossIpTest,
+ ::testing::Values(vpx_mbpost_proc_across_ip_vsx));
+
INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcDownTest,
::testing::Values(vpx_mbpost_proc_down_vsx));
#endif // HAVE_VSX
0x0C, 0x0E, 0x18, 0x19, 0x1A, 0x1B,
0x1C, 0x1D, 0x1E, 0x1F };
-static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,
- 0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
- 0x0C, 0x0D, 0x1C, 0x1D };
+// C: s[c + 7]
+static INLINE int16x8_t next7l_s16(uint8x16_t c) {
+ static const uint8x16_t next7_perm = {
+ 0x07, 0x10, 0x08, 0x11, 0x09, 0x12, 0x0A, 0x13,
+ 0x0B, 0x14, 0x0C, 0x15, 0x0D, 0x16, 0x0E, 0x17,
+ };
+ return (int16x8_t)vec_perm(c, vec_zeros_u8, next7_perm);
+}
+
+// Slide across window and add.
+static INLINE int16x8_t slide_sum_s16(int16x8_t x) {
+ // x = A B C D E F G H
+ //
+ // 0 A B C D E F G
+ const int16x8_t sum1 = vec_add(x, vec_slo(x, vec_splats((int8_t)(2 << 3))));
+ // 0 0 A B C D E F
+ const int16x8_t sum2 = vec_add(vec_slo(x, vec_splats((int8_t)(4 << 3))),
+ // 0 0 0 A B C D E
+ vec_slo(x, vec_splats((int8_t)(6 << 3))));
+ // 0 0 0 0 A B C D
+ const int16x8_t sum3 = vec_add(vec_slo(x, vec_splats((int8_t)(8 << 3))),
+ // 0 0 0 0 0 A B C
+ vec_slo(x, vec_splats((int8_t)(10 << 3))));
+ // 0 0 0 0 0 0 A B
+ const int16x8_t sum4 = vec_add(vec_slo(x, vec_splats((int8_t)(12 << 3))),
+ // 0 0 0 0 0 0 0 A
+ vec_slo(x, vec_splats((int8_t)(14 << 3))));
+ return vec_add(vec_add(sum1, sum2), vec_add(sum3, sum4));
+}
+
+// Slide across window and add.
+static INLINE int32x4_t slide_sumsq_s32(int32x4_t xsq_even, int32x4_t xsq_odd) {
+ // 0 A C E
+ // + 0 B D F
+ int32x4_t sumsq_1 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(4 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(4 << 3))));
+ // 0 0 A C
+ // + 0 0 B D
+ int32x4_t sumsq_2 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(8 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(8 << 3))));
+ // 0 0 0 A
+ // + 0 0 0 B
+ int32x4_t sumsq_3 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(12 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(12 << 3))));
+ sumsq_1 = vec_add(sumsq_1, xsq_even);
+ sumsq_2 = vec_add(sumsq_2, sumsq_3);
+ return vec_add(sumsq_1, sumsq_2);
+}
+
+// C: (b + sum + val) >> 4
+static INLINE int16x8_t filter_s16(int16x8_t b, int16x8_t sum, int16x8_t val) {
+ return vec_sra(vec_add(vec_add(b, sum), val), vec_splats((uint16_t)4));
+}
+
+// C: sumsq * 15 - sum * sum
+static INLINE bool16x8_t mask_s16(int32x4_t sumsq_even, int32x4_t sumsq_odd,
+ int16x8_t sum, int32x4_t lim) {
+ static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,
+ 0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
+ 0x0C, 0x0D, 0x1C, 0x1D };
+ const int32x4_t sumsq_odd_scaled =
+ vec_mul(sumsq_odd, vec_splats((int32_t)15));
+ const int32x4_t sumsq_even_scaled =
+ vec_mul(sumsq_even, vec_splats((int32_t)15));
+ const int32x4_t thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
+ const int32x4_t thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
+
+ const bool32x4_t mask_odd = vec_cmplt(thres_odd, lim);
+ const bool32x4_t mask_even = vec_cmplt(thres_even, lim);
+ return vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
+}
+
+void vpx_mbpost_proc_across_ip_vsx(unsigned char *src, int pitch, int rows,
+ int cols, int flimit) {
+ int row, col;
+ const int32x4_t lim = vec_splats(flimit);
+
+ // 8 columns are processed at a time.
+ assert(cols % 8 == 0);
+
+ for (row = 0; row < rows; row++) {
+ // The sum is signed and requires at most 13 bits.
+ // (8 bits + sign) * 15 (4 bits)
+ int16x8_t sum;
+ // The sum of squares requires at most 20 bits.
+ // (16 bits + sign) * 15 (4 bits)
+ int32x4_t sumsq_even, sumsq_odd;
+
+ // Fill left context with first col.
+ int16x8_t left_ctx = vec_splats((int16_t)src[0]);
+ int16_t s = src[0] * 9;
+ int32_t ssq = src[0] * src[0] * 9 + 16;
+
+ // Fill the next 6 columns of the sliding window with cols 2 to 7.
+ for (col = 1; col <= 6; ++col) {
+ s += src[col];
+ ssq += src[col] * src[col];
+ }
+ // Set this sum to every element in the window.
+ sum = vec_splats(s);
+ sumsq_even = vec_splats(ssq);
+ sumsq_odd = vec_splats(ssq);
+
+ for (col = 0; col < cols; col += 8) {
+ bool16x8_t mask;
+ int16x8_t filtered, masked;
+ uint8x16_t out;
+
+ const uint8x16_t val = vec_vsx_ld(0, src + col);
+ const int16x8_t val_high = unpack_to_s16_h(val);
+
+ // C: s[c + 7]
+ const int16x8_t right_ctx = (col + 8 == cols)
+ ? vec_splats((int16_t)src[col + 7])
+ : next7l_s16(val);
+
+ // C: x = s[c + 7] - s[c - 8];
+ const int16x8_t x = vec_sub(right_ctx, left_ctx);
+ const int32x4_t xsq_even =
+ vec_sub(vec_mule(right_ctx, right_ctx), vec_mule(left_ctx, left_ctx));
+ const int32x4_t xsq_odd =
+ vec_sub(vec_mulo(right_ctx, right_ctx), vec_mulo(left_ctx, left_ctx));
+
+ const int32x4_t sumsq_tmp = slide_sumsq_s32(xsq_even, xsq_odd);
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_even = vec_add(sumsq_even, sumsq_tmp);
+ // B D F G
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_odd = vec_add(sumsq_odd, vec_add(sumsq_tmp, xsq_odd));
+
+ sum = vec_add(sum, slide_sum_s16(x));
+
+ // C: (8 + sum + s[c]) >> 4
+ filtered = filter_s16(vec_splats((int16_t)8), sum, val_high);
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
+ masked = vec_sel(val_high, filtered, mask);
+
+ out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, src + col), load_merge);
+ vec_vsx_st(out, 0, src + col);
+
+ // Update window sum and square sum
+ sum = vec_splat(sum, 7);
+ sumsq_even = vec_splat(sumsq_odd, 3);
+ sumsq_odd = vec_splat(sumsq_odd, 3);
+
+ // C: s[c - 8] (for next iteration)
+ left_ctx = val_high;
+ }
+ src += pitch;
+ }
+}
void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols,
int flimit) {
for (row = 0; row < rows; row++) {
int32x4_t d15_even, d15_odd, d0_even, d0_odd;
- int32x4_t sumsq_odd_scaled, sumsq_even_scaled;
- int32x4_t thres_odd, thres_even;
- bool32x4_t mask_odd, mask_even;
bool16x8_t mask;
int16x8_t filtered, masked;
uint8x16_t out;
sumsq_odd = vec_add(sumsq_odd, vec_sub(d15_odd, d0_odd));
sumsq_even = vec_add(sumsq_even, vec_sub(d15_even, d0_even));
- // C: sumsq * 15 - sum * sum
- sumsq_odd_scaled = vec_mul(sumsq_odd, vec_splats((int32_t)15));
- sumsq_even_scaled = vec_mul(sumsq_even, vec_splats((int32_t)15));
- thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
- thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
-
// C: (vpx_rv[(r & 127) + (c & 7)] + sum + s[0]) >> 4
- filtered = vec_add(vec_add(rv, sum), window[8]);
- filtered = vec_sra(filtered, vec_splats((uint16_t)4));
+ filtered = filter_s16(rv, sum, window[8]);
- mask_odd = vec_cmplt(thres_odd, lim);
- mask_even = vec_cmplt(thres_even, lim);
- mask = vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
masked = vec_sel(window[8], filtered, mask);
// TODO(ltrudeau) If cols % 16 == 0, we could just process 16 per
projects / platform / upstream / libvpx.git / commitdiff