RECON_AND_STORE(dest + 6 * stride, in6);
RECON_AND_STORE(dest + 7 * stride, in7);
}
+
+static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {
+ const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
+ // idct constants for each stage
+ const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
+ const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
+ const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
+ const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
+ const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
+ const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
+ const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
+ const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
+
+ const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
+ const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
+ const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
+ const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
+
+ const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
+ const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
+ const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
+ const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
+ const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
+ const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
+ const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
+ const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
+
+ const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
+ const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
+ const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
+ const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+ const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
+ const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+
+ const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+ __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
+ stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15,
+ stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23,
+ stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31;
+ __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+
+ /* Stage1 */
+
+ stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0);
+ stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1);
+
+ stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6);
+ stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7);
+
+ stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8);
+ stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9);
+
+ stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14);
+ stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15);
+
+ /* Stage2 */
+
+ stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0);
+ stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1);
+
+ stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6);
+ stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7);
+
+ /* Stage3 */
+ {
+ const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]);
+ const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]);
+ const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]);
+ const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]);
+
+ const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]);
+ const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]);
+ const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]);
+ const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]);
+
+ stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0);
+ stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1);
+
+ MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4,
+ stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18],
+ stp1[29])
+ MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8,
+ stg3_9, stg3_10, stg3_8, stp1[21], stp1[26],
+ stp1[22], stp1[25])
+ }
+
+ /* Stage4 */
+ {
+ const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15);
+ const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15);
+ const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12);
+ const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12);
+
+ stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0);
+ stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0
+ stp1[2] = stp1[0];
+ stp1[3] = stp1[1];
+
+ MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5,
+ stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13)
+
+ stp2_16 = _mm_add_epi16(stp1[16], stp1[19]);
+ stp2_17 = _mm_add_epi16(stp1[17], stp1[18]);
+ stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]);
+ stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]);
+ stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]);
+ stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]);
+ stp2_22 = _mm_add_epi16(stp1[22], stp1[21]);
+ stp2_23 = _mm_add_epi16(stp1[23], stp1[20]);
+
+ stp2_24 = _mm_add_epi16(stp1[24], stp1[27]);
+ stp2_25 = _mm_add_epi16(stp1[25], stp1[26]);
+ stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]);
+ stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]);
+ stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]);
+ stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]);
+ stp2_30 = _mm_add_epi16(stp1[29], stp1[30]);
+ stp2_31 = _mm_add_epi16(stp1[28], stp1[31]);
+ }
+
+ /* Stage5 */
+ {
+ // Note:
+ // #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass
+ // SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop
+ // to 23198 and -23197, respectively.
+#define AVOID_OVERFLOW (1)
+
+#if AVOID_OVERFLOW
+ const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]);
+ const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]);
+#endif
+ const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
+ const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
+
+ const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28);
+ const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28);
+ const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
+ const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
+
+ const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
+ const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
+
+#if AVOID_OVERFLOW
+ tmp0 = _mm_madd_epi16(lo_6_5, stg4_1);
+ tmp1 = _mm_madd_epi16(hi_6_5, stg4_1);
+ tmp2 = _mm_madd_epi16(lo_6_5, stg4_0);
+ tmp3 = _mm_madd_epi16(hi_6_5, stg4_0);
+
+ tmp0 = _mm_add_epi32(tmp0, rounding);
+ tmp1 = _mm_add_epi32(tmp1, rounding);
+ tmp2 = _mm_add_epi32(tmp2, rounding);
+ tmp3 = _mm_add_epi32(tmp3, rounding);
+
+ tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
+ tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
+ tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
+ tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
+
+ stp1[5] = _mm_packs_epi32(tmp0, tmp1);
+ stp1[6] = _mm_packs_epi32(tmp2, tmp3);
+#else
+ tmp0 = _mm_sub_epi16(stp1[7], stp1[4]);
+ tmp1 = _mm_adds_epi16(stp1[7], stp1[4]);
+ stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0);
+ stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0);
+#endif
+
+ stp1[8] = _mm_add_epi16(stp2_8, stp2_11);
+ stp1[9] = _mm_add_epi16(stp2_9, stp2_10);
+ stp1[10] = _mm_sub_epi16(stp2_9, stp2_10);
+ stp1[11] = _mm_sub_epi16(stp2_8, stp2_11);
+ stp1[12] = _mm_sub_epi16(stp2_15, stp2_12);
+ stp1[13] = _mm_sub_epi16(stp2_14, stp2_13);
+ stp1[14] = _mm_add_epi16(stp2_14, stp2_13);
+ stp1[15] = _mm_add_epi16(stp2_15, stp2_12);
+
+ MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4,
+ stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19],
+ stp1[28])
+ MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6,
+ stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21],
+ stp1[26])
+
+ stp1[16] = stp2_16;
+ stp1[17] = stp2_17;
+ stp1[22] = stp2_22;
+ stp1[23] = stp2_23;
+ stp1[24] = stp2_24;
+ stp1[25] = stp2_25;
+ stp1[30] = stp2_30;
+ stp1[31] = stp2_31;
+ }
+
+ /* Stage6 */
+ {
+#if AVOID_OVERFLOW
+ const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]);
+ const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]);
+ const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]);
+ const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]);
+#endif
+
+ stp2_0 = _mm_add_epi16(stp1[0], stp1[7]);
+ stp2_1 = _mm_add_epi16(stp1[1], stp1[6]);
+ stp2_2 = _mm_add_epi16(stp1[2], stp1[5]);
+ stp2_3 = _mm_add_epi16(stp1[3], stp1[4]);
+ stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]);
+ stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]);
+ stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]);
+ stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]);
+
+ stp2_8 = stp1[8];
+ stp2_9 = stp1[9];
+ stp2_14 = stp1[14];
+ stp2_15 = stp1[15];
+
+#if AVOID_OVERFLOW
+ MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0,
+ stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11,
+ stp2_12)
+#else
+ tmp0 = _mm_add_epi16(stp1[10], stp1[13]);
+ tmp1 = _mm_sub_epi16(stp1[13], stp1[10]);
+ tmp2 = _mm_add_epi16(stp1[11], stp1[12]);
+ tmp3 = _mm_sub_epi16(stp1[12], stp1[11]);
+
+ stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0);
+ stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0);
+ stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0);
+ stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0);
+
+#endif
+
+ stp2_16 = _mm_add_epi16(stp1[16], stp1[23]);
+ stp2_17 = _mm_add_epi16(stp1[17], stp1[22]);
+ stp2_18 = _mm_add_epi16(stp1[18], stp1[21]);
+ stp2_19 = _mm_add_epi16(stp1[19], stp1[20]);
+ stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]);
+ stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]);
+ stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]);
+ stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]);
+
+ stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]);
+ stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]);
+ stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]);
+ stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]);
+ stp2_28 = _mm_add_epi16(stp1[27], stp1[28]);
+ stp2_29 = _mm_add_epi16(stp1[26], stp1[29]);
+ stp2_30 = _mm_add_epi16(stp1[25], stp1[30]);
+ stp2_31 = _mm_add_epi16(stp1[24], stp1[31]);
+ }
+
+ /* Stage7 */
+ {
+#if AVOID_OVERFLOW
+ const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
+ const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
+ const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
+ const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
+
+ const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
+ const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
+ const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24);
+ const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24);
+#endif
+ stp1[0] = _mm_add_epi16(stp2_0, stp2_15);
+ stp1[1] = _mm_add_epi16(stp2_1, stp2_14);
+ stp1[2] = _mm_add_epi16(stp2_2, stp2_13);
+ stp1[3] = _mm_add_epi16(stp2_3, stp2_12);
+ stp1[4] = _mm_add_epi16(stp2_4, stp2_11);
+ stp1[5] = _mm_add_epi16(stp2_5, stp2_10);
+ stp1[6] = _mm_add_epi16(stp2_6, stp2_9);
+ stp1[7] = _mm_add_epi16(stp2_7, stp2_8);
+ stp1[8] = _mm_sub_epi16(stp2_7, stp2_8);
+ stp1[9] = _mm_sub_epi16(stp2_6, stp2_9);
+ stp1[10] = _mm_sub_epi16(stp2_5, stp2_10);
+ stp1[11] = _mm_sub_epi16(stp2_4, stp2_11);
+ stp1[12] = _mm_sub_epi16(stp2_3, stp2_12);
+ stp1[13] = _mm_sub_epi16(stp2_2, stp2_13);
+ stp1[14] = _mm_sub_epi16(stp2_1, stp2_14);
+ stp1[15] = _mm_sub_epi16(stp2_0, stp2_15);
+
+ stp1[16] = stp2_16;
+ stp1[17] = stp2_17;
+ stp1[18] = stp2_18;
+ stp1[19] = stp2_19;
+
+#if AVOID_OVERFLOW
+ MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0,
+ stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21],
+ stp1[26])
+ MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0,
+ stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23],
+ stp1[24])
+#else
+ tmp0 = _mm_add_epi16(stp2_20, stp2_27);
+ tmp1 = _mm_sub_epi16(stp2_27, stp2_20);
+ tmp2 = _mm_add_epi16(stp2_21, stp2_26);
+ tmp3 = _mm_sub_epi16(stp2_26, stp2_21);
+
+ stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0);
+ stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0);
+ stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0);
+ stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0);
+
+ tmp0 = _mm_add_epi16(stp2_22, stp2_25);
+ tmp1 = _mm_sub_epi16(stp2_25, stp2_22);
+ tmp2 = _mm_add_epi16(stp2_23, stp2_24);
+ tmp3 = _mm_sub_epi16(stp2_24, stp2_23);
+
+ stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0);
+ stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0);
+ stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0);
+ stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0);
+#endif
+
+ stp1[28] = stp2_28;
+ stp1[29] = stp2_29;
+ stp1[30] = stp2_30;
+ stp1[31] = stp2_31;
+ }
+#undef AVOID_OVERFLOW
+}
+
+// Only upper-left 8x8 has non-zero coeff
+void vpx_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ const __m128i zero = _mm_setzero_si128();
+ const __m128i final_rounding = _mm_set1_epi16(1 << 5);
+ __m128i in[32], col[32];
+ __m128i stp1[32];
+ int i;
+
+ // Load input data. Only need to load the top left 8x8 block.
+ in[0] = load_input_data(input);
+ in[1] = load_input_data(input + 32);
+ in[2] = load_input_data(input + 64);
+ in[3] = load_input_data(input + 96);
+ in[4] = load_input_data(input + 128);
+ in[5] = load_input_data(input + 160);
+ in[6] = load_input_data(input + 192);
+ in[7] = load_input_data(input + 224);
+
+ array_transpose_8x8(in, in);
+ idct32_34(in, stp1);
+
+ // 1_D: Store 32 intermediate results for each 8x32 block.
+ col[0] = _mm_add_epi16(stp1[0], stp1[31]);
+ col[1] = _mm_add_epi16(stp1[1], stp1[30]);
+ col[2] = _mm_add_epi16(stp1[2], stp1[29]);
+ col[3] = _mm_add_epi16(stp1[3], stp1[28]);
+ col[4] = _mm_add_epi16(stp1[4], stp1[27]);
+ col[5] = _mm_add_epi16(stp1[5], stp1[26]);
+ col[6] = _mm_add_epi16(stp1[6], stp1[25]);
+ col[7] = _mm_add_epi16(stp1[7], stp1[24]);
+ col[8] = _mm_add_epi16(stp1[8], stp1[23]);
+ col[9] = _mm_add_epi16(stp1[9], stp1[22]);
+ col[10] = _mm_add_epi16(stp1[10], stp1[21]);
+ col[11] = _mm_add_epi16(stp1[11], stp1[20]);
+ col[12] = _mm_add_epi16(stp1[12], stp1[19]);
+ col[13] = _mm_add_epi16(stp1[13], stp1[18]);
+ col[14] = _mm_add_epi16(stp1[14], stp1[17]);
+ col[15] = _mm_add_epi16(stp1[15], stp1[16]);
+ col[16] = _mm_sub_epi16(stp1[15], stp1[16]);
+ col[17] = _mm_sub_epi16(stp1[14], stp1[17]);
+ col[18] = _mm_sub_epi16(stp1[13], stp1[18]);
+ col[19] = _mm_sub_epi16(stp1[12], stp1[19]);
+ col[20] = _mm_sub_epi16(stp1[11], stp1[20]);
+ col[21] = _mm_sub_epi16(stp1[10], stp1[21]);
+ col[22] = _mm_sub_epi16(stp1[9], stp1[22]);
+ col[23] = _mm_sub_epi16(stp1[8], stp1[23]);
+ col[24] = _mm_sub_epi16(stp1[7], stp1[24]);
+ col[25] = _mm_sub_epi16(stp1[6], stp1[25]);
+ col[26] = _mm_sub_epi16(stp1[5], stp1[26]);
+ col[27] = _mm_sub_epi16(stp1[4], stp1[27]);
+ col[28] = _mm_sub_epi16(stp1[3], stp1[28]);
+ col[29] = _mm_sub_epi16(stp1[2], stp1[29]);
+ col[30] = _mm_sub_epi16(stp1[1], stp1[30]);
+ col[31] = _mm_sub_epi16(stp1[0], stp1[31]);
+ for (i = 0; i < 4; i++) {
+ int j;
+ // Transpose 32x8 block to 8x32 block
+ array_transpose_8x8(col + i * 8, in);
+ idct32_34(in, stp1);
+
+ // 2_D: Calculate the results and store them to destination.
+ in[0] = _mm_add_epi16(stp1[0], stp1[31]);
+ in[1] = _mm_add_epi16(stp1[1], stp1[30]);
+ in[2] = _mm_add_epi16(stp1[2], stp1[29]);
+ in[3] = _mm_add_epi16(stp1[3], stp1[28]);
+ in[4] = _mm_add_epi16(stp1[4], stp1[27]);
+ in[5] = _mm_add_epi16(stp1[5], stp1[26]);
+ in[6] = _mm_add_epi16(stp1[6], stp1[25]);
+ in[7] = _mm_add_epi16(stp1[7], stp1[24]);
+ in[8] = _mm_add_epi16(stp1[8], stp1[23]);
+ in[9] = _mm_add_epi16(stp1[9], stp1[22]);
+ in[10] = _mm_add_epi16(stp1[10], stp1[21]);
+ in[11] = _mm_add_epi16(stp1[11], stp1[20]);
+ in[12] = _mm_add_epi16(stp1[12], stp1[19]);
+ in[13] = _mm_add_epi16(stp1[13], stp1[18]);
+ in[14] = _mm_add_epi16(stp1[14], stp1[17]);
+ in[15] = _mm_add_epi16(stp1[15], stp1[16]);
+ in[16] = _mm_sub_epi16(stp1[15], stp1[16]);
+ in[17] = _mm_sub_epi16(stp1[14], stp1[17]);
+ in[18] = _mm_sub_epi16(stp1[13], stp1[18]);
+ in[19] = _mm_sub_epi16(stp1[12], stp1[19]);
+ in[20] = _mm_sub_epi16(stp1[11], stp1[20]);
+ in[21] = _mm_sub_epi16(stp1[10], stp1[21]);
+ in[22] = _mm_sub_epi16(stp1[9], stp1[22]);
+ in[23] = _mm_sub_epi16(stp1[8], stp1[23]);
+ in[24] = _mm_sub_epi16(stp1[7], stp1[24]);
+ in[25] = _mm_sub_epi16(stp1[6], stp1[25]);
+ in[26] = _mm_sub_epi16(stp1[5], stp1[26]);
+ in[27] = _mm_sub_epi16(stp1[4], stp1[27]);
+ in[28] = _mm_sub_epi16(stp1[3], stp1[28]);
+ in[29] = _mm_sub_epi16(stp1[2], stp1[29]);
+ in[30] = _mm_sub_epi16(stp1[1], stp1[30]);
+ in[31] = _mm_sub_epi16(stp1[0], stp1[31]);
+
+ for (j = 0; j < 32; ++j) {
+ // Final rounding and shift
+ in[j] = _mm_adds_epi16(in[j], final_rounding);
+ in[j] = _mm_srai_epi16(in[j], 6);
+ RECON_AND_STORE(dest + j * stride, in[j]);
+ }
+
+ dest += 8;
+ }
+}
projects / platform / upstream / libvpx.git / commitdiff