Update butterfly() in idct x86 optimizations.

Change-Id: Ic73e03bab9fdc085146f52094014db4af36ad701

diff --git a/vpx_dsp/x86/inv_txfm_sse2.c b/vpx_dsp/x86/inv_txfm_sse2.c
index 1df1c9d..64e068c 100644 (file)
--- a/vpx_dsp/x86/inv_txfm_sse2.c
+++ b/vpx_dsp/x86/inv_txfm_sse2.c
@@ -1324,30 +1324,17 @@ static void idct32_full_8x32_quarter_1(const __m128i *in /*in[32]*/,
   __m128i u0, u1, u2, u3, u4, u5, u6, u7;  // stp1_
   __m128i v0, v1, v2, v3, v4, v5, v6, v7;  // stp2_
 
-  {
-    const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-    const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64);
-    const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64);
-    const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64);
-    butterfly(&in[4], &in[28], &stg3_0, &stg3_1, &u4, &u7);
-    butterfly(&in[20], &in[12], &stg3_2, &stg3_3, &u5, &u6);
-  }
+  butterfly(in[4], in[28], (int)cospi_28_64, (int)cospi_4_64, &u4, &u7);
+  butterfly(in[20], in[12], (int)cospi_12_64, (int)cospi_20_64, &u5, &u6);
 
   v4 = _mm_add_epi16(u4, u5);
   v5 = _mm_sub_epi16(u4, u5);
   v6 = _mm_sub_epi16(u7, u6);
   v7 = _mm_add_epi16(u7, u6);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-    const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64);
-    const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64);
-    butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6);
-
-    butterfly(&in[0], &in[16], &stg4_0, &stg4_1, &u0, &u1);
-    butterfly(&in[8], &in[24], &stg4_2, &stg4_3, &u2, &u3);
-  }
+  butterfly(v6, v5, (int)cospi_16_64, (int)cospi_16_64, &v5, &v6);
+  butterfly(in[0], in[16], (int)cospi_16_64, (int)cospi_16_64, &u1, &u0);
+  butterfly(in[8], in[24], (int)cospi_24_64, (int)cospi_8_64, &u2, &u3);
 
   v0 = _mm_add_epi16(u0, u3);
   v1 = _mm_add_epi16(u1, u2);
@@ -1372,41 +1359,24 @@ static void idct32_full_8x32_quarter_2(const __m128i *in /*in[32]*/,
   __m128i u8, u9, u10, u11, u12, u13, u14, u15;  // stp2_
   __m128i v8, v9, v10, v11, v12, v13, v14, v15;  // stp1_
 
-  {
-    const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64);
-    const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64);
-    const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64);
-    const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64);
-    butterfly(&in[2], &in[30], &stg2_0, &stg2_1, &u8, &u15);
-    butterfly(&in[18], &in[14], &stg2_2, &stg2_3, &u9, &u14);
-  }
+  butterfly(in[2], in[30], (int)cospi_30_64, (int)cospi_2_64, &u8, &u15);
+  butterfly(in[18], in[14], (int)cospi_14_64, (int)cospi_18_64, &u9, &u14);
 
   v8 = _mm_add_epi16(u8, u9);
   v9 = _mm_sub_epi16(u8, u9);
   v14 = _mm_sub_epi16(u15, u14);
   v15 = _mm_add_epi16(u15, u14);
 
-  {
-    const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64);
-    const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64);
-    const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64);
-    const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64);
-    butterfly(&in[10], &in[22], &stg2_4, &stg2_5, &u10, &u13);
-    butterfly(&in[26], &in[6], &stg2_6, &stg2_7, &u11, &u12);
-  }
+  butterfly(in[10], in[22], (int)cospi_22_64, (int)cospi_10_64, &u10, &u13);
+  butterfly(in[26], in[6], (int)cospi_6_64, (int)cospi_26_64, &u11, &u12);
 
   v10 = _mm_sub_epi16(u11, u10);
   v11 = _mm_add_epi16(u11, u10);
   v12 = _mm_add_epi16(u12, u13);
   v13 = _mm_sub_epi16(u12, u13);
 
-  {
-    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    butterfly_self(&v9, &v14, &stg4_4, &stg4_5);
-    butterfly_self(&v10, &v13, &stg4_6, &stg4_4);
-  }
+  butterfly(v14, v9, (int)cospi_24_64, (int)cospi_8_64, &v9, &v14);
+  butterfly(v13, v10, -(int)cospi_8_64, (int)cospi_24_64, &v10, &v13);
 
   out[0] = _mm_add_epi16(v8, v11);
   out[1] = _mm_add_epi16(v9, v10);
@@ -1418,12 +1388,10 @@ static void idct32_full_8x32_quarter_2(const __m128i *in /*in[32]*/,
   out[4] = _mm_sub_epi16(v15, v12);
   out[5] = _mm_sub_epi16(v14, v13);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-    butterfly_self(&out[2], &out[5], &stg6_0, &stg4_0);
-    butterfly_self(&out[3], &out[4], &stg6_0, &stg4_0);
-  }
+  butterfly(out[5], out[2], (int)cospi_16_64, (int)cospi_16_64, &out[2],
+            &out[5]);
+  butterfly(out[4], out[3], (int)cospi_16_64, (int)cospi_16_64, &out[3],
+            &out[4]);
 }
 
 // For each 8x32 block __m128i in[32],
@@ -1439,34 +1407,16 @@ static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/,
   __m128i u16, u17, u18, u19, u20, u21, u22, u23;
   __m128i u24, u25, u26, u27, u28, u29, u30, u31;
 
-  {
-    const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64);
-    const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64);
-    const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64);
-    const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64);
-    const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64);
-    const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64);
-    const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64);
-    const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64);
-    const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64);
-    const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64);
-    const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64);
-    const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64);
-    const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64);
-    const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64);
-    const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64);
-    const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64);
-    butterfly(&in[1], &in[31], &stg1_0, &stg1_1, &u16, &u31);
-    butterfly(&in[17], &in[15], &stg1_2, &stg1_3, &u17, &u30);
-    butterfly(&in[9], &in[23], &stg1_4, &stg1_5, &u18, &u29);
-    butterfly(&in[25], &in[7], &stg1_6, &stg1_7, &u19, &u28);
-
-    butterfly(&in[5], &in[27], &stg1_8, &stg1_9, &u20, &u27);
-    butterfly(&in[21], &in[11], &stg1_10, &stg1_11, &u21, &u26);
-
-    butterfly(&in[13], &in[19], &stg1_12, &stg1_13, &u22, &u25);
-    butterfly(&in[29], &in[3], &stg1_14, &stg1_15, &u23, &u24);
-  }
+  butterfly(in[1], in[31], (int)cospi_31_64, (int)cospi_1_64, &u16, &u31);
+  butterfly(in[17], in[15], (int)cospi_15_64, (int)cospi_17_64, &u17, &u30);
+  butterfly(in[9], in[23], (int)cospi_23_64, (int)cospi_9_64, &u18, &u29);
+  butterfly(in[25], in[7], (int)cospi_7_64, (int)cospi_25_64, &u19, &u28);
+
+  butterfly(in[5], in[27], (int)cospi_27_64, (int)cospi_5_64, &u20, &u27);
+  butterfly(in[21], in[11], (int)cospi_11_64, (int)cospi_21_64, &u21, &u26);
+
+  butterfly(in[13], in[19], (int)cospi_19_64, (int)cospi_13_64, &u22, &u25);
+  butterfly(in[29], in[3], (int)cospi_3_64, (int)cospi_29_64, &u23, &u24);
 
   v16 = _mm_add_epi16(u16, u17);
   v17 = _mm_sub_epi16(u16, u17);
@@ -1488,18 +1438,10 @@ static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/,
   v30 = _mm_sub_epi16(u31, u30);
   v31 = _mm_add_epi16(u31, u30);
 
-  {
-    const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
-    const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
-    const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
-    const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-    const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
-    const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
-    butterfly_self(&v17, &v30, &stg3_4, &stg3_5);
-    butterfly_self(&v18, &v29, &stg3_6, &stg3_4);
-    butterfly_self(&v21, &v26, &stg3_8, &stg3_9);
-    butterfly_self(&v22, &v25, &stg3_10, &stg3_8);
-  }
+  butterfly(v30, v17, (int)cospi_28_64, (int)cospi_4_64, &v17, &v30);
+  butterfly(v29, v18, -(int)cospi_4_64, (int)cospi_28_64, &v18, &v29);
+  butterfly(v26, v21, (int)cospi_12_64, (int)cospi_20_64, &v21, &v26);
+  butterfly(v25, v22, -(int)cospi_20_64, (int)cospi_12_64, &v22, &v25);
 
   u16 = _mm_add_epi16(v16, v19);
   u17 = _mm_add_epi16(v17, v18);
@@ -1520,15 +1462,10 @@ static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/,
   u30 = _mm_add_epi16(v29, v30);
   u31 = _mm_add_epi16(v28, v31);
 
-  {
-    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    butterfly_self(&u18, &u29, &stg4_4, &stg4_5);
-    butterfly_self(&u19, &u28, &stg4_4, &stg4_5);
-    butterfly_self(&u20, &u27, &stg4_6, &stg4_4);
-    butterfly_self(&u21, &u26, &stg4_6, &stg4_4);
-  }
+  butterfly(u29, u18, (int)cospi_24_64, (int)cospi_8_64, &u18, &u29);
+  butterfly(u28, u19, (int)cospi_24_64, (int)cospi_8_64, &u19, &u28);
+  butterfly(u27, u20, -(int)cospi_8_64, (int)cospi_24_64, &u20, &u27);
+  butterfly(u26, u21, -(int)cospi_8_64, (int)cospi_24_64, &u21, &u26);
 
   out[0] = _mm_add_epi16(u16, u23);
   out[1] = _mm_add_epi16(u17, u22);
@@ -1548,14 +1485,14 @@ static void idct32_full_8x32_quarter_3_4(const __m128i *in /*in[32]*/,
   out[14] = _mm_add_epi16(u25, u30);
   out[15] = _mm_add_epi16(u24, u31);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-    butterfly_self(&out[4], &out[11], &stg6_0, &stg4_0);
-    butterfly_self(&out[5], &out[10], &stg6_0, &stg4_0);
-    butterfly_self(&out[6], &out[9], &stg6_0, &stg4_0);
-    butterfly_self(&out[7], &out[8], &stg6_0, &stg4_0);
-  }
+  butterfly(out[11], out[4], (int)cospi_16_64, (int)cospi_16_64, &out[4],
+            &out[11]);
+  butterfly(out[10], out[5], (int)cospi_16_64, (int)cospi_16_64, &out[5],
+            &out[10]);
+  butterfly(out[9], out[6], (int)cospi_16_64, (int)cospi_16_64, &out[6],
+            &out[9]);
+  butterfly(out[8], out[7], (int)cospi_16_64, (int)cospi_16_64, &out[7],
+            &out[8]);
 }
 
 static void idct32_full_8x32_quarter_1_2(const __m128i *in /*in[32]*/,

diff --git a/vpx_dsp/x86/inv_txfm_sse2.h b/vpx_dsp/x86/inv_txfm_sse2.h
index 3d8acc0..cdfc958 100644 (file)
--- a/vpx_dsp/x86/inv_txfm_sse2.h
+++ b/vpx_dsp/x86/inv_txfm_sse2.h
@@ -90,6 +90,8 @@ static INLINE void multiplication_and_add(const __m128i in0, const __m128i in1,
   *res1 = idct_calc_wraplow_sse2(lo, hi, cst1);
 }
 
+#define butterfly multiplication_and_add
+
 // Functions to allow 8 bit optimisations to be used when profile 0 is used with
 // highbitdepth enabled
 static INLINE __m128i load_input_data4(const tran_low_t *data) {
@@ -308,47 +310,6 @@ static INLINE void add_sub_butterfly(const __m128i *in, __m128i *out,
   }
 }
 
-#define BUTTERFLY_PAIR(x0, x1, co0, co1)         \
-  do {                                           \
-    tmp0 = _mm_madd_epi16(x0, co0);              \
-    tmp1 = _mm_madd_epi16(x1, co0);              \
-    tmp2 = _mm_madd_epi16(x0, co1);              \
-    tmp3 = _mm_madd_epi16(x1, co1);              \
-    tmp0 = _mm_add_epi32(tmp0, rounding);        \
-    tmp1 = _mm_add_epi32(tmp1, rounding);        \
-    tmp2 = _mm_add_epi32(tmp2, rounding);        \
-    tmp3 = _mm_add_epi32(tmp3, rounding);        \
-    tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
-    tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
-    tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
-    tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
-  } while (0)
-
-static INLINE void butterfly(const __m128i *x0, const __m128i *x1,
-                             const __m128i *c0, const __m128i *c1, __m128i *y0,
-                             __m128i *y1) {
-  __m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
-  const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
-
-  u0 = _mm_unpacklo_epi16(*x0, *x1);
-  u1 = _mm_unpackhi_epi16(*x0, *x1);
-  BUTTERFLY_PAIR(u0, u1, *c0, *c1);
-  *y0 = _mm_packs_epi32(tmp0, tmp1);
-  *y1 = _mm_packs_epi32(tmp2, tmp3);
-}
-
-static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i *c0,
-                                  const __m128i *c1) {
-  __m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
-  const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
-
-  u0 = _mm_unpacklo_epi16(*x0, *x1);
-  u1 = _mm_unpackhi_epi16(*x0, *x1);
-  BUTTERFLY_PAIR(u0, u1, *c0, *c1);
-  *x0 = _mm_packs_epi32(tmp0, tmp1);
-  *x1 = _mm_packs_epi32(tmp2, tmp3);
-}
-
 static INLINE void idct8(const __m128i *const in /*in[8]*/,
                          __m128i *const out /*out[8]*/) {
   __m128i step1[8], step2[8];

diff --git a/vpx_dsp/x86/inv_txfm_ssse3.c b/vpx_dsp/x86/inv_txfm_ssse3.c
index f9b37fe..f1b838a 100644 (file)
--- a/vpx_dsp/x86/inv_txfm_ssse3.c
+++ b/vpx_dsp/x86/inv_txfm_ssse3.c
@@ -38,14 +38,7 @@ static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {
   const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
   const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
 
-  const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
   const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
-  const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-  const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-  const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-
-  const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
   __m128i u0, u1, u2, u3, u4, u5, u6, u7;
   __m128i x0, x1, x4, x5, x6, x7;
   __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15;
@@ -64,10 +57,12 @@ static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {
   stp1[15] = _mm_sub_epi16(v0, v15);
 
   // in[2], in[6]
-  u0 = _mm_mulhrs_epi16(in[2], stk2_0);             // stp2_8
-  u1 = _mm_mulhrs_epi16(in[6], stk2_6);             // stp2_11
-  butterfly(&u0, &u2, &stg4_4, &stg4_5, &u4, &u5);  // stp2_9, stp2_14
-  butterfly(&u1, &u3, &stg4_6, &stg4_4, &u6, &u7);  // stp2_10, stp2_13
+  u0 = _mm_mulhrs_epi16(in[2], stk2_0);  // stp2_8
+  u1 = _mm_mulhrs_epi16(in[6], stk2_6);  // stp2_11
+  butterfly(u2, u0, (int)cospi_24_64, (int)cospi_8_64, &u4,
+            &u5);  // stp2_9, stp2_14
+  butterfly(u3, u1, -(int)cospi_8_64, (int)cospi_24_64, &u6,
+            &u7);  // stp2_10, stp2_13
 
   v8 = _mm_add_epi16(u0, u1);
   v9 = _mm_add_epi16(u4, u6);
@@ -77,14 +72,14 @@ static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {
   v13 = _mm_sub_epi16(u5, u7);
   v14 = _mm_add_epi16(u5, u7);
 
-  butterfly_self(&v10, &v13, &stg6_0, &stg4_0);
-  butterfly_self(&v11, &v12, &stg6_0, &stg4_0);
+  butterfly(v13, v10, (int)cospi_16_64, (int)cospi_16_64, &v10, &v13);
+  butterfly(v12, v11, (int)cospi_16_64, (int)cospi_16_64, &v11, &v12);
 
   // 1, 14
   x1 = _mm_mulhrs_epi16(in[0], stk4_0);  // stp1[1], stk4_1 = stk4_0
   // stp1[2] = stp1[0], stp1[3] = stp1[1]
   x4 = _mm_mulhrs_epi16(in[4], stk3_0);  // stp1[4]
-  butterfly(&x7, &x4, &stg4_1, &stg4_0, &x5, &x6);
+  butterfly(x7, x4, (int)cospi_16_64, (int)cospi_16_64, &x5, &x6);
   v1 = _mm_add_epi16(x1, x6);  // stp2_1
   v2 = _mm_add_epi16(x0, x5);  // stp2_2
   stp1[1] = _mm_add_epi16(v1, v14);
@@ -125,19 +120,6 @@ static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {
   const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
   const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
   const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
-  const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
-  const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
-  const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
-  const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-  const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
-  const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
-
-  const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-  const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-
-  const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
   __m128i v16, v17, v18, v19, v20, v21, v22, v23;
   __m128i v24, v25, v26, v27, v28, v29, v30, v31;
   __m128i u16, u17, u18, u19, u20, u21, u22, u23;
@@ -155,10 +137,10 @@ static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {
   v23 = _mm_mulhrs_epi16(in[3], stk1_14);
   v24 = _mm_mulhrs_epi16(in[3], stk1_15);
 
-  butterfly(&v16, &v31, &stg3_4, &stg3_5, &v17, &v30);
-  butterfly(&v19, &v28, &stg3_6, &stg3_4, &v18, &v29);
-  butterfly(&v20, &v27, &stg3_8, &stg3_9, &v21, &v26);
-  butterfly(&v23, &v24, &stg3_10, &stg3_8, &v22, &v25);
+  butterfly(v31, v16, (int)cospi_28_64, (int)cospi_4_64, &v17, &v30);
+  butterfly(v28, v19, -(int)cospi_4_64, (int)cospi_28_64, &v18, &v29);
+  butterfly(v27, v20, (int)cospi_12_64, (int)cospi_20_64, &v21, &v26);
+  butterfly(v24, v23, -(int)cospi_20_64, (int)cospi_12_64, &v22, &v25);
 
   u16 = _mm_add_epi16(v16, v19);
   u17 = _mm_add_epi16(v17, v18);
@@ -177,10 +159,10 @@ static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {
   u29 = _mm_sub_epi16(v30, v29);
   u30 = _mm_add_epi16(v29, v30);
 
-  butterfly_self(&u18, &u29, &stg4_4, &stg4_5);
-  butterfly_self(&u19, &u28, &stg4_4, &stg4_5);
-  butterfly_self(&u20, &u27, &stg4_6, &stg4_4);
-  butterfly_self(&u21, &u26, &stg4_6, &stg4_4);
+  butterfly(u29, u18, (int)cospi_24_64, (int)cospi_8_64, &u18, &u29);
+  butterfly(u28, u19, (int)cospi_24_64, (int)cospi_8_64, &u19, &u28);
+  butterfly(u27, u20, -(int)cospi_8_64, (int)cospi_24_64, &u20, &u27);
+  butterfly(u26, u21, -(int)cospi_8_64, (int)cospi_24_64, &u21, &u26);
 
   stp1[16] = _mm_add_epi16(u16, u23);
   stp1[23] = _mm_sub_epi16(u16, u23);
@@ -206,10 +188,14 @@ static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {
   stp1[27] = _mm_sub_epi16(u28, u27);
   stp1[28] = _mm_add_epi16(u27, u28);
 
-  butterfly_self(&stp1[20], &stp1[27], &stg6_0, &stg4_0);
-  butterfly_self(&stp1[21], &stp1[26], &stg6_0, &stg4_0);
-  butterfly_self(&stp1[22], &stp1[25], &stg6_0, &stg4_0);
-  butterfly_self(&stp1[23], &stp1[24], &stg6_0, &stg4_0);
+  butterfly(stp1[27], stp1[20], (int)cospi_16_64, (int)cospi_16_64, &stp1[20],
+            &stp1[27]);
+  butterfly(stp1[26], stp1[21], (int)cospi_16_64, (int)cospi_16_64, &stp1[21],
+            &stp1[26]);
+  butterfly(stp1[25], stp1[22], (int)cospi_16_64, (int)cospi_16_64, &stp1[22],
+            &stp1[25]);
+  butterfly(stp1[24], stp1[23], (int)cospi_16_64, (int)cospi_16_64, &stp1[23],
+            &stp1[24]);
 }
 
 // Only upper-left 8x8 has non-zero coeff
@@ -309,11 +295,7 @@ static void idct32_8x32_135_quarter_1(const __m128i *in /*in[16]*/,
   v6 = _mm_sub_epi16(u7, u6);
   v7 = _mm_add_epi16(u7, u6);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-    butterfly(&v6, &v5, &stg4_1, &stg4_0, &v5, &v6);
-  }
+  butterfly(v6, v5, (int)cospi_16_64, (int)cospi_16_64, &v5, &v6);
 
   out[0] = _mm_add_epi16(v0, v7);
   out[1] = _mm_add_epi16(v1, v6);
@@ -358,13 +340,8 @@ static void idct32_8x32_135_quarter_2(const __m128i *in /*in[16]*/,
   v14 = _mm_sub_epi16(u15, u14);
   v15 = _mm_add_epi16(u15, u14);
 
-  {
-    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    butterfly_self(&v9, &v14, &stg4_4, &stg4_5);
-    butterfly_self(&v10, &v13, &stg4_6, &stg4_4);
-  }
+  butterfly(v14, v9, (int)cospi_24_64, (int)cospi_8_64, &v9, &v14);
+  butterfly(v13, v10, -(int)cospi_8_64, (int)cospi_24_64, &v10, &v13);
 
   out[0] = _mm_add_epi16(v8, v11);
   out[1] = _mm_add_epi16(v9, v10);
@@ -375,12 +352,10 @@ static void idct32_8x32_135_quarter_2(const __m128i *in /*in[16]*/,
   out[6] = _mm_add_epi16(v14, v13);
   out[7] = _mm_add_epi16(v15, v12);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-    butterfly_self(&out[2], &out[5], &stg6_0, &stg4_0);
-    butterfly_self(&out[3], &out[4], &stg6_0, &stg4_0);
-  }
+  butterfly(out[5], out[2], (int)cospi_16_64, (int)cospi_16_64, &out[2],
+            &out[5]);
+  butterfly(out[4], out[3], (int)cospi_16_64, (int)cospi_16_64, &out[3],
+            &out[4]);
 }
 
 // 8x32 block even indexed 8 inputs of in[16],
@@ -462,19 +437,10 @@ static void idct32_8x32_quarter_3_4(const __m128i *in /*in[16]*/,
   v30 = _mm_sub_epi16(u31, u30);
   v31 = _mm_add_epi16(u31, u30);
 
-  {
-    const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
-    const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
-    const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
-    const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-    const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
-    const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
-
-    butterfly_self(&v17, &v30, &stg3_4, &stg3_5);
-    butterfly_self(&v18, &v29, &stg3_6, &stg3_4);
-    butterfly_self(&v21, &v26, &stg3_8, &stg3_9);
-    butterfly_self(&v22, &v25, &stg3_10, &stg3_8);
-  }
+  butterfly(v30, v17, (int)cospi_28_64, (int)cospi_4_64, &v17, &v30);
+  butterfly(v29, v18, -(int)cospi_4_64, (int)cospi_28_64, &v18, &v29);
+  butterfly(v26, v21, (int)cospi_12_64, (int)cospi_20_64, &v21, &v26);
+  butterfly(v25, v22, -(int)cospi_20_64, (int)cospi_12_64, &v22, &v25);
 
   u16 = _mm_add_epi16(v16, v19);
   u17 = _mm_add_epi16(v17, v18);
@@ -494,15 +460,10 @@ static void idct32_8x32_quarter_3_4(const __m128i *in /*in[16]*/,
   u30 = _mm_add_epi16(v29, v30);
   u31 = _mm_add_epi16(v28, v31);
 
-  {
-    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    butterfly_self(&u18, &u29, &stg4_4, &stg4_5);
-    butterfly_self(&u19, &u28, &stg4_4, &stg4_5);
-    butterfly_self(&u20, &u27, &stg4_6, &stg4_4);
-    butterfly_self(&u21, &u26, &stg4_6, &stg4_4);
-  }
+  butterfly(u29, u18, (int)cospi_24_64, (int)cospi_8_64, &u18, &u29);
+  butterfly(u28, u19, (int)cospi_24_64, (int)cospi_8_64, &u19, &u28);
+  butterfly(u27, u20, -(int)cospi_8_64, (int)cospi_24_64, &u20, &u27);
+  butterfly(u26, u21, -(int)cospi_8_64, (int)cospi_24_64, &u21, &u26);
 
   out[0] = _mm_add_epi16(u16, u23);
   out[1] = _mm_add_epi16(u17, u22);
@@ -522,14 +483,10 @@ static void idct32_8x32_quarter_3_4(const __m128i *in /*in[16]*/,
   out[14] = _mm_add_epi16(u25, u30);
   out[15] = _mm_add_epi16(u24, u31);
 
-  {
-    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-    butterfly(&v20, &v27, &stg6_0, &stg4_0, &out[4], &out[11]);
-    butterfly(&v21, &v26, &stg6_0, &stg4_0, &out[5], &out[10]);
-    butterfly(&v22, &v25, &stg6_0, &stg4_0, &out[6], &out[9]);
-    butterfly(&v23, &v24, &stg6_0, &stg4_0, &out[7], &out[8]);
-  }
+  butterfly(v27, v20, (int)cospi_16_64, (int)cospi_16_64, &out[4], &out[11]);
+  butterfly(v26, v21, (int)cospi_16_64, (int)cospi_16_64, &out[5], &out[10]);
+  butterfly(v25, v22, (int)cospi_16_64, (int)cospi_16_64, &out[6], &out[9]);
+  butterfly(v24, v23, (int)cospi_16_64, (int)cospi_16_64, &out[7], &out[8]);
 }
 
 // 8x16 block, input __m128i in[16], output __m128i in[32]