simd/loongson/jidctint-mmi.c

   1 /*
   2  * Loongson MMI optimizations for libjpeg-turbo
   3  *
   4  * Copyright (C) 2014-2015, 2018, D. R. Commander.  All Rights Reserved.
   5  * Copyright (C) 2016-2017, Loongson Technology Corporation Limited, BeiJing.
   6  *                          All Rights Reserved.
   7  * Authors:  ZhuChen     <zhuchen@loongson.cn>
   8  *           CaiWanwei   <caiwanwei@loongson.cn>
   9  *           SunZhangzhi <sunzhangzhi-cq@loongson.cn>
  10  *
  11  * Based on the x86 SIMD extension for IJG JPEG library
  12  * Copyright (C) 1999-2006, MIYASAKA Masaru.
  13  *
  14  * This software is provided 'as-is', without any express or implied
  15  * warranty.  In no event will the authors be held liable for any damages
  16  * arising from the use of this software.
  17  *
  18  * Permission is granted to anyone to use this software for any purpose,
  19  * including commercial applications, and to alter it and redistribute it
  20  * freely, subject to the following restrictions:
  21  *
  22  * 1. The origin of this software must not be misrepresented; you must not
  23  *    claim that you wrote the original software. If you use this software
  24  *    in a product, an acknowledgment in the product documentation would be
  25  *    appreciated but is not required.
  26  * 2. Altered source versions must be plainly marked as such, and must not be
  27  *    misrepresented as being the original software.
  28  * 3. This notice may not be removed or altered from any source distribution.
  29  */
  30
  31 /* SLOW INTEGER INVERSE DCT */
  32
  33 #include "jsimd_mmi.h"
  34
  35
  36 #define CONST_BITS  13
  37 #define PASS1_BITS  2
  38 #define DESCALE_P1  (CONST_BITS - PASS1_BITS)
  39 #define DESCALE_P2  (CONST_BITS + PASS1_BITS + 3)
  40 #define CENTERJSAMPLE  128
  41
  42 #define FIX_0_298  ((short)2446)  /* FIX(0.298631336) */
  43 #define FIX_0_390  ((short)3196)  /* FIX(0.390180644) */
  44 #define FIX_0_899  ((short)7373)  /* FIX(0.899976223) */
  45 #define FIX_0_541  ((short)4433)  /* FIX(0.541196100) */
  46 #define FIX_0_765  ((short)6270)  /* FIX(0.765366865) */
  47 #define FIX_1_175  ((short)9633)  /* FIX(1.175875602) */
  48 #define FIX_1_501  ((short)12299) /* FIX(1.501321110) */
  49 #define FIX_1_847  ((short)15137) /* FIX(1.847759065) */
  50 #define FIX_1_961  ((short)16069) /* FIX(1.961570560) */
  51 #define FIX_2_053  ((short)16819) /* FIX(2.053119869) */
  52 #define FIX_2_562  ((short)20995) /* FIX(2.562915447) */
  53 #define FIX_3_072  ((short)25172) /* FIX(3.072711026) */
  54
  55 enum const_index {
  56   index_PW_F130_F054,
  57   index_PW_F054_MF130,
  58   index_PW_MF078_F117,
  59   index_PW_F117_F078,
  60   index_PW_MF060_MF089,
  61   index_PW_MF089_F060,
  62   index_PW_MF050_MF256,
  63   index_PW_MF256_F050,
  64   index_PD_DESCALE_P1,
  65   index_PD_DESCALE_P2,
  66   index_PB_CENTERJSAMP
  67 };
  68
  69 static uint64_t const_value[] = {
  70   _uint64_set_pi16(FIX_0_541, (FIX_0_541 + FIX_0_765),
  71                    FIX_0_541, (FIX_0_541 + FIX_0_765)),
  72   _uint64_set_pi16((FIX_0_541 - FIX_1_847), FIX_0_541,
  73                    (FIX_0_541 - FIX_1_847), FIX_0_541),
  74   _uint64_set_pi16(FIX_1_175, (FIX_1_175 - FIX_1_961),
  75                    FIX_1_175, (FIX_1_175 - FIX_1_961)),
  76   _uint64_set_pi16((FIX_1_175 - FIX_0_390), FIX_1_175,
  77                    (FIX_1_175 - FIX_0_390), FIX_1_175),
  78   _uint64_set_pi16(-FIX_0_899, (FIX_0_298 - FIX_0_899),
  79                    -FIX_0_899, (FIX_0_298 - FIX_0_899)),
  80   _uint64_set_pi16((FIX_1_501 - FIX_0_899), -FIX_0_899,
  81                    (FIX_1_501 - FIX_0_899), -FIX_0_899),
  82   _uint64_set_pi16(-FIX_2_562, (FIX_2_053 - FIX_2_562),
  83                    -FIX_2_562, (FIX_2_053 - FIX_2_562)),
  84   _uint64_set_pi16((FIX_3_072 - FIX_2_562), -FIX_2_562,
  85                    (FIX_3_072 - FIX_2_562), -FIX_2_562),
  86   _uint64_set_pi32((1 << (DESCALE_P1 - 1)), (1 << (DESCALE_P1 - 1))),
  87   _uint64_set_pi32((1 << (DESCALE_P2 - 1)), (1 << (DESCALE_P2 - 1))),
  88   _uint64_set_pi8(CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE,
  89                   CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE)
  90 };
  91
  92 #define PW_F130_F054    get_const_value(index_PW_F130_F054)
  93 #define PW_F054_MF130   get_const_value(index_PW_F054_MF130)
  94 #define PW_MF078_F117   get_const_value(index_PW_MF078_F117)
  95 #define PW_F117_F078    get_const_value(index_PW_F117_F078)
  96 #define PW_MF060_MF089  get_const_value(index_PW_MF060_MF089)
  97 #define PW_MF089_F060   get_const_value(index_PW_MF089_F060)
  98 #define PW_MF050_MF256  get_const_value(index_PW_MF050_MF256)
  99 #define PW_MF256_F050   get_const_value(index_PW_MF256_F050)
 100 #define PD_DESCALE_P1   get_const_value(index_PD_DESCALE_P1)
 101 #define PD_DESCALE_P2   get_const_value(index_PD_DESCALE_P2)
 102 #define PB_CENTERJSAMP  get_const_value(index_PB_CENTERJSAMP)
 103
 104
 105 #define test_m32_zero(mm32)  (!(*(uint32_t *)&mm32))
 106 #define test_m64_zero(mm64)  (!(*(uint64_t *)&mm64))
 107
 108
 109 #define DO_IDCT_COMMON(PASS) { \
 110   __m64 tmp0_3l, tmp0_3h, tmp1_2l, tmp1_2h; \
 111   __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \
 112   __m64 z34l, z34h, z3l, z3h, z4l, z4h, z3, z4; \
 113   __m64 out0l, out0h, out1l, out1h, out2l, out2h, out3l, out3h; \
 114   __m64 out4l, out4h, out5l, out5h, out6l, out6h, out7l, out7h; \
 115   \
 116   z3 = _mm_add_pi16(tmp0, tmp2); \
 117   z4 = _mm_add_pi16(tmp1, tmp3); \
 118   \
 119   /* (Original) \
 120    * z5 = (z3 + z4) * 1.175875602; \
 121    * z3 = z3 * -1.961570560;  z4 = z4 * -0.390180644; \
 122    * z3 += z5;  z4 += z5; \
 123    * \
 124    * (This implementation) \
 125    * z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602; \
 126    * z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644); \
 127    */ \
 128   \
 129   z34l = _mm_unpacklo_pi16(z3, z4); \
 130   z34h = _mm_unpackhi_pi16(z3, z4); \
 131   z3l = _mm_madd_pi16(z34l, PW_MF078_F117); \
 132   z3h = _mm_madd_pi16(z34h, PW_MF078_F117); \
 133   z4l = _mm_madd_pi16(z34l, PW_F117_F078); \
 134   z4h = _mm_madd_pi16(z34h, PW_F117_F078); \
 135   \
 136   /* (Original) \
 137    * z1 = tmp0 + tmp3;  z2 = tmp1 + tmp2; \
 138    * tmp0 = tmp0 * 0.298631336;  tmp1 = tmp1 * 2.053119869; \
 139    * tmp2 = tmp2 * 3.072711026;  tmp3 = tmp3 * 1.501321110; \
 140    * z1 = z1 * -0.899976223;  z2 = z2 * -2.562915447; \
 141    * tmp0 += z1 + z3;  tmp1 += z2 + z4; \
 142    * tmp2 += z2 + z3;  tmp3 += z1 + z4; \
 143    * \
 144    * (This implementation) \
 145    * tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223; \
 146    * tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447; \
 147    * tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447); \
 148    * tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223); \
 149    * tmp0 += z3;  tmp1 += z4; \
 150    * tmp2 += z3;  tmp3 += z4; \
 151    */ \
 152   \
 153   tmp0_3l = _mm_unpacklo_pi16(tmp0, tmp3); \
 154   tmp0_3h = _mm_unpackhi_pi16(tmp0, tmp3); \
 155   \
 156   tmp0l = _mm_madd_pi16(tmp0_3l, PW_MF060_MF089); \
 157   tmp0h = _mm_madd_pi16(tmp0_3h, PW_MF060_MF089); \
 158   tmp3l = _mm_madd_pi16(tmp0_3l, PW_MF089_F060); \
 159   tmp3h = _mm_madd_pi16(tmp0_3h, PW_MF089_F060); \
 160   \
 161   tmp0l = _mm_add_pi32(tmp0l, z3l); \
 162   tmp0h = _mm_add_pi32(tmp0h, z3h); \
 163   tmp3l = _mm_add_pi32(tmp3l, z4l); \
 164   tmp3h = _mm_add_pi32(tmp3h, z4h); \
 165   \
 166   tmp1_2l = _mm_unpacklo_pi16(tmp1, tmp2); \
 167   tmp1_2h = _mm_unpackhi_pi16(tmp1, tmp2); \
 168   \
 169   tmp1l = _mm_madd_pi16(tmp1_2l, PW_MF050_MF256); \
 170   tmp1h = _mm_madd_pi16(tmp1_2h, PW_MF050_MF256); \
 171   tmp2l = _mm_madd_pi16(tmp1_2l, PW_MF256_F050); \
 172   tmp2h = _mm_madd_pi16(tmp1_2h, PW_MF256_F050); \
 173   \
 174   tmp1l = _mm_add_pi32(tmp1l, z4l); \
 175   tmp1h = _mm_add_pi32(tmp1h, z4h); \
 176   tmp2l = _mm_add_pi32(tmp2l, z3l); \
 177   tmp2h = _mm_add_pi32(tmp2h, z3h); \
 178   \
 179   /* Final output stage */ \
 180   \
 181   out0l = _mm_add_pi32(tmp10l, tmp3l); \
 182   out0h = _mm_add_pi32(tmp10h, tmp3h); \
 183   out7l = _mm_sub_pi32(tmp10l, tmp3l); \
 184   out7h = _mm_sub_pi32(tmp10h, tmp3h); \
 185   \
 186   out0l = _mm_add_pi32(out0l, PD_DESCALE_P##PASS); \
 187   out0h = _mm_add_pi32(out0h, PD_DESCALE_P##PASS); \
 188   out0l = _mm_srai_pi32(out0l, DESCALE_P##PASS); \
 189   out0h = _mm_srai_pi32(out0h, DESCALE_P##PASS); \
 190   \
 191   out7l = _mm_add_pi32(out7l, PD_DESCALE_P##PASS); \
 192   out7h = _mm_add_pi32(out7h, PD_DESCALE_P##PASS); \
 193   out7l = _mm_srai_pi32(out7l, DESCALE_P##PASS); \
 194   out7h = _mm_srai_pi32(out7h, DESCALE_P##PASS); \
 195   \
 196   out0 = _mm_packs_pi32(out0l, out0h); \
 197   out7 = _mm_packs_pi32(out7l, out7h); \
 198   \
 199   out1l = _mm_add_pi32(tmp11l, tmp2l); \
 200   out1h = _mm_add_pi32(tmp11h, tmp2h); \
 201   out6l = _mm_sub_pi32(tmp11l, tmp2l); \
 202   out6h = _mm_sub_pi32(tmp11h, tmp2h); \
 203   \
 204   out1l = _mm_add_pi32(out1l, PD_DESCALE_P##PASS); \
 205   out1h = _mm_add_pi32(out1h, PD_DESCALE_P##PASS); \
 206   out1l = _mm_srai_pi32(out1l, DESCALE_P##PASS); \
 207   out1h = _mm_srai_pi32(out1h, DESCALE_P##PASS); \
 208   \
 209   out6l = _mm_add_pi32(out6l, PD_DESCALE_P##PASS); \
 210   out6h = _mm_add_pi32(out6h, PD_DESCALE_P##PASS); \
 211   out6l = _mm_srai_pi32(out6l, DESCALE_P##PASS); \
 212   out6h = _mm_srai_pi32(out6h, DESCALE_P##PASS); \
 213   \
 214   out1 = _mm_packs_pi32(out1l, out1h); \
 215   out6 = _mm_packs_pi32(out6l, out6h); \
 216   \
 217   out2l = _mm_add_pi32(tmp12l, tmp1l); \
 218   out2h = _mm_add_pi32(tmp12h, tmp1h); \
 219   out5l = _mm_sub_pi32(tmp12l, tmp1l); \
 220   out5h = _mm_sub_pi32(tmp12h, tmp1h); \
 221   \
 222   out2l = _mm_add_pi32(out2l, PD_DESCALE_P##PASS); \
 223   out2h = _mm_add_pi32(out2h, PD_DESCALE_P##PASS); \
 224   out2l = _mm_srai_pi32(out2l, DESCALE_P##PASS); \
 225   out2h = _mm_srai_pi32(out2h, DESCALE_P##PASS); \
 226   \
 227   out5l = _mm_add_pi32(out5l, PD_DESCALE_P##PASS); \
 228   out5h = _mm_add_pi32(out5h, PD_DESCALE_P##PASS); \
 229   out5l = _mm_srai_pi32(out5l, DESCALE_P##PASS); \
 230   out5h = _mm_srai_pi32(out5h, DESCALE_P##PASS); \
 231   \
 232   out2 = _mm_packs_pi32(out2l, out2h); \
 233   out5 = _mm_packs_pi32(out5l, out5h); \
 234   \
 235   out3l = _mm_add_pi32(tmp13l, tmp0l); \
 236   out3h = _mm_add_pi32(tmp13h, tmp0h); \
 237   \
 238   out4l = _mm_sub_pi32(tmp13l, tmp0l); \
 239   out4h = _mm_sub_pi32(tmp13h, tmp0h); \
 240   \
 241   out3l = _mm_add_pi32(out3l, PD_DESCALE_P##PASS); \
 242   out3h = _mm_add_pi32(out3h, PD_DESCALE_P##PASS); \
 243   out3l = _mm_srai_pi32(out3l, DESCALE_P##PASS); \
 244   out3h = _mm_srai_pi32(out3h, DESCALE_P##PASS); \
 245   \
 246   out4l = _mm_add_pi32(out4l, PD_DESCALE_P##PASS); \
 247   out4h = _mm_add_pi32(out4h, PD_DESCALE_P##PASS); \
 248   out4l = _mm_srai_pi32(out4l, DESCALE_P##PASS); \
 249   out4h = _mm_srai_pi32(out4h, DESCALE_P##PASS); \
 250   \
 251   out3 = _mm_packs_pi32(out3l, out3h); \
 252   out4 = _mm_packs_pi32(out4l, out4h); \
 253 }
 254
 255 #define DO_IDCT_PASS1(iter) { \
 256   __m64 col0l, col1l, col2l, col3l, col4l, col5l, col6l, col7l; \
 257   __m64 quant0l, quant1l, quant2l, quant3l; \
 258   __m64 quant4l, quant5l, quant6l, quant7l; \
 259   __m64 z23, z2, z3, z23l, z23h; \
 260   __m64 row01a, row01b, row01c, row01d, row23a, row23b, row23c, row23d; \
 261   __m64 row0l, row0h, row1l, row1h, row2l, row2h, row3l, row3h; \
 262   __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \
 263   __m64 tmp10l, tmp10h, tmp11l, tmp11h, tmp12l, tmp12h, tmp13l, tmp13h; \
 264   __m32 col0a, col1a, mm0; \
 265   \
 266   col0a = _mm_load_si32((__m32 *)&inptr[DCTSIZE * 1]); \
 267   col1a = _mm_load_si32((__m32 *)&inptr[DCTSIZE * 2]); \
 268   mm0 = _mm_or_si32(col0a, col1a); \
 269   \
 270   if (test_m32_zero(mm0)) { \
 271     __m64 mm1, mm2; \
 272     \
 273     col0l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 0]); \
 274     col1l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 1]); \
 275     col2l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 2]); \
 276     col3l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 3]); \
 277     col4l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 4]); \
 278     col5l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 5]); \
 279     col6l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 6]); \
 280     col7l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 7]); \
 281     \
 282     mm1 = _mm_or_si64(col1l, col3l); \
 283     mm2 = _mm_or_si64(col2l, col4l); \
 284     mm1 = _mm_or_si64(mm1, col5l); \
 285     mm2 = _mm_or_si64(mm2, col6l); \
 286     mm1 = _mm_or_si64(mm1, col7l); \
 287     mm1 = _mm_or_si64(mm1, mm2); \
 288     \
 289     if (test_m64_zero(mm1)) { \
 290       __m64 dcval, dcvall, dcvalh, row0, row1, row2, row3; \
 291       \
 292       /* AC terms all zero */ \
 293       \
 294       quant0l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 0]); \
 295       \
 296       dcval = _mm_mullo_pi16(col0l, quant0l); \
 297       dcval = _mm_slli_pi16(dcval, PASS1_BITS);  /* dcval=(00 10 20 30) */ \
 298       \
 299       dcvall = _mm_unpacklo_pi16(dcval, dcval);  /* dcvall=(00 00 10 10) */ \
 300       dcvalh = _mm_unpackhi_pi16(dcval, dcval);  /* dcvalh=(20 20 30 30) */ \
 301       \
 302       row0 = _mm_unpacklo_pi32(dcvall, dcvall);  /* row0=(00 00 00 00) */ \
 303       row1 = _mm_unpackhi_pi32(dcvall, dcvall);  /* row1=(10 10 10 10) */ \
 304       row2 = _mm_unpacklo_pi32(dcvalh, dcvalh);  /* row2=(20 20 20 20) */ \
 305       row3 = _mm_unpackhi_pi32(dcvalh, dcvalh);  /* row3=(30 30 30 30) */ \
 306       \
 307       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0], row0); \
 308       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0 + 4], row0); \
 309       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1], row1); \
 310       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1 + 4], row1); \
 311       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2], row2); \
 312       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2 + 4], row2); \
 313       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3], row3); \
 314       _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3 + 4], row3); \
 315       \
 316       goto nextcolumn##iter; \
 317     } \
 318   } \
 319   \
 320   /* Even part \
 321    * \
 322    * (Original) \
 323    * z1 = (z2 + z3) * 0.541196100; \
 324    * tmp2 = z1 + z3 * -1.847759065; \
 325    * tmp3 = z1 + z2 * 0.765366865; \
 326    * \
 327    * (This implementation) \
 328    * tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065); \
 329    * tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100; \
 330    */ \
 331   \
 332   col0l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 0]);  /* (00 10 20 30) */ \
 333   col2l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 2]);  /* (02 12 22 32) */ \
 334   col4l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 4]);  /* (04 14 24 34) */ \
 335   col6l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 6]);  /* (06 16 26 36) */ \
 336   \
 337   quant0l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 0]); \
 338   quant2l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 2]); \
 339   quant4l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 4]); \
 340   quant6l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 6]); \
 341   \
 342   z2 = _mm_mullo_pi16(col2l, quant2l); \
 343   z3 = _mm_mullo_pi16(col6l, quant6l); \
 344   \
 345   z23l = _mm_unpacklo_pi16(z2, z3); \
 346   z23h = _mm_unpackhi_pi16(z2, z3); \
 347   tmp3l = _mm_madd_pi16(z23l, PW_F130_F054); \
 348   tmp3h = _mm_madd_pi16(z23h, PW_F130_F054); \
 349   tmp2l = _mm_madd_pi16(z23l, PW_F054_MF130); \
 350   tmp2h = _mm_madd_pi16(z23h, PW_F054_MF130); \
 351   \
 352   z2 = _mm_mullo_pi16(col0l, quant0l); \
 353   z3 = _mm_mullo_pi16(col4l, quant4l); \
 354   \
 355   z23 = _mm_add_pi16(z2, z3); \
 356   tmp0l = _mm_loadlo_pi16_f(z23); \
 357   tmp0h = _mm_loadhi_pi16_f(z23); \
 358   tmp0l = _mm_srai_pi32(tmp0l, (16 - CONST_BITS)); \
 359   tmp0h = _mm_srai_pi32(tmp0h, (16 - CONST_BITS)); \
 360   \
 361   tmp10l = _mm_add_pi32(tmp0l, tmp3l); \
 362   tmp10h = _mm_add_pi32(tmp0h, tmp3h); \
 363   tmp13l = _mm_sub_pi32(tmp0l, tmp3l); \
 364   tmp13h = _mm_sub_pi32(tmp0h, tmp3h); \
 365   \
 366   z23 = _mm_sub_pi16(z2, z3); \
 367   tmp1l = _mm_loadlo_pi16_f(z23); \
 368   tmp1h = _mm_loadhi_pi16_f(z23); \
 369   tmp1l = _mm_srai_pi32(tmp1l, (16 - CONST_BITS)); \
 370   tmp1h = _mm_srai_pi32(tmp1h, (16 - CONST_BITS)); \
 371   \
 372   tmp11l = _mm_add_pi32(tmp1l, tmp2l); \
 373   tmp11h = _mm_add_pi32(tmp1h, tmp2h); \
 374   tmp12l = _mm_sub_pi32(tmp1l, tmp2l); \
 375   tmp12h = _mm_sub_pi32(tmp1h, tmp2h); \
 376   \
 377   /* Odd part */ \
 378   \
 379   col1l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 1]);  /* (01 11 21 31) */ \
 380   col3l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 3]);  /* (03 13 23 33) */ \
 381   col5l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 5]);  /* (05 15 25 35) */ \
 382   col7l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 7]);  /* (07 17 27 37) */ \
 383   \
 384   quant1l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 1]); \
 385   quant3l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 3]); \
 386   quant5l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 5]); \
 387   quant7l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 7]); \
 388   \
 389   tmp0 = _mm_mullo_pi16(col7l, quant7l); \
 390   tmp1 = _mm_mullo_pi16(col5l, quant5l); \
 391   tmp2 = _mm_mullo_pi16(col3l, quant3l); \
 392   tmp3 = _mm_mullo_pi16(col1l, quant1l); \
 393   \
 394   DO_IDCT_COMMON(1) \
 395   \
 396   /* out0=(00 10 20 30), out1=(01 11 21 31) */ \
 397   /* out2=(02 12 22 32), out3=(03 13 23 33) */ \
 398   /* out4=(04 14 24 34), out5=(05 15 25 35) */ \
 399   /* out6=(06 16 26 36), out7=(07 17 27 37) */ \
 400   \
 401   /* Transpose coefficients */ \
 402   \
 403   row01a = _mm_unpacklo_pi16(out0, out1);     /* row01a=(00 01 10 11) */ \
 404   row23a = _mm_unpackhi_pi16(out0, out1);     /* row23a=(20 21 30 31) */ \
 405   row01d = _mm_unpacklo_pi16(out6, out7);     /* row01d=(06 07 16 17) */ \
 406   row23d = _mm_unpackhi_pi16(out6, out7);     /* row23d=(26 27 36 37) */ \
 407   \
 408   row01b = _mm_unpacklo_pi16(out2, out3);     /* row01b=(02 03 12 13) */ \
 409   row23b = _mm_unpackhi_pi16(out2, out3);     /* row23b=(22 23 32 33) */ \
 410   row01c = _mm_unpacklo_pi16(out4, out5);     /* row01c=(04 05 14 15) */ \
 411   row23c = _mm_unpackhi_pi16(out4, out5);     /* row23c=(24 25 34 35) */ \
 412   \
 413   row0l = _mm_unpacklo_pi32(row01a, row01b);  /* row0l=(00 01 02 03) */ \
 414   row1l = _mm_unpackhi_pi32(row01a, row01b);  /* row1l=(10 11 12 13) */ \
 415   row2l = _mm_unpacklo_pi32(row23a, row23b);  /* row2l=(20 21 22 23) */ \
 416   row3l = _mm_unpackhi_pi32(row23a, row23b);  /* row3l=(30 31 32 33) */ \
 417   \
 418   row0h = _mm_unpacklo_pi32(row01c, row01d);  /* row0h=(04 05 06 07) */ \
 419   row1h = _mm_unpackhi_pi32(row01c, row01d);  /* row1h=(14 15 16 17) */ \
 420   row2h = _mm_unpacklo_pi32(row23c, row23d);  /* row2h=(24 25 26 27) */ \
 421   row3h = _mm_unpackhi_pi32(row23c, row23d);  /* row3h=(34 35 36 37) */ \
 422   \
 423   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0], row0l); \
 424   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0 + 4], row0h); \
 425   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1], row1l); \
 426   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1 + 4], row1h); \
 427   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2], row2l); \
 428   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2 + 4], row2h); \
 429   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3], row3l); \
 430   _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3 + 4], row3h); \
 431 }
 432
 433 #define DO_IDCT_PASS2(ctr) { \
 434   __m64 row0l, row1l, row2l, row3l, row4l, row5l, row6l, row7l; \
 435   __m64 z23, z23l, z23h; \
 436   __m64 col0123a, col0123b, col0123c, col0123d; \
 437   __m64 col01l, col01h, col23l, col23h, row06, row17, row24, row35; \
 438   __m64 col0, col1, col2, col3; \
 439   __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \
 440   __m64 tmp10l, tmp10h, tmp11l, tmp11h, tmp12l, tmp12h, tmp13l, tmp13h; \
 441   \
 442   row0l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 0]);  /* (00 01 02 03) */ \
 443   row1l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 1]);  /* (10 11 12 13) */ \
 444   row2l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 2]);  /* (20 21 22 23) */ \
 445   row3l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 3]);  /* (30 31 32 33) */ \
 446   row4l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 4]);  /* (40 41 42 43) */ \
 447   row5l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 5]);  /* (50 51 52 53) */ \
 448   row6l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 6]);  /* (60 61 62 63) */ \
 449   row7l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 7]);  /* (70 71 72 73) */ \
 450   \
 451   /* Even part \
 452    * \
 453    * (Original) \
 454    * z1 = (z2 + z3) * 0.541196100; \
 455    * tmp2 = z1 + z3 * -1.847759065; \
 456    * tmp3 = z1 + z2 * 0.765366865; \
 457    * \
 458    * (This implementation) \
 459    * tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065); \
 460    * tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100; \
 461    */ \
 462   \
 463   z23l = _mm_unpacklo_pi16(row2l, row6l); \
 464   z23h = _mm_unpackhi_pi16(row2l, row6l); \
 465   \
 466   tmp3l = _mm_madd_pi16(z23l, PW_F130_F054); \
 467   tmp3h = _mm_madd_pi16(z23h, PW_F130_F054); \
 468   tmp2l = _mm_madd_pi16(z23l, PW_F054_MF130); \
 469   tmp2h = _mm_madd_pi16(z23h, PW_F054_MF130); \
 470   \
 471   z23 = _mm_add_pi16(row0l, row4l); \
 472   tmp0l = _mm_loadlo_pi16_f(z23); \
 473   tmp0h = _mm_loadhi_pi16_f(z23); \
 474   tmp0l = _mm_srai_pi32(tmp0l, (16 - CONST_BITS)); \
 475   tmp0h = _mm_srai_pi32(tmp0h, (16 - CONST_BITS)); \
 476   \
 477   tmp10l = _mm_add_pi32(tmp0l, tmp3l); \
 478   tmp10h = _mm_add_pi32(tmp0h, tmp3h); \
 479   tmp13l = _mm_sub_pi32(tmp0l, tmp3l); \
 480   tmp13h = _mm_sub_pi32(tmp0h, tmp3h); \
 481   \
 482   z23 = _mm_sub_pi16(row0l, row4l); \
 483   tmp1l = _mm_loadlo_pi16_f(z23); \
 484   tmp1h = _mm_loadhi_pi16_f(z23); \
 485   tmp1l = _mm_srai_pi32(tmp1l, (16 - CONST_BITS)); \
 486   tmp1h = _mm_srai_pi32(tmp1h, (16 - CONST_BITS)); \
 487   \
 488   tmp11l = _mm_add_pi32(tmp1l, tmp2l); \
 489   tmp11h = _mm_add_pi32(tmp1h, tmp2h); \
 490   tmp12l = _mm_sub_pi32(tmp1l, tmp2l); \
 491   tmp12h = _mm_sub_pi32(tmp1h, tmp2h); \
 492   \
 493   /* Odd part */ \
 494   \
 495   tmp0 = row7l; \
 496   tmp1 = row5l; \
 497   tmp2 = row3l; \
 498   tmp3 = row1l; \
 499   \
 500   DO_IDCT_COMMON(2) \
 501   \
 502   /* out0=(00 01 02 03), out1=(10 11 12 13) */ \
 503   /* out2=(20 21 22 23), out3=(30 31 32 33) */ \
 504   /* out4=(40 41 42 43), out5=(50 51 52 53) */ \
 505   /* out6=(60 61 62 63), out7=(70 71 72 73) */ \
 506   \
 507   row06 = _mm_packs_pi16(out0, out6);  /* row06=(00 01 02 03 60 61 62 63) */ \
 508   row17 = _mm_packs_pi16(out1, out7);  /* row17=(10 11 12 13 70 71 72 73) */ \
 509   row24 = _mm_packs_pi16(out2, out4);  /* row24=(20 21 22 23 40 41 42 43) */ \
 510   row35 = _mm_packs_pi16(out3, out5);  /* row35=(30 31 32 33 50 51 52 53) */ \
 511   \
 512   row06 = _mm_add_pi8(row06, PB_CENTERJSAMP); \
 513   row17 = _mm_add_pi8(row17, PB_CENTERJSAMP); \
 514   row24 = _mm_add_pi8(row24, PB_CENTERJSAMP); \
 515   row35 = _mm_add_pi8(row35, PB_CENTERJSAMP); \
 516   \
 517   /* Transpose coefficients */ \
 518   \
 519   col0123a = _mm_unpacklo_pi8(row06, row17);  /* col0123a=(00 10 01 11 02 12 03 13) */ \
 520   col0123d = _mm_unpackhi_pi8(row06, row17);  /* col0123d=(60 70 61 71 62 72 63 73) */ \
 521   col0123b = _mm_unpacklo_pi8(row24, row35);  /* col0123b=(20 30 21 31 22 32 23 33) */ \
 522   col0123c = _mm_unpackhi_pi8(row24, row35);  /* col0123c=(40 50 41 51 42 52 43 53) */ \
 523   \
 524   col01l = _mm_unpacklo_pi16(col0123a, col0123b);  /* col01l=(00 10 20 30 01 11 21 31) */ \
 525   col23l = _mm_unpackhi_pi16(col0123a, col0123b);  /* col23l=(02 12 22 32 03 13 23 33) */ \
 526   col01h = _mm_unpacklo_pi16(col0123c, col0123d);  /* col01h=(40 50 60 70 41 51 61 71) */ \
 527   col23h = _mm_unpackhi_pi16(col0123c, col0123d);  /* col23h=(42 52 62 72 43 53 63 73) */ \
 528   \
 529   col0 = _mm_unpacklo_pi32(col01l, col01h);   /* col0=(00 10 20 30 40 50 60 70) */ \
 530   col1 = _mm_unpackhi_pi32(col01l, col01h);   /* col1=(01 11 21 31 41 51 61 71) */ \
 531   col2 = _mm_unpacklo_pi32(col23l, col23h);   /* col2=(02 12 22 32 42 52 62 72) */ \
 532   col3 = _mm_unpackhi_pi32(col23l, col23h);   /* col3=(03 13 23 33 43 53 63 73) */ \
 533   \
 534   _mm_store_si64((__m64 *)(output_buf[ctr + 0] + output_col), col0); \
 535   _mm_store_si64((__m64 *)(output_buf[ctr + 1] + output_col), col1); \
 536   _mm_store_si64((__m64 *)(output_buf[ctr + 2] + output_col), col2); \
 537   _mm_store_si64((__m64 *)(output_buf[ctr + 3] + output_col), col3); \
 538 }
 539
 540 void jsimd_idct_islow_mmi(void *dct_table, JCOEFPTR coef_block,
 541                           JSAMPARRAY output_buf, JDIMENSION output_col)
 542 {
 543   __m64 tmp0, tmp1, tmp2, tmp3;
 544   __m64 out0, out1, out2, out3, out4, out5, out6, out7;
 545   JCOEFPTR inptr;
 546   ISLOW_MULT_TYPE *quantptr;
 547   JCOEF *wsptr;
 548   JCOEF workspace[DCTSIZE2];  /* buffers data between passes */
 549
 550   /* Pass 1: process columns. */
 551
 552   inptr = coef_block;
 553   quantptr = (ISLOW_MULT_TYPE *)dct_table;
 554   wsptr = workspace;
 555
 556   DO_IDCT_PASS1(1)
 557 nextcolumn1:
 558   inptr += 4;
 559   quantptr += 4;
 560   wsptr += DCTSIZE * 4;
 561   DO_IDCT_PASS1(2)
 562 nextcolumn2:
 563
 564   /* Pass 2: process rows. */
 565
 566   wsptr = workspace;
 567
 568   DO_IDCT_PASS2(0)
 569   wsptr += 4;
 570   DO_IDCT_PASS2(4)
 571 }