simd/jidctred-sse2.asm

   1 ;
   2 ; jidctred.asm - reduced-size IDCT (SSE2)
   3 ;
   4 ; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
   5 ;
   6 ; Based on the x86 SIMD extension for IJG JPEG library
   7 ; Copyright (C) 1999-2006, MIYASAKA Masaru.
   8 ; For conditions of distribution and use, see copyright notice in jsimdext.inc
   9 ;
  10 ; This file should be assembled with NASM (Netwide Assembler),
  11 ; can *not* be assembled with Microsoft's MASM or any compatible
  12 ; assembler (including Borland's Turbo Assembler).
  13 ; NASM is available from http://nasm.sourceforge.net/ or
  14 ; http://sourceforge.net/project/showfiles.php?group_id=6208
  15 ;
  16 ; This file contains inverse-DCT routines that produce reduced-size
  17 ; output: either 4x4 or 2x2 pixels from an 8x8 DCT block.
  18 ; The following code is based directly on the IJG's original jidctred.c;
  19 ; see the jidctred.c for more details.
  20 ;
  21 ; [TAB8]
  22
  23 %include "jsimdext.inc"
  24 %include "jdct.inc"
  25
  26 ; --------------------------------------------------------------------------
  27
  28 %define CONST_BITS      13
  29 %define PASS1_BITS      2
  30
  31 %define DESCALE_P1_4    (CONST_BITS-PASS1_BITS+1)
  32 %define DESCALE_P2_4    (CONST_BITS+PASS1_BITS+3+1)
  33 %define DESCALE_P1_2    (CONST_BITS-PASS1_BITS+2)
  34 %define DESCALE_P2_2    (CONST_BITS+PASS1_BITS+3+2)
  35
  36 %if CONST_BITS == 13
  37 F_0_211 equ      1730           ; FIX(0.211164243)
  38 F_0_509 equ      4176           ; FIX(0.509795579)
  39 F_0_601 equ      4926           ; FIX(0.601344887)
  40 F_0_720 equ      5906           ; FIX(0.720959822)
  41 F_0_765 equ      6270           ; FIX(0.765366865)
  42 F_0_850 equ      6967           ; FIX(0.850430095)
  43 F_0_899 equ      7373           ; FIX(0.899976223)
  44 F_1_061 equ      8697           ; FIX(1.061594337)
  45 F_1_272 equ     10426           ; FIX(1.272758580)
  46 F_1_451 equ     11893           ; FIX(1.451774981)
  47 F_1_847 equ     15137           ; FIX(1.847759065)
  48 F_2_172 equ     17799           ; FIX(2.172734803)
  49 F_2_562 equ     20995           ; FIX(2.562915447)
  50 F_3_624 equ     29692           ; FIX(3.624509785)
  51 %else
  52 ; NASM cannot do compile-time arithmetic on floating-point constants.
  53 %define DESCALE(x,n)  (((x)+(1<<((n)-1)))>>(n))
  54 F_0_211 equ     DESCALE( 226735879,30-CONST_BITS)       ; FIX(0.211164243)
  55 F_0_509 equ     DESCALE( 547388834,30-CONST_BITS)       ; FIX(0.509795579)
  56 F_0_601 equ     DESCALE( 645689155,30-CONST_BITS)       ; FIX(0.601344887)
  57 F_0_720 equ     DESCALE( 774124714,30-CONST_BITS)       ; FIX(0.720959822)
  58 F_0_765 equ     DESCALE( 821806413,30-CONST_BITS)       ; FIX(0.765366865)
  59 F_0_850 equ     DESCALE( 913142361,30-CONST_BITS)       ; FIX(0.850430095)
  60 F_0_899 equ     DESCALE( 966342111,30-CONST_BITS)       ; FIX(0.899976223)
  61 F_1_061 equ     DESCALE(1139878239,30-CONST_BITS)       ; FIX(1.061594337)
  62 F_1_272 equ     DESCALE(1366614119,30-CONST_BITS)       ; FIX(1.272758580)
  63 F_1_451 equ     DESCALE(1558831516,30-CONST_BITS)       ; FIX(1.451774981)
  64 F_1_847 equ     DESCALE(1984016188,30-CONST_BITS)       ; FIX(1.847759065)
  65 F_2_172 equ     DESCALE(2332956230,30-CONST_BITS)       ; FIX(2.172734803)
  66 F_2_562 equ     DESCALE(2751909506,30-CONST_BITS)       ; FIX(2.562915447)
  67 F_3_624 equ     DESCALE(3891787747,30-CONST_BITS)       ; FIX(3.624509785)
  68 %endif
  69
  70 ; --------------------------------------------------------------------------
  71         SECTION SEG_CONST
  72
  73         alignz  16
  74         global  EXTN(jconst_idct_red_sse2)
  75
  76 EXTN(jconst_idct_red_sse2):
  77
  78 PW_F184_MF076   times 4 dw  F_1_847,-F_0_765
  79 PW_F256_F089    times 4 dw  F_2_562, F_0_899
  80 PW_F106_MF217   times 4 dw  F_1_061,-F_2_172
  81 PW_MF060_MF050  times 4 dw -F_0_601,-F_0_509
  82 PW_F145_MF021   times 4 dw  F_1_451,-F_0_211
  83 PW_F362_MF127   times 4 dw  F_3_624,-F_1_272
  84 PW_F085_MF072   times 4 dw  F_0_850,-F_0_720
  85 PD_DESCALE_P1_4 times 4 dd  1 << (DESCALE_P1_4-1)
  86 PD_DESCALE_P2_4 times 4 dd  1 << (DESCALE_P2_4-1)
  87 PD_DESCALE_P1_2 times 4 dd  1 << (DESCALE_P1_2-1)
  88 PD_DESCALE_P2_2 times 4 dd  1 << (DESCALE_P2_2-1)
  89 PB_CENTERJSAMP  times 16 db CENTERJSAMPLE
  90
  91         alignz  16
  92
  93 ; --------------------------------------------------------------------------
  94         SECTION SEG_TEXT
  95         BITS    32
  96 ;
  97 ; Perform dequantization and inverse DCT on one block of coefficients,
  98 ; producing a reduced-size 4x4 output block.
  99 ;
 100 ; GLOBAL(void)
 101 ; jsimd_idct_4x4_sse2 (void *dct_table, JCOEFPTR coef_block,
 102 ;                      JSAMPARRAY output_buf, JDIMENSION output_col)
 103 ;
 104
 105 %define dct_table(b)    (b)+8           ; void *dct_table
 106 %define coef_block(b)   (b)+12          ; JCOEFPTR coef_block
 107 %define output_buf(b)   (b)+16          ; JSAMPARRAY output_buf
 108 %define output_col(b)   (b)+20          ; JDIMENSION output_col
 109
 110 %define original_ebp    ebp+0
 111 %define wk(i)           ebp-(WK_NUM-(i))*SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
 112 %define WK_NUM          2
 113
 114         align   16
 115         global  EXTN(jsimd_idct_4x4_sse2)
 116
 117 EXTN(jsimd_idct_4x4_sse2):
 118         push    ebp
 119         mov     eax,esp                         ; eax = original ebp
 120         sub     esp, byte 4
 121         and     esp, byte (-SIZEOF_XMMWORD)     ; align to 128 bits
 122         mov     [esp],eax
 123         mov     ebp,esp                         ; ebp = aligned ebp
 124         lea     esp, [wk(0)]
 125         pushpic ebx
 126 ;       push    ecx             ; unused
 127 ;       push    edx             ; need not be preserved
 128         push    esi
 129         push    edi
 130
 131         get_GOT ebx             ; get GOT address
 132
 133         ; ---- Pass 1: process columns from input.
 134
 135 ;       mov     eax, [original_ebp]
 136         mov     edx, POINTER [dct_table(eax)]           ; quantptr
 137         mov     esi, JCOEFPTR [coef_block(eax)]         ; inptr
 138
 139 %ifndef NO_ZERO_COLUMN_TEST_4X4_SSE2
 140         mov     eax, DWORD [DWBLOCK(1,0,esi,SIZEOF_JCOEF)]
 141         or      eax, DWORD [DWBLOCK(2,0,esi,SIZEOF_JCOEF)]
 142         jnz     short .columnDCT
 143
 144         movdqa  xmm0, XMMWORD [XMMBLOCK(1,0,esi,SIZEOF_JCOEF)]
 145         movdqa  xmm1, XMMWORD [XMMBLOCK(2,0,esi,SIZEOF_JCOEF)]
 146         por     xmm0, XMMWORD [XMMBLOCK(3,0,esi,SIZEOF_JCOEF)]
 147         por     xmm1, XMMWORD [XMMBLOCK(5,0,esi,SIZEOF_JCOEF)]
 148         por     xmm0, XMMWORD [XMMBLOCK(6,0,esi,SIZEOF_JCOEF)]
 149         por     xmm1, XMMWORD [XMMBLOCK(7,0,esi,SIZEOF_JCOEF)]
 150         por     xmm0,xmm1
 151         packsswb xmm0,xmm0
 152         packsswb xmm0,xmm0
 153         movd    eax,xmm0
 154         test    eax,eax
 155         jnz     short .columnDCT
 156
 157         ; -- AC terms all zero
 158
 159         movdqa  xmm0, XMMWORD [XMMBLOCK(0,0,esi,SIZEOF_JCOEF)]
 160         pmullw  xmm0, XMMWORD [XMMBLOCK(0,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 161
 162         psllw   xmm0,PASS1_BITS
 163
 164         movdqa    xmm3,xmm0     ; xmm0=in0=(00 01 02 03 04 05 06 07)
 165         punpcklwd xmm0,xmm0     ; xmm0=(00 00 01 01 02 02 03 03)
 166         punpckhwd xmm3,xmm3     ; xmm3=(04 04 05 05 06 06 07 07)
 167
 168         pshufd  xmm1,xmm0,0x50  ; xmm1=[col0 col1]=(00 00 00 00 01 01 01 01)
 169         pshufd  xmm0,xmm0,0xFA  ; xmm0=[col2 col3]=(02 02 02 02 03 03 03 03)
 170         pshufd  xmm6,xmm3,0x50  ; xmm6=[col4 col5]=(04 04 04 04 05 05 05 05)
 171         pshufd  xmm3,xmm3,0xFA  ; xmm3=[col6 col7]=(06 06 06 06 07 07 07 07)
 172
 173         jmp     near .column_end
 174         alignx  16,7
 175 %endif
 176 .columnDCT:
 177
 178         ; -- Odd part
 179
 180         movdqa  xmm0, XMMWORD [XMMBLOCK(1,0,esi,SIZEOF_JCOEF)]
 181         movdqa  xmm1, XMMWORD [XMMBLOCK(3,0,esi,SIZEOF_JCOEF)]
 182         pmullw  xmm0, XMMWORD [XMMBLOCK(1,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 183         pmullw  xmm1, XMMWORD [XMMBLOCK(3,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 184         movdqa  xmm2, XMMWORD [XMMBLOCK(5,0,esi,SIZEOF_JCOEF)]
 185         movdqa  xmm3, XMMWORD [XMMBLOCK(7,0,esi,SIZEOF_JCOEF)]
 186         pmullw  xmm2, XMMWORD [XMMBLOCK(5,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 187         pmullw  xmm3, XMMWORD [XMMBLOCK(7,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 188
 189         movdqa    xmm4,xmm0
 190         movdqa    xmm5,xmm0
 191         punpcklwd xmm4,xmm1
 192         punpckhwd xmm5,xmm1
 193         movdqa    xmm0,xmm4
 194         movdqa    xmm1,xmm5
 195         pmaddwd   xmm4,[GOTOFF(ebx,PW_F256_F089)]       ; xmm4=(tmp2L)
 196         pmaddwd   xmm5,[GOTOFF(ebx,PW_F256_F089)]       ; xmm5=(tmp2H)
 197         pmaddwd   xmm0,[GOTOFF(ebx,PW_F106_MF217)]      ; xmm0=(tmp0L)
 198         pmaddwd   xmm1,[GOTOFF(ebx,PW_F106_MF217)]      ; xmm1=(tmp0H)
 199
 200         movdqa    xmm6,xmm2
 201         movdqa    xmm7,xmm2
 202         punpcklwd xmm6,xmm3
 203         punpckhwd xmm7,xmm3
 204         movdqa    xmm2,xmm6
 205         movdqa    xmm3,xmm7
 206         pmaddwd   xmm6,[GOTOFF(ebx,PW_MF060_MF050)]     ; xmm6=(tmp2L)
 207         pmaddwd   xmm7,[GOTOFF(ebx,PW_MF060_MF050)]     ; xmm7=(tmp2H)
 208         pmaddwd   xmm2,[GOTOFF(ebx,PW_F145_MF021)]      ; xmm2=(tmp0L)
 209         pmaddwd   xmm3,[GOTOFF(ebx,PW_F145_MF021)]      ; xmm3=(tmp0H)
 210
 211         paddd   xmm6,xmm4               ; xmm6=tmp2L
 212         paddd   xmm7,xmm5               ; xmm7=tmp2H
 213         paddd   xmm2,xmm0               ; xmm2=tmp0L
 214         paddd   xmm3,xmm1               ; xmm3=tmp0H
 215
 216         movdqa  XMMWORD [wk(0)], xmm2   ; wk(0)=tmp0L
 217         movdqa  XMMWORD [wk(1)], xmm3   ; wk(1)=tmp0H
 218
 219         ; -- Even part
 220
 221         movdqa  xmm4, XMMWORD [XMMBLOCK(0,0,esi,SIZEOF_JCOEF)]
 222         movdqa  xmm5, XMMWORD [XMMBLOCK(2,0,esi,SIZEOF_JCOEF)]
 223         movdqa  xmm0, XMMWORD [XMMBLOCK(6,0,esi,SIZEOF_JCOEF)]
 224         pmullw  xmm4, XMMWORD [XMMBLOCK(0,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 225         pmullw  xmm5, XMMWORD [XMMBLOCK(2,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 226         pmullw  xmm0, XMMWORD [XMMBLOCK(6,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 227
 228         pxor      xmm1,xmm1
 229         pxor      xmm2,xmm2
 230         punpcklwd xmm1,xmm4             ; xmm1=tmp0L
 231         punpckhwd xmm2,xmm4             ; xmm2=tmp0H
 232         psrad     xmm1,(16-CONST_BITS-1) ; psrad xmm1,16 & pslld xmm1,CONST_BITS+1
 233         psrad     xmm2,(16-CONST_BITS-1) ; psrad xmm2,16 & pslld xmm2,CONST_BITS+1
 234
 235         movdqa    xmm3,xmm5             ; xmm5=in2=z2
 236         punpcklwd xmm5,xmm0             ; xmm0=in6=z3
 237         punpckhwd xmm3,xmm0
 238         pmaddwd   xmm5,[GOTOFF(ebx,PW_F184_MF076)]      ; xmm5=tmp2L
 239         pmaddwd   xmm3,[GOTOFF(ebx,PW_F184_MF076)]      ; xmm3=tmp2H
 240
 241         movdqa  xmm4,xmm1
 242         movdqa  xmm0,xmm2
 243         paddd   xmm1,xmm5               ; xmm1=tmp10L
 244         paddd   xmm2,xmm3               ; xmm2=tmp10H
 245         psubd   xmm4,xmm5               ; xmm4=tmp12L
 246         psubd   xmm0,xmm3               ; xmm0=tmp12H
 247
 248         ; -- Final output stage
 249
 250         movdqa  xmm5,xmm1
 251         movdqa  xmm3,xmm2
 252         paddd   xmm1,xmm6               ; xmm1=data0L
 253         paddd   xmm2,xmm7               ; xmm2=data0H
 254         psubd   xmm5,xmm6               ; xmm5=data3L
 255         psubd   xmm3,xmm7               ; xmm3=data3H
 256
 257         movdqa  xmm6,[GOTOFF(ebx,PD_DESCALE_P1_4)]      ; xmm6=[PD_DESCALE_P1_4]
 258
 259         paddd   xmm1,xmm6
 260         paddd   xmm2,xmm6
 261         psrad   xmm1,DESCALE_P1_4
 262         psrad   xmm2,DESCALE_P1_4
 263         paddd   xmm5,xmm6
 264         paddd   xmm3,xmm6
 265         psrad   xmm5,DESCALE_P1_4
 266         psrad   xmm3,DESCALE_P1_4
 267
 268         packssdw  xmm1,xmm2             ; xmm1=data0=(00 01 02 03 04 05 06 07)
 269         packssdw  xmm5,xmm3             ; xmm5=data3=(30 31 32 33 34 35 36 37)
 270
 271         movdqa  xmm7, XMMWORD [wk(0)]   ; xmm7=tmp0L
 272         movdqa  xmm6, XMMWORD [wk(1)]   ; xmm6=tmp0H
 273
 274         movdqa  xmm2,xmm4
 275         movdqa  xmm3,xmm0
 276         paddd   xmm4,xmm7               ; xmm4=data1L
 277         paddd   xmm0,xmm6               ; xmm0=data1H
 278         psubd   xmm2,xmm7               ; xmm2=data2L
 279         psubd   xmm3,xmm6               ; xmm3=data2H
 280
 281         movdqa  xmm7,[GOTOFF(ebx,PD_DESCALE_P1_4)]      ; xmm7=[PD_DESCALE_P1_4]
 282
 283         paddd   xmm4,xmm7
 284         paddd   xmm0,xmm7
 285         psrad   xmm4,DESCALE_P1_4
 286         psrad   xmm0,DESCALE_P1_4
 287         paddd   xmm2,xmm7
 288         paddd   xmm3,xmm7
 289         psrad   xmm2,DESCALE_P1_4
 290         psrad   xmm3,DESCALE_P1_4
 291
 292         packssdw  xmm4,xmm0             ; xmm4=data1=(10 11 12 13 14 15 16 17)
 293         packssdw  xmm2,xmm3             ; xmm2=data2=(20 21 22 23 24 25 26 27)
 294
 295         movdqa    xmm6,xmm1     ; transpose coefficients(phase 1)
 296         punpcklwd xmm1,xmm4     ; xmm1=(00 10 01 11 02 12 03 13)
 297         punpckhwd xmm6,xmm4     ; xmm6=(04 14 05 15 06 16 07 17)
 298         movdqa    xmm7,xmm2     ; transpose coefficients(phase 1)
 299         punpcklwd xmm2,xmm5     ; xmm2=(20 30 21 31 22 32 23 33)
 300         punpckhwd xmm7,xmm5     ; xmm7=(24 34 25 35 26 36 27 37)
 301
 302         movdqa    xmm0,xmm1     ; transpose coefficients(phase 2)
 303         punpckldq xmm1,xmm2     ; xmm1=[col0 col1]=(00 10 20 30 01 11 21 31)
 304         punpckhdq xmm0,xmm2     ; xmm0=[col2 col3]=(02 12 22 32 03 13 23 33)
 305         movdqa    xmm3,xmm6     ; transpose coefficients(phase 2)
 306         punpckldq xmm6,xmm7     ; xmm6=[col4 col5]=(04 14 24 34 05 15 25 35)
 307         punpckhdq xmm3,xmm7     ; xmm3=[col6 col7]=(06 16 26 36 07 17 27 37)
 308 .column_end:
 309
 310         ; -- Prefetch the next coefficient block
 311
 312         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 0*32]
 313         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 1*32]
 314         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 2*32]
 315         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 3*32]
 316
 317         ; ---- Pass 2: process rows, store into output array.
 318
 319         mov     eax, [original_ebp]
 320         mov     edi, JSAMPARRAY [output_buf(eax)]       ; (JSAMPROW *)
 321         mov     eax, JDIMENSION [output_col(eax)]
 322
 323         ; -- Even part
 324
 325         pxor      xmm4,xmm4
 326         punpcklwd xmm4,xmm1             ; xmm4=tmp0
 327         psrad     xmm4,(16-CONST_BITS-1) ; psrad xmm4,16 & pslld xmm4,CONST_BITS+1
 328
 329         ; -- Odd part
 330
 331         punpckhwd xmm1,xmm0
 332         punpckhwd xmm6,xmm3
 333         movdqa    xmm5,xmm1
 334         movdqa    xmm2,xmm6
 335         pmaddwd   xmm1,[GOTOFF(ebx,PW_F256_F089)]       ; xmm1=(tmp2)
 336         pmaddwd   xmm6,[GOTOFF(ebx,PW_MF060_MF050)]     ; xmm6=(tmp2)
 337         pmaddwd   xmm5,[GOTOFF(ebx,PW_F106_MF217)]      ; xmm5=(tmp0)
 338         pmaddwd   xmm2,[GOTOFF(ebx,PW_F145_MF021)]      ; xmm2=(tmp0)
 339
 340         paddd     xmm6,xmm1             ; xmm6=tmp2
 341         paddd     xmm2,xmm5             ; xmm2=tmp0
 342
 343         ; -- Even part
 344
 345         punpcklwd xmm0,xmm3
 346         pmaddwd   xmm0,[GOTOFF(ebx,PW_F184_MF076)]      ; xmm0=tmp2
 347
 348         movdqa    xmm7,xmm4
 349         paddd     xmm4,xmm0             ; xmm4=tmp10
 350         psubd     xmm7,xmm0             ; xmm7=tmp12
 351
 352         ; -- Final output stage
 353
 354         movdqa  xmm1,[GOTOFF(ebx,PD_DESCALE_P2_4)]      ; xmm1=[PD_DESCALE_P2_4]
 355
 356         movdqa  xmm5,xmm4
 357         movdqa  xmm3,xmm7
 358         paddd   xmm4,xmm6               ; xmm4=data0=(00 10 20 30)
 359         paddd   xmm7,xmm2               ; xmm7=data1=(01 11 21 31)
 360         psubd   xmm5,xmm6               ; xmm5=data3=(03 13 23 33)
 361         psubd   xmm3,xmm2               ; xmm3=data2=(02 12 22 32)
 362
 363         paddd   xmm4,xmm1
 364         paddd   xmm7,xmm1
 365         psrad   xmm4,DESCALE_P2_4
 366         psrad   xmm7,DESCALE_P2_4
 367         paddd   xmm5,xmm1
 368         paddd   xmm3,xmm1
 369         psrad   xmm5,DESCALE_P2_4
 370         psrad   xmm3,DESCALE_P2_4
 371
 372         packssdw  xmm4,xmm3             ; xmm4=(00 10 20 30 02 12 22 32)
 373         packssdw  xmm7,xmm5             ; xmm7=(01 11 21 31 03 13 23 33)
 374
 375         movdqa    xmm0,xmm4             ; transpose coefficients(phase 1)
 376         punpcklwd xmm4,xmm7             ; xmm4=(00 01 10 11 20 21 30 31)
 377         punpckhwd xmm0,xmm7             ; xmm0=(02 03 12 13 22 23 32 33)
 378
 379         movdqa    xmm6,xmm4             ; transpose coefficients(phase 2)
 380         punpckldq xmm4,xmm0             ; xmm4=(00 01 02 03 10 11 12 13)
 381         punpckhdq xmm6,xmm0             ; xmm6=(20 21 22 23 30 31 32 33)
 382
 383         packsswb  xmm4,xmm6             ; xmm4=(00 01 02 03 10 11 12 13 20 ..)
 384         paddb     xmm4,[GOTOFF(ebx,PB_CENTERJSAMP)]
 385
 386         pshufd    xmm2,xmm4,0x39        ; xmm2=(10 11 12 13 20 21 22 23 30 ..)
 387         pshufd    xmm1,xmm4,0x4E        ; xmm1=(20 21 22 23 30 31 32 33 00 ..)
 388         pshufd    xmm3,xmm4,0x93        ; xmm3=(30 31 32 33 00 01 02 03 10 ..)
 389
 390         mov     edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]
 391         mov     esi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]
 392         movd    XMM_DWORD [edx+eax*SIZEOF_JSAMPLE], xmm4
 393         movd    XMM_DWORD [esi+eax*SIZEOF_JSAMPLE], xmm2
 394         mov     edx, JSAMPROW [edi+2*SIZEOF_JSAMPROW]
 395         mov     esi, JSAMPROW [edi+3*SIZEOF_JSAMPROW]
 396         movd    XMM_DWORD [edx+eax*SIZEOF_JSAMPLE], xmm1
 397         movd    XMM_DWORD [esi+eax*SIZEOF_JSAMPLE], xmm3
 398
 399         pop     edi
 400         pop     esi
 401 ;       pop     edx             ; need not be preserved
 402 ;       pop     ecx             ; unused
 403         poppic  ebx
 404         mov     esp,ebp         ; esp <- aligned ebp
 405         pop     esp             ; esp <- original ebp
 406         pop     ebp
 407         ret
 408
 409
 410 ; --------------------------------------------------------------------------
 411 ;
 412 ; Perform dequantization and inverse DCT on one block of coefficients,
 413 ; producing a reduced-size 2x2 output block.
 414 ;
 415 ; GLOBAL(void)
 416 ; jsimd_idct_2x2_sse2 (void *dct_table, JCOEFPTR coef_block,
 417 ;                      JSAMPARRAY output_buf, JDIMENSION output_col)
 418 ;
 419
 420 %define dct_table(b)    (b)+8           ; void *dct_table
 421 %define coef_block(b)   (b)+12          ; JCOEFPTR coef_block
 422 %define output_buf(b)   (b)+16          ; JSAMPARRAY output_buf
 423 %define output_col(b)   (b)+20          ; JDIMENSION output_col
 424
 425         align   16
 426         global  EXTN(jsimd_idct_2x2_sse2)
 427
 428 EXTN(jsimd_idct_2x2_sse2):
 429         push    ebp
 430         mov     ebp,esp
 431         push    ebx
 432 ;       push    ecx             ; need not be preserved
 433 ;       push    edx             ; need not be preserved
 434         push    esi
 435         push    edi
 436
 437         get_GOT ebx             ; get GOT address
 438
 439         ; ---- Pass 1: process columns from input.
 440
 441         mov     edx, POINTER [dct_table(ebp)]           ; quantptr
 442         mov     esi, JCOEFPTR [coef_block(ebp)]         ; inptr
 443
 444         ; | input:                  | result:        |
 445         ; | 00 01 ** 03 ** 05 ** 07 |                |
 446         ; | 10 11 ** 13 ** 15 ** 17 |                |
 447         ; | ** ** ** ** ** ** ** ** |                |
 448         ; | 30 31 ** 33 ** 35 ** 37 | A0 A1 A3 A5 A7 |
 449         ; | ** ** ** ** ** ** ** ** | B0 B1 B3 B5 B7 |
 450         ; | 50 51 ** 53 ** 55 ** 57 |                |
 451         ; | ** ** ** ** ** ** ** ** |                |
 452         ; | 70 71 ** 73 ** 75 ** 77 |                |
 453
 454         ; -- Odd part
 455
 456         movdqa  xmm0, XMMWORD [XMMBLOCK(1,0,esi,SIZEOF_JCOEF)]
 457         movdqa  xmm1, XMMWORD [XMMBLOCK(3,0,esi,SIZEOF_JCOEF)]
 458         pmullw  xmm0, XMMWORD [XMMBLOCK(1,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 459         pmullw  xmm1, XMMWORD [XMMBLOCK(3,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 460         movdqa  xmm2, XMMWORD [XMMBLOCK(5,0,esi,SIZEOF_JCOEF)]
 461         movdqa  xmm3, XMMWORD [XMMBLOCK(7,0,esi,SIZEOF_JCOEF)]
 462         pmullw  xmm2, XMMWORD [XMMBLOCK(5,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 463         pmullw  xmm3, XMMWORD [XMMBLOCK(7,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 464
 465         ; xmm0=(10 11 ** 13 ** 15 ** 17), xmm1=(30 31 ** 33 ** 35 ** 37)
 466         ; xmm2=(50 51 ** 53 ** 55 ** 57), xmm3=(70 71 ** 73 ** 75 ** 77)
 467
 468         pcmpeqd   xmm7,xmm7
 469         pslld     xmm7,WORD_BIT         ; xmm7={0x0000 0xFFFF 0x0000 0xFFFF ..}
 470
 471         movdqa    xmm4,xmm0             ; xmm4=(10 11 ** 13 ** 15 ** 17)
 472         movdqa    xmm5,xmm2             ; xmm5=(50 51 ** 53 ** 55 ** 57)
 473         punpcklwd xmm4,xmm1             ; xmm4=(10 30 11 31 ** ** 13 33)
 474         punpcklwd xmm5,xmm3             ; xmm5=(50 70 51 71 ** ** 53 73)
 475         pmaddwd   xmm4,[GOTOFF(ebx,PW_F362_MF127)]
 476         pmaddwd   xmm5,[GOTOFF(ebx,PW_F085_MF072)]
 477
 478         psrld   xmm0,WORD_BIT           ; xmm0=(11 -- 13 -- 15 -- 17 --)
 479         pand    xmm1,xmm7               ; xmm1=(-- 31 -- 33 -- 35 -- 37)
 480         psrld   xmm2,WORD_BIT           ; xmm2=(51 -- 53 -- 55 -- 57 --)
 481         pand    xmm3,xmm7               ; xmm3=(-- 71 -- 73 -- 75 -- 77)
 482         por     xmm0,xmm1               ; xmm0=(11 31 13 33 15 35 17 37)
 483         por     xmm2,xmm3               ; xmm2=(51 71 53 73 55 75 57 77)
 484         pmaddwd xmm0,[GOTOFF(ebx,PW_F362_MF127)]
 485         pmaddwd xmm2,[GOTOFF(ebx,PW_F085_MF072)]
 486
 487         paddd   xmm4,xmm5               ; xmm4=tmp0[col0 col1 **** col3]
 488         paddd   xmm0,xmm2               ; xmm0=tmp0[col1 col3 col5 col7]
 489
 490         ; -- Even part
 491
 492         movdqa  xmm6, XMMWORD [XMMBLOCK(0,0,esi,SIZEOF_JCOEF)]
 493         pmullw  xmm6, XMMWORD [XMMBLOCK(0,0,edx,SIZEOF_ISLOW_MULT_TYPE)]
 494
 495         ; xmm6=(00 01 ** 03 ** 05 ** 07)
 496
 497         movdqa  xmm1,xmm6               ; xmm1=(00 01 ** 03 ** 05 ** 07)
 498         pslld   xmm6,WORD_BIT           ; xmm6=(-- 00 -- ** -- ** -- **)
 499         pand    xmm1,xmm7               ; xmm1=(-- 01 -- 03 -- 05 -- 07)
 500         psrad   xmm6,(WORD_BIT-CONST_BITS-2) ; xmm6=tmp10[col0 **** **** ****]
 501         psrad   xmm1,(WORD_BIT-CONST_BITS-2) ; xmm1=tmp10[col1 col3 col5 col7]
 502
 503         ; -- Final output stage
 504
 505         movdqa  xmm3,xmm6
 506         movdqa  xmm5,xmm1
 507         paddd   xmm6,xmm4       ; xmm6=data0[col0 **** **** ****]=(A0 ** ** **)
 508         paddd   xmm1,xmm0       ; xmm1=data0[col1 col3 col5 col7]=(A1 A3 A5 A7)
 509         psubd   xmm3,xmm4       ; xmm3=data1[col0 **** **** ****]=(B0 ** ** **)
 510         psubd   xmm5,xmm0       ; xmm5=data1[col1 col3 col5 col7]=(B1 B3 B5 B7)
 511
 512         movdqa  xmm2,[GOTOFF(ebx,PD_DESCALE_P1_2)]      ; xmm2=[PD_DESCALE_P1_2]
 513
 514         punpckldq  xmm6,xmm3            ; xmm6=(A0 B0 ** **)
 515
 516         movdqa     xmm7,xmm1
 517         punpcklqdq xmm1,xmm5            ; xmm1=(A1 A3 B1 B3)
 518         punpckhqdq xmm7,xmm5            ; xmm7=(A5 A7 B5 B7)
 519
 520         paddd   xmm6,xmm2
 521         psrad   xmm6,DESCALE_P1_2
 522
 523         paddd   xmm1,xmm2
 524         paddd   xmm7,xmm2
 525         psrad   xmm1,DESCALE_P1_2
 526         psrad   xmm7,DESCALE_P1_2
 527
 528         ; -- Prefetch the next coefficient block
 529
 530         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 0*32]
 531         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 1*32]
 532         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 2*32]
 533         prefetchnta [esi + DCTSIZE2*SIZEOF_JCOEF + 3*32]
 534
 535         ; ---- Pass 2: process rows, store into output array.
 536
 537         mov     edi, JSAMPARRAY [output_buf(ebp)]       ; (JSAMPROW *)
 538         mov     eax, JDIMENSION [output_col(ebp)]
 539
 540         ; | input:| result:|
 541         ; | A0 B0 |        |
 542         ; | A1 B1 | C0 C1  |
 543         ; | A3 B3 | D0 D1  |
 544         ; | A5 B5 |        |
 545         ; | A7 B7 |        |
 546
 547         ; -- Odd part
 548
 549         packssdw  xmm1,xmm1             ; xmm1=(A1 A3 B1 B3 A1 A3 B1 B3)
 550         packssdw  xmm7,xmm7             ; xmm7=(A5 A7 B5 B7 A5 A7 B5 B7)
 551         pmaddwd   xmm1,[GOTOFF(ebx,PW_F362_MF127)]
 552         pmaddwd   xmm7,[GOTOFF(ebx,PW_F085_MF072)]
 553
 554         paddd     xmm1,xmm7             ; xmm1=tmp0[row0 row1 row0 row1]
 555
 556         ; -- Even part
 557
 558         pslld     xmm6,(CONST_BITS+2)   ; xmm6=tmp10[row0 row1 **** ****]
 559
 560         ; -- Final output stage
 561
 562         movdqa    xmm4,xmm6
 563         paddd     xmm6,xmm1     ; xmm6=data0[row0 row1 **** ****]=(C0 C1 ** **)
 564         psubd     xmm4,xmm1     ; xmm4=data1[row0 row1 **** ****]=(D0 D1 ** **)
 565
 566         punpckldq xmm6,xmm4     ; xmm6=(C0 D0 C1 D1)
 567
 568         paddd     xmm6,[GOTOFF(ebx,PD_DESCALE_P2_2)]
 569         psrad     xmm6,DESCALE_P2_2
 570
 571         packssdw  xmm6,xmm6             ; xmm6=(C0 D0 C1 D1 C0 D0 C1 D1)
 572         packsswb  xmm6,xmm6             ; xmm6=(C0 D0 C1 D1 C0 D0 C1 D1 ..)
 573         paddb     xmm6,[GOTOFF(ebx,PB_CENTERJSAMP)]
 574
 575         pextrw  ebx,xmm6,0x00           ; ebx=(C0 D0 -- --)
 576         pextrw  ecx,xmm6,0x01           ; ecx=(C1 D1 -- --)
 577
 578         mov     edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]
 579         mov     esi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]
 580         mov     WORD [edx+eax*SIZEOF_JSAMPLE], bx
 581         mov     WORD [esi+eax*SIZEOF_JSAMPLE], cx
 582
 583         pop     edi
 584         pop     esi
 585 ;       pop     edx             ; need not be preserved
 586 ;       pop     ecx             ; need not be preserved
 587         pop     ebx
 588         pop     ebp
 589         ret
 590
 591 ; For some reason, the OS X linker does not honor the request to align the
 592 ; segment unless we do this.
 593         align   16