src/third_party/boringssl/win-x86_64/crypto/aes/aesni-x86_64.asm

   1 default rel
   2 %define XMMWORD
   3 %define YMMWORD
   4 %define ZMMWORD
   5 section .text code align=64
   6
   7 EXTERN  OPENSSL_ia32cap_P
   8 global  aesni_encrypt
   9
  10 ALIGN   16
  11 aesni_encrypt:
  12         movups  xmm2,XMMWORD[rcx]
  13         mov     eax,DWORD[240+r8]
  14         movups  xmm0,XMMWORD[r8]
  15         movups  xmm1,XMMWORD[16+r8]
  16         lea     r8,[32+r8]
  17         xorps   xmm2,xmm0
  18 $L$oop_enc1_1:
  19 DB      102,15,56,220,209
  20         dec     eax
  21         movups  xmm1,XMMWORD[r8]
  22         lea     r8,[16+r8]
  23         jnz     NEAR $L$oop_enc1_1
  24 DB      102,15,56,221,209
  25         movups  XMMWORD[rdx],xmm2
  26         DB      0F3h,0C3h               ;repret
  27
  28
  29 global  aesni_decrypt
  30
  31 ALIGN   16
  32 aesni_decrypt:
  33         movups  xmm2,XMMWORD[rcx]
  34         mov     eax,DWORD[240+r8]
  35         movups  xmm0,XMMWORD[r8]
  36         movups  xmm1,XMMWORD[16+r8]
  37         lea     r8,[32+r8]
  38         xorps   xmm2,xmm0
  39 $L$oop_dec1_2:
  40 DB      102,15,56,222,209
  41         dec     eax
  42         movups  xmm1,XMMWORD[r8]
  43         lea     r8,[16+r8]
  44         jnz     NEAR $L$oop_dec1_2
  45 DB      102,15,56,223,209
  46         movups  XMMWORD[rdx],xmm2
  47         DB      0F3h,0C3h               ;repret
  48
  49
  50 ALIGN   16
  51 _aesni_encrypt2:
  52         movups  xmm0,XMMWORD[rcx]
  53         shl     eax,4
  54         movups  xmm1,XMMWORD[16+rcx]
  55         xorps   xmm2,xmm0
  56         xorps   xmm3,xmm0
  57         movups  xmm0,XMMWORD[32+rcx]
  58         lea     rcx,[32+rax*1+rcx]
  59         neg     rax
  60         add     rax,16
  61
  62 $L$enc_loop2:
  63 DB      102,15,56,220,209
  64 DB      102,15,56,220,217
  65         movups  xmm1,XMMWORD[rax*1+rcx]
  66         add     rax,32
  67 DB      102,15,56,220,208
  68 DB      102,15,56,220,216
  69         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
  70         jnz     NEAR $L$enc_loop2
  71
  72 DB      102,15,56,220,209
  73 DB      102,15,56,220,217
  74 DB      102,15,56,221,208
  75 DB      102,15,56,221,216
  76         DB      0F3h,0C3h               ;repret
  77
  78
  79 ALIGN   16
  80 _aesni_decrypt2:
  81         movups  xmm0,XMMWORD[rcx]
  82         shl     eax,4
  83         movups  xmm1,XMMWORD[16+rcx]
  84         xorps   xmm2,xmm0
  85         xorps   xmm3,xmm0
  86         movups  xmm0,XMMWORD[32+rcx]
  87         lea     rcx,[32+rax*1+rcx]
  88         neg     rax
  89         add     rax,16
  90
  91 $L$dec_loop2:
  92 DB      102,15,56,222,209
  93 DB      102,15,56,222,217
  94         movups  xmm1,XMMWORD[rax*1+rcx]
  95         add     rax,32
  96 DB      102,15,56,222,208
  97 DB      102,15,56,222,216
  98         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
  99         jnz     NEAR $L$dec_loop2
 100
 101 DB      102,15,56,222,209
 102 DB      102,15,56,222,217
 103 DB      102,15,56,223,208
 104 DB      102,15,56,223,216
 105         DB      0F3h,0C3h               ;repret
 106
 107
 108 ALIGN   16
 109 _aesni_encrypt3:
 110         movups  xmm0,XMMWORD[rcx]
 111         shl     eax,4
 112         movups  xmm1,XMMWORD[16+rcx]
 113         xorps   xmm2,xmm0
 114         xorps   xmm3,xmm0
 115         xorps   xmm4,xmm0
 116         movups  xmm0,XMMWORD[32+rcx]
 117         lea     rcx,[32+rax*1+rcx]
 118         neg     rax
 119         add     rax,16
 120
 121 $L$enc_loop3:
 122 DB      102,15,56,220,209
 123 DB      102,15,56,220,217
 124 DB      102,15,56,220,225
 125         movups  xmm1,XMMWORD[rax*1+rcx]
 126         add     rax,32
 127 DB      102,15,56,220,208
 128 DB      102,15,56,220,216
 129 DB      102,15,56,220,224
 130         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 131         jnz     NEAR $L$enc_loop3
 132
 133 DB      102,15,56,220,209
 134 DB      102,15,56,220,217
 135 DB      102,15,56,220,225
 136 DB      102,15,56,221,208
 137 DB      102,15,56,221,216
 138 DB      102,15,56,221,224
 139         DB      0F3h,0C3h               ;repret
 140
 141
 142 ALIGN   16
 143 _aesni_decrypt3:
 144         movups  xmm0,XMMWORD[rcx]
 145         shl     eax,4
 146         movups  xmm1,XMMWORD[16+rcx]
 147         xorps   xmm2,xmm0
 148         xorps   xmm3,xmm0
 149         xorps   xmm4,xmm0
 150         movups  xmm0,XMMWORD[32+rcx]
 151         lea     rcx,[32+rax*1+rcx]
 152         neg     rax
 153         add     rax,16
 154
 155 $L$dec_loop3:
 156 DB      102,15,56,222,209
 157 DB      102,15,56,222,217
 158 DB      102,15,56,222,225
 159         movups  xmm1,XMMWORD[rax*1+rcx]
 160         add     rax,32
 161 DB      102,15,56,222,208
 162 DB      102,15,56,222,216
 163 DB      102,15,56,222,224
 164         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 165         jnz     NEAR $L$dec_loop3
 166
 167 DB      102,15,56,222,209
 168 DB      102,15,56,222,217
 169 DB      102,15,56,222,225
 170 DB      102,15,56,223,208
 171 DB      102,15,56,223,216
 172 DB      102,15,56,223,224
 173         DB      0F3h,0C3h               ;repret
 174
 175
 176 ALIGN   16
 177 _aesni_encrypt4:
 178         movups  xmm0,XMMWORD[rcx]
 179         shl     eax,4
 180         movups  xmm1,XMMWORD[16+rcx]
 181         xorps   xmm2,xmm0
 182         xorps   xmm3,xmm0
 183         xorps   xmm4,xmm0
 184         xorps   xmm5,xmm0
 185         movups  xmm0,XMMWORD[32+rcx]
 186         lea     rcx,[32+rax*1+rcx]
 187         neg     rax
 188 DB      0x0f,0x1f,0x00
 189         add     rax,16
 190
 191 $L$enc_loop4:
 192 DB      102,15,56,220,209
 193 DB      102,15,56,220,217
 194 DB      102,15,56,220,225
 195 DB      102,15,56,220,233
 196         movups  xmm1,XMMWORD[rax*1+rcx]
 197         add     rax,32
 198 DB      102,15,56,220,208
 199 DB      102,15,56,220,216
 200 DB      102,15,56,220,224
 201 DB      102,15,56,220,232
 202         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 203         jnz     NEAR $L$enc_loop4
 204
 205 DB      102,15,56,220,209
 206 DB      102,15,56,220,217
 207 DB      102,15,56,220,225
 208 DB      102,15,56,220,233
 209 DB      102,15,56,221,208
 210 DB      102,15,56,221,216
 211 DB      102,15,56,221,224
 212 DB      102,15,56,221,232
 213         DB      0F3h,0C3h               ;repret
 214
 215
 216 ALIGN   16
 217 _aesni_decrypt4:
 218         movups  xmm0,XMMWORD[rcx]
 219         shl     eax,4
 220         movups  xmm1,XMMWORD[16+rcx]
 221         xorps   xmm2,xmm0
 222         xorps   xmm3,xmm0
 223         xorps   xmm4,xmm0
 224         xorps   xmm5,xmm0
 225         movups  xmm0,XMMWORD[32+rcx]
 226         lea     rcx,[32+rax*1+rcx]
 227         neg     rax
 228 DB      0x0f,0x1f,0x00
 229         add     rax,16
 230
 231 $L$dec_loop4:
 232 DB      102,15,56,222,209
 233 DB      102,15,56,222,217
 234 DB      102,15,56,222,225
 235 DB      102,15,56,222,233
 236         movups  xmm1,XMMWORD[rax*1+rcx]
 237         add     rax,32
 238 DB      102,15,56,222,208
 239 DB      102,15,56,222,216
 240 DB      102,15,56,222,224
 241 DB      102,15,56,222,232
 242         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 243         jnz     NEAR $L$dec_loop4
 244
 245 DB      102,15,56,222,209
 246 DB      102,15,56,222,217
 247 DB      102,15,56,222,225
 248 DB      102,15,56,222,233
 249 DB      102,15,56,223,208
 250 DB      102,15,56,223,216
 251 DB      102,15,56,223,224
 252 DB      102,15,56,223,232
 253         DB      0F3h,0C3h               ;repret
 254
 255
 256 ALIGN   16
 257 _aesni_encrypt6:
 258         movups  xmm0,XMMWORD[rcx]
 259         shl     eax,4
 260         movups  xmm1,XMMWORD[16+rcx]
 261         xorps   xmm2,xmm0
 262         pxor    xmm3,xmm0
 263         pxor    xmm4,xmm0
 264 DB      102,15,56,220,209
 265         lea     rcx,[32+rax*1+rcx]
 266         neg     rax
 267 DB      102,15,56,220,217
 268         pxor    xmm5,xmm0
 269         pxor    xmm6,xmm0
 270 DB      102,15,56,220,225
 271         pxor    xmm7,xmm0
 272         add     rax,16
 273 DB      102,15,56,220,233
 274 DB      102,15,56,220,241
 275 DB      102,15,56,220,249
 276         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 277         jmp     NEAR $L$enc_loop6_enter
 278 ALIGN   16
 279 $L$enc_loop6:
 280 DB      102,15,56,220,209
 281 DB      102,15,56,220,217
 282 DB      102,15,56,220,225
 283 DB      102,15,56,220,233
 284 DB      102,15,56,220,241
 285 DB      102,15,56,220,249
 286 $L$enc_loop6_enter:
 287         movups  xmm1,XMMWORD[rax*1+rcx]
 288         add     rax,32
 289 DB      102,15,56,220,208
 290 DB      102,15,56,220,216
 291 DB      102,15,56,220,224
 292 DB      102,15,56,220,232
 293 DB      102,15,56,220,240
 294 DB      102,15,56,220,248
 295         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 296         jnz     NEAR $L$enc_loop6
 297
 298 DB      102,15,56,220,209
 299 DB      102,15,56,220,217
 300 DB      102,15,56,220,225
 301 DB      102,15,56,220,233
 302 DB      102,15,56,220,241
 303 DB      102,15,56,220,249
 304 DB      102,15,56,221,208
 305 DB      102,15,56,221,216
 306 DB      102,15,56,221,224
 307 DB      102,15,56,221,232
 308 DB      102,15,56,221,240
 309 DB      102,15,56,221,248
 310         DB      0F3h,0C3h               ;repret
 311
 312
 313 ALIGN   16
 314 _aesni_decrypt6:
 315         movups  xmm0,XMMWORD[rcx]
 316         shl     eax,4
 317         movups  xmm1,XMMWORD[16+rcx]
 318         xorps   xmm2,xmm0
 319         pxor    xmm3,xmm0
 320         pxor    xmm4,xmm0
 321 DB      102,15,56,222,209
 322         lea     rcx,[32+rax*1+rcx]
 323         neg     rax
 324 DB      102,15,56,222,217
 325         pxor    xmm5,xmm0
 326         pxor    xmm6,xmm0
 327 DB      102,15,56,222,225
 328         pxor    xmm7,xmm0
 329         add     rax,16
 330 DB      102,15,56,222,233
 331 DB      102,15,56,222,241
 332 DB      102,15,56,222,249
 333         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 334         jmp     NEAR $L$dec_loop6_enter
 335 ALIGN   16
 336 $L$dec_loop6:
 337 DB      102,15,56,222,209
 338 DB      102,15,56,222,217
 339 DB      102,15,56,222,225
 340 DB      102,15,56,222,233
 341 DB      102,15,56,222,241
 342 DB      102,15,56,222,249
 343 $L$dec_loop6_enter:
 344         movups  xmm1,XMMWORD[rax*1+rcx]
 345         add     rax,32
 346 DB      102,15,56,222,208
 347 DB      102,15,56,222,216
 348 DB      102,15,56,222,224
 349 DB      102,15,56,222,232
 350 DB      102,15,56,222,240
 351 DB      102,15,56,222,248
 352         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 353         jnz     NEAR $L$dec_loop6
 354
 355 DB      102,15,56,222,209
 356 DB      102,15,56,222,217
 357 DB      102,15,56,222,225
 358 DB      102,15,56,222,233
 359 DB      102,15,56,222,241
 360 DB      102,15,56,222,249
 361 DB      102,15,56,223,208
 362 DB      102,15,56,223,216
 363 DB      102,15,56,223,224
 364 DB      102,15,56,223,232
 365 DB      102,15,56,223,240
 366 DB      102,15,56,223,248
 367         DB      0F3h,0C3h               ;repret
 368
 369
 370 ALIGN   16
 371 _aesni_encrypt8:
 372         movups  xmm0,XMMWORD[rcx]
 373         shl     eax,4
 374         movups  xmm1,XMMWORD[16+rcx]
 375         xorps   xmm2,xmm0
 376         xorps   xmm3,xmm0
 377         pxor    xmm4,xmm0
 378         pxor    xmm5,xmm0
 379         pxor    xmm6,xmm0
 380         lea     rcx,[32+rax*1+rcx]
 381         neg     rax
 382 DB      102,15,56,220,209
 383         add     rax,16
 384         pxor    xmm7,xmm0
 385 DB      102,15,56,220,217
 386         pxor    xmm8,xmm0
 387         pxor    xmm9,xmm0
 388 DB      102,15,56,220,225
 389 DB      102,15,56,220,233
 390 DB      102,15,56,220,241
 391 DB      102,15,56,220,249
 392 DB      102,68,15,56,220,193
 393 DB      102,68,15,56,220,201
 394         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 395         jmp     NEAR $L$enc_loop8_enter
 396 ALIGN   16
 397 $L$enc_loop8:
 398 DB      102,15,56,220,209
 399 DB      102,15,56,220,217
 400 DB      102,15,56,220,225
 401 DB      102,15,56,220,233
 402 DB      102,15,56,220,241
 403 DB      102,15,56,220,249
 404 DB      102,68,15,56,220,193
 405 DB      102,68,15,56,220,201
 406 $L$enc_loop8_enter:
 407         movups  xmm1,XMMWORD[rax*1+rcx]
 408         add     rax,32
 409 DB      102,15,56,220,208
 410 DB      102,15,56,220,216
 411 DB      102,15,56,220,224
 412 DB      102,15,56,220,232
 413 DB      102,15,56,220,240
 414 DB      102,15,56,220,248
 415 DB      102,68,15,56,220,192
 416 DB      102,68,15,56,220,200
 417         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 418         jnz     NEAR $L$enc_loop8
 419
 420 DB      102,15,56,220,209
 421 DB      102,15,56,220,217
 422 DB      102,15,56,220,225
 423 DB      102,15,56,220,233
 424 DB      102,15,56,220,241
 425 DB      102,15,56,220,249
 426 DB      102,68,15,56,220,193
 427 DB      102,68,15,56,220,201
 428 DB      102,15,56,221,208
 429 DB      102,15,56,221,216
 430 DB      102,15,56,221,224
 431 DB      102,15,56,221,232
 432 DB      102,15,56,221,240
 433 DB      102,15,56,221,248
 434 DB      102,68,15,56,221,192
 435 DB      102,68,15,56,221,200
 436         DB      0F3h,0C3h               ;repret
 437
 438
 439 ALIGN   16
 440 _aesni_decrypt8:
 441         movups  xmm0,XMMWORD[rcx]
 442         shl     eax,4
 443         movups  xmm1,XMMWORD[16+rcx]
 444         xorps   xmm2,xmm0
 445         xorps   xmm3,xmm0
 446         pxor    xmm4,xmm0
 447         pxor    xmm5,xmm0
 448         pxor    xmm6,xmm0
 449         lea     rcx,[32+rax*1+rcx]
 450         neg     rax
 451 DB      102,15,56,222,209
 452         add     rax,16
 453         pxor    xmm7,xmm0
 454 DB      102,15,56,222,217
 455         pxor    xmm8,xmm0
 456         pxor    xmm9,xmm0
 457 DB      102,15,56,222,225
 458 DB      102,15,56,222,233
 459 DB      102,15,56,222,241
 460 DB      102,15,56,222,249
 461 DB      102,68,15,56,222,193
 462 DB      102,68,15,56,222,201
 463         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 464         jmp     NEAR $L$dec_loop8_enter
 465 ALIGN   16
 466 $L$dec_loop8:
 467 DB      102,15,56,222,209
 468 DB      102,15,56,222,217
 469 DB      102,15,56,222,225
 470 DB      102,15,56,222,233
 471 DB      102,15,56,222,241
 472 DB      102,15,56,222,249
 473 DB      102,68,15,56,222,193
 474 DB      102,68,15,56,222,201
 475 $L$dec_loop8_enter:
 476         movups  xmm1,XMMWORD[rax*1+rcx]
 477         add     rax,32
 478 DB      102,15,56,222,208
 479 DB      102,15,56,222,216
 480 DB      102,15,56,222,224
 481 DB      102,15,56,222,232
 482 DB      102,15,56,222,240
 483 DB      102,15,56,222,248
 484 DB      102,68,15,56,222,192
 485 DB      102,68,15,56,222,200
 486         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 487         jnz     NEAR $L$dec_loop8
 488
 489 DB      102,15,56,222,209
 490 DB      102,15,56,222,217
 491 DB      102,15,56,222,225
 492 DB      102,15,56,222,233
 493 DB      102,15,56,222,241
 494 DB      102,15,56,222,249
 495 DB      102,68,15,56,222,193
 496 DB      102,68,15,56,222,201
 497 DB      102,15,56,223,208
 498 DB      102,15,56,223,216
 499 DB      102,15,56,223,224
 500 DB      102,15,56,223,232
 501 DB      102,15,56,223,240
 502 DB      102,15,56,223,248
 503 DB      102,68,15,56,223,192
 504 DB      102,68,15,56,223,200
 505         DB      0F3h,0C3h               ;repret
 506
 507 global  aesni_ecb_encrypt
 508
 509 ALIGN   16
 510 aesni_ecb_encrypt:
 511         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 512         mov     QWORD[16+rsp],rsi
 513         mov     rax,rsp
 514 $L$SEH_begin_aesni_ecb_encrypt:
 515         mov     rdi,rcx
 516         mov     rsi,rdx
 517         mov     rdx,r8
 518         mov     rcx,r9
 519         mov     r8,QWORD[40+rsp]
 520
 521
 522         and     rdx,-16
 523         jz      NEAR $L$ecb_ret
 524
 525         mov     eax,DWORD[240+rcx]
 526         movups  xmm0,XMMWORD[rcx]
 527         mov     r11,rcx
 528         mov     r10d,eax
 529         test    r8d,r8d
 530         jz      NEAR $L$ecb_decrypt
 531
 532         cmp     rdx,0x80
 533         jb      NEAR $L$ecb_enc_tail
 534
 535         movdqu  xmm2,XMMWORD[rdi]
 536         movdqu  xmm3,XMMWORD[16+rdi]
 537         movdqu  xmm4,XMMWORD[32+rdi]
 538         movdqu  xmm5,XMMWORD[48+rdi]
 539         movdqu  xmm6,XMMWORD[64+rdi]
 540         movdqu  xmm7,XMMWORD[80+rdi]
 541         movdqu  xmm8,XMMWORD[96+rdi]
 542         movdqu  xmm9,XMMWORD[112+rdi]
 543         lea     rdi,[128+rdi]
 544         sub     rdx,0x80
 545         jmp     NEAR $L$ecb_enc_loop8_enter
 546 ALIGN   16
 547 $L$ecb_enc_loop8:
 548         movups  XMMWORD[rsi],xmm2
 549         mov     rcx,r11
 550         movdqu  xmm2,XMMWORD[rdi]
 551         mov     eax,r10d
 552         movups  XMMWORD[16+rsi],xmm3
 553         movdqu  xmm3,XMMWORD[16+rdi]
 554         movups  XMMWORD[32+rsi],xmm4
 555         movdqu  xmm4,XMMWORD[32+rdi]
 556         movups  XMMWORD[48+rsi],xmm5
 557         movdqu  xmm5,XMMWORD[48+rdi]
 558         movups  XMMWORD[64+rsi],xmm6
 559         movdqu  xmm6,XMMWORD[64+rdi]
 560         movups  XMMWORD[80+rsi],xmm7
 561         movdqu  xmm7,XMMWORD[80+rdi]
 562         movups  XMMWORD[96+rsi],xmm8
 563         movdqu  xmm8,XMMWORD[96+rdi]
 564         movups  XMMWORD[112+rsi],xmm9
 565         lea     rsi,[128+rsi]
 566         movdqu  xmm9,XMMWORD[112+rdi]
 567         lea     rdi,[128+rdi]
 568 $L$ecb_enc_loop8_enter:
 569
 570         call    _aesni_encrypt8
 571
 572         sub     rdx,0x80
 573         jnc     NEAR $L$ecb_enc_loop8
 574
 575         movups  XMMWORD[rsi],xmm2
 576         mov     rcx,r11
 577         movups  XMMWORD[16+rsi],xmm3
 578         mov     eax,r10d
 579         movups  XMMWORD[32+rsi],xmm4
 580         movups  XMMWORD[48+rsi],xmm5
 581         movups  XMMWORD[64+rsi],xmm6
 582         movups  XMMWORD[80+rsi],xmm7
 583         movups  XMMWORD[96+rsi],xmm8
 584         movups  XMMWORD[112+rsi],xmm9
 585         lea     rsi,[128+rsi]
 586         add     rdx,0x80
 587         jz      NEAR $L$ecb_ret
 588
 589 $L$ecb_enc_tail:
 590         movups  xmm2,XMMWORD[rdi]
 591         cmp     rdx,0x20
 592         jb      NEAR $L$ecb_enc_one
 593         movups  xmm3,XMMWORD[16+rdi]
 594         je      NEAR $L$ecb_enc_two
 595         movups  xmm4,XMMWORD[32+rdi]
 596         cmp     rdx,0x40
 597         jb      NEAR $L$ecb_enc_three
 598         movups  xmm5,XMMWORD[48+rdi]
 599         je      NEAR $L$ecb_enc_four
 600         movups  xmm6,XMMWORD[64+rdi]
 601         cmp     rdx,0x60
 602         jb      NEAR $L$ecb_enc_five
 603         movups  xmm7,XMMWORD[80+rdi]
 604         je      NEAR $L$ecb_enc_six
 605         movdqu  xmm8,XMMWORD[96+rdi]
 606         call    _aesni_encrypt8
 607         movups  XMMWORD[rsi],xmm2
 608         movups  XMMWORD[16+rsi],xmm3
 609         movups  XMMWORD[32+rsi],xmm4
 610         movups  XMMWORD[48+rsi],xmm5
 611         movups  XMMWORD[64+rsi],xmm6
 612         movups  XMMWORD[80+rsi],xmm7
 613         movups  XMMWORD[96+rsi],xmm8
 614         jmp     NEAR $L$ecb_ret
 615 ALIGN   16
 616 $L$ecb_enc_one:
 617         movups  xmm0,XMMWORD[rcx]
 618         movups  xmm1,XMMWORD[16+rcx]
 619         lea     rcx,[32+rcx]
 620         xorps   xmm2,xmm0
 621 $L$oop_enc1_3:
 622 DB      102,15,56,220,209
 623         dec     eax
 624         movups  xmm1,XMMWORD[rcx]
 625         lea     rcx,[16+rcx]
 626         jnz     NEAR $L$oop_enc1_3
 627 DB      102,15,56,221,209
 628         movups  XMMWORD[rsi],xmm2
 629         jmp     NEAR $L$ecb_ret
 630 ALIGN   16
 631 $L$ecb_enc_two:
 632         call    _aesni_encrypt2
 633         movups  XMMWORD[rsi],xmm2
 634         movups  XMMWORD[16+rsi],xmm3
 635         jmp     NEAR $L$ecb_ret
 636 ALIGN   16
 637 $L$ecb_enc_three:
 638         call    _aesni_encrypt3
 639         movups  XMMWORD[rsi],xmm2
 640         movups  XMMWORD[16+rsi],xmm3
 641         movups  XMMWORD[32+rsi],xmm4
 642         jmp     NEAR $L$ecb_ret
 643 ALIGN   16
 644 $L$ecb_enc_four:
 645         call    _aesni_encrypt4
 646         movups  XMMWORD[rsi],xmm2
 647         movups  XMMWORD[16+rsi],xmm3
 648         movups  XMMWORD[32+rsi],xmm4
 649         movups  XMMWORD[48+rsi],xmm5
 650         jmp     NEAR $L$ecb_ret
 651 ALIGN   16
 652 $L$ecb_enc_five:
 653         xorps   xmm7,xmm7
 654         call    _aesni_encrypt6
 655         movups  XMMWORD[rsi],xmm2
 656         movups  XMMWORD[16+rsi],xmm3
 657         movups  XMMWORD[32+rsi],xmm4
 658         movups  XMMWORD[48+rsi],xmm5
 659         movups  XMMWORD[64+rsi],xmm6
 660         jmp     NEAR $L$ecb_ret
 661 ALIGN   16
 662 $L$ecb_enc_six:
 663         call    _aesni_encrypt6
 664         movups  XMMWORD[rsi],xmm2
 665         movups  XMMWORD[16+rsi],xmm3
 666         movups  XMMWORD[32+rsi],xmm4
 667         movups  XMMWORD[48+rsi],xmm5
 668         movups  XMMWORD[64+rsi],xmm6
 669         movups  XMMWORD[80+rsi],xmm7
 670         jmp     NEAR $L$ecb_ret
 671
 672 ALIGN   16
 673 $L$ecb_decrypt:
 674         cmp     rdx,0x80
 675         jb      NEAR $L$ecb_dec_tail
 676
 677         movdqu  xmm2,XMMWORD[rdi]
 678         movdqu  xmm3,XMMWORD[16+rdi]
 679         movdqu  xmm4,XMMWORD[32+rdi]
 680         movdqu  xmm5,XMMWORD[48+rdi]
 681         movdqu  xmm6,XMMWORD[64+rdi]
 682         movdqu  xmm7,XMMWORD[80+rdi]
 683         movdqu  xmm8,XMMWORD[96+rdi]
 684         movdqu  xmm9,XMMWORD[112+rdi]
 685         lea     rdi,[128+rdi]
 686         sub     rdx,0x80
 687         jmp     NEAR $L$ecb_dec_loop8_enter
 688 ALIGN   16
 689 $L$ecb_dec_loop8:
 690         movups  XMMWORD[rsi],xmm2
 691         mov     rcx,r11
 692         movdqu  xmm2,XMMWORD[rdi]
 693         mov     eax,r10d
 694         movups  XMMWORD[16+rsi],xmm3
 695         movdqu  xmm3,XMMWORD[16+rdi]
 696         movups  XMMWORD[32+rsi],xmm4
 697         movdqu  xmm4,XMMWORD[32+rdi]
 698         movups  XMMWORD[48+rsi],xmm5
 699         movdqu  xmm5,XMMWORD[48+rdi]
 700         movups  XMMWORD[64+rsi],xmm6
 701         movdqu  xmm6,XMMWORD[64+rdi]
 702         movups  XMMWORD[80+rsi],xmm7
 703         movdqu  xmm7,XMMWORD[80+rdi]
 704         movups  XMMWORD[96+rsi],xmm8
 705         movdqu  xmm8,XMMWORD[96+rdi]
 706         movups  XMMWORD[112+rsi],xmm9
 707         lea     rsi,[128+rsi]
 708         movdqu  xmm9,XMMWORD[112+rdi]
 709         lea     rdi,[128+rdi]
 710 $L$ecb_dec_loop8_enter:
 711
 712         call    _aesni_decrypt8
 713
 714         movups  xmm0,XMMWORD[r11]
 715         sub     rdx,0x80
 716         jnc     NEAR $L$ecb_dec_loop8
 717
 718         movups  XMMWORD[rsi],xmm2
 719         mov     rcx,r11
 720         movups  XMMWORD[16+rsi],xmm3
 721         mov     eax,r10d
 722         movups  XMMWORD[32+rsi],xmm4
 723         movups  XMMWORD[48+rsi],xmm5
 724         movups  XMMWORD[64+rsi],xmm6
 725         movups  XMMWORD[80+rsi],xmm7
 726         movups  XMMWORD[96+rsi],xmm8
 727         movups  XMMWORD[112+rsi],xmm9
 728         lea     rsi,[128+rsi]
 729         add     rdx,0x80
 730         jz      NEAR $L$ecb_ret
 731
 732 $L$ecb_dec_tail:
 733         movups  xmm2,XMMWORD[rdi]
 734         cmp     rdx,0x20
 735         jb      NEAR $L$ecb_dec_one
 736         movups  xmm3,XMMWORD[16+rdi]
 737         je      NEAR $L$ecb_dec_two
 738         movups  xmm4,XMMWORD[32+rdi]
 739         cmp     rdx,0x40
 740         jb      NEAR $L$ecb_dec_three
 741         movups  xmm5,XMMWORD[48+rdi]
 742         je      NEAR $L$ecb_dec_four
 743         movups  xmm6,XMMWORD[64+rdi]
 744         cmp     rdx,0x60
 745         jb      NEAR $L$ecb_dec_five
 746         movups  xmm7,XMMWORD[80+rdi]
 747         je      NEAR $L$ecb_dec_six
 748         movups  xmm8,XMMWORD[96+rdi]
 749         movups  xmm0,XMMWORD[rcx]
 750         call    _aesni_decrypt8
 751         movups  XMMWORD[rsi],xmm2
 752         movups  XMMWORD[16+rsi],xmm3
 753         movups  XMMWORD[32+rsi],xmm4
 754         movups  XMMWORD[48+rsi],xmm5
 755         movups  XMMWORD[64+rsi],xmm6
 756         movups  XMMWORD[80+rsi],xmm7
 757         movups  XMMWORD[96+rsi],xmm8
 758         jmp     NEAR $L$ecb_ret
 759 ALIGN   16
 760 $L$ecb_dec_one:
 761         movups  xmm0,XMMWORD[rcx]
 762         movups  xmm1,XMMWORD[16+rcx]
 763         lea     rcx,[32+rcx]
 764         xorps   xmm2,xmm0
 765 $L$oop_dec1_4:
 766 DB      102,15,56,222,209
 767         dec     eax
 768         movups  xmm1,XMMWORD[rcx]
 769         lea     rcx,[16+rcx]
 770         jnz     NEAR $L$oop_dec1_4
 771 DB      102,15,56,223,209
 772         movups  XMMWORD[rsi],xmm2
 773         jmp     NEAR $L$ecb_ret
 774 ALIGN   16
 775 $L$ecb_dec_two:
 776         call    _aesni_decrypt2
 777         movups  XMMWORD[rsi],xmm2
 778         movups  XMMWORD[16+rsi],xmm3
 779         jmp     NEAR $L$ecb_ret
 780 ALIGN   16
 781 $L$ecb_dec_three:
 782         call    _aesni_decrypt3
 783         movups  XMMWORD[rsi],xmm2
 784         movups  XMMWORD[16+rsi],xmm3
 785         movups  XMMWORD[32+rsi],xmm4
 786         jmp     NEAR $L$ecb_ret
 787 ALIGN   16
 788 $L$ecb_dec_four:
 789         call    _aesni_decrypt4
 790         movups  XMMWORD[rsi],xmm2
 791         movups  XMMWORD[16+rsi],xmm3
 792         movups  XMMWORD[32+rsi],xmm4
 793         movups  XMMWORD[48+rsi],xmm5
 794         jmp     NEAR $L$ecb_ret
 795 ALIGN   16
 796 $L$ecb_dec_five:
 797         xorps   xmm7,xmm7
 798         call    _aesni_decrypt6
 799         movups  XMMWORD[rsi],xmm2
 800         movups  XMMWORD[16+rsi],xmm3
 801         movups  XMMWORD[32+rsi],xmm4
 802         movups  XMMWORD[48+rsi],xmm5
 803         movups  XMMWORD[64+rsi],xmm6
 804         jmp     NEAR $L$ecb_ret
 805 ALIGN   16
 806 $L$ecb_dec_six:
 807         call    _aesni_decrypt6
 808         movups  XMMWORD[rsi],xmm2
 809         movups  XMMWORD[16+rsi],xmm3
 810         movups  XMMWORD[32+rsi],xmm4
 811         movups  XMMWORD[48+rsi],xmm5
 812         movups  XMMWORD[64+rsi],xmm6
 813         movups  XMMWORD[80+rsi],xmm7
 814
 815 $L$ecb_ret:
 816         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 817         mov     rsi,QWORD[16+rsp]
 818         DB      0F3h,0C3h               ;repret
 819 $L$SEH_end_aesni_ecb_encrypt:
 820 global  aesni_ccm64_encrypt_blocks
 821
 822 ALIGN   16
 823 aesni_ccm64_encrypt_blocks:
 824         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 825         mov     QWORD[16+rsp],rsi
 826         mov     rax,rsp
 827 $L$SEH_begin_aesni_ccm64_encrypt_blocks:
 828         mov     rdi,rcx
 829         mov     rsi,rdx
 830         mov     rdx,r8
 831         mov     rcx,r9
 832         mov     r8,QWORD[40+rsp]
 833         mov     r9,QWORD[48+rsp]
 834
 835
 836         lea     rsp,[((-88))+rsp]
 837         movaps  XMMWORD[rsp],xmm6
 838         movaps  XMMWORD[16+rsp],xmm7
 839         movaps  XMMWORD[32+rsp],xmm8
 840         movaps  XMMWORD[48+rsp],xmm9
 841 $L$ccm64_enc_body:
 842         mov     eax,DWORD[240+rcx]
 843         movdqu  xmm6,XMMWORD[r8]
 844         movdqa  xmm9,XMMWORD[$L$increment64]
 845         movdqa  xmm7,XMMWORD[$L$bswap_mask]
 846
 847         shl     eax,4
 848         mov     r10d,16
 849         lea     r11,[rcx]
 850         movdqu  xmm3,XMMWORD[r9]
 851         movdqa  xmm2,xmm6
 852         lea     rcx,[32+rax*1+rcx]
 853 DB      102,15,56,0,247
 854         sub     r10,rax
 855         jmp     NEAR $L$ccm64_enc_outer
 856 ALIGN   16
 857 $L$ccm64_enc_outer:
 858         movups  xmm0,XMMWORD[r11]
 859         mov     rax,r10
 860         movups  xmm8,XMMWORD[rdi]
 861
 862         xorps   xmm2,xmm0
 863         movups  xmm1,XMMWORD[16+r11]
 864         xorps   xmm0,xmm8
 865         xorps   xmm3,xmm0
 866         movups  xmm0,XMMWORD[32+r11]
 867
 868 $L$ccm64_enc2_loop:
 869 DB      102,15,56,220,209
 870 DB      102,15,56,220,217
 871         movups  xmm1,XMMWORD[rax*1+rcx]
 872         add     rax,32
 873 DB      102,15,56,220,208
 874 DB      102,15,56,220,216
 875         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 876         jnz     NEAR $L$ccm64_enc2_loop
 877 DB      102,15,56,220,209
 878 DB      102,15,56,220,217
 879         paddq   xmm6,xmm9
 880         dec     rdx
 881 DB      102,15,56,221,208
 882 DB      102,15,56,221,216
 883
 884         lea     rdi,[16+rdi]
 885         xorps   xmm8,xmm2
 886         movdqa  xmm2,xmm6
 887         movups  XMMWORD[rsi],xmm8
 888 DB      102,15,56,0,215
 889         lea     rsi,[16+rsi]
 890         jnz     NEAR $L$ccm64_enc_outer
 891
 892         movups  XMMWORD[r9],xmm3
 893         movaps  xmm6,XMMWORD[rsp]
 894         movaps  xmm7,XMMWORD[16+rsp]
 895         movaps  xmm8,XMMWORD[32+rsp]
 896         movaps  xmm9,XMMWORD[48+rsp]
 897         lea     rsp,[88+rsp]
 898 $L$ccm64_enc_ret:
 899         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
 900         mov     rsi,QWORD[16+rsp]
 901         DB      0F3h,0C3h               ;repret
 902 $L$SEH_end_aesni_ccm64_encrypt_blocks:
 903 global  aesni_ccm64_decrypt_blocks
 904
 905 ALIGN   16
 906 aesni_ccm64_decrypt_blocks:
 907         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
 908         mov     QWORD[16+rsp],rsi
 909         mov     rax,rsp
 910 $L$SEH_begin_aesni_ccm64_decrypt_blocks:
 911         mov     rdi,rcx
 912         mov     rsi,rdx
 913         mov     rdx,r8
 914         mov     rcx,r9
 915         mov     r8,QWORD[40+rsp]
 916         mov     r9,QWORD[48+rsp]
 917
 918
 919         lea     rsp,[((-88))+rsp]
 920         movaps  XMMWORD[rsp],xmm6
 921         movaps  XMMWORD[16+rsp],xmm7
 922         movaps  XMMWORD[32+rsp],xmm8
 923         movaps  XMMWORD[48+rsp],xmm9
 924 $L$ccm64_dec_body:
 925         mov     eax,DWORD[240+rcx]
 926         movups  xmm6,XMMWORD[r8]
 927         movdqu  xmm3,XMMWORD[r9]
 928         movdqa  xmm9,XMMWORD[$L$increment64]
 929         movdqa  xmm7,XMMWORD[$L$bswap_mask]
 930
 931         movaps  xmm2,xmm6
 932         mov     r10d,eax
 933         mov     r11,rcx
 934 DB      102,15,56,0,247
 935         movups  xmm0,XMMWORD[rcx]
 936         movups  xmm1,XMMWORD[16+rcx]
 937         lea     rcx,[32+rcx]
 938         xorps   xmm2,xmm0
 939 $L$oop_enc1_5:
 940 DB      102,15,56,220,209
 941         dec     eax
 942         movups  xmm1,XMMWORD[rcx]
 943         lea     rcx,[16+rcx]
 944         jnz     NEAR $L$oop_enc1_5
 945 DB      102,15,56,221,209
 946         shl     r10d,4
 947         mov     eax,16
 948         movups  xmm8,XMMWORD[rdi]
 949         paddq   xmm6,xmm9
 950         lea     rdi,[16+rdi]
 951         sub     rax,r10
 952         lea     rcx,[32+r10*1+r11]
 953         mov     r10,rax
 954         jmp     NEAR $L$ccm64_dec_outer
 955 ALIGN   16
 956 $L$ccm64_dec_outer:
 957         xorps   xmm8,xmm2
 958         movdqa  xmm2,xmm6
 959         movups  XMMWORD[rsi],xmm8
 960         lea     rsi,[16+rsi]
 961 DB      102,15,56,0,215
 962
 963         sub     rdx,1
 964         jz      NEAR $L$ccm64_dec_break
 965
 966         movups  xmm0,XMMWORD[r11]
 967         mov     rax,r10
 968         movups  xmm1,XMMWORD[16+r11]
 969         xorps   xmm8,xmm0
 970         xorps   xmm2,xmm0
 971         xorps   xmm3,xmm8
 972         movups  xmm0,XMMWORD[32+r11]
 973         jmp     NEAR $L$ccm64_dec2_loop
 974 ALIGN   16
 975 $L$ccm64_dec2_loop:
 976 DB      102,15,56,220,209
 977 DB      102,15,56,220,217
 978         movups  xmm1,XMMWORD[rax*1+rcx]
 979         add     rax,32
 980 DB      102,15,56,220,208
 981 DB      102,15,56,220,216
 982         movups  xmm0,XMMWORD[((-16))+rax*1+rcx]
 983         jnz     NEAR $L$ccm64_dec2_loop
 984         movups  xmm8,XMMWORD[rdi]
 985         paddq   xmm6,xmm9
 986 DB      102,15,56,220,209
 987 DB      102,15,56,220,217
 988 DB      102,15,56,221,208
 989 DB      102,15,56,221,216
 990         lea     rdi,[16+rdi]
 991         jmp     NEAR $L$ccm64_dec_outer
 992
 993 ALIGN   16
 994 $L$ccm64_dec_break:
 995
 996         mov     eax,DWORD[240+r11]
 997         movups  xmm0,XMMWORD[r11]
 998         movups  xmm1,XMMWORD[16+r11]
 999         xorps   xmm8,xmm0
1000         lea     r11,[32+r11]
1001         xorps   xmm3,xmm8
1002 $L$oop_enc1_6:
1003 DB      102,15,56,220,217
1004         dec     eax
1005         movups  xmm1,XMMWORD[r11]
1006         lea     r11,[16+r11]
1007         jnz     NEAR $L$oop_enc1_6
1008 DB      102,15,56,221,217
1009         movups  XMMWORD[r9],xmm3
1010         movaps  xmm6,XMMWORD[rsp]
1011         movaps  xmm7,XMMWORD[16+rsp]
1012         movaps  xmm8,XMMWORD[32+rsp]
1013         movaps  xmm9,XMMWORD[48+rsp]
1014         lea     rsp,[88+rsp]
1015 $L$ccm64_dec_ret:
1016         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
1017         mov     rsi,QWORD[16+rsp]
1018         DB      0F3h,0C3h               ;repret
1019 $L$SEH_end_aesni_ccm64_decrypt_blocks:
1020 global  aesni_ctr32_encrypt_blocks
1021
1022 ALIGN   16
1023 aesni_ctr32_encrypt_blocks:
1024         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
1025         mov     QWORD[16+rsp],rsi
1026         mov     rax,rsp
1027 $L$SEH_begin_aesni_ctr32_encrypt_blocks:
1028         mov     rdi,rcx
1029         mov     rsi,rdx
1030         mov     rdx,r8
1031         mov     rcx,r9
1032         mov     r8,QWORD[40+rsp]
1033
1034
1035         lea     rax,[rsp]
1036         push    rbp
1037         sub     rsp,288
1038         and     rsp,-16
1039         movaps  XMMWORD[(-168)+rax],xmm6
1040         movaps  XMMWORD[(-152)+rax],xmm7
1041         movaps  XMMWORD[(-136)+rax],xmm8
1042         movaps  XMMWORD[(-120)+rax],xmm9
1043         movaps  XMMWORD[(-104)+rax],xmm10
1044         movaps  XMMWORD[(-88)+rax],xmm11
1045         movaps  XMMWORD[(-72)+rax],xmm12
1046         movaps  XMMWORD[(-56)+rax],xmm13
1047         movaps  XMMWORD[(-40)+rax],xmm14
1048         movaps  XMMWORD[(-24)+rax],xmm15
1049 $L$ctr32_body:
1050         lea     rbp,[((-8))+rax]
1051
1052         cmp     rdx,1
1053         je      NEAR $L$ctr32_one_shortcut
1054
1055         movdqu  xmm2,XMMWORD[r8]
1056         movdqu  xmm0,XMMWORD[rcx]
1057         mov     r8d,DWORD[12+r8]
1058         pxor    xmm2,xmm0
1059         mov     r11d,DWORD[12+rcx]
1060         movdqa  XMMWORD[rsp],xmm2
1061         bswap   r8d
1062         movdqa  xmm3,xmm2
1063         movdqa  xmm4,xmm2
1064         movdqa  xmm5,xmm2
1065         movdqa  XMMWORD[64+rsp],xmm2
1066         movdqa  XMMWORD[80+rsp],xmm2
1067         movdqa  XMMWORD[96+rsp],xmm2
1068         mov     r10,rdx
1069         movdqa  XMMWORD[112+rsp],xmm2
1070
1071         lea     rax,[1+r8]
1072         lea     rdx,[2+r8]
1073         bswap   eax
1074         bswap   edx
1075         xor     eax,r11d
1076         xor     edx,r11d
1077 DB      102,15,58,34,216,3
1078         lea     rax,[3+r8]
1079         movdqa  XMMWORD[16+rsp],xmm3
1080 DB      102,15,58,34,226,3
1081         bswap   eax
1082         mov     rdx,r10
1083         lea     r10,[4+r8]
1084         movdqa  XMMWORD[32+rsp],xmm4
1085         xor     eax,r11d
1086         bswap   r10d
1087 DB      102,15,58,34,232,3
1088         xor     r10d,r11d
1089         movdqa  XMMWORD[48+rsp],xmm5
1090         lea     r9,[5+r8]
1091         mov     DWORD[((64+12))+rsp],r10d
1092         bswap   r9d
1093         lea     r10,[6+r8]
1094         mov     eax,DWORD[240+rcx]
1095         xor     r9d,r11d
1096         bswap   r10d
1097         mov     DWORD[((80+12))+rsp],r9d
1098         xor     r10d,r11d
1099         lea     r9,[7+r8]
1100         mov     DWORD[((96+12))+rsp],r10d
1101         bswap   r9d
1102         mov     r10d,DWORD[((OPENSSL_ia32cap_P+4))]
1103         xor     r9d,r11d
1104         and     r10d,71303168
1105         mov     DWORD[((112+12))+rsp],r9d
1106
1107         movups  xmm1,XMMWORD[16+rcx]
1108
1109         movdqa  xmm6,XMMWORD[64+rsp]
1110         movdqa  xmm7,XMMWORD[80+rsp]
1111
1112         cmp     rdx,8
1113         jb      NEAR $L$ctr32_tail
1114
1115         sub     rdx,6
1116         cmp     r10d,4194304
1117         je      NEAR $L$ctr32_6x
1118
1119         lea     rcx,[128+rcx]
1120         sub     rdx,2
1121         jmp     NEAR $L$ctr32_loop8
1122
1123 ALIGN   16
1124 $L$ctr32_6x:
1125         shl     eax,4
1126         mov     r10d,48
1127         bswap   r11d
1128         lea     rcx,[32+rax*1+rcx]
1129         sub     r10,rax
1130         jmp     NEAR $L$ctr32_loop6
1131
1132 ALIGN   16
1133 $L$ctr32_loop6:
1134         add     r8d,6
1135         movups  xmm0,XMMWORD[((-48))+r10*1+rcx]
1136 DB      102,15,56,220,209
1137         mov     eax,r8d
1138         xor     eax,r11d
1139 DB      102,15,56,220,217
1140 DB      0x0f,0x38,0xf1,0x44,0x24,12
1141         lea     eax,[1+r8]
1142 DB      102,15,56,220,225
1143         xor     eax,r11d
1144 DB      0x0f,0x38,0xf1,0x44,0x24,28
1145 DB      102,15,56,220,233
1146         lea     eax,[2+r8]
1147         xor     eax,r11d
1148 DB      102,15,56,220,241
1149 DB      0x0f,0x38,0xf1,0x44,0x24,44
1150         lea     eax,[3+r8]
1151 DB      102,15,56,220,249
1152         movups  xmm1,XMMWORD[((-32))+r10*1+rcx]
1153         xor     eax,r11d
1154
1155 DB      102,15,56,220,208
1156 DB      0x0f,0x38,0xf1,0x44,0x24,60
1157         lea     eax,[4+r8]
1158 DB      102,15,56,220,216
1159         xor     eax,r11d
1160 DB      0x0f,0x38,0xf1,0x44,0x24,76
1161 DB      102,15,56,220,224
1162         lea     eax,[5+r8]
1163         xor     eax,r11d
1164 DB      102,15,56,220,232
1165 DB      0x0f,0x38,0xf1,0x44,0x24,92
1166         mov     rax,r10
1167 DB      102,15,56,220,240
1168 DB      102,15,56,220,248
1169         movups  xmm0,XMMWORD[((-16))+r10*1+rcx]
1170
1171         call    $L$enc_loop6
1172
1173         movdqu  xmm8,XMMWORD[rdi]
1174         movdqu  xmm9,XMMWORD[16+rdi]
1175         movdqu  xmm10,XMMWORD[32+rdi]
1176         movdqu  xmm11,XMMWORD[48+rdi]
1177         movdqu  xmm12,XMMWORD[64+rdi]
1178         movdqu  xmm13,XMMWORD[80+rdi]
1179         lea     rdi,[96+rdi]
1180         movups  xmm1,XMMWORD[((-64))+r10*1+rcx]
1181         pxor    xmm8,xmm2
1182         movaps  xmm2,XMMWORD[rsp]
1183         pxor    xmm9,xmm3
1184         movaps  xmm3,XMMWORD[16+rsp]
1185         pxor    xmm10,xmm4
1186         movaps  xmm4,XMMWORD[32+rsp]
1187         pxor    xmm11,xmm5
1188         movaps  xmm5,XMMWORD[48+rsp]
1189         pxor    xmm12,xmm6
1190         movaps  xmm6,XMMWORD[64+rsp]
1191         pxor    xmm13,xmm7
1192         movaps  xmm7,XMMWORD[80+rsp]
1193         movdqu  XMMWORD[rsi],xmm8
1194         movdqu  XMMWORD[16+rsi],xmm9
1195         movdqu  XMMWORD[32+rsi],xmm10
1196         movdqu  XMMWORD[48+rsi],xmm11
1197         movdqu  XMMWORD[64+rsi],xmm12
1198         movdqu  XMMWORD[80+rsi],xmm13
1199         lea     rsi,[96+rsi]
1200
1201         sub     rdx,6
1202         jnc     NEAR $L$ctr32_loop6
1203
1204         add     rdx,6
1205         jz      NEAR $L$ctr32_done
1206
1207         lea     eax,[((-48))+r10]
1208         lea     rcx,[((-80))+r10*1+rcx]
1209         neg     eax
1210         shr     eax,4
1211         jmp     NEAR $L$ctr32_tail
1212
1213 ALIGN   32
1214 $L$ctr32_loop8:
1215         add     r8d,8
1216         movdqa  xmm8,XMMWORD[96+rsp]
1217 DB      102,15,56,220,209
1218         mov     r9d,r8d
1219         movdqa  xmm9,XMMWORD[112+rsp]
1220 DB      102,15,56,220,217
1221         bswap   r9d
1222         movups  xmm0,XMMWORD[((32-128))+rcx]
1223 DB      102,15,56,220,225
1224         xor     r9d,r11d
1225         nop
1226 DB      102,15,56,220,233
1227         mov     DWORD[((0+12))+rsp],r9d
1228         lea     r9,[1+r8]
1229 DB      102,15,56,220,241
1230 DB      102,15,56,220,249
1231 DB      102,68,15,56,220,193
1232 DB      102,68,15,56,220,201
1233         movups  xmm1,XMMWORD[((48-128))+rcx]
1234         bswap   r9d
1235 DB      102,15,56,220,208
1236 DB      102,15,56,220,216
1237         xor     r9d,r11d
1238 DB      0x66,0x90
1239 DB      102,15,56,220,224
1240 DB      102,15,56,220,232
1241         mov     DWORD[((16+12))+rsp],r9d
1242         lea     r9,[2+r8]
1243 DB      102,15,56,220,240
1244 DB      102,15,56,220,248
1245 DB      102,68,15,56,220,192
1246 DB      102,68,15,56,220,200
1247         movups  xmm0,XMMWORD[((64-128))+rcx]
1248         bswap   r9d
1249 DB      102,15,56,220,209
1250 DB      102,15,56,220,217
1251         xor     r9d,r11d
1252 DB      0x66,0x90
1253 DB      102,15,56,220,225
1254 DB      102,15,56,220,233
1255         mov     DWORD[((32+12))+rsp],r9d
1256         lea     r9,[3+r8]
1257 DB      102,15,56,220,241
1258 DB      102,15,56,220,249
1259 DB      102,68,15,56,220,193
1260 DB      102,68,15,56,220,201
1261         movups  xmm1,XMMWORD[((80-128))+rcx]
1262         bswap   r9d
1263 DB      102,15,56,220,208
1264 DB      102,15,56,220,216
1265         xor     r9d,r11d
1266 DB      0x66,0x90
1267 DB      102,15,56,220,224
1268 DB      102,15,56,220,232
1269         mov     DWORD[((48+12))+rsp],r9d
1270         lea     r9,[4+r8]
1271 DB      102,15,56,220,240
1272 DB      102,15,56,220,248
1273 DB      102,68,15,56,220,192
1274 DB      102,68,15,56,220,200
1275         movups  xmm0,XMMWORD[((96-128))+rcx]
1276         bswap   r9d
1277 DB      102,15,56,220,209
1278 DB      102,15,56,220,217
1279         xor     r9d,r11d
1280 DB      0x66,0x90
1281 DB      102,15,56,220,225
1282 DB      102,15,56,220,233
1283         mov     DWORD[((64+12))+rsp],r9d
1284         lea     r9,[5+r8]
1285 DB      102,15,56,220,241
1286 DB      102,15,56,220,249
1287 DB      102,68,15,56,220,193
1288 DB      102,68,15,56,220,201
1289         movups  xmm1,XMMWORD[((112-128))+rcx]
1290         bswap   r9d
1291 DB      102,15,56,220,208
1292 DB      102,15,56,220,216
1293         xor     r9d,r11d
1294 DB      0x66,0x90
1295 DB      102,15,56,220,224
1296 DB      102,15,56,220,232
1297         mov     DWORD[((80+12))+rsp],r9d
1298         lea     r9,[6+r8]
1299 DB      102,15,56,220,240
1300 DB      102,15,56,220,248
1301 DB      102,68,15,56,220,192
1302 DB      102,68,15,56,220,200
1303         movups  xmm0,XMMWORD[((128-128))+rcx]
1304         bswap   r9d
1305 DB      102,15,56,220,209
1306 DB      102,15,56,220,217
1307         xor     r9d,r11d
1308 DB      0x66,0x90
1309 DB      102,15,56,220,225
1310 DB      102,15,56,220,233
1311         mov     DWORD[((96+12))+rsp],r9d
1312         lea     r9,[7+r8]
1313 DB      102,15,56,220,241
1314 DB      102,15,56,220,249
1315 DB      102,68,15,56,220,193
1316 DB      102,68,15,56,220,201
1317         movups  xmm1,XMMWORD[((144-128))+rcx]
1318         bswap   r9d
1319 DB      102,15,56,220,208
1320 DB      102,15,56,220,216
1321 DB      102,15,56,220,224
1322         xor     r9d,r11d
1323         movdqu  xmm10,XMMWORD[rdi]
1324 DB      102,15,56,220,232
1325         mov     DWORD[((112+12))+rsp],r9d
1326         cmp     eax,11
1327 DB      102,15,56,220,240
1328 DB      102,15,56,220,248
1329 DB      102,68,15,56,220,192
1330 DB      102,68,15,56,220,200
1331         movups  xmm0,XMMWORD[((160-128))+rcx]
1332
1333         jb      NEAR $L$ctr32_enc_done
1334
1335 DB      102,15,56,220,209
1336 DB      102,15,56,220,217
1337 DB      102,15,56,220,225
1338 DB      102,15,56,220,233
1339 DB      102,15,56,220,241
1340 DB      102,15,56,220,249
1341 DB      102,68,15,56,220,193
1342 DB      102,68,15,56,220,201
1343         movups  xmm1,XMMWORD[((176-128))+rcx]
1344
1345 DB      102,15,56,220,208
1346 DB      102,15,56,220,216
1347 DB      102,15,56,220,224
1348 DB      102,15,56,220,232
1349 DB      102,15,56,220,240
1350 DB      102,15,56,220,248
1351 DB      102,68,15,56,220,192
1352 DB      102,68,15,56,220,200
1353         movups  xmm0,XMMWORD[((192-128))+rcx]
1354         je      NEAR $L$ctr32_enc_done
1355
1356 DB      102,15,56,220,209
1357 DB      102,15,56,220,217
1358 DB      102,15,56,220,225
1359 DB      102,15,56,220,233
1360 DB      102,15,56,220,241
1361 DB      102,15,56,220,249
1362 DB      102,68,15,56,220,193
1363 DB      102,68,15,56,220,201
1364         movups  xmm1,XMMWORD[((208-128))+rcx]
1365
1366 DB      102,15,56,220,208
1367 DB      102,15,56,220,216
1368 DB      102,15,56,220,224
1369 DB      102,15,56,220,232
1370 DB      102,15,56,220,240
1371 DB      102,15,56,220,248
1372 DB      102,68,15,56,220,192
1373 DB      102,68,15,56,220,200
1374         movups  xmm0,XMMWORD[((224-128))+rcx]
1375         jmp     NEAR $L$ctr32_enc_done
1376
1377 ALIGN   16
1378 $L$ctr32_enc_done:
1379         movdqu  xmm11,XMMWORD[16+rdi]
1380         pxor    xmm10,xmm0
1381         movdqu  xmm12,XMMWORD[32+rdi]
1382         pxor    xmm11,xmm0
1383         movdqu  xmm13,XMMWORD[48+rdi]
1384         pxor    xmm12,xmm0
1385         movdqu  xmm14,XMMWORD[64+rdi]
1386         pxor    xmm13,xmm0
1387         movdqu  xmm15,XMMWORD[80+rdi]
1388         pxor    xmm14,xmm0
1389         pxor    xmm15,xmm0
1390 DB      102,15,56,220,209
1391 DB      102,15,56,220,217
1392 DB      102,15,56,220,225
1393 DB      102,15,56,220,233
1394 DB      102,15,56,220,241
1395 DB      102,15,56,220,249
1396 DB      102,68,15,56,220,193
1397 DB      102,68,15,56,220,201
1398         movdqu  xmm1,XMMWORD[96+rdi]
1399         lea     rdi,[128+rdi]
1400
1401 DB      102,65,15,56,221,210
1402         pxor    xmm1,xmm0
1403         movdqu  xmm10,XMMWORD[((112-128))+rdi]
1404 DB      102,65,15,56,221,219
1405         pxor    xmm10,xmm0
1406         movdqa  xmm11,XMMWORD[rsp]
1407 DB      102,65,15,56,221,228
1408 DB      102,65,15,56,221,237
1409         movdqa  xmm12,XMMWORD[16+rsp]
1410         movdqa  xmm13,XMMWORD[32+rsp]
1411 DB      102,65,15,56,221,246
1412 DB      102,65,15,56,221,255
1413         movdqa  xmm14,XMMWORD[48+rsp]
1414         movdqa  xmm15,XMMWORD[64+rsp]
1415 DB      102,68,15,56,221,193
1416         movdqa  xmm0,XMMWORD[80+rsp]
1417         movups  xmm1,XMMWORD[((16-128))+rcx]
1418 DB      102,69,15,56,221,202
1419
1420         movups  XMMWORD[rsi],xmm2
1421         movdqa  xmm2,xmm11
1422         movups  XMMWORD[16+rsi],xmm3
1423         movdqa  xmm3,xmm12
1424         movups  XMMWORD[32+rsi],xmm4
1425         movdqa  xmm4,xmm13
1426         movups  XMMWORD[48+rsi],xmm5
1427         movdqa  xmm5,xmm14
1428         movups  XMMWORD[64+rsi],xmm6
1429         movdqa  xmm6,xmm15
1430         movups  XMMWORD[80+rsi],xmm7
1431         movdqa  xmm7,xmm0
1432         movups  XMMWORD[96+rsi],xmm8
1433         movups  XMMWORD[112+rsi],xmm9
1434         lea     rsi,[128+rsi]
1435
1436         sub     rdx,8
1437         jnc     NEAR $L$ctr32_loop8
1438
1439         add     rdx,8
1440         jz      NEAR $L$ctr32_done
1441         lea     rcx,[((-128))+rcx]
1442
1443 $L$ctr32_tail:
1444         lea     rcx,[16+rcx]
1445         cmp     rdx,4
1446         jb      NEAR $L$ctr32_loop3
1447         je      NEAR $L$ctr32_loop4
1448
1449         shl     eax,4
1450         movdqa  xmm8,XMMWORD[96+rsp]
1451         pxor    xmm9,xmm9
1452
1453         movups  xmm0,XMMWORD[16+rcx]
1454 DB      102,15,56,220,209
1455 DB      102,15,56,220,217
1456         lea     rcx,[((32-16))+rax*1+rcx]
1457         neg     rax
1458 DB      102,15,56,220,225
1459         add     rax,16
1460         movups  xmm10,XMMWORD[rdi]
1461 DB      102,15,56,220,233
1462 DB      102,15,56,220,241
1463         movups  xmm11,XMMWORD[16+rdi]
1464         movups  xmm12,XMMWORD[32+rdi]
1465 DB      102,15,56,220,249
1466 DB      102,68,15,56,220,193
1467
1468         call    $L$enc_loop8_enter
1469
1470         movdqu  xmm13,XMMWORD[48+rdi]
1471         pxor    xmm2,xmm10
1472         movdqu  xmm10,XMMWORD[64+rdi]
1473         pxor    xmm3,xmm11
1474         movdqu  XMMWORD[rsi],xmm2
1475         pxor    xmm4,xmm12
1476         movdqu  XMMWORD[16+rsi],xmm3
1477         pxor    xmm5,xmm13
1478         movdqu  XMMWORD[32+rsi],xmm4
1479         pxor    xmm6,xmm10
1480         movdqu  XMMWORD[48+rsi],xmm5
1481         movdqu  XMMWORD[64+rsi],xmm6
1482         cmp     rdx,6
1483         jb      NEAR $L$ctr32_done
1484
1485         movups  xmm11,XMMWORD[80+rdi]
1486         xorps   xmm7,xmm11
1487         movups  XMMWORD[80+rsi],xmm7
1488         je      NEAR $L$ctr32_done
1489
1490         movups  xmm12,XMMWORD[96+rdi]
1491         xorps   xmm8,xmm12
1492         movups  XMMWORD[96+rsi],xmm8
1493         jmp     NEAR $L$ctr32_done
1494
1495 ALIGN   32
1496 $L$ctr32_loop4:
1497 DB      102,15,56,220,209
1498         lea     rcx,[16+rcx]
1499         dec     eax
1500 DB      102,15,56,220,217
1501 DB      102,15,56,220,225
1502 DB      102,15,56,220,233
1503         movups  xmm1,XMMWORD[rcx]
1504         jnz     NEAR $L$ctr32_loop4
1505 DB      102,15,56,221,209
1506 DB      102,15,56,221,217
1507         movups  xmm10,XMMWORD[rdi]
1508         movups  xmm11,XMMWORD[16+rdi]
1509 DB      102,15,56,221,225
1510 DB      102,15,56,221,233
1511         movups  xmm12,XMMWORD[32+rdi]
1512         movups  xmm13,XMMWORD[48+rdi]
1513
1514         xorps   xmm2,xmm10
1515         movups  XMMWORD[rsi],xmm2
1516         xorps   xmm3,xmm11
1517         movups  XMMWORD[16+rsi],xmm3
1518         pxor    xmm4,xmm12
1519         movdqu  XMMWORD[32+rsi],xmm4
1520         pxor    xmm5,xmm13
1521         movdqu  XMMWORD[48+rsi],xmm5
1522         jmp     NEAR $L$ctr32_done
1523
1524 ALIGN   32
1525 $L$ctr32_loop3:
1526 DB      102,15,56,220,209
1527         lea     rcx,[16+rcx]
1528         dec     eax
1529 DB      102,15,56,220,217
1530 DB      102,15,56,220,225
1531         movups  xmm1,XMMWORD[rcx]
1532         jnz     NEAR $L$ctr32_loop3
1533 DB      102,15,56,221,209
1534 DB      102,15,56,221,217
1535 DB      102,15,56,221,225
1536
1537         movups  xmm10,XMMWORD[rdi]
1538         xorps   xmm2,xmm10
1539         movups  XMMWORD[rsi],xmm2
1540         cmp     rdx,2
1541         jb      NEAR $L$ctr32_done
1542
1543         movups  xmm11,XMMWORD[16+rdi]
1544         xorps   xmm3,xmm11
1545         movups  XMMWORD[16+rsi],xmm3
1546         je      NEAR $L$ctr32_done
1547
1548         movups  xmm12,XMMWORD[32+rdi]
1549         xorps   xmm4,xmm12
1550         movups  XMMWORD[32+rsi],xmm4
1551         jmp     NEAR $L$ctr32_done
1552
1553 ALIGN   16
1554 $L$ctr32_one_shortcut:
1555         movups  xmm2,XMMWORD[r8]
1556         movups  xmm10,XMMWORD[rdi]
1557         mov     eax,DWORD[240+rcx]
1558         movups  xmm0,XMMWORD[rcx]
1559         movups  xmm1,XMMWORD[16+rcx]
1560         lea     rcx,[32+rcx]
1561         xorps   xmm2,xmm0
1562 $L$oop_enc1_7:
1563 DB      102,15,56,220,209
1564         dec     eax
1565         movups  xmm1,XMMWORD[rcx]
1566         lea     rcx,[16+rcx]
1567         jnz     NEAR $L$oop_enc1_7
1568 DB      102,15,56,221,209
1569         xorps   xmm2,xmm10
1570         movups  XMMWORD[rsi],xmm2
1571         jmp     NEAR $L$ctr32_done
1572
1573 ALIGN   16
1574 $L$ctr32_done:
1575         movaps  xmm6,XMMWORD[((-160))+rbp]
1576         movaps  xmm7,XMMWORD[((-144))+rbp]
1577         movaps  xmm8,XMMWORD[((-128))+rbp]
1578         movaps  xmm9,XMMWORD[((-112))+rbp]
1579         movaps  xmm10,XMMWORD[((-96))+rbp]
1580         movaps  xmm11,XMMWORD[((-80))+rbp]
1581         movaps  xmm12,XMMWORD[((-64))+rbp]
1582         movaps  xmm13,XMMWORD[((-48))+rbp]
1583         movaps  xmm14,XMMWORD[((-32))+rbp]
1584         movaps  xmm15,XMMWORD[((-16))+rbp]
1585         lea     rsp,[rbp]
1586         pop     rbp
1587 $L$ctr32_epilogue:
1588         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
1589         mov     rsi,QWORD[16+rsp]
1590         DB      0F3h,0C3h               ;repret
1591 $L$SEH_end_aesni_ctr32_encrypt_blocks:
1592 global  aesni_xts_encrypt
1593
1594 ALIGN   16
1595 aesni_xts_encrypt:
1596         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
1597         mov     QWORD[16+rsp],rsi
1598         mov     rax,rsp
1599 $L$SEH_begin_aesni_xts_encrypt:
1600         mov     rdi,rcx
1601         mov     rsi,rdx
1602         mov     rdx,r8
1603         mov     rcx,r9
1604         mov     r8,QWORD[40+rsp]
1605         mov     r9,QWORD[48+rsp]
1606
1607
1608         lea     rax,[rsp]
1609         push    rbp
1610         sub     rsp,272
1611         and     rsp,-16
1612         movaps  XMMWORD[(-168)+rax],xmm6
1613         movaps  XMMWORD[(-152)+rax],xmm7
1614         movaps  XMMWORD[(-136)+rax],xmm8
1615         movaps  XMMWORD[(-120)+rax],xmm9
1616         movaps  XMMWORD[(-104)+rax],xmm10
1617         movaps  XMMWORD[(-88)+rax],xmm11
1618         movaps  XMMWORD[(-72)+rax],xmm12
1619         movaps  XMMWORD[(-56)+rax],xmm13
1620         movaps  XMMWORD[(-40)+rax],xmm14
1621         movaps  XMMWORD[(-24)+rax],xmm15
1622 $L$xts_enc_body:
1623         lea     rbp,[((-8))+rax]
1624         movups  xmm2,XMMWORD[r9]
1625         mov     eax,DWORD[240+r8]
1626         mov     r10d,DWORD[240+rcx]
1627         movups  xmm0,XMMWORD[r8]
1628         movups  xmm1,XMMWORD[16+r8]
1629         lea     r8,[32+r8]
1630         xorps   xmm2,xmm0
1631 $L$oop_enc1_8:
1632 DB      102,15,56,220,209
1633         dec     eax
1634         movups  xmm1,XMMWORD[r8]
1635         lea     r8,[16+r8]
1636         jnz     NEAR $L$oop_enc1_8
1637 DB      102,15,56,221,209
1638         movups  xmm0,XMMWORD[rcx]
1639         mov     r11,rcx
1640         mov     eax,r10d
1641         shl     r10d,4
1642         mov     r9,rdx
1643         and     rdx,-16
1644
1645         movups  xmm1,XMMWORD[16+r10*1+rcx]
1646
1647         movdqa  xmm8,XMMWORD[$L$xts_magic]
1648         movdqa  xmm15,xmm2
1649         pshufd  xmm9,xmm2,0x5f
1650         pxor    xmm1,xmm0
1651         movdqa  xmm14,xmm9
1652         paddd   xmm9,xmm9
1653         movdqa  xmm10,xmm15
1654         psrad   xmm14,31
1655         paddq   xmm15,xmm15
1656         pand    xmm14,xmm8
1657         pxor    xmm10,xmm0
1658         pxor    xmm15,xmm14
1659         movdqa  xmm14,xmm9
1660         paddd   xmm9,xmm9
1661         movdqa  xmm11,xmm15
1662         psrad   xmm14,31
1663         paddq   xmm15,xmm15
1664         pand    xmm14,xmm8
1665         pxor    xmm11,xmm0
1666         pxor    xmm15,xmm14
1667         movdqa  xmm14,xmm9
1668         paddd   xmm9,xmm9
1669         movdqa  xmm12,xmm15
1670         psrad   xmm14,31
1671         paddq   xmm15,xmm15
1672         pand    xmm14,xmm8
1673         pxor    xmm12,xmm0
1674         pxor    xmm15,xmm14
1675         movdqa  xmm14,xmm9
1676         paddd   xmm9,xmm9
1677         movdqa  xmm13,xmm15
1678         psrad   xmm14,31
1679         paddq   xmm15,xmm15
1680         pand    xmm14,xmm8
1681         pxor    xmm13,xmm0
1682         pxor    xmm15,xmm14
1683         movdqa  xmm14,xmm15
1684         psrad   xmm9,31
1685         paddq   xmm15,xmm15
1686         pand    xmm9,xmm8
1687         pxor    xmm14,xmm0
1688         pxor    xmm15,xmm9
1689         movaps  XMMWORD[96+rsp],xmm1
1690
1691         sub     rdx,16*6
1692         jc      NEAR $L$xts_enc_short
1693
1694         mov     eax,16+96
1695         lea     rcx,[32+r10*1+r11]
1696         sub     rax,r10
1697         movups  xmm1,XMMWORD[16+r11]
1698         mov     r10,rax
1699         lea     r8,[$L$xts_magic]
1700         jmp     NEAR $L$xts_enc_grandloop
1701
1702 ALIGN   32
1703 $L$xts_enc_grandloop:
1704         movdqu  xmm2,XMMWORD[rdi]
1705         movdqa  xmm8,xmm0
1706         movdqu  xmm3,XMMWORD[16+rdi]
1707         pxor    xmm2,xmm10
1708         movdqu  xmm4,XMMWORD[32+rdi]
1709         pxor    xmm3,xmm11
1710 DB      102,15,56,220,209
1711         movdqu  xmm5,XMMWORD[48+rdi]
1712         pxor    xmm4,xmm12
1713 DB      102,15,56,220,217
1714         movdqu  xmm6,XMMWORD[64+rdi]
1715         pxor    xmm5,xmm13
1716 DB      102,15,56,220,225
1717         movdqu  xmm7,XMMWORD[80+rdi]
1718         pxor    xmm8,xmm15
1719         movdqa  xmm9,XMMWORD[96+rsp]
1720         pxor    xmm6,xmm14
1721 DB      102,15,56,220,233
1722         movups  xmm0,XMMWORD[32+r11]
1723         lea     rdi,[96+rdi]
1724         pxor    xmm7,xmm8
1725
1726         pxor    xmm10,xmm9
1727 DB      102,15,56,220,241
1728         pxor    xmm11,xmm9
1729         movdqa  XMMWORD[rsp],xmm10
1730 DB      102,15,56,220,249
1731         movups  xmm1,XMMWORD[48+r11]
1732         pxor    xmm12,xmm9
1733
1734 DB      102,15,56,220,208
1735         pxor    xmm13,xmm9
1736         movdqa  XMMWORD[16+rsp],xmm11
1737 DB      102,15,56,220,216
1738         pxor    xmm14,xmm9
1739         movdqa  XMMWORD[32+rsp],xmm12
1740 DB      102,15,56,220,224
1741 DB      102,15,56,220,232
1742         pxor    xmm8,xmm9
1743         movdqa  XMMWORD[64+rsp],xmm14
1744 DB      102,15,56,220,240
1745 DB      102,15,56,220,248
1746         movups  xmm0,XMMWORD[64+r11]
1747         movdqa  XMMWORD[80+rsp],xmm8
1748         pshufd  xmm9,xmm15,0x5f
1749         jmp     NEAR $L$xts_enc_loop6
1750 ALIGN   32
1751 $L$xts_enc_loop6:
1752 DB      102,15,56,220,209
1753 DB      102,15,56,220,217
1754 DB      102,15,56,220,225
1755 DB      102,15,56,220,233
1756 DB      102,15,56,220,241
1757 DB      102,15,56,220,249
1758         movups  xmm1,XMMWORD[((-64))+rax*1+rcx]
1759         add     rax,32
1760
1761 DB      102,15,56,220,208
1762 DB      102,15,56,220,216
1763 DB      102,15,56,220,224
1764 DB      102,15,56,220,232
1765 DB      102,15,56,220,240
1766 DB      102,15,56,220,248
1767         movups  xmm0,XMMWORD[((-80))+rax*1+rcx]
1768         jnz     NEAR $L$xts_enc_loop6
1769
1770         movdqa  xmm8,XMMWORD[r8]
1771         movdqa  xmm14,xmm9
1772         paddd   xmm9,xmm9
1773 DB      102,15,56,220,209
1774         paddq   xmm15,xmm15
1775         psrad   xmm14,31
1776 DB      102,15,56,220,217
1777         pand    xmm14,xmm8
1778         movups  xmm10,XMMWORD[r11]
1779 DB      102,15,56,220,225
1780 DB      102,15,56,220,233
1781 DB      102,15,56,220,241
1782         pxor    xmm15,xmm14
1783         movaps  xmm11,xmm10
1784 DB      102,15,56,220,249
1785         movups  xmm1,XMMWORD[((-64))+rcx]
1786
1787         movdqa  xmm14,xmm9
1788 DB      102,15,56,220,208
1789         paddd   xmm9,xmm9
1790         pxor    xmm10,xmm15
1791 DB      102,15,56,220,216
1792         psrad   xmm14,31
1793         paddq   xmm15,xmm15
1794 DB      102,15,56,220,224
1795 DB      102,15,56,220,232
1796         pand    xmm14,xmm8
1797         movaps  xmm12,xmm11
1798 DB      102,15,56,220,240
1799         pxor    xmm15,xmm14
1800         movdqa  xmm14,xmm9
1801 DB      102,15,56,220,248
1802         movups  xmm0,XMMWORD[((-48))+rcx]
1803
1804         paddd   xmm9,xmm9
1805 DB      102,15,56,220,209
1806         pxor    xmm11,xmm15
1807         psrad   xmm14,31
1808 DB      102,15,56,220,217
1809         paddq   xmm15,xmm15
1810         pand    xmm14,xmm8
1811 DB      102,15,56,220,225
1812 DB      102,15,56,220,233
1813         movdqa  XMMWORD[48+rsp],xmm13
1814         pxor    xmm15,xmm14
1815 DB      102,15,56,220,241
1816         movaps  xmm13,xmm12
1817         movdqa  xmm14,xmm9
1818 DB      102,15,56,220,249
1819         movups  xmm1,XMMWORD[((-32))+rcx]
1820
1821         paddd   xmm9,xmm9
1822 DB      102,15,56,220,208
1823         pxor    xmm12,xmm15
1824         psrad   xmm14,31
1825 DB      102,15,56,220,216
1826         paddq   xmm15,xmm15
1827         pand    xmm14,xmm8
1828 DB      102,15,56,220,224
1829 DB      102,15,56,220,232
1830 DB      102,15,56,220,240
1831         pxor    xmm15,xmm14
1832         movaps  xmm14,xmm13
1833 DB      102,15,56,220,248
1834
1835         movdqa  xmm0,xmm9
1836         paddd   xmm9,xmm9
1837 DB      102,15,56,220,209
1838         pxor    xmm13,xmm15
1839         psrad   xmm0,31
1840 DB      102,15,56,220,217
1841         paddq   xmm15,xmm15
1842         pand    xmm0,xmm8
1843 DB      102,15,56,220,225
1844 DB      102,15,56,220,233
1845         pxor    xmm15,xmm0
1846         movups  xmm0,XMMWORD[r11]
1847 DB      102,15,56,220,241
1848 DB      102,15,56,220,249
1849         movups  xmm1,XMMWORD[16+r11]
1850
1851         pxor    xmm14,xmm15
1852 DB      102,15,56,221,84,36,0
1853         psrad   xmm9,31
1854         paddq   xmm15,xmm15
1855 DB      102,15,56,221,92,36,16
1856 DB      102,15,56,221,100,36,32
1857         pand    xmm9,xmm8
1858         mov     rax,r10
1859 DB      102,15,56,221,108,36,48
1860 DB      102,15,56,221,116,36,64
1861 DB      102,15,56,221,124,36,80
1862         pxor    xmm15,xmm9
1863
1864         lea     rsi,[96+rsi]
1865         movups  XMMWORD[(-96)+rsi],xmm2
1866         movups  XMMWORD[(-80)+rsi],xmm3
1867         movups  XMMWORD[(-64)+rsi],xmm4
1868         movups  XMMWORD[(-48)+rsi],xmm5
1869         movups  XMMWORD[(-32)+rsi],xmm6
1870         movups  XMMWORD[(-16)+rsi],xmm7
1871         sub     rdx,16*6
1872         jnc     NEAR $L$xts_enc_grandloop
1873
1874         mov     eax,16+96
1875         sub     eax,r10d
1876         mov     rcx,r11
1877         shr     eax,4
1878
1879 $L$xts_enc_short:
1880         mov     r10d,eax
1881         pxor    xmm10,xmm0
1882         add     rdx,16*6
1883         jz      NEAR $L$xts_enc_done
1884
1885         pxor    xmm11,xmm0
1886         cmp     rdx,0x20
1887         jb      NEAR $L$xts_enc_one
1888         pxor    xmm12,xmm0
1889         je      NEAR $L$xts_enc_two
1890
1891         pxor    xmm13,xmm0
1892         cmp     rdx,0x40
1893         jb      NEAR $L$xts_enc_three
1894         pxor    xmm14,xmm0
1895         je      NEAR $L$xts_enc_four
1896
1897         movdqu  xmm2,XMMWORD[rdi]
1898         movdqu  xmm3,XMMWORD[16+rdi]
1899         movdqu  xmm4,XMMWORD[32+rdi]
1900         pxor    xmm2,xmm10
1901         movdqu  xmm5,XMMWORD[48+rdi]
1902         pxor    xmm3,xmm11
1903         movdqu  xmm6,XMMWORD[64+rdi]
1904         lea     rdi,[80+rdi]
1905         pxor    xmm4,xmm12
1906         pxor    xmm5,xmm13
1907         pxor    xmm6,xmm14
1908
1909         call    _aesni_encrypt6
1910
1911         xorps   xmm2,xmm10
1912         movdqa  xmm10,xmm15
1913         xorps   xmm3,xmm11
1914         xorps   xmm4,xmm12
1915         movdqu  XMMWORD[rsi],xmm2
1916         xorps   xmm5,xmm13
1917         movdqu  XMMWORD[16+rsi],xmm3
1918         xorps   xmm6,xmm14
1919         movdqu  XMMWORD[32+rsi],xmm4
1920         movdqu  XMMWORD[48+rsi],xmm5
1921         movdqu  XMMWORD[64+rsi],xmm6
1922         lea     rsi,[80+rsi]
1923         jmp     NEAR $L$xts_enc_done
1924
1925 ALIGN   16
1926 $L$xts_enc_one:
1927         movups  xmm2,XMMWORD[rdi]
1928         lea     rdi,[16+rdi]
1929         xorps   xmm2,xmm10
1930         movups  xmm0,XMMWORD[rcx]
1931         movups  xmm1,XMMWORD[16+rcx]
1932         lea     rcx,[32+rcx]
1933         xorps   xmm2,xmm0
1934 $L$oop_enc1_9:
1935 DB      102,15,56,220,209
1936         dec     eax
1937         movups  xmm1,XMMWORD[rcx]
1938         lea     rcx,[16+rcx]
1939         jnz     NEAR $L$oop_enc1_9
1940 DB      102,15,56,221,209
1941         xorps   xmm2,xmm10
1942         movdqa  xmm10,xmm11
1943         movups  XMMWORD[rsi],xmm2
1944         lea     rsi,[16+rsi]
1945         jmp     NEAR $L$xts_enc_done
1946
1947 ALIGN   16
1948 $L$xts_enc_two:
1949         movups  xmm2,XMMWORD[rdi]
1950         movups  xmm3,XMMWORD[16+rdi]
1951         lea     rdi,[32+rdi]
1952         xorps   xmm2,xmm10
1953         xorps   xmm3,xmm11
1954
1955         call    _aesni_encrypt2
1956
1957         xorps   xmm2,xmm10
1958         movdqa  xmm10,xmm12
1959         xorps   xmm3,xmm11
1960         movups  XMMWORD[rsi],xmm2
1961         movups  XMMWORD[16+rsi],xmm3
1962         lea     rsi,[32+rsi]
1963         jmp     NEAR $L$xts_enc_done
1964
1965 ALIGN   16
1966 $L$xts_enc_three:
1967         movups  xmm2,XMMWORD[rdi]
1968         movups  xmm3,XMMWORD[16+rdi]
1969         movups  xmm4,XMMWORD[32+rdi]
1970         lea     rdi,[48+rdi]
1971         xorps   xmm2,xmm10
1972         xorps   xmm3,xmm11
1973         xorps   xmm4,xmm12
1974
1975         call    _aesni_encrypt3
1976
1977         xorps   xmm2,xmm10
1978         movdqa  xmm10,xmm13
1979         xorps   xmm3,xmm11
1980         xorps   xmm4,xmm12
1981         movups  XMMWORD[rsi],xmm2
1982         movups  XMMWORD[16+rsi],xmm3
1983         movups  XMMWORD[32+rsi],xmm4
1984         lea     rsi,[48+rsi]
1985         jmp     NEAR $L$xts_enc_done
1986
1987 ALIGN   16
1988 $L$xts_enc_four:
1989         movups  xmm2,XMMWORD[rdi]
1990         movups  xmm3,XMMWORD[16+rdi]
1991         movups  xmm4,XMMWORD[32+rdi]
1992         xorps   xmm2,xmm10
1993         movups  xmm5,XMMWORD[48+rdi]
1994         lea     rdi,[64+rdi]
1995         xorps   xmm3,xmm11
1996         xorps   xmm4,xmm12
1997         xorps   xmm5,xmm13
1998
1999         call    _aesni_encrypt4
2000
2001         pxor    xmm2,xmm10
2002         movdqa  xmm10,xmm14
2003         pxor    xmm3,xmm11
2004         pxor    xmm4,xmm12
2005         movdqu  XMMWORD[rsi],xmm2
2006         pxor    xmm5,xmm13
2007         movdqu  XMMWORD[16+rsi],xmm3
2008         movdqu  XMMWORD[32+rsi],xmm4
2009         movdqu  XMMWORD[48+rsi],xmm5
2010         lea     rsi,[64+rsi]
2011         jmp     NEAR $L$xts_enc_done
2012
2013 ALIGN   16
2014 $L$xts_enc_done:
2015         and     r9,15
2016         jz      NEAR $L$xts_enc_ret
2017         mov     rdx,r9
2018
2019 $L$xts_enc_steal:
2020         movzx   eax,BYTE[rdi]
2021         movzx   ecx,BYTE[((-16))+rsi]
2022         lea     rdi,[1+rdi]
2023         mov     BYTE[((-16))+rsi],al
2024         mov     BYTE[rsi],cl
2025         lea     rsi,[1+rsi]
2026         sub     rdx,1
2027         jnz     NEAR $L$xts_enc_steal
2028
2029         sub     rsi,r9
2030         mov     rcx,r11
2031         mov     eax,r10d
2032
2033         movups  xmm2,XMMWORD[((-16))+rsi]
2034         xorps   xmm2,xmm10
2035         movups  xmm0,XMMWORD[rcx]
2036         movups  xmm1,XMMWORD[16+rcx]
2037         lea     rcx,[32+rcx]
2038         xorps   xmm2,xmm0
2039 $L$oop_enc1_10:
2040 DB      102,15,56,220,209
2041         dec     eax
2042         movups  xmm1,XMMWORD[rcx]
2043         lea     rcx,[16+rcx]
2044         jnz     NEAR $L$oop_enc1_10
2045 DB      102,15,56,221,209
2046         xorps   xmm2,xmm10
2047         movups  XMMWORD[(-16)+rsi],xmm2
2048
2049 $L$xts_enc_ret:
2050         movaps  xmm6,XMMWORD[((-160))+rbp]
2051         movaps  xmm7,XMMWORD[((-144))+rbp]
2052         movaps  xmm8,XMMWORD[((-128))+rbp]
2053         movaps  xmm9,XMMWORD[((-112))+rbp]
2054         movaps  xmm10,XMMWORD[((-96))+rbp]
2055         movaps  xmm11,XMMWORD[((-80))+rbp]
2056         movaps  xmm12,XMMWORD[((-64))+rbp]
2057         movaps  xmm13,XMMWORD[((-48))+rbp]
2058         movaps  xmm14,XMMWORD[((-32))+rbp]
2059         movaps  xmm15,XMMWORD[((-16))+rbp]
2060         lea     rsp,[rbp]
2061         pop     rbp
2062 $L$xts_enc_epilogue:
2063         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
2064         mov     rsi,QWORD[16+rsp]
2065         DB      0F3h,0C3h               ;repret
2066 $L$SEH_end_aesni_xts_encrypt:
2067 global  aesni_xts_decrypt
2068
2069 ALIGN   16
2070 aesni_xts_decrypt:
2071         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
2072         mov     QWORD[16+rsp],rsi
2073         mov     rax,rsp
2074 $L$SEH_begin_aesni_xts_decrypt:
2075         mov     rdi,rcx
2076         mov     rsi,rdx
2077         mov     rdx,r8
2078         mov     rcx,r9
2079         mov     r8,QWORD[40+rsp]
2080         mov     r9,QWORD[48+rsp]
2081
2082
2083         lea     rax,[rsp]
2084         push    rbp
2085         sub     rsp,272
2086         and     rsp,-16
2087         movaps  XMMWORD[(-168)+rax],xmm6
2088         movaps  XMMWORD[(-152)+rax],xmm7
2089         movaps  XMMWORD[(-136)+rax],xmm8
2090         movaps  XMMWORD[(-120)+rax],xmm9
2091         movaps  XMMWORD[(-104)+rax],xmm10
2092         movaps  XMMWORD[(-88)+rax],xmm11
2093         movaps  XMMWORD[(-72)+rax],xmm12
2094         movaps  XMMWORD[(-56)+rax],xmm13
2095         movaps  XMMWORD[(-40)+rax],xmm14
2096         movaps  XMMWORD[(-24)+rax],xmm15
2097 $L$xts_dec_body:
2098         lea     rbp,[((-8))+rax]
2099         movups  xmm2,XMMWORD[r9]
2100         mov     eax,DWORD[240+r8]
2101         mov     r10d,DWORD[240+rcx]
2102         movups  xmm0,XMMWORD[r8]
2103         movups  xmm1,XMMWORD[16+r8]
2104         lea     r8,[32+r8]
2105         xorps   xmm2,xmm0
2106 $L$oop_enc1_11:
2107 DB      102,15,56,220,209
2108         dec     eax
2109         movups  xmm1,XMMWORD[r8]
2110         lea     r8,[16+r8]
2111         jnz     NEAR $L$oop_enc1_11
2112 DB      102,15,56,221,209
2113         xor     eax,eax
2114         test    rdx,15
2115         setnz   al
2116         shl     rax,4
2117         sub     rdx,rax
2118
2119         movups  xmm0,XMMWORD[rcx]
2120         mov     r11,rcx
2121         mov     eax,r10d
2122         shl     r10d,4
2123         mov     r9,rdx
2124         and     rdx,-16
2125
2126         movups  xmm1,XMMWORD[16+r10*1+rcx]
2127
2128         movdqa  xmm8,XMMWORD[$L$xts_magic]
2129         movdqa  xmm15,xmm2
2130         pshufd  xmm9,xmm2,0x5f
2131         pxor    xmm1,xmm0
2132         movdqa  xmm14,xmm9
2133         paddd   xmm9,xmm9
2134         movdqa  xmm10,xmm15
2135         psrad   xmm14,31
2136         paddq   xmm15,xmm15
2137         pand    xmm14,xmm8
2138         pxor    xmm10,xmm0
2139         pxor    xmm15,xmm14
2140         movdqa  xmm14,xmm9
2141         paddd   xmm9,xmm9
2142         movdqa  xmm11,xmm15
2143         psrad   xmm14,31
2144         paddq   xmm15,xmm15
2145         pand    xmm14,xmm8
2146         pxor    xmm11,xmm0
2147         pxor    xmm15,xmm14
2148         movdqa  xmm14,xmm9
2149         paddd   xmm9,xmm9
2150         movdqa  xmm12,xmm15
2151         psrad   xmm14,31
2152         paddq   xmm15,xmm15
2153         pand    xmm14,xmm8
2154         pxor    xmm12,xmm0
2155         pxor    xmm15,xmm14
2156         movdqa  xmm14,xmm9
2157         paddd   xmm9,xmm9
2158         movdqa  xmm13,xmm15
2159         psrad   xmm14,31
2160         paddq   xmm15,xmm15
2161         pand    xmm14,xmm8
2162         pxor    xmm13,xmm0
2163         pxor    xmm15,xmm14
2164         movdqa  xmm14,xmm15
2165         psrad   xmm9,31
2166         paddq   xmm15,xmm15
2167         pand    xmm9,xmm8
2168         pxor    xmm14,xmm0
2169         pxor    xmm15,xmm9
2170         movaps  XMMWORD[96+rsp],xmm1
2171
2172         sub     rdx,16*6
2173         jc      NEAR $L$xts_dec_short
2174
2175         mov     eax,16+96
2176         lea     rcx,[32+r10*1+r11]
2177         sub     rax,r10
2178         movups  xmm1,XMMWORD[16+r11]
2179         mov     r10,rax
2180         lea     r8,[$L$xts_magic]
2181         jmp     NEAR $L$xts_dec_grandloop
2182
2183 ALIGN   32
2184 $L$xts_dec_grandloop:
2185         movdqu  xmm2,XMMWORD[rdi]
2186         movdqa  xmm8,xmm0
2187         movdqu  xmm3,XMMWORD[16+rdi]
2188         pxor    xmm2,xmm10
2189         movdqu  xmm4,XMMWORD[32+rdi]
2190         pxor    xmm3,xmm11
2191 DB      102,15,56,222,209
2192         movdqu  xmm5,XMMWORD[48+rdi]
2193         pxor    xmm4,xmm12
2194 DB      102,15,56,222,217
2195         movdqu  xmm6,XMMWORD[64+rdi]
2196         pxor    xmm5,xmm13
2197 DB      102,15,56,222,225
2198         movdqu  xmm7,XMMWORD[80+rdi]
2199         pxor    xmm8,xmm15
2200         movdqa  xmm9,XMMWORD[96+rsp]
2201         pxor    xmm6,xmm14
2202 DB      102,15,56,222,233
2203         movups  xmm0,XMMWORD[32+r11]
2204         lea     rdi,[96+rdi]
2205         pxor    xmm7,xmm8
2206
2207         pxor    xmm10,xmm9
2208 DB      102,15,56,222,241
2209         pxor    xmm11,xmm9
2210         movdqa  XMMWORD[rsp],xmm10
2211 DB      102,15,56,222,249
2212         movups  xmm1,XMMWORD[48+r11]
2213         pxor    xmm12,xmm9
2214
2215 DB      102,15,56,222,208
2216         pxor    xmm13,xmm9
2217         movdqa  XMMWORD[16+rsp],xmm11
2218 DB      102,15,56,222,216
2219         pxor    xmm14,xmm9
2220         movdqa  XMMWORD[32+rsp],xmm12
2221 DB      102,15,56,222,224
2222 DB      102,15,56,222,232
2223         pxor    xmm8,xmm9
2224         movdqa  XMMWORD[64+rsp],xmm14
2225 DB      102,15,56,222,240
2226 DB      102,15,56,222,248
2227         movups  xmm0,XMMWORD[64+r11]
2228         movdqa  XMMWORD[80+rsp],xmm8
2229         pshufd  xmm9,xmm15,0x5f
2230         jmp     NEAR $L$xts_dec_loop6
2231 ALIGN   32
2232 $L$xts_dec_loop6:
2233 DB      102,15,56,222,209
2234 DB      102,15,56,222,217
2235 DB      102,15,56,222,225
2236 DB      102,15,56,222,233
2237 DB      102,15,56,222,241
2238 DB      102,15,56,222,249
2239         movups  xmm1,XMMWORD[((-64))+rax*1+rcx]
2240         add     rax,32
2241
2242 DB      102,15,56,222,208
2243 DB      102,15,56,222,216
2244 DB      102,15,56,222,224
2245 DB      102,15,56,222,232
2246 DB      102,15,56,222,240
2247 DB      102,15,56,222,248
2248         movups  xmm0,XMMWORD[((-80))+rax*1+rcx]
2249         jnz     NEAR $L$xts_dec_loop6
2250
2251         movdqa  xmm8,XMMWORD[r8]
2252         movdqa  xmm14,xmm9
2253         paddd   xmm9,xmm9
2254 DB      102,15,56,222,209
2255         paddq   xmm15,xmm15
2256         psrad   xmm14,31
2257 DB      102,15,56,222,217
2258         pand    xmm14,xmm8
2259         movups  xmm10,XMMWORD[r11]
2260 DB      102,15,56,222,225
2261 DB      102,15,56,222,233
2262 DB      102,15,56,222,241
2263         pxor    xmm15,xmm14
2264         movaps  xmm11,xmm10
2265 DB      102,15,56,222,249
2266         movups  xmm1,XMMWORD[((-64))+rcx]
2267
2268         movdqa  xmm14,xmm9
2269 DB      102,15,56,222,208
2270         paddd   xmm9,xmm9
2271         pxor    xmm10,xmm15
2272 DB      102,15,56,222,216
2273         psrad   xmm14,31
2274         paddq   xmm15,xmm15
2275 DB      102,15,56,222,224
2276 DB      102,15,56,222,232
2277         pand    xmm14,xmm8
2278         movaps  xmm12,xmm11
2279 DB      102,15,56,222,240
2280         pxor    xmm15,xmm14
2281         movdqa  xmm14,xmm9
2282 DB      102,15,56,222,248
2283         movups  xmm0,XMMWORD[((-48))+rcx]
2284
2285         paddd   xmm9,xmm9
2286 DB      102,15,56,222,209
2287         pxor    xmm11,xmm15
2288         psrad   xmm14,31
2289 DB      102,15,56,222,217
2290         paddq   xmm15,xmm15
2291         pand    xmm14,xmm8
2292 DB      102,15,56,222,225
2293 DB      102,15,56,222,233
2294         movdqa  XMMWORD[48+rsp],xmm13
2295         pxor    xmm15,xmm14
2296 DB      102,15,56,222,241
2297         movaps  xmm13,xmm12
2298         movdqa  xmm14,xmm9
2299 DB      102,15,56,222,249
2300         movups  xmm1,XMMWORD[((-32))+rcx]
2301
2302         paddd   xmm9,xmm9
2303 DB      102,15,56,222,208
2304         pxor    xmm12,xmm15
2305         psrad   xmm14,31
2306 DB      102,15,56,222,216
2307         paddq   xmm15,xmm15
2308         pand    xmm14,xmm8
2309 DB      102,15,56,222,224
2310 DB      102,15,56,222,232
2311 DB      102,15,56,222,240
2312         pxor    xmm15,xmm14
2313         movaps  xmm14,xmm13
2314 DB      102,15,56,222,248
2315
2316         movdqa  xmm0,xmm9
2317         paddd   xmm9,xmm9
2318 DB      102,15,56,222,209
2319         pxor    xmm13,xmm15
2320         psrad   xmm0,31
2321 DB      102,15,56,222,217
2322         paddq   xmm15,xmm15
2323         pand    xmm0,xmm8
2324 DB      102,15,56,222,225
2325 DB      102,15,56,222,233
2326         pxor    xmm15,xmm0
2327         movups  xmm0,XMMWORD[r11]
2328 DB      102,15,56,222,241
2329 DB      102,15,56,222,249
2330         movups  xmm1,XMMWORD[16+r11]
2331
2332         pxor    xmm14,xmm15
2333 DB      102,15,56,223,84,36,0
2334         psrad   xmm9,31
2335         paddq   xmm15,xmm15
2336 DB      102,15,56,223,92,36,16
2337 DB      102,15,56,223,100,36,32
2338         pand    xmm9,xmm8
2339         mov     rax,r10
2340 DB      102,15,56,223,108,36,48
2341 DB      102,15,56,223,116,36,64
2342 DB      102,15,56,223,124,36,80
2343         pxor    xmm15,xmm9
2344
2345         lea     rsi,[96+rsi]
2346         movups  XMMWORD[(-96)+rsi],xmm2
2347         movups  XMMWORD[(-80)+rsi],xmm3
2348         movups  XMMWORD[(-64)+rsi],xmm4
2349         movups  XMMWORD[(-48)+rsi],xmm5
2350         movups  XMMWORD[(-32)+rsi],xmm6
2351         movups  XMMWORD[(-16)+rsi],xmm7
2352         sub     rdx,16*6
2353         jnc     NEAR $L$xts_dec_grandloop
2354
2355         mov     eax,16+96
2356         sub     eax,r10d
2357         mov     rcx,r11
2358         shr     eax,4
2359
2360 $L$xts_dec_short:
2361         mov     r10d,eax
2362         pxor    xmm10,xmm0
2363         pxor    xmm11,xmm0
2364         add     rdx,16*6
2365         jz      NEAR $L$xts_dec_done
2366
2367         pxor    xmm12,xmm0
2368         cmp     rdx,0x20
2369         jb      NEAR $L$xts_dec_one
2370         pxor    xmm13,xmm0
2371         je      NEAR $L$xts_dec_two
2372
2373         pxor    xmm14,xmm0
2374         cmp     rdx,0x40
2375         jb      NEAR $L$xts_dec_three
2376         je      NEAR $L$xts_dec_four
2377
2378         movdqu  xmm2,XMMWORD[rdi]
2379         movdqu  xmm3,XMMWORD[16+rdi]
2380         movdqu  xmm4,XMMWORD[32+rdi]
2381         pxor    xmm2,xmm10
2382         movdqu  xmm5,XMMWORD[48+rdi]
2383         pxor    xmm3,xmm11
2384         movdqu  xmm6,XMMWORD[64+rdi]
2385         lea     rdi,[80+rdi]
2386         pxor    xmm4,xmm12
2387         pxor    xmm5,xmm13
2388         pxor    xmm6,xmm14
2389
2390         call    _aesni_decrypt6
2391
2392         xorps   xmm2,xmm10
2393         xorps   xmm3,xmm11
2394         xorps   xmm4,xmm12
2395         movdqu  XMMWORD[rsi],xmm2
2396         xorps   xmm5,xmm13
2397         movdqu  XMMWORD[16+rsi],xmm3
2398         xorps   xmm6,xmm14
2399         movdqu  XMMWORD[32+rsi],xmm4
2400         pxor    xmm14,xmm14
2401         movdqu  XMMWORD[48+rsi],xmm5
2402         pcmpgtd xmm14,xmm15
2403         movdqu  XMMWORD[64+rsi],xmm6
2404         lea     rsi,[80+rsi]
2405         pshufd  xmm11,xmm14,0x13
2406         and     r9,15
2407         jz      NEAR $L$xts_dec_ret
2408
2409         movdqa  xmm10,xmm15
2410         paddq   xmm15,xmm15
2411         pand    xmm11,xmm8
2412         pxor    xmm11,xmm15
2413         jmp     NEAR $L$xts_dec_done2
2414
2415 ALIGN   16
2416 $L$xts_dec_one:
2417         movups  xmm2,XMMWORD[rdi]
2418         lea     rdi,[16+rdi]
2419         xorps   xmm2,xmm10
2420         movups  xmm0,XMMWORD[rcx]
2421         movups  xmm1,XMMWORD[16+rcx]
2422         lea     rcx,[32+rcx]
2423         xorps   xmm2,xmm0
2424 $L$oop_dec1_12:
2425 DB      102,15,56,222,209
2426         dec     eax
2427         movups  xmm1,XMMWORD[rcx]
2428         lea     rcx,[16+rcx]
2429         jnz     NEAR $L$oop_dec1_12
2430 DB      102,15,56,223,209
2431         xorps   xmm2,xmm10
2432         movdqa  xmm10,xmm11
2433         movups  XMMWORD[rsi],xmm2
2434         movdqa  xmm11,xmm12
2435         lea     rsi,[16+rsi]
2436         jmp     NEAR $L$xts_dec_done
2437
2438 ALIGN   16
2439 $L$xts_dec_two:
2440         movups  xmm2,XMMWORD[rdi]
2441         movups  xmm3,XMMWORD[16+rdi]
2442         lea     rdi,[32+rdi]
2443         xorps   xmm2,xmm10
2444         xorps   xmm3,xmm11
2445
2446         call    _aesni_decrypt2
2447
2448         xorps   xmm2,xmm10
2449         movdqa  xmm10,xmm12
2450         xorps   xmm3,xmm11
2451         movdqa  xmm11,xmm13
2452         movups  XMMWORD[rsi],xmm2
2453         movups  XMMWORD[16+rsi],xmm3
2454         lea     rsi,[32+rsi]
2455         jmp     NEAR $L$xts_dec_done
2456
2457 ALIGN   16
2458 $L$xts_dec_three:
2459         movups  xmm2,XMMWORD[rdi]
2460         movups  xmm3,XMMWORD[16+rdi]
2461         movups  xmm4,XMMWORD[32+rdi]
2462         lea     rdi,[48+rdi]
2463         xorps   xmm2,xmm10
2464         xorps   xmm3,xmm11
2465         xorps   xmm4,xmm12
2466
2467         call    _aesni_decrypt3
2468
2469         xorps   xmm2,xmm10
2470         movdqa  xmm10,xmm13
2471         xorps   xmm3,xmm11
2472         movdqa  xmm11,xmm14
2473         xorps   xmm4,xmm12
2474         movups  XMMWORD[rsi],xmm2
2475         movups  XMMWORD[16+rsi],xmm3
2476         movups  XMMWORD[32+rsi],xmm4
2477         lea     rsi,[48+rsi]
2478         jmp     NEAR $L$xts_dec_done
2479
2480 ALIGN   16
2481 $L$xts_dec_four:
2482         movups  xmm2,XMMWORD[rdi]
2483         movups  xmm3,XMMWORD[16+rdi]
2484         movups  xmm4,XMMWORD[32+rdi]
2485         xorps   xmm2,xmm10
2486         movups  xmm5,XMMWORD[48+rdi]
2487         lea     rdi,[64+rdi]
2488         xorps   xmm3,xmm11
2489         xorps   xmm4,xmm12
2490         xorps   xmm5,xmm13
2491
2492         call    _aesni_decrypt4
2493
2494         pxor    xmm2,xmm10
2495         movdqa  xmm10,xmm14
2496         pxor    xmm3,xmm11
2497         movdqa  xmm11,xmm15
2498         pxor    xmm4,xmm12
2499         movdqu  XMMWORD[rsi],xmm2
2500         pxor    xmm5,xmm13
2501         movdqu  XMMWORD[16+rsi],xmm3
2502         movdqu  XMMWORD[32+rsi],xmm4
2503         movdqu  XMMWORD[48+rsi],xmm5
2504         lea     rsi,[64+rsi]
2505         jmp     NEAR $L$xts_dec_done
2506
2507 ALIGN   16
2508 $L$xts_dec_done:
2509         and     r9,15
2510         jz      NEAR $L$xts_dec_ret
2511 $L$xts_dec_done2:
2512         mov     rdx,r9
2513         mov     rcx,r11
2514         mov     eax,r10d
2515
2516         movups  xmm2,XMMWORD[rdi]
2517         xorps   xmm2,xmm11
2518         movups  xmm0,XMMWORD[rcx]
2519         movups  xmm1,XMMWORD[16+rcx]
2520         lea     rcx,[32+rcx]
2521         xorps   xmm2,xmm0
2522 $L$oop_dec1_13:
2523 DB      102,15,56,222,209
2524         dec     eax
2525         movups  xmm1,XMMWORD[rcx]
2526         lea     rcx,[16+rcx]
2527         jnz     NEAR $L$oop_dec1_13
2528 DB      102,15,56,223,209
2529         xorps   xmm2,xmm11
2530         movups  XMMWORD[rsi],xmm2
2531
2532 $L$xts_dec_steal:
2533         movzx   eax,BYTE[16+rdi]
2534         movzx   ecx,BYTE[rsi]
2535         lea     rdi,[1+rdi]
2536         mov     BYTE[rsi],al
2537         mov     BYTE[16+rsi],cl
2538         lea     rsi,[1+rsi]
2539         sub     rdx,1
2540         jnz     NEAR $L$xts_dec_steal
2541
2542         sub     rsi,r9
2543         mov     rcx,r11
2544         mov     eax,r10d
2545
2546         movups  xmm2,XMMWORD[rsi]
2547         xorps   xmm2,xmm10
2548         movups  xmm0,XMMWORD[rcx]
2549         movups  xmm1,XMMWORD[16+rcx]
2550         lea     rcx,[32+rcx]
2551         xorps   xmm2,xmm0
2552 $L$oop_dec1_14:
2553 DB      102,15,56,222,209
2554         dec     eax
2555         movups  xmm1,XMMWORD[rcx]
2556         lea     rcx,[16+rcx]
2557         jnz     NEAR $L$oop_dec1_14
2558 DB      102,15,56,223,209
2559         xorps   xmm2,xmm10
2560         movups  XMMWORD[rsi],xmm2
2561
2562 $L$xts_dec_ret:
2563         movaps  xmm6,XMMWORD[((-160))+rbp]
2564         movaps  xmm7,XMMWORD[((-144))+rbp]
2565         movaps  xmm8,XMMWORD[((-128))+rbp]
2566         movaps  xmm9,XMMWORD[((-112))+rbp]
2567         movaps  xmm10,XMMWORD[((-96))+rbp]
2568         movaps  xmm11,XMMWORD[((-80))+rbp]
2569         movaps  xmm12,XMMWORD[((-64))+rbp]
2570         movaps  xmm13,XMMWORD[((-48))+rbp]
2571         movaps  xmm14,XMMWORD[((-32))+rbp]
2572         movaps  xmm15,XMMWORD[((-16))+rbp]
2573         lea     rsp,[rbp]
2574         pop     rbp
2575 $L$xts_dec_epilogue:
2576         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
2577         mov     rsi,QWORD[16+rsp]
2578         DB      0F3h,0C3h               ;repret
2579 $L$SEH_end_aesni_xts_decrypt:
2580 global  aesni_cbc_encrypt
2581
2582 ALIGN   16
2583 aesni_cbc_encrypt:
2584         mov     QWORD[8+rsp],rdi        ;WIN64 prologue
2585         mov     QWORD[16+rsp],rsi
2586         mov     rax,rsp
2587 $L$SEH_begin_aesni_cbc_encrypt:
2588         mov     rdi,rcx
2589         mov     rsi,rdx
2590         mov     rdx,r8
2591         mov     rcx,r9
2592         mov     r8,QWORD[40+rsp]
2593         mov     r9,QWORD[48+rsp]
2594
2595
2596         test    rdx,rdx
2597         jz      NEAR $L$cbc_ret
2598
2599         mov     r10d,DWORD[240+rcx]
2600         mov     r11,rcx
2601         test    r9d,r9d
2602         jz      NEAR $L$cbc_decrypt
2603
2604         movups  xmm2,XMMWORD[r8]
2605         mov     eax,r10d
2606         cmp     rdx,16
2607         jb      NEAR $L$cbc_enc_tail
2608         sub     rdx,16
2609         jmp     NEAR $L$cbc_enc_loop
2610 ALIGN   16
2611 $L$cbc_enc_loop:
2612         movups  xmm3,XMMWORD[rdi]
2613         lea     rdi,[16+rdi]
2614
2615         movups  xmm0,XMMWORD[rcx]
2616         movups  xmm1,XMMWORD[16+rcx]
2617         xorps   xmm3,xmm0
2618         lea     rcx,[32+rcx]
2619         xorps   xmm2,xmm3
2620 $L$oop_enc1_15:
2621 DB      102,15,56,220,209
2622         dec     eax
2623         movups  xmm1,XMMWORD[rcx]
2624         lea     rcx,[16+rcx]
2625         jnz     NEAR $L$oop_enc1_15
2626 DB      102,15,56,221,209
2627         mov     eax,r10d
2628         mov     rcx,r11
2629         movups  XMMWORD[rsi],xmm2
2630         lea     rsi,[16+rsi]
2631         sub     rdx,16
2632         jnc     NEAR $L$cbc_enc_loop
2633         add     rdx,16
2634         jnz     NEAR $L$cbc_enc_tail
2635         movups  XMMWORD[r8],xmm2
2636         jmp     NEAR $L$cbc_ret
2637
2638 $L$cbc_enc_tail:
2639         mov     rcx,rdx
2640         xchg    rsi,rdi
2641         DD      0x9066A4F3
2642         mov     ecx,16
2643         sub     rcx,rdx
2644         xor     eax,eax
2645         DD      0x9066AAF3
2646         lea     rdi,[((-16))+rdi]
2647         mov     eax,r10d
2648         mov     rsi,rdi
2649         mov     rcx,r11
2650         xor     rdx,rdx
2651         jmp     NEAR $L$cbc_enc_loop
2652
2653 ALIGN   16
2654 $L$cbc_decrypt:
2655         lea     rax,[rsp]
2656         push    rbp
2657         sub     rsp,176
2658         and     rsp,-16
2659         movaps  XMMWORD[16+rsp],xmm6
2660         movaps  XMMWORD[32+rsp],xmm7
2661         movaps  XMMWORD[48+rsp],xmm8
2662         movaps  XMMWORD[64+rsp],xmm9
2663         movaps  XMMWORD[80+rsp],xmm10
2664         movaps  XMMWORD[96+rsp],xmm11
2665         movaps  XMMWORD[112+rsp],xmm12
2666         movaps  XMMWORD[128+rsp],xmm13
2667         movaps  XMMWORD[144+rsp],xmm14
2668         movaps  XMMWORD[160+rsp],xmm15
2669 $L$cbc_decrypt_body:
2670         lea     rbp,[((-8))+rax]
2671         movups  xmm10,XMMWORD[r8]
2672         mov     eax,r10d
2673         cmp     rdx,0x50
2674         jbe     NEAR $L$cbc_dec_tail
2675
2676         movups  xmm0,XMMWORD[rcx]
2677         movdqu  xmm2,XMMWORD[rdi]
2678         movdqu  xmm3,XMMWORD[16+rdi]
2679         movdqa  xmm11,xmm2
2680         movdqu  xmm4,XMMWORD[32+rdi]
2681         movdqa  xmm12,xmm3
2682         movdqu  xmm5,XMMWORD[48+rdi]
2683         movdqa  xmm13,xmm4
2684         movdqu  xmm6,XMMWORD[64+rdi]
2685         movdqa  xmm14,xmm5
2686         movdqu  xmm7,XMMWORD[80+rdi]
2687         movdqa  xmm15,xmm6
2688         mov     r9d,DWORD[((OPENSSL_ia32cap_P+4))]
2689         cmp     rdx,0x70
2690         jbe     NEAR $L$cbc_dec_six_or_seven
2691
2692         and     r9d,71303168
2693         sub     rdx,0x50
2694         cmp     r9d,4194304
2695         je      NEAR $L$cbc_dec_loop6_enter
2696         sub     rdx,0x20
2697         lea     rcx,[112+rcx]
2698         jmp     NEAR $L$cbc_dec_loop8_enter
2699 ALIGN   16
2700 $L$cbc_dec_loop8:
2701         movups  XMMWORD[rsi],xmm9
2702         lea     rsi,[16+rsi]
2703 $L$cbc_dec_loop8_enter:
2704         movdqu  xmm8,XMMWORD[96+rdi]
2705         pxor    xmm2,xmm0
2706         movdqu  xmm9,XMMWORD[112+rdi]
2707         pxor    xmm3,xmm0
2708         movups  xmm1,XMMWORD[((16-112))+rcx]
2709         pxor    xmm4,xmm0
2710         xor     r11,r11
2711         cmp     rdx,0x70
2712         pxor    xmm5,xmm0
2713         pxor    xmm6,xmm0
2714         pxor    xmm7,xmm0
2715         pxor    xmm8,xmm0
2716
2717 DB      102,15,56,222,209
2718         pxor    xmm9,xmm0
2719         movups  xmm0,XMMWORD[((32-112))+rcx]
2720 DB      102,15,56,222,217
2721 DB      102,15,56,222,225
2722 DB      102,15,56,222,233
2723 DB      102,15,56,222,241
2724 DB      102,15,56,222,249
2725 DB      102,68,15,56,222,193
2726         setnc   r11b
2727         shl     r11,7
2728 DB      102,68,15,56,222,201
2729         add     r11,rdi
2730         movups  xmm1,XMMWORD[((48-112))+rcx]
2731 DB      102,15,56,222,208
2732 DB      102,15,56,222,216
2733 DB      102,15,56,222,224
2734 DB      102,15,56,222,232
2735 DB      102,15,56,222,240
2736 DB      102,15,56,222,248
2737 DB      102,68,15,56,222,192
2738 DB      102,68,15,56,222,200
2739         movups  xmm0,XMMWORD[((64-112))+rcx]
2740         nop
2741 DB      102,15,56,222,209
2742 DB      102,15,56,222,217
2743 DB      102,15,56,222,225
2744 DB      102,15,56,222,233
2745 DB      102,15,56,222,241
2746 DB      102,15,56,222,249
2747 DB      102,68,15,56,222,193
2748 DB      102,68,15,56,222,201
2749         movups  xmm1,XMMWORD[((80-112))+rcx]
2750         nop
2751 DB      102,15,56,222,208
2752 DB      102,15,56,222,216
2753 DB      102,15,56,222,224
2754 DB      102,15,56,222,232
2755 DB      102,15,56,222,240
2756 DB      102,15,56,222,248
2757 DB      102,68,15,56,222,192
2758 DB      102,68,15,56,222,200
2759         movups  xmm0,XMMWORD[((96-112))+rcx]
2760         nop
2761 DB      102,15,56,222,209
2762 DB      102,15,56,222,217
2763 DB      102,15,56,222,225
2764 DB      102,15,56,222,233
2765 DB      102,15,56,222,241
2766 DB      102,15,56,222,249
2767 DB      102,68,15,56,222,193
2768 DB      102,68,15,56,222,201
2769         movups  xmm1,XMMWORD[((112-112))+rcx]
2770         nop
2771 DB      102,15,56,222,208
2772 DB      102,15,56,222,216
2773 DB      102,15,56,222,224
2774 DB      102,15,56,222,232
2775 DB      102,15,56,222,240
2776 DB      102,15,56,222,248
2777 DB      102,68,15,56,222,192
2778 DB      102,68,15,56,222,200
2779         movups  xmm0,XMMWORD[((128-112))+rcx]
2780         nop
2781 DB      102,15,56,222,209
2782 DB      102,15,56,222,217
2783 DB      102,15,56,222,225
2784 DB      102,15,56,222,233
2785 DB      102,15,56,222,241
2786 DB      102,15,56,222,249
2787 DB      102,68,15,56,222,193
2788 DB      102,68,15,56,222,201
2789         movups  xmm1,XMMWORD[((144-112))+rcx]
2790         cmp     eax,11
2791 DB      102,15,56,222,208
2792 DB      102,15,56,222,216
2793 DB      102,15,56,222,224
2794 DB      102,15,56,222,232
2795 DB      102,15,56,222,240
2796 DB      102,15,56,222,248
2797 DB      102,68,15,56,222,192
2798 DB      102,68,15,56,222,200
2799         movups  xmm0,XMMWORD[((160-112))+rcx]
2800         jb      NEAR $L$cbc_dec_done
2801 DB      102,15,56,222,209
2802 DB      102,15,56,222,217
2803 DB      102,15,56,222,225
2804 DB      102,15,56,222,233
2805 DB      102,15,56,222,241
2806 DB      102,15,56,222,249
2807 DB      102,68,15,56,222,193
2808 DB      102,68,15,56,222,201
2809         movups  xmm1,XMMWORD[((176-112))+rcx]
2810         nop
2811 DB      102,15,56,222,208
2812 DB      102,15,56,222,216
2813 DB      102,15,56,222,224
2814 DB      102,15,56,222,232
2815 DB      102,15,56,222,240
2816 DB      102,15,56,222,248
2817 DB      102,68,15,56,222,192
2818 DB      102,68,15,56,222,200
2819         movups  xmm0,XMMWORD[((192-112))+rcx]
2820         je      NEAR $L$cbc_dec_done
2821 DB      102,15,56,222,209
2822 DB      102,15,56,222,217
2823 DB      102,15,56,222,225
2824 DB      102,15,56,222,233
2825 DB      102,15,56,222,241
2826 DB      102,15,56,222,249
2827 DB      102,68,15,56,222,193
2828 DB      102,68,15,56,222,201
2829         movups  xmm1,XMMWORD[((208-112))+rcx]
2830         nop
2831 DB      102,15,56,222,208
2832 DB      102,15,56,222,216
2833 DB      102,15,56,222,224
2834 DB      102,15,56,222,232
2835 DB      102,15,56,222,240
2836 DB      102,15,56,222,248
2837 DB      102,68,15,56,222,192
2838 DB      102,68,15,56,222,200
2839         movups  xmm0,XMMWORD[((224-112))+rcx]
2840         jmp     NEAR $L$cbc_dec_done
2841 ALIGN   16
2842 $L$cbc_dec_done:
2843 DB      102,15,56,222,209
2844 DB      102,15,56,222,217
2845         pxor    xmm10,xmm0
2846         pxor    xmm11,xmm0
2847 DB      102,15,56,222,225
2848 DB      102,15,56,222,233
2849         pxor    xmm12,xmm0
2850         pxor    xmm13,xmm0
2851 DB      102,15,56,222,241
2852 DB      102,15,56,222,249
2853         pxor    xmm14,xmm0
2854         pxor    xmm15,xmm0
2855 DB      102,68,15,56,222,193
2856 DB      102,68,15,56,222,201
2857         movdqu  xmm1,XMMWORD[80+rdi]
2858
2859 DB      102,65,15,56,223,210
2860         movdqu  xmm10,XMMWORD[96+rdi]
2861         pxor    xmm1,xmm0
2862 DB      102,65,15,56,223,219
2863         pxor    xmm10,xmm0
2864         movdqu  xmm0,XMMWORD[112+rdi]
2865 DB      102,65,15,56,223,228
2866         lea     rdi,[128+rdi]
2867         movdqu  xmm11,XMMWORD[r11]
2868 DB      102,65,15,56,223,237
2869 DB      102,65,15,56,223,246
2870         movdqu  xmm12,XMMWORD[16+r11]
2871         movdqu  xmm13,XMMWORD[32+r11]
2872 DB      102,65,15,56,223,255
2873 DB      102,68,15,56,223,193
2874         movdqu  xmm14,XMMWORD[48+r11]
2875         movdqu  xmm15,XMMWORD[64+r11]
2876 DB      102,69,15,56,223,202
2877         movdqa  xmm10,xmm0
2878         movdqu  xmm1,XMMWORD[80+r11]
2879         movups  xmm0,XMMWORD[((-112))+rcx]
2880
2881         movups  XMMWORD[rsi],xmm2
2882         movdqa  xmm2,xmm11
2883         movups  XMMWORD[16+rsi],xmm3
2884         movdqa  xmm3,xmm12
2885         movups  XMMWORD[32+rsi],xmm4
2886         movdqa  xmm4,xmm13
2887         movups  XMMWORD[48+rsi],xmm5
2888         movdqa  xmm5,xmm14
2889         movups  XMMWORD[64+rsi],xmm6
2890         movdqa  xmm6,xmm15
2891         movups  XMMWORD[80+rsi],xmm7
2892         movdqa  xmm7,xmm1
2893         movups  XMMWORD[96+rsi],xmm8
2894         lea     rsi,[112+rsi]
2895
2896         sub     rdx,0x80
2897         ja      NEAR $L$cbc_dec_loop8
2898
2899         movaps  xmm2,xmm9
2900         lea     rcx,[((-112))+rcx]
2901         add     rdx,0x70
2902         jle     NEAR $L$cbc_dec_tail_collected
2903         movups  XMMWORD[rsi],xmm9
2904         lea     rsi,[16+rsi]
2905         cmp     rdx,0x50
2906         jbe     NEAR $L$cbc_dec_tail
2907
2908         movaps  xmm2,xmm11
2909 $L$cbc_dec_six_or_seven:
2910         cmp     rdx,0x60
2911         ja      NEAR $L$cbc_dec_seven
2912
2913         movaps  xmm8,xmm7
2914         call    _aesni_decrypt6
2915         pxor    xmm2,xmm10
2916         movaps  xmm10,xmm8
2917         pxor    xmm3,xmm11
2918         movdqu  XMMWORD[rsi],xmm2
2919         pxor    xmm4,xmm12
2920         movdqu  XMMWORD[16+rsi],xmm3
2921         pxor    xmm5,xmm13
2922         movdqu  XMMWORD[32+rsi],xmm4
2923         pxor    xmm6,xmm14
2924         movdqu  XMMWORD[48+rsi],xmm5
2925         pxor    xmm7,xmm15
2926         movdqu  XMMWORD[64+rsi],xmm6
2927         lea     rsi,[80+rsi]
2928         movdqa  xmm2,xmm7
2929         jmp     NEAR $L$cbc_dec_tail_collected
2930
2931 ALIGN   16
2932 $L$cbc_dec_seven:
2933         movups  xmm8,XMMWORD[96+rdi]
2934         xorps   xmm9,xmm9
2935         call    _aesni_decrypt8
2936         movups  xmm9,XMMWORD[80+rdi]
2937         pxor    xmm2,xmm10
2938         movups  xmm10,XMMWORD[96+rdi]
2939         pxor    xmm3,xmm11
2940         movdqu  XMMWORD[rsi],xmm2
2941         pxor    xmm4,xmm12
2942         movdqu  XMMWORD[16+rsi],xmm3
2943         pxor    xmm5,xmm13
2944         movdqu  XMMWORD[32+rsi],xmm4
2945         pxor    xmm6,xmm14
2946         movdqu  XMMWORD[48+rsi],xmm5
2947         pxor    xmm7,xmm15
2948         movdqu  XMMWORD[64+rsi],xmm6
2949         pxor    xmm8,xmm9
2950         movdqu  XMMWORD[80+rsi],xmm7
2951         lea     rsi,[96+rsi]
2952         movdqa  xmm2,xmm8
2953         jmp     NEAR $L$cbc_dec_tail_collected
2954
2955 ALIGN   16
2956 $L$cbc_dec_loop6:
2957         movups  XMMWORD[rsi],xmm7
2958         lea     rsi,[16+rsi]
2959         movdqu  xmm2,XMMWORD[rdi]
2960         movdqu  xmm3,XMMWORD[16+rdi]
2961         movdqa  xmm11,xmm2
2962         movdqu  xmm4,XMMWORD[32+rdi]
2963         movdqa  xmm12,xmm3
2964         movdqu  xmm5,XMMWORD[48+rdi]
2965         movdqa  xmm13,xmm4
2966         movdqu  xmm6,XMMWORD[64+rdi]
2967         movdqa  xmm14,xmm5
2968         movdqu  xmm7,XMMWORD[80+rdi]
2969         movdqa  xmm15,xmm6
2970 $L$cbc_dec_loop6_enter:
2971         lea     rdi,[96+rdi]
2972         movdqa  xmm8,xmm7
2973
2974         call    _aesni_decrypt6
2975
2976         pxor    xmm2,xmm10
2977         movdqa  xmm10,xmm8
2978         pxor    xmm3,xmm11
2979         movdqu  XMMWORD[rsi],xmm2
2980         pxor    xmm4,xmm12
2981         movdqu  XMMWORD[16+rsi],xmm3
2982         pxor    xmm5,xmm13
2983         movdqu  XMMWORD[32+rsi],xmm4
2984         pxor    xmm6,xmm14
2985         mov     rcx,r11
2986         movdqu  XMMWORD[48+rsi],xmm5
2987         pxor    xmm7,xmm15
2988         mov     eax,r10d
2989         movdqu  XMMWORD[64+rsi],xmm6
2990         lea     rsi,[80+rsi]
2991         sub     rdx,0x60
2992         ja      NEAR $L$cbc_dec_loop6
2993
2994         movdqa  xmm2,xmm7
2995         add     rdx,0x50
2996         jle     NEAR $L$cbc_dec_tail_collected
2997         movups  XMMWORD[rsi],xmm7
2998         lea     rsi,[16+rsi]
2999
3000 $L$cbc_dec_tail:
3001         movups  xmm2,XMMWORD[rdi]
3002         sub     rdx,0x10
3003         jbe     NEAR $L$cbc_dec_one
3004
3005         movups  xmm3,XMMWORD[16+rdi]
3006         movaps  xmm11,xmm2
3007         sub     rdx,0x10
3008         jbe     NEAR $L$cbc_dec_two
3009
3010         movups  xmm4,XMMWORD[32+rdi]
3011         movaps  xmm12,xmm3
3012         sub     rdx,0x10
3013         jbe     NEAR $L$cbc_dec_three
3014
3015         movups  xmm5,XMMWORD[48+rdi]
3016         movaps  xmm13,xmm4
3017         sub     rdx,0x10
3018         jbe     NEAR $L$cbc_dec_four
3019
3020         movups  xmm6,XMMWORD[64+rdi]
3021         movaps  xmm14,xmm5
3022         movaps  xmm15,xmm6
3023         xorps   xmm7,xmm7
3024         call    _aesni_decrypt6
3025         pxor    xmm2,xmm10
3026         movaps  xmm10,xmm15
3027         pxor    xmm3,xmm11
3028         movdqu  XMMWORD[rsi],xmm2
3029         pxor    xmm4,xmm12
3030         movdqu  XMMWORD[16+rsi],xmm3
3031         pxor    xmm5,xmm13
3032         movdqu  XMMWORD[32+rsi],xmm4
3033         pxor    xmm6,xmm14
3034         movdqu  XMMWORD[48+rsi],xmm5
3035         lea     rsi,[64+rsi]
3036         movdqa  xmm2,xmm6
3037         sub     rdx,0x10
3038         jmp     NEAR $L$cbc_dec_tail_collected
3039
3040 ALIGN   16
3041 $L$cbc_dec_one:
3042         movaps  xmm11,xmm2
3043         movups  xmm0,XMMWORD[rcx]
3044         movups  xmm1,XMMWORD[16+rcx]
3045         lea     rcx,[32+rcx]
3046         xorps   xmm2,xmm0
3047 $L$oop_dec1_16:
3048 DB      102,15,56,222,209
3049         dec     eax
3050         movups  xmm1,XMMWORD[rcx]
3051         lea     rcx,[16+rcx]
3052         jnz     NEAR $L$oop_dec1_16
3053 DB      102,15,56,223,209
3054         xorps   xmm2,xmm10
3055         movaps  xmm10,xmm11
3056         jmp     NEAR $L$cbc_dec_tail_collected
3057 ALIGN   16
3058 $L$cbc_dec_two:
3059         movaps  xmm12,xmm3
3060         call    _aesni_decrypt2
3061         pxor    xmm2,xmm10
3062         movaps  xmm10,xmm12
3063         pxor    xmm3,xmm11
3064         movdqu  XMMWORD[rsi],xmm2
3065         movdqa  xmm2,xmm3
3066         lea     rsi,[16+rsi]
3067         jmp     NEAR $L$cbc_dec_tail_collected
3068 ALIGN   16
3069 $L$cbc_dec_three:
3070         movaps  xmm13,xmm4
3071         call    _aesni_decrypt3
3072         pxor    xmm2,xmm10
3073         movaps  xmm10,xmm13
3074         pxor    xmm3,xmm11
3075         movdqu  XMMWORD[rsi],xmm2
3076         pxor    xmm4,xmm12
3077         movdqu  XMMWORD[16+rsi],xmm3
3078         movdqa  xmm2,xmm4
3079         lea     rsi,[32+rsi]
3080         jmp     NEAR $L$cbc_dec_tail_collected
3081 ALIGN   16
3082 $L$cbc_dec_four:
3083         movaps  xmm14,xmm5
3084         call    _aesni_decrypt4
3085         pxor    xmm2,xmm10
3086         movaps  xmm10,xmm14
3087         pxor    xmm3,xmm11
3088         movdqu  XMMWORD[rsi],xmm2
3089         pxor    xmm4,xmm12
3090         movdqu  XMMWORD[16+rsi],xmm3
3091         pxor    xmm5,xmm13
3092         movdqu  XMMWORD[32+rsi],xmm4
3093         movdqa  xmm2,xmm5
3094         lea     rsi,[48+rsi]
3095         jmp     NEAR $L$cbc_dec_tail_collected
3096
3097 ALIGN   16
3098 $L$cbc_dec_tail_collected:
3099         movups  XMMWORD[r8],xmm10
3100         and     rdx,15
3101         jnz     NEAR $L$cbc_dec_tail_partial
3102         movups  XMMWORD[rsi],xmm2
3103         jmp     NEAR $L$cbc_dec_ret
3104 ALIGN   16
3105 $L$cbc_dec_tail_partial:
3106         movaps  XMMWORD[rsp],xmm2
3107         mov     rcx,16
3108         mov     rdi,rsi
3109         sub     rcx,rdx
3110         lea     rsi,[rsp]
3111         DD      0x9066A4F3
3112
3113 $L$cbc_dec_ret:
3114         movaps  xmm6,XMMWORD[16+rsp]
3115         movaps  xmm7,XMMWORD[32+rsp]
3116         movaps  xmm8,XMMWORD[48+rsp]
3117         movaps  xmm9,XMMWORD[64+rsp]
3118         movaps  xmm10,XMMWORD[80+rsp]
3119         movaps  xmm11,XMMWORD[96+rsp]
3120         movaps  xmm12,XMMWORD[112+rsp]
3121         movaps  xmm13,XMMWORD[128+rsp]
3122         movaps  xmm14,XMMWORD[144+rsp]
3123         movaps  xmm15,XMMWORD[160+rsp]
3124         lea     rsp,[rbp]
3125         pop     rbp
3126 $L$cbc_ret:
3127         mov     rdi,QWORD[8+rsp]        ;WIN64 epilogue
3128         mov     rsi,QWORD[16+rsp]
3129         DB      0F3h,0C3h               ;repret
3130 $L$SEH_end_aesni_cbc_encrypt:
3131 global  aesni_set_decrypt_key
3132
3133 ALIGN   16
3134 aesni_set_decrypt_key:
3135 DB      0x48,0x83,0xEC,0x08
3136         call    __aesni_set_encrypt_key
3137         shl     edx,4
3138         test    eax,eax
3139         jnz     NEAR $L$dec_key_ret
3140         lea     rcx,[16+rdx*1+r8]
3141
3142         movups  xmm0,XMMWORD[r8]
3143         movups  xmm1,XMMWORD[rcx]
3144         movups  XMMWORD[rcx],xmm0
3145         movups  XMMWORD[r8],xmm1
3146         lea     r8,[16+r8]
3147         lea     rcx,[((-16))+rcx]
3148
3149 $L$dec_key_inverse:
3150         movups  xmm0,XMMWORD[r8]
3151         movups  xmm1,XMMWORD[rcx]
3152 DB      102,15,56,219,192
3153 DB      102,15,56,219,201
3154         lea     r8,[16+r8]
3155         lea     rcx,[((-16))+rcx]
3156         movups  XMMWORD[16+rcx],xmm0
3157         movups  XMMWORD[(-16)+r8],xmm1
3158         cmp     rcx,r8
3159         ja      NEAR $L$dec_key_inverse
3160
3161         movups  xmm0,XMMWORD[r8]
3162 DB      102,15,56,219,192
3163         movups  XMMWORD[rcx],xmm0
3164 $L$dec_key_ret:
3165         add     rsp,8
3166         DB      0F3h,0C3h               ;repret
3167 $L$SEH_end_set_decrypt_key:
3168
3169 global  aesni_set_encrypt_key
3170
3171 ALIGN   16
3172 aesni_set_encrypt_key:
3173 __aesni_set_encrypt_key:
3174 DB      0x48,0x83,0xEC,0x08
3175         mov     rax,-1
3176         test    rcx,rcx
3177         jz      NEAR $L$enc_key_ret
3178         test    r8,r8
3179         jz      NEAR $L$enc_key_ret
3180
3181         movups  xmm0,XMMWORD[rcx]
3182         xorps   xmm4,xmm4
3183         lea     rax,[16+r8]
3184         cmp     edx,256
3185         je      NEAR $L$14rounds
3186         cmp     edx,192
3187         je      NEAR $L$12rounds
3188         cmp     edx,128
3189         jne     NEAR $L$bad_keybits
3190
3191 $L$10rounds:
3192         mov     edx,9
3193         movups  XMMWORD[r8],xmm0
3194 DB      102,15,58,223,200,1
3195         call    $L$key_expansion_128_cold
3196 DB      102,15,58,223,200,2
3197         call    $L$key_expansion_128
3198 DB      102,15,58,223,200,4
3199         call    $L$key_expansion_128
3200 DB      102,15,58,223,200,8
3201         call    $L$key_expansion_128
3202 DB      102,15,58,223,200,16
3203         call    $L$key_expansion_128
3204 DB      102,15,58,223,200,32
3205         call    $L$key_expansion_128
3206 DB      102,15,58,223,200,64
3207         call    $L$key_expansion_128
3208 DB      102,15,58,223,200,128
3209         call    $L$key_expansion_128
3210 DB      102,15,58,223,200,27
3211         call    $L$key_expansion_128
3212 DB      102,15,58,223,200,54
3213         call    $L$key_expansion_128
3214         movups  XMMWORD[rax],xmm0
3215         mov     DWORD[80+rax],edx
3216         xor     eax,eax
3217         jmp     NEAR $L$enc_key_ret
3218
3219 ALIGN   16
3220 $L$12rounds:
3221         movq    xmm2,QWORD[16+rcx]
3222         mov     edx,11
3223         movups  XMMWORD[r8],xmm0
3224 DB      102,15,58,223,202,1
3225         call    $L$key_expansion_192a_cold
3226 DB      102,15,58,223,202,2
3227         call    $L$key_expansion_192b
3228 DB      102,15,58,223,202,4
3229         call    $L$key_expansion_192a
3230 DB      102,15,58,223,202,8
3231         call    $L$key_expansion_192b
3232 DB      102,15,58,223,202,16
3233         call    $L$key_expansion_192a
3234 DB      102,15,58,223,202,32
3235         call    $L$key_expansion_192b
3236 DB      102,15,58,223,202,64
3237         call    $L$key_expansion_192a
3238 DB      102,15,58,223,202,128
3239         call    $L$key_expansion_192b
3240         movups  XMMWORD[rax],xmm0
3241         mov     DWORD[48+rax],edx
3242         xor     rax,rax
3243         jmp     NEAR $L$enc_key_ret
3244
3245 ALIGN   16
3246 $L$14rounds:
3247         movups  xmm2,XMMWORD[16+rcx]
3248         mov     edx,13
3249         lea     rax,[16+rax]
3250         movups  XMMWORD[r8],xmm0
3251         movups  XMMWORD[16+r8],xmm2
3252 DB      102,15,58,223,202,1
3253         call    $L$key_expansion_256a_cold
3254 DB      102,15,58,223,200,1
3255         call    $L$key_expansion_256b
3256 DB      102,15,58,223,202,2
3257         call    $L$key_expansion_256a
3258 DB      102,15,58,223,200,2
3259         call    $L$key_expansion_256b
3260 DB      102,15,58,223,202,4
3261         call    $L$key_expansion_256a
3262 DB      102,15,58,223,200,4
3263         call    $L$key_expansion_256b
3264 DB      102,15,58,223,202,8
3265         call    $L$key_expansion_256a
3266 DB      102,15,58,223,200,8
3267         call    $L$key_expansion_256b
3268 DB      102,15,58,223,202,16
3269         call    $L$key_expansion_256a
3270 DB      102,15,58,223,200,16
3271         call    $L$key_expansion_256b
3272 DB      102,15,58,223,202,32
3273         call    $L$key_expansion_256a
3274 DB      102,15,58,223,200,32
3275         call    $L$key_expansion_256b
3276 DB      102,15,58,223,202,64
3277         call    $L$key_expansion_256a
3278         movups  XMMWORD[rax],xmm0
3279         mov     DWORD[16+rax],edx
3280         xor     rax,rax
3281         jmp     NEAR $L$enc_key_ret
3282
3283 ALIGN   16
3284 $L$bad_keybits:
3285         mov     rax,-2
3286 $L$enc_key_ret:
3287         add     rsp,8
3288         DB      0F3h,0C3h               ;repret
3289 $L$SEH_end_set_encrypt_key:
3290
3291 ALIGN   16
3292 $L$key_expansion_128:
3293         movups  XMMWORD[rax],xmm0
3294         lea     rax,[16+rax]
3295 $L$key_expansion_128_cold:
3296         shufps  xmm4,xmm0,16
3297         xorps   xmm0,xmm4
3298         shufps  xmm4,xmm0,140
3299         xorps   xmm0,xmm4
3300         shufps  xmm1,xmm1,255
3301         xorps   xmm0,xmm1
3302         DB      0F3h,0C3h               ;repret
3303
3304 ALIGN   16
3305 $L$key_expansion_192a:
3306         movups  XMMWORD[rax],xmm0
3307         lea     rax,[16+rax]
3308 $L$key_expansion_192a_cold:
3309         movaps  xmm5,xmm2
3310 $L$key_expansion_192b_warm:
3311         shufps  xmm4,xmm0,16
3312         movdqa  xmm3,xmm2
3313         xorps   xmm0,xmm4
3314         shufps  xmm4,xmm0,140
3315         pslldq  xmm3,4
3316         xorps   xmm0,xmm4
3317         pshufd  xmm1,xmm1,85
3318         pxor    xmm2,xmm3
3319         pxor    xmm0,xmm1
3320         pshufd  xmm3,xmm0,255
3321         pxor    xmm2,xmm3
3322         DB      0F3h,0C3h               ;repret
3323
3324 ALIGN   16
3325 $L$key_expansion_192b:
3326         movaps  xmm3,xmm0
3327         shufps  xmm5,xmm0,68
3328         movups  XMMWORD[rax],xmm5
3329         shufps  xmm3,xmm2,78
3330         movups  XMMWORD[16+rax],xmm3
3331         lea     rax,[32+rax]
3332         jmp     NEAR $L$key_expansion_192b_warm
3333
3334 ALIGN   16
3335 $L$key_expansion_256a:
3336         movups  XMMWORD[rax],xmm2
3337         lea     rax,[16+rax]
3338 $L$key_expansion_256a_cold:
3339         shufps  xmm4,xmm0,16
3340         xorps   xmm0,xmm4
3341         shufps  xmm4,xmm0,140
3342         xorps   xmm0,xmm4
3343         shufps  xmm1,xmm1,255
3344         xorps   xmm0,xmm1
3345         DB      0F3h,0C3h               ;repret
3346
3347 ALIGN   16
3348 $L$key_expansion_256b:
3349         movups  XMMWORD[rax],xmm0
3350         lea     rax,[16+rax]
3351
3352         shufps  xmm4,xmm2,16
3353         xorps   xmm2,xmm4
3354         shufps  xmm4,xmm2,140
3355         xorps   xmm2,xmm4
3356         shufps  xmm1,xmm1,170
3357         xorps   xmm2,xmm1
3358         DB      0F3h,0C3h               ;repret
3359
3360
3361 ALIGN   64
3362 $L$bswap_mask:
3363 DB      15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
3364 $L$increment32:
3365         DD      6,6,6,0
3366 $L$increment64:
3367         DD      1,0,0,0
3368 $L$xts_magic:
3369         DD      0x87,0,1,0
3370 $L$increment1:
3371 DB      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
3372
3373 DB      65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69
3374 DB      83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83
3375 DB      32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
3376 DB      115,108,46,111,114,103,62,0
3377 ALIGN   64
3378 EXTERN  __imp_RtlVirtualUnwind
3379
3380 ALIGN   16
3381 ecb_se_handler:
3382         push    rsi
3383         push    rdi
3384         push    rbx
3385         push    rbp
3386         push    r12
3387         push    r13
3388         push    r14
3389         push    r15
3390         pushfq
3391         sub     rsp,64
3392
3393         mov     rax,QWORD[152+r8]
3394
3395         jmp     NEAR $L$common_seh_tail
3396
3397
3398
3399 ALIGN   16
3400 ccm64_se_handler:
3401         push    rsi
3402         push    rdi
3403         push    rbx
3404         push    rbp
3405         push    r12
3406         push    r13
3407         push    r14
3408         push    r15
3409         pushfq
3410         sub     rsp,64
3411
3412         mov     rax,QWORD[120+r8]
3413         mov     rbx,QWORD[248+r8]
3414
3415         mov     rsi,QWORD[8+r9]
3416         mov     r11,QWORD[56+r9]
3417
3418         mov     r10d,DWORD[r11]
3419         lea     r10,[r10*1+rsi]
3420         cmp     rbx,r10
3421         jb      NEAR $L$common_seh_tail
3422
3423         mov     rax,QWORD[152+r8]
3424
3425         mov     r10d,DWORD[4+r11]
3426         lea     r10,[r10*1+rsi]
3427         cmp     rbx,r10
3428         jae     NEAR $L$common_seh_tail
3429
3430         lea     rsi,[rax]
3431         lea     rdi,[512+r8]
3432         mov     ecx,8
3433         DD      0xa548f3fc
3434         lea     rax,[88+rax]
3435
3436         jmp     NEAR $L$common_seh_tail
3437
3438
3439
3440 ALIGN   16
3441 ctr_xts_se_handler:
3442         push    rsi
3443         push    rdi
3444         push    rbx
3445         push    rbp
3446         push    r12
3447         push    r13
3448         push    r14
3449         push    r15
3450         pushfq
3451         sub     rsp,64
3452
3453         mov     rax,QWORD[120+r8]
3454         mov     rbx,QWORD[248+r8]
3455
3456         mov     rsi,QWORD[8+r9]
3457         mov     r11,QWORD[56+r9]
3458
3459         mov     r10d,DWORD[r11]
3460         lea     r10,[r10*1+rsi]
3461         cmp     rbx,r10
3462         jb      NEAR $L$common_seh_tail
3463
3464         mov     rax,QWORD[152+r8]
3465
3466         mov     r10d,DWORD[4+r11]
3467         lea     r10,[r10*1+rsi]
3468         cmp     rbx,r10
3469         jae     NEAR $L$common_seh_tail
3470
3471         mov     rax,QWORD[160+r8]
3472         lea     rsi,[((-160))+rax]
3473         lea     rdi,[512+r8]
3474         mov     ecx,20
3475         DD      0xa548f3fc
3476
3477         jmp     NEAR $L$common_rbp_tail
3478
3479
3480 ALIGN   16
3481 cbc_se_handler:
3482         push    rsi
3483         push    rdi
3484         push    rbx
3485         push    rbp
3486         push    r12
3487         push    r13
3488         push    r14
3489         push    r15
3490         pushfq
3491         sub     rsp,64
3492
3493         mov     rax,QWORD[152+r8]
3494         mov     rbx,QWORD[248+r8]
3495
3496         lea     r10,[$L$cbc_decrypt]
3497         cmp     rbx,r10
3498         jb      NEAR $L$common_seh_tail
3499
3500         lea     r10,[$L$cbc_decrypt_body]
3501         cmp     rbx,r10
3502         jb      NEAR $L$restore_cbc_rax
3503
3504         lea     r10,[$L$cbc_ret]
3505         cmp     rbx,r10
3506         jae     NEAR $L$common_seh_tail
3507
3508         lea     rsi,[16+rax]
3509         lea     rdi,[512+r8]
3510         mov     ecx,20
3511         DD      0xa548f3fc
3512
3513 $L$common_rbp_tail:
3514         mov     rax,QWORD[160+r8]
3515         mov     rbp,QWORD[rax]
3516         lea     rax,[8+rax]
3517         mov     QWORD[160+r8],rbp
3518         jmp     NEAR $L$common_seh_tail
3519
3520 $L$restore_cbc_rax:
3521         mov     rax,QWORD[120+r8]
3522
3523 $L$common_seh_tail:
3524         mov     rdi,QWORD[8+rax]
3525         mov     rsi,QWORD[16+rax]
3526         mov     QWORD[152+r8],rax
3527         mov     QWORD[168+r8],rsi
3528         mov     QWORD[176+r8],rdi
3529
3530         mov     rdi,QWORD[40+r9]
3531         mov     rsi,r8
3532         mov     ecx,154
3533         DD      0xa548f3fc
3534
3535         mov     rsi,r9
3536         xor     rcx,rcx
3537         mov     rdx,QWORD[8+rsi]
3538         mov     r8,QWORD[rsi]
3539         mov     r9,QWORD[16+rsi]
3540         mov     r10,QWORD[40+rsi]
3541         lea     r11,[56+rsi]
3542         lea     r12,[24+rsi]
3543         mov     QWORD[32+rsp],r10
3544         mov     QWORD[40+rsp],r11
3545         mov     QWORD[48+rsp],r12
3546         mov     QWORD[56+rsp],rcx
3547         call    QWORD[__imp_RtlVirtualUnwind]
3548
3549         mov     eax,1
3550         add     rsp,64
3551         popfq
3552         pop     r15
3553         pop     r14
3554         pop     r13
3555         pop     r12
3556         pop     rbp
3557         pop     rbx
3558         pop     rdi
3559         pop     rsi
3560         DB      0F3h,0C3h               ;repret
3561
3562
3563 section .pdata rdata align=4
3564 ALIGN   4
3565         DD      $L$SEH_begin_aesni_ecb_encrypt wrt ..imagebase
3566         DD      $L$SEH_end_aesni_ecb_encrypt wrt ..imagebase
3567         DD      $L$SEH_info_ecb wrt ..imagebase
3568
3569         DD      $L$SEH_begin_aesni_ccm64_encrypt_blocks wrt ..imagebase
3570         DD      $L$SEH_end_aesni_ccm64_encrypt_blocks wrt ..imagebase
3571         DD      $L$SEH_info_ccm64_enc wrt ..imagebase
3572
3573         DD      $L$SEH_begin_aesni_ccm64_decrypt_blocks wrt ..imagebase
3574         DD      $L$SEH_end_aesni_ccm64_decrypt_blocks wrt ..imagebase
3575         DD      $L$SEH_info_ccm64_dec wrt ..imagebase
3576
3577         DD      $L$SEH_begin_aesni_ctr32_encrypt_blocks wrt ..imagebase
3578         DD      $L$SEH_end_aesni_ctr32_encrypt_blocks wrt ..imagebase
3579         DD      $L$SEH_info_ctr32 wrt ..imagebase
3580
3581         DD      $L$SEH_begin_aesni_xts_encrypt wrt ..imagebase
3582         DD      $L$SEH_end_aesni_xts_encrypt wrt ..imagebase
3583         DD      $L$SEH_info_xts_enc wrt ..imagebase
3584
3585         DD      $L$SEH_begin_aesni_xts_decrypt wrt ..imagebase
3586         DD      $L$SEH_end_aesni_xts_decrypt wrt ..imagebase
3587         DD      $L$SEH_info_xts_dec wrt ..imagebase
3588         DD      $L$SEH_begin_aesni_cbc_encrypt wrt ..imagebase
3589         DD      $L$SEH_end_aesni_cbc_encrypt wrt ..imagebase
3590         DD      $L$SEH_info_cbc wrt ..imagebase
3591
3592         DD      aesni_set_decrypt_key wrt ..imagebase
3593         DD      $L$SEH_end_set_decrypt_key wrt ..imagebase
3594         DD      $L$SEH_info_key wrt ..imagebase
3595
3596         DD      aesni_set_encrypt_key wrt ..imagebase
3597         DD      $L$SEH_end_set_encrypt_key wrt ..imagebase
3598         DD      $L$SEH_info_key wrt ..imagebase
3599 section .xdata rdata align=8
3600 ALIGN   8
3601 $L$SEH_info_ecb:
3602 DB      9,0,0,0
3603         DD      ecb_se_handler wrt ..imagebase
3604 $L$SEH_info_ccm64_enc:
3605 DB      9,0,0,0
3606         DD      ccm64_se_handler wrt ..imagebase
3607         DD      $L$ccm64_enc_body wrt ..imagebase,$L$ccm64_enc_ret wrt ..imagebase
3608 $L$SEH_info_ccm64_dec:
3609 DB      9,0,0,0
3610         DD      ccm64_se_handler wrt ..imagebase
3611         DD      $L$ccm64_dec_body wrt ..imagebase,$L$ccm64_dec_ret wrt ..imagebase
3612 $L$SEH_info_ctr32:
3613 DB      9,0,0,0
3614         DD      ctr_xts_se_handler wrt ..imagebase
3615         DD      $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase
3616 $L$SEH_info_xts_enc:
3617 DB      9,0,0,0
3618         DD      ctr_xts_se_handler wrt ..imagebase
3619         DD      $L$xts_enc_body wrt ..imagebase,$L$xts_enc_epilogue wrt ..imagebase
3620 $L$SEH_info_xts_dec:
3621 DB      9,0,0,0
3622         DD      ctr_xts_se_handler wrt ..imagebase
3623         DD      $L$xts_dec_body wrt ..imagebase,$L$xts_dec_epilogue wrt ..imagebase
3624 $L$SEH_info_cbc:
3625 DB      9,0,0,0
3626         DD      cbc_se_handler wrt ..imagebase
3627 $L$SEH_info_key:
3628 DB      0x01,0x04,0x01,0x00
3629 DB      0x04,0x02,0x00,0x00