kernel/power/cgemm_logic_power10.S

   1 /***************************************************************************
   2 Copyright (c) 2013-2020, The OpenBLAS Project
   3 All rights reserved.
   4 Redistribution and use in source and binary forms, with or without
   5 modification, are permitted provided that the following conditions are
   6 met:
   7 1. Redistributions of source code must retain the above copyright
   8 notice, this list of conditions and the following disclaimer.
   9 2. Redistributions in binary form must reproduce the above copyright
  10 notice, this list of conditions and the following disclaimer in
  11 the documentation and/or other materials provided with the
  12 distribution.
  13 3. Neither the name of the OpenBLAS project nor the names of
  14 its contributors may be used to endorse or promote products
  15 derived from this software without specific prior written permission.
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  25 USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 *****************************************************************************/
  27
  28 #define MY_ALIGN .align 3
  29 b CGEMM_L4
  30 /*                MINI SUBROUTINES                            */
  31 /*                4x8 MAIN 128x+2 LOOP                     */
  32
  33
  34 CGEMM_L4x8_LMAIN_SUB:
  35 /*----------------------------------------*/
  36     mtctr   T8
  37     LOAD4x8_2
  38     MY_ALIGN
  39 CGEMM_L4x8_LOOP:
  40 /*----------------------------------------*/
  41     dcbt    AO, PRE
  42     dcbt    BO, PRE
  43     KERNEL4x8_L2 128,64,0,0
  44 CGEMM_L4x8_K128:
  45 /*----------------------------------------*/
  46     KERNEL4x8_L2 128,64,1,0
  47     dcbt    AO, T2
  48     KERNEL4x8_L2 128,64,2,0
  49     KERNEL4x8_L2 128,64,3,0
  50     dcbt    AO, T3
  51     dcbt    BO, T2
  52     KERNEL4x8_L2 128,64,4,0
  53     KERNEL4x8_L2 128,64,5,0
  54     dcbt    AO, T4
  55     KERNEL4x8_L2 128,64,6,0
  56     KERNEL4x8_L2 128,64,7,0
  57     dcbt    AO, T5
  58     dcbt    BO, T3
  59     KERNEL4x8_L2 128,64,8,0
  60     KERNEL4x8_L2 128,64,9,0
  61     KERNEL4x8_L2 128,64,10,0
  62     KERNEL4x8_L2 128,64,11,0
  63     dcbt    BO, T4
  64     KERNEL4x8_L2 128,64,12,0
  65     KERNEL4x8_L2 128,64,13,0
  66     KERNEL4x8_L2 128,64,14,0
  67     KERNEL4x8_L2 128,64,15,0
  68     KERNEL4x8_L2 128,64,16,0
  69     KERNEL4x8_L2 128,64,17,0
  70     KERNEL4x8_L2 128,64,18,0
  71     KERNEL4x8_L2 128,64,19,0
  72     KERNEL4x8_L2 128,64,20,0
  73     KERNEL4x8_L2 128,64,21,0
  74     KERNEL4x8_L2 128,64,22,0
  75     KERNEL4x8_L2 128,64,23,0
  76     KERNEL4x8_L2 128,64,24,0
  77     KERNEL4x8_L2 128,64,25,0
  78     KERNEL4x8_L2 128,64,26,0
  79     KERNEL4x8_L2 128,64,27,0
  80     KERNEL4x8_L2 128,64,28,0
  81     KERNEL4x8_L2 128,64,29,0
  82     KERNEL4x8_L2 128,64,30,0
  83     KERNEL4x8_L2 128,64,31,0
  84     KERNEL4x8_L2 128,64,32,0
  85     KERNEL4x8_L2 128,64,33,0
  86     KERNEL4x8_L2 128,64,34,0
  87     KERNEL4x8_L2 128,64,35,0
  88     KERNEL4x8_L2 128,64,36,0
  89     KERNEL4x8_L2 128,64,37,0
  90     KERNEL4x8_L2 128,64,38,0
  91     KERNEL4x8_L2 128,64,39,0
  92     KERNEL4x8_L2 128,64,40,0
  93     KERNEL4x8_L2 128,64,41,0
  94     KERNEL4x8_L2 128,64,42,0
  95     KERNEL4x8_L2 128,64,43,0
  96     KERNEL4x8_L2 128,64,44,0
  97     KERNEL4x8_L2 128,64,45,0
  98     KERNEL4x8_L2 128,64,46,0
  99     KERNEL4x8_L2 128,64,47,0
 100     KERNEL4x8_L2 128,64,48,0
 101     KERNEL4x8_L2 128,64,49,0
 102     KERNEL4x8_L2 128,64,50,0
 103     KERNEL4x8_L2 128,64,51,0
 104     KERNEL4x8_L2 128,64,52,0
 105     KERNEL4x8_L2 128,64,53,0
 106     KERNEL4x8_L2 128,64,54,0
 107     KERNEL4x8_L2 128,64,55,0
 108     KERNEL4x8_L2 128,64,56,0
 109     KERNEL4x8_L2 128,64,57,0
 110     KERNEL4x8_L2 128,64,58,0
 111     KERNEL4x8_L2 128,64,59,0
 112     KERNEL4x8_L2 128,64,60,0
 113     KERNEL4x8_L2 128,64,61,0
 114     KERNEL4x8_L2 128,64,62,0
 115     KERNEL4x8_L2 128,64,63,1
 116     bdnz    CGEMM_L4x8_LOOP
 117     MY_ALIGN
 118 CGEMM_L4x8_LOOP_END:
 119 /*----------------------------------------*/
 120     END4x8_2
 121     blr
 122     MY_ALIGN
 123
 124
 125 CGEMM_4x8_L64_SUB:
 126 /*----------------------------------------*/
 127     LOAD4x8_2
 128     dcbt    AO, PRE
 129     dcbt    BO, PRE
 130     KERNEL4x8_L2 128,64,0,0
 131     KERNEL4x8_L2 128,64,1,0
 132     dcbt    AO, T2
 133     KERNEL4x8_L2 128,64,2,0
 134     KERNEL4x8_L2 128,64,3,0
 135     dcbt    AO, T3
 136     dcbt    BO, T2
 137     KERNEL4x8_L2 128,64,4,0
 138     KERNEL4x8_L2 128,64,5,0
 139     dcbt    AO, T4
 140     KERNEL4x8_L2 128,64,6,0
 141     KERNEL4x8_L2 128,64,7,0
 142     dcbt    AO, T5
 143     dcbt    BO, T3
 144     KERNEL4x8_L2 128,64,8,0
 145     KERNEL4x8_L2 128,64,9,0
 146     KERNEL4x8_L2 128,64,10,0
 147     KERNEL4x8_L2 128,64,11,0
 148     dcbt    BO, T4
 149     KERNEL4x8_L2 128,64,12,0
 150     KERNEL4x8_L2 128,64,13,0
 151     KERNEL4x8_L2 128,64,14,0
 152     KERNEL4x8_L2 128,64,15,0
 153     KERNEL4x8_L2 128,64,16,0
 154     KERNEL4x8_L2 128,64,17,0
 155     KERNEL4x8_L2 128,64,18,0
 156     KERNEL4x8_L2 128,64,19,0
 157     KERNEL4x8_L2 128,64,20,0
 158     KERNEL4x8_L2 128,64,21,0
 159     KERNEL4x8_L2 128,64,22,0
 160     KERNEL4x8_L2 128,64,23,0
 161     KERNEL4x8_L2 128,64,24,0
 162     KERNEL4x8_L2 128,64,25,0
 163     KERNEL4x8_L2 128,64,26,0
 164     KERNEL4x8_L2 128,64,27,0
 165     KERNEL4x8_L2 128,64,28,0
 166     KERNEL4x8_L2 128,64,29,0
 167     KERNEL4x8_L2 128,64,30,0
 168     KERNEL4x8_E2 128,64,31,1
 169     blr
 170     MY_ALIGN
 171
 172
 173 CGEMM_4x8_L32_SUB:
 174 /*----------------------------------------*/
 175     LOAD4x8_2
 176     dcbt    AO, PRE
 177     dcbt    BO, PRE
 178     KERNEL4x8_L2 128,64,0,0
 179     KERNEL4x8_L2 128,64,1,0
 180     dcbt    AO, T2
 181     KERNEL4x8_L2 128,64,2,0
 182     KERNEL4x8_L2 128,64,3,0
 183     dcbt    AO, T3
 184     dcbt    BO, T2
 185     KERNEL4x8_L2 128,64,4,0
 186     KERNEL4x8_L2 128,64,5,0
 187     dcbt    AO, T4
 188     KERNEL4x8_L2 128,64,6,0
 189     KERNEL4x8_L2 128,64,7,0
 190     dcbt    AO, T5
 191     dcbt    BO, T3
 192     KERNEL4x8_L2 128,64,8,0
 193     KERNEL4x8_L2 128,64,9,0
 194     KERNEL4x8_L2 128,64,10,0
 195     KERNEL4x8_L2 128,64,11,0
 196     dcbt    BO, T4
 197     KERNEL4x8_L2 128,64,12,0
 198     KERNEL4x8_L2 128,64,13,0
 199     KERNEL4x8_L2 128,64,14,0
 200     KERNEL4x8_E2 128,64,15,1
 201     blr
 202     MY_ALIGN
 203
 204
 205 CGEMM_4x8_L16_SUB:
 206 /*----------------------------------------*/
 207     LOAD4x8_2
 208     dcbt    AO, PRE
 209     dcbt    BO, PRE
 210     KERNEL4x8_L2 128,64,0,0
 211     KERNEL4x8_L2 128,64,1,0
 212     dcbt    AO, T2
 213     KERNEL4x8_L2 128,64,2,0
 214     KERNEL4x8_L2 128,64,3,0
 215     dcbt    AO, T3
 216     dcbt    BO, T2
 217     KERNEL4x8_L2 128,64,4,0
 218     KERNEL4x8_L2 128,64,5,0
 219     dcbt    AO, T4
 220     KERNEL4x8_L2 128,64,6,0
 221     KERNEL4x8_E2 128,64,7,1
 222     blr
 223     MY_ALIGN
 224
 225
 226 CGEMM_4x4_LMAIN_SUB:
 227 /*----------------------------------------*/
 228     mtctr   T8
 229     LOAD4x4_2
 230     MY_ALIGN
 231 CGEMM_L4x4_LOOP:
 232 /*----------------------------------------*/
 233     KERNEL4x4_L2 64,64,0,0
 234 CGEMM_L4x4_K32:
 235 /*----------------------------------------*/
 236     KERNEL4x4_L2 64,64,1,0
 237     KERNEL4x4_L2 64,64,2,0
 238     KERNEL4x4_L2 64,64,3,0
 239     KERNEL4x4_L2 64,64,4,0
 240     KERNEL4x4_L2 64,64,5,0
 241     KERNEL4x4_L2 64,64,6,0
 242     KERNEL4x4_L2 64,64,7,0
 243     KERNEL4x4_L2 64,64,8,0
 244     KERNEL4x4_L2 64,64,9,0
 245     KERNEL4x4_L2 64,64,10,0
 246     KERNEL4x4_L2 64,64,11,0
 247     KERNEL4x4_L2 64,64,12,0
 248     KERNEL4x4_L2 64,64,13,0
 249     KERNEL4x4_L2 64,64,14,0
 250     KERNEL4x4_L2 64,64,15,1
 251     bdnz    CGEMM_L4x4_LOOP
 252     MY_ALIGN
 253 CGEMM_L4x4_LOOP_END:
 254 /*----------------------------------------*/
 255     END4x4_2
 256     blr
 257     MY_ALIGN
 258
 259
 260 CGEMM_4x4_L16_SUB:
 261 /*----------------------------------------*/
 262     LOAD4x4_2
 263     KERNEL4x4_L2 64,64,0,0
 264     KERNEL4x4_L2 64,64,1,0
 265     KERNEL4x4_L2 64,64,2,0
 266     KERNEL4x4_L2 64,64,3,0
 267     KERNEL4x4_L2 64,64,4,0
 268     KERNEL4x4_L2 64,64,5,0
 269     KERNEL4x4_L2 64,64,6,0
 270     KERNEL4x4_E2 64,64,7,1
 271     blr
 272     MY_ALIGN
 273
 274
 275 CGEMM_4x4_L8_SUB:
 276 /*----------------------------------------*/
 277     LOAD4x4_2
 278     KERNEL4x4_L2 64,64,0,0
 279     KERNEL4x4_L2 64,64,1,0
 280     KERNEL4x4_L2 64,64,2,0
 281     KERNEL4x4_E2 64,64,3,1
 282     blr
 283
 284
 285 CGEMM_4x2_LMAIN_SUB:
 286 /*----------------------------------------*/
 287     mtctr   T8
 288     LOAD4x2_2
 289     MY_ALIGN
 290 CGEMM_L4x2_LOOP:
 291 /*----------------------------------------*/
 292     KERNEL4x2_L2 32,64,0,0
 293 CGEMM_L4x2_K32:
 294 /*----------------------------------------*/
 295     KERNEL4x2_L2 32,64,1,0
 296     KERNEL4x2_L2 32,64,2,0
 297     KERNEL4x2_L2 32,64,3,0
 298     KERNEL4x2_L2 32,64,4,0
 299     KERNEL4x2_L2 32,64,5,0
 300     KERNEL4x2_L2 32,64,6,0
 301     KERNEL4x2_L2 32,64,7,0
 302     KERNEL4x2_L2 32,64,8,0
 303     KERNEL4x2_L2 32,64,9,0
 304     KERNEL4x2_L2 32,64,10,0
 305     KERNEL4x2_L2 32,64,11,0
 306     KERNEL4x2_L2 32,64,12,0
 307     KERNEL4x2_L2 32,64,13,0
 308     KERNEL4x2_L2 32,64,14,0
 309     KERNEL4x2_L2 32,64,15,1
 310     bdnz    CGEMM_L4x2_LOOP
 311     MY_ALIGN
 312
 313
 314 CGEMM_L4x2_LOOP_END:
 315 /*----------------------------------------*/
 316     END4x2_2
 317     blr
 318     MY_ALIGN
 319 CGEMM_4x2_L16_SUB:
 320 /*----------------------------------------*/
 321     LOAD4x2_2
 322     KERNEL4x2_L2 32,64,0,0
 323     KERNEL4x2_L2 32,64,1,0
 324     KERNEL4x2_L2 32,64,2,0
 325     KERNEL4x2_L2 32,64,3,0
 326     KERNEL4x2_L2 32,64,4,0
 327     KERNEL4x2_L2 32,64,5,0
 328     KERNEL4x2_L2 32,64,6,0
 329     KERNEL4x2_E2 32,64,7,1
 330     blr
 331     MY_ALIGN
 332 CGEMM_4x2_L8_SUB:
 333 /*----------------------------------------*/
 334     LOAD4x2_2
 335     KERNEL4x2_L2 32,64,0,0
 336     KERNEL4x2_L2 32,64,1,0
 337     KERNEL4x2_L2 32,64,2,0
 338     KERNEL4x2_E2 32,64,3,1
 339     blr
 340
 341
 342 CGEMM_4x1_LMAIN_SUB:
 343 /*----------------------------------------*/
 344     mtctr   T8
 345     LOAD4x1_2
 346     MY_ALIGN
 347 CGEMM_L4x1_LOOP:
 348 /*----------------------------------------*/
 349     KERNEL4x1_L2 16,64,0,0
 350 CGEMM_L4x1_K32:
 351 /*----------------------------------------*/
 352     KERNEL4x1_L2 16,64,1,0
 353     KERNEL4x1_L2 16,64,2,0
 354     KERNEL4x1_L2 16,64,3,0
 355     KERNEL4x1_L2 16,64,4,0
 356     KERNEL4x1_L2 16,64,5,0
 357     KERNEL4x1_L2 16,64,6,0
 358     KERNEL4x1_L2 16,64,7,0
 359     KERNEL4x1_L2 16,64,8,0
 360     KERNEL4x1_L2 16,64,9,0
 361     KERNEL4x1_L2 16,64,10,0
 362     KERNEL4x1_L2 16,64,11,0
 363     KERNEL4x1_L2 16,64,12,0
 364     KERNEL4x1_L2 16,64,13,0
 365     KERNEL4x1_L2 16,64,14,0
 366     KERNEL4x1_L2 16,64,15,1
 367     bdnz    CGEMM_L4x1_LOOP
 368     MY_ALIGN
 369 CGEMM_L4x1_LOOP_END:
 370 /*----------------------------------------*/
 371     END4x1_2
 372     blr
 373
 374     MY_ALIGN
 375 CGEMM_4x1_L16_SUB:
 376 /*----------------------------------------*/
 377     LOAD4x1_2
 378     KERNEL4x1_L2 16,64,0,0
 379     KERNEL4x1_L2 16,64,1,0
 380     KERNEL4x1_L2 16,64,2,0
 381     KERNEL4x1_L2 16,64,3,0
 382     KERNEL4x1_L2 16,64,4,0
 383     KERNEL4x1_L2 16,64,5,0
 384     KERNEL4x1_L2 16,64,6,0
 385     KERNEL4x1_E2 16,64,7,1
 386     blr
 387     MY_ALIGN
 388
 389
 390 CGEMM_4x1_L8_SUB:
 391 /*----------------------------------------*/
 392     LOAD4x1_2
 393     KERNEL4x1_L2 16,64,0,0
 394     KERNEL4x1_L2 16,64,1,0
 395     KERNEL4x1_L2 16,64,2,0
 396     KERNEL4x1_E2 16,64,3,1
 397     blr
 398
 399
 400
 401 /*             MAIN LOOP BEGINS               */
 402     MY_ALIGN
 403
 404
 405 CGEMM_L4:
 406 /*----------------------------------------*/
 407 #if defined(TRMMKERNEL) && !defined(LEFT)
 408     neg TEMP_REG, OFFSET
 409 #endif
 410     /* Pre set value in vs57 as 0xffff0000ffff0000 for masking */
 411     vspltisb v24, -1
 412     vspltisb v25, 0
 413     xxsldwi vs57, vs56, vs57, 1
 414     xxpermdi vs57, vs57, vs57, 3
 415     srawi.    J,  N,  2
 416     ble   CGEMM_L4_END
 417
 418
 419 CGEMM_L4_BEGIN:
 420 /*----------------------------------------*/
 421     mr    CO, C
 422     slwi    T1, LDC , 2
 423     add     T2,C,LDC
 424     mr    AO, A
 425     add   C,  C,  T1
 426 #if defined(TRMMKERNEL) && defined(LEFT)
 427     mr TEMP_REG, OFFSET  /*off = offset;*/
 428 #endif
 429     srawi.    I,  M,  3
 430     ble   CGEMM_L4x8_END
 431     dcbt    CO,r0  /*just prefetch*/
 432     dcbt    T2,r0
 433
 434
 435 CGEMM_L4x8_BEGIN:
 436 /*----------------------------------------*/
 437 #if defined(TRMMKERNEL)
 438     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4
 439 #else
 440     mr    BO, B
 441     dcbt    B,  r0
 442 #endif
 443     dcbt    AO, r0
 444 #if defined(TRMMKERNEL)
 445     REFRESH_TEMP_BK T6,K,TEMP_REG,8,4
 446     mr T1, T6
 447 /* TEMPS FOR PREFETCH */
 448     li T2, 1024
 449     li T3, 1024+512
 450     addi T1,T1, -2
 451 /* TEMPS FOR PREFETCH */
 452     li T4, 2048
 453     li T5, 2048+512
 454     srawi.   T8, T1, 7 /**(T1-2) % 128x */
 455 #else
 456     mr T1, K
 457 /* TEMPS FOR PREFETCH */
 458     li T2, 1024
 459     li T3, 1024+512
 460     addi T1,T1, -2
 461 /* TEMPS FOR PREFETCH */
 462     li T4, 2048
 463     li T5, 2048+512
 464     srawi.   T8, T1, 7 /**(K-2) % 128x */
 465 #endif
 466     ZERO4x8
 467     ble   CGEMM_L4x8_SUB0
 468     bl CGEMM_L4x8_LMAIN_SUB
 469     andi.   L,  T1, 127
 470     ble   CGEMM_L4x8_SAVE
 471     b   CGEMM_L4x8_SUB2
 472
 473
 474 CGEMM_L4x8_SUB0:
 475 /*----------------------------------------*/
 476 #if defined(TRMMKERNEL)
 477     andi.   L,  T6, 255
 478     cmpwi   T6,129
 479 #else
 480     andi.   L,  K,  255
 481     cmpwi   K,129
 482 #endif
 483     li T8,1
 484     bne CMP4x8_128K
 485     addi BO,BO,-32
 486     addi AO,AO,-64
 487     LOAD4x8O 64,32
 488     END4x8_WITHOUT_ADD
 489     LOAD4x8_2O  128, 64
 490     mtctr   T8
 491     bl CGEMM_L4x8_K128
 492     b CGEMM_L4x8_SAVE
 493     CMP4x8_128K:
 494 /*----------------------------------------*/
 495 #if defined(TRMMKERNEL)
 496     cmpwi   T6,128
 497 #else
 498     cmpwi   K,128
 499 #endif
 500     bne CGEMM_L4x8_SUB2
 501     MY_ALIGN
 502     mtctr   T8
 503     addi BO,BO,-64
 504     addi AO,AO,-128
 505     LOAD4x8_2O 128,64
 506     bl CGEMM_L4x8_K128
 507     b CGEMM_L4x8_SAVE
 508     MY_ALIGN
 509
 510
 511 CGEMM_L4x8_SUB2:
 512 /*----------------------------------------*/
 513     andi.      T1,L, 64
 514     ble CGEMM_L4x8_SUB2_32
 515     bl  CGEMM_4x8_L64_SUB
 516     MY_ALIGN
 517
 518
 519 CGEMM_L4x8_SUB2_32:
 520 /*----------------------------------------*/
 521     andi.      T1,L, 32
 522     ble CGEMM_L4x8_SUB2_16
 523     bl  CGEMM_4x8_L32_SUB
 524     MY_ALIGN
 525
 526
 527 CGEMM_L4x8_SUB2_16:
 528 /*----------------------------------------*/
 529     andi.      T1,L, 16
 530     ble CGEMM_L4x8_SUB2_8
 531     bl  CGEMM_4x8_L16_SUB
 532     MY_ALIGN
 533
 534
 535 CGEMM_L4x8_SUB2_8:
 536 /*----------------------------------------*/
 537     andi.      T1,L, 8
 538     ble CGEMM_L4x8_SUB2_4
 539     LOAD4x8_2
 540     KERNEL4x8_L2  128,64, 0,0
 541     KERNEL4x8_L2  128,64, 1,0
 542     KERNEL4x8_L2  128,64, 2,0
 543     KERNEL4x8_E2  128,64, 3,1
 544     MY_ALIGN
 545
 546
 547 CGEMM_L4x8_SUB2_4:
 548 /*----------------------------------------*/
 549     andi.      T1,L, 4
 550     ble CGEMM_L4x8_SUB2_2
 551     LOAD4x8_2
 552     KERNEL4x8_L2  128,64, 0,0
 553     KERNEL4x8_E2  128,64, 1,1
 554     MY_ALIGN
 555
 556
 557 CGEMM_L4x8_SUB2_2:
 558 /*----------------------------------------*/
 559     andi.      T1,L, 2
 560     ble CGEMM_L4x8_SUB2_1
 561     LOAD4x8_2
 562     KERNEL4x8_E2  128,64, 0,1
 563     MY_ALIGN
 564
 565
 566 CGEMM_L4x8_SUB2_1:
 567 /*----------------------------------------*/
 568     andi.      T1,L, 1
 569     ble CGEMM_L4x8_SAVE
 570     KERNEL4x8
 571
 572     MY_ALIGN
 573 CGEMM_L4x8_SAVE:
 574 /*----------------------------------------*/
 575     addic.    I,  I,  -1
 576     MY_ALIGN
 577     SAVE4x8
 578 #if defined(TRMMKERNEL)
 579     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,4
 580 #endif
 581     bgt   CGEMM_L4x8_BEGIN
 582     andi.   T2, M,  7
 583     ble   CGEMM_L4x1_END
 584     andi.   T1, M,  4
 585     ble   CGEMM_L4x4_END
 586     b   CGEMM_L4x4_BEGIN
 587     MY_ALIGN
 588
 589
 590 CGEMM_L4x8_END:
 591 /*----------------------------------------*/
 592
 593
 594 CGEMM_L4x4_BEGIN:
 595 /*----------------------------------------*/
 596     andi.   T2, M,  7
 597     ble   CGEMM_L4x1_END
 598     andi.   T1, M,  4
 599     ble   CGEMM_L4x4_END
 600 #if defined(TRMMKERNEL)
 601     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4
 602 #else
 603     mr    BO, B
 604 #endif
 605 #if defined(TRMMKERNEL)
 606     REFRESH_TEMP_BK T6,K,TEMP_REG,4,4
 607     mr T1, T6
 608     addi T1,T1, -2
 609     srawi.   T8, T1, 5 /**(T1-2) % 32x */
 610 #else
 611     mr T1, K
 612     addi T1,T1, -2
 613     srawi.   T8, T1, 5 /**(K-2) % 32x */
 614 #endif
 615     ZERO4x4
 616     ble   CGEMM_L4x4_SUB0
 617     bl CGEMM_4x4_LMAIN_SUB
 618     andi.   L,  T1, 31
 619     ble   CGEMM_L4x4_SAVE
 620     b    CGEMM_L4x4_SUB2
 621
 622
 623 CGEMM_L4x4_SUB0:
 624 /*----------------------------------------*/
 625 #if defined(TRMMKERNEL)
 626     andi.   L,  T6, 63
 627     cmpwi   T6,33
 628 #else
 629     andi.   L,  K,  63
 630     cmpwi   K,33
 631 #endif
 632     li T8,1
 633     bne CMP4x4_32K
 634     addi BO,BO,-32
 635     addi AO,AO,-32
 636     LOAD4x4O 32,32
 637     END4x4_WITHOUT_ADD
 638     LOAD4x4_2O  64, 64
 639     mtctr   T8
 640     bl CGEMM_L4x4_K32
 641     b CGEMM_L4x4_SAVE
 642     CMP4x4_32K:
 643 /*----------------------------------------*/
 644 #if defined(TRMMKERNEL)
 645     cmpwi   T6,32
 646 #else
 647     cmpwi   K,32
 648 #endif
 649     bne CGEMM_L4x4_SUB2
 650     MY_ALIGN
 651     mtctr   T8
 652     addi BO,BO,-64
 653     addi AO,AO,-64
 654     LOAD4x4_2O 64,64
 655     bl CGEMM_L4x4_K32
 656     b CGEMM_L4x4_SAVE
 657     MY_ALIGN
 658     MY_ALIGN
 659
 660
 661 CGEMM_L4x4_SUB2:
 662 /*----------------------------------------*/
 663     andi.      T1,L, 16
 664     ble CGEMM_L4x4_SUB2_8
 665     bl  CGEMM_4x4_L16_SUB
 666     MY_ALIGN
 667
 668
 669 CGEMM_L4x4_SUB2_8:
 670 /*----------------------------------------*/
 671     andi.      T1,L, 8
 672     ble CGEMM_L4x4_SUB2_4
 673     bl CGEMM_4x4_L8_SUB
 674     MY_ALIGN
 675
 676
 677 CGEMM_L4x4_SUB2_4:
 678 /*----------------------------------------*/
 679     andi.      T1,L, 4
 680     ble CGEMM_L4x4_SUB2_2
 681     LOAD4x4_2
 682     KERNEL4x4_L2  64,64, 0,0
 683     KERNEL4x4_E2  64,64, 1,1
 684     MY_ALIGN
 685
 686
 687 CGEMM_L4x4_SUB2_2:
 688 /*----------------------------------------*/
 689     andi.      T1,L, 2
 690     ble CGEMM_L4x4_SUB2_1
 691     LOAD4x4_2
 692     KERNEL4x4_E2  64,64, 0,1
 693     MY_ALIGN
 694
 695
 696 CGEMM_L4x4_SUB2_1:
 697 /*----------------------------------------*/
 698     andi.      T1,L, 1
 699     ble CGEMM_L4x4_SAVE
 700     KERNEL4x4
 701
 702
 703 CGEMM_L4x4_SAVE:
 704 /*----------------------------------------*/
 705     SAVE4x4
 706 #if defined(TRMMKERNEL)
 707     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,4
 708 #endif
 709
 710
 711 CGEMM_L4x4_END:
 712 /*----------------------------------------*/
 713
 714
 715 CGEMM_L4x2_BEGIN:
 716 /*----------------------------------------*/
 717     andi.   T1, M,  2
 718     ble   CGEMM_L4x2_END
 719 #if defined(TRMMKERNEL)
 720     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4
 721 #else
 722     mr    BO, B
 723 #endif
 724 #if defined(TRMMKERNEL)
 725     REFRESH_TEMP_BK T6,K,TEMP_REG,2,4
 726     mr T1, T6
 727     addi T1,T1, -2
 728     srawi.   T8, T1, 5 /**(T1-2) % 32x */
 729 #else
 730     mr T1, K
 731     addi T1,T1, -2
 732     srawi.   T8, T1, 5 /**(K-2) % 32x */
 733 #endif
 734     ZERO4x2
 735     ble   CGEMM_L4x2_SUB0
 736     bl CGEMM_4x2_LMAIN_SUB
 737     andi.   L,  T1, 31
 738     ble   CGEMM_L4x2_SAVE
 739     b   CGEMM_L4x2_SUB2
 740
 741
 742 CGEMM_L4x2_SUB0:
 743 /*----------------------------------------*/
 744 #if defined(TRMMKERNEL)
 745     andi.   L,  T6, 63
 746     cmpwi   T6,33
 747 #else
 748     andi.   L,  K,  63
 749     cmpwi   K,33
 750 #endif
 751     li T8,1
 752     bne CMP4x2_32K
 753     addi BO,BO,-32
 754     addi AO,AO,-16
 755     LOAD4x2O 16,32
 756     END4x2_WITHOUT_ADD
 757     LOAD4x2_2O  32, 64
 758     mtctr   T8
 759     bl CGEMM_L4x2_K32
 760     b CGEMM_L4x2_SAVE
 761     CMP4x2_32K:
 762 /*----------------------------------------*/
 763 #if defined(TRMMKERNEL)
 764     cmpwi   T6,32
 765 #else
 766     cmpwi   K,32
 767 #endif
 768     bne CGEMM_L4x2_SUB2
 769     MY_ALIGN
 770     mtctr   T8
 771     addi BO,BO,-64
 772     addi AO,AO,-32
 773     LOAD4x2_2O 32,64
 774     bl CGEMM_L4x2_K32
 775     b CGEMM_L4x2_SAVE
 776     MY_ALIGN
 777     MY_ALIGN
 778
 779
 780 CGEMM_L4x2_SUB2:
 781 /*----------------------------------------*/
 782     andi.      T1,L, 16
 783     ble CGEMM_L4x2_SUB2_8
 784     bl CGEMM_4x2_L16_SUB
 785     MY_ALIGN
 786
 787
 788 CGEMM_L4x2_SUB2_8:
 789 /*----------------------------------------*/
 790     andi.      T1,L, 8
 791     ble CGEMM_L4x2_SUB2_4
 792     bl CGEMM_4x2_L8_SUB
 793     MY_ALIGN
 794
 795
 796 CGEMM_L4x2_SUB2_4:
 797 /*----------------------------------------*/
 798     andi.      T1,L, 4
 799     ble CGEMM_L4x2_SUB2_2
 800     LOAD4x2_2
 801     KERNEL4x2_L2  32,64, 0,0
 802     KERNEL4x2_E2  32,64, 1,1
 803     MY_ALIGN
 804
 805
 806 CGEMM_L4x2_SUB2_2:
 807 /*----------------------------------------*/
 808     andi.      T1,L, 2
 809     ble CGEMM_L4x2_SUB2_1
 810     LOAD4x2_2
 811     KERNEL4x2_E2  32,64, 0,1
 812     MY_ALIGN
 813
 814
 815 CGEMM_L4x2_SUB2_1:
 816 /*----------------------------------------*/
 817     andi.      T1,L, 1
 818     ble CGEMM_L4x2_SAVE
 819     KERNEL4x2
 820
 821     MY_ALIGN
 822 CGEMM_L4x2_SAVE:
 823 /*----------------------------------------*/
 824     SAVE4x2
 825 #if defined(TRMMKERNEL)
 826     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,4
 827 #endif
 828
 829
 830 CGEMM_L4x2_END:
 831 /*----------------------------------------*/
 832
 833
 834 CGEMM_L4x1_BEGIN:
 835 /*----------------------------------------*/
 836     andi.   T1, M,  1
 837     ble   CGEMM_L4x1_END
 838 #if defined(TRMMKERNEL)
 839     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4
 840 #else
 841     mr    BO, B
 842 #endif
 843 #if defined(TRMMKERNEL)
 844     REFRESH_TEMP_BK T6,K,TEMP_REG,1,4
 845     mr T1, T6
 846     addi T1,T1, -2
 847     srawi.   T8, T1, 5 /**(T1-2) % 32x */
 848 #else
 849     mr T1, K
 850     addi T1,T1, -2
 851     srawi.   T8, T1, 5 /**(K-2) % 32x */
 852 #endif
 853     ZERO4x1
 854     ble   CGEMM_L4x1_SUB0
 855     bl CGEMM_4x1_LMAIN_SUB
 856     andi.   L,  T1, 31
 857     ble   CGEMM_L4x1_SAVE
 858     b   CGEMM_L4x1_SUB2
 859
 860
 861 CGEMM_L4x1_SUB0:
 862 /*----------------------------------------*/
 863 #if defined(TRMMKERNEL)
 864     andi.   L,  T6, 63
 865     cmpwi   T6,33
 866 #else
 867     andi.   L,  K,  63
 868     cmpwi   K,33
 869 #endif
 870     li T8,1
 871     bne CMP4x1_32K
 872     addi BO,BO,-32
 873     addi AO,AO,-8
 874     LOAD4x1O 8,32
 875     END4x1_WITHOUT_ADD
 876     LOAD4x1_2O  16, 64
 877     mtctr   T8
 878     bl CGEMM_L4x1_K32
 879     b CGEMM_L4x1_SAVE
 880     CMP4x1_32K:
 881 /*----------------------------------------*/
 882 #if defined(TRMMKERNEL)
 883     cmpwi   T6,32
 884 #else
 885     cmpwi   K,32
 886 #endif
 887     bne CGEMM_L4x1_SUB2
 888     MY_ALIGN
 889     mtctr   T8
 890     addi BO,BO,-64
 891     addi AO,AO,-16
 892     LOAD4x1_2O 16,64
 893     bl CGEMM_L4x1_K32
 894     b CGEMM_L4x1_SAVE
 895     MY_ALIGN
 896     MY_ALIGN
 897
 898
 899 CGEMM_L4x1_SUB2:
 900 /*----------------------------------------*/
 901     andi.      T1,L, 16
 902     ble CGEMM_L4x1_SUB2_8
 903     bl CGEMM_4x1_L16_SUB
 904     MY_ALIGN
 905
 906
 907 CGEMM_L4x1_SUB2_8:
 908 /*----------------------------------------*/
 909     andi.      T1,L, 8
 910     ble CGEMM_L4x1_SUB2_4
 911     bl CGEMM_4x1_L8_SUB
 912     MY_ALIGN
 913
 914
 915 CGEMM_L4x1_SUB2_4:
 916 /*----------------------------------------*/
 917     andi.      T1,L, 4
 918     ble CGEMM_L4x1_SUB2_2
 919     LOAD4x1_2
 920     KERNEL4x1_L2  16,64, 0,0
 921     KERNEL4x1_E2  16,64, 1,1
 922     MY_ALIGN
 923
 924
 925 CGEMM_L4x1_SUB2_2:
 926 /*----------------------------------------*/
 927     andi.      T1,L, 2
 928     ble CGEMM_L4x1_SUB2_1
 929     LOAD4x1_2
 930     KERNEL4x1_E2  16,64, 0,1
 931     MY_ALIGN
 932
 933
 934 CGEMM_L4x1_SUB2_1:
 935 /*----------------------------------------*/
 936     andi.      T1,L, 1
 937     ble CGEMM_L4x1_SAVE
 938     KERNEL4x1
 939
 940     MY_ALIGN
 941 CGEMM_L4x1_SAVE:
 942 /*----------------------------------------*/
 943
 944     SAVE4x1
 945 #if defined(TRMMKERNEL)
 946     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,4
 947 #endif
 948
 949
 950 CGEMM_L4x1_END:
 951 /*----------------------------------------*/
 952     slwi    T1, K,  5
 953     addic.    J,  J,  -1
 954     add   B,  B,  T1
 955 #if defined(TRMMKERNEL) && !defined(LEFT)
 956     addi TEMP_REG, TEMP_REG, 4
 957 #endif
 958     bgt   CGEMM_L4_BEGIN
 959
 960
 961 CGEMM_L4_END:
 962
 963 b CGEMM_L2
 964 /*                MINI SUBROUTINES                            */
 965 /*                2x8 MAIN 128x+2 LOOP                     */
 966
 967
 968 CGEMM_L2x8_LMAIN_SUB:
 969 /*----------------------------------------*/
 970     mtctr   T8
 971     LOAD2x8_2
 972     MY_ALIGN
 973 CGEMM_L2x8_LOOP:
 974 /*----------------------------------------*/
 975     dcbt    AO, PRE
 976     dcbt    BO, PRE
 977     KERNEL2x8_L2 128,32,0,0
 978 CGEMM_L2x8_K128:
 979 /*----------------------------------------*/
 980     KERNEL2x8_L2 128,32,1,0
 981     dcbt    AO, T2
 982     KERNEL2x8_L2 128,32,2,0
 983     KERNEL2x8_L2 128,32,3,0
 984     dcbt    AO, T3
 985     dcbt    BO, T2
 986     KERNEL2x8_L2 128,32,4,0
 987     KERNEL2x8_L2 128,32,5,0
 988     dcbt    AO, T4
 989     KERNEL2x8_L2 128,32,6,0
 990     KERNEL2x8_L2 128,32,7,0
 991     dcbt    AO, T5
 992     dcbt    BO, T3
 993     KERNEL2x8_L2 128,32,8,0
 994     KERNEL2x8_L2 128,32,9,0
 995     KERNEL2x8_L2 128,32,10,0
 996     KERNEL2x8_L2 128,32,11,0
 997     dcbt    BO, T4
 998     KERNEL2x8_L2 128,32,12,0
 999     KERNEL2x8_L2 128,32,13,0
1000     KERNEL2x8_L2 128,32,14,0
1001     KERNEL2x8_L2 128,32,15,0
1002     KERNEL2x8_L2 128,32,16,0
1003     KERNEL2x8_L2 128,32,17,0
1004     KERNEL2x8_L2 128,32,18,0
1005     KERNEL2x8_L2 128,32,19,0
1006     KERNEL2x8_L2 128,32,20,0
1007     KERNEL2x8_L2 128,32,21,0
1008     KERNEL2x8_L2 128,32,22,0
1009     KERNEL2x8_L2 128,32,23,0
1010     KERNEL2x8_L2 128,32,24,0
1011     KERNEL2x8_L2 128,32,25,0
1012     KERNEL2x8_L2 128,32,26,0
1013     KERNEL2x8_L2 128,32,27,0
1014     KERNEL2x8_L2 128,32,28,0
1015     KERNEL2x8_L2 128,32,29,0
1016     KERNEL2x8_L2 128,32,30,0
1017     KERNEL2x8_L2 128,32,31,0
1018     KERNEL2x8_L2 128,32,32,0
1019     KERNEL2x8_L2 128,32,33,0
1020     KERNEL2x8_L2 128,32,34,0
1021     KERNEL2x8_L2 128,32,35,0
1022     KERNEL2x8_L2 128,32,36,0
1023     KERNEL2x8_L2 128,32,37,0
1024     KERNEL2x8_L2 128,32,38,0
1025     KERNEL2x8_L2 128,32,39,0
1026     KERNEL2x8_L2 128,32,40,0
1027     KERNEL2x8_L2 128,32,41,0
1028     KERNEL2x8_L2 128,32,42,0
1029     KERNEL2x8_L2 128,32,43,0
1030     KERNEL2x8_L2 128,32,44,0
1031     KERNEL2x8_L2 128,32,45,0
1032     KERNEL2x8_L2 128,32,46,0
1033     KERNEL2x8_L2 128,32,47,0
1034     KERNEL2x8_L2 128,32,48,0
1035     KERNEL2x8_L2 128,32,49,0
1036     KERNEL2x8_L2 128,32,50,0
1037     KERNEL2x8_L2 128,32,51,0
1038     KERNEL2x8_L2 128,32,52,0
1039     KERNEL2x8_L2 128,32,53,0
1040     KERNEL2x8_L2 128,32,54,0
1041     KERNEL2x8_L2 128,32,55,0
1042     KERNEL2x8_L2 128,32,56,0
1043     KERNEL2x8_L2 128,32,57,0
1044     KERNEL2x8_L2 128,32,58,0
1045     KERNEL2x8_L2 128,32,59,0
1046     KERNEL2x8_L2 128,32,60,0
1047     KERNEL2x8_L2 128,32,61,0
1048     KERNEL2x8_L2 128,32,62,0
1049     KERNEL2x8_L2 128,32,63,1
1050     bdnz    CGEMM_L2x8_LOOP
1051     MY_ALIGN
1052 CGEMM_L2x8_LOOP_END:
1053 /*----------------------------------------*/
1054     END2x8_2
1055     blr
1056     MY_ALIGN
1057
1058
1059 CGEMM_2x8_L64_SUB:
1060 /*----------------------------------------*/
1061     LOAD2x8_2
1062     dcbt    AO, PRE
1063     dcbt    BO, PRE
1064     KERNEL2x8_L2 128,32,0,0
1065     KERNEL2x8_L2 128,32,1,0
1066     dcbt    AO, T2
1067     KERNEL2x8_L2 128,32,2,0
1068     KERNEL2x8_L2 128,32,3,0
1069     dcbt    AO, T3
1070     dcbt    BO, T2
1071     KERNEL2x8_L2 128,32,4,0
1072     KERNEL2x8_L2 128,32,5,0
1073     dcbt    AO, T4
1074     KERNEL2x8_L2 128,32,6,0
1075     KERNEL2x8_L2 128,32,7,0
1076     dcbt    AO, T5
1077     dcbt    BO, T3
1078     KERNEL2x8_L2 128,32,8,0
1079     KERNEL2x8_L2 128,32,9,0
1080     KERNEL2x8_L2 128,32,10,0
1081     KERNEL2x8_L2 128,32,11,0
1082     dcbt    BO, T4
1083     KERNEL2x8_L2 128,32,12,0
1084     KERNEL2x8_L2 128,32,13,0
1085     KERNEL2x8_L2 128,32,14,0
1086     KERNEL2x8_L2 128,32,15,0
1087     KERNEL2x8_L2 128,32,16,0
1088     KERNEL2x8_L2 128,32,17,0
1089     KERNEL2x8_L2 128,32,18,0
1090     KERNEL2x8_L2 128,32,19,0
1091     KERNEL2x8_L2 128,32,20,0
1092     KERNEL2x8_L2 128,32,21,0
1093     KERNEL2x8_L2 128,32,22,0
1094     KERNEL2x8_L2 128,32,23,0
1095     KERNEL2x8_L2 128,32,24,0
1096     KERNEL2x8_L2 128,32,25,0
1097     KERNEL2x8_L2 128,32,26,0
1098     KERNEL2x8_L2 128,32,27,0
1099     KERNEL2x8_L2 128,32,28,0
1100     KERNEL2x8_L2 128,32,29,0
1101     KERNEL2x8_L2 128,32,30,0
1102     KERNEL2x8_E2 128,32,31,1
1103     blr
1104     MY_ALIGN
1105
1106
1107 CGEMM_2x8_L32_SUB:
1108 /*----------------------------------------*/
1109     LOAD2x8_2
1110     dcbt    AO, PRE
1111     dcbt    BO, PRE
1112     KERNEL2x8_L2 128,32,0,0
1113     KERNEL2x8_L2 128,32,1,0
1114     dcbt    AO, T2
1115     KERNEL2x8_L2 128,32,2,0
1116     KERNEL2x8_L2 128,32,3,0
1117     dcbt    AO, T3
1118     dcbt    BO, T2
1119     KERNEL2x8_L2 128,32,4,0
1120     KERNEL2x8_L2 128,32,5,0
1121     dcbt    AO, T4
1122     KERNEL2x8_L2 128,32,6,0
1123     KERNEL2x8_L2 128,32,7,0
1124     dcbt    AO, T5
1125     dcbt    BO, T3
1126     KERNEL2x8_L2 128,32,8,0
1127     KERNEL2x8_L2 128,32,9,0
1128     KERNEL2x8_L2 128,32,10,0
1129     KERNEL2x8_L2 128,32,11,0
1130     dcbt    BO, T4
1131     KERNEL2x8_L2 128,32,12,0
1132     KERNEL2x8_L2 128,32,13,0
1133     KERNEL2x8_L2 128,32,14,0
1134     KERNEL2x8_E2 128,32,15,1
1135     blr
1136     MY_ALIGN
1137
1138
1139 CGEMM_2x8_L16_SUB:
1140 /*----------------------------------------*/
1141     LOAD2x8_2
1142     dcbt    AO, PRE
1143     dcbt    BO, PRE
1144     KERNEL2x8_L2 128,32,0,0
1145     KERNEL2x8_L2 128,32,1,0
1146     dcbt    AO, T2
1147     KERNEL2x8_L2 128,32,2,0
1148     KERNEL2x8_L2 128,32,3,0
1149     dcbt    AO, T3
1150     dcbt    BO, T2
1151     KERNEL2x8_L2 128,32,4,0
1152     KERNEL2x8_L2 128,32,5,0
1153     dcbt    AO, T4
1154     KERNEL2x8_L2 128,32,6,0
1155     KERNEL2x8_E2 128,32,7,1
1156     blr
1157     MY_ALIGN
1158
1159
1160 CGEMM_2x4_LMAIN_SUB:
1161 /*----------------------------------------*/
1162     mtctr   T8
1163     LOAD2x4_2
1164     MY_ALIGN
1165 CGEMM_L2x4_LOOP:
1166 /*----------------------------------------*/
1167     KERNEL2x4_L2 64,32,0,0
1168 CGEMM_L2x4_K32:
1169 /*----------------------------------------*/
1170     KERNEL2x4_L2 64,32,1,0
1171     KERNEL2x4_L2 64,32,2,0
1172     KERNEL2x4_L2 64,32,3,0
1173     KERNEL2x4_L2 64,32,4,0
1174     KERNEL2x4_L2 64,32,5,0
1175     KERNEL2x4_L2 64,32,6,0
1176     KERNEL2x4_L2 64,32,7,0
1177     KERNEL2x4_L2 64,32,8,0
1178     KERNEL2x4_L2 64,32,9,0
1179     KERNEL2x4_L2 64,32,10,0
1180     KERNEL2x4_L2 64,32,11,0
1181     KERNEL2x4_L2 64,32,12,0
1182     KERNEL2x4_L2 64,32,13,0
1183     KERNEL2x4_L2 64,32,14,0
1184     KERNEL2x4_L2 64,32,15,1
1185     bdnz    CGEMM_L2x4_LOOP
1186     MY_ALIGN
1187 CGEMM_L2x4_LOOP_END:
1188 /*----------------------------------------*/
1189     END2x4_2
1190     blr
1191     MY_ALIGN
1192
1193
1194 CGEMM_2x4_L16_SUB:
1195 /*----------------------------------------*/
1196     LOAD2x4_2
1197     KERNEL2x4_L2 64,32,0,0
1198     KERNEL2x4_L2 64,32,1,0
1199     KERNEL2x4_L2 64,32,2,0
1200     KERNEL2x4_L2 64,32,3,0
1201     KERNEL2x4_L2 64,32,4,0
1202     KERNEL2x4_L2 64,32,5,0
1203     KERNEL2x4_L2 64,32,6,0
1204     KERNEL2x4_E2 64,32,7,1
1205     blr
1206     MY_ALIGN
1207
1208
1209 CGEMM_2x4_L8_SUB:
1210 /*----------------------------------------*/
1211     LOAD2x4_2
1212     KERNEL2x4_L2 64,32,0,0
1213     KERNEL2x4_L2 64,32,1,0
1214     KERNEL2x4_L2 64,32,2,0
1215     KERNEL2x4_E2 64,32,3,1
1216     blr
1217
1218
1219 CGEMM_2x2_LMAIN_SUB:
1220 /*----------------------------------------*/
1221     mtctr   T8
1222     LOAD2x2_2
1223     MY_ALIGN
1224 CGEMM_L2x2_LOOP:
1225 /*----------------------------------------*/
1226     KERNEL2x2_L2 32,32,0,0
1227 CGEMM_L2x2_K32:
1228 /*----------------------------------------*/
1229     KERNEL2x2_L2 32,32,1,0
1230     KERNEL2x2_L2 32,32,2,0
1231     KERNEL2x2_L2 32,32,3,0
1232     KERNEL2x2_L2 32,32,4,0
1233     KERNEL2x2_L2 32,32,5,0
1234     KERNEL2x2_L2 32,32,6,0
1235     KERNEL2x2_L2 32,32,7,0
1236     KERNEL2x2_L2 32,32,8,0
1237     KERNEL2x2_L2 32,32,9,0
1238     KERNEL2x2_L2 32,32,10,0
1239     KERNEL2x2_L2 32,32,11,0
1240     KERNEL2x2_L2 32,32,12,0
1241     KERNEL2x2_L2 32,32,13,0
1242     KERNEL2x2_L2 32,32,14,0
1243     KERNEL2x2_L2 32,32,15,1
1244     bdnz    CGEMM_L2x2_LOOP
1245     MY_ALIGN
1246
1247
1248 CGEMM_L2x2_LOOP_END:
1249 /*----------------------------------------*/
1250     END2x2_2
1251     blr
1252     MY_ALIGN
1253 CGEMM_2x2_L16_SUB:
1254 /*----------------------------------------*/
1255     LOAD2x2_2
1256     KERNEL2x2_L2 32,32,0,0
1257     KERNEL2x2_L2 32,32,1,0
1258     KERNEL2x2_L2 32,32,2,0
1259     KERNEL2x2_L2 32,32,3,0
1260     KERNEL2x2_L2 32,32,4,0
1261     KERNEL2x2_L2 32,32,5,0
1262     KERNEL2x2_L2 32,32,6,0
1263     KERNEL2x2_E2 32,32,7,1
1264     blr
1265     MY_ALIGN
1266 CGEMM_2x2_L8_SUB:
1267 /*----------------------------------------*/
1268     LOAD2x2_2
1269     KERNEL2x2_L2 32,32,0,0
1270     KERNEL2x2_L2 32,32,1,0
1271     KERNEL2x2_L2 32,32,2,0
1272     KERNEL2x2_E2 32,32,3,1
1273     blr
1274
1275
1276 CGEMM_2x1_LMAIN_SUB:
1277 /*----------------------------------------*/
1278     mtctr   T8
1279     LOAD2x1_2
1280     MY_ALIGN
1281 CGEMM_L2x1_LOOP:
1282 /*----------------------------------------*/
1283     KERNEL2x1_L2 16,32,0,0
1284 CGEMM_L2x1_K32:
1285 /*----------------------------------------*/
1286     KERNEL2x1_L2 16,32,1,0
1287     KERNEL2x1_L2 16,32,2,0
1288     KERNEL2x1_L2 16,32,3,0
1289     KERNEL2x1_L2 16,32,4,0
1290     KERNEL2x1_L2 16,32,5,0
1291     KERNEL2x1_L2 16,32,6,0
1292     KERNEL2x1_L2 16,32,7,0
1293     KERNEL2x1_L2 16,32,8,0
1294     KERNEL2x1_L2 16,32,9,0
1295     KERNEL2x1_L2 16,32,10,0
1296     KERNEL2x1_L2 16,32,11,0
1297     KERNEL2x1_L2 16,32,12,0
1298     KERNEL2x1_L2 16,32,13,0
1299     KERNEL2x1_L2 16,32,14,0
1300     KERNEL2x1_L2 16,32,15,1
1301     bdnz    CGEMM_L2x1_LOOP
1302     MY_ALIGN
1303 CGEMM_L2x1_LOOP_END:
1304 /*----------------------------------------*/
1305     END2x1_2
1306     blr
1307
1308     MY_ALIGN
1309 CGEMM_2x1_L16_SUB:
1310 /*----------------------------------------*/
1311     LOAD2x1_2
1312     KERNEL2x1_L2 16,32,0,0
1313     KERNEL2x1_L2 16,32,1,0
1314     KERNEL2x1_L2 16,32,2,0
1315     KERNEL2x1_L2 16,32,3,0
1316     KERNEL2x1_L2 16,32,4,0
1317     KERNEL2x1_L2 16,32,5,0
1318     KERNEL2x1_L2 16,32,6,0
1319     KERNEL2x1_E2 16,32,7,1
1320     blr
1321     MY_ALIGN
1322
1323
1324 CGEMM_2x1_L8_SUB:
1325 /*----------------------------------------*/
1326     LOAD2x1_2
1327     KERNEL2x1_L2 16,32,0,0
1328     KERNEL2x1_L2 16,32,1,0
1329     KERNEL2x1_L2 16,32,2,0
1330     KERNEL2x1_E2 16,32,3,1
1331     blr
1332
1333
1334
1335 /*             MAIN LOOP BEGINS               */
1336     MY_ALIGN
1337
1338
1339 CGEMM_L2:
1340 /*----------------------------------------*/
1341
1342     andi.    J,  N,  2
1343     ble   CGEMM_L2_END
1344
1345
1346 CGEMM_L2_BEGIN:
1347 /*----------------------------------------*/
1348     mr    CO, C
1349     slwi    T1, LDC , 1
1350     add     T2,C,LDC
1351     mr    AO, A
1352     add   C,  C,  T1
1353 #if defined(TRMMKERNEL) && defined(LEFT)
1354     mr TEMP_REG, OFFSET  /*off = offset;*/
1355 #endif
1356     srawi.    I,  M, 3
1357     ble   CGEMM_L2x8_END
1358     dcbt    CO,r0  /*just prefetch*/
1359     dcbt    T2,r0
1360
1361
1362 CGEMM_L2x8_BEGIN:
1363 /*----------------------------------------*/
1364 #if defined(TRMMKERNEL)
1365     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2
1366 #else
1367     mr    BO, B
1368     dcbt    B,  r0
1369 #endif
1370     dcbt    AO, r0
1371 #if defined(TRMMKERNEL)
1372     REFRESH_TEMP_BK T6,K,TEMP_REG,8,2
1373     mr T1, T6
1374 /* TEMPS FOR PREFETCH */
1375     li T2, 1024
1376     li T3, 1024+512
1377     addi T1,T1, -2
1378 /* TEMPS FOR PREFETCH */
1379     li T4, 2048
1380     li T5, 2048+512
1381     srawi.   T8, T1, 7 /**(T1-2) % 128x */
1382 #else
1383     mr T1, K
1384 /* TEMPS FOR PREFETCH */
1385     li T2, 1024
1386     li T3, 1024+512
1387     addi T1,T1, -2
1388 /* TEMPS FOR PREFETCH */
1389     li T4, 2048
1390     li T5, 2048+512
1391     srawi.   T8, T1, 7 /**(K-2) % 128x */
1392 #endif
1393     ZERO2x8
1394     ble   CGEMM_L2x8_SUB0
1395     bl CGEMM_L2x8_LMAIN_SUB
1396     andi.   L,  T1, 127
1397     ble   CGEMM_L2x8_SAVE
1398     b   CGEMM_L2x8_SUB2
1399
1400
1401 CGEMM_L2x8_SUB0:
1402 /*----------------------------------------*/
1403 #if defined(TRMMKERNEL)
1404     andi.   L,  T6, 255
1405     cmpwi   T6,129
1406 #else
1407     andi.   L,  K,  255
1408     cmpwi   K,129
1409 #endif
1410     li T8,1
1411     bne CMP2x8_128K
1412     addi BO,BO,-16
1413     addi AO,AO,-64
1414     LOAD2x8O 64,16
1415     END2x8_WITHOUT_ADD
1416     LOAD2x8_2O  128, 32
1417     mtctr   T8
1418     bl CGEMM_L2x8_K128
1419     b CGEMM_L2x8_SAVE
1420     CMP2x8_128K:
1421 /*----------------------------------------*/
1422 #if defined(TRMMKERNEL)
1423     cmpwi   T6,128
1424 #else
1425     cmpwi   K,128
1426 #endif
1427     bne CGEMM_L2x8_SUB2
1428     MY_ALIGN
1429     mtctr   T8
1430     addi BO,BO,-32
1431     addi AO,AO,-128
1432     LOAD2x8_2O 128,32
1433     bl CGEMM_L2x8_K128
1434     b CGEMM_L2x8_SAVE
1435     MY_ALIGN
1436
1437
1438 CGEMM_L2x8_SUB2:
1439 /*----------------------------------------*/
1440     andi.      T1,L, 64
1441     ble CGEMM_L2x8_SUB2_32
1442     bl  CGEMM_2x8_L64_SUB
1443     MY_ALIGN
1444
1445
1446 CGEMM_L2x8_SUB2_32:
1447 /*----------------------------------------*/
1448     andi.      T1,L, 32
1449     ble CGEMM_L2x8_SUB2_16
1450     bl  CGEMM_2x8_L32_SUB
1451     MY_ALIGN
1452
1453
1454 CGEMM_L2x8_SUB2_16:
1455 /*----------------------------------------*/
1456     andi.      T1,L, 16
1457     ble CGEMM_L2x8_SUB2_8
1458     bl  CGEMM_2x8_L16_SUB
1459     MY_ALIGN
1460
1461
1462 CGEMM_L2x8_SUB2_8:
1463 /*----------------------------------------*/
1464     andi.      T1,L, 8
1465     ble CGEMM_L2x8_SUB2_4
1466     LOAD2x8_2
1467     KERNEL2x8_L2  128,32, 0,0
1468     KERNEL2x8_L2  128,32, 1,0
1469     KERNEL2x8_L2  128,32, 2,0
1470     KERNEL2x8_E2  128,32, 3,1
1471     MY_ALIGN
1472
1473
1474 CGEMM_L2x8_SUB2_4:
1475 /*----------------------------------------*/
1476     andi.      T1,L, 4
1477     ble CGEMM_L2x8_SUB2_2
1478     LOAD2x8_2
1479     KERNEL2x8_L2  128,32, 0,0
1480     KERNEL2x8_E2  128,32, 1,1
1481     MY_ALIGN
1482
1483
1484 CGEMM_L2x8_SUB2_2:
1485 /*----------------------------------------*/
1486     andi.      T1,L, 2
1487     ble CGEMM_L2x8_SUB2_1
1488     LOAD2x8_2
1489     KERNEL2x8_E2  128,32, 0,1
1490     MY_ALIGN
1491
1492
1493 CGEMM_L2x8_SUB2_1:
1494 /*----------------------------------------*/
1495     andi.      T1,L, 1
1496     ble CGEMM_L2x8_SAVE
1497     KERNEL2x8
1498
1499     MY_ALIGN
1500 CGEMM_L2x8_SAVE:
1501 /*----------------------------------------*/
1502     addic.    I,  I,  -1
1503     MY_ALIGN
1504     SAVE2x8
1505 #if defined(TRMMKERNEL)
1506     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,2
1507 #endif
1508     bgt   CGEMM_L2x8_BEGIN
1509     andi.   T2, M,  7
1510     ble   CGEMM_L2x1_END
1511     andi.   T1, M,  4
1512     ble   CGEMM_L2x4_END
1513     b   CGEMM_L2x4_BEGIN
1514     MY_ALIGN
1515
1516
1517 CGEMM_L2x8_END:
1518 /*----------------------------------------*/
1519
1520
1521 CGEMM_L2x4_BEGIN:
1522 /*----------------------------------------*/
1523     andi.   T2, M,  7
1524     ble   CGEMM_L2x1_END
1525     andi.   T1, M,  4
1526     ble   CGEMM_L2x4_END
1527 #if defined(TRMMKERNEL)
1528     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2
1529 #else
1530     mr    BO, B
1531 #endif
1532 #if defined(TRMMKERNEL)
1533     REFRESH_TEMP_BK T6,K,TEMP_REG,4,2
1534     mr T1, T6
1535     addi T1,T1, -2
1536     srawi.   T8, T1, 5 /**(T1-2) % 32x */
1537 #else
1538     mr T1, K
1539     addi T1,T1, -2
1540     srawi.   T8, T1, 5 /**(K-2) % 32x */
1541 #endif
1542     ZERO2x4
1543     ble   CGEMM_L2x4_SUB0
1544     bl CGEMM_2x4_LMAIN_SUB
1545     andi.   L,  T1, 31
1546     ble   CGEMM_L2x4_SAVE
1547     b    CGEMM_L2x4_SUB2
1548
1549
1550 CGEMM_L2x4_SUB0:
1551 /*----------------------------------------*/
1552 #if defined(TRMMKERNEL)
1553     andi.   L,  T6, 63
1554     cmpwi   T6,33
1555 #else
1556     andi.   L,  K,  63
1557     cmpwi   K,33
1558 #endif
1559     li T8,1
1560     bne CMP2x4_32K
1561     addi BO,BO,-16
1562     addi AO,AO,-32
1563     LOAD2x4O 32,16
1564     END2x4_WITHOUT_ADD
1565     LOAD2x4_2O  64, 32
1566     mtctr   T8
1567     bl CGEMM_L2x4_K32
1568     b CGEMM_L2x4_SAVE
1569     CMP2x4_32K:
1570 /*----------------------------------------*/
1571 #if defined(TRMMKERNEL)
1572     cmpwi   T6,32
1573 #else
1574     cmpwi   K,32
1575 #endif
1576     bne CGEMM_L2x4_SUB2
1577     MY_ALIGN
1578     mtctr   T8
1579     addi BO,BO,-32
1580     addi AO,AO,-64
1581     LOAD2x4_2O 64,32
1582     bl CGEMM_L2x4_K32
1583     b CGEMM_L2x4_SAVE
1584     MY_ALIGN
1585     MY_ALIGN
1586
1587
1588 CGEMM_L2x4_SUB2:
1589 /*----------------------------------------*/
1590     andi.      T1,L, 16
1591     ble CGEMM_L2x4_SUB2_8
1592     bl  CGEMM_2x4_L16_SUB
1593     MY_ALIGN
1594
1595
1596 CGEMM_L2x4_SUB2_8:
1597 /*----------------------------------------*/
1598     andi.      T1,L, 8
1599     ble CGEMM_L2x4_SUB2_4
1600     bl CGEMM_2x4_L8_SUB
1601     MY_ALIGN
1602
1603
1604 CGEMM_L2x4_SUB2_4:
1605 /*----------------------------------------*/
1606     andi.      T1,L, 4
1607     ble CGEMM_L2x4_SUB2_2
1608     LOAD2x4_2
1609     KERNEL2x4_L2  64,32, 0,0
1610     KERNEL2x4_E2  64,32, 1,1
1611     MY_ALIGN
1612
1613
1614 CGEMM_L2x4_SUB2_2:
1615 /*----------------------------------------*/
1616     andi.      T1,L, 2
1617     ble CGEMM_L2x4_SUB2_1
1618     LOAD2x4_2
1619     KERNEL2x4_E2  64,32, 0,1
1620     MY_ALIGN
1621
1622
1623 CGEMM_L2x4_SUB2_1:
1624 /*----------------------------------------*/
1625     andi.      T1,L, 1
1626     ble CGEMM_L2x4_SAVE
1627     KERNEL2x4
1628
1629
1630 CGEMM_L2x4_SAVE:
1631 /*----------------------------------------*/
1632     SAVE2x4
1633 #if defined(TRMMKERNEL)
1634     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,2
1635 #endif
1636
1637
1638 CGEMM_L2x4_END:
1639 /*----------------------------------------*/
1640
1641
1642 CGEMM_L2x2_BEGIN:
1643 /*----------------------------------------*/
1644     andi.   T1, M,  2
1645     ble   CGEMM_L2x2_END
1646 #if defined(TRMMKERNEL)
1647     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2
1648 #else
1649     mr    BO, B
1650 #endif
1651 #if defined(TRMMKERNEL)
1652     REFRESH_TEMP_BK T6,K,TEMP_REG,2,2
1653     mr T1, T6
1654     addi T1,T1, -2
1655     srawi.   T8, T1, 5 /**(T1-2) % 32x */
1656 #else
1657     mr T1, K
1658     addi T1,T1, -2
1659     srawi.   T8, T1, 5 /**(K-2) % 32x */
1660 #endif
1661     ZERO2x2
1662     ble   CGEMM_L2x2_SUB0
1663     bl CGEMM_2x2_LMAIN_SUB
1664     andi.   L,  T1, 31
1665     ble   CGEMM_L2x2_SAVE
1666     b   CGEMM_L2x2_SUB2
1667
1668
1669 CGEMM_L2x2_SUB0:
1670 /*----------------------------------------*/
1671 #if defined(TRMMKERNEL)
1672     andi.   L,  T6, 63
1673     cmpwi   T6,33
1674 #else
1675     andi.   L,  K,  63
1676     cmpwi   K,33
1677 #endif
1678     li T8,1
1679     bne CMP2x2_32K
1680     addi BO,BO,-16
1681     addi AO,AO,-16
1682     LOAD2x2O 16,16
1683     END2x2_WITHOUT_ADD
1684     LOAD2x2_2O  32, 32
1685     mtctr   T8
1686     bl CGEMM_L2x2_K32
1687     b CGEMM_L2x2_SAVE
1688     CMP2x2_32K:
1689 /*----------------------------------------*/
1690 #if defined(TRMMKERNEL)
1691     cmpwi   T6,32
1692 #else
1693     cmpwi   K,32
1694 #endif
1695     bne CGEMM_L2x2_SUB2
1696     MY_ALIGN
1697     mtctr   T8
1698     addi BO,BO,-32
1699     addi AO,AO,-32
1700     LOAD2x2_2O 32,32
1701     bl CGEMM_L2x2_K32
1702     b CGEMM_L2x2_SAVE
1703     MY_ALIGN
1704     MY_ALIGN
1705
1706
1707 CGEMM_L2x2_SUB2:
1708 /*----------------------------------------*/
1709     andi.      T1,L, 16
1710     ble CGEMM_L2x2_SUB2_8
1711     bl CGEMM_2x2_L16_SUB
1712     MY_ALIGN
1713
1714
1715 CGEMM_L2x2_SUB2_8:
1716 /*----------------------------------------*/
1717     andi.      T1,L, 8
1718     ble CGEMM_L2x2_SUB2_4
1719     bl CGEMM_2x2_L8_SUB
1720     MY_ALIGN
1721
1722
1723 CGEMM_L2x2_SUB2_4:
1724 /*----------------------------------------*/
1725     andi.      T1,L, 4
1726     ble CGEMM_L2x2_SUB2_2
1727     LOAD2x2_2
1728     KERNEL2x2_L2  32,32, 0,0
1729     KERNEL2x2_E2  32,32, 1,1
1730     MY_ALIGN
1731
1732
1733 CGEMM_L2x2_SUB2_2:
1734 /*----------------------------------------*/
1735     andi.      T1,L, 2
1736     ble CGEMM_L2x2_SUB2_1
1737     LOAD2x2_2
1738     KERNEL2x2_E2  32,32, 0,1
1739     MY_ALIGN
1740
1741
1742 CGEMM_L2x2_SUB2_1:
1743 /*----------------------------------------*/
1744     andi.      T1,L, 1
1745     ble CGEMM_L2x2_SAVE
1746     KERNEL2x2
1747
1748     MY_ALIGN
1749 CGEMM_L2x2_SAVE:
1750 /*----------------------------------------*/
1751     SAVE2x2
1752 #if defined(TRMMKERNEL)
1753     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,2
1754 #endif
1755
1756
1757 CGEMM_L2x2_END:
1758 /*----------------------------------------*/
1759
1760
1761 CGEMM_L2x1_BEGIN:
1762 /*----------------------------------------*/
1763     andi.   T1, M,  1
1764     ble   CGEMM_L2x1_END
1765 #if defined(TRMMKERNEL)
1766     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2
1767 #else
1768     mr    BO, B
1769 #endif
1770 #if defined(TRMMKERNEL)
1771     REFRESH_TEMP_BK T6,K,TEMP_REG,1,2
1772     mr T1, T6
1773     addi T1,T1, -2
1774     srawi.   T8, T1, 5 /**(T1-2) % 32x */
1775 #else
1776     mr T1, K
1777     addi T1,T1, -2
1778     srawi.   T8, T1, 5 /**(K-2) % 32x */
1779 #endif
1780     ZERO2x1
1781     ble   CGEMM_L2x1_SUB0
1782     bl CGEMM_2x1_LMAIN_SUB
1783     andi.   L,  T1, 31
1784     ble   CGEMM_L2x1_SAVE
1785     b   CGEMM_L2x1_SUB2
1786
1787
1788 CGEMM_L2x1_SUB0:
1789 /*----------------------------------------*/
1790 #if defined(TRMMKERNEL)
1791     andi.   L,  T6, 63
1792     cmpwi   T6,33
1793 #else
1794     andi.   L,  K,  63
1795     cmpwi   K,33
1796 #endif
1797     li T8,1
1798     bne CMP2x1_32K
1799     addi BO,BO,-16
1800     addi AO,AO,-8
1801     LOAD2x1O 8,16
1802     END2x1_WITHOUT_ADD
1803     LOAD2x1_2O  16, 32
1804     mtctr   T8
1805     bl CGEMM_L2x1_K32
1806     b CGEMM_L2x1_SAVE
1807     CMP2x1_32K:
1808 /*----------------------------------------*/
1809 #if defined(TRMMKERNEL)
1810     cmpwi   T6,32
1811 #else
1812     cmpwi   K,32
1813 #endif
1814     bne CGEMM_L2x1_SUB2
1815     MY_ALIGN
1816     mtctr   T8
1817     addi BO,BO,-32
1818     addi AO,AO,-16
1819     LOAD2x1_2O 16,32
1820     bl CGEMM_L2x1_K32
1821     b CGEMM_L2x1_SAVE
1822     MY_ALIGN
1823     MY_ALIGN
1824
1825
1826 CGEMM_L2x1_SUB2:
1827 /*----------------------------------------*/
1828     andi.      T1,L, 16
1829     ble CGEMM_L2x1_SUB2_8
1830     bl CGEMM_2x1_L16_SUB
1831     MY_ALIGN
1832
1833
1834 CGEMM_L2x1_SUB2_8:
1835 /*----------------------------------------*/
1836     andi.      T1,L, 8
1837     ble CGEMM_L2x1_SUB2_4
1838     bl CGEMM_2x1_L8_SUB
1839     MY_ALIGN
1840
1841
1842 CGEMM_L2x1_SUB2_4:
1843 /*----------------------------------------*/
1844     andi.      T1,L, 4
1845     ble CGEMM_L2x1_SUB2_2
1846     LOAD2x1_2
1847     KERNEL2x1_L2  16,32, 0,0
1848     KERNEL2x1_E2  16,32, 1,1
1849     MY_ALIGN
1850
1851
1852 CGEMM_L2x1_SUB2_2:
1853 /*----------------------------------------*/
1854     andi.      T1,L, 2
1855     ble CGEMM_L2x1_SUB2_1
1856     LOAD2x1_2
1857     KERNEL2x1_E2  16,32, 0,1
1858     MY_ALIGN
1859
1860
1861 CGEMM_L2x1_SUB2_1:
1862 /*----------------------------------------*/
1863     andi.      T1,L, 1
1864     ble CGEMM_L2x1_SAVE
1865     KERNEL2x1
1866
1867     MY_ALIGN
1868 CGEMM_L2x1_SAVE:
1869 /*----------------------------------------*/
1870
1871     SAVE2x1
1872 #if defined(TRMMKERNEL)
1873     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,2
1874 #endif
1875
1876
1877 CGEMM_L2x1_END:
1878 /*----------------------------------------*/
1879     slwi    T1, K,  4
1880
1881     add   B,  B,  T1
1882 #if defined(TRMMKERNEL) && !defined(LEFT)
1883     addi TEMP_REG, TEMP_REG, 2
1884 #endif
1885
1886 CGEMM_L2_END:
1887
1888
1889 b CGEMM_L1
1890 /*                MINI SUBROUTINES                            */
1891 /*                1x8 MAIN 128x+2 LOOP                     */
1892
1893
1894 CGEMM_L1x8_LMAIN_SUB:
1895 /*----------------------------------------*/
1896     mtctr   T8
1897     LOAD1x8_2
1898     MY_ALIGN
1899 CGEMM_L1x8_LOOP:
1900 /*----------------------------------------*/
1901     dcbt    AO, PRE
1902     dcbt    BO, PRE
1903     KERNEL1x8_L2 128,16,0,0
1904 CGEMM_L1x8_K128:
1905 /*----------------------------------------*/
1906     KERNEL1x8_L2 128,16,1,0
1907     dcbt    AO, T2
1908     KERNEL1x8_L2 128,16,2,0
1909     KERNEL1x8_L2 128,16,3,0
1910     dcbt    AO, T3
1911     dcbt    BO, T2
1912     KERNEL1x8_L2 128,16,4,0
1913     KERNEL1x8_L2 128,16,5,0
1914     dcbt    AO, T4
1915     KERNEL1x8_L2 128,16,6,0
1916     KERNEL1x8_L2 128,16,7,0
1917     dcbt    AO, T5
1918     dcbt    BO, T3
1919     KERNEL1x8_L2 128,16,8,0
1920     KERNEL1x8_L2 128,16,9,0
1921     KERNEL1x8_L2 128,16,10,0
1922     KERNEL1x8_L2 128,16,11,0
1923     dcbt    BO, T4
1924     KERNEL1x8_L2 128,16,12,0
1925     KERNEL1x8_L2 128,16,13,0
1926     KERNEL1x8_L2 128,16,14,0
1927     KERNEL1x8_L2 128,16,15,0
1928     KERNEL1x8_L2 128,16,16,0
1929     KERNEL1x8_L2 128,16,17,0
1930     KERNEL1x8_L2 128,16,18,0
1931     KERNEL1x8_L2 128,16,19,0
1932     KERNEL1x8_L2 128,16,20,0
1933     KERNEL1x8_L2 128,16,21,0
1934     KERNEL1x8_L2 128,16,22,0
1935     KERNEL1x8_L2 128,16,23,0
1936     KERNEL1x8_L2 128,16,24,0
1937     KERNEL1x8_L2 128,16,25,0
1938     KERNEL1x8_L2 128,16,26,0
1939     KERNEL1x8_L2 128,16,27,0
1940     KERNEL1x8_L2 128,16,28,0
1941     KERNEL1x8_L2 128,16,29,0
1942     KERNEL1x8_L2 128,16,30,0
1943     KERNEL1x8_L2 128,16,31,0
1944     KERNEL1x8_L2 128,16,32,0
1945     KERNEL1x8_L2 128,16,33,0
1946     KERNEL1x8_L2 128,16,34,0
1947     KERNEL1x8_L2 128,16,35,0
1948     KERNEL1x8_L2 128,16,36,0
1949     KERNEL1x8_L2 128,16,37,0
1950     KERNEL1x8_L2 128,16,38,0
1951     KERNEL1x8_L2 128,16,39,0
1952     KERNEL1x8_L2 128,16,40,0
1953     KERNEL1x8_L2 128,16,41,0
1954     KERNEL1x8_L2 128,16,42,0
1955     KERNEL1x8_L2 128,16,43,0
1956     KERNEL1x8_L2 128,16,44,0
1957     KERNEL1x8_L2 128,16,45,0
1958     KERNEL1x8_L2 128,16,46,0
1959     KERNEL1x8_L2 128,16,47,0
1960     KERNEL1x8_L2 128,16,48,0
1961     KERNEL1x8_L2 128,16,49,0
1962     KERNEL1x8_L2 128,16,50,0
1963     KERNEL1x8_L2 128,16,51,0
1964     KERNEL1x8_L2 128,16,52,0
1965     KERNEL1x8_L2 128,16,53,0
1966     KERNEL1x8_L2 128,16,54,0
1967     KERNEL1x8_L2 128,16,55,0
1968     KERNEL1x8_L2 128,16,56,0
1969     KERNEL1x8_L2 128,16,57,0
1970     KERNEL1x8_L2 128,16,58,0
1971     KERNEL1x8_L2 128,16,59,0
1972     KERNEL1x8_L2 128,16,60,0
1973     KERNEL1x8_L2 128,16,61,0
1974     KERNEL1x8_L2 128,16,62,0
1975     KERNEL1x8_L2 128,16,63,1
1976     bdnz    CGEMM_L1x8_LOOP
1977     MY_ALIGN
1978 CGEMM_L1x8_LOOP_END:
1979 /*----------------------------------------*/
1980     END1x8_2
1981     blr
1982     MY_ALIGN
1983
1984
1985 CGEMM_1x8_L64_SUB:
1986 /*----------------------------------------*/
1987     LOAD1x8_2
1988     dcbt    AO, PRE
1989     dcbt    BO, PRE
1990     KERNEL1x8_L2 128,16,0,0
1991     KERNEL1x8_L2 128,16,1,0
1992     dcbt    AO, T2
1993     KERNEL1x8_L2 128,16,2,0
1994     KERNEL1x8_L2 128,16,3,0
1995     dcbt    AO, T3
1996     dcbt    BO, T2
1997     KERNEL1x8_L2 128,16,4,0
1998     KERNEL1x8_L2 128,16,5,0
1999     dcbt    AO, T4
2000     KERNEL1x8_L2 128,16,6,0
2001     KERNEL1x8_L2 128,16,7,0
2002     dcbt    AO, T5
2003     dcbt    BO, T3
2004     KERNEL1x8_L2 128,16,8,0
2005     KERNEL1x8_L2 128,16,9,0
2006     KERNEL1x8_L2 128,16,10,0
2007     KERNEL1x8_L2 128,16,11,0
2008     dcbt    BO, T4
2009     KERNEL1x8_L2 128,16,12,0
2010     KERNEL1x8_L2 128,16,13,0
2011     KERNEL1x8_L2 128,16,14,0
2012     KERNEL1x8_L2 128,16,15,0
2013     KERNEL1x8_L2 128,16,16,0
2014     KERNEL1x8_L2 128,16,17,0
2015     KERNEL1x8_L2 128,16,18,0
2016     KERNEL1x8_L2 128,16,19,0
2017     KERNEL1x8_L2 128,16,20,0
2018     KERNEL1x8_L2 128,16,21,0
2019     KERNEL1x8_L2 128,16,22,0
2020     KERNEL1x8_L2 128,16,23,0
2021     KERNEL1x8_L2 128,16,24,0
2022     KERNEL1x8_L2 128,16,25,0
2023     KERNEL1x8_L2 128,16,26,0
2024     KERNEL1x8_L2 128,16,27,0
2025     KERNEL1x8_L2 128,16,28,0
2026     KERNEL1x8_L2 128,16,29,0
2027     KERNEL1x8_L2 128,16,30,0
2028     KERNEL1x8_E2 128,16,31,1
2029     blr
2030     MY_ALIGN
2031
2032
2033 CGEMM_1x8_L32_SUB:
2034 /*----------------------------------------*/
2035     LOAD1x8_2
2036     dcbt    AO, PRE
2037     dcbt    BO, PRE
2038     KERNEL1x8_L2 128,16,0,0
2039     KERNEL1x8_L2 128,16,1,0
2040     dcbt    AO, T2
2041     KERNEL1x8_L2 128,16,2,0
2042     KERNEL1x8_L2 128,16,3,0
2043     dcbt    AO, T3
2044     dcbt    BO, T2
2045     KERNEL1x8_L2 128,16,4,0
2046     KERNEL1x8_L2 128,16,5,0
2047     dcbt    AO, T4
2048     KERNEL1x8_L2 128,16,6,0
2049     KERNEL1x8_L2 128,16,7,0
2050     dcbt    AO, T5
2051     dcbt    BO, T3
2052     KERNEL1x8_L2 128,16,8,0
2053     KERNEL1x8_L2 128,16,9,0
2054     KERNEL1x8_L2 128,16,10,0
2055     KERNEL1x8_L2 128,16,11,0
2056     dcbt    BO, T4
2057     KERNEL1x8_L2 128,16,12,0
2058     KERNEL1x8_L2 128,16,13,0
2059     KERNEL1x8_L2 128,16,14,0
2060     KERNEL1x8_E2 128,16,15,1
2061     blr
2062     MY_ALIGN
2063
2064
2065 CGEMM_1x8_L16_SUB:
2066 /*----------------------------------------*/
2067     LOAD1x8_2
2068     dcbt    AO, PRE
2069     dcbt    BO, PRE
2070     KERNEL1x8_L2 128,16,0,0
2071     KERNEL1x8_L2 128,16,1,0
2072     dcbt    AO, T2
2073     KERNEL1x8_L2 128,16,2,0
2074     KERNEL1x8_L2 128,16,3,0
2075     dcbt    AO, T3
2076     dcbt    BO, T2
2077     KERNEL1x8_L2 128,16,4,0
2078     KERNEL1x8_L2 128,16,5,0
2079     dcbt    AO, T4
2080     KERNEL1x8_L2 128,16,6,0
2081     KERNEL1x8_E2 128,16,7,1
2082     blr
2083     MY_ALIGN
2084
2085
2086 CGEMM_1x4_LMAIN_SUB:
2087 /*----------------------------------------*/
2088     mtctr   T8
2089     LOAD1x4_2
2090     MY_ALIGN
2091 CGEMM_L1x4_LOOP:
2092 /*----------------------------------------*/
2093     KERNEL1x4_L2 64,16,0,0
2094 CGEMM_L1x4_K32:
2095 /*----------------------------------------*/
2096     KERNEL1x4_L2 64,16,1,0
2097     KERNEL1x4_L2 64,16,2,0
2098     KERNEL1x4_L2 64,16,3,0
2099     KERNEL1x4_L2 64,16,4,0
2100     KERNEL1x4_L2 64,16,5,0
2101     KERNEL1x4_L2 64,16,6,0
2102     KERNEL1x4_L2 64,16,7,0
2103     KERNEL1x4_L2 64,16,8,0
2104     KERNEL1x4_L2 64,16,9,0
2105     KERNEL1x4_L2 64,16,10,0
2106     KERNEL1x4_L2 64,16,11,0
2107     KERNEL1x4_L2 64,16,12,0
2108     KERNEL1x4_L2 64,16,13,0
2109     KERNEL1x4_L2 64,16,14,0
2110     KERNEL1x4_L2 64,16,15,1
2111     bdnz    CGEMM_L1x4_LOOP
2112     MY_ALIGN
2113 CGEMM_L1x4_LOOP_END:
2114 /*----------------------------------------*/
2115     END1x4_2
2116     blr
2117     MY_ALIGN
2118
2119
2120 CGEMM_1x4_L16_SUB:
2121 /*----------------------------------------*/
2122     LOAD1x4_2
2123     KERNEL1x4_L2 64,16,0,0
2124     KERNEL1x4_L2 64,16,1,0
2125     KERNEL1x4_L2 64,16,2,0
2126     KERNEL1x4_L2 64,16,3,0
2127     KERNEL1x4_L2 64,16,4,0
2128     KERNEL1x4_L2 64,16,5,0
2129     KERNEL1x4_L2 64,16,6,0
2130     KERNEL1x4_E2 64,16,7,1
2131     blr
2132     MY_ALIGN
2133
2134
2135 CGEMM_1x4_L8_SUB:
2136 /*----------------------------------------*/
2137     LOAD1x4_2
2138     KERNEL1x4_L2 64,16,0,0
2139     KERNEL1x4_L2 64,16,1,0
2140     KERNEL1x4_L2 64,16,2,0
2141     KERNEL1x4_E2 64,16,3,1
2142     blr
2143
2144
2145 CGEMM_1x2_LMAIN_SUB:
2146 /*----------------------------------------*/
2147     mtctr   T8
2148     LOAD1x2_2
2149     MY_ALIGN
2150 CGEMM_L1x2_LOOP:
2151 /*----------------------------------------*/
2152     KERNEL1x2_L2 32,16,0,0
2153 CGEMM_L1x2_K32:
2154 /*----------------------------------------*/
2155     KERNEL1x2_L2 32,16,1,0
2156     KERNEL1x2_L2 32,16,2,0
2157     KERNEL1x2_L2 32,16,3,0
2158     KERNEL1x2_L2 32,16,4,0
2159     KERNEL1x2_L2 32,16,5,0
2160     KERNEL1x2_L2 32,16,6,0
2161     KERNEL1x2_L2 32,16,7,0
2162     KERNEL1x2_L2 32,16,8,0
2163     KERNEL1x2_L2 32,16,9,0
2164     KERNEL1x2_L2 32,16,10,0
2165     KERNEL1x2_L2 32,16,11,0
2166     KERNEL1x2_L2 32,16,12,0
2167     KERNEL1x2_L2 32,16,13,0
2168     KERNEL1x2_L2 32,16,14,0
2169     KERNEL1x2_L2 32,16,15,1
2170     bdnz    CGEMM_L1x2_LOOP
2171     MY_ALIGN
2172
2173
2174 CGEMM_L1x2_LOOP_END:
2175 /*----------------------------------------*/
2176     END1x2_2
2177     blr
2178     MY_ALIGN
2179 CGEMM_1x2_L16_SUB:
2180 /*----------------------------------------*/
2181     LOAD1x2_2
2182     KERNEL1x2_L2 32,16,0,0
2183     KERNEL1x2_L2 32,16,1,0
2184     KERNEL1x2_L2 32,16,2,0
2185     KERNEL1x2_L2 32,16,3,0
2186     KERNEL1x2_L2 32,16,4,0
2187     KERNEL1x2_L2 32,16,5,0
2188     KERNEL1x2_L2 32,16,6,0
2189     KERNEL1x2_E2 32,16,7,1
2190     blr
2191     MY_ALIGN
2192 CGEMM_1x2_L8_SUB:
2193 /*----------------------------------------*/
2194     LOAD1x2_2
2195     KERNEL1x2_L2 32,16,0,0
2196     KERNEL1x2_L2 32,16,1,0
2197     KERNEL1x2_L2 32,16,2,0
2198     KERNEL1x2_E2 32,16,3,1
2199     blr
2200
2201
2202 CGEMM_1x1_LMAIN_SUB:
2203 /*----------------------------------------*/
2204     mtctr   T8
2205     LOAD1x1_2
2206     MY_ALIGN
2207 CGEMM_L1x1_LOOP:
2208 /*----------------------------------------*/
2209     KERNEL1x1_L2 16,16,0,0
2210 CGEMM_L1x1_K32:
2211 /*----------------------------------------*/
2212     KERNEL1x1_L2 16,16,1,0
2213     KERNEL1x1_L2 16,16,2,0
2214     KERNEL1x1_L2 16,16,3,0
2215     KERNEL1x1_L2 16,16,4,0
2216     KERNEL1x1_L2 16,16,5,0
2217     KERNEL1x1_L2 16,16,6,0
2218     KERNEL1x1_L2 16,16,7,0
2219     KERNEL1x1_L2 16,16,8,0
2220     KERNEL1x1_L2 16,16,9,0
2221     KERNEL1x1_L2 16,16,10,0
2222     KERNEL1x1_L2 16,16,11,0
2223     KERNEL1x1_L2 16,16,12,0
2224     KERNEL1x1_L2 16,16,13,0
2225     KERNEL1x1_L2 16,16,14,0
2226     KERNEL1x1_L2 16,16,15,1
2227     bdnz    CGEMM_L1x1_LOOP
2228     MY_ALIGN
2229 CGEMM_L1x1_LOOP_END:
2230 /*----------------------------------------*/
2231     END1x1_2
2232     blr
2233
2234     MY_ALIGN
2235 CGEMM_1x1_L16_SUB:
2236 /*----------------------------------------*/
2237     LOAD1x1_2
2238     KERNEL1x1_L2 16,16,0,0
2239     KERNEL1x1_L2 16,16,1,0
2240     KERNEL1x1_L2 16,16,2,0
2241     KERNEL1x1_L2 16,16,3,0
2242     KERNEL1x1_L2 16,16,4,0
2243     KERNEL1x1_L2 16,16,5,0
2244     KERNEL1x1_L2 16,16,6,0
2245     KERNEL1x1_E2 16,16,7,1
2246     blr
2247     MY_ALIGN
2248
2249
2250 CGEMM_1x1_L8_SUB:
2251 /*----------------------------------------*/
2252     LOAD1x1_2
2253     KERNEL1x1_L2 16,16,0,0
2254     KERNEL1x1_L2 16,16,1,0
2255     KERNEL1x1_L2 16,16,2,0
2256     KERNEL1x1_E2 16,16,3,1
2257     blr
2258
2259
2260
2261 /*             MAIN LOOP BEGINS               */
2262     MY_ALIGN
2263
2264
2265 CGEMM_L1:
2266 /*----------------------------------------*/
2267
2268     andi.    J,  N,  1
2269     ble   CGEMM_L1_END
2270
2271 CGEMM_L1_BEGIN:
2272 /*----------------------------------------*/
2273     mr    CO, C
2274     add     T2,C,LDC
2275     mr    AO, A
2276     add   C,  C,  T1
2277 #if defined(TRMMKERNEL) && defined(LEFT)
2278     mr TEMP_REG, OFFSET  /*off = offset;*/
2279 #endif
2280     srawi.    I,  M,  3
2281     ble   CGEMM_L1x8_END
2282     dcbt    CO,r0  /*just prefetch*/
2283     dcbt    T2,r0
2284
2285
2286 CGEMM_L1x8_BEGIN:
2287 /*----------------------------------------*/
2288 #if defined(TRMMKERNEL)
2289     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1
2290 #else
2291     mr    BO, B
2292     dcbt    B,  r0
2293 #endif
2294     dcbt    AO, r0
2295 #if defined(TRMMKERNEL)
2296     REFRESH_TEMP_BK T6,K,TEMP_REG,8,1
2297     mr T1, T6
2298 /* TEMPS FOR PREFETCH */
2299     li T2, 1024
2300     li T3, 1024+512
2301     addi T1,T1, -2
2302 /* TEMPS FOR PREFETCH */
2303     li T4, 2048
2304     li T5, 2048+512
2305     srawi.   T8, T1, 7 /**(T1-2) % 128x */
2306 #else
2307     mr T1, K
2308 /* TEMPS FOR PREFETCH */
2309     li T2, 1024
2310     li T3, 1024+512
2311     addi T1,T1, -2
2312 /* TEMPS FOR PREFETCH */
2313     li T4, 2048
2314     li T5, 2048+512
2315     srawi.   T8, T1, 7 /**(K-2) % 128x */
2316 #endif
2317     ZERO1x8
2318     ble   CGEMM_L1x8_SUB0
2319     bl CGEMM_L1x8_LMAIN_SUB
2320     andi.   L,  T1, 127
2321     ble   CGEMM_L1x8_SAVE
2322     b   CGEMM_L1x8_SUB2
2323
2324
2325 CGEMM_L1x8_SUB0:
2326 /*----------------------------------------*/
2327 #if defined(TRMMKERNEL)
2328     andi.   L,  T6, 255
2329     cmpwi   T6,129
2330 #else
2331     andi.   L,  K,  255
2332     cmpwi   K,129
2333 #endif
2334     li T8,1
2335     bne CMP1x8_128K
2336     addi BO,BO,-8
2337     addi AO,AO,-64
2338     LOAD1x8O 64,8
2339     END1x8_WITHOUT_ADD
2340     LOAD1x8_2O  128, 16
2341     mtctr   T8
2342     bl CGEMM_L1x8_K128
2343     b CGEMM_L1x8_SAVE
2344     CMP1x8_128K:
2345 /*----------------------------------------*/
2346 #if defined(TRMMKERNEL)
2347     cmpwi   T6,128
2348 #else
2349     cmpwi   K,128
2350 #endif
2351     bne CGEMM_L1x8_SUB2
2352     MY_ALIGN
2353     mtctr   T8
2354     addi BO,BO,-16
2355     addi AO,AO,-128
2356     LOAD1x8_2O 128,16
2357     bl CGEMM_L1x8_K128
2358     b CGEMM_L1x8_SAVE
2359     MY_ALIGN
2360
2361
2362 CGEMM_L1x8_SUB2:
2363 /*----------------------------------------*/
2364     andi.      T1,L, 64
2365     ble CGEMM_L1x8_SUB2_32
2366     bl  CGEMM_1x8_L64_SUB
2367     MY_ALIGN
2368
2369
2370 CGEMM_L1x8_SUB2_32:
2371 /*----------------------------------------*/
2372     andi.      T1,L, 32
2373     ble CGEMM_L1x8_SUB2_16
2374     bl  CGEMM_1x8_L32_SUB
2375     MY_ALIGN
2376
2377
2378 CGEMM_L1x8_SUB2_16:
2379 /*----------------------------------------*/
2380     andi.      T1,L, 16
2381     ble CGEMM_L1x8_SUB2_8
2382     bl  CGEMM_1x8_L16_SUB
2383     MY_ALIGN
2384
2385
2386 CGEMM_L1x8_SUB2_8:
2387 /*----------------------------------------*/
2388     andi.      T1,L, 8
2389     ble CGEMM_L1x8_SUB2_4
2390     LOAD1x8_2
2391     KERNEL1x8_L2  128,16, 0,0
2392     KERNEL1x8_L2  128,16, 1,0
2393     KERNEL1x8_L2  128,16, 2,0
2394     KERNEL1x8_E2  128,16, 3,1
2395     MY_ALIGN
2396
2397
2398 CGEMM_L1x8_SUB2_4:
2399 /*----------------------------------------*/
2400     andi.      T1,L, 4
2401     ble CGEMM_L1x8_SUB2_2
2402     LOAD1x8_2
2403     KERNEL1x8_L2  128,16, 0,0
2404     KERNEL1x8_E2  128,16, 1,1
2405     MY_ALIGN
2406
2407
2408 CGEMM_L1x8_SUB2_2:
2409 /*----------------------------------------*/
2410     andi.      T1,L, 2
2411     ble CGEMM_L1x8_SUB2_1
2412     LOAD1x8_2
2413     KERNEL1x8_E2  128,16, 0,1
2414     MY_ALIGN
2415
2416
2417 CGEMM_L1x8_SUB2_1:
2418 /*----------------------------------------*/
2419     andi.      T1,L, 1
2420     ble CGEMM_L1x8_SAVE
2421     KERNEL1x8
2422
2423     MY_ALIGN
2424 CGEMM_L1x8_SAVE:
2425 /*----------------------------------------*/
2426     addic.    I,  I,  -1
2427     MY_ALIGN
2428     SAVE1x8
2429 #if defined(TRMMKERNEL)
2430     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,8,1
2431 #endif
2432     bgt   CGEMM_L1x8_BEGIN
2433     andi.   T2, M,  7
2434     ble   CGEMM_L1x1_END
2435     andi.   T1, M,  4
2436     ble   CGEMM_L1x4_END
2437     b   CGEMM_L1x4_BEGIN
2438     MY_ALIGN
2439
2440
2441 CGEMM_L1x8_END:
2442 /*----------------------------------------*/
2443
2444
2445 CGEMM_L1x4_BEGIN:
2446 /*----------------------------------------*/
2447     andi.   T2, M,  7
2448     ble   CGEMM_L1x1_END
2449     andi.   T1, M,  4
2450     ble   CGEMM_L1x4_END
2451 #if defined(TRMMKERNEL)
2452     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1
2453 #else
2454     mr    BO, B
2455 #endif
2456 #if defined(TRMMKERNEL)
2457     REFRESH_TEMP_BK T6,K,TEMP_REG,4,1
2458     mr T1, T6
2459     addi T1,T1, -2
2460     srawi.   T8, T1, 5 /**(T1-2) % 31x */
2461 #else
2462     mr T1, K
2463     addi T1,T1, -2
2464     srawi.   T8, T1, 5 /**(K-2) % 31x */
2465 #endif
2466     ZERO1x4
2467     ble   CGEMM_L1x4_SUB0
2468     bl CGEMM_1x4_LMAIN_SUB
2469     andi.   L,  T1, 31
2470     ble   CGEMM_L1x4_SAVE
2471     b    CGEMM_L1x4_SUB2
2472
2473
2474 CGEMM_L1x4_SUB0:
2475 /*----------------------------------------*/
2476 #if defined(TRMMKERNEL)
2477     andi.   L,  T6, 63
2478     cmpwi   T6,33
2479 #else
2480     andi.   L,  K,  63
2481     cmpwi   K,33
2482 #endif
2483     li T8,1
2484     bne CMP1x4_32K
2485     addi BO,BO,-8
2486     addi AO,AO,-32
2487     LOAD1x4O 32,8
2488     END1x4_WITHOUT_ADD
2489     LOAD1x4_2O  64, 16
2490     mtctr   T8
2491     bl CGEMM_L1x4_K32
2492     b CGEMM_L1x4_SAVE
2493     CMP1x4_32K:
2494 /*----------------------------------------*/
2495 #if defined(TRMMKERNEL)
2496     cmpwi   T6,32
2497 #else
2498     cmpwi   K,32
2499 #endif
2500     bne CGEMM_L1x4_SUB2
2501     MY_ALIGN
2502     mtctr   T8
2503     addi BO,BO,-16
2504     addi AO,AO,-64
2505     LOAD1x4_2O 64,16
2506     bl CGEMM_L1x4_K32
2507     b CGEMM_L1x4_SAVE
2508     MY_ALIGN
2509     MY_ALIGN
2510
2511
2512 CGEMM_L1x4_SUB2:
2513 /*----------------------------------------*/
2514     andi.      T1,L, 16
2515     ble CGEMM_L1x4_SUB2_8
2516     bl  CGEMM_1x4_L16_SUB
2517     MY_ALIGN
2518
2519
2520 CGEMM_L1x4_SUB2_8:
2521 /*----------------------------------------*/
2522     andi.      T1,L, 8
2523     ble CGEMM_L1x4_SUB2_4
2524     bl CGEMM_1x4_L8_SUB
2525     MY_ALIGN
2526
2527
2528 CGEMM_L1x4_SUB2_4:
2529 /*----------------------------------------*/
2530     andi.      T1,L, 4
2531     ble CGEMM_L1x4_SUB2_2
2532     LOAD1x4_2
2533     KERNEL1x4_L2  64,16, 0,0
2534     KERNEL1x4_E2  64,16, 1,1
2535     MY_ALIGN
2536
2537
2538 CGEMM_L1x4_SUB2_2:
2539 /*----------------------------------------*/
2540     andi.      T1,L, 2
2541     ble CGEMM_L1x4_SUB2_1
2542     LOAD1x4_2
2543     KERNEL1x4_E2  64,16, 0,1
2544     MY_ALIGN
2545
2546
2547 CGEMM_L1x4_SUB2_1:
2548 /*----------------------------------------*/
2549     andi.      T1,L, 1
2550     ble CGEMM_L1x4_SAVE
2551     KERNEL1x4
2552
2553
2554 CGEMM_L1x4_SAVE:
2555 /*----------------------------------------*/
2556     SAVE1x4
2557 #if defined(TRMMKERNEL)
2558     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,4,1
2559 #endif
2560
2561
2562 CGEMM_L1x4_END:
2563 /*----------------------------------------*/
2564
2565
2566 CGEMM_L1x2_BEGIN:
2567 /*----------------------------------------*/
2568     andi.   T1, M,  2
2569     ble   CGEMM_L1x2_END
2570 #if defined(TRMMKERNEL)
2571     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1
2572 #else
2573     mr    BO, B
2574 #endif
2575 #if defined(TRMMKERNEL)
2576     REFRESH_TEMP_BK T6,K,TEMP_REG,2,1
2577     mr T1, T6
2578     addi T1,T1, -2
2579     srawi.   T8, T1, 5 /**(T1-2) % 31x */
2580 #else
2581     mr T1, K
2582     addi T1,T1, -2
2583     srawi.   T8, T1, 5 /**(K-2) % 31x */
2584 #endif
2585     ZERO1x2
2586     ble   CGEMM_L1x2_SUB0
2587     bl CGEMM_1x2_LMAIN_SUB
2588     andi.   L,  T1, 31
2589     ble   CGEMM_L1x2_SAVE
2590     b   CGEMM_L1x2_SUB2
2591
2592
2593 CGEMM_L1x2_SUB0:
2594 /*----------------------------------------*/
2595 #if defined(TRMMKERNEL)
2596     andi.   L,  T6, 63
2597     cmpwi   T6,33
2598 #else
2599     andi.   L,  K,  63
2600     cmpwi   K,33
2601 #endif
2602     li T8,1
2603     bne CMP1x2_32K
2604     addi BO,BO,-8
2605     addi AO,AO,-16
2606     LOAD1x2O 16,8
2607     END1x2_WITHOUT_ADD
2608     LOAD1x2_2O  32, 16
2609     mtctr   T8
2610     bl CGEMM_L1x2_K32
2611     b CGEMM_L1x2_SAVE
2612     CMP1x2_32K:
2613 /*----------------------------------------*/
2614 #if defined(TRMMKERNEL)
2615     cmpwi   T6,32
2616 #else
2617     cmpwi   K,32
2618 #endif
2619     bne CGEMM_L1x2_SUB2
2620     MY_ALIGN
2621     mtctr   T8
2622     addi BO,BO,-16
2623     addi AO,AO,-32
2624     LOAD1x2_2O 32,16
2625     bl CGEMM_L1x2_K32
2626     b CGEMM_L1x2_SAVE
2627     MY_ALIGN
2628     MY_ALIGN
2629
2630
2631 CGEMM_L1x2_SUB2:
2632 /*----------------------------------------*/
2633     andi.      T1,L, 16
2634     ble CGEMM_L1x2_SUB2_8
2635     bl CGEMM_1x2_L16_SUB
2636     MY_ALIGN
2637
2638
2639 CGEMM_L1x2_SUB2_8:
2640 /*----------------------------------------*/
2641     andi.      T1,L, 8
2642     ble CGEMM_L1x2_SUB2_4
2643     bl CGEMM_1x2_L8_SUB
2644     MY_ALIGN
2645
2646
2647 CGEMM_L1x2_SUB2_4:
2648 /*----------------------------------------*/
2649     andi.      T1,L, 4
2650     ble CGEMM_L1x2_SUB2_2
2651     LOAD1x2_2
2652     KERNEL1x2_L2  32,16, 0,0
2653     KERNEL1x2_E2  32,16, 1,1
2654     MY_ALIGN
2655
2656
2657 CGEMM_L1x2_SUB2_2:
2658 /*----------------------------------------*/
2659     andi.      T1,L, 2
2660     ble CGEMM_L1x2_SUB2_1
2661     LOAD1x2_2
2662     KERNEL1x2_E2  32,16, 0,1
2663     MY_ALIGN
2664
2665
2666 CGEMM_L1x2_SUB2_1:
2667 /*----------------------------------------*/
2668     andi.      T1,L, 1
2669     ble CGEMM_L1x2_SAVE
2670     KERNEL1x2
2671
2672     MY_ALIGN
2673 CGEMM_L1x2_SAVE:
2674 /*----------------------------------------*/
2675     SAVE1x2
2676 #if defined(TRMMKERNEL)
2677     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,2,1
2678 #endif
2679
2680
2681 CGEMM_L1x2_END:
2682 /*----------------------------------------*/
2683
2684
2685 CGEMM_L1x1_BEGIN:
2686 /*----------------------------------------*/
2687     andi.   T1, M,  1
2688     ble   CGEMM_L1x1_END
2689 #if defined(TRMMKERNEL)
2690     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1
2691 #else
2692     mr    BO, B
2693 #endif
2694 #if defined(TRMMKERNEL)
2695     REFRESH_TEMP_BK T6,K,TEMP_REG,1,1
2696     mr T1, T6
2697     addi T1,T1, -2
2698     srawi.   T8, T1, 5 /**(T1-2) % 31x */
2699 #else
2700     mr T1, K
2701     addi T1,T1, -2
2702     srawi.   T8, T1, 5 /**(K-2) % 31x */
2703 #endif
2704     ZERO1x1
2705     ble   CGEMM_L1x1_SUB0
2706     bl CGEMM_1x1_LMAIN_SUB
2707     andi.   L,  T1, 31
2708     ble   CGEMM_L1x1_SAVE
2709     b   CGEMM_L1x1_SUB2
2710
2711
2712 CGEMM_L1x1_SUB0:
2713 /*----------------------------------------*/
2714 #if defined(TRMMKERNEL)
2715     andi.   L,  T6, 63
2716     cmpwi   T6,33
2717 #else
2718     andi.   L,  K,  63
2719     cmpwi   K,33
2720 #endif
2721     li T8,1
2722     bne CMP1x1_32K
2723     addi BO,BO,-8
2724     addi AO,AO,-8
2725     LOAD1x1O 8,8
2726     END1x1_WITHOUT_ADD
2727     LOAD1x1_2O  16, 16
2728     mtctr   T8
2729     bl CGEMM_L1x1_K32
2730     b CGEMM_L1x1_SAVE
2731     CMP1x1_32K:
2732 /*----------------------------------------*/
2733 #if defined(TRMMKERNEL)
2734     cmpwi   T6,32
2735 #else
2736     cmpwi   K,32
2737 #endif
2738     bne CGEMM_L1x1_SUB2
2739     MY_ALIGN
2740     mtctr   T8
2741     addi BO,BO,-16
2742     addi AO,AO,-16
2743     LOAD1x1_2O 16,16
2744     bl CGEMM_L1x1_K32
2745     b CGEMM_L1x1_SAVE
2746     MY_ALIGN
2747     MY_ALIGN
2748
2749
2750 CGEMM_L1x1_SUB2:
2751 /*----------------------------------------*/
2752     andi.      T1,L, 16
2753     ble CGEMM_L1x1_SUB2_8
2754     bl CGEMM_1x1_L16_SUB
2755     MY_ALIGN
2756
2757
2758 CGEMM_L1x1_SUB2_8:
2759 /*----------------------------------------*/
2760     andi.      T1,L, 8
2761     ble CGEMM_L1x1_SUB2_4
2762     bl CGEMM_1x1_L8_SUB
2763     MY_ALIGN
2764
2765
2766 CGEMM_L1x1_SUB2_4:
2767 /*----------------------------------------*/
2768     andi.      T1,L, 4
2769     ble CGEMM_L1x1_SUB2_2
2770     LOAD1x1_2
2771     KERNEL1x1_L2  16,16, 0,0
2772     KERNEL1x1_E2  16,16, 1,1
2773     MY_ALIGN
2774
2775
2776 CGEMM_L1x1_SUB2_2:
2777 /*----------------------------------------*/
2778     andi.      T1,L, 2
2779     ble CGEMM_L1x1_SUB2_1
2780     LOAD1x1_2
2781     KERNEL1x1_E2  16,16, 0,1
2782     MY_ALIGN
2783
2784
2785 CGEMM_L1x1_SUB2_1:
2786 /*----------------------------------------*/
2787     andi.      T1,L, 1
2788     ble CGEMM_L1x1_SAVE
2789     KERNEL1x1
2790
2791     MY_ALIGN
2792 CGEMM_L1x1_SAVE:
2793 /*----------------------------------------*/
2794
2795     SAVE1x1
2796 #if defined(TRMMKERNEL)
2797     REFRESH_AFTER_SAVE T6,K,TEMP_REG,BO,AO,1,1
2798 #endif
2799
2800
2801 CGEMM_L1x1_END:
2802 /*----------------------------------------*/
2803     slwi    T1, K,  3
2804
2805     add   B,  B,  T1
2806 #if defined(TRMMKERNEL) && !defined(LEFT)
2807     addi TEMP_REG, TEMP_REG, 1
2808 #endif
2809
2810 CGEMM_L1_END:
2811
2812
2813
2814