1 /*****************************************************************************
2 Copyright (c) 2011-2014, The OpenBLAS Project
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
9 1. Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in
14 the documentation and/or other materials provided with the
16 3. Neither the name of the OpenBLAS project nor the names of
17 its contributors may be used to endorse or promote products
18 derived from this software without specific prior written
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
27 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
30 USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 **********************************************************************************/
34 /*********************************************************************/
35 /* Copyright 2009, 2010 The University of Texas at Austin. */
36 /* All rights reserved. */
38 /* Redistribution and use in source and binary forms, with or */
39 /* without modification, are permitted provided that the following */
40 /* conditions are met: */
42 /* 1. Redistributions of source code must retain the above */
43 /* copyright notice, this list of conditions and the following */
46 /* 2. Redistributions in binary form must reproduce the above */
47 /* copyright notice, this list of conditions and the following */
48 /* disclaimer in the documentation and/or other materials */
49 /* provided with the distribution. */
51 /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
52 /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
53 /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
54 /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
55 /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
56 /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
57 /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
58 /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
59 /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
60 /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
61 /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
62 /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
63 /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
64 /* POSSIBILITY OF SUCH DAMAGE. */
66 /* The views and conclusions contained in the software and */
67 /* documentation are those of the authors and should not be */
68 /* interpreted as representing official policies, either expressed */
69 /* or implied, of The University of Texas at Austin. */
70 /*********************************************************************/
76 #define SBGEMM_DEFAULT_UNROLL_N 4
77 #define SBGEMM_DEFAULT_UNROLL_M 8
78 #define SBGEMM_DEFAULT_UNROLL_MN 32
79 #define SBGEMM_DEFAULT_P 256
80 #define SBGEMM_DEFAULT_R 256
81 #define SBGEMM_DEFAULT_Q 256
87 #define GEMM_DEFAULT_OFFSET_A 64
88 #define GEMM_DEFAULT_OFFSET_B 256
89 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x01ffffUL
91 #define SGEMM_DEFAULT_UNROLL_N 4
92 #define DGEMM_DEFAULT_UNROLL_N 4
93 #define QGEMM_DEFAULT_UNROLL_N 2
94 #define CGEMM_DEFAULT_UNROLL_N 2
95 #define ZGEMM_DEFAULT_UNROLL_N 2
96 #define XGEMM_DEFAULT_UNROLL_N 1
99 #define SGEMM_DEFAULT_UNROLL_M 4
100 #define DGEMM_DEFAULT_UNROLL_M 2
101 #define QGEMM_DEFAULT_UNROLL_M 2
102 #define CGEMM_DEFAULT_UNROLL_M 2
103 #define ZGEMM_DEFAULT_UNROLL_M 1
104 #define XGEMM_DEFAULT_UNROLL_M 1
106 #define SGEMM_DEFAULT_UNROLL_M 8
107 #define DGEMM_DEFAULT_UNROLL_M 4
108 #define QGEMM_DEFAULT_UNROLL_M 2
109 #define CGEMM_DEFAULT_UNROLL_M 4
110 #define ZGEMM_DEFAULT_UNROLL_M 2
111 #define XGEMM_DEFAULT_UNROLL_M 1
114 #define SGEMM_DEFAULT_P sgemm_p
115 #define DGEMM_DEFAULT_P dgemm_p
116 #define QGEMM_DEFAULT_P qgemm_p
117 #define CGEMM_DEFAULT_P cgemm_p
118 #define ZGEMM_DEFAULT_P zgemm_p
119 #define XGEMM_DEFAULT_P xgemm_p
121 #define SGEMM_DEFAULT_R sgemm_r
122 #define DGEMM_DEFAULT_R dgemm_r
123 #define QGEMM_DEFAULT_R qgemm_r
124 #define CGEMM_DEFAULT_R cgemm_r
125 #define ZGEMM_DEFAULT_R zgemm_r
126 #define XGEMM_DEFAULT_R xgemm_r
130 #define SGEMM_DEFAULT_Q 248
131 #define DGEMM_DEFAULT_Q 248
132 #define QGEMM_DEFAULT_Q 248
133 #define CGEMM_DEFAULT_Q 248
134 #define ZGEMM_DEFAULT_Q 248
135 #define XGEMM_DEFAULT_Q 248
139 #define SGEMM_DEFAULT_Q 240
140 #define DGEMM_DEFAULT_Q 240
141 #define QGEMM_DEFAULT_Q 240
142 #define CGEMM_DEFAULT_Q 240
143 #define ZGEMM_DEFAULT_Q 240
144 #define XGEMM_DEFAULT_Q 240
150 #define HAVE_EXCLUSIVE_CACHE
154 #if defined(BARCELONA) || defined(SHANGHAI) || defined(BOBCAT)
159 #define GEMM_DEFAULT_OFFSET_A 64
160 #define GEMM_DEFAULT_OFFSET_B 832
161 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0fffUL
163 #define SGEMM_DEFAULT_UNROLL_N 4
164 #define DGEMM_DEFAULT_UNROLL_N 4
165 #define QGEMM_DEFAULT_UNROLL_N 2
166 #define CGEMM_DEFAULT_UNROLL_N 2
167 #define ZGEMM_DEFAULT_UNROLL_N 2
168 #define XGEMM_DEFAULT_UNROLL_N 1
171 #define SGEMM_DEFAULT_UNROLL_M 4
172 #define DGEMM_DEFAULT_UNROLL_M 2
173 #define QGEMM_DEFAULT_UNROLL_M 2
174 #define CGEMM_DEFAULT_UNROLL_M 2
175 #define ZGEMM_DEFAULT_UNROLL_M 1
176 #define XGEMM_DEFAULT_UNROLL_M 1
178 #define SGEMM_DEFAULT_UNROLL_M 8
179 #define DGEMM_DEFAULT_UNROLL_M 4
180 #define QGEMM_DEFAULT_UNROLL_M 2
181 #define CGEMM_DEFAULT_UNROLL_M 4
182 #define ZGEMM_DEFAULT_UNROLL_M 2
183 #define XGEMM_DEFAULT_UNROLL_M 1
187 #define SGEMM_DEFAULT_P 496
188 #define DGEMM_DEFAULT_P 248
189 #define QGEMM_DEFAULT_P 124
190 #define CGEMM_DEFAULT_P 248
191 #define ZGEMM_DEFAULT_P 124
192 #define XGEMM_DEFAULT_P 62
194 #define SGEMM_DEFAULT_Q 248
195 #define DGEMM_DEFAULT_Q 248
196 #define QGEMM_DEFAULT_Q 248
197 #define CGEMM_DEFAULT_Q 248
198 #define ZGEMM_DEFAULT_Q 248
199 #define XGEMM_DEFAULT_Q 248
203 #define SGEMM_DEFAULT_P 448
204 #define DGEMM_DEFAULT_P 224
205 #define QGEMM_DEFAULT_P 112
206 #define CGEMM_DEFAULT_P 224
207 #define ZGEMM_DEFAULT_P 112
208 #define XGEMM_DEFAULT_P 56
210 #define SGEMM_DEFAULT_Q 224
211 #define DGEMM_DEFAULT_Q 224
212 #define QGEMM_DEFAULT_Q 224
213 #define CGEMM_DEFAULT_Q 224
214 #define ZGEMM_DEFAULT_Q 224
215 #define XGEMM_DEFAULT_Q 224
219 #define SGEMM_DEFAULT_R sgemm_r
220 #define QGEMM_DEFAULT_R qgemm_r
221 #define DGEMM_DEFAULT_R dgemm_r
222 #define CGEMM_DEFAULT_R cgemm_r
223 #define ZGEMM_DEFAULT_R zgemm_r
224 #define XGEMM_DEFAULT_R xgemm_r
227 #define HAVE_EXCLUSIVE_CACHE
229 #define GEMM_THREAD gemm_thread_mn
239 #define GEMM_DEFAULT_OFFSET_A 64
240 #define GEMM_DEFAULT_OFFSET_B 832
241 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0fffUL
245 #define QGEMM_DEFAULT_UNROLL_N 2
246 #define CGEMM_DEFAULT_UNROLL_N 2
247 #define ZGEMM_DEFAULT_UNROLL_N 2
248 #define XGEMM_DEFAULT_UNROLL_N 1
251 #define SGEMM_DEFAULT_UNROLL_N 4
252 #define DGEMM_DEFAULT_UNROLL_N 4
253 #define SGEMM_DEFAULT_UNROLL_M 4
254 #define DGEMM_DEFAULT_UNROLL_M 2
255 #define QGEMM_DEFAULT_UNROLL_M 2
256 #define CGEMM_DEFAULT_UNROLL_M 2
257 #define ZGEMM_DEFAULT_UNROLL_M 1
258 #define XGEMM_DEFAULT_UNROLL_M 1
260 #define SGEMM_DEFAULT_UNROLL_N 2
261 #define DGEMM_DEFAULT_UNROLL_N 2
262 #define SGEMM_DEFAULT_UNROLL_M 16
263 #define DGEMM_DEFAULT_UNROLL_M 8
264 #define QGEMM_DEFAULT_UNROLL_M 2
265 #define CGEMM_DEFAULT_UNROLL_M 4
266 #define ZGEMM_DEFAULT_UNROLL_M 2
267 #define XGEMM_DEFAULT_UNROLL_M 1
268 #define CGEMM3M_DEFAULT_UNROLL_N 4
269 #define CGEMM3M_DEFAULT_UNROLL_M 8
270 #define ZGEMM3M_DEFAULT_UNROLL_N 4
271 #define ZGEMM3M_DEFAULT_UNROLL_M 4
273 #define DGEMM_DEFAULT_UNROLL_MN 16
274 #define GEMV_UNROLL 8
278 #if defined(ARCH_X86_64)
279 #define SGEMM_DEFAULT_P 768
280 #define DGEMM_DEFAULT_P 384
282 #define SGEMM_DEFAULT_P 448
283 #define DGEMM_DEFAULT_P 224
286 #define QGEMM_DEFAULT_P 112
287 #define CGEMM_DEFAULT_P 224
288 #define ZGEMM_DEFAULT_P 112
289 #define XGEMM_DEFAULT_P 56
291 #if defined(ARCH_X86_64)
292 #define SGEMM_DEFAULT_Q 168
293 #define DGEMM_DEFAULT_Q 168
295 #define SGEMM_DEFAULT_Q 224
296 #define DGEMM_DEFAULT_Q 224
299 #define QGEMM_DEFAULT_Q 224
300 #define CGEMM_DEFAULT_Q 224
301 #define ZGEMM_DEFAULT_Q 224
302 #define XGEMM_DEFAULT_Q 224
304 #define CGEMM3M_DEFAULT_P 448
305 #define ZGEMM3M_DEFAULT_P 224
306 #define XGEMM3M_DEFAULT_P 112
307 #define CGEMM3M_DEFAULT_Q 224
308 #define ZGEMM3M_DEFAULT_Q 224
309 #define XGEMM3M_DEFAULT_Q 224
310 #define CGEMM3M_DEFAULT_R 12288
311 #define ZGEMM3M_DEFAULT_R 12288
312 #define XGEMM3M_DEFAULT_R 12288
314 #define SGEMM_DEFAULT_R sgemm_r
315 #define QGEMM_DEFAULT_R qgemm_r
316 #define DGEMM_DEFAULT_R dgemm_r
317 #define CGEMM_DEFAULT_R cgemm_r
318 #define ZGEMM_DEFAULT_R zgemm_r
319 #define XGEMM_DEFAULT_R xgemm_r
322 #define HAVE_EXCLUSIVE_CACHE
324 #define GEMM_THREAD gemm_thread_mn
332 #define GEMM_DEFAULT_OFFSET_A 64
333 #define GEMM_DEFAULT_OFFSET_B 832
334 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0fffUL
338 #define QGEMM_DEFAULT_UNROLL_N 2
339 #define CGEMM_DEFAULT_UNROLL_N 2
340 #define ZGEMM_DEFAULT_UNROLL_N 2
341 #define XGEMM_DEFAULT_UNROLL_N 1
344 #define SGEMM_DEFAULT_UNROLL_N 4
345 #define DGEMM_DEFAULT_UNROLL_N 4
346 #define SGEMM_DEFAULT_UNROLL_M 4
347 #define DGEMM_DEFAULT_UNROLL_M 2
348 #define QGEMM_DEFAULT_UNROLL_M 2
349 #define CGEMM_DEFAULT_UNROLL_M 2
350 #define ZGEMM_DEFAULT_UNROLL_M 1
351 #define XGEMM_DEFAULT_UNROLL_M 1
353 #define SGEMM_DEFAULT_UNROLL_N 2
354 #define DGEMM_DEFAULT_UNROLL_N 2
355 #define SGEMM_DEFAULT_UNROLL_M 16
356 #define DGEMM_DEFAULT_UNROLL_M 8
357 #define QGEMM_DEFAULT_UNROLL_M 2
358 #define CGEMM_DEFAULT_UNROLL_M 4
359 #define ZGEMM_DEFAULT_UNROLL_M 2
360 #define XGEMM_DEFAULT_UNROLL_M 1
361 #define CGEMM3M_DEFAULT_UNROLL_N 4
362 #define CGEMM3M_DEFAULT_UNROLL_M 8
363 #define ZGEMM3M_DEFAULT_UNROLL_N 4
364 #define ZGEMM3M_DEFAULT_UNROLL_M 4
365 #define GEMV_UNROLL 8
368 #if defined(ARCH_X86_64)
369 #define SGEMM_DEFAULT_P 768
370 #define DGEMM_DEFAULT_P 768
371 #define ZGEMM_DEFAULT_P 384
372 #define CGEMM_DEFAULT_P 768
374 #define SGEMM_DEFAULT_P 448
375 #define DGEMM_DEFAULT_P 480
376 #define ZGEMM_DEFAULT_P 112
377 #define CGEMM_DEFAULT_P 224
379 #define QGEMM_DEFAULT_P 112
380 #define XGEMM_DEFAULT_P 56
382 #if defined(ARCH_X86_64)
383 #define SGEMM_DEFAULT_Q 192
384 #define DGEMM_DEFAULT_Q 168
385 #define ZGEMM_DEFAULT_Q 168
386 #define CGEMM_DEFAULT_Q 168
388 #define SGEMM_DEFAULT_Q 224
389 #define DGEMM_DEFAULT_Q 224
390 #define ZGEMM_DEFAULT_Q 224
391 #define CGEMM_DEFAULT_Q 224
393 #define QGEMM_DEFAULT_Q 224
394 #define XGEMM_DEFAULT_Q 224
396 #define CGEMM3M_DEFAULT_P 448
397 #define ZGEMM3M_DEFAULT_P 224
398 #define XGEMM3M_DEFAULT_P 112
399 #define CGEMM3M_DEFAULT_Q 224
400 #define ZGEMM3M_DEFAULT_Q 224
401 #define XGEMM3M_DEFAULT_Q 224
402 #define CGEMM3M_DEFAULT_R 12288
403 #define ZGEMM3M_DEFAULT_R 12288
404 #define XGEMM3M_DEFAULT_R 12288
406 #define SGEMM_DEFAULT_R 12288
407 #define QGEMM_DEFAULT_R qgemm_r
408 #define DGEMM_DEFAULT_R 12288
409 #define CGEMM_DEFAULT_R cgemm_r
410 #define ZGEMM_DEFAULT_R zgemm_r
411 #define XGEMM_DEFAULT_R xgemm_r
414 #define HAVE_EXCLUSIVE_CACHE
416 #define GEMM_THREAD gemm_thread_mn
424 #define GEMM_DEFAULT_OFFSET_A 64
425 #define GEMM_DEFAULT_OFFSET_B 832
426 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0fffUL
430 #define QGEMM_DEFAULT_UNROLL_N 2
431 #define CGEMM_DEFAULT_UNROLL_N 2
432 #define ZGEMM_DEFAULT_UNROLL_N 2
433 #define XGEMM_DEFAULT_UNROLL_N 1
436 #define SGEMM_DEFAULT_UNROLL_N 4
437 #define DGEMM_DEFAULT_UNROLL_N 4
438 #define SGEMM_DEFAULT_UNROLL_M 4
439 #define DGEMM_DEFAULT_UNROLL_M 2
440 #define QGEMM_DEFAULT_UNROLL_M 2
441 #define CGEMM_DEFAULT_UNROLL_M 2
442 #define ZGEMM_DEFAULT_UNROLL_M 1
443 #define XGEMM_DEFAULT_UNROLL_M 1
445 #define SGEMM_DEFAULT_UNROLL_N 2
446 #define DGEMM_DEFAULT_UNROLL_N 2
447 #define SGEMM_DEFAULT_UNROLL_M 16
448 #define DGEMM_DEFAULT_UNROLL_M 8
449 #define QGEMM_DEFAULT_UNROLL_M 2
450 #define CGEMM_DEFAULT_UNROLL_M 4
451 #define ZGEMM_DEFAULT_UNROLL_M 2
452 #define XGEMM_DEFAULT_UNROLL_M 1
453 #define CGEMM3M_DEFAULT_UNROLL_N 4
454 #define CGEMM3M_DEFAULT_UNROLL_M 8
455 #define ZGEMM3M_DEFAULT_UNROLL_N 4
456 #define ZGEMM3M_DEFAULT_UNROLL_M 4
457 #define GEMV_UNROLL 8
460 #if defined(ARCH_X86_64)
461 #define SGEMM_DEFAULT_P 768
462 #define DGEMM_DEFAULT_P 576
463 #define ZGEMM_DEFAULT_P 288
464 #define CGEMM_DEFAULT_P 576
466 #define SGEMM_DEFAULT_P 448
467 #define DGEMM_DEFAULT_P 480
468 #define ZGEMM_DEFAULT_P 112
469 #define CGEMM_DEFAULT_P 224
471 #define QGEMM_DEFAULT_P 112
472 #define XGEMM_DEFAULT_P 56
474 #if defined(ARCH_X86_64)
475 #define SGEMM_DEFAULT_Q 192
476 #define DGEMM_DEFAULT_Q 160
477 #define ZGEMM_DEFAULT_Q 160
478 #define CGEMM_DEFAULT_Q 160
480 #define SGEMM_DEFAULT_Q 224
481 #define DGEMM_DEFAULT_Q 224
482 #define ZGEMM_DEFAULT_Q 224
483 #define CGEMM_DEFAULT_Q 224
485 #define QGEMM_DEFAULT_Q 224
486 #define XGEMM_DEFAULT_Q 224
488 #define CGEMM3M_DEFAULT_P 448
489 #define ZGEMM3M_DEFAULT_P 224
490 #define XGEMM3M_DEFAULT_P 112
491 #define CGEMM3M_DEFAULT_Q 224
492 #define ZGEMM3M_DEFAULT_Q 224
493 #define XGEMM3M_DEFAULT_Q 224
494 #define CGEMM3M_DEFAULT_R 12288
495 #define ZGEMM3M_DEFAULT_R 12288
496 #define XGEMM3M_DEFAULT_R 12288
498 #define SGEMM_DEFAULT_R 12288
499 #define QGEMM_DEFAULT_R qgemm_r
500 #define DGEMM_DEFAULT_R 12288
501 #define CGEMM_DEFAULT_R cgemm_r
502 #define ZGEMM_DEFAULT_R zgemm_r
503 #define XGEMM_DEFAULT_R xgemm_r
506 #define HAVE_EXCLUSIVE_CACHE
508 #define GEMM_THREAD gemm_thread_mn
517 #define GEMM_DEFAULT_OFFSET_A 64
518 #define GEMM_DEFAULT_OFFSET_B 832
519 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0fffUL
523 #define QGEMM_DEFAULT_UNROLL_N 2
524 #define CGEMM_DEFAULT_UNROLL_N 2
525 #define ZGEMM_DEFAULT_UNROLL_N 2
526 #define XGEMM_DEFAULT_UNROLL_N 1
529 #define SGEMM_DEFAULT_UNROLL_N 4
530 #define DGEMM_DEFAULT_UNROLL_N 4
531 #define SGEMM_DEFAULT_UNROLL_M 4
532 #define DGEMM_DEFAULT_UNROLL_M 2
533 #define QGEMM_DEFAULT_UNROLL_M 2
534 #define CGEMM_DEFAULT_UNROLL_M 2
535 #define ZGEMM_DEFAULT_UNROLL_M 1
536 #define XGEMM_DEFAULT_UNROLL_M 1
538 #define SGEMM_DEFAULT_UNROLL_N 2
539 #define DGEMM_DEFAULT_UNROLL_N 2
540 #define SGEMM_DEFAULT_UNROLL_M 16
541 #define DGEMM_DEFAULT_UNROLL_M 8
542 #define QGEMM_DEFAULT_UNROLL_M 2
543 #define CGEMM_DEFAULT_UNROLL_M 4
544 #define ZGEMM_DEFAULT_UNROLL_M 2
545 #define XGEMM_DEFAULT_UNROLL_M 1
546 #define CGEMM3M_DEFAULT_UNROLL_N 4
547 #define CGEMM3M_DEFAULT_UNROLL_M 8
548 #define ZGEMM3M_DEFAULT_UNROLL_N 4
549 #define ZGEMM3M_DEFAULT_UNROLL_M 4
550 #define GEMV_UNROLL 8
553 #if defined(ARCH_X86_64)
554 #define SGEMM_DEFAULT_P 768
555 #define DGEMM_DEFAULT_P 576
556 #define ZGEMM_DEFAULT_P 288
557 #define CGEMM_DEFAULT_P 576
559 #define SGEMM_DEFAULT_P 448
560 #define DGEMM_DEFAULT_P 480
561 #define ZGEMM_DEFAULT_P 112
562 #define CGEMM_DEFAULT_P 224
564 #define QGEMM_DEFAULT_P 112
565 #define XGEMM_DEFAULT_P 56
567 #if defined(ARCH_X86_64)
568 #define SGEMM_DEFAULT_Q 192
569 #define DGEMM_DEFAULT_Q 160
570 #define ZGEMM_DEFAULT_Q 160
571 #define CGEMM_DEFAULT_Q 160
573 #define SGEMM_DEFAULT_Q 224
574 #define DGEMM_DEFAULT_Q 224
575 #define ZGEMM_DEFAULT_Q 224
576 #define CGEMM_DEFAULT_Q 224
578 #define QGEMM_DEFAULT_Q 224
579 #define XGEMM_DEFAULT_Q 224
581 #define CGEMM3M_DEFAULT_P 448
582 #define ZGEMM3M_DEFAULT_P 224
583 #define XGEMM3M_DEFAULT_P 112
584 #define CGEMM3M_DEFAULT_Q 224
585 #define ZGEMM3M_DEFAULT_Q 224
586 #define XGEMM3M_DEFAULT_Q 224
587 #define CGEMM3M_DEFAULT_R 12288
588 #define ZGEMM3M_DEFAULT_R 12288
589 #define XGEMM3M_DEFAULT_R 12288
591 #define SGEMM_DEFAULT_R 12288
592 #define QGEMM_DEFAULT_R qgemm_r
593 #define DGEMM_DEFAULT_R 12288
594 #define CGEMM_DEFAULT_R cgemm_r
595 #define ZGEMM_DEFAULT_R zgemm_r
596 #define XGEMM_DEFAULT_R xgemm_r
599 #define HAVE_EXCLUSIVE_CACHE
601 #define GEMM_THREAD gemm_thread_mn
609 #define GEMM_DEFAULT_OFFSET_A 0
610 #define GEMM_DEFAULT_OFFSET_B 0
611 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
615 #define SWITCH_RATIO 16
619 #define SGEMM_DEFAULT_UNROLL_M 4
620 #define DGEMM_DEFAULT_UNROLL_M 2
621 #define QGEMM_DEFAULT_UNROLL_M 2
622 #define CGEMM_DEFAULT_UNROLL_M 2
623 #define ZGEMM_DEFAULT_UNROLL_M 1
624 #define XGEMM_DEFAULT_UNROLL_M 1
626 #define SGEMM_DEFAULT_UNROLL_N 4
627 #define DGEMM_DEFAULT_UNROLL_N 4
628 #define QGEMM_DEFAULT_UNROLL_N 2
629 #define CGEMM_DEFAULT_UNROLL_N 2
630 #define ZGEMM_DEFAULT_UNROLL_N 2
631 #define XGEMM_DEFAULT_UNROLL_N 1
635 #define SGEMM_DEFAULT_UNROLL_M 8
636 #define DGEMM_DEFAULT_UNROLL_M 4
637 #define QGEMM_DEFAULT_UNROLL_M 2
638 #define CGEMM_DEFAULT_UNROLL_M 8
639 #define ZGEMM_DEFAULT_UNROLL_M 4
640 #define XGEMM_DEFAULT_UNROLL_M 1
642 #define SGEMM_DEFAULT_UNROLL_N 4
643 #define DGEMM_DEFAULT_UNROLL_N 8
644 #define QGEMM_DEFAULT_UNROLL_N 2
645 #define CGEMM_DEFAULT_UNROLL_N 2
646 #define ZGEMM_DEFAULT_UNROLL_N 2
647 #define XGEMM_DEFAULT_UNROLL_N 1
649 #define SGEMM_DEFAULT_UNROLL_MN 32
650 #define DGEMM_DEFAULT_UNROLL_MN 32
656 #define SGEMM_DEFAULT_P 512
657 #define SGEMM_DEFAULT_R sgemm_r
658 #define DGEMM_DEFAULT_P 512
659 #define DGEMM_DEFAULT_R dgemm_r
660 #define QGEMM_DEFAULT_P 504
661 #define QGEMM_DEFAULT_R qgemm_r
662 #define CGEMM_DEFAULT_P 128
663 #define CGEMM_DEFAULT_R 1024
664 #define ZGEMM_DEFAULT_P 512
665 #define ZGEMM_DEFAULT_R zgemm_r
666 #define XGEMM_DEFAULT_P 252
667 #define XGEMM_DEFAULT_R xgemm_r
668 #define SGEMM_DEFAULT_Q 256
669 #define DGEMM_DEFAULT_Q 256
670 #define QGEMM_DEFAULT_Q 128
671 #define CGEMM_DEFAULT_Q 256
672 #define ZGEMM_DEFAULT_Q 192
673 #define XGEMM_DEFAULT_Q 128
677 #define SGEMM_DEFAULT_P 320
678 #define DGEMM_DEFAULT_P 512
679 #define CGEMM_DEFAULT_P 256
680 #define ZGEMM_DEFAULT_P 192
683 #define SGEMM_DEFAULT_Q 320
684 #define DGEMM_DEFAULT_Q 128
686 #define SGEMM_DEFAULT_Q 320
687 #define DGEMM_DEFAULT_Q 256
689 #define CGEMM_DEFAULT_Q 256
690 #define ZGEMM_DEFAULT_Q 192
692 #define SGEMM_DEFAULT_R sgemm_r
693 #define DGEMM_DEFAULT_R 13824
694 #define CGEMM_DEFAULT_R cgemm_r
695 #define ZGEMM_DEFAULT_R zgemm_r
697 #define QGEMM_DEFAULT_Q 128
698 #define QGEMM_DEFAULT_P 504
699 #define QGEMM_DEFAULT_R qgemm_r
700 #define XGEMM_DEFAULT_P 252
701 #define XGEMM_DEFAULT_R xgemm_r
702 #define XGEMM_DEFAULT_Q 128
704 #define CGEMM3M_DEFAULT_UNROLL_N 4
705 #define CGEMM3M_DEFAULT_UNROLL_M 8
706 #define ZGEMM3M_DEFAULT_UNROLL_N 4
707 #define ZGEMM3M_DEFAULT_UNROLL_M 4
709 #define CGEMM3M_DEFAULT_P 320
710 #define ZGEMM3M_DEFAULT_P 256
711 #define XGEMM3M_DEFAULT_P 112
712 #define CGEMM3M_DEFAULT_Q 320
713 #define ZGEMM3M_DEFAULT_Q 256
714 #define XGEMM3M_DEFAULT_Q 224
715 #define CGEMM3M_DEFAULT_R 12288
716 #define ZGEMM3M_DEFAULT_R 12288
717 #define XGEMM3M_DEFAULT_R 12288
728 #define GEMM_DEFAULT_OFFSET_A 0
729 #define GEMM_DEFAULT_OFFSET_B 384
730 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
732 #define SGEMM_DEFAULT_UNROLL_N 4
733 #define DGEMM_DEFAULT_UNROLL_N 4
734 #define QGEMM_DEFAULT_UNROLL_N 2
735 #define CGEMM_DEFAULT_UNROLL_N 2
736 #define ZGEMM_DEFAULT_UNROLL_N 2
737 #define XGEMM_DEFAULT_UNROLL_N 1
739 #define SGEMM_DEFAULT_UNROLL_M 2
740 #define DGEMM_DEFAULT_UNROLL_M 1
741 #define QGEMM_DEFAULT_UNROLL_M 2
742 #define CGEMM_DEFAULT_UNROLL_M 1
743 #define ZGEMM_DEFAULT_UNROLL_M 1
744 #define XGEMM_DEFAULT_UNROLL_M 1
746 #define SGEMM_DEFAULT_R sgemm_r
747 #define DGEMM_DEFAULT_R dgemm_r
748 #define QGEMM_DEFAULT_R qgemm_r
749 #define CGEMM_DEFAULT_R cgemm_r
750 #define ZGEMM_DEFAULT_R zgemm_r
751 #define XGEMM_DEFAULT_R xgemm_r
753 #define SGEMM_DEFAULT_P 208
754 #define DGEMM_DEFAULT_P 104
755 #define QGEMM_DEFAULT_P 56
756 #define CGEMM_DEFAULT_P 104
757 #define ZGEMM_DEFAULT_P 56
758 #define XGEMM_DEFAULT_P 28
760 #define SGEMM_DEFAULT_Q 208
761 #define DGEMM_DEFAULT_Q 208
762 #define QGEMM_DEFAULT_Q 208
763 #define CGEMM_DEFAULT_Q 208
764 #define ZGEMM_DEFAULT_Q 208
765 #define XGEMM_DEFAULT_Q 208
768 #define HAVE_EXCLUSIVE_CACHE
776 #define GEMM_DEFAULT_OFFSET_A 0
777 #define GEMM_DEFAULT_OFFSET_B 256
778 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
780 #define SGEMM_DEFAULT_UNROLL_N 4
781 #define DGEMM_DEFAULT_UNROLL_N 4
782 #define QGEMM_DEFAULT_UNROLL_N 2
783 #define CGEMM_DEFAULT_UNROLL_N 2
784 #define ZGEMM_DEFAULT_UNROLL_N 2
785 #define XGEMM_DEFAULT_UNROLL_N 1
787 #define SGEMM_DEFAULT_UNROLL_M 2
788 #define DGEMM_DEFAULT_UNROLL_M 1
789 #define QGEMM_DEFAULT_UNROLL_M 2
790 #define CGEMM_DEFAULT_UNROLL_M 1
791 #define ZGEMM_DEFAULT_UNROLL_M 1
792 #define XGEMM_DEFAULT_UNROLL_M 1
794 #define SGEMM_DEFAULT_R sgemm_r
795 #define DGEMM_DEFAULT_R dgemm_r
796 #define QGEMM_DEFAULT_R qgemm_r
797 #define CGEMM_DEFAULT_R cgemm_r
798 #define ZGEMM_DEFAULT_R zgemm_r
799 #define XGEMM_DEFAULT_R xgemm_r
801 #define SGEMM_DEFAULT_P 128
802 #define DGEMM_DEFAULT_P 128
803 #define QGEMM_DEFAULT_P 128
804 #define CGEMM_DEFAULT_P 128
805 #define ZGEMM_DEFAULT_P 128
806 #define XGEMM_DEFAULT_P 128
808 #define SGEMM_DEFAULT_Q 512
809 #define DGEMM_DEFAULT_Q 256
810 #define QGEMM_DEFAULT_Q 256
811 #define CGEMM_DEFAULT_Q 256
812 #define ZGEMM_DEFAULT_Q 128
813 #define XGEMM_DEFAULT_Q 128
823 #define GEMM_DEFAULT_OFFSET_A 64
824 #define GEMM_DEFAULT_OFFSET_B 256
825 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x01ffffUL
828 #define SGEMM_DEFAULT_UNROLL_N 4
829 #define DGEMM_DEFAULT_UNROLL_N 4
830 #define QGEMM_DEFAULT_UNROLL_N 2
831 #define CGEMM_DEFAULT_UNROLL_N 2
832 #define ZGEMM_DEFAULT_UNROLL_N 2
833 #define XGEMM_DEFAULT_UNROLL_N 1
835 #define SGEMM_DEFAULT_UNROLL_M 4
836 #define DGEMM_DEFAULT_UNROLL_M 2
837 #define QGEMM_DEFAULT_UNROLL_M 2
838 #define CGEMM_DEFAULT_UNROLL_M 2
839 #define ZGEMM_DEFAULT_UNROLL_M 1
840 #define XGEMM_DEFAULT_UNROLL_M 1
842 #define SGEMM_DEFAULT_UNROLL_N 8
843 #define DGEMM_DEFAULT_UNROLL_N 4
844 #define QGEMM_DEFAULT_UNROLL_N 2
845 #define CGEMM_DEFAULT_UNROLL_N 4
846 #define ZGEMM_DEFAULT_UNROLL_N 2
847 #define XGEMM_DEFAULT_UNROLL_N 1
849 #define SGEMM_DEFAULT_UNROLL_M 4
850 #define DGEMM_DEFAULT_UNROLL_M 4
851 #define QGEMM_DEFAULT_UNROLL_M 2
852 #define CGEMM_DEFAULT_UNROLL_M 2
853 #define ZGEMM_DEFAULT_UNROLL_M 2
854 #define XGEMM_DEFAULT_UNROLL_M 1
857 #define SGEMM_DEFAULT_P 288
858 #define DGEMM_DEFAULT_P 288
859 #define QGEMM_DEFAULT_P 288
860 #define CGEMM_DEFAULT_P 288
861 #define ZGEMM_DEFAULT_P 288
862 #define XGEMM_DEFAULT_P 288
864 #define SGEMM_DEFAULT_R sgemm_r
865 #define DGEMM_DEFAULT_R dgemm_r
866 #define QGEMM_DEFAULT_R qgemm_r
867 #define CGEMM_DEFAULT_R cgemm_r
868 #define ZGEMM_DEFAULT_R zgemm_r
869 #define XGEMM_DEFAULT_R xgemm_r
871 #define SGEMM_DEFAULT_Q 256
872 #define DGEMM_DEFAULT_Q 128
873 #define QGEMM_DEFAULT_Q 64
874 #define CGEMM_DEFAULT_Q 128
875 #define ZGEMM_DEFAULT_Q 64
876 #define XGEMM_DEFAULT_Q 32
879 #define HAVE_EXCLUSIVE_CACHE
883 #if defined(PENTIUM) || defined(PENTIUM2) || defined(PENTIUM3)
892 #define GEMM_DEFAULT_OFFSET_A 0
893 #define GEMM_DEFAULT_OFFSET_B 0
894 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
897 #define SGEMM_DEFAULT_UNROLL_M 8
898 #define CGEMM_DEFAULT_UNROLL_M 4
900 #define SGEMM_DEFAULT_UNROLL_M 4
901 #define CGEMM_DEFAULT_UNROLL_M 2
903 #define DGEMM_DEFAULT_UNROLL_M 2
904 #define SGEMM_DEFAULT_UNROLL_N 2
905 #define DGEMM_DEFAULT_UNROLL_N 2
906 #define QGEMM_DEFAULT_UNROLL_M 2
907 #define QGEMM_DEFAULT_UNROLL_N 2
908 #define CGEMM_DEFAULT_UNROLL_N 1
909 #define ZGEMM_DEFAULT_UNROLL_M 1
910 #define ZGEMM_DEFAULT_UNROLL_N 1
911 #define XGEMM_DEFAULT_UNROLL_M 1
912 #define XGEMM_DEFAULT_UNROLL_N 1
914 #define SGEMM_DEFAULT_P sgemm_p
915 #define SGEMM_DEFAULT_Q 256
916 #define SGEMM_DEFAULT_R sgemm_r
918 #define DGEMM_DEFAULT_P dgemm_p
919 #define DGEMM_DEFAULT_Q 256
920 #define DGEMM_DEFAULT_R dgemm_r
922 #define QGEMM_DEFAULT_P qgemm_p
923 #define QGEMM_DEFAULT_Q 256
924 #define QGEMM_DEFAULT_R qgemm_r
926 #define CGEMM_DEFAULT_P cgemm_p
927 #define CGEMM_DEFAULT_Q 256
928 #define CGEMM_DEFAULT_R cgemm_r
930 #define ZGEMM_DEFAULT_P zgemm_p
931 #define ZGEMM_DEFAULT_Q 256
932 #define ZGEMM_DEFAULT_R zgemm_r
934 #define XGEMM_DEFAULT_P xgemm_p
935 #define XGEMM_DEFAULT_Q 256
936 #define XGEMM_DEFAULT_R xgemm_r
947 #define GEMM_DEFAULT_OFFSET_A 0
948 #define GEMM_DEFAULT_OFFSET_B 0
949 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
952 #define SGEMM_DEFAULT_UNROLL_M 4
953 #define SGEMM_DEFAULT_UNROLL_N 4
954 #define DGEMM_DEFAULT_UNROLL_M 2
955 #define DGEMM_DEFAULT_UNROLL_N 4
956 #define QGEMM_DEFAULT_UNROLL_M 2
957 #define QGEMM_DEFAULT_UNROLL_N 2
958 #define CGEMM_DEFAULT_UNROLL_M 2
959 #define CGEMM_DEFAULT_UNROLL_N 2
960 #define ZGEMM_DEFAULT_UNROLL_M 1
961 #define ZGEMM_DEFAULT_UNROLL_N 2
962 #define XGEMM_DEFAULT_UNROLL_M 1
963 #define XGEMM_DEFAULT_UNROLL_N 1
965 #define SGEMM_DEFAULT_UNROLL_M 8
966 #define SGEMM_DEFAULT_UNROLL_N 2
967 #define DGEMM_DEFAULT_UNROLL_M 2
968 #define DGEMM_DEFAULT_UNROLL_N 2
969 #define QGEMM_DEFAULT_UNROLL_M 2
970 #define QGEMM_DEFAULT_UNROLL_N 2
971 #define CGEMM_DEFAULT_UNROLL_M 4
972 #define CGEMM_DEFAULT_UNROLL_N 1
973 #define ZGEMM_DEFAULT_UNROLL_M 1
974 #define ZGEMM_DEFAULT_UNROLL_N 1
975 #define XGEMM_DEFAULT_UNROLL_M 1
976 #define XGEMM_DEFAULT_UNROLL_N 1
980 #define SGEMM_DEFAULT_P sgemm_p
981 #define SGEMM_DEFAULT_Q 256
982 #define SGEMM_DEFAULT_R sgemm_r
984 #define DGEMM_DEFAULT_P dgemm_p
985 #define DGEMM_DEFAULT_Q 256
986 #define DGEMM_DEFAULT_R dgemm_r
988 #define QGEMM_DEFAULT_P qgemm_p
989 #define QGEMM_DEFAULT_Q 256
990 #define QGEMM_DEFAULT_R qgemm_r
992 #define CGEMM_DEFAULT_P cgemm_p
993 #define CGEMM_DEFAULT_Q 256
994 #define CGEMM_DEFAULT_R cgemm_r
996 #define ZGEMM_DEFAULT_P zgemm_p
997 #define ZGEMM_DEFAULT_Q 256
998 #define ZGEMM_DEFAULT_R zgemm_r
1000 #define XGEMM_DEFAULT_P xgemm_p
1001 #define XGEMM_DEFAULT_Q 256
1002 #define XGEMM_DEFAULT_R xgemm_r
1007 #ifdef CORE_NORTHWOOD
1012 #define GEMM_DEFAULT_OFFSET_A 0
1013 #define GEMM_DEFAULT_OFFSET_B 32
1015 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
1019 #define SGEMM_DEFAULT_UNROLL_M 8
1020 #define DGEMM_DEFAULT_UNROLL_M 4
1021 #define QGEMM_DEFAULT_UNROLL_M 2
1022 #define CGEMM_DEFAULT_UNROLL_M 4
1023 #define ZGEMM_DEFAULT_UNROLL_M 2
1024 #define XGEMM_DEFAULT_UNROLL_M 1
1026 #define SGEMM_DEFAULT_UNROLL_N 2
1027 #define DGEMM_DEFAULT_UNROLL_N 2
1028 #define QGEMM_DEFAULT_UNROLL_N 2
1029 #define CGEMM_DEFAULT_UNROLL_N 1
1030 #define ZGEMM_DEFAULT_UNROLL_N 1
1031 #define XGEMM_DEFAULT_UNROLL_N 1
1033 #define SGEMM_DEFAULT_P sgemm_p
1034 #define SGEMM_DEFAULT_R sgemm_r
1036 #define DGEMM_DEFAULT_P dgemm_p
1037 #define DGEMM_DEFAULT_R dgemm_r
1039 #define QGEMM_DEFAULT_P qgemm_p
1040 #define QGEMM_DEFAULT_R qgemm_r
1042 #define CGEMM_DEFAULT_P cgemm_p
1043 #define CGEMM_DEFAULT_R cgemm_r
1045 #define ZGEMM_DEFAULT_P zgemm_p
1046 #define ZGEMM_DEFAULT_R zgemm_r
1048 #define XGEMM_DEFAULT_P xgemm_p
1049 #define XGEMM_DEFAULT_R xgemm_r
1051 #define SGEMM_DEFAULT_Q 128
1052 #define DGEMM_DEFAULT_Q 128
1053 #define QGEMM_DEFAULT_Q 128
1054 #define CGEMM_DEFAULT_Q 128
1055 #define ZGEMM_DEFAULT_Q 128
1056 #define XGEMM_DEFAULT_Q 128
1059 #ifdef CORE_PRESCOTT
1065 #define GEMM_DEFAULT_OFFSET_A 128
1066 #define GEMM_DEFAULT_OFFSET_B 192
1068 #define GEMM_DEFAULT_OFFSET_A 0
1069 #define GEMM_DEFAULT_OFFSET_B 256
1072 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
1077 #define SGEMM_DEFAULT_UNROLL_M 4
1078 #define DGEMM_DEFAULT_UNROLL_M 2
1079 #define QGEMM_DEFAULT_UNROLL_M 2
1080 #define CGEMM_DEFAULT_UNROLL_M 2
1081 #define ZGEMM_DEFAULT_UNROLL_M 1
1082 #define XGEMM_DEFAULT_UNROLL_M 1
1084 #define SGEMM_DEFAULT_UNROLL_M 8
1085 #define DGEMM_DEFAULT_UNROLL_M 4
1086 #define QGEMM_DEFAULT_UNROLL_M 2
1087 #define CGEMM_DEFAULT_UNROLL_M 4
1088 #define ZGEMM_DEFAULT_UNROLL_M 2
1089 #define XGEMM_DEFAULT_UNROLL_M 1
1092 #define SGEMM_DEFAULT_UNROLL_N 4
1093 #define DGEMM_DEFAULT_UNROLL_N 4
1094 #define QGEMM_DEFAULT_UNROLL_N 2
1095 #define CGEMM_DEFAULT_UNROLL_N 2
1096 #define ZGEMM_DEFAULT_UNROLL_N 2
1097 #define XGEMM_DEFAULT_UNROLL_N 1
1099 #define SGEMM_DEFAULT_P sgemm_p
1100 #define SGEMM_DEFAULT_R sgemm_r
1102 #define DGEMM_DEFAULT_P dgemm_p
1103 #define DGEMM_DEFAULT_R dgemm_r
1105 #define QGEMM_DEFAULT_P qgemm_p
1106 #define QGEMM_DEFAULT_R qgemm_r
1108 #define CGEMM_DEFAULT_P cgemm_p
1109 #define CGEMM_DEFAULT_R cgemm_r
1111 #define ZGEMM_DEFAULT_P zgemm_p
1112 #define ZGEMM_DEFAULT_R zgemm_r
1114 #define XGEMM_DEFAULT_P xgemm_p
1115 #define XGEMM_DEFAULT_R xgemm_r
1117 #define SGEMM_DEFAULT_Q 128
1118 #define DGEMM_DEFAULT_Q 128
1119 #define QGEMM_DEFAULT_Q 128
1120 #define CGEMM_DEFAULT_Q 128
1121 #define ZGEMM_DEFAULT_Q 128
1122 #define XGEMM_DEFAULT_Q 128
1130 #define GEMM_DEFAULT_OFFSET_A 448
1131 #define GEMM_DEFAULT_OFFSET_B 128
1132 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1136 #define SWITCH_RATIO 4
1139 #define SGEMM_DEFAULT_UNROLL_M 8
1140 #define DGEMM_DEFAULT_UNROLL_M 4
1141 #define QGEMM_DEFAULT_UNROLL_M 2
1142 #define CGEMM_DEFAULT_UNROLL_M 4
1143 #define ZGEMM_DEFAULT_UNROLL_M 2
1144 #define XGEMM_DEFAULT_UNROLL_M 1
1146 #define SGEMM_DEFAULT_UNROLL_N 2
1147 #define DGEMM_DEFAULT_UNROLL_N 2
1148 #define QGEMM_DEFAULT_UNROLL_N 2
1149 #define CGEMM_DEFAULT_UNROLL_N 1
1150 #define ZGEMM_DEFAULT_UNROLL_N 1
1151 #define XGEMM_DEFAULT_UNROLL_N 1
1153 #define MASK(a, b) ((((a) + (b) - 1) / (b)) * (b))
1156 #define SGEMM_DEFAULT_UNROLL_M 8
1157 #define DGEMM_DEFAULT_UNROLL_M 4
1158 #define QGEMM_DEFAULT_UNROLL_M 2
1159 #define CGEMM_DEFAULT_UNROLL_M 4
1160 #define ZGEMM_DEFAULT_UNROLL_M 2
1161 #define XGEMM_DEFAULT_UNROLL_M 1
1163 #define SGEMM_DEFAULT_UNROLL_N 4
1164 #define DGEMM_DEFAULT_UNROLL_N 4
1165 #define QGEMM_DEFAULT_UNROLL_N 2
1166 #define CGEMM_DEFAULT_UNROLL_N 2
1167 #define ZGEMM_DEFAULT_UNROLL_N 2
1168 #define XGEMM_DEFAULT_UNROLL_N 1
1171 #define SGEMM_DEFAULT_P sgemm_p
1172 #define SGEMM_DEFAULT_R sgemm_r
1174 #define DGEMM_DEFAULT_P dgemm_p
1175 #define DGEMM_DEFAULT_R dgemm_r
1177 #define QGEMM_DEFAULT_P qgemm_p
1178 #define QGEMM_DEFAULT_R qgemm_r
1180 #define CGEMM_DEFAULT_P cgemm_p
1181 #define CGEMM_DEFAULT_R cgemm_r
1183 #define ZGEMM_DEFAULT_P zgemm_p
1184 #define ZGEMM_DEFAULT_R zgemm_r
1186 #define XGEMM_DEFAULT_P xgemm_p
1187 #define XGEMM_DEFAULT_R xgemm_r
1189 #define SGEMM_DEFAULT_Q 256
1190 #define DGEMM_DEFAULT_Q 256
1191 #define QGEMM_DEFAULT_Q 256
1192 #define CGEMM_DEFAULT_Q 256
1193 #define ZGEMM_DEFAULT_Q 256
1194 #define XGEMM_DEFAULT_Q 256
1203 #define GEMM_DEFAULT_OFFSET_A 128
1204 #define GEMM_DEFAULT_OFFSET_B 0
1205 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1209 #define SWITCH_RATIO 4
1212 #define SGEMM_DEFAULT_UNROLL_M 4
1213 #define DGEMM_DEFAULT_UNROLL_M 2
1214 #define QGEMM_DEFAULT_UNROLL_M 2
1215 #define CGEMM_DEFAULT_UNROLL_M 2
1216 #define ZGEMM_DEFAULT_UNROLL_M 1
1217 #define XGEMM_DEFAULT_UNROLL_M 1
1219 #define SGEMM_DEFAULT_UNROLL_N 4
1220 #define DGEMM_DEFAULT_UNROLL_N 4
1221 #define QGEMM_DEFAULT_UNROLL_N 2
1222 #define CGEMM_DEFAULT_UNROLL_N 2
1223 #define ZGEMM_DEFAULT_UNROLL_N 2
1224 #define XGEMM_DEFAULT_UNROLL_N 1
1226 #define SGEMM_DEFAULT_UNROLL_M 8
1227 #define DGEMM_DEFAULT_UNROLL_M 4
1228 #define QGEMM_DEFAULT_UNROLL_M 2
1229 #define CGEMM_DEFAULT_UNROLL_M 4
1230 #define ZGEMM_DEFAULT_UNROLL_M 2
1231 #define XGEMM_DEFAULT_UNROLL_M 1
1233 #define SGEMM_DEFAULT_UNROLL_N 4
1234 #define DGEMM_DEFAULT_UNROLL_N 4
1235 #define QGEMM_DEFAULT_UNROLL_N 2
1236 #define CGEMM_DEFAULT_UNROLL_N 2
1237 #define ZGEMM_DEFAULT_UNROLL_N 2
1238 #define XGEMM_DEFAULT_UNROLL_N 1
1241 #define SGEMM_DEFAULT_P sgemm_p
1242 #define SGEMM_DEFAULT_R sgemm_r
1244 #define DGEMM_DEFAULT_P dgemm_p
1245 #define DGEMM_DEFAULT_R dgemm_r
1247 #define QGEMM_DEFAULT_P qgemm_p
1248 #define QGEMM_DEFAULT_R qgemm_r
1250 #define CGEMM_DEFAULT_P cgemm_p
1251 #define CGEMM_DEFAULT_R cgemm_r
1253 #define ZGEMM_DEFAULT_P zgemm_p
1254 #define ZGEMM_DEFAULT_R zgemm_r
1256 #define XGEMM_DEFAULT_P xgemm_p
1257 #define XGEMM_DEFAULT_R xgemm_r
1259 #define SGEMM_DEFAULT_Q 512
1260 #define DGEMM_DEFAULT_Q 256
1261 #define QGEMM_DEFAULT_Q 128
1262 #define CGEMM_DEFAULT_Q 512
1263 #define ZGEMM_DEFAULT_Q 256
1264 #define XGEMM_DEFAULT_Q 128
1266 #define GETRF_FACTOR 0.75
1274 #define GEMM_DEFAULT_OFFSET_A 128
1275 #define GEMM_DEFAULT_OFFSET_B 0
1276 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1280 #define SWITCH_RATIO 4
1283 #define SGEMM_DEFAULT_UNROLL_M 4
1284 #define DGEMM_DEFAULT_UNROLL_M 2
1285 #define QGEMM_DEFAULT_UNROLL_M 2
1286 #define CGEMM_DEFAULT_UNROLL_M 2
1287 #define ZGEMM_DEFAULT_UNROLL_M 1
1288 #define XGEMM_DEFAULT_UNROLL_M 1
1290 #define SGEMM_DEFAULT_UNROLL_N 4
1291 #define DGEMM_DEFAULT_UNROLL_N 4
1292 #define QGEMM_DEFAULT_UNROLL_N 2
1293 #define CGEMM_DEFAULT_UNROLL_N 2
1294 #define ZGEMM_DEFAULT_UNROLL_N 2
1295 #define XGEMM_DEFAULT_UNROLL_N 1
1297 #define SGEMM_DEFAULT_UNROLL_M 8
1298 #define DGEMM_DEFAULT_UNROLL_M 4
1299 #define QGEMM_DEFAULT_UNROLL_M 2
1300 #define CGEMM_DEFAULT_UNROLL_M 4
1301 #define ZGEMM_DEFAULT_UNROLL_M 2
1302 #define XGEMM_DEFAULT_UNROLL_M 1
1304 #define SGEMM_DEFAULT_UNROLL_N 4
1305 #define DGEMM_DEFAULT_UNROLL_N 4
1306 #define QGEMM_DEFAULT_UNROLL_N 2
1307 #define CGEMM_DEFAULT_UNROLL_N 2
1308 #define ZGEMM_DEFAULT_UNROLL_N 2
1309 #define XGEMM_DEFAULT_UNROLL_N 1
1312 #define SGEMM_DEFAULT_P sgemm_p
1313 #define SGEMM_DEFAULT_R sgemm_r
1315 #define DGEMM_DEFAULT_P dgemm_p
1316 #define DGEMM_DEFAULT_R dgemm_r
1318 #define QGEMM_DEFAULT_P qgemm_p
1319 #define QGEMM_DEFAULT_R qgemm_r
1321 #define CGEMM_DEFAULT_P cgemm_p
1322 #define CGEMM_DEFAULT_R cgemm_r
1324 #define ZGEMM_DEFAULT_P zgemm_p
1325 #define ZGEMM_DEFAULT_R zgemm_r
1327 #define XGEMM_DEFAULT_P xgemm_p
1328 #define XGEMM_DEFAULT_R xgemm_r
1330 #define SGEMM_DEFAULT_Q 768
1331 #define DGEMM_DEFAULT_Q 384
1332 #define QGEMM_DEFAULT_Q 192
1333 #define CGEMM_DEFAULT_Q 768
1334 #define ZGEMM_DEFAULT_Q 384
1335 #define XGEMM_DEFAULT_Q 192
1337 #define GETRF_FACTOR 0.75
1338 #define GEMM_THREAD gemm_thread_mn
1346 #define GEMM_DEFAULT_OFFSET_A 32
1347 #define GEMM_DEFAULT_OFFSET_B 0
1348 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1352 #define SWITCH_RATIO 4
1355 #define SGEMM_DEFAULT_UNROLL_M 4
1356 #define DGEMM_DEFAULT_UNROLL_M 2
1357 #define QGEMM_DEFAULT_UNROLL_M 2
1358 #define CGEMM_DEFAULT_UNROLL_M 2
1359 #define ZGEMM_DEFAULT_UNROLL_M 1
1360 #define XGEMM_DEFAULT_UNROLL_M 1
1362 #define SGEMM_DEFAULT_UNROLL_N 4
1363 #define DGEMM_DEFAULT_UNROLL_N 4
1364 #define QGEMM_DEFAULT_UNROLL_N 2
1365 #define CGEMM_DEFAULT_UNROLL_N 2
1366 #define ZGEMM_DEFAULT_UNROLL_N 2
1367 #define XGEMM_DEFAULT_UNROLL_N 1
1369 #define SGEMM_DEFAULT_UNROLL_M 4
1370 #define DGEMM_DEFAULT_UNROLL_M 2
1371 #define QGEMM_DEFAULT_UNROLL_M 2
1372 #define CGEMM_DEFAULT_UNROLL_M 2
1373 #define ZGEMM_DEFAULT_UNROLL_M 1
1374 #define XGEMM_DEFAULT_UNROLL_M 1
1376 #define SGEMM_DEFAULT_UNROLL_N 8
1377 #define DGEMM_DEFAULT_UNROLL_N 8
1378 #define QGEMM_DEFAULT_UNROLL_N 2
1379 #define CGEMM_DEFAULT_UNROLL_N 4
1380 #define ZGEMM_DEFAULT_UNROLL_N 4
1381 #define XGEMM_DEFAULT_UNROLL_N 1
1384 #define SGEMM_DEFAULT_P 504
1385 #define SGEMM_DEFAULT_R sgemm_r
1387 #define DGEMM_DEFAULT_P 504
1388 #define DGEMM_DEFAULT_R dgemm_r
1390 #define QGEMM_DEFAULT_P 504
1391 #define QGEMM_DEFAULT_R qgemm_r
1393 #define CGEMM_DEFAULT_P 252
1394 #define CGEMM_DEFAULT_R cgemm_r
1396 #define ZGEMM_DEFAULT_P 252
1397 #define ZGEMM_DEFAULT_R zgemm_r
1399 #define XGEMM_DEFAULT_P 252
1400 #define XGEMM_DEFAULT_R xgemm_r
1402 #define SGEMM_DEFAULT_Q 512
1403 #define DGEMM_DEFAULT_Q 256
1404 #define QGEMM_DEFAULT_Q 128
1405 #define CGEMM_DEFAULT_Q 512
1406 #define ZGEMM_DEFAULT_Q 256
1407 #define XGEMM_DEFAULT_Q 128
1409 #define GETRF_FACTOR 0.72
1419 #define GEMM_DEFAULT_OFFSET_A 0
1420 #define GEMM_DEFAULT_OFFSET_B 0
1421 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1425 #define SWITCH_RATIO 4
1428 #define SGEMM_DEFAULT_UNROLL_M 4
1429 #define DGEMM_DEFAULT_UNROLL_M 2
1430 #define QGEMM_DEFAULT_UNROLL_M 2
1431 #define CGEMM_DEFAULT_UNROLL_M 2
1432 #define ZGEMM_DEFAULT_UNROLL_M 1
1433 #define XGEMM_DEFAULT_UNROLL_M 1
1435 #define SGEMM_DEFAULT_UNROLL_N 4
1436 #define DGEMM_DEFAULT_UNROLL_N 4
1437 #define QGEMM_DEFAULT_UNROLL_N 2
1438 #define CGEMM_DEFAULT_UNROLL_N 2
1439 #define ZGEMM_DEFAULT_UNROLL_N 2
1440 #define XGEMM_DEFAULT_UNROLL_N 1
1442 #define SGEMM_DEFAULT_UNROLL_M 16
1443 #define DGEMM_DEFAULT_UNROLL_M 8
1444 #define QGEMM_DEFAULT_UNROLL_M 2
1445 #define CGEMM_DEFAULT_UNROLL_M 8
1446 #define ZGEMM_DEFAULT_UNROLL_M 1
1447 #define XGEMM_DEFAULT_UNROLL_M 1
1449 #define SGEMM_DEFAULT_UNROLL_N 4
1450 #define DGEMM_DEFAULT_UNROLL_N 4
1451 #define QGEMM_DEFAULT_UNROLL_N 2
1452 #define CGEMM_DEFAULT_UNROLL_N 2
1453 #define ZGEMM_DEFAULT_UNROLL_N 4
1454 #define XGEMM_DEFAULT_UNROLL_N 1
1457 #define SGEMM_DEFAULT_P 768
1458 #define SGEMM_DEFAULT_R sgemm_r
1459 /*#define SGEMM_DEFAULT_R 1024*/
1461 #define DGEMM_DEFAULT_P 512
1462 #define DGEMM_DEFAULT_R dgemm_r
1463 /*#define DGEMM_DEFAULT_R 1024*/
1465 #define QGEMM_DEFAULT_P 504
1466 #define QGEMM_DEFAULT_R qgemm_r
1468 #define CGEMM_DEFAULT_P 768
1469 #define CGEMM_DEFAULT_R cgemm_r
1470 /*#define CGEMM_DEFAULT_R 1024*/
1472 #define ZGEMM_DEFAULT_P 512
1473 #define ZGEMM_DEFAULT_R zgemm_r
1474 /*#define ZGEMM_DEFAULT_R 1024*/
1476 #define XGEMM_DEFAULT_P 252
1477 #define XGEMM_DEFAULT_R xgemm_r
1479 #define SGEMM_DEFAULT_Q 384
1480 #define DGEMM_DEFAULT_Q 256
1481 #define QGEMM_DEFAULT_Q 128
1482 #define CGEMM_DEFAULT_Q 512
1483 #define ZGEMM_DEFAULT_Q 192
1484 #define XGEMM_DEFAULT_Q 128
1486 #define CGEMM3M_DEFAULT_UNROLL_N 8
1487 #define CGEMM3M_DEFAULT_UNROLL_M 4
1488 #define ZGEMM3M_DEFAULT_UNROLL_N 8
1489 #define ZGEMM3M_DEFAULT_UNROLL_M 2
1491 #define CGEMM3M_DEFAULT_P 448
1492 #define ZGEMM3M_DEFAULT_P 224
1493 #define XGEMM3M_DEFAULT_P 112
1494 #define CGEMM3M_DEFAULT_Q 224
1495 #define ZGEMM3M_DEFAULT_Q 224
1496 #define XGEMM3M_DEFAULT_Q 224
1497 #define CGEMM3M_DEFAULT_R 12288
1498 #define ZGEMM3M_DEFAULT_R 12288
1499 #define XGEMM3M_DEFAULT_R 12288
1503 #define GETRF_FACTOR 0.72
1512 #define GEMM_DEFAULT_OFFSET_A 0
1513 #define GEMM_DEFAULT_OFFSET_B 0
1514 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1518 #if defined(XDOUBLE) || defined(DOUBLE)
1519 #define SWITCH_RATIO 4
1520 #define GEMM_PREFERED_SIZE 4
1522 #define SWITCH_RATIO 8
1523 #define GEMM_PREFERED_SIZE 8
1528 #define SGEMM_DEFAULT_UNROLL_M 4
1529 #define DGEMM_DEFAULT_UNROLL_M 2
1530 #define QGEMM_DEFAULT_UNROLL_M 2
1531 #define CGEMM_DEFAULT_UNROLL_M 2
1532 #define ZGEMM_DEFAULT_UNROLL_M 1
1533 #define XGEMM_DEFAULT_UNROLL_M 1
1535 #define SGEMM_DEFAULT_UNROLL_N 4
1536 #define DGEMM_DEFAULT_UNROLL_N 4
1537 #define QGEMM_DEFAULT_UNROLL_N 2
1538 #define CGEMM_DEFAULT_UNROLL_N 2
1539 #define ZGEMM_DEFAULT_UNROLL_N 2
1540 #define XGEMM_DEFAULT_UNROLL_N 1
1544 #define SGEMM_DEFAULT_UNROLL_M 8
1545 #define DGEMM_DEFAULT_UNROLL_M 4
1546 #define QGEMM_DEFAULT_UNROLL_M 2
1547 #define CGEMM_DEFAULT_UNROLL_M 8
1548 #define ZGEMM_DEFAULT_UNROLL_M 4
1549 #define XGEMM_DEFAULT_UNROLL_M 1
1551 #define SGEMM_DEFAULT_UNROLL_N 4
1552 #define DGEMM_DEFAULT_UNROLL_N 8
1553 #define QGEMM_DEFAULT_UNROLL_N 2
1554 #define CGEMM_DEFAULT_UNROLL_N 2
1555 #define ZGEMM_DEFAULT_UNROLL_N 2
1556 #define XGEMM_DEFAULT_UNROLL_N 1
1558 #define SGEMM_DEFAULT_UNROLL_MN 32
1559 #define DGEMM_DEFAULT_UNROLL_MN 32
1565 #define SGEMM_DEFAULT_P 512
1566 #define SGEMM_DEFAULT_R sgemm_r
1567 #define DGEMM_DEFAULT_P 512
1568 #define DGEMM_DEFAULT_R dgemm_r
1569 #define QGEMM_DEFAULT_P 504
1570 #define QGEMM_DEFAULT_R qgemm_r
1571 #define CGEMM_DEFAULT_P 128
1572 #define CGEMM_DEFAULT_R 1024
1573 #define ZGEMM_DEFAULT_P 512
1574 #define ZGEMM_DEFAULT_R zgemm_r
1575 #define XGEMM_DEFAULT_P 252
1576 #define XGEMM_DEFAULT_R xgemm_r
1577 #define SGEMM_DEFAULT_Q 256
1578 #define DGEMM_DEFAULT_Q 256
1579 #define QGEMM_DEFAULT_Q 128
1580 #define CGEMM_DEFAULT_Q 256
1581 #define ZGEMM_DEFAULT_Q 192
1582 #define XGEMM_DEFAULT_Q 128
1586 #define SGEMM_DEFAULT_P 320
1587 #define DGEMM_DEFAULT_P 512
1588 #define CGEMM_DEFAULT_P 256
1589 #define ZGEMM_DEFAULT_P 192
1592 #define SGEMM_DEFAULT_Q 320
1593 #define DGEMM_DEFAULT_Q 128
1595 #define SGEMM_DEFAULT_Q 320
1596 #define DGEMM_DEFAULT_Q 256
1598 #define CGEMM_DEFAULT_Q 256
1599 #define ZGEMM_DEFAULT_Q 192
1601 #define SGEMM_DEFAULT_R sgemm_r
1602 #define DGEMM_DEFAULT_R 13824
1603 #define CGEMM_DEFAULT_R cgemm_r
1604 #define ZGEMM_DEFAULT_R zgemm_r
1606 #define QGEMM_DEFAULT_Q 128
1607 #define QGEMM_DEFAULT_P 504
1608 #define QGEMM_DEFAULT_R qgemm_r
1609 #define XGEMM_DEFAULT_P 252
1610 #define XGEMM_DEFAULT_R xgemm_r
1611 #define XGEMM_DEFAULT_Q 128
1613 #define CGEMM3M_DEFAULT_UNROLL_N 4
1614 #define CGEMM3M_DEFAULT_UNROLL_M 8
1615 #define ZGEMM3M_DEFAULT_UNROLL_N 4
1616 #define ZGEMM3M_DEFAULT_UNROLL_M 4
1618 #define CGEMM3M_DEFAULT_P 320
1619 #define ZGEMM3M_DEFAULT_P 256
1620 #define XGEMM3M_DEFAULT_P 112
1621 #define CGEMM3M_DEFAULT_Q 320
1622 #define ZGEMM3M_DEFAULT_Q 256
1623 #define XGEMM3M_DEFAULT_Q 224
1624 #define CGEMM3M_DEFAULT_R 12288
1625 #define ZGEMM3M_DEFAULT_R 12288
1626 #define XGEMM3M_DEFAULT_R 12288
1638 #define GEMM_DEFAULT_OFFSET_A 0
1639 #define GEMM_DEFAULT_OFFSET_B 0
1640 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
1644 #if defined(XDOUBLE) || defined(DOUBLE)
1645 #define SWITCH_RATIO 8
1646 #define GEMM_PREFERED_SIZE 8
1648 #define SWITCH_RATIO 16
1649 #define GEMM_PREFERED_SIZE 16
1651 #define USE_SGEMM_KERNEL_DIRECT 1
1655 #define SGEMM_DEFAULT_UNROLL_M 4
1656 #define DGEMM_DEFAULT_UNROLL_M 2
1657 #define QGEMM_DEFAULT_UNROLL_M 2
1658 #define CGEMM_DEFAULT_UNROLL_M 2
1659 #define ZGEMM_DEFAULT_UNROLL_M 1
1660 #define XGEMM_DEFAULT_UNROLL_M 1
1662 #define SGEMM_DEFAULT_UNROLL_N 4
1663 #define DGEMM_DEFAULT_UNROLL_N 4
1664 #define QGEMM_DEFAULT_UNROLL_N 2
1665 #define CGEMM_DEFAULT_UNROLL_N 2
1666 #define ZGEMM_DEFAULT_UNROLL_N 2
1667 #define XGEMM_DEFAULT_UNROLL_N 1
1671 #define SGEMM_DEFAULT_UNROLL_M 16
1673 #define DGEMM_DEFAULT_UNROLL_M 4
1675 #define DGEMM_DEFAULT_UNROLL_M 16
1677 #define QGEMM_DEFAULT_UNROLL_M 2
1678 #define CGEMM_DEFAULT_UNROLL_M 8
1679 #define ZGEMM_DEFAULT_UNROLL_M 4
1680 #define XGEMM_DEFAULT_UNROLL_M 1
1682 #define SGEMM_DEFAULT_UNROLL_N 4
1684 #define DGEMM_DEFAULT_UNROLL_N 8
1686 #define DGEMM_DEFAULT_UNROLL_N 2
1688 #define QGEMM_DEFAULT_UNROLL_N 2
1689 #define CGEMM_DEFAULT_UNROLL_N 2
1690 #define ZGEMM_DEFAULT_UNROLL_N 2
1691 #define XGEMM_DEFAULT_UNROLL_N 1
1693 #define SGEMM_DEFAULT_UNROLL_MN 32
1694 #define DGEMM_DEFAULT_UNROLL_MN 32
1699 #define SGEMM_DEFAULT_P 512
1700 #define SGEMM_DEFAULT_R sgemm_r
1701 #define DGEMM_DEFAULT_P 512
1702 #define DGEMM_DEFAULT_R dgemm_r
1703 #define QGEMM_DEFAULT_P 504
1704 #define QGEMM_DEFAULT_R qgemm_r
1705 #define CGEMM_DEFAULT_P 128
1706 #define CGEMM_DEFAULT_R 1024
1707 #define ZGEMM_DEFAULT_P 512
1708 #define ZGEMM_DEFAULT_R zgemm_r
1709 #define XGEMM_DEFAULT_P 252
1710 #define XGEMM_DEFAULT_R xgemm_r
1711 #define SGEMM_DEFAULT_Q 256
1712 #define DGEMM_DEFAULT_Q 256
1713 #define QGEMM_DEFAULT_Q 128
1714 #define CGEMM_DEFAULT_Q 256
1715 #define ZGEMM_DEFAULT_Q 192
1716 #define XGEMM_DEFAULT_Q 128
1720 #define SGEMM_DEFAULT_P 448
1721 #ifndef DYNAMIC_ARCH
1722 #define DGEMM_DEFAULT_P 192
1724 #define DGEMM_DEFAULT_P 384
1726 #define CGEMM_DEFAULT_P 384
1727 #define ZGEMM_DEFAULT_P 256
1729 #define SGEMM_DEFAULT_Q 448
1730 #ifndef DYNAMIC_ARCH
1731 #define DGEMM_DEFAULT_Q 384
1733 #define DGEMM_DEFAULT_Q 168
1735 #define CGEMM_DEFAULT_Q 192
1736 #define ZGEMM_DEFAULT_Q 128
1738 #define SGEMM_DEFAULT_R sgemm_r
1739 #ifndef DYNAMIC_ARCH
1740 #define DGEMM_DEFAULT_R 8640
1742 #define DGEMM_DEFAULT_R 13824
1744 #define CGEMM_DEFAULT_R cgemm_r
1745 #define ZGEMM_DEFAULT_R zgemm_r
1747 #define QGEMM_DEFAULT_Q 128
1748 #define QGEMM_DEFAULT_P 504
1749 #define QGEMM_DEFAULT_R qgemm_r
1750 #define XGEMM_DEFAULT_P 252
1751 #define XGEMM_DEFAULT_R xgemm_r
1752 #define XGEMM_DEFAULT_Q 128
1754 #define CGEMM3M_DEFAULT_UNROLL_N 4
1755 #define CGEMM3M_DEFAULT_UNROLL_M 8
1756 #define ZGEMM3M_DEFAULT_UNROLL_N 4
1757 #define ZGEMM3M_DEFAULT_UNROLL_M 4
1759 #define CGEMM3M_DEFAULT_P 320
1760 #define ZGEMM3M_DEFAULT_P 256
1761 #define XGEMM3M_DEFAULT_P 112
1762 #define CGEMM3M_DEFAULT_Q 320
1763 #define ZGEMM3M_DEFAULT_Q 256
1764 #define XGEMM3M_DEFAULT_Q 224
1765 #define CGEMM3M_DEFAULT_R 12288
1766 #define ZGEMM3M_DEFAULT_R 12288
1767 #define XGEMM3M_DEFAULT_R 12288
1774 #ifdef SAPPHIRERAPIDS
1779 #define GEMM_DEFAULT_OFFSET_A 0
1780 #define GEMM_DEFAULT_OFFSET_B 0
1781 #define GEMM_DEFAULT_ALIGN 0x03fffUL
1785 #if defined(XDOUBLE) || defined(DOUBLE)
1786 #define SWITCH_RATIO 8
1787 #define GEMM_PREFERED_SIZE 8
1789 #define SWITCH_RATIO 16
1790 #define GEMM_PREFERED_SIZE 16
1792 #define USE_SGEMM_KERNEL_DIRECT 1
1794 #undef SBGEMM_DEFAULT_UNROLL_N
1795 #undef SBGEMM_DEFAULT_UNROLL_M
1796 #undef SBGEMM_DEFAULT_P
1797 #undef SBGEMM_DEFAULT_R
1798 #undef SBGEMM_DEFAULT_Q
1799 // FIXME: actually UNROLL_M = UNROLL_N = 16
1800 // If M and N is equal, OpenBLAS will reuse OCOPY as ICOPY.
1801 // But for AMX, they are not the same, set UNROLL_M = 32 to workaround
1802 #define SBGEMM_DEFAULT_UNROLL_N 16
1803 #define SBGEMM_DEFAULT_UNROLL_M 32
1804 #define SBGEMM_DEFAULT_P 256
1805 #define SBGEMM_DEFAULT_Q 1024
1806 #define SBGEMM_DEFAULT_R sbgemm_r
1810 #define SGEMM_DEFAULT_UNROLL_M 4
1811 #define DGEMM_DEFAULT_UNROLL_M 2
1812 #define QGEMM_DEFAULT_UNROLL_M 2
1813 #define CGEMM_DEFAULT_UNROLL_M 2
1814 #define ZGEMM_DEFAULT_UNROLL_M 1
1815 #define XGEMM_DEFAULT_UNROLL_M 1
1817 #define SGEMM_DEFAULT_UNROLL_N 4
1818 #define DGEMM_DEFAULT_UNROLL_N 4
1819 #define QGEMM_DEFAULT_UNROLL_N 2
1820 #define CGEMM_DEFAULT_UNROLL_N 2
1821 #define ZGEMM_DEFAULT_UNROLL_N 2
1822 #define XGEMM_DEFAULT_UNROLL_N 1
1826 #define SGEMM_DEFAULT_UNROLL_M 16
1827 #define DGEMM_DEFAULT_UNROLL_M 16
1828 #define QGEMM_DEFAULT_UNROLL_M 2
1829 #define CGEMM_DEFAULT_UNROLL_M 8
1830 #define ZGEMM_DEFAULT_UNROLL_M 4
1831 #define XGEMM_DEFAULT_UNROLL_M 1
1833 #define SGEMM_DEFAULT_UNROLL_N 4
1834 #define DGEMM_DEFAULT_UNROLL_N 2
1835 #define QGEMM_DEFAULT_UNROLL_N 2
1836 #define CGEMM_DEFAULT_UNROLL_N 2
1837 #define ZGEMM_DEFAULT_UNROLL_N 2
1838 #define XGEMM_DEFAULT_UNROLL_N 1
1840 #define SGEMM_DEFAULT_UNROLL_MN 32
1841 #define DGEMM_DEFAULT_UNROLL_MN 32
1846 #define SGEMM_DEFAULT_P 512
1847 #define SGEMM_DEFAULT_R sgemm_r
1848 #define DGEMM_DEFAULT_P 512
1849 #define DGEMM_DEFAULT_R dgemm_r
1850 #define QGEMM_DEFAULT_P 504
1851 #define QGEMM_DEFAULT_R qgemm_r
1852 #define CGEMM_DEFAULT_P 128
1853 #define CGEMM_DEFAULT_R 1024
1854 #define ZGEMM_DEFAULT_P 512
1855 #define ZGEMM_DEFAULT_R zgemm_r
1856 #define XGEMM_DEFAULT_P 252
1857 #define XGEMM_DEFAULT_R xgemm_r
1858 #define SGEMM_DEFAULT_Q 256
1859 #define DGEMM_DEFAULT_Q 256
1860 #define QGEMM_DEFAULT_Q 128
1861 #define CGEMM_DEFAULT_Q 256
1862 #define ZGEMM_DEFAULT_Q 192
1863 #define XGEMM_DEFAULT_Q 128
1867 #define SGEMM_DEFAULT_P 640
1868 #define DGEMM_DEFAULT_P 192
1869 #define CGEMM_DEFAULT_P 384
1870 #define ZGEMM_DEFAULT_P 256
1872 #define SGEMM_DEFAULT_Q 320
1873 #define DGEMM_DEFAULT_Q 384
1874 #define CGEMM_DEFAULT_Q 192
1875 #define ZGEMM_DEFAULT_Q 128
1877 #define SGEMM_DEFAULT_R sgemm_r
1878 #define DGEMM_DEFAULT_R 8640
1879 #define CGEMM_DEFAULT_R cgemm_r
1880 #define ZGEMM_DEFAULT_R zgemm_r
1882 #define QGEMM_DEFAULT_Q 128
1883 #define QGEMM_DEFAULT_P 504
1884 #define QGEMM_DEFAULT_R qgemm_r
1885 #define XGEMM_DEFAULT_P 252
1886 #define XGEMM_DEFAULT_R xgemm_r
1887 #define XGEMM_DEFAULT_Q 128
1889 #define CGEMM3M_DEFAULT_UNROLL_N 4
1890 #define CGEMM3M_DEFAULT_UNROLL_M 8
1891 #define ZGEMM3M_DEFAULT_UNROLL_N 4
1892 #define ZGEMM3M_DEFAULT_UNROLL_M 4
1894 #define CGEMM3M_DEFAULT_P 320
1895 #define ZGEMM3M_DEFAULT_P 256
1896 #define XGEMM3M_DEFAULT_P 112
1897 #define CGEMM3M_DEFAULT_Q 320
1898 #define ZGEMM3M_DEFAULT_Q 256
1899 #define XGEMM3M_DEFAULT_Q 224
1900 #define CGEMM3M_DEFAULT_R 12288
1901 #define ZGEMM3M_DEFAULT_R 12288
1902 #define XGEMM3M_DEFAULT_R 12288
1912 #define GEMM_DEFAULT_OFFSET_A 0
1913 #define GEMM_DEFAULT_OFFSET_B 0
1914 #define GEMM_DEFAULT_ALIGN 0x03fffUL
1918 #if defined(XDOUBLE) || defined(DOUBLE)
1919 #define SWITCH_RATIO 8
1920 #define GEMM_PREFERED_SIZE 8
1922 #define SWITCH_RATIO 16
1923 #define GEMM_PREFERED_SIZE 16
1925 #define USE_SGEMM_KERNEL_DIRECT 1
1927 #undef SBGEMM_DEFAULT_UNROLL_N
1928 #undef SBGEMM_DEFAULT_UNROLL_M
1929 #undef SBGEMM_DEFAULT_P
1930 #undef SBGEMM_DEFAULT_R
1931 #undef SBGEMM_DEFAULT_Q
1932 #define SBGEMM_DEFAULT_UNROLL_N 4
1933 #define SBGEMM_DEFAULT_UNROLL_M 16
1934 #define SBGEMM_DEFAULT_P 384
1935 #define SBGEMM_DEFAULT_Q 768
1936 #define SBGEMM_DEFAULT_R sbgemm_r
1940 #define SGEMM_DEFAULT_UNROLL_M 4
1941 #define DGEMM_DEFAULT_UNROLL_M 2
1942 #define QGEMM_DEFAULT_UNROLL_M 2
1943 #define CGEMM_DEFAULT_UNROLL_M 2
1944 #define ZGEMM_DEFAULT_UNROLL_M 1
1945 #define XGEMM_DEFAULT_UNROLL_M 1
1947 #define SGEMM_DEFAULT_UNROLL_N 4
1948 #define DGEMM_DEFAULT_UNROLL_N 4
1949 #define QGEMM_DEFAULT_UNROLL_N 2
1950 #define CGEMM_DEFAULT_UNROLL_N 2
1951 #define ZGEMM_DEFAULT_UNROLL_N 2
1952 #define XGEMM_DEFAULT_UNROLL_N 1
1956 #define SGEMM_DEFAULT_UNROLL_M 16
1957 #define DGEMM_DEFAULT_UNROLL_M 16
1958 #define QGEMM_DEFAULT_UNROLL_M 2
1959 #define CGEMM_DEFAULT_UNROLL_M 8
1960 #define ZGEMM_DEFAULT_UNROLL_M 4
1961 #define XGEMM_DEFAULT_UNROLL_M 1
1963 #define SGEMM_DEFAULT_UNROLL_N 4
1964 #define DGEMM_DEFAULT_UNROLL_N 2
1965 #define QGEMM_DEFAULT_UNROLL_N 2
1966 #define CGEMM_DEFAULT_UNROLL_N 2
1967 #define ZGEMM_DEFAULT_UNROLL_N 2
1968 #define XGEMM_DEFAULT_UNROLL_N 1
1970 #define SGEMM_DEFAULT_UNROLL_MN 32
1971 #define DGEMM_DEFAULT_UNROLL_MN 32
1976 #define SGEMM_DEFAULT_P 512
1977 #define SGEMM_DEFAULT_R sgemm_r
1978 #define DGEMM_DEFAULT_P 512
1979 #define DGEMM_DEFAULT_R dgemm_r
1980 #define QGEMM_DEFAULT_P 504
1981 #define QGEMM_DEFAULT_R qgemm_r
1982 #define CGEMM_DEFAULT_P 128
1983 #define CGEMM_DEFAULT_R 1024
1984 #define ZGEMM_DEFAULT_P 512
1985 #define ZGEMM_DEFAULT_R zgemm_r
1986 #define XGEMM_DEFAULT_P 252
1987 #define XGEMM_DEFAULT_R xgemm_r
1988 #define SGEMM_DEFAULT_Q 256
1989 #define DGEMM_DEFAULT_Q 256
1990 #define QGEMM_DEFAULT_Q 128
1991 #define CGEMM_DEFAULT_Q 256
1992 #define ZGEMM_DEFAULT_Q 192
1993 #define XGEMM_DEFAULT_Q 128
1997 #define SGEMM_DEFAULT_P 640
1998 #define DGEMM_DEFAULT_P 192
1999 #define CGEMM_DEFAULT_P 384
2000 #define ZGEMM_DEFAULT_P 256
2002 #define SGEMM_DEFAULT_Q 320
2003 #define DGEMM_DEFAULT_Q 384
2004 #define CGEMM_DEFAULT_Q 192
2005 #define ZGEMM_DEFAULT_Q 128
2007 #define SGEMM_DEFAULT_R sgemm_r
2008 #define DGEMM_DEFAULT_R 8640
2009 #define CGEMM_DEFAULT_R cgemm_r
2010 #define ZGEMM_DEFAULT_R zgemm_r
2012 #define QGEMM_DEFAULT_Q 128
2013 #define QGEMM_DEFAULT_P 504
2014 #define QGEMM_DEFAULT_R qgemm_r
2015 #define XGEMM_DEFAULT_P 252
2016 #define XGEMM_DEFAULT_R xgemm_r
2017 #define XGEMM_DEFAULT_Q 128
2019 #define CGEMM3M_DEFAULT_UNROLL_N 4
2020 #define CGEMM3M_DEFAULT_UNROLL_M 8
2021 #define ZGEMM3M_DEFAULT_UNROLL_N 4
2022 #define ZGEMM3M_DEFAULT_UNROLL_M 4
2024 #define CGEMM3M_DEFAULT_P 320
2025 #define ZGEMM3M_DEFAULT_P 256
2026 #define XGEMM3M_DEFAULT_P 112
2027 #define CGEMM3M_DEFAULT_Q 320
2028 #define ZGEMM3M_DEFAULT_Q 256
2029 #define XGEMM3M_DEFAULT_Q 224
2030 #define CGEMM3M_DEFAULT_R 12288
2031 #define ZGEMM3M_DEFAULT_R 12288
2032 #define XGEMM3M_DEFAULT_R 12288
2043 #define GEMM_DEFAULT_OFFSET_A 64
2044 #define GEMM_DEFAULT_OFFSET_B 0
2045 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
2050 #define SGEMM_DEFAULT_UNROLL_M 4
2051 #define DGEMM_DEFAULT_UNROLL_M 2
2052 #define QGEMM_DEFAULT_UNROLL_M 2
2053 #define CGEMM_DEFAULT_UNROLL_M 2
2054 #define ZGEMM_DEFAULT_UNROLL_M 1
2055 #define XGEMM_DEFAULT_UNROLL_M 1
2057 #define SGEMM_DEFAULT_UNROLL_M 8
2058 #define DGEMM_DEFAULT_UNROLL_M 4
2059 #define QGEMM_DEFAULT_UNROLL_M 2
2060 #define CGEMM_DEFAULT_UNROLL_M 4
2061 #define ZGEMM_DEFAULT_UNROLL_M 2
2062 #define XGEMM_DEFAULT_UNROLL_M 1
2065 #define SGEMM_DEFAULT_UNROLL_N 4
2066 #define DGEMM_DEFAULT_UNROLL_N 2
2067 #define QGEMM_DEFAULT_UNROLL_N 2
2068 #define CGEMM_DEFAULT_UNROLL_N 2
2069 #define ZGEMM_DEFAULT_UNROLL_N 1
2070 #define XGEMM_DEFAULT_UNROLL_N 1
2072 #define SGEMM_DEFAULT_P sgemm_p
2073 #define SGEMM_DEFAULT_R sgemm_r
2075 #define DGEMM_DEFAULT_P dgemm_p
2076 #define DGEMM_DEFAULT_R dgemm_r
2078 #define QGEMM_DEFAULT_P qgemm_p
2079 #define QGEMM_DEFAULT_R qgemm_r
2081 #define CGEMM_DEFAULT_P cgemm_p
2082 #define CGEMM_DEFAULT_R cgemm_r
2084 #define ZGEMM_DEFAULT_P zgemm_p
2085 #define ZGEMM_DEFAULT_R zgemm_r
2087 #define XGEMM_DEFAULT_P xgemm_p
2088 #define XGEMM_DEFAULT_R xgemm_r
2090 #define SGEMM_DEFAULT_Q 256
2091 #define DGEMM_DEFAULT_Q 256
2092 #define QGEMM_DEFAULT_Q 256
2093 #define CGEMM_DEFAULT_Q 256
2094 #define ZGEMM_DEFAULT_Q 256
2095 #define XGEMM_DEFAULT_Q 256
2105 #define GEMM_DEFAULT_OFFSET_A 0
2106 #define GEMM_DEFAULT_OFFSET_B 128
2107 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
2109 #define SGEMM_DEFAULT_UNROLL_M 8
2110 #define SGEMM_DEFAULT_UNROLL_N 8
2111 #define DGEMM_DEFAULT_UNROLL_M 8
2112 #define DGEMM_DEFAULT_UNROLL_N 8
2113 #define QGEMM_DEFAULT_UNROLL_M 8
2114 #define QGEMM_DEFAULT_UNROLL_N 8
2115 #define CGEMM_DEFAULT_UNROLL_M 4
2116 #define CGEMM_DEFAULT_UNROLL_N 4
2117 #define ZGEMM_DEFAULT_UNROLL_M 4
2118 #define ZGEMM_DEFAULT_UNROLL_N 4
2119 #define XGEMM_DEFAULT_UNROLL_M 4
2120 #define XGEMM_DEFAULT_UNROLL_N 4
2122 #define SGEMM_DEFAULT_P sgemm_p
2123 #define DGEMM_DEFAULT_P dgemm_p
2124 #define QGEMM_DEFAULT_P qgemm_p
2125 #define CGEMM_DEFAULT_P cgemm_p
2126 #define ZGEMM_DEFAULT_P zgemm_p
2127 #define XGEMM_DEFAULT_P xgemm_p
2129 #define SGEMM_DEFAULT_Q 1024
2130 #define DGEMM_DEFAULT_Q 1024
2131 #define QGEMM_DEFAULT_Q 1024
2132 #define CGEMM_DEFAULT_Q 1024
2133 #define ZGEMM_DEFAULT_Q 1024
2134 #define XGEMM_DEFAULT_Q 1024
2136 #define SGEMM_DEFAULT_R sgemm_r
2137 #define DGEMM_DEFAULT_R dgemm_r
2138 #define QGEMM_DEFAULT_R qgemm_r
2139 #define CGEMM_DEFAULT_R cgemm_r
2140 #define ZGEMM_DEFAULT_R zgemm_r
2141 #define XGEMM_DEFAULT_R xgemm_r
2145 #define GETRF_FACTOR 0.65
2149 #if defined(EV4) || defined(EV5) || defined(EV6)
2159 #define GEMM_DEFAULT_OFFSET_A 512
2160 #define GEMM_DEFAULT_OFFSET_B 512
2161 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
2163 #define SGEMM_DEFAULT_UNROLL_M 4
2164 #define SGEMM_DEFAULT_UNROLL_N 4
2165 #define DGEMM_DEFAULT_UNROLL_M 4
2166 #define DGEMM_DEFAULT_UNROLL_N 4
2167 #define CGEMM_DEFAULT_UNROLL_M 2
2168 #define CGEMM_DEFAULT_UNROLL_N 2
2169 #define ZGEMM_DEFAULT_UNROLL_M 2
2170 #define ZGEMM_DEFAULT_UNROLL_N 2
2175 #define SGEMM_DEFAULT_P 32
2176 #define SGEMM_DEFAULT_Q 112
2177 #define SGEMM_DEFAULT_R 256
2179 #define DGEMM_DEFAULT_P 32
2180 #define DGEMM_DEFAULT_Q 56
2181 #define DGEMM_DEFAULT_R 256
2183 #define CGEMM_DEFAULT_P 32
2184 #define CGEMM_DEFAULT_Q 64
2185 #define CGEMM_DEFAULT_R 240
2187 #define ZGEMM_DEFAULT_P 32
2188 #define ZGEMM_DEFAULT_Q 32
2189 #define ZGEMM_DEFAULT_R 240
2193 #define SGEMM_DEFAULT_P 64
2194 #define SGEMM_DEFAULT_Q 256
2196 #define DGEMM_DEFAULT_P 64
2197 #define DGEMM_DEFAULT_Q 128
2199 #define CGEMM_DEFAULT_P 64
2200 #define CGEMM_DEFAULT_Q 128
2202 #define ZGEMM_DEFAULT_P 64
2203 #define ZGEMM_DEFAULT_Q 64
2207 #define SGEMM_DEFAULT_P 256
2208 #define SGEMM_DEFAULT_Q 512
2210 #define DGEMM_DEFAULT_P 256
2211 #define DGEMM_DEFAULT_Q 256
2213 #define CGEMM_DEFAULT_P 256
2214 #define CGEMM_DEFAULT_Q 256
2216 #define ZGEMM_DEFAULT_P 128
2217 #define ZGEMM_DEFAULT_Q 256
2227 #define GEMM_DEFAULT_OFFSET_A 0
2228 #define GEMM_DEFAULT_OFFSET_B 8192
2229 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
2231 #define SGEMM_DEFAULT_UNROLL_M 16
2232 #define SGEMM_DEFAULT_UNROLL_N 4
2233 #define DGEMM_DEFAULT_UNROLL_M 4
2234 #define DGEMM_DEFAULT_UNROLL_N 4
2235 #define CGEMM_DEFAULT_UNROLL_M 8
2236 #define CGEMM_DEFAULT_UNROLL_N 2
2237 #define ZGEMM_DEFAULT_UNROLL_M 2
2238 #define ZGEMM_DEFAULT_UNROLL_N 2
2240 #define SGEMM_DEFAULT_P 128
2241 #define DGEMM_DEFAULT_P 128
2242 #define CGEMM_DEFAULT_P 128
2243 #define ZGEMM_DEFAULT_P 128
2245 #define SGEMM_DEFAULT_Q 512
2246 #define DGEMM_DEFAULT_Q 256
2247 #define CGEMM_DEFAULT_Q 256
2248 #define ZGEMM_DEFAULT_Q 128
2254 #define GEMM_DEFAULT_OFFSET_A 0
2255 #define GEMM_DEFAULT_OFFSET_B 1024
2256 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2258 #define SGEMM_DEFAULT_UNROLL_M 16
2259 #define SGEMM_DEFAULT_UNROLL_N 4
2260 #define DGEMM_DEFAULT_UNROLL_M 4
2261 #define DGEMM_DEFAULT_UNROLL_N 4
2262 #define CGEMM_DEFAULT_UNROLL_M 2
2263 #define CGEMM_DEFAULT_UNROLL_N 2
2264 #define ZGEMM_DEFAULT_UNROLL_M 2
2265 #define ZGEMM_DEFAULT_UNROLL_N 2
2267 #define SGEMM_DEFAULT_P 256
2268 #define DGEMM_DEFAULT_P 128
2269 #define CGEMM_DEFAULT_P 128
2270 #define ZGEMM_DEFAULT_P 64
2272 #define SGEMM_DEFAULT_Q 256
2273 #define DGEMM_DEFAULT_Q 256
2274 #define CGEMM_DEFAULT_Q 256
2275 #define ZGEMM_DEFAULT_Q 256
2285 #define GEMM_DEFAULT_OFFSET_A 2688
2286 #define GEMM_DEFAULT_OFFSET_B 3072
2287 #define GEMM_DEFAULT_ALIGN 0x03fffUL
2289 #if defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2290 #define SGEMM_DEFAULT_UNROLL_M 4
2292 #define SGEMM_DEFAULT_UNROLL_M 16
2294 #define SGEMM_DEFAULT_UNROLL_N 4
2295 #define DGEMM_DEFAULT_UNROLL_M 4
2296 #define DGEMM_DEFAULT_UNROLL_N 4
2297 #if defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2298 #define CGEMM_DEFAULT_UNROLL_M 2
2300 #define CGEMM_DEFAULT_UNROLL_M 8
2302 #define CGEMM_DEFAULT_UNROLL_N 2
2303 #define ZGEMM_DEFAULT_UNROLL_M 2
2304 #define ZGEMM_DEFAULT_UNROLL_N 2
2306 #if defined(OS_LINUX) || defined(OS_DARWIN) || defined(OS_FREEBSD)
2307 #if L2_SIZE == 1024976
2308 #define SGEMM_DEFAULT_P 320
2309 #define DGEMM_DEFAULT_P 256
2310 #define CGEMM_DEFAULT_P 256
2311 #define ZGEMM_DEFAULT_P 256
2313 #define SGEMM_DEFAULT_P 176
2314 #define DGEMM_DEFAULT_P 176
2315 #define CGEMM_DEFAULT_P 176
2316 #define ZGEMM_DEFAULT_P 176
2320 #define SGEMM_DEFAULT_Q 512
2321 #define DGEMM_DEFAULT_Q 256
2322 #define CGEMM_DEFAULT_Q 256
2323 #define ZGEMM_DEFAULT_Q 128
2334 #define GEMM_DEFAULT_OFFSET_A (32 * 0)
2335 #define GEMM_DEFAULT_OFFSET_B (32 * 0)
2336 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2338 #define SGEMM_DEFAULT_UNROLL_M 4
2339 #define SGEMM_DEFAULT_UNROLL_N 4
2340 #define DGEMM_DEFAULT_UNROLL_M 4
2341 #define DGEMM_DEFAULT_UNROLL_N 4
2342 #define CGEMM_DEFAULT_UNROLL_M 2
2343 #define CGEMM_DEFAULT_UNROLL_N 2
2344 #define ZGEMM_DEFAULT_UNROLL_M 2
2345 #define ZGEMM_DEFAULT_UNROLL_N 2
2347 #define SGEMM_DEFAULT_P 512
2348 #define DGEMM_DEFAULT_P 512
2349 #define CGEMM_DEFAULT_P 512
2350 #define ZGEMM_DEFAULT_P 512
2352 #define SGEMM_DEFAULT_Q 1024
2353 #define DGEMM_DEFAULT_Q 512
2354 #define CGEMM_DEFAULT_Q 512
2355 #define ZGEMM_DEFAULT_Q 256
2357 #define SGEMM_DEFAULT_R SGEMM_DEFAULT_P
2358 #define DGEMM_DEFAULT_R DGEMM_DEFAULT_P
2359 #define CGEMM_DEFAULT_R CGEMM_DEFAULT_P
2360 #define ZGEMM_DEFAULT_R ZGEMM_DEFAULT_P
2370 #define GEMM_DEFAULT_OFFSET_A (32 * 0)
2371 #define GEMM_DEFAULT_OFFSET_B (32 * 0)
2372 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2374 #define SGEMM_DEFAULT_UNROLL_M 8
2375 #define SGEMM_DEFAULT_UNROLL_N 4
2376 #define DGEMM_DEFAULT_UNROLL_M 8
2377 #define DGEMM_DEFAULT_UNROLL_N 4
2378 #define CGEMM_DEFAULT_UNROLL_M 4
2379 #define CGEMM_DEFAULT_UNROLL_N 2
2380 #define ZGEMM_DEFAULT_UNROLL_M 4
2381 #define ZGEMM_DEFAULT_UNROLL_N 2
2383 #define SGEMM_DEFAULT_P 128
2384 #define DGEMM_DEFAULT_P 128
2385 #define CGEMM_DEFAULT_P 128
2386 #define ZGEMM_DEFAULT_P 128
2388 #define SGEMM_DEFAULT_Q 4096
2389 #define DGEMM_DEFAULT_Q 3072
2390 #define CGEMM_DEFAULT_Q 2048
2391 #define ZGEMM_DEFAULT_Q 1024
2393 #define SGEMM_DEFAULT_Q 512
2394 #define DGEMM_DEFAULT_Q 256
2395 #define CGEMM_DEFAULT_Q 256
2396 #define ZGEMM_DEFAULT_Q 128
2404 #if defined(POWER3) || defined(POWER4) || defined(POWER5)
2405 #define GEMM_DEFAULT_OFFSET_A 0
2406 #define GEMM_DEFAULT_OFFSET_B 2048
2407 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2409 #define SGEMM_DEFAULT_UNROLL_M 4
2410 #define SGEMM_DEFAULT_UNROLL_N 4
2411 #define DGEMM_DEFAULT_UNROLL_M 4
2412 #define DGEMM_DEFAULT_UNROLL_N 4
2413 #define CGEMM_DEFAULT_UNROLL_M 2
2414 #define CGEMM_DEFAULT_UNROLL_N 2
2415 #define ZGEMM_DEFAULT_UNROLL_M 2
2416 #define ZGEMM_DEFAULT_UNROLL_N 2
2423 #define SGEMM_DEFAULT_P 256
2424 #define SGEMM_DEFAULT_Q 432
2425 #define SGEMM_DEFAULT_R 1012
2427 #define DGEMM_DEFAULT_P 256
2428 #define DGEMM_DEFAULT_Q 216
2429 #define DGEMM_DEFAULT_R 1012
2431 #define CGEMM_DEFAULT_P 256
2432 #define CGEMM_DEFAULT_Q 104
2433 #define CGEMM_DEFAULT_R 1012
2435 #define ZGEMM_DEFAULT_P 256
2436 #define ZGEMM_DEFAULT_Q 104
2437 #define ZGEMM_DEFAULT_R 1012
2441 #ifdef ALLOC_HUGETLB
2442 #define SGEMM_DEFAULT_P 184
2443 #define DGEMM_DEFAULT_P 184
2444 #define CGEMM_DEFAULT_P 184
2445 #define ZGEMM_DEFAULT_P 184
2447 #define SGEMM_DEFAULT_P 144
2448 #define DGEMM_DEFAULT_P 144
2449 #define CGEMM_DEFAULT_P 144
2450 #define ZGEMM_DEFAULT_P 144
2453 #define SGEMM_DEFAULT_Q 256
2454 #define CGEMM_DEFAULT_Q 256
2455 #define DGEMM_DEFAULT_Q 256
2456 #define ZGEMM_DEFAULT_Q 256
2460 #ifdef ALLOC_HUGETLB
2461 #define SGEMM_DEFAULT_P 512
2462 #define DGEMM_DEFAULT_P 256
2463 #define CGEMM_DEFAULT_P 256
2464 #define ZGEMM_DEFAULT_P 128
2466 #define SGEMM_DEFAULT_P 320
2467 #define DGEMM_DEFAULT_P 160
2468 #define CGEMM_DEFAULT_P 160
2469 #define ZGEMM_DEFAULT_P 80
2472 #define SGEMM_DEFAULT_Q 256
2473 #define CGEMM_DEFAULT_Q 256
2474 #define DGEMM_DEFAULT_Q 256
2475 #define ZGEMM_DEFAULT_Q 256
2487 #define GEMM_DEFAULT_OFFSET_A 384
2488 #define GEMM_DEFAULT_OFFSET_B 1024
2489 #define GEMM_DEFAULT_ALIGN 0x03fffUL
2491 #define SGEMM_DEFAULT_UNROLL_M 4
2492 #define SGEMM_DEFAULT_UNROLL_N 4
2493 #define DGEMM_DEFAULT_UNROLL_M 4
2494 #define DGEMM_DEFAULT_UNROLL_N 4
2495 #define CGEMM_DEFAULT_UNROLL_M 2
2496 #define CGEMM_DEFAULT_UNROLL_N 4
2497 #define ZGEMM_DEFAULT_UNROLL_M 2
2498 #define ZGEMM_DEFAULT_UNROLL_N 4
2500 #define SGEMM_DEFAULT_P 992
2501 #define DGEMM_DEFAULT_P 480
2502 #define CGEMM_DEFAULT_P 488
2503 #define ZGEMM_DEFAULT_P 248
2505 #define SGEMM_DEFAULT_Q 504
2506 #define DGEMM_DEFAULT_Q 504
2507 #define CGEMM_DEFAULT_Q 400
2508 #define ZGEMM_DEFAULT_Q 400
2519 #define GEMM_DEFAULT_OFFSET_A 0
2520 #define GEMM_DEFAULT_OFFSET_B 65536
2522 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2523 #if defined(__32BIT__)
2524 #warning using BINARY32==POWER6
2525 #define SGEMM_DEFAULT_UNROLL_M 4
2526 #define SGEMM_DEFAULT_UNROLL_N 4
2527 #define DGEMM_DEFAULT_UNROLL_M 4
2528 #define DGEMM_DEFAULT_UNROLL_N 4
2529 #define CGEMM_DEFAULT_UNROLL_M 2
2530 #define CGEMM_DEFAULT_UNROLL_N 4
2531 #define ZGEMM_DEFAULT_UNROLL_M 2
2532 #define ZGEMM_DEFAULT_UNROLL_N 4
2534 #define SGEMM_DEFAULT_UNROLL_M 16
2535 #define SGEMM_DEFAULT_UNROLL_N 8
2536 #define DGEMM_DEFAULT_UNROLL_M 16
2537 #define DGEMM_DEFAULT_UNROLL_N 4
2538 #define CGEMM_DEFAULT_UNROLL_M 8
2539 #define CGEMM_DEFAULT_UNROLL_N 4
2540 #define ZGEMM_DEFAULT_UNROLL_M 8
2541 #define ZGEMM_DEFAULT_UNROLL_N 2
2543 #define SGEMM_DEFAULT_P 1280UL
2544 #define DGEMM_DEFAULT_P 640UL
2545 #define CGEMM_DEFAULT_P 640UL
2546 #define ZGEMM_DEFAULT_P 320UL
2548 #define SGEMM_DEFAULT_Q 640UL
2549 #define DGEMM_DEFAULT_Q 720UL
2550 #define CGEMM_DEFAULT_Q 640UL
2551 #define ZGEMM_DEFAULT_Q 640UL
2554 #define SGEMM_DEFAULT_R SGEMM_DEFAULT_P
2555 #define DGEMM_DEFAULT_R DGEMM_DEFAULT_P
2556 #define CGEMM_DEFAULT_R CGEMM_DEFAULT_P
2557 #define ZGEMM_DEFAULT_R ZGEMM_DEFAULT_P
2559 #define SGEMM_DEFAULT_R 4096
2560 #define DGEMM_DEFAULT_R 4096
2561 #define CGEMM_DEFAULT_R 4096
2562 #define ZGEMM_DEFAULT_R 4096
2573 #define GEMM_DEFAULT_OFFSET_A 0
2574 #define GEMM_DEFAULT_OFFSET_B 65536
2575 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2577 #define SWITCH_RATIO 16
2578 #define GEMM_PREFERED_SIZE 16
2580 #define SGEMM_DEFAULT_UNROLL_M 16
2581 #define SGEMM_DEFAULT_UNROLL_N 8
2582 #define DGEMM_DEFAULT_UNROLL_M 16
2583 #define DGEMM_DEFAULT_UNROLL_N 4
2584 #define CGEMM_DEFAULT_UNROLL_M 8
2585 #define CGEMM_DEFAULT_UNROLL_N 4
2586 #define ZGEMM_DEFAULT_UNROLL_M 8
2587 #define ZGEMM_DEFAULT_UNROLL_N 2
2589 #define SGEMM_DEFAULT_P 832
2590 #define DGEMM_DEFAULT_P 128
2591 #define CGEMM_DEFAULT_P 512
2592 #define ZGEMM_DEFAULT_P 256
2594 #define SGEMM_DEFAULT_Q 1026
2595 #define DGEMM_DEFAULT_Q 384
2596 #define CGEMM_DEFAULT_Q 1026
2597 #define ZGEMM_DEFAULT_Q 1026
2599 #define SGEMM_DEFAULT_R 4096
2600 #define DGEMM_DEFAULT_R 4096
2601 #define CGEMM_DEFAULT_R 4096
2602 #define ZGEMM_DEFAULT_R 4096
2608 #if defined(POWER10)
2612 #define GEMM_DEFAULT_OFFSET_A 0
2613 #define GEMM_DEFAULT_OFFSET_B 65536
2614 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2616 #define SWITCH_RATIO 16
2617 #define GEMM_PREFERED_SIZE 16
2619 #define SGEMM_DEFAULT_UNROLL_M 16
2620 #define SGEMM_DEFAULT_UNROLL_N 8
2621 #if defined(HAVE_GAS) && (HAVE_GAS == 1)
2622 #define DGEMM_DEFAULT_UNROLL_M 16
2623 #define DGEMM_DEFAULT_UNROLL_N 4
2625 #define DGEMM_DEFAULT_UNROLL_M 8
2626 #define DGEMM_DEFAULT_UNROLL_N 8
2628 #define CGEMM_DEFAULT_UNROLL_M 8
2629 #define CGEMM_DEFAULT_UNROLL_N 4
2630 #define ZGEMM_DEFAULT_UNROLL_M 8
2631 #define ZGEMM_DEFAULT_UNROLL_N 2
2633 #define SGEMM_DEFAULT_P 512
2634 #define DGEMM_DEFAULT_P 384
2635 #define CGEMM_DEFAULT_P 512
2636 #define ZGEMM_DEFAULT_P 256
2638 #define SGEMM_DEFAULT_Q 512
2639 #define DGEMM_DEFAULT_Q 512
2640 #define CGEMM_DEFAULT_Q 384
2641 #define ZGEMM_DEFAULT_Q 384
2643 #define SGEMM_DEFAULT_R 4096
2644 #define DGEMM_DEFAULT_R 4096
2645 #define CGEMM_DEFAULT_R 4096
2646 #define ZGEMM_DEFAULT_R 4096
2650 #undef SBGEMM_DEFAULT_UNROLL_N
2651 #undef SBGEMM_DEFAULT_UNROLL_M
2652 #undef SBGEMM_DEFAULT_P
2653 #undef SBGEMM_DEFAULT_R
2654 #undef SBGEMM_DEFAULT_Q
2655 #define SBGEMM_DEFAULT_UNROLL_M 16
2656 #define SBGEMM_DEFAULT_UNROLL_N 8
2657 #define SBGEMM_DEFAULT_P 832
2658 #define SBGEMM_DEFAULT_Q 1026
2659 #define SBGEMM_DEFAULT_R 4096
2662 #if defined(SPARC) && defined(V7)
2667 #define GEMM_DEFAULT_OFFSET_A 0
2668 #define GEMM_DEFAULT_OFFSET_B 2048
2669 #define GEMM_DEFAULT_ALIGN 0x03fffUL
2671 #define SGEMM_DEFAULT_UNROLL_M 2
2672 #define SGEMM_DEFAULT_UNROLL_N 8
2673 #define DGEMM_DEFAULT_UNROLL_M 2
2674 #define DGEMM_DEFAULT_UNROLL_N 8
2675 #define CGEMM_DEFAULT_UNROLL_M 1
2676 #define CGEMM_DEFAULT_UNROLL_N 4
2677 #define ZGEMM_DEFAULT_UNROLL_M 1
2678 #define ZGEMM_DEFAULT_UNROLL_N 4
2680 #define SGEMM_DEFAULT_P 256
2681 #define DGEMM_DEFAULT_P 256
2682 #define CGEMM_DEFAULT_P 256
2683 #define ZGEMM_DEFAULT_P 256
2685 #define SGEMM_DEFAULT_Q 512
2686 #define DGEMM_DEFAULT_Q 256
2687 #define CGEMM_DEFAULT_Q 256
2688 #define ZGEMM_DEFAULT_Q 128
2691 #define GEMM_THREAD gemm_thread_mn
2694 #if (defined(SPARC) && defined(V9)) || defined(__sparc_v9__)
2699 #define GEMM_DEFAULT_OFFSET_A 0
2700 #define GEMM_DEFAULT_OFFSET_B 2048
2701 #define GEMM_DEFAULT_ALIGN 0x03fffUL
2703 #define SGEMM_DEFAULT_UNROLL_M 4
2704 #define SGEMM_DEFAULT_UNROLL_N 4
2705 #define DGEMM_DEFAULT_UNROLL_M 4
2706 #define DGEMM_DEFAULT_UNROLL_N 4
2707 #define CGEMM_DEFAULT_UNROLL_M 2
2708 #define CGEMM_DEFAULT_UNROLL_N 2
2709 #define ZGEMM_DEFAULT_UNROLL_M 2
2710 #define ZGEMM_DEFAULT_UNROLL_N 2
2712 #define SGEMM_DEFAULT_P 512
2713 #define DGEMM_DEFAULT_P 512
2714 #define CGEMM_DEFAULT_P 512
2715 #define ZGEMM_DEFAULT_P 512
2717 #define SGEMM_DEFAULT_Q 1024
2718 #define DGEMM_DEFAULT_Q 512
2719 #define CGEMM_DEFAULT_Q 512
2720 #define ZGEMM_DEFAULT_Q 256
2730 #define GEMM_DEFAULT_OFFSET_A 0
2731 #define GEMM_DEFAULT_OFFSET_B 0
2732 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
2734 #define SGEMM_DEFAULT_UNROLL_M 2
2735 #define SGEMM_DEFAULT_UNROLL_N 8
2736 #define DGEMM_DEFAULT_UNROLL_M 2
2737 #define DGEMM_DEFAULT_UNROLL_N 8
2738 #define CGEMM_DEFAULT_UNROLL_M 1
2739 #define CGEMM_DEFAULT_UNROLL_N 4
2740 #define ZGEMM_DEFAULT_UNROLL_M 1
2741 #define ZGEMM_DEFAULT_UNROLL_N 4
2743 #define SGEMM_DEFAULT_P 108
2744 #define DGEMM_DEFAULT_P 112
2745 #define CGEMM_DEFAULT_P 108
2746 #define ZGEMM_DEFAULT_P 112
2748 #define SGEMM_DEFAULT_Q 288
2749 #define DGEMM_DEFAULT_Q 144
2750 #define CGEMM_DEFAULT_Q 144
2751 #define ZGEMM_DEFAULT_Q 72
2753 #define SGEMM_DEFAULT_R 2000
2754 #define DGEMM_DEFAULT_R 2000
2755 #define CGEMM_DEFAULT_R 2000
2756 #define ZGEMM_DEFAULT_R 2000
2761 #if defined(LOONGSON3R4)
2765 #define GEMM_DEFAULT_OFFSET_A 0
2766 #define GEMM_DEFAULT_OFFSET_B 0
2767 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
2770 #define SGEMM_DEFAULT_UNROLL_M 8
2771 #define SGEMM_DEFAULT_UNROLL_N 8
2773 #define DGEMM_DEFAULT_UNROLL_M 8
2774 #define DGEMM_DEFAULT_UNROLL_N 4
2776 #define CGEMM_DEFAULT_UNROLL_M 8
2777 #define CGEMM_DEFAULT_UNROLL_N 4
2779 #define ZGEMM_DEFAULT_UNROLL_M 4
2780 #define ZGEMM_DEFAULT_UNROLL_N 4
2782 #define SGEMM_DEFAULT_UNROLL_M 8
2783 #define SGEMM_DEFAULT_UNROLL_N 4
2785 #define DGEMM_DEFAULT_UNROLL_M 4
2786 #define DGEMM_DEFAULT_UNROLL_N 4
2788 #define CGEMM_DEFAULT_UNROLL_M 4
2789 #define CGEMM_DEFAULT_UNROLL_N 2
2791 #define ZGEMM_DEFAULT_UNROLL_M 2
2792 #define ZGEMM_DEFAULT_UNROLL_N 2
2795 #define SGEMM_DEFAULT_P 64
2796 #define DGEMM_DEFAULT_P 44
2797 #define CGEMM_DEFAULT_P 64
2798 #define ZGEMM_DEFAULT_P 32
2800 #define SGEMM_DEFAULT_Q 192
2801 #define DGEMM_DEFAULT_Q 92
2802 #define CGEMM_DEFAULT_Q 128
2803 #define ZGEMM_DEFAULT_Q 80
2805 #define SGEMM_DEFAULT_R 640
2806 #define DGEMM_DEFAULT_R dgemm_r
2807 #define CGEMM_DEFAULT_R 640
2808 #define ZGEMM_DEFAULT_R 640
2810 #define GEMM_OFFSET_A1 0x10000
2811 #define GEMM_OFFSET_B1 0x100000
2816 #if defined(LOONGSON3R3)
2817 ////Copy from SICORTEX
2821 #define GEMM_DEFAULT_OFFSET_A 0
2822 #define GEMM_DEFAULT_OFFSET_B 0
2823 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
2825 #define SGEMM_DEFAULT_UNROLL_M 8
2826 #define SGEMM_DEFAULT_UNROLL_N 4
2828 #define DGEMM_DEFAULT_UNROLL_M 4
2829 #define DGEMM_DEFAULT_UNROLL_N 4
2831 #define CGEMM_DEFAULT_UNROLL_M 4
2832 #define CGEMM_DEFAULT_UNROLL_N 2
2834 #define ZGEMM_DEFAULT_UNROLL_M 2
2835 #define ZGEMM_DEFAULT_UNROLL_N 2
2837 #define SGEMM_DEFAULT_P 64
2838 #define DGEMM_DEFAULT_P 44
2839 #define CGEMM_DEFAULT_P 64
2840 #define ZGEMM_DEFAULT_P 32
2842 #define SGEMM_DEFAULT_Q 192
2843 #define DGEMM_DEFAULT_Q 92
2844 #define CGEMM_DEFAULT_Q 128
2845 #define ZGEMM_DEFAULT_Q 80
2847 #define SGEMM_DEFAULT_R 640
2848 #define DGEMM_DEFAULT_R dgemm_r
2849 #define CGEMM_DEFAULT_R 640
2850 #define ZGEMM_DEFAULT_R 640
2852 #define GEMM_OFFSET_A1 0x10000
2853 #define GEMM_OFFSET_B1 0x100000
2858 #if defined (LOONGSON3R5)
2862 #define GEMM_DEFAULT_OFFSET_A 0
2863 #define GEMM_DEFAULT_OFFSET_B 0
2864 #define GEMM_DEFAULT_ALIGN 0x0ffffUL
2866 #define SGEMM_DEFAULT_UNROLL_N 8
2867 #define DGEMM_DEFAULT_UNROLL_N 4
2868 #define QGEMM_DEFAULT_UNROLL_N 2
2869 #define CGEMM_DEFAULT_UNROLL_N 4
2870 #define ZGEMM_DEFAULT_UNROLL_N 4
2871 #define XGEMM_DEFAULT_UNROLL_N 1
2873 #define SGEMM_DEFAULT_UNROLL_M 2
2874 #define DGEMM_DEFAULT_UNROLL_M 16
2875 #define QGEMM_DEFAULT_UNROLL_M 2
2876 #define CGEMM_DEFAULT_UNROLL_M 1
2877 #define ZGEMM_DEFAULT_UNROLL_M 1
2878 #define XGEMM_DEFAULT_UNROLL_M 1
2880 #define SGEMM_DEFAULT_P sgemm_p
2881 #define DGEMM_DEFAULT_P 32
2882 #define QGEMM_DEFAULT_P qgemm_p
2883 #define CGEMM_DEFAULT_P cgemm_p
2884 #define ZGEMM_DEFAULT_P zgemm_p
2885 #define XGEMM_DEFAULT_P xgemm_p
2887 #define SGEMM_DEFAULT_R sgemm_r
2888 #define DGEMM_DEFAULT_R 858
2889 #define QGEMM_DEFAULT_R qgemm_r
2890 #define CGEMM_DEFAULT_R cgemm_r
2891 #define ZGEMM_DEFAULT_R zgemm_r
2892 #define XGEMM_DEFAULT_R xgemm_r
2894 #define SGEMM_DEFAULT_Q 128
2895 #define DGEMM_DEFAULT_Q 152
2896 #define QGEMM_DEFAULT_Q 128
2897 #define CGEMM_DEFAULT_Q 128
2898 #define ZGEMM_DEFAULT_Q 128
2899 #define XGEMM_DEFAULT_Q 128
2904 #if defined(P5600) || defined(MIPS1004K) || defined(MIPS24K) || defined(I6400) || defined(P6600) || defined(I6500)
2908 #define GEMM_DEFAULT_OFFSET_A 0
2909 #define GEMM_DEFAULT_OFFSET_B 0
2910 #define GEMM_DEFAULT_ALIGN (BLASLONG) 0x03fffUL
2912 #if defined(HAVE_MSA) && !defined(NO_MSA)
2913 #define SGEMM_DEFAULT_UNROLL_M 8
2914 #define SGEMM_DEFAULT_UNROLL_N 8
2916 #define DGEMM_DEFAULT_UNROLL_M 8
2917 #define DGEMM_DEFAULT_UNROLL_N 4
2919 #define CGEMM_DEFAULT_UNROLL_M 8
2920 #define CGEMM_DEFAULT_UNROLL_N 4
2922 #define ZGEMM_DEFAULT_UNROLL_M 4
2923 #define ZGEMM_DEFAULT_UNROLL_N 4
2925 #define SGEMM_DEFAULT_UNROLL_M 2
2926 #define SGEMM_DEFAULT_UNROLL_N 2
2928 #define DGEMM_DEFAULT_UNROLL_M 2
2929 #define DGEMM_DEFAULT_UNROLL_N 2
2931 #define CGEMM_DEFAULT_UNROLL_M 2
2932 #define CGEMM_DEFAULT_UNROLL_N 2
2934 #define ZGEMM_DEFAULT_UNROLL_M 2
2935 #define ZGEMM_DEFAULT_UNROLL_N 2
2938 #define SGEMM_DEFAULT_P 128
2939 #define DGEMM_DEFAULT_P 128
2940 #define CGEMM_DEFAULT_P 96
2941 #define ZGEMM_DEFAULT_P 64
2943 #define SGEMM_DEFAULT_Q 240
2944 #define DGEMM_DEFAULT_Q 120
2945 #define CGEMM_DEFAULT_Q 120
2946 #define ZGEMM_DEFAULT_Q 120
2948 #define SGEMM_DEFAULT_R 12288
2949 #define DGEMM_DEFAULT_R 8192
2950 #define CGEMM_DEFAULT_R 4096
2951 #define ZGEMM_DEFAULT_R 4096
2956 #ifdef RISCV64_GENERIC
2957 #define GEMM_DEFAULT_OFFSET_A 0
2958 #define GEMM_DEFAULT_OFFSET_B 0
2959 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
2961 #define SGEMM_DEFAULT_UNROLL_M 2
2962 #define SGEMM_DEFAULT_UNROLL_N 2
2964 #define DGEMM_DEFAULT_UNROLL_M 2
2965 #define DGEMM_DEFAULT_UNROLL_N 2
2967 #define CGEMM_DEFAULT_UNROLL_M 2
2968 #define CGEMM_DEFAULT_UNROLL_N 2
2970 #define ZGEMM_DEFAULT_UNROLL_M 2
2971 #define ZGEMM_DEFAULT_UNROLL_N 2
2973 #define SGEMM_DEFAULT_P 128
2974 #define DGEMM_DEFAULT_P 128
2975 #define CGEMM_DEFAULT_P 96
2976 #define ZGEMM_DEFAULT_P 64
2978 #define SGEMM_DEFAULT_Q 240
2979 #define DGEMM_DEFAULT_Q 120
2980 #define CGEMM_DEFAULT_Q 120
2981 #define ZGEMM_DEFAULT_Q 120
2983 #define SGEMM_DEFAULT_R 12288
2984 #define DGEMM_DEFAULT_R 8192
2985 #define CGEMM_DEFAULT_R 4096
2986 #define ZGEMM_DEFAULT_R 4096
2990 #define GEMM_DEFAULT_OFFSET_A 0
2991 #define GEMM_DEFAULT_OFFSET_B 0
2996 #define GEMM_DEFAULT_OFFSET_A 0
2997 #define GEMM_DEFAULT_OFFSET_B 0
2998 #define GEMM_DEFAULT_ALIGN 0x03fffUL
3000 #define SGEMM_DEFAULT_UNROLL_M 16
3001 #define SGEMM_DEFAULT_UNROLL_N 4
3003 #define DGEMM_DEFAULT_UNROLL_M 8
3004 #define DGEMM_DEFAULT_UNROLL_N 4
3006 #define CGEMM_DEFAULT_UNROLL_M 2
3007 #define CGEMM_DEFAULT_UNROLL_N 2
3009 #define ZGEMM_DEFAULT_UNROLL_M 2
3010 #define ZGEMM_DEFAULT_UNROLL_N 2
3012 #define SGEMM_DEFAULT_P 160
3013 #define DGEMM_DEFAULT_P 160
3014 #define CGEMM_DEFAULT_P 96
3015 #define ZGEMM_DEFAULT_P 64
3017 #define SGEMM_DEFAULT_Q 240
3018 #define DGEMM_DEFAULT_Q 128
3019 #define CGEMM_DEFAULT_Q 120
3020 #define ZGEMM_DEFAULT_Q 120
3022 #define SGEMM_DEFAULT_R 12288
3023 #define DGEMM_DEFAULT_R 8192
3024 #define CGEMM_DEFAULT_R 4096
3025 #define ZGEMM_DEFAULT_R 4096
3029 #define GEMM_DEFAULT_OFFSET_A 0
3030 #define GEMM_DEFAULT_OFFSET_B 0
3038 #define GEMM_DEFAULT_OFFSET_A 0
3039 #define GEMM_DEFAULT_OFFSET_B 0
3040 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3042 #define SGEMM_DEFAULT_UNROLL_M 4
3043 #define SGEMM_DEFAULT_UNROLL_N 4
3045 #define DGEMM_DEFAULT_UNROLL_M 4
3046 #define DGEMM_DEFAULT_UNROLL_N 4
3048 #define CGEMM_DEFAULT_UNROLL_M 2
3049 #define CGEMM_DEFAULT_UNROLL_N 2
3051 #define ZGEMM_DEFAULT_UNROLL_M 2
3052 #define ZGEMM_DEFAULT_UNROLL_N 2
3054 #define SGEMM_DEFAULT_P 128
3055 #define DGEMM_DEFAULT_P 128
3056 #define CGEMM_DEFAULT_P 96
3057 #define ZGEMM_DEFAULT_P 64
3059 #define SGEMM_DEFAULT_Q 240
3060 #define DGEMM_DEFAULT_Q 120
3061 #define CGEMM_DEFAULT_Q 120
3062 #define ZGEMM_DEFAULT_Q 120
3064 #define SGEMM_DEFAULT_R 12288
3065 #define DGEMM_DEFAULT_R 8192
3066 #define CGEMM_DEFAULT_R 4096
3067 #define ZGEMM_DEFAULT_R 4096
3079 #define GEMM_DEFAULT_OFFSET_A 0
3080 #define GEMM_DEFAULT_OFFSET_B 0
3081 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3083 #define SGEMM_DEFAULT_UNROLL_M 4
3084 #define SGEMM_DEFAULT_UNROLL_N 2
3086 #define DGEMM_DEFAULT_UNROLL_M 4
3087 #define DGEMM_DEFAULT_UNROLL_N 2
3089 #define CGEMM_DEFAULT_UNROLL_M 2
3090 #define CGEMM_DEFAULT_UNROLL_N 2
3092 #define ZGEMM_DEFAULT_UNROLL_M 2
3093 #define ZGEMM_DEFAULT_UNROLL_N 2
3095 #define SGEMM_DEFAULT_P 128
3096 #define DGEMM_DEFAULT_P 128
3097 #define CGEMM_DEFAULT_P 96
3098 #define ZGEMM_DEFAULT_P 64
3100 #define SGEMM_DEFAULT_Q 240
3101 #define DGEMM_DEFAULT_Q 120
3102 #define CGEMM_DEFAULT_Q 120
3103 #define ZGEMM_DEFAULT_Q 120
3105 #define SGEMM_DEFAULT_R 12288
3106 #define DGEMM_DEFAULT_R 8192
3107 #define CGEMM_DEFAULT_R 4096
3108 #define ZGEMM_DEFAULT_R 4096
3114 /* Common ARMv8 parameters */
3120 #define GEMM_DEFAULT_OFFSET_A 0
3121 #define GEMM_DEFAULT_OFFSET_B 0
3123 /* Use explicit casting for win64 as LLP64 datamodel is used */
3124 #define GEMM_DEFAULT_ALIGN (BLASULONG)0x03fffUL
3126 #define GEMM_DEFAULT_ALIGN 0x03fffUL
3131 #if defined(CORTEXA57) || \
3132 defined(CORTEXA72) || defined(CORTEXA73) || \
3133 defined(FALKOR) || defined(TSV110) || defined(EMAG8180) || defined(VORTEX)
3135 #define SGEMM_DEFAULT_UNROLL_M 16
3136 #define SGEMM_DEFAULT_UNROLL_N 4
3138 #define DGEMM_DEFAULT_UNROLL_M 8
3139 #define DGEMM_DEFAULT_UNROLL_N 4
3141 #define CGEMM_DEFAULT_UNROLL_M 8
3142 #define CGEMM_DEFAULT_UNROLL_N 4
3144 #define ZGEMM_DEFAULT_UNROLL_M 4
3145 #define ZGEMM_DEFAULT_UNROLL_N 4
3147 /*FIXME: this should be using the cache size, but there is currently no easy way to
3148 query that on ARM. So if getarch counted more than 8 cores we simply assume the host
3149 is a big desktop or server with abundant cache rather than a phone or embedded device */
3150 #if NUM_CORES > 8 || defined(TSV110) || defined(EMAG8180) || defined(VORTEX)
3151 #define SGEMM_DEFAULT_P 512
3152 #define DGEMM_DEFAULT_P 256
3153 #define CGEMM_DEFAULT_P 256
3154 #define ZGEMM_DEFAULT_P 128
3156 #define SGEMM_DEFAULT_Q 1024
3157 #define DGEMM_DEFAULT_Q 512
3158 #define CGEMM_DEFAULT_Q 512
3159 #define ZGEMM_DEFAULT_Q 512
3161 #define SGEMM_DEFAULT_P 128
3162 #define DGEMM_DEFAULT_P 160
3163 #define CGEMM_DEFAULT_P 128
3164 #define ZGEMM_DEFAULT_P 128
3166 #define SGEMM_DEFAULT_Q 352
3167 #define DGEMM_DEFAULT_Q 128
3168 #define CGEMM_DEFAULT_Q 224
3169 #define ZGEMM_DEFAULT_Q 112
3172 #define SGEMM_DEFAULT_R 4096
3173 #define DGEMM_DEFAULT_R 4096
3174 #define CGEMM_DEFAULT_R 4096
3175 #define ZGEMM_DEFAULT_R 2048
3177 #elif defined(CORTEXA53) || defined(CORTEXA55)
3179 #define SGEMM_DEFAULT_UNROLL_M 8
3180 #define SGEMM_DEFAULT_UNROLL_N 8
3182 #define DGEMM_DEFAULT_UNROLL_M 4
3183 #define DGEMM_DEFAULT_UNROLL_N 4
3185 #define CGEMM_DEFAULT_UNROLL_M 8
3186 #define CGEMM_DEFAULT_UNROLL_N 4
3188 #define ZGEMM_DEFAULT_UNROLL_M 4
3189 #define ZGEMM_DEFAULT_UNROLL_N 4
3191 #define SGEMM_DEFAULT_P 256
3192 #define DGEMM_DEFAULT_P 160
3193 #define CGEMM_DEFAULT_P 128
3194 #define ZGEMM_DEFAULT_P 128
3196 #define SGEMM_DEFAULT_Q 256
3197 #define DGEMM_DEFAULT_Q 128
3198 #define CGEMM_DEFAULT_Q 224
3199 #define ZGEMM_DEFAULT_Q 112
3201 #define SGEMM_DEFAULT_R 4096
3202 #define DGEMM_DEFAULT_R 4096
3203 #define CGEMM_DEFAULT_R 4096
3204 #define ZGEMM_DEFAULT_R 2048
3206 #elif defined(THUNDERX)
3208 #define SGEMM_DEFAULT_UNROLL_M 4
3209 #define SGEMM_DEFAULT_UNROLL_N 4
3211 #define DGEMM_DEFAULT_UNROLL_M 2
3212 #define DGEMM_DEFAULT_UNROLL_N 2
3214 #define CGEMM_DEFAULT_UNROLL_M 2
3215 #define CGEMM_DEFAULT_UNROLL_N 2
3217 #define ZGEMM_DEFAULT_UNROLL_M 2
3218 #define ZGEMM_DEFAULT_UNROLL_N 2
3220 #define SGEMM_DEFAULT_P 128
3221 #define DGEMM_DEFAULT_P 128
3222 #define CGEMM_DEFAULT_P 96
3223 #define ZGEMM_DEFAULT_P 64
3225 #define SGEMM_DEFAULT_Q 240
3226 #define DGEMM_DEFAULT_Q 120
3227 #define CGEMM_DEFAULT_Q 120
3228 #define ZGEMM_DEFAULT_Q 120
3230 #define SGEMM_DEFAULT_R 12288
3231 #define DGEMM_DEFAULT_R 8192
3232 #define CGEMM_DEFAULT_R 4096
3233 #define ZGEMM_DEFAULT_R 4096
3235 #elif defined(THUNDERX2T99)
3237 #define SGEMM_DEFAULT_UNROLL_M 16
3238 #define SGEMM_DEFAULT_UNROLL_N 4
3240 #define DGEMM_DEFAULT_UNROLL_M 8
3241 #define DGEMM_DEFAULT_UNROLL_N 4
3243 #define CGEMM_DEFAULT_UNROLL_M 8
3244 #define CGEMM_DEFAULT_UNROLL_N 4
3246 #define ZGEMM_DEFAULT_UNROLL_M 4
3247 #define ZGEMM_DEFAULT_UNROLL_N 4
3249 #define SGEMM_DEFAULT_P 128
3250 #define DGEMM_DEFAULT_P 160
3251 #define CGEMM_DEFAULT_P 128
3252 #define ZGEMM_DEFAULT_P 128
3254 #define SGEMM_DEFAULT_Q 352
3255 #define DGEMM_DEFAULT_Q 128
3256 #define CGEMM_DEFAULT_Q 224
3257 #define ZGEMM_DEFAULT_Q 112
3259 #define SGEMM_DEFAULT_R 4096
3260 #define DGEMM_DEFAULT_R 4096
3261 #define CGEMM_DEFAULT_R 4096
3262 #define ZGEMM_DEFAULT_R 4096
3264 #elif defined(THUNDERX3T110)
3266 #define SGEMM_DEFAULT_UNROLL_M 16
3267 #define SGEMM_DEFAULT_UNROLL_N 4
3269 #define DGEMM_DEFAULT_UNROLL_M 8
3270 #define DGEMM_DEFAULT_UNROLL_N 4
3272 #define CGEMM_DEFAULT_UNROLL_M 8
3273 #define CGEMM_DEFAULT_UNROLL_N 4
3275 #define ZGEMM_DEFAULT_UNROLL_M 4
3276 #define ZGEMM_DEFAULT_UNROLL_N 4
3278 #define SGEMM_DEFAULT_P 128
3279 #define DGEMM_DEFAULT_P 320
3280 #define CGEMM_DEFAULT_P 128
3281 #define ZGEMM_DEFAULT_P 128
3283 #define SGEMM_DEFAULT_Q 352
3284 #define DGEMM_DEFAULT_Q 128
3285 #define CGEMM_DEFAULT_Q 224
3286 #define ZGEMM_DEFAULT_Q 112
3288 #define SGEMM_DEFAULT_R 4096
3289 #define DGEMM_DEFAULT_R 4096
3290 #define CGEMM_DEFAULT_R 4096
3291 #define ZGEMM_DEFAULT_R 4096
3293 #elif defined(NEOVERSEN1)
3295 #define SGEMM_DEFAULT_UNROLL_M 16
3296 #define SGEMM_DEFAULT_UNROLL_N 4
3298 #define DGEMM_DEFAULT_UNROLL_M 8
3299 #define DGEMM_DEFAULT_UNROLL_N 4
3301 #define CGEMM_DEFAULT_UNROLL_M 8
3302 #define CGEMM_DEFAULT_UNROLL_N 4
3304 #define ZGEMM_DEFAULT_UNROLL_M 4
3305 #define ZGEMM_DEFAULT_UNROLL_N 4
3307 #define SGEMM_DEFAULT_P 128
3308 #define DGEMM_DEFAULT_P 160
3309 #define CGEMM_DEFAULT_P 128
3310 #define ZGEMM_DEFAULT_P 128
3312 #define SGEMM_DEFAULT_Q 352
3313 #define DGEMM_DEFAULT_Q 128
3314 #define CGEMM_DEFAULT_Q 224
3315 #define ZGEMM_DEFAULT_Q 112
3317 #define SGEMM_DEFAULT_R 4096
3318 #define DGEMM_DEFAULT_R 4096
3319 #define CGEMM_DEFAULT_R 4096
3320 #define ZGEMM_DEFAULT_R 4096
3322 #elif defined(NEOVERSEV1)
3324 #define SGEMM_DEFAULT_UNROLL_M 16
3325 #define SGEMM_DEFAULT_UNROLL_N 4
3327 #define DGEMM_DEFAULT_UNROLL_M 8
3328 #define DGEMM_DEFAULT_UNROLL_N 4
3330 #define CGEMM_DEFAULT_UNROLL_M 8
3331 #define CGEMM_DEFAULT_UNROLL_N 4
3333 #define ZGEMM_DEFAULT_UNROLL_M 4
3334 #define ZGEMM_DEFAULT_UNROLL_N 4
3336 #define SGEMM_DEFAULT_P 128
3337 #define DGEMM_DEFAULT_P 160
3338 #define CGEMM_DEFAULT_P 128
3339 #define ZGEMM_DEFAULT_P 128
3341 #define SGEMM_DEFAULT_Q 352
3342 #define DGEMM_DEFAULT_Q 128
3343 #define CGEMM_DEFAULT_Q 224
3344 #define ZGEMM_DEFAULT_Q 112
3346 #define SGEMM_DEFAULT_R 4096
3347 #define DGEMM_DEFAULT_R 4096
3348 #define CGEMM_DEFAULT_R 4096
3349 #define ZGEMM_DEFAULT_R 4096
3351 #elif defined(NEOVERSEN2)
3353 #define SGEMM_DEFAULT_UNROLL_M 16
3354 #define SGEMM_DEFAULT_UNROLL_N 4
3356 #define DGEMM_DEFAULT_UNROLL_M 8
3357 #define DGEMM_DEFAULT_UNROLL_N 4
3359 #define CGEMM_DEFAULT_UNROLL_M 8
3360 #define CGEMM_DEFAULT_UNROLL_N 4
3362 #define ZGEMM_DEFAULT_UNROLL_M 4
3363 #define ZGEMM_DEFAULT_UNROLL_N 4
3365 #define SGEMM_DEFAULT_P 128
3366 #define DGEMM_DEFAULT_P 160
3367 #define CGEMM_DEFAULT_P 128
3368 #define ZGEMM_DEFAULT_P 128
3370 #define SGEMM_DEFAULT_Q 352
3371 #define DGEMM_DEFAULT_Q 128
3372 #define CGEMM_DEFAULT_Q 224
3373 #define ZGEMM_DEFAULT_Q 112
3375 #define SGEMM_DEFAULT_R 4096
3376 #define DGEMM_DEFAULT_R 4096
3377 #define CGEMM_DEFAULT_R 4096
3378 #define ZGEMM_DEFAULT_R 4096
3380 #elif defined(ARMV8SVE) || defined(A64FX)
3382 /* When all BLAS3 routines are implemeted with SVE, SGEMM_DEFAULT_UNROLL_M should be "sve_vl".
3383 Until then, just keep it different than DGEMM_DEFAULT_UNROLL_N to keep copy routines in both directions seperated. */
3384 #define SGEMM_DEFAULT_UNROLL_M 4
3385 #define SGEMM_DEFAULT_UNROLL_N 8
3386 /* SGEMM_UNROLL_MN is calculated as max(SGEMM_UNROLL_M, SGEMM_UNROLL_N)
3387 * Since we don't define SGEMM_UNROLL_M correctly we have to manually set this macro.
3388 * If SVE size is ever more than 1024, this should be increased also. */
3389 #define SGEMM_DEFAULT_UNROLL_MN 32
3391 /* When all BLAS3 routines are implemeted with SVE, DGEMM_DEFAULT_UNROLL_M should be "sve_vl".
3392 Until then, just keep it different than DGEMM_DEFAULT_UNROLL_N to keep copy routines in both directions seperated. */
3393 #define DGEMM_DEFAULT_UNROLL_M 2
3394 #define DGEMM_DEFAULT_UNROLL_N 8
3396 #define DGEMM_DEFAULT_UNROLL_MN 32
3398 #define CGEMM_DEFAULT_UNROLL_M 2
3399 #define CGEMM_DEFAULT_UNROLL_N 4
3400 #define CGEMM_DEFAULT_UNROLL_MN 16
3402 #define ZGEMM_DEFAULT_UNROLL_M 2
3403 #define ZGEMM_DEFAULT_UNROLL_N 4
3404 #define ZGEMM_DEFAULT_UNROLL_MN 16
3406 #define SGEMM_DEFAULT_P 128
3407 #define DGEMM_DEFAULT_P 160
3408 #define CGEMM_DEFAULT_P 128
3409 #define ZGEMM_DEFAULT_P 128
3411 #define SGEMM_DEFAULT_Q 352
3412 #define DGEMM_DEFAULT_Q 128
3413 #define CGEMM_DEFAULT_Q 224
3414 #define ZGEMM_DEFAULT_Q 112
3416 #define SGEMM_DEFAULT_R 4096
3417 #define DGEMM_DEFAULT_R 4096
3418 #define CGEMM_DEFAULT_R 4096
3419 #define ZGEMM_DEFAULT_R 4096
3421 #else /* Other/undetected ARMv8 cores */
3423 #define SGEMM_DEFAULT_UNROLL_M 16
3424 #define SGEMM_DEFAULT_UNROLL_N 4
3426 #define DGEMM_DEFAULT_UNROLL_M 8
3427 #define DGEMM_DEFAULT_UNROLL_N 4
3429 #define CGEMM_DEFAULT_UNROLL_M 8
3430 #define CGEMM_DEFAULT_UNROLL_N 4
3432 #define ZGEMM_DEFAULT_UNROLL_M 4
3433 #define ZGEMM_DEFAULT_UNROLL_N 4
3435 #define SGEMM_DEFAULT_P 128
3436 #define DGEMM_DEFAULT_P 160
3437 #define CGEMM_DEFAULT_P 128
3438 #define ZGEMM_DEFAULT_P 128
3440 #define SGEMM_DEFAULT_Q 352
3441 #define DGEMM_DEFAULT_Q 128
3442 #define CGEMM_DEFAULT_Q 224
3443 #define ZGEMM_DEFAULT_Q 112
3445 #define SGEMM_DEFAULT_R 4096
3446 #define DGEMM_DEFAULT_R 4096
3447 #define CGEMM_DEFAULT_R 4096
3448 #define ZGEMM_DEFAULT_R 4096
3459 #define GEMM_DEFAULT_OFFSET_A 0
3460 #define GEMM_DEFAULT_OFFSET_B 0
3461 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3463 #define SGEMM_DEFAULT_UNROLL_M 2
3464 #define SGEMM_DEFAULT_UNROLL_N 2
3466 #define DGEMM_DEFAULT_UNROLL_M 2
3467 #define DGEMM_DEFAULT_UNROLL_N 2
3469 #define CGEMM_DEFAULT_UNROLL_M 2
3470 #define CGEMM_DEFAULT_UNROLL_N 2
3472 #define ZGEMM_DEFAULT_UNROLL_M 2
3473 #define ZGEMM_DEFAULT_UNROLL_N 2
3475 #define SGEMM_DEFAULT_P 128
3476 #define DGEMM_DEFAULT_P 128
3477 #define CGEMM_DEFAULT_P 96
3478 #define ZGEMM_DEFAULT_P 64
3480 #define SGEMM_DEFAULT_Q 240
3481 #define DGEMM_DEFAULT_Q 120
3482 #define CGEMM_DEFAULT_Q 120
3483 #define ZGEMM_DEFAULT_Q 120
3485 #define SGEMM_DEFAULT_R 12288
3486 #define DGEMM_DEFAULT_R 8192
3487 #define CGEMM_DEFAULT_R 4096
3488 #define ZGEMM_DEFAULT_R 4096
3500 #define GEMM_DEFAULT_OFFSET_A 0
3501 #define GEMM_DEFAULT_OFFSET_B 0
3502 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3504 #define SGEMM_DEFAULT_UNROLL_M 4
3505 #define SGEMM_DEFAULT_UNROLL_N 4
3507 #define DGEMM_DEFAULT_UNROLL_M 4
3508 #define DGEMM_DEFAULT_UNROLL_N 4
3510 #define CGEMM_DEFAULT_UNROLL_M 2
3511 #define CGEMM_DEFAULT_UNROLL_N 2
3513 #define ZGEMM_DEFAULT_UNROLL_M 2
3514 #define ZGEMM_DEFAULT_UNROLL_N 2
3516 #define SGEMM_DEFAULT_P 128
3517 #define DGEMM_DEFAULT_P 128
3518 #define CGEMM_DEFAULT_P 96
3519 #define ZGEMM_DEFAULT_P 64
3521 #define SGEMM_DEFAULT_Q 240
3522 #define DGEMM_DEFAULT_Q 120
3523 #define CGEMM_DEFAULT_Q 120
3524 #define ZGEMM_DEFAULT_Q 120
3526 #define SGEMM_DEFAULT_R 12288
3527 #define DGEMM_DEFAULT_R 8192
3528 #define CGEMM_DEFAULT_R 4096
3529 #define ZGEMM_DEFAULT_R 4096
3541 #define GEMM_DEFAULT_OFFSET_A 0
3542 #define GEMM_DEFAULT_OFFSET_B 0
3543 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3545 #define SGEMM_DEFAULT_UNROLL_M 4
3546 #define SGEMM_DEFAULT_UNROLL_N 4
3548 #define DGEMM_DEFAULT_UNROLL_M 4
3549 #define DGEMM_DEFAULT_UNROLL_N 4
3551 #define CGEMM_DEFAULT_UNROLL_M 2
3552 #define CGEMM_DEFAULT_UNROLL_N 2
3554 #define ZGEMM_DEFAULT_UNROLL_M 2
3555 #define ZGEMM_DEFAULT_UNROLL_N 2
3557 #define SGEMM_DEFAULT_P 128
3558 #define DGEMM_DEFAULT_P 128
3559 #define CGEMM_DEFAULT_P 96
3560 #define ZGEMM_DEFAULT_P 64
3562 #define SGEMM_DEFAULT_Q 240
3563 #define DGEMM_DEFAULT_Q 120
3564 #define CGEMM_DEFAULT_Q 120
3565 #define ZGEMM_DEFAULT_Q 120
3567 #define SGEMM_DEFAULT_R 12288
3568 #define DGEMM_DEFAULT_R 8192
3569 #define CGEMM_DEFAULT_R 4096
3570 #define ZGEMM_DEFAULT_R 4096
3578 #if defined(ZARCH_GENERIC)
3582 #define GEMM_DEFAULT_OFFSET_A 0
3583 #define GEMM_DEFAULT_OFFSET_B 0
3584 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3586 #define SGEMM_DEFAULT_UNROLL_M 2
3587 #define SGEMM_DEFAULT_UNROLL_N 2
3589 #define DGEMM_DEFAULT_UNROLL_M 2
3590 #define DGEMM_DEFAULT_UNROLL_N 2
3592 #define CGEMM_DEFAULT_UNROLL_M 2
3593 #define CGEMM_DEFAULT_UNROLL_N 2
3595 #define ZGEMM_DEFAULT_UNROLL_M 2
3596 #define ZGEMM_DEFAULT_UNROLL_N 2
3598 #define SGEMM_DEFAULT_P 128
3599 #define DGEMM_DEFAULT_P 128
3600 #define CGEMM_DEFAULT_P 96
3601 #define ZGEMM_DEFAULT_P 64
3603 #define SGEMM_DEFAULT_Q 240
3604 #define DGEMM_DEFAULT_Q 120
3605 #define CGEMM_DEFAULT_Q 120
3606 #define ZGEMM_DEFAULT_Q 120
3608 #define SGEMM_DEFAULT_R 12288
3609 #define DGEMM_DEFAULT_R 8192
3610 #define CGEMM_DEFAULT_R 4096
3611 #define ZGEMM_DEFAULT_R 4096
3621 #define GEMM_DEFAULT_OFFSET_A 0
3622 #define GEMM_DEFAULT_OFFSET_B 0
3623 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL
3625 #define SGEMM_DEFAULT_UNROLL_M 8
3626 #define SGEMM_DEFAULT_UNROLL_N 4
3628 #define DGEMM_DEFAULT_UNROLL_M 8
3629 #define DGEMM_DEFAULT_UNROLL_N 4
3631 #define CGEMM_DEFAULT_UNROLL_M 4
3632 #define CGEMM_DEFAULT_UNROLL_N 4
3634 #define ZGEMM_DEFAULT_UNROLL_M 4
3635 #define ZGEMM_DEFAULT_UNROLL_N 4
3637 #define SGEMM_DEFAULT_P 456
3638 #define DGEMM_DEFAULT_P 320
3639 #define CGEMM_DEFAULT_P 480
3640 #define ZGEMM_DEFAULT_P 224
3642 #define SGEMM_DEFAULT_Q 488
3643 #define DGEMM_DEFAULT_Q 384
3644 #define CGEMM_DEFAULT_Q 128
3645 #define ZGEMM_DEFAULT_Q 352
3647 #define SGEMM_DEFAULT_R 8192
3648 #define DGEMM_DEFAULT_R 4096
3649 #define CGEMM_DEFAULT_R 4096
3650 #define ZGEMM_DEFAULT_R 2048
3661 #define GEMM_DEFAULT_OFFSET_A 0
3662 #define GEMM_DEFAULT_OFFSET_B 0
3663 #define GEMM_DEFAULT_ALIGN 0x03fffUL
3665 #define SGEMM_DEFAULT_UNROLL_M 16
3666 #define SGEMM_DEFAULT_UNROLL_N 4
3668 #define DGEMM_DEFAULT_UNROLL_M 8
3669 #define DGEMM_DEFAULT_UNROLL_N 4
3671 #define CGEMM_DEFAULT_UNROLL_M 4
3672 #define CGEMM_DEFAULT_UNROLL_N 4
3674 #define ZGEMM_DEFAULT_UNROLL_M 4
3675 #define ZGEMM_DEFAULT_UNROLL_N 4
3677 #define SGEMM_DEFAULT_P 480
3678 #define DGEMM_DEFAULT_P 320
3679 #define CGEMM_DEFAULT_P 480
3680 #define ZGEMM_DEFAULT_P 224
3682 #define SGEMM_DEFAULT_Q 512
3683 #define DGEMM_DEFAULT_Q 384
3684 #define CGEMM_DEFAULT_Q 128
3685 #define ZGEMM_DEFAULT_Q 352
3687 #define SGEMM_DEFAULT_R 8192
3688 #define DGEMM_DEFAULT_R 4096
3689 #define CGEMM_DEFAULT_R 4096
3690 #define ZGEMM_DEFAULT_R 2048
3703 #define GEMM_DEFAULT_OFFSET_A 0
3704 #define GEMM_DEFAULT_OFFSET_B 0
3705 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x0ffffUL
3707 #define SGEMM_DEFAULT_UNROLL_N 2
3708 #define DGEMM_DEFAULT_UNROLL_N 2
3709 #define QGEMM_DEFAULT_UNROLL_N 2
3710 #define CGEMM_DEFAULT_UNROLL_N 2
3711 #define ZGEMM_DEFAULT_UNROLL_N 2
3712 #define XGEMM_DEFAULT_UNROLL_N 1
3715 #define SGEMM_DEFAULT_UNROLL_M 2
3716 #define DGEMM_DEFAULT_UNROLL_M 2
3717 #define QGEMM_DEFAULT_UNROLL_M 2
3718 #define CGEMM_DEFAULT_UNROLL_M 2
3719 #define ZGEMM_DEFAULT_UNROLL_M 2
3720 #define XGEMM_DEFAULT_UNROLL_M 1
3722 #define SGEMM_DEFAULT_UNROLL_M 2
3723 #define DGEMM_DEFAULT_UNROLL_M 2
3724 #define QGEMM_DEFAULT_UNROLL_M 2
3725 #define CGEMM_DEFAULT_UNROLL_M 2
3726 #define ZGEMM_DEFAULT_UNROLL_M 2
3727 #define XGEMM_DEFAULT_UNROLL_M 1
3731 #define SGEMM_DEFAULT_P 128
3732 #define DGEMM_DEFAULT_P 128
3733 #define CGEMM_DEFAULT_P 96
3734 #define ZGEMM_DEFAULT_P 64
3735 #define SGEMM_DEFAULT_Q 240
3736 #define DGEMM_DEFAULT_Q 120
3737 #define CGEMM_DEFAULT_Q 120
3738 #define ZGEMM_DEFAULT_Q 120
3739 #define SGEMM_DEFAULT_R 12288
3740 #define DGEMM_DEFAULT_R 8192
3741 #define CGEMM_DEFAULT_R 4096
3742 #define ZGEMM_DEFAULT_R 4096
3744 #define SGEMM_DEFAULT_P sgemm_p
3745 #define DGEMM_DEFAULT_P dgemm_p
3746 #define QGEMM_DEFAULT_P qgemm_p
3747 #define CGEMM_DEFAULT_P cgemm_p
3748 #define ZGEMM_DEFAULT_P zgemm_p
3749 #define XGEMM_DEFAULT_P xgemm_p
3751 #define SGEMM_DEFAULT_R sgemm_r
3752 #define DGEMM_DEFAULT_R dgemm_r
3753 #define QGEMM_DEFAULT_R qgemm_r
3754 #define CGEMM_DEFAULT_R cgemm_r
3755 #define ZGEMM_DEFAULT_R zgemm_r
3756 #define XGEMM_DEFAULT_R xgemm_r
3758 #define SGEMM_DEFAULT_Q 128
3759 #define DGEMM_DEFAULT_Q 128
3760 #define QGEMM_DEFAULT_Q 128
3761 #define CGEMM_DEFAULT_Q 128
3762 #define ZGEMM_DEFAULT_Q 128
3763 #define XGEMM_DEFAULT_Q 128
3770 #ifndef QGEMM_DEFAULT_UNROLL_M
3771 #define QGEMM_DEFAULT_UNROLL_M 2
3774 #ifndef QGEMM_DEFAULT_UNROLL_N
3775 #define QGEMM_DEFAULT_UNROLL_N 2
3778 #ifndef XGEMM_DEFAULT_UNROLL_M
3779 #define XGEMM_DEFAULT_UNROLL_M 2
3782 #ifndef XGEMM_DEFAULT_UNROLL_N
3783 #define XGEMM_DEFAULT_UNROLL_N 2
3787 #define SHUFPD_0 shufps $0x44,
3788 #define SHUFPD_1 shufps $0x4e,
3789 #define SHUFPD_2 shufps $0xe4,
3790 #define SHUFPD_3 shufps $0xee,
3794 #define SHUFPD_0 shufpd $0,
3798 #define SHUFPD_1 shufpd $1,
3802 #define SHUFPD_2 shufpd $2,
3806 #define SHUFPD_3 shufpd $3,
3810 #define SHUFPS_39 shufps $0x39,