1 /*********************************************************************/
2 /* Copyright 2009, 2010 The University of Texas at Austin. */
3 /* All rights reserved. */
5 /* Redistribution and use in source and binary forms, with or */
6 /* without modification, are permitted provided that the following */
7 /* conditions are met: */
9 /* 1. Redistributions of source code must retain the above */
10 /* copyright notice, this list of conditions and the following */
13 /* 2. Redistributions in binary form must reproduce the above */
14 /* copyright notice, this list of conditions and the following */
15 /* disclaimer in the documentation and/or other materials */
16 /* provided with the distribution. */
18 /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
19 /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
20 /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
21 /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
22 /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
23 /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
24 /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
25 /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
26 /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
27 /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
28 /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
29 /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
30 /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
31 /* POSSIBILITY OF SUCH DAMAGE. */
33 /* The views and conclusions contained in the software and */
34 /* documentation are those of the authors and should not be */
35 /* interpreted as representing official policies, either expressed */
36 /* or implied, of The University of Texas at Austin. */
37 /*********************************************************************/
56 static void __inline blas_lock(volatile BLASULONG *address){
61 while (*address) {YIELDING;};
63 #if defined(_MSC_VER) && !defined(__clang__)
64 // use intrinsic instead of inline assembly
65 ret = _InterlockedExchange((volatile LONG *)address, 1);
76 : "=r"(ret), "=m"(*address)
77 : "0"(1), "m"(*address)
84 #define BLAS_LOCK_DEFINED
86 static __inline unsigned long long rpcc(void){
87 #if defined(_MSC_VER) && !defined(__clang__)
88 return __rdtsc(); // use MSVC intrinsic
92 __asm__ __volatile__ ("rdtsc" : "=a" (a), "=d" (d));
94 return ((unsigned long long)a + ((unsigned long long)d << 32));
99 static __inline unsigned long getstackaddr(void){
100 #if defined(_MSC_VER) && !defined(__clang__)
101 return (unsigned long)_ReturnAddress(); // use MSVC intrinsic
105 __asm__ __volatile__ ("mov %%esp, %0"
106 : "=r"(addr) : : "memory");
113 static __inline long double sqrt_long(long double val) {
114 #if defined(_MSC_VER) && !defined(__clang__)
115 return sqrt(val); // not sure if this will use fsqrt
119 __asm__ __volatile__ ("fldt %1\n"
121 "fstpt %0\n" : "=m" (result) : "m"(val));
126 #define SQRT(a) sqrt_long(a)
128 /* This is due to gcc's bug */
129 void cpuid(int op, int *eax, int *ebx, int *ecx, int *edx);
133 static __inline int WhereAmI(void){
134 int eax, ebx, ecx, edx;
137 cpuid(1, &eax, &ebx, &ecx, &edx);
138 apicid = BITMASK(ebx, 24, 0xff);
143 #ifdef ENABLE_SSE_EXCEPTION
145 #define IDEBUG_START \
147 unsigned int fp_sse_mode, new_fp_mode; \
148 __asm__ __volatile__ ("stmxcsr %0" : "=m" (fp_sse_mode) : ); \
149 new_fp_mode = fp_sse_mode & ~0xd00; \
150 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (new_fp_mode) );
153 __asm__ __volatile__ ("ldmxcsr %0" : : "m" (fp_sse_mode) ); \
159 #define GET_IMAGE(res) __asm__ __volatile__("fstpt %0" : "=m"(res) : : "memory")
160 #elif defined(DOUBLE)
161 #define GET_IMAGE(res) __asm__ __volatile__("fstpl %0" : "=m"(res) : : "memory")
163 #define GET_IMAGE(res) __asm__ __volatile__("fstps %0" : "=m"(res) : : "memory");
166 #define GET_IMAGE_CANCEL __asm__ __volatile__ ("ffree %st")
169 extern unsigned int blas_quick_divide_table[];
171 static __inline int blas_quickdivide(unsigned int x, unsigned int y){
175 if (y <= 1) return x;
177 #if defined(_MSC_VER) && !defined(__clang__)
181 #if (MAX_CPU_NUMBER > 64)
188 y = blas_quick_divide_table[y];
190 __asm__ __volatile__ ("mull %0" :"=d" (result), "+a"(x): "0" (y));
200 #define PAGESIZE ( 4 << 10)
202 #define HUGE_PAGESIZE ( 4 << 20)
204 #define BUFFER_SIZE (16 << 20)
208 #if defined(DOUBLE) || defined(XDOUBLE)
210 #define MMXSTORE movq
213 #define MMXSTORE movd
216 #if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
217 //Enable some optimization for barcelona.
218 #define BARCELONA_OPTIMIZATION
221 #if defined(HAVE_3DNOW)
223 #elif defined(HAVE_MMX)
231 #if defined(CORE2) || defined(PENTIUM4)
232 #define movapd movaps
235 #define BRANCH .byte 0x3e
236 #define NOBRANCH .byte 0x2e
237 #define PADDING .byte 0x66;
242 #define LOCAL_BUFFER_SIZE QLOCAL_BUFFER_SIZE
244 #define LOCAL_BUFFER_SIZE DLOCAL_BUFFER_SIZE
246 #define LOCAL_BUFFER_SIZE SLOCAL_BUFFER_SIZE
250 #define LOCAL_BUFFER_SIZE XLOCAL_BUFFER_SIZE
252 #define LOCAL_BUFFER_SIZE ZLOCAL_BUFFER_SIZE
254 #define LOCAL_BUFFER_SIZE CLOCAL_BUFFER_SIZE
258 #if defined(OS_WINDOWS)
259 #if LOCAL_BUFFER_SIZE > 16384
260 #define STACK_TOUCHING \
261 movl $0, 4096 * 4(%esp);\
262 movl $0, 4096 * 3(%esp);\
263 movl $0, 4096 * 2(%esp);\
264 movl $0, 4096 * 1(%esp);
265 #elif LOCAL_BUFFER_SIZE > 12288
266 #define STACK_TOUCHING \
267 movl $0, 4096 * 3(%esp);\
268 movl $0, 4096 * 2(%esp);\
269 movl $0, 4096 * 1(%esp);
270 #elif LOCAL_BUFFER_SIZE > 8192
271 #define STACK_TOUCHING \
272 movl $0, 4096 * 2(%esp);\
273 movl $0, 4096 * 1(%esp);
274 #elif LOCAL_BUFFER_SIZE > 4096
275 #define STACK_TOUCHING \
276 movl $0, 4096 * 1(%esp);
278 #define STACK_TOUCHING
281 #define STACK_TOUCHING
285 #define REALNAME ASMNAME
287 #define REALNAME ASMFNAME
290 #if defined(F_INTERFACE_PATHSCALE) || defined(F_INTERFACE_OPEN64)
291 #define RETURN_BY_STRUCT
292 #elif defined(F_INTERFACE_GFORT) || defined(F_INTERFACE_G95)
293 #define RETURN_BY_COMPLEX
295 #define RETURN_BY_STACK
299 #define PROLOGUE .text;.align 5; .globl REALNAME; REALNAME:
300 #define EPILOGUE .subsections_via_symbols
304 #if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)
305 #define SAVEREGISTERS \
307 movups %xmm6, 0(%esp);\
308 movups %xmm7, 16(%esp)
310 #define RESTOREREGISTERS \
311 movups 0(%esp), %xmm6;\
312 movups 16(%esp), %xmm7;\
315 #define SAVEREGISTERS
316 #define RESTOREREGISTERS
319 #if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)
324 .def REALNAME;.scl 2;.type 32;.endef; \
330 #define EPILOGUE .end
332 #define EPILOGUE .end REALNAME
336 #if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(__ELF__)
341 .type REALNAME, @function; \
345 #define PROFCODE call mcount
351 .size REALNAME, .-REALNAME; \
352 .section .note.GNU-stack,"",@progbits
364 #elif defined(DOUBLE)
388 #define ALIGN_2 .align 2
389 #define ALIGN_3 .align 3
390 #define ALIGN_4 .align 4
391 #define ALIGN_5 .align 5
396 #define ALIGN_2 .align 4
400 #define ALIGN_3 .align 8
404 #define ALIGN_4 .align 16
408 #define ALIGN_5 .align 32
412 #define ALIGN_6 .align 64
415 // Because Clang didn't support ffreep, we directly use the opcode.
416 // Please check out http://www.sandpile.org/x86/opc_fpu.htm
418 #define ffreep .byte 0xdf, 0xc0 #