glib/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2013 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76
  77 /* ------------------------------------------------------------------ */
  78 /* VERSION NUMBER OF VALGRIND                                         */
  79 /* ------------------------------------------------------------------ */
  80
  81 /* Specify Valgrind's version number, so that user code can
  82    conditionally compile based on our version number.  Note that these
  83    were introduced at version 3.6 and so do not exist in version 3.5
  84    or earlier.  The recommended way to use them to check for "version
  85    X.Y or later" is (eg)
  86
  87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  88     && (__VALGRIND_MAJOR__ > 3                                   \
  89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  90 */
  91 #define __VALGRIND_MAJOR__    3
  92 #define __VALGRIND_MINOR__    10
  93
  94
  95 #include <stdarg.h>
  96
  97 /* Nb: this file might be included in a file compiled with -ansi.  So
  98    we can't use C++ style "//" comments nor the "asm" keyword (instead
  99    use "__asm__"). */
 100
 101 /* Derive some tags indicating what the target platform is.  Note
 102    that in this file we're using the compiler's CPP symbols for
 103    identifying architectures, which are different to the ones we use
 104    within the rest of Valgrind.  Note, __powerpc__ is active for both
 105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 106    latter (on Linux, that is).
 107
 108    Misc note: how to find out what's predefined in gcc by default:
 109    gcc -Wp,-dM somefile.c
 110 */
 111 #undef PLAT_x86_darwin
 112 #undef PLAT_amd64_darwin
 113 #undef PLAT_x86_win32
 114 #undef PLAT_amd64_win64
 115 #undef PLAT_x86_linux
 116 #undef PLAT_amd64_linux
 117 #undef PLAT_ppc32_linux
 118 #undef PLAT_ppc64_linux
 119 #undef PLAT_arm_linux
 120 #undef PLAT_arm64_linux
 121 #undef PLAT_s390x_linux
 122 #undef PLAT_mips32_linux
 123 #undef PLAT_mips64_linux
 124
 125
 126 #if defined(__APPLE__) && defined(__i386__)
 127 #  define PLAT_x86_darwin 1
 128 #elif defined(__APPLE__) && defined(__x86_64__)
 129 #  define PLAT_amd64_darwin 1
 130 #elif (defined(__MINGW32__) && !defined(__MINGW64__)) \
 131       || defined(__CYGWIN32__) \
 132       || (defined(_WIN32) && defined(_M_IX86))
 133 #  define PLAT_x86_win32 1
 134 #elif defined(__MINGW64__) \
 135       || (defined(_WIN64) && defined(_M_X64))
 136 #  define PLAT_amd64_win64 1
 137 #elif defined(__linux__) && defined(__i386__)
 138 #  define PLAT_x86_linux 1
 139 #elif defined(__linux__) && defined(__x86_64__)
 140 #  define PLAT_amd64_linux 1
 141 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 142 #  define PLAT_ppc32_linux 1
 143 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 144 #  define PLAT_ppc64_linux 1
 145 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
 146 #  define PLAT_arm_linux 1
 147 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
 148 #  define PLAT_arm64_linux 1
 149 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
 150 #  define PLAT_s390x_linux 1
 151 #elif defined(__linux__) && defined(__mips__) && (__mips==64)
 152 #  define PLAT_mips64_linux 1
 153 #elif defined(__linux__) && defined(__mips__) && (__mips!=64)
 154 #  define PLAT_mips32_linux 1
 155 #else
 156 /* If we're not compiling for our target platform, don't generate
 157    any inline asms.  */
 158 #  if !defined(NVALGRIND)
 159 #    define NVALGRIND 1
 160 #  endif
 161 #endif
 162
 163
 164 /* ------------------------------------------------------------------ */
 165 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 166 /* in here of use to end-users -- skip to the next section.           */
 167 /* ------------------------------------------------------------------ */
 168
 169 /*
 170  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
 171  * request. Accepts both pointers and integers as arguments.
 172  *
 173  * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
 174  * client request that does not return a value.
 175
 176  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
 177  * client request and whose value equals the client request result.  Accepts
 178  * both pointers and integers as arguments.  Note that such calls are not
 179  * necessarily pure functions -- they may have side effects.
 180  */
 181
 182 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
 183                                    _zzq_request, _zzq_arg1, _zzq_arg2,  \
 184                                    _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
 185   do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),   \
 186                         (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
 187                         (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
 188
 189 #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1,        \
 190                            _zzq_arg2,  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
 191   do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                        \
 192                     (_zzq_request), (_zzq_arg1), (_zzq_arg2),           \
 193                     (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
 194
 195 #if defined(NVALGRIND)
 196
 197 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 198    from the compiled code (analogous to NDEBUG's effects on
 199    assert()) */
 200 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 201         _zzq_default, _zzq_request,                               \
 202         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 203       (_zzq_default)
 204
 205 #else  /* ! NVALGRIND */
 206
 207 /* The following defines the magic code sequences which the JITter
 208    spots and handles magically.  Don't look too closely at them as
 209    they will rot your brain.
 210
 211    The assembly code sequences for all architectures is in this one
 212    file.  This is because this file must be stand-alone, and we don't
 213    want to have multiple files.
 214
 215    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 216    value gets put in the return slot, so that everything works when
 217    this is executed not under Valgrind.  Args are passed in a memory
 218    block, and so there's no intrinsic limit to the number that could
 219    be passed, but it's currently five.
 220
 221    The macro args are:
 222       _zzq_rlval    result lvalue
 223       _zzq_default  default value (result returned when running on real CPU)
 224       _zzq_request  request code
 225       _zzq_arg1..5  request params
 226
 227    The other two macros are used to support function wrapping, and are
 228    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 229    guest's NRADDR pseudo-register and whatever other information is
 230    needed to safely run the call original from the wrapper: on
 231    ppc64-linux, the R2 value at the divert point is also needed.  This
 232    information is abstracted into a user-visible type, OrigFn.
 233
 234    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 235    guest, but guarantees that the branch instruction will not be
 236    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 237    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 238    complete inline asm, since it needs to be combined with more magic
 239    inline asm stuff to be useful.
 240 */
 241
 242 /* ------------------------- x86-{linux,darwin} ---------------- */
 243
 244 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
 245     ||  (defined(PLAT_x86_win32) && defined(__GNUC__))
 246
 247 typedef
 248    struct {
 249       unsigned int nraddr; /* where's the code? */
 250    }
 251    OrigFn;
 252
 253 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 254                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 255                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 256
 257 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 258         _zzq_default, _zzq_request,                               \
 259         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 260   __extension__                                                   \
 261   ({volatile unsigned int _zzq_args[6];                           \
 262     volatile unsigned int _zzq_result;                            \
 263     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 264     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 265     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 266     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 267     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 268     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 269     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 270                      /* %EDX = client_request ( %EAX ) */         \
 271                      "xchgl %%ebx,%%ebx"                          \
 272                      : "=d" (_zzq_result)                         \
 273                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 274                      : "cc", "memory"                             \
 275                     );                                            \
 276     _zzq_result;                                                  \
 277   })
 278
 279 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 280   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 281     volatile unsigned int __addr;                                 \
 282     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 283                      /* %EAX = guest_NRADDR */                    \
 284                      "xchgl %%ecx,%%ecx"                          \
 285                      : "=a" (__addr)                              \
 286                      :                                            \
 287                      : "cc", "memory"                             \
 288                     );                                            \
 289     _zzq_orig->nraddr = __addr;                                   \
 290   }
 291
 292 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 293                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 294                      /* call-noredir *%EAX */                     \
 295                      "xchgl %%edx,%%edx\n\t"
 296
 297 #define VALGRIND_VEX_INJECT_IR()                                 \
 298  do {                                                            \
 299     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 300                      "xchgl %%edi,%%edi\n\t"                     \
 301                      : : : "cc", "memory"                        \
 302                     );                                           \
 303  } while (0)
 304
 305 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) */
 306
 307 /* ------------------------- x86-Win32 ------------------------- */
 308
 309 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
 310
 311 typedef
 312    struct {
 313       unsigned int nraddr; /* where's the code? */
 314    }
 315    OrigFn;
 316
 317 #if defined(_MSC_VER)
 318
 319 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 320                      __asm rol edi, 3  __asm rol edi, 13          \
 321                      __asm rol edi, 29 __asm rol edi, 19
 322
 323 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 324         _zzq_default, _zzq_request,                               \
 325         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 326     valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
 327         (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
 328         (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
 329         (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
 330
 331 static __inline uintptr_t
 332 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
 333                                 uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
 334                                 uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
 335                                 uintptr_t _zzq_arg5)
 336 {
 337     volatile uintptr_t _zzq_args[6];
 338     volatile unsigned int _zzq_result;
 339     _zzq_args[0] = (uintptr_t)(_zzq_request);
 340     _zzq_args[1] = (uintptr_t)(_zzq_arg1);
 341     _zzq_args[2] = (uintptr_t)(_zzq_arg2);
 342     _zzq_args[3] = (uintptr_t)(_zzq_arg3);
 343     _zzq_args[4] = (uintptr_t)(_zzq_arg4);
 344     _zzq_args[5] = (uintptr_t)(_zzq_arg5);
 345     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
 346             __SPECIAL_INSTRUCTION_PREAMBLE
 347             /* %EDX = client_request ( %EAX ) */
 348             __asm xchg ebx,ebx
 349             __asm mov _zzq_result, edx
 350     }
 351     return _zzq_result;
 352 }
 353
 354 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 355   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 356     volatile unsigned int __addr;                                 \
 357     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
 358             /* %EAX = guest_NRADDR */                             \
 359             __asm xchg ecx,ecx                                    \
 360             __asm mov __addr, eax                                 \
 361     }                                                             \
 362     _zzq_orig->nraddr = __addr;                                   \
 363   }
 364
 365 #define VALGRIND_CALL_NOREDIR_EAX ERROR
 366
 367 #define VALGRIND_VEX_INJECT_IR()                                 \
 368  do {                                                            \
 369     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                       \
 370             __asm xchg edi,edi                                   \
 371     }                                                            \
 372  } while (0)
 373
 374 #else
 375 #error Unsupported compiler.
 376 #endif
 377
 378 #endif /* PLAT_x86_win32 */
 379
 380 /* ------------------------ amd64-{linux,darwin} --------------- */
 381
 382 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
 383     ||  (defined(PLAT_amd64_win64) && defined(__GNUC__))
 384
 385 typedef
 386    struct {
 387       unsigned long long int nraddr; /* where's the code? */
 388    }
 389    OrigFn;
 390
 391 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 392                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 393                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 394
 395 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 396         _zzq_default, _zzq_request,                               \
 397         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 398     __extension__                                                 \
 399     ({ volatile unsigned long long int _zzq_args[6];              \
 400     volatile unsigned long long int _zzq_result;                  \
 401     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 402     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 403     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 404     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 405     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 406     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 407     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 408                      /* %RDX = client_request ( %RAX ) */         \
 409                      "xchgq %%rbx,%%rbx"                          \
 410                      : "=d" (_zzq_result)                         \
 411                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 412                      : "cc", "memory"                             \
 413                     );                                            \
 414     _zzq_result;                                                  \
 415     })
 416
 417 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 418   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 419     volatile unsigned long long int __addr;                       \
 420     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 421                      /* %RAX = guest_NRADDR */                    \
 422                      "xchgq %%rcx,%%rcx"                          \
 423                      : "=a" (__addr)                              \
 424                      :                                            \
 425                      : "cc", "memory"                             \
 426                     );                                            \
 427     _zzq_orig->nraddr = __addr;                                   \
 428   }
 429
 430 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 431                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 432                      /* call-noredir *%RAX */                     \
 433                      "xchgq %%rdx,%%rdx\n\t"
 434
 435 #define VALGRIND_VEX_INJECT_IR()                                 \
 436  do {                                                            \
 437     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 438                      "xchgq %%rdi,%%rdi\n\t"                     \
 439                      : : : "cc", "memory"                        \
 440                     );                                           \
 441  } while (0)
 442
 443 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 444
 445 /* ------------------------- amd64-Win64 ------------------------- */
 446
 447 #if defined(PLAT_amd64_win64) && !defined(__GNUC__)
 448
 449 #error Unsupported compiler.
 450
 451 #endif /* PLAT_amd64_win64 */
 452
 453 /* ------------------------ ppc32-linux ------------------------ */
 454
 455 #if defined(PLAT_ppc32_linux)
 456
 457 typedef
 458    struct {
 459       unsigned int nraddr; /* where's the code? */
 460    }
 461    OrigFn;
 462
 463 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 464                     "rlwinm 0,0,3,0,31  ; rlwinm 0,0,13,0,31\n\t" \
 465                     "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
 466
 467 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 468         _zzq_default, _zzq_request,                               \
 469         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 470                                                                   \
 471     __extension__                                                 \
 472   ({         unsigned int  _zzq_args[6];                          \
 473              unsigned int  _zzq_result;                           \
 474              unsigned int* _zzq_ptr;                              \
 475     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 476     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 477     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 478     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 479     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 480     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 481     _zzq_ptr = _zzq_args;                                         \
 482     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 483                      "mr 4,%2\n\t" /*ptr*/                        \
 484                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 485                      /* %R3 = client_request ( %R4 ) */           \
 486                      "or 1,1,1\n\t"                               \
 487                      "mr %0,3"     /*result*/                     \
 488                      : "=b" (_zzq_result)                         \
 489                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 490                      : "cc", "memory", "r3", "r4");               \
 491     _zzq_result;                                                  \
 492     })
 493
 494 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 495   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 496     unsigned int __addr;                                          \
 497     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 498                      /* %R3 = guest_NRADDR */                     \
 499                      "or 2,2,2\n\t"                               \
 500                      "mr %0,3"                                    \
 501                      : "=b" (__addr)                              \
 502                      :                                            \
 503                      : "cc", "memory", "r3"                       \
 504                     );                                            \
 505     _zzq_orig->nraddr = __addr;                                   \
 506   }
 507
 508 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 509                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 510                      /* branch-and-link-to-noredir *%R11 */       \
 511                      "or 3,3,3\n\t"
 512
 513 #define VALGRIND_VEX_INJECT_IR()                                 \
 514  do {                                                            \
 515     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 516                      "or 5,5,5\n\t"                              \
 517                     );                                           \
 518  } while (0)
 519
 520 #endif /* PLAT_ppc32_linux */
 521
 522 /* ------------------------ ppc64-linux ------------------------ */
 523
 524 #if defined(PLAT_ppc64_linux)
 525
 526 typedef
 527    struct {
 528       unsigned long long int nraddr; /* where's the code? */
 529       unsigned long long int r2;  /* what tocptr do we need? */
 530    }
 531    OrigFn;
 532
 533 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 534                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 535                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 536
 537 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 538         _zzq_default, _zzq_request,                               \
 539         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 540                                                                   \
 541   __extension__                                                   \
 542   ({         unsigned long long int  _zzq_args[6];                \
 543              unsigned long long int  _zzq_result;                 \
 544              unsigned long long int* _zzq_ptr;                    \
 545     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 546     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 547     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 548     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 549     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 550     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 551     _zzq_ptr = _zzq_args;                                         \
 552     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 553                      "mr 4,%2\n\t" /*ptr*/                        \
 554                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 555                      /* %R3 = client_request ( %R4 ) */           \
 556                      "or 1,1,1\n\t"                               \
 557                      "mr %0,3"     /*result*/                     \
 558                      : "=b" (_zzq_result)                         \
 559                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 560                      : "cc", "memory", "r3", "r4");               \
 561     _zzq_result;                                                  \
 562   })
 563
 564 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 565   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 566     unsigned long long int __addr;                                \
 567     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 568                      /* %R3 = guest_NRADDR */                     \
 569                      "or 2,2,2\n\t"                               \
 570                      "mr %0,3"                                    \
 571                      : "=b" (__addr)                              \
 572                      :                                            \
 573                      : "cc", "memory", "r3"                       \
 574                     );                                            \
 575     _zzq_orig->nraddr = __addr;                                   \
 576     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 577                      /* %R3 = guest_NRADDR_GPR2 */                \
 578                      "or 4,4,4\n\t"                               \
 579                      "mr %0,3"                                    \
 580                      : "=b" (__addr)                              \
 581                      :                                            \
 582                      : "cc", "memory", "r3"                       \
 583                     );                                            \
 584     _zzq_orig->r2 = __addr;                                       \
 585   }
 586
 587 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 588                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 589                      /* branch-and-link-to-noredir *%R11 */       \
 590                      "or 3,3,3\n\t"
 591
 592 #define VALGRIND_VEX_INJECT_IR()                                 \
 593  do {                                                            \
 594     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 595                      "or 5,5,5\n\t"                              \
 596                     );                                           \
 597  } while (0)
 598
 599 #endif /* PLAT_ppc64_linux */
 600
 601 /* ------------------------- arm-linux ------------------------- */
 602
 603 #if defined(PLAT_arm_linux)
 604
 605 typedef
 606    struct {
 607       unsigned int nraddr; /* where's the code? */
 608    }
 609    OrigFn;
 610
 611 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 612             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 613             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 614
 615 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 616         _zzq_default, _zzq_request,                               \
 617         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 618                                                                   \
 619   __extension__                                                   \
 620   ({volatile unsigned int  _zzq_args[6];                          \
 621     volatile unsigned int  _zzq_result;                           \
 622     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 623     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 624     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 625     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 626     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 627     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 628     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 629                      "mov r4, %2\n\t" /*ptr*/                     \
 630                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 631                      /* R3 = client_request ( R4 ) */             \
 632                      "orr r10, r10, r10\n\t"                      \
 633                      "mov %0, r3"     /*result*/                  \
 634                      : "=r" (_zzq_result)                         \
 635                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 636                      : "cc","memory", "r3", "r4");                \
 637     _zzq_result;                                                  \
 638   })
 639
 640 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 641   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 642     unsigned int __addr;                                          \
 643     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 644                      /* R3 = guest_NRADDR */                      \
 645                      "orr r11, r11, r11\n\t"                      \
 646                      "mov %0, r3"                                 \
 647                      : "=r" (__addr)                              \
 648                      :                                            \
 649                      : "cc", "memory", "r3"                       \
 650                     );                                            \
 651     _zzq_orig->nraddr = __addr;                                   \
 652   }
 653
 654 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 655                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 656                      /* branch-and-link-to-noredir *%R4 */        \
 657                      "orr r12, r12, r12\n\t"
 658
 659 #define VALGRIND_VEX_INJECT_IR()                                 \
 660  do {                                                            \
 661     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 662                      "orr r9, r9, r9\n\t"                        \
 663                      : : : "cc", "memory"                        \
 664                     );                                           \
 665  } while (0)
 666
 667 #endif /* PLAT_arm_linux */
 668
 669 /* ------------------------ arm64-linux ------------------------- */
 670
 671 #if defined(PLAT_arm64_linux)
 672
 673 typedef
 674    struct {
 675       unsigned long long int nraddr; /* where's the code? */
 676    }
 677    OrigFn;
 678
 679 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 680             "ror x12, x12, #3  ;  ror x12, x12, #13 \n\t"         \
 681             "ror x12, x12, #51 ;  ror x12, x12, #61 \n\t"
 682
 683 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 684         _zzq_default, _zzq_request,                               \
 685         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 686                                                                   \
 687   __extension__                                                   \
 688   ({volatile unsigned long long int  _zzq_args[6];                \
 689     volatile unsigned long long int  _zzq_result;                 \
 690     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 691     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 692     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 693     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 694     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 695     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 696     __asm__ volatile("mov x3, %1\n\t" /*default*/                 \
 697                      "mov x4, %2\n\t" /*ptr*/                     \
 698                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 699                      /* X3 = client_request ( X4 ) */             \
 700                      "orr x10, x10, x10\n\t"                      \
 701                      "mov %0, x3"     /*result*/                  \
 702                      : "=r" (_zzq_result)                         \
 703                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 704                      : "cc","memory", "x3", "x4");                \
 705     _zzq_result;                                                  \
 706   })
 707
 708 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 709   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 710     unsigned long long int __addr;                                \
 711     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 712                      /* X3 = guest_NRADDR */                      \
 713                      "orr x11, x11, x11\n\t"                      \
 714                      "mov %0, x3"                                 \
 715                      : "=r" (__addr)                              \
 716                      :                                            \
 717                      : "cc", "memory", "x3"                       \
 718                     );                                            \
 719     _zzq_orig->nraddr = __addr;                                   \
 720   }
 721
 722 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                    \
 723                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 724                      /* branch-and-link-to-noredir X8 */          \
 725                      "orr x12, x12, x12\n\t"
 726
 727 #define VALGRIND_VEX_INJECT_IR()                                 \
 728  do {                                                            \
 729     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 730                      "orr x9, x9, x9\n\t"                        \
 731                      : : : "cc", "memory"                        \
 732                     );                                           \
 733  } while (0)
 734
 735 #endif /* PLAT_arm64_linux */
 736
 737 /* ------------------------ s390x-linux ------------------------ */
 738
 739 #if defined(PLAT_s390x_linux)
 740
 741 typedef
 742   struct {
 743      unsigned long long int nraddr; /* where's the code? */
 744   }
 745   OrigFn;
 746
 747 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
 748  * code. This detection is implemented in platform specific toIR.c
 749  * (e.g. VEX/priv/guest_s390_decoder.c).
 750  */
 751 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
 752                      "lr 15,15\n\t"                              \
 753                      "lr 1,1\n\t"                                \
 754                      "lr 2,2\n\t"                                \
 755                      "lr 3,3\n\t"
 756
 757 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
 758 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
 759 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
 760 #define __VEX_INJECT_IR_CODE  "lr 5,5\n\t"
 761
 762 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
 763        _zzq_default, _zzq_request,                               \
 764        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 765   __extension__                                                  \
 766  ({volatile unsigned long long int _zzq_args[6];                 \
 767    volatile unsigned long long int _zzq_result;                  \
 768    _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 769    _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 770    _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 771    _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 772    _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 773    _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 774    __asm__ volatile(/* r2 = args */                              \
 775                     "lgr 2,%1\n\t"                               \
 776                     /* r3 = default */                           \
 777                     "lgr 3,%2\n\t"                               \
 778                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 779                     __CLIENT_REQUEST_CODE                        \
 780                     /* results = r3 */                           \
 781                     "lgr %0, 3\n\t"                              \
 782                     : "=d" (_zzq_result)                         \
 783                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 784                     : "cc", "2", "3", "memory"                   \
 785                    );                                            \
 786    _zzq_result;                                                  \
 787  })
 788
 789 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
 790  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 791    volatile unsigned long long int __addr;                       \
 792    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 793                     __GET_NR_CONTEXT_CODE                        \
 794                     "lgr %0, 3\n\t"                              \
 795                     : "=a" (__addr)                              \
 796                     :                                            \
 797                     : "cc", "3", "memory"                        \
 798                    );                                            \
 799    _zzq_orig->nraddr = __addr;                                   \
 800  }
 801
 802 #define VALGRIND_CALL_NOREDIR_R1                                 \
 803                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 804                     __CALL_NO_REDIR_CODE
 805
 806 #define VALGRIND_VEX_INJECT_IR()                                 \
 807  do {                                                            \
 808     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 809                      __VEX_INJECT_IR_CODE);                      \
 810  } while (0)
 811
 812 #endif /* PLAT_s390x_linux */
 813
 814 /* ------------------------- mips32-linux ---------------- */
 815
 816 #if defined(PLAT_mips32_linux)
 817
 818 typedef
 819    struct {
 820       unsigned int nraddr; /* where's the code? */
 821    }
 822    OrigFn;
 823
 824 /* .word  0x342
 825  * .word  0x742
 826  * .word  0xC2
 827  * .word  0x4C2*/
 828 #define __SPECIAL_INSTRUCTION_PREAMBLE          \
 829                      "srl $0, $0, 13\n\t"       \
 830                      "srl $0, $0, 29\n\t"       \
 831                      "srl $0, $0, 3\n\t"        \
 832                      "srl $0, $0, 19\n\t"
 833
 834 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 835        _zzq_default, _zzq_request,                                \
 836        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
 837   __extension__                                                   \
 838   ({ volatile unsigned int _zzq_args[6];                          \
 839     volatile unsigned int _zzq_result;                            \
 840     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 841     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 842     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 843     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 844     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 845     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 846         __asm__ volatile("move $11, %1\n\t" /*default*/           \
 847                      "move $12, %2\n\t" /*ptr*/                   \
 848                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 849                      /* T3 = client_request ( T4 ) */             \
 850                      "or $13, $13, $13\n\t"                       \
 851                      "move %0, $11\n\t"     /*result*/            \
 852                      : "=r" (_zzq_result)                         \
 853                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 854                      : "$11", "$12");                             \
 855     _zzq_result;                                                  \
 856   })
 857
 858 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 859   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 860     volatile unsigned int __addr;                                 \
 861     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 862                      /* %t9 = guest_NRADDR */                     \
 863                      "or $14, $14, $14\n\t"                       \
 864                      "move %0, $11"     /*result*/                \
 865                      : "=r" (__addr)                              \
 866                      :                                            \
 867                      : "$11"                                      \
 868                     );                                            \
 869     _zzq_orig->nraddr = __addr;                                   \
 870   }
 871
 872 #define VALGRIND_CALL_NOREDIR_T9                                 \
 873                      __SPECIAL_INSTRUCTION_PREAMBLE              \
 874                      /* call-noredir *%t9 */                     \
 875                      "or $15, $15, $15\n\t"
 876
 877 #define VALGRIND_VEX_INJECT_IR()                                 \
 878  do {                                                            \
 879     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
 880                      "or $11, $11, $11\n\t"                      \
 881                     );                                           \
 882  } while (0)
 883
 884
 885 #endif /* PLAT_mips32_linux */
 886
 887 /* ------------------------- mips64-linux ---------------- */
 888
 889 #if defined(PLAT_mips64_linux)
 890
 891 typedef
 892    struct {
 893       unsigned long long nraddr; /* where's the code? */
 894    }
 895    OrigFn;
 896
 897 /* dsll $0,$0, 3
 898  * dsll $0,$0, 13
 899  * dsll $0,$0, 29
 900  * dsll $0,$0, 19*/
 901 #define __SPECIAL_INSTRUCTION_PREAMBLE                              \
 902                      "dsll $0,$0, 3 ; dsll $0,$0,13\n\t"            \
 903                      "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
 904
 905 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                            \
 906        _zzq_default, _zzq_request,                                  \
 907        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)       \
 908   __extension__                                                     \
 909   ({ volatile unsigned long long int _zzq_args[6];                  \
 910     volatile unsigned long long int _zzq_result;                    \
 911     _zzq_args[0] = (unsigned long long int)(_zzq_request);          \
 912     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);             \
 913     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);             \
 914     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);             \
 915     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);             \
 916     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);             \
 917         __asm__ volatile("move $11, %1\n\t" /*default*/             \
 918                          "move $12, %2\n\t" /*ptr*/                 \
 919                          __SPECIAL_INSTRUCTION_PREAMBLE             \
 920                          /* $11 = client_request ( $12 ) */         \
 921                          "or $13, $13, $13\n\t"                     \
 922                          "move %0, $11\n\t"     /*result*/          \
 923                          : "=r" (_zzq_result)                       \
 924                          : "r" (_zzq_default), "r" (&_zzq_args[0])  \
 925                          : "$11", "$12");                           \
 926     _zzq_result;                                                    \
 927   })
 928
 929 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
 930   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
 931     volatile unsigned long long int __addr;                         \
 932     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
 933                      /* $11 = guest_NRADDR */                       \
 934                      "or $14, $14, $14\n\t"                         \
 935                      "move %0, $11"     /*result*/                  \
 936                      : "=r" (__addr)                                \
 937                      :                                              \
 938                      : "$11");                                      \
 939     _zzq_orig->nraddr = __addr;                                     \
 940   }
 941
 942 #define VALGRIND_CALL_NOREDIR_T9                                    \
 943                      __SPECIAL_INSTRUCTION_PREAMBLE                 \
 944                      /* call-noredir $25 */                         \
 945                      "or $15, $15, $15\n\t"
 946
 947 #define VALGRIND_VEX_INJECT_IR()                                    \
 948  do {                                                               \
 949     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
 950                      "or $11, $11, $11\n\t"                         \
 951                     );                                              \
 952  } while (0)
 953
 954 #endif /* PLAT_mips64_linux */
 955
 956 /* Insert assembly code for other platforms here... */
 957
 958 #endif /* NVALGRIND */
 959
 960
 961 /* ------------------------------------------------------------------ */
 962 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 963 /* ugly.  It's the least-worst tradeoff I can think of.               */
 964 /* ------------------------------------------------------------------ */
 965
 966 /* This section defines magic (a.k.a appalling-hack) macros for doing
 967    guaranteed-no-redirection macros, so as to get from function
 968    wrappers to the functions they are wrapping.  The whole point is to
 969    construct standard call sequences, but to do the call itself with a
 970    special no-redirect call pseudo-instruction that the JIT
 971    understands and handles specially.  This section is long and
 972    repetitious, and I can't see a way to make it shorter.
 973
 974    The naming scheme is as follows:
 975
 976       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 977
 978    'W' stands for "word" and 'v' for "void".  Hence there are
 979    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 980    and for each, the possibility of returning a word-typed result, or
 981    no result.
 982 */
 983
 984 /* Use these to write the name of your wrapper.  NOTE: duplicates
 985    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h.  NOTE also: inserts
 986    the default behaviour equivalance class tag "0000" into the name.
 987    See pub_tool_redir.h for details -- normally you don't need to
 988    think about this, though. */
 989
 990 /* Use an extra level of macroisation so as to ensure the soname/fnname
 991    args are fully macro-expanded before pasting them together. */
 992 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 993
 994 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 995    VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
 996
 997 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 998    VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
 999
1000 /* Use this macro from within a wrapper function to collect the
1001    context (address and possibly other info) of the original function.
1002    Once you have that you can then use it in one of the CALL_FN_
1003    macros.  The type of the argument _lval is OrigFn. */
1004 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
1005
1006 /* Also provide end-user facilities for function replacement, rather
1007    than wrapping.  A replacement function differs from a wrapper in
1008    that it has no way to get hold of the original function being
1009    called, and hence no way to call onwards to it.  In a replacement
1010    function, VALGRIND_GET_ORIG_FN always returns zero. */
1011
1012 #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname)                 \
1013    VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
1014
1015 #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname)                 \
1016    VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
1017
1018 /* Derivatives of the main macros below, for calling functions
1019    returning void. */
1020
1021 #define CALL_FN_v_v(fnptr)                                        \
1022    do { volatile unsigned long _junk;                             \
1023         CALL_FN_W_v(_junk,fnptr); } while (0)
1024
1025 #define CALL_FN_v_W(fnptr, arg1)                                  \
1026    do { volatile unsigned long _junk;                             \
1027         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
1028
1029 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
1030    do { volatile unsigned long _junk;                             \
1031         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
1032
1033 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
1034    do { volatile unsigned long _junk;                             \
1035         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
1036
1037 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
1038    do { volatile unsigned long _junk;                             \
1039         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
1040
1041 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
1042    do { volatile unsigned long _junk;                             \
1043         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
1044
1045 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
1046    do { volatile unsigned long _junk;                             \
1047         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
1048
1049 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
1050    do { volatile unsigned long _junk;                             \
1051         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
1052
1053 /* ------------------------- x86-{linux,darwin} ---------------- */
1054
1055 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
1056
1057 /* These regs are trashed by the hidden call.  No need to mention eax
1058    as gcc can already see that, plus causes gcc to bomb. */
1059 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
1060
1061 /* Macros to save and align the stack before making a function
1062    call and restore it afterwards as gcc may not keep the stack
1063    pointer aligned if it doesn't realise calls are being made
1064    to other functions. */
1065
1066 #define VALGRIND_ALIGN_STACK               \
1067       "movl %%esp,%%edi\n\t"               \
1068       "andl $0xfffffff0,%%esp\n\t"
1069 #define VALGRIND_RESTORE_STACK             \
1070       "movl %%edi,%%esp\n\t"
1071
1072 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
1073    long) == 4. */
1074
1075 #define CALL_FN_W_v(lval, orig)                                   \
1076    do {                                                           \
1077       volatile OrigFn        _orig = (orig);                      \
1078       volatile unsigned long _argvec[1];                          \
1079       volatile unsigned long _res;                                \
1080       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1081       __asm__ volatile(                                           \
1082          VALGRIND_ALIGN_STACK                                     \
1083          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1084          VALGRIND_CALL_NOREDIR_EAX                                \
1085          VALGRIND_RESTORE_STACK                                   \
1086          : /*out*/   "=a" (_res)                                  \
1087          : /*in*/    "a" (&_argvec[0])                            \
1088          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1089       );                                                          \
1090       lval = (__typeof__(lval)) _res;                             \
1091    } while (0)
1092
1093 #define CALL_FN_W_W(lval, orig, arg1)                             \
1094    do {                                                           \
1095       volatile OrigFn        _orig = (orig);                      \
1096       volatile unsigned long _argvec[2];                          \
1097       volatile unsigned long _res;                                \
1098       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1099       _argvec[1] = (unsigned long)(arg1);                         \
1100       __asm__ volatile(                                           \
1101          VALGRIND_ALIGN_STACK                                     \
1102          "subl $12, %%esp\n\t"                                    \
1103          "pushl 4(%%eax)\n\t"                                     \
1104          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1105          VALGRIND_CALL_NOREDIR_EAX                                \
1106          VALGRIND_RESTORE_STACK                                   \
1107          : /*out*/   "=a" (_res)                                  \
1108          : /*in*/    "a" (&_argvec[0])                            \
1109          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1110       );                                                          \
1111       lval = (__typeof__(lval)) _res;                             \
1112    } while (0)
1113
1114 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1115    do {                                                           \
1116       volatile OrigFn        _orig = (orig);                      \
1117       volatile unsigned long _argvec[3];                          \
1118       volatile unsigned long _res;                                \
1119       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1120       _argvec[1] = (unsigned long)(arg1);                         \
1121       _argvec[2] = (unsigned long)(arg2);                         \
1122       __asm__ volatile(                                           \
1123          VALGRIND_ALIGN_STACK                                     \
1124          "subl $8, %%esp\n\t"                                     \
1125          "pushl 8(%%eax)\n\t"                                     \
1126          "pushl 4(%%eax)\n\t"                                     \
1127          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1128          VALGRIND_CALL_NOREDIR_EAX                                \
1129          VALGRIND_RESTORE_STACK                                   \
1130          : /*out*/   "=a" (_res)                                  \
1131          : /*in*/    "a" (&_argvec[0])                            \
1132          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1133       );                                                          \
1134       lval = (__typeof__(lval)) _res;                             \
1135    } while (0)
1136
1137 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1138    do {                                                           \
1139       volatile OrigFn        _orig = (orig);                      \
1140       volatile unsigned long _argvec[4];                          \
1141       volatile unsigned long _res;                                \
1142       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1143       _argvec[1] = (unsigned long)(arg1);                         \
1144       _argvec[2] = (unsigned long)(arg2);                         \
1145       _argvec[3] = (unsigned long)(arg3);                         \
1146       __asm__ volatile(                                           \
1147          VALGRIND_ALIGN_STACK                                     \
1148          "subl $4, %%esp\n\t"                                     \
1149          "pushl 12(%%eax)\n\t"                                    \
1150          "pushl 8(%%eax)\n\t"                                     \
1151          "pushl 4(%%eax)\n\t"                                     \
1152          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1153          VALGRIND_CALL_NOREDIR_EAX                                \
1154          VALGRIND_RESTORE_STACK                                   \
1155          : /*out*/   "=a" (_res)                                  \
1156          : /*in*/    "a" (&_argvec[0])                            \
1157          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1158       );                                                          \
1159       lval = (__typeof__(lval)) _res;                             \
1160    } while (0)
1161
1162 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1163    do {                                                           \
1164       volatile OrigFn        _orig = (orig);                      \
1165       volatile unsigned long _argvec[5];                          \
1166       volatile unsigned long _res;                                \
1167       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1168       _argvec[1] = (unsigned long)(arg1);                         \
1169       _argvec[2] = (unsigned long)(arg2);                         \
1170       _argvec[3] = (unsigned long)(arg3);                         \
1171       _argvec[4] = (unsigned long)(arg4);                         \
1172       __asm__ volatile(                                           \
1173          VALGRIND_ALIGN_STACK                                     \
1174          "pushl 16(%%eax)\n\t"                                    \
1175          "pushl 12(%%eax)\n\t"                                    \
1176          "pushl 8(%%eax)\n\t"                                     \
1177          "pushl 4(%%eax)\n\t"                                     \
1178          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1179          VALGRIND_CALL_NOREDIR_EAX                                \
1180          VALGRIND_RESTORE_STACK                                   \
1181          : /*out*/   "=a" (_res)                                  \
1182          : /*in*/    "a" (&_argvec[0])                            \
1183          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1184       );                                                          \
1185       lval = (__typeof__(lval)) _res;                             \
1186    } while (0)
1187
1188 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1189    do {                                                           \
1190       volatile OrigFn        _orig = (orig);                      \
1191       volatile unsigned long _argvec[6];                          \
1192       volatile unsigned long _res;                                \
1193       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1194       _argvec[1] = (unsigned long)(arg1);                         \
1195       _argvec[2] = (unsigned long)(arg2);                         \
1196       _argvec[3] = (unsigned long)(arg3);                         \
1197       _argvec[4] = (unsigned long)(arg4);                         \
1198       _argvec[5] = (unsigned long)(arg5);                         \
1199       __asm__ volatile(                                           \
1200          VALGRIND_ALIGN_STACK                                     \
1201          "subl $12, %%esp\n\t"                                    \
1202          "pushl 20(%%eax)\n\t"                                    \
1203          "pushl 16(%%eax)\n\t"                                    \
1204          "pushl 12(%%eax)\n\t"                                    \
1205          "pushl 8(%%eax)\n\t"                                     \
1206          "pushl 4(%%eax)\n\t"                                     \
1207          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1208          VALGRIND_CALL_NOREDIR_EAX                                \
1209          VALGRIND_RESTORE_STACK                                   \
1210          : /*out*/   "=a" (_res)                                  \
1211          : /*in*/    "a" (&_argvec[0])                            \
1212          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1213       );                                                          \
1214       lval = (__typeof__(lval)) _res;                             \
1215    } while (0)
1216
1217 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1218    do {                                                           \
1219       volatile OrigFn        _orig = (orig);                      \
1220       volatile unsigned long _argvec[7];                          \
1221       volatile unsigned long _res;                                \
1222       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1223       _argvec[1] = (unsigned long)(arg1);                         \
1224       _argvec[2] = (unsigned long)(arg2);                         \
1225       _argvec[3] = (unsigned long)(arg3);                         \
1226       _argvec[4] = (unsigned long)(arg4);                         \
1227       _argvec[5] = (unsigned long)(arg5);                         \
1228       _argvec[6] = (unsigned long)(arg6);                         \
1229       __asm__ volatile(                                           \
1230          VALGRIND_ALIGN_STACK                                     \
1231          "subl $8, %%esp\n\t"                                     \
1232          "pushl 24(%%eax)\n\t"                                    \
1233          "pushl 20(%%eax)\n\t"                                    \
1234          "pushl 16(%%eax)\n\t"                                    \
1235          "pushl 12(%%eax)\n\t"                                    \
1236          "pushl 8(%%eax)\n\t"                                     \
1237          "pushl 4(%%eax)\n\t"                                     \
1238          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1239          VALGRIND_CALL_NOREDIR_EAX                                \
1240          VALGRIND_RESTORE_STACK                                   \
1241          : /*out*/   "=a" (_res)                                  \
1242          : /*in*/    "a" (&_argvec[0])                            \
1243          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1244       );                                                          \
1245       lval = (__typeof__(lval)) _res;                             \
1246    } while (0)
1247
1248 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1249                                  arg7)                            \
1250    do {                                                           \
1251       volatile OrigFn        _orig = (orig);                      \
1252       volatile unsigned long _argvec[8];                          \
1253       volatile unsigned long _res;                                \
1254       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1255       _argvec[1] = (unsigned long)(arg1);                         \
1256       _argvec[2] = (unsigned long)(arg2);                         \
1257       _argvec[3] = (unsigned long)(arg3);                         \
1258       _argvec[4] = (unsigned long)(arg4);                         \
1259       _argvec[5] = (unsigned long)(arg5);                         \
1260       _argvec[6] = (unsigned long)(arg6);                         \
1261       _argvec[7] = (unsigned long)(arg7);                         \
1262       __asm__ volatile(                                           \
1263          VALGRIND_ALIGN_STACK                                     \
1264          "subl $4, %%esp\n\t"                                     \
1265          "pushl 28(%%eax)\n\t"                                    \
1266          "pushl 24(%%eax)\n\t"                                    \
1267          "pushl 20(%%eax)\n\t"                                    \
1268          "pushl 16(%%eax)\n\t"                                    \
1269          "pushl 12(%%eax)\n\t"                                    \
1270          "pushl 8(%%eax)\n\t"                                     \
1271          "pushl 4(%%eax)\n\t"                                     \
1272          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1273          VALGRIND_CALL_NOREDIR_EAX                                \
1274          VALGRIND_RESTORE_STACK                                   \
1275          : /*out*/   "=a" (_res)                                  \
1276          : /*in*/    "a" (&_argvec[0])                            \
1277          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1278       );                                                          \
1279       lval = (__typeof__(lval)) _res;                             \
1280    } while (0)
1281
1282 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1283                                  arg7,arg8)                       \
1284    do {                                                           \
1285       volatile OrigFn        _orig = (orig);                      \
1286       volatile unsigned long _argvec[9];                          \
1287       volatile unsigned long _res;                                \
1288       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1289       _argvec[1] = (unsigned long)(arg1);                         \
1290       _argvec[2] = (unsigned long)(arg2);                         \
1291       _argvec[3] = (unsigned long)(arg3);                         \
1292       _argvec[4] = (unsigned long)(arg4);                         \
1293       _argvec[5] = (unsigned long)(arg5);                         \
1294       _argvec[6] = (unsigned long)(arg6);                         \
1295       _argvec[7] = (unsigned long)(arg7);                         \
1296       _argvec[8] = (unsigned long)(arg8);                         \
1297       __asm__ volatile(                                           \
1298          VALGRIND_ALIGN_STACK                                     \
1299          "pushl 32(%%eax)\n\t"                                    \
1300          "pushl 28(%%eax)\n\t"                                    \
1301          "pushl 24(%%eax)\n\t"                                    \
1302          "pushl 20(%%eax)\n\t"                                    \
1303          "pushl 16(%%eax)\n\t"                                    \
1304          "pushl 12(%%eax)\n\t"                                    \
1305          "pushl 8(%%eax)\n\t"                                     \
1306          "pushl 4(%%eax)\n\t"                                     \
1307          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1308          VALGRIND_CALL_NOREDIR_EAX                                \
1309          VALGRIND_RESTORE_STACK                                   \
1310          : /*out*/   "=a" (_res)                                  \
1311          : /*in*/    "a" (&_argvec[0])                            \
1312          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1313       );                                                          \
1314       lval = (__typeof__(lval)) _res;                             \
1315    } while (0)
1316
1317 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1318                                  arg7,arg8,arg9)                  \
1319    do {                                                           \
1320       volatile OrigFn        _orig = (orig);                      \
1321       volatile unsigned long _argvec[10];                         \
1322       volatile unsigned long _res;                                \
1323       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1324       _argvec[1] = (unsigned long)(arg1);                         \
1325       _argvec[2] = (unsigned long)(arg2);                         \
1326       _argvec[3] = (unsigned long)(arg3);                         \
1327       _argvec[4] = (unsigned long)(arg4);                         \
1328       _argvec[5] = (unsigned long)(arg5);                         \
1329       _argvec[6] = (unsigned long)(arg6);                         \
1330       _argvec[7] = (unsigned long)(arg7);                         \
1331       _argvec[8] = (unsigned long)(arg8);                         \
1332       _argvec[9] = (unsigned long)(arg9);                         \
1333       __asm__ volatile(                                           \
1334          VALGRIND_ALIGN_STACK                                     \
1335          "subl $12, %%esp\n\t"                                    \
1336          "pushl 36(%%eax)\n\t"                                    \
1337          "pushl 32(%%eax)\n\t"                                    \
1338          "pushl 28(%%eax)\n\t"                                    \
1339          "pushl 24(%%eax)\n\t"                                    \
1340          "pushl 20(%%eax)\n\t"                                    \
1341          "pushl 16(%%eax)\n\t"                                    \
1342          "pushl 12(%%eax)\n\t"                                    \
1343          "pushl 8(%%eax)\n\t"                                     \
1344          "pushl 4(%%eax)\n\t"                                     \
1345          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1346          VALGRIND_CALL_NOREDIR_EAX                                \
1347          VALGRIND_RESTORE_STACK                                   \
1348          : /*out*/   "=a" (_res)                                  \
1349          : /*in*/    "a" (&_argvec[0])                            \
1350          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1351       );                                                          \
1352       lval = (__typeof__(lval)) _res;                             \
1353    } while (0)
1354
1355 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1356                                   arg7,arg8,arg9,arg10)           \
1357    do {                                                           \
1358       volatile OrigFn        _orig = (orig);                      \
1359       volatile unsigned long _argvec[11];                         \
1360       volatile unsigned long _res;                                \
1361       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1362       _argvec[1] = (unsigned long)(arg1);                         \
1363       _argvec[2] = (unsigned long)(arg2);                         \
1364       _argvec[3] = (unsigned long)(arg3);                         \
1365       _argvec[4] = (unsigned long)(arg4);                         \
1366       _argvec[5] = (unsigned long)(arg5);                         \
1367       _argvec[6] = (unsigned long)(arg6);                         \
1368       _argvec[7] = (unsigned long)(arg7);                         \
1369       _argvec[8] = (unsigned long)(arg8);                         \
1370       _argvec[9] = (unsigned long)(arg9);                         \
1371       _argvec[10] = (unsigned long)(arg10);                       \
1372       __asm__ volatile(                                           \
1373          VALGRIND_ALIGN_STACK                                     \
1374          "subl $8, %%esp\n\t"                                     \
1375          "pushl 40(%%eax)\n\t"                                    \
1376          "pushl 36(%%eax)\n\t"                                    \
1377          "pushl 32(%%eax)\n\t"                                    \
1378          "pushl 28(%%eax)\n\t"                                    \
1379          "pushl 24(%%eax)\n\t"                                    \
1380          "pushl 20(%%eax)\n\t"                                    \
1381          "pushl 16(%%eax)\n\t"                                    \
1382          "pushl 12(%%eax)\n\t"                                    \
1383          "pushl 8(%%eax)\n\t"                                     \
1384          "pushl 4(%%eax)\n\t"                                     \
1385          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1386          VALGRIND_CALL_NOREDIR_EAX                                \
1387          VALGRIND_RESTORE_STACK                                   \
1388          : /*out*/   "=a" (_res)                                  \
1389          : /*in*/    "a" (&_argvec[0])                            \
1390          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1391       );                                                          \
1392       lval = (__typeof__(lval)) _res;                             \
1393    } while (0)
1394
1395 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1396                                   arg6,arg7,arg8,arg9,arg10,      \
1397                                   arg11)                          \
1398    do {                                                           \
1399       volatile OrigFn        _orig = (orig);                      \
1400       volatile unsigned long _argvec[12];                         \
1401       volatile unsigned long _res;                                \
1402       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1403       _argvec[1] = (unsigned long)(arg1);                         \
1404       _argvec[2] = (unsigned long)(arg2);                         \
1405       _argvec[3] = (unsigned long)(arg3);                         \
1406       _argvec[4] = (unsigned long)(arg4);                         \
1407       _argvec[5] = (unsigned long)(arg5);                         \
1408       _argvec[6] = (unsigned long)(arg6);                         \
1409       _argvec[7] = (unsigned long)(arg7);                         \
1410       _argvec[8] = (unsigned long)(arg8);                         \
1411       _argvec[9] = (unsigned long)(arg9);                         \
1412       _argvec[10] = (unsigned long)(arg10);                       \
1413       _argvec[11] = (unsigned long)(arg11);                       \
1414       __asm__ volatile(                                           \
1415          VALGRIND_ALIGN_STACK                                     \
1416          "subl $4, %%esp\n\t"                                     \
1417          "pushl 44(%%eax)\n\t"                                    \
1418          "pushl 40(%%eax)\n\t"                                    \
1419          "pushl 36(%%eax)\n\t"                                    \
1420          "pushl 32(%%eax)\n\t"                                    \
1421          "pushl 28(%%eax)\n\t"                                    \
1422          "pushl 24(%%eax)\n\t"                                    \
1423          "pushl 20(%%eax)\n\t"                                    \
1424          "pushl 16(%%eax)\n\t"                                    \
1425          "pushl 12(%%eax)\n\t"                                    \
1426          "pushl 8(%%eax)\n\t"                                     \
1427          "pushl 4(%%eax)\n\t"                                     \
1428          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1429          VALGRIND_CALL_NOREDIR_EAX                                \
1430          VALGRIND_RESTORE_STACK                                   \
1431          : /*out*/   "=a" (_res)                                  \
1432          : /*in*/    "a" (&_argvec[0])                            \
1433          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1434       );                                                          \
1435       lval = (__typeof__(lval)) _res;                             \
1436    } while (0)
1437
1438 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1439                                   arg6,arg7,arg8,arg9,arg10,      \
1440                                   arg11,arg12)                    \
1441    do {                                                           \
1442       volatile OrigFn        _orig = (orig);                      \
1443       volatile unsigned long _argvec[13];                         \
1444       volatile unsigned long _res;                                \
1445       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1446       _argvec[1] = (unsigned long)(arg1);                         \
1447       _argvec[2] = (unsigned long)(arg2);                         \
1448       _argvec[3] = (unsigned long)(arg3);                         \
1449       _argvec[4] = (unsigned long)(arg4);                         \
1450       _argvec[5] = (unsigned long)(arg5);                         \
1451       _argvec[6] = (unsigned long)(arg6);                         \
1452       _argvec[7] = (unsigned long)(arg7);                         \
1453       _argvec[8] = (unsigned long)(arg8);                         \
1454       _argvec[9] = (unsigned long)(arg9);                         \
1455       _argvec[10] = (unsigned long)(arg10);                       \
1456       _argvec[11] = (unsigned long)(arg11);                       \
1457       _argvec[12] = (unsigned long)(arg12);                       \
1458       __asm__ volatile(                                           \
1459          VALGRIND_ALIGN_STACK                                     \
1460          "pushl 48(%%eax)\n\t"                                    \
1461          "pushl 44(%%eax)\n\t"                                    \
1462          "pushl 40(%%eax)\n\t"                                    \
1463          "pushl 36(%%eax)\n\t"                                    \
1464          "pushl 32(%%eax)\n\t"                                    \
1465          "pushl 28(%%eax)\n\t"                                    \
1466          "pushl 24(%%eax)\n\t"                                    \
1467          "pushl 20(%%eax)\n\t"                                    \
1468          "pushl 16(%%eax)\n\t"                                    \
1469          "pushl 12(%%eax)\n\t"                                    \
1470          "pushl 8(%%eax)\n\t"                                     \
1471          "pushl 4(%%eax)\n\t"                                     \
1472          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1473          VALGRIND_CALL_NOREDIR_EAX                                \
1474          VALGRIND_RESTORE_STACK                                   \
1475          : /*out*/   "=a" (_res)                                  \
1476          : /*in*/    "a" (&_argvec[0])                            \
1477          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1478       );                                                          \
1479       lval = (__typeof__(lval)) _res;                             \
1480    } while (0)
1481
1482 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1483
1484 /* ------------------------ amd64-{linux,darwin} --------------- */
1485
1486 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1487
1488 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1489
1490 /* These regs are trashed by the hidden call. */
1491 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1492                             "rdi", "r8", "r9", "r10", "r11"
1493
1494 /* This is all pretty complex.  It's so as to make stack unwinding
1495    work reliably.  See bug 243270.  The basic problem is the sub and
1496    add of 128 of %rsp in all of the following macros.  If gcc believes
1497    the CFA is in %rsp, then unwinding may fail, because what's at the
1498    CFA is not what gcc "expected" when it constructs the CFIs for the
1499    places where the macros are instantiated.
1500
1501    But we can't just add a CFI annotation to increase the CFA offset
1502    by 128, to match the sub of 128 from %rsp, because we don't know
1503    whether gcc has chosen %rsp as the CFA at that point, or whether it
1504    has chosen some other register (eg, %rbp).  In the latter case,
1505    adding a CFI annotation to change the CFA offset is simply wrong.
1506
1507    So the solution is to get hold of the CFA using
1508    __builtin_dwarf_cfa(), put it in a known register, and add a
1509    CFI annotation to say what the register is.  We choose %rbp for
1510    this (perhaps perversely), because:
1511
1512    (1) %rbp is already subject to unwinding.  If a new register was
1513        chosen then the unwinder would have to unwind it in all stack
1514        traces, which is expensive, and
1515
1516    (2) %rbp is already subject to precise exception updates in the
1517        JIT.  If a new register was chosen, we'd have to have precise
1518        exceptions for it too, which reduces performance of the
1519        generated code.
1520
1521    However .. one extra complication.  We can't just whack the result
1522    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1523    list of trashed registers at the end of the inline assembly
1524    fragments; gcc won't allow %rbp to appear in that list.  Hence
1525    instead we need to stash %rbp in %r15 for the duration of the asm,
1526    and say that %r15 is trashed instead.  gcc seems happy to go with
1527    that.
1528
1529    Oh .. and this all needs to be conditionalised so that it is
1530    unchanged from before this commit, when compiled with older gccs
1531    that don't support __builtin_dwarf_cfa.  Furthermore, since
1532    this header file is freestanding, it has to be independent of
1533    config.h, and so the following conditionalisation cannot depend on
1534    configure time checks.
1535
1536    Although it's not clear from
1537    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1538    this expression excludes Darwin.
1539    .cfi directives in Darwin assembly appear to be completely
1540    different and I haven't investigated how they work.
1541
1542    For even more entertainment value, note we have to use the
1543    completely undocumented __builtin_dwarf_cfa(), which appears to
1544    really compute the CFA, whereas __builtin_frame_address(0) claims
1545    to but actually doesn't.  See
1546    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1547 */
1548 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1549 #  define __FRAME_POINTER                                         \
1550       ,"r"(__builtin_dwarf_cfa())
1551 #  define VALGRIND_CFI_PROLOGUE                                   \
1552       "movq %%rbp, %%r15\n\t"                                     \
1553       "movq %2, %%rbp\n\t"                                        \
1554       ".cfi_remember_state\n\t"                                   \
1555       ".cfi_def_cfa rbp, 0\n\t"
1556 #  define VALGRIND_CFI_EPILOGUE                                   \
1557       "movq %%r15, %%rbp\n\t"                                     \
1558       ".cfi_restore_state\n\t"
1559 #else
1560 #  define __FRAME_POINTER
1561 #  define VALGRIND_CFI_PROLOGUE
1562 #  define VALGRIND_CFI_EPILOGUE
1563 #endif
1564
1565 /* Macros to save and align the stack before making a function
1566    call and restore it afterwards as gcc may not keep the stack
1567    pointer aligned if it doesn't realise calls are being made
1568    to other functions. */
1569
1570 #define VALGRIND_ALIGN_STACK               \
1571       "movq %%rsp,%%r14\n\t"               \
1572       "andq $0xfffffffffffffff0,%%rsp\n\t"
1573 #define VALGRIND_RESTORE_STACK             \
1574       "movq %%r14,%%rsp\n\t"
1575
1576 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1577    long) == 8. */
1578
1579 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1580    macros.  In order not to trash the stack redzone, we need to drop
1581    %rsp by 128 before the hidden call, and restore afterwards.  The
1582    nastyness is that it is only by luck that the stack still appears
1583    to be unwindable during the hidden call - since then the behaviour
1584    of any routine using this macro does not match what the CFI data
1585    says.  Sigh.
1586
1587    Why is this important?  Imagine that a wrapper has a stack
1588    allocated local, and passes to the hidden call, a pointer to it.
1589    Because gcc does not know about the hidden call, it may allocate
1590    that local in the redzone.  Unfortunately the hidden call may then
1591    trash it before it comes to use it.  So we must step clear of the
1592    redzone, for the duration of the hidden call, to make it safe.
1593
1594    Probably the same problem afflicts the other redzone-style ABIs too
1595    (ppc64-linux); but for those, the stack is
1596    self describing (none of this CFI nonsense) so at least messing
1597    with the stack pointer doesn't give a danger of non-unwindable
1598    stack. */
1599
1600 #define CALL_FN_W_v(lval, orig)                                        \
1601    do {                                                                \
1602       volatile OrigFn        _orig = (orig);                           \
1603       volatile unsigned long _argvec[1];                               \
1604       volatile unsigned long _res;                                     \
1605       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1606       __asm__ volatile(                                                \
1607          VALGRIND_CFI_PROLOGUE                                         \
1608          VALGRIND_ALIGN_STACK                                          \
1609          "subq $128,%%rsp\n\t"                                         \
1610          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1611          VALGRIND_CALL_NOREDIR_RAX                                     \
1612          VALGRIND_RESTORE_STACK                                        \
1613          VALGRIND_CFI_EPILOGUE                                         \
1614          : /*out*/   "=a" (_res)                                       \
1615          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1616          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1617       );                                                               \
1618       lval = (__typeof__(lval)) _res;                                  \
1619    } while (0)
1620
1621 #define CALL_FN_W_W(lval, orig, arg1)                                  \
1622    do {                                                                \
1623       volatile OrigFn        _orig = (orig);                           \
1624       volatile unsigned long _argvec[2];                               \
1625       volatile unsigned long _res;                                     \
1626       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1627       _argvec[1] = (unsigned long)(arg1);                              \
1628       __asm__ volatile(                                                \
1629          VALGRIND_CFI_PROLOGUE                                         \
1630          VALGRIND_ALIGN_STACK                                          \
1631          "subq $128,%%rsp\n\t"                                         \
1632          "movq 8(%%rax), %%rdi\n\t"                                    \
1633          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1634          VALGRIND_CALL_NOREDIR_RAX                                     \
1635          VALGRIND_RESTORE_STACK                                        \
1636          VALGRIND_CFI_EPILOGUE                                         \
1637          : /*out*/   "=a" (_res)                                       \
1638          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1639          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1640       );                                                               \
1641       lval = (__typeof__(lval)) _res;                                  \
1642    } while (0)
1643
1644 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                            \
1645    do {                                                                \
1646       volatile OrigFn        _orig = (orig);                           \
1647       volatile unsigned long _argvec[3];                               \
1648       volatile unsigned long _res;                                     \
1649       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1650       _argvec[1] = (unsigned long)(arg1);                              \
1651       _argvec[2] = (unsigned long)(arg2);                              \
1652       __asm__ volatile(                                                \
1653          VALGRIND_CFI_PROLOGUE                                         \
1654          VALGRIND_ALIGN_STACK                                          \
1655          "subq $128,%%rsp\n\t"                                         \
1656          "movq 16(%%rax), %%rsi\n\t"                                   \
1657          "movq 8(%%rax), %%rdi\n\t"                                    \
1658          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1659          VALGRIND_CALL_NOREDIR_RAX                                     \
1660          VALGRIND_RESTORE_STACK                                        \
1661          VALGRIND_CFI_EPILOGUE                                         \
1662          : /*out*/   "=a" (_res)                                       \
1663          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1664          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1665       );                                                               \
1666       lval = (__typeof__(lval)) _res;                                  \
1667    } while (0)
1668
1669 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                      \
1670    do {                                                                \
1671       volatile OrigFn        _orig = (orig);                           \
1672       volatile unsigned long _argvec[4];                               \
1673       volatile unsigned long _res;                                     \
1674       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1675       _argvec[1] = (unsigned long)(arg1);                              \
1676       _argvec[2] = (unsigned long)(arg2);                              \
1677       _argvec[3] = (unsigned long)(arg3);                              \
1678       __asm__ volatile(                                                \
1679          VALGRIND_CFI_PROLOGUE                                         \
1680          VALGRIND_ALIGN_STACK                                          \
1681          "subq $128,%%rsp\n\t"                                         \
1682          "movq 24(%%rax), %%rdx\n\t"                                   \
1683          "movq 16(%%rax), %%rsi\n\t"                                   \
1684          "movq 8(%%rax), %%rdi\n\t"                                    \
1685          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1686          VALGRIND_CALL_NOREDIR_RAX                                     \
1687          VALGRIND_RESTORE_STACK                                        \
1688          VALGRIND_CFI_EPILOGUE                                         \
1689          : /*out*/   "=a" (_res)                                       \
1690          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1691          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1692       );                                                               \
1693       lval = (__typeof__(lval)) _res;                                  \
1694    } while (0)
1695
1696 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)                \
1697    do {                                                                \
1698       volatile OrigFn        _orig = (orig);                           \
1699       volatile unsigned long _argvec[5];                               \
1700       volatile unsigned long _res;                                     \
1701       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1702       _argvec[1] = (unsigned long)(arg1);                              \
1703       _argvec[2] = (unsigned long)(arg2);                              \
1704       _argvec[3] = (unsigned long)(arg3);                              \
1705       _argvec[4] = (unsigned long)(arg4);                              \
1706       __asm__ volatile(                                                \
1707          VALGRIND_CFI_PROLOGUE                                         \
1708          VALGRIND_ALIGN_STACK                                          \
1709          "subq $128,%%rsp\n\t"                                         \
1710          "movq 32(%%rax), %%rcx\n\t"                                   \
1711          "movq 24(%%rax), %%rdx\n\t"                                   \
1712          "movq 16(%%rax), %%rsi\n\t"                                   \
1713          "movq 8(%%rax), %%rdi\n\t"                                    \
1714          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1715          VALGRIND_CALL_NOREDIR_RAX                                     \
1716          VALGRIND_RESTORE_STACK                                        \
1717          VALGRIND_CFI_EPILOGUE                                         \
1718          : /*out*/   "=a" (_res)                                       \
1719          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1720          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1721       );                                                               \
1722       lval = (__typeof__(lval)) _res;                                  \
1723    } while (0)
1724
1725 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)             \
1726    do {                                                                \
1727       volatile OrigFn        _orig = (orig);                           \
1728       volatile unsigned long _argvec[6];                               \
1729       volatile unsigned long _res;                                     \
1730       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1731       _argvec[1] = (unsigned long)(arg1);                              \
1732       _argvec[2] = (unsigned long)(arg2);                              \
1733       _argvec[3] = (unsigned long)(arg3);                              \
1734       _argvec[4] = (unsigned long)(arg4);                              \
1735       _argvec[5] = (unsigned long)(arg5);                              \
1736       __asm__ volatile(                                                \
1737          VALGRIND_CFI_PROLOGUE                                         \
1738          VALGRIND_ALIGN_STACK                                          \
1739          "subq $128,%%rsp\n\t"                                         \
1740          "movq 40(%%rax), %%r8\n\t"                                    \
1741          "movq 32(%%rax), %%rcx\n\t"                                   \
1742          "movq 24(%%rax), %%rdx\n\t"                                   \
1743          "movq 16(%%rax), %%rsi\n\t"                                   \
1744          "movq 8(%%rax), %%rdi\n\t"                                    \
1745          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1746          VALGRIND_CALL_NOREDIR_RAX                                     \
1747          VALGRIND_RESTORE_STACK                                        \
1748          VALGRIND_CFI_EPILOGUE                                         \
1749          : /*out*/   "=a" (_res)                                       \
1750          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1751          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1752       );                                                               \
1753       lval = (__typeof__(lval)) _res;                                  \
1754    } while (0)
1755
1756 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)        \
1757    do {                                                                \
1758       volatile OrigFn        _orig = (orig);                           \
1759       volatile unsigned long _argvec[7];                               \
1760       volatile unsigned long _res;                                     \
1761       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1762       _argvec[1] = (unsigned long)(arg1);                              \
1763       _argvec[2] = (unsigned long)(arg2);                              \
1764       _argvec[3] = (unsigned long)(arg3);                              \
1765       _argvec[4] = (unsigned long)(arg4);                              \
1766       _argvec[5] = (unsigned long)(arg5);                              \
1767       _argvec[6] = (unsigned long)(arg6);                              \
1768       __asm__ volatile(                                                \
1769          VALGRIND_CFI_PROLOGUE                                         \
1770          VALGRIND_ALIGN_STACK                                          \
1771          "subq $128,%%rsp\n\t"                                         \
1772          "movq 48(%%rax), %%r9\n\t"                                    \
1773          "movq 40(%%rax), %%r8\n\t"                                    \
1774          "movq 32(%%rax), %%rcx\n\t"                                   \
1775          "movq 24(%%rax), %%rdx\n\t"                                   \
1776          "movq 16(%%rax), %%rsi\n\t"                                   \
1777          "movq 8(%%rax), %%rdi\n\t"                                    \
1778          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1779          VALGRIND_CALL_NOREDIR_RAX                                     \
1780          VALGRIND_RESTORE_STACK                                        \
1781          VALGRIND_CFI_EPILOGUE                                         \
1782          : /*out*/   "=a" (_res)                                       \
1783          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1784          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1785       );                                                               \
1786       lval = (__typeof__(lval)) _res;                                  \
1787    } while (0)
1788
1789 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1790                                  arg7)                                 \
1791    do {                                                                \
1792       volatile OrigFn        _orig = (orig);                           \
1793       volatile unsigned long _argvec[8];                               \
1794       volatile unsigned long _res;                                     \
1795       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1796       _argvec[1] = (unsigned long)(arg1);                              \
1797       _argvec[2] = (unsigned long)(arg2);                              \
1798       _argvec[3] = (unsigned long)(arg3);                              \
1799       _argvec[4] = (unsigned long)(arg4);                              \
1800       _argvec[5] = (unsigned long)(arg5);                              \
1801       _argvec[6] = (unsigned long)(arg6);                              \
1802       _argvec[7] = (unsigned long)(arg7);                              \
1803       __asm__ volatile(                                                \
1804          VALGRIND_CFI_PROLOGUE                                         \
1805          VALGRIND_ALIGN_STACK                                          \
1806          "subq $136,%%rsp\n\t"                                         \
1807          "pushq 56(%%rax)\n\t"                                         \
1808          "movq 48(%%rax), %%r9\n\t"                                    \
1809          "movq 40(%%rax), %%r8\n\t"                                    \
1810          "movq 32(%%rax), %%rcx\n\t"                                   \
1811          "movq 24(%%rax), %%rdx\n\t"                                   \
1812          "movq 16(%%rax), %%rsi\n\t"                                   \
1813          "movq 8(%%rax), %%rdi\n\t"                                    \
1814          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1815          VALGRIND_CALL_NOREDIR_RAX                                     \
1816          VALGRIND_RESTORE_STACK                                        \
1817          VALGRIND_CFI_EPILOGUE                                         \
1818          : /*out*/   "=a" (_res)                                       \
1819          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1820          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1821       );                                                               \
1822       lval = (__typeof__(lval)) _res;                                  \
1823    } while (0)
1824
1825 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1826                                  arg7,arg8)                            \
1827    do {                                                                \
1828       volatile OrigFn        _orig = (orig);                           \
1829       volatile unsigned long _argvec[9];                               \
1830       volatile unsigned long _res;                                     \
1831       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1832       _argvec[1] = (unsigned long)(arg1);                              \
1833       _argvec[2] = (unsigned long)(arg2);                              \
1834       _argvec[3] = (unsigned long)(arg3);                              \
1835       _argvec[4] = (unsigned long)(arg4);                              \
1836       _argvec[5] = (unsigned long)(arg5);                              \
1837       _argvec[6] = (unsigned long)(arg6);                              \
1838       _argvec[7] = (unsigned long)(arg7);                              \
1839       _argvec[8] = (unsigned long)(arg8);                              \
1840       __asm__ volatile(                                                \
1841          VALGRIND_CFI_PROLOGUE                                         \
1842          VALGRIND_ALIGN_STACK                                          \
1843          "subq $128,%%rsp\n\t"                                         \
1844          "pushq 64(%%rax)\n\t"                                         \
1845          "pushq 56(%%rax)\n\t"                                         \
1846          "movq 48(%%rax), %%r9\n\t"                                    \
1847          "movq 40(%%rax), %%r8\n\t"                                    \
1848          "movq 32(%%rax), %%rcx\n\t"                                   \
1849          "movq 24(%%rax), %%rdx\n\t"                                   \
1850          "movq 16(%%rax), %%rsi\n\t"                                   \
1851          "movq 8(%%rax), %%rdi\n\t"                                    \
1852          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1853          VALGRIND_CALL_NOREDIR_RAX                                     \
1854          VALGRIND_RESTORE_STACK                                        \
1855          VALGRIND_CFI_EPILOGUE                                         \
1856          : /*out*/   "=a" (_res)                                       \
1857          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1858          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1859       );                                                               \
1860       lval = (__typeof__(lval)) _res;                                  \
1861    } while (0)
1862
1863 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1864                                  arg7,arg8,arg9)                       \
1865    do {                                                                \
1866       volatile OrigFn        _orig = (orig);                           \
1867       volatile unsigned long _argvec[10];                              \
1868       volatile unsigned long _res;                                     \
1869       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1870       _argvec[1] = (unsigned long)(arg1);                              \
1871       _argvec[2] = (unsigned long)(arg2);                              \
1872       _argvec[3] = (unsigned long)(arg3);                              \
1873       _argvec[4] = (unsigned long)(arg4);                              \
1874       _argvec[5] = (unsigned long)(arg5);                              \
1875       _argvec[6] = (unsigned long)(arg6);                              \
1876       _argvec[7] = (unsigned long)(arg7);                              \
1877       _argvec[8] = (unsigned long)(arg8);                              \
1878       _argvec[9] = (unsigned long)(arg9);                              \
1879       __asm__ volatile(                                                \
1880          VALGRIND_CFI_PROLOGUE                                         \
1881          VALGRIND_ALIGN_STACK                                          \
1882          "subq $136,%%rsp\n\t"                                         \
1883          "pushq 72(%%rax)\n\t"                                         \
1884          "pushq 64(%%rax)\n\t"                                         \
1885          "pushq 56(%%rax)\n\t"                                         \
1886          "movq 48(%%rax), %%r9\n\t"                                    \
1887          "movq 40(%%rax), %%r8\n\t"                                    \
1888          "movq 32(%%rax), %%rcx\n\t"                                   \
1889          "movq 24(%%rax), %%rdx\n\t"                                   \
1890          "movq 16(%%rax), %%rsi\n\t"                                   \
1891          "movq 8(%%rax), %%rdi\n\t"                                    \
1892          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1893          VALGRIND_CALL_NOREDIR_RAX                                     \
1894          VALGRIND_RESTORE_STACK                                        \
1895          VALGRIND_CFI_EPILOGUE                                         \
1896          : /*out*/   "=a" (_res)                                       \
1897          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1898          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1899       );                                                               \
1900       lval = (__typeof__(lval)) _res;                                  \
1901    } while (0)
1902
1903 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
1904                                   arg7,arg8,arg9,arg10)                \
1905    do {                                                                \
1906       volatile OrigFn        _orig = (orig);                           \
1907       volatile unsigned long _argvec[11];                              \
1908       volatile unsigned long _res;                                     \
1909       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1910       _argvec[1] = (unsigned long)(arg1);                              \
1911       _argvec[2] = (unsigned long)(arg2);                              \
1912       _argvec[3] = (unsigned long)(arg3);                              \
1913       _argvec[4] = (unsigned long)(arg4);                              \
1914       _argvec[5] = (unsigned long)(arg5);                              \
1915       _argvec[6] = (unsigned long)(arg6);                              \
1916       _argvec[7] = (unsigned long)(arg7);                              \
1917       _argvec[8] = (unsigned long)(arg8);                              \
1918       _argvec[9] = (unsigned long)(arg9);                              \
1919       _argvec[10] = (unsigned long)(arg10);                            \
1920       __asm__ volatile(                                                \
1921          VALGRIND_CFI_PROLOGUE                                         \
1922          VALGRIND_ALIGN_STACK                                          \
1923          "subq $128,%%rsp\n\t"                                         \
1924          "pushq 80(%%rax)\n\t"                                         \
1925          "pushq 72(%%rax)\n\t"                                         \
1926          "pushq 64(%%rax)\n\t"                                         \
1927          "pushq 56(%%rax)\n\t"                                         \
1928          "movq 48(%%rax), %%r9\n\t"                                    \
1929          "movq 40(%%rax), %%r8\n\t"                                    \
1930          "movq 32(%%rax), %%rcx\n\t"                                   \
1931          "movq 24(%%rax), %%rdx\n\t"                                   \
1932          "movq 16(%%rax), %%rsi\n\t"                                   \
1933          "movq 8(%%rax), %%rdi\n\t"                                    \
1934          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1935          VALGRIND_CALL_NOREDIR_RAX                                     \
1936          VALGRIND_RESTORE_STACK                                        \
1937          VALGRIND_CFI_EPILOGUE                                         \
1938          : /*out*/   "=a" (_res)                                       \
1939          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1940          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1941       );                                                               \
1942       lval = (__typeof__(lval)) _res;                                  \
1943    } while (0)
1944
1945 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
1946                                   arg7,arg8,arg9,arg10,arg11)          \
1947    do {                                                                \
1948       volatile OrigFn        _orig = (orig);                           \
1949       volatile unsigned long _argvec[12];                              \
1950       volatile unsigned long _res;                                     \
1951       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1952       _argvec[1] = (unsigned long)(arg1);                              \
1953       _argvec[2] = (unsigned long)(arg2);                              \
1954       _argvec[3] = (unsigned long)(arg3);                              \
1955       _argvec[4] = (unsigned long)(arg4);                              \
1956       _argvec[5] = (unsigned long)(arg5);                              \
1957       _argvec[6] = (unsigned long)(arg6);                              \
1958       _argvec[7] = (unsigned long)(arg7);                              \
1959       _argvec[8] = (unsigned long)(arg8);                              \
1960       _argvec[9] = (unsigned long)(arg9);                              \
1961       _argvec[10] = (unsigned long)(arg10);                            \
1962       _argvec[11] = (unsigned long)(arg11);                            \
1963       __asm__ volatile(                                                \
1964          VALGRIND_CFI_PROLOGUE                                         \
1965          VALGRIND_ALIGN_STACK                                          \
1966          "subq $136,%%rsp\n\t"                                         \
1967          "pushq 88(%%rax)\n\t"                                         \
1968          "pushq 80(%%rax)\n\t"                                         \
1969          "pushq 72(%%rax)\n\t"                                         \
1970          "pushq 64(%%rax)\n\t"                                         \
1971          "pushq 56(%%rax)\n\t"                                         \
1972          "movq 48(%%rax), %%r9\n\t"                                    \
1973          "movq 40(%%rax), %%r8\n\t"                                    \
1974          "movq 32(%%rax), %%rcx\n\t"                                   \
1975          "movq 24(%%rax), %%rdx\n\t"                                   \
1976          "movq 16(%%rax), %%rsi\n\t"                                   \
1977          "movq 8(%%rax), %%rdi\n\t"                                    \
1978          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1979          VALGRIND_CALL_NOREDIR_RAX                                     \
1980          VALGRIND_RESTORE_STACK                                        \
1981          VALGRIND_CFI_EPILOGUE                                         \
1982          : /*out*/   "=a" (_res)                                       \
1983          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1985       );                                                               \
1986       lval = (__typeof__(lval)) _res;                                  \
1987    } while (0)
1988
1989 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
1990                                 arg7,arg8,arg9,arg10,arg11,arg12)      \
1991    do {                                                                \
1992       volatile OrigFn        _orig = (orig);                           \
1993       volatile unsigned long _argvec[13];                              \
1994       volatile unsigned long _res;                                     \
1995       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1996       _argvec[1] = (unsigned long)(arg1);                              \
1997       _argvec[2] = (unsigned long)(arg2);                              \
1998       _argvec[3] = (unsigned long)(arg3);                              \
1999       _argvec[4] = (unsigned long)(arg4);                              \
2000       _argvec[5] = (unsigned long)(arg5);                              \
2001       _argvec[6] = (unsigned long)(arg6);                              \
2002       _argvec[7] = (unsigned long)(arg7);                              \
2003       _argvec[8] = (unsigned long)(arg8);                              \
2004       _argvec[9] = (unsigned long)(arg9);                              \
2005       _argvec[10] = (unsigned long)(arg10);                            \
2006       _argvec[11] = (unsigned long)(arg11);                            \
2007       _argvec[12] = (unsigned long)(arg12);                            \
2008       __asm__ volatile(                                                \
2009          VALGRIND_CFI_PROLOGUE                                         \
2010          VALGRIND_ALIGN_STACK                                          \
2011          "subq $128,%%rsp\n\t"                                         \
2012          "pushq 96(%%rax)\n\t"                                         \
2013          "pushq 88(%%rax)\n\t"                                         \
2014          "pushq 80(%%rax)\n\t"                                         \
2015          "pushq 72(%%rax)\n\t"                                         \
2016          "pushq 64(%%rax)\n\t"                                         \
2017          "pushq 56(%%rax)\n\t"                                         \
2018          "movq 48(%%rax), %%r9\n\t"                                    \
2019          "movq 40(%%rax), %%r8\n\t"                                    \
2020          "movq 32(%%rax), %%rcx\n\t"                                   \
2021          "movq 24(%%rax), %%rdx\n\t"                                   \
2022          "movq 16(%%rax), %%rsi\n\t"                                   \
2023          "movq 8(%%rax), %%rdi\n\t"                                    \
2024          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2025          VALGRIND_CALL_NOREDIR_RAX                                     \
2026          VALGRIND_RESTORE_STACK                                        \
2027          VALGRIND_CFI_EPILOGUE                                         \
2028          : /*out*/   "=a" (_res)                                       \
2029          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2030          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2031       );                                                               \
2032       lval = (__typeof__(lval)) _res;                                  \
2033    } while (0)
2034
2035 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
2036
2037 /* ------------------------ ppc32-linux ------------------------ */
2038
2039 #if defined(PLAT_ppc32_linux)
2040
2041 /* This is useful for finding out about the on-stack stuff:
2042
2043    extern int f9  ( int,int,int,int,int,int,int,int,int );
2044    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
2045    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
2046    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
2047
2048    int g9 ( void ) {
2049       return f9(11,22,33,44,55,66,77,88,99);
2050    }
2051    int g10 ( void ) {
2052       return f10(11,22,33,44,55,66,77,88,99,110);
2053    }
2054    int g11 ( void ) {
2055       return f11(11,22,33,44,55,66,77,88,99,110,121);
2056    }
2057    int g12 ( void ) {
2058       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
2059    }
2060 */
2061
2062 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2063
2064 /* These regs are trashed by the hidden call. */
2065 #define __CALLER_SAVED_REGS                                       \
2066    "lr", "ctr", "xer",                                            \
2067    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2068    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2069    "r11", "r12", "r13"
2070
2071 /* Macros to save and align the stack before making a function
2072    call and restore it afterwards as gcc may not keep the stack
2073    pointer aligned if it doesn't realise calls are being made
2074    to other functions. */
2075
2076 #define VALGRIND_ALIGN_STACK               \
2077       "mr 28,1\n\t"                        \
2078       "rlwinm 1,1,0,0,27\n\t"
2079 #define VALGRIND_RESTORE_STACK             \
2080       "mr 1,28\n\t"
2081
2082 /* These CALL_FN_ macros assume that on ppc32-linux,
2083    sizeof(unsigned long) == 4. */
2084
2085 #define CALL_FN_W_v(lval, orig)                                   \
2086    do {                                                           \
2087       volatile OrigFn        _orig = (orig);                      \
2088       volatile unsigned long _argvec[1];                          \
2089       volatile unsigned long _res;                                \
2090       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2091       __asm__ volatile(                                           \
2092          VALGRIND_ALIGN_STACK                                     \
2093          "mr 11,%1\n\t"                                           \
2094          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2095          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2096          VALGRIND_RESTORE_STACK                                   \
2097          "mr %0,3"                                                \
2098          : /*out*/   "=r" (_res)                                  \
2099          : /*in*/    "r" (&_argvec[0])                            \
2100          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2101       );                                                          \
2102       lval = (__typeof__(lval)) _res;                             \
2103    } while (0)
2104
2105 #define CALL_FN_W_W(lval, orig, arg1)                             \
2106    do {                                                           \
2107       volatile OrigFn        _orig = (orig);                      \
2108       volatile unsigned long _argvec[2];                          \
2109       volatile unsigned long _res;                                \
2110       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2111       _argvec[1] = (unsigned long)arg1;                           \
2112       __asm__ volatile(                                           \
2113          VALGRIND_ALIGN_STACK                                     \
2114          "mr 11,%1\n\t"                                           \
2115          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2116          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2117          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2118          VALGRIND_RESTORE_STACK                                   \
2119          "mr %0,3"                                                \
2120          : /*out*/   "=r" (_res)                                  \
2121          : /*in*/    "r" (&_argvec[0])                            \
2122          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2123       );                                                          \
2124       lval = (__typeof__(lval)) _res;                             \
2125    } while (0)
2126
2127 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2128    do {                                                           \
2129       volatile OrigFn        _orig = (orig);                      \
2130       volatile unsigned long _argvec[3];                          \
2131       volatile unsigned long _res;                                \
2132       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2133       _argvec[1] = (unsigned long)arg1;                           \
2134       _argvec[2] = (unsigned long)arg2;                           \
2135       __asm__ volatile(                                           \
2136          VALGRIND_ALIGN_STACK                                     \
2137          "mr 11,%1\n\t"                                           \
2138          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2139          "lwz 4,8(11)\n\t"                                        \
2140          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2141          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2142          VALGRIND_RESTORE_STACK                                   \
2143          "mr %0,3"                                                \
2144          : /*out*/   "=r" (_res)                                  \
2145          : /*in*/    "r" (&_argvec[0])                            \
2146          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2147       );                                                          \
2148       lval = (__typeof__(lval)) _res;                             \
2149    } while (0)
2150
2151 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2152    do {                                                           \
2153       volatile OrigFn        _orig = (orig);                      \
2154       volatile unsigned long _argvec[4];                          \
2155       volatile unsigned long _res;                                \
2156       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2157       _argvec[1] = (unsigned long)arg1;                           \
2158       _argvec[2] = (unsigned long)arg2;                           \
2159       _argvec[3] = (unsigned long)arg3;                           \
2160       __asm__ volatile(                                           \
2161          VALGRIND_ALIGN_STACK                                     \
2162          "mr 11,%1\n\t"                                           \
2163          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2164          "lwz 4,8(11)\n\t"                                        \
2165          "lwz 5,12(11)\n\t"                                       \
2166          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2167          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2168          VALGRIND_RESTORE_STACK                                   \
2169          "mr %0,3"                                                \
2170          : /*out*/   "=r" (_res)                                  \
2171          : /*in*/    "r" (&_argvec[0])                            \
2172          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2173       );                                                          \
2174       lval = (__typeof__(lval)) _res;                             \
2175    } while (0)
2176
2177 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2178    do {                                                           \
2179       volatile OrigFn        _orig = (orig);                      \
2180       volatile unsigned long _argvec[5];                          \
2181       volatile unsigned long _res;                                \
2182       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2183       _argvec[1] = (unsigned long)arg1;                           \
2184       _argvec[2] = (unsigned long)arg2;                           \
2185       _argvec[3] = (unsigned long)arg3;                           \
2186       _argvec[4] = (unsigned long)arg4;                           \
2187       __asm__ volatile(                                           \
2188          VALGRIND_ALIGN_STACK                                     \
2189          "mr 11,%1\n\t"                                           \
2190          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2191          "lwz 4,8(11)\n\t"                                        \
2192          "lwz 5,12(11)\n\t"                                       \
2193          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2194          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2195          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2196          VALGRIND_RESTORE_STACK                                   \
2197          "mr %0,3"                                                \
2198          : /*out*/   "=r" (_res)                                  \
2199          : /*in*/    "r" (&_argvec[0])                            \
2200          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2201       );                                                          \
2202       lval = (__typeof__(lval)) _res;                             \
2203    } while (0)
2204
2205 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2206    do {                                                           \
2207       volatile OrigFn        _orig = (orig);                      \
2208       volatile unsigned long _argvec[6];                          \
2209       volatile unsigned long _res;                                \
2210       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2211       _argvec[1] = (unsigned long)arg1;                           \
2212       _argvec[2] = (unsigned long)arg2;                           \
2213       _argvec[3] = (unsigned long)arg3;                           \
2214       _argvec[4] = (unsigned long)arg4;                           \
2215       _argvec[5] = (unsigned long)arg5;                           \
2216       __asm__ volatile(                                           \
2217          VALGRIND_ALIGN_STACK                                     \
2218          "mr 11,%1\n\t"                                           \
2219          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2220          "lwz 4,8(11)\n\t"                                        \
2221          "lwz 5,12(11)\n\t"                                       \
2222          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2223          "lwz 7,20(11)\n\t"                                       \
2224          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2225          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2226          VALGRIND_RESTORE_STACK                                   \
2227          "mr %0,3"                                                \
2228          : /*out*/   "=r" (_res)                                  \
2229          : /*in*/    "r" (&_argvec[0])                            \
2230          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2231       );                                                          \
2232       lval = (__typeof__(lval)) _res;                             \
2233    } while (0)
2234
2235 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2236    do {                                                           \
2237       volatile OrigFn        _orig = (orig);                      \
2238       volatile unsigned long _argvec[7];                          \
2239       volatile unsigned long _res;                                \
2240       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2241       _argvec[1] = (unsigned long)arg1;                           \
2242       _argvec[2] = (unsigned long)arg2;                           \
2243       _argvec[3] = (unsigned long)arg3;                           \
2244       _argvec[4] = (unsigned long)arg4;                           \
2245       _argvec[5] = (unsigned long)arg5;                           \
2246       _argvec[6] = (unsigned long)arg6;                           \
2247       __asm__ volatile(                                           \
2248          VALGRIND_ALIGN_STACK                                     \
2249          "mr 11,%1\n\t"                                           \
2250          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2251          "lwz 4,8(11)\n\t"                                        \
2252          "lwz 5,12(11)\n\t"                                       \
2253          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2254          "lwz 7,20(11)\n\t"                                       \
2255          "lwz 8,24(11)\n\t"                                       \
2256          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2257          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2258          VALGRIND_RESTORE_STACK                                   \
2259          "mr %0,3"                                                \
2260          : /*out*/   "=r" (_res)                                  \
2261          : /*in*/    "r" (&_argvec[0])                            \
2262          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2263       );                                                          \
2264       lval = (__typeof__(lval)) _res;                             \
2265    } while (0)
2266
2267 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2268                                  arg7)                            \
2269    do {                                                           \
2270       volatile OrigFn        _orig = (orig);                      \
2271       volatile unsigned long _argvec[8];                          \
2272       volatile unsigned long _res;                                \
2273       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2274       _argvec[1] = (unsigned long)arg1;                           \
2275       _argvec[2] = (unsigned long)arg2;                           \
2276       _argvec[3] = (unsigned long)arg3;                           \
2277       _argvec[4] = (unsigned long)arg4;                           \
2278       _argvec[5] = (unsigned long)arg5;                           \
2279       _argvec[6] = (unsigned long)arg6;                           \
2280       _argvec[7] = (unsigned long)arg7;                           \
2281       __asm__ volatile(                                           \
2282          VALGRIND_ALIGN_STACK                                     \
2283          "mr 11,%1\n\t"                                           \
2284          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2285          "lwz 4,8(11)\n\t"                                        \
2286          "lwz 5,12(11)\n\t"                                       \
2287          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2288          "lwz 7,20(11)\n\t"                                       \
2289          "lwz 8,24(11)\n\t"                                       \
2290          "lwz 9,28(11)\n\t"                                       \
2291          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2292          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2293          VALGRIND_RESTORE_STACK                                   \
2294          "mr %0,3"                                                \
2295          : /*out*/   "=r" (_res)                                  \
2296          : /*in*/    "r" (&_argvec[0])                            \
2297          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2298       );                                                          \
2299       lval = (__typeof__(lval)) _res;                             \
2300    } while (0)
2301
2302 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2303                                  arg7,arg8)                       \
2304    do {                                                           \
2305       volatile OrigFn        _orig = (orig);                      \
2306       volatile unsigned long _argvec[9];                          \
2307       volatile unsigned long _res;                                \
2308       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2309       _argvec[1] = (unsigned long)arg1;                           \
2310       _argvec[2] = (unsigned long)arg2;                           \
2311       _argvec[3] = (unsigned long)arg3;                           \
2312       _argvec[4] = (unsigned long)arg4;                           \
2313       _argvec[5] = (unsigned long)arg5;                           \
2314       _argvec[6] = (unsigned long)arg6;                           \
2315       _argvec[7] = (unsigned long)arg7;                           \
2316       _argvec[8] = (unsigned long)arg8;                           \
2317       __asm__ volatile(                                           \
2318          VALGRIND_ALIGN_STACK                                     \
2319          "mr 11,%1\n\t"                                           \
2320          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2321          "lwz 4,8(11)\n\t"                                        \
2322          "lwz 5,12(11)\n\t"                                       \
2323          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2324          "lwz 7,20(11)\n\t"                                       \
2325          "lwz 8,24(11)\n\t"                                       \
2326          "lwz 9,28(11)\n\t"                                       \
2327          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2328          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2329          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2330          VALGRIND_RESTORE_STACK                                   \
2331          "mr %0,3"                                                \
2332          : /*out*/   "=r" (_res)                                  \
2333          : /*in*/    "r" (&_argvec[0])                            \
2334          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2335       );                                                          \
2336       lval = (__typeof__(lval)) _res;                             \
2337    } while (0)
2338
2339 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2340                                  arg7,arg8,arg9)                  \
2341    do {                                                           \
2342       volatile OrigFn        _orig = (orig);                      \
2343       volatile unsigned long _argvec[10];                         \
2344       volatile unsigned long _res;                                \
2345       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2346       _argvec[1] = (unsigned long)arg1;                           \
2347       _argvec[2] = (unsigned long)arg2;                           \
2348       _argvec[3] = (unsigned long)arg3;                           \
2349       _argvec[4] = (unsigned long)arg4;                           \
2350       _argvec[5] = (unsigned long)arg5;                           \
2351       _argvec[6] = (unsigned long)arg6;                           \
2352       _argvec[7] = (unsigned long)arg7;                           \
2353       _argvec[8] = (unsigned long)arg8;                           \
2354       _argvec[9] = (unsigned long)arg9;                           \
2355       __asm__ volatile(                                           \
2356          VALGRIND_ALIGN_STACK                                     \
2357          "mr 11,%1\n\t"                                           \
2358          "addi 1,1,-16\n\t"                                       \
2359          /* arg9 */                                               \
2360          "lwz 3,36(11)\n\t"                                       \
2361          "stw 3,8(1)\n\t"                                         \
2362          /* args1-8 */                                            \
2363          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2364          "lwz 4,8(11)\n\t"                                        \
2365          "lwz 5,12(11)\n\t"                                       \
2366          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2367          "lwz 7,20(11)\n\t"                                       \
2368          "lwz 8,24(11)\n\t"                                       \
2369          "lwz 9,28(11)\n\t"                                       \
2370          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2371          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2372          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2373          VALGRIND_RESTORE_STACK                                   \
2374          "mr %0,3"                                                \
2375          : /*out*/   "=r" (_res)                                  \
2376          : /*in*/    "r" (&_argvec[0])                            \
2377          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2378       );                                                          \
2379       lval = (__typeof__(lval)) _res;                             \
2380    } while (0)
2381
2382 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2383                                   arg7,arg8,arg9,arg10)           \
2384    do {                                                           \
2385       volatile OrigFn        _orig = (orig);                      \
2386       volatile unsigned long _argvec[11];                         \
2387       volatile unsigned long _res;                                \
2388       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2389       _argvec[1] = (unsigned long)arg1;                           \
2390       _argvec[2] = (unsigned long)arg2;                           \
2391       _argvec[3] = (unsigned long)arg3;                           \
2392       _argvec[4] = (unsigned long)arg4;                           \
2393       _argvec[5] = (unsigned long)arg5;                           \
2394       _argvec[6] = (unsigned long)arg6;                           \
2395       _argvec[7] = (unsigned long)arg7;                           \
2396       _argvec[8] = (unsigned long)arg8;                           \
2397       _argvec[9] = (unsigned long)arg9;                           \
2398       _argvec[10] = (unsigned long)arg10;                         \
2399       __asm__ volatile(                                           \
2400          VALGRIND_ALIGN_STACK                                     \
2401          "mr 11,%1\n\t"                                           \
2402          "addi 1,1,-16\n\t"                                       \
2403          /* arg10 */                                              \
2404          "lwz 3,40(11)\n\t"                                       \
2405          "stw 3,12(1)\n\t"                                        \
2406          /* arg9 */                                               \
2407          "lwz 3,36(11)\n\t"                                       \
2408          "stw 3,8(1)\n\t"                                         \
2409          /* args1-8 */                                            \
2410          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2411          "lwz 4,8(11)\n\t"                                        \
2412          "lwz 5,12(11)\n\t"                                       \
2413          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2414          "lwz 7,20(11)\n\t"                                       \
2415          "lwz 8,24(11)\n\t"                                       \
2416          "lwz 9,28(11)\n\t"                                       \
2417          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2418          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2419          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2420          VALGRIND_RESTORE_STACK                                   \
2421          "mr %0,3"                                                \
2422          : /*out*/   "=r" (_res)                                  \
2423          : /*in*/    "r" (&_argvec[0])                            \
2424          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2425       );                                                          \
2426       lval = (__typeof__(lval)) _res;                             \
2427    } while (0)
2428
2429 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2430                                   arg7,arg8,arg9,arg10,arg11)     \
2431    do {                                                           \
2432       volatile OrigFn        _orig = (orig);                      \
2433       volatile unsigned long _argvec[12];                         \
2434       volatile unsigned long _res;                                \
2435       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2436       _argvec[1] = (unsigned long)arg1;                           \
2437       _argvec[2] = (unsigned long)arg2;                           \
2438       _argvec[3] = (unsigned long)arg3;                           \
2439       _argvec[4] = (unsigned long)arg4;                           \
2440       _argvec[5] = (unsigned long)arg5;                           \
2441       _argvec[6] = (unsigned long)arg6;                           \
2442       _argvec[7] = (unsigned long)arg7;                           \
2443       _argvec[8] = (unsigned long)arg8;                           \
2444       _argvec[9] = (unsigned long)arg9;                           \
2445       _argvec[10] = (unsigned long)arg10;                         \
2446       _argvec[11] = (unsigned long)arg11;                         \
2447       __asm__ volatile(                                           \
2448          VALGRIND_ALIGN_STACK                                     \
2449          "mr 11,%1\n\t"                                           \
2450          "addi 1,1,-32\n\t"                                       \
2451          /* arg11 */                                              \
2452          "lwz 3,44(11)\n\t"                                       \
2453          "stw 3,16(1)\n\t"                                        \
2454          /* arg10 */                                              \
2455          "lwz 3,40(11)\n\t"                                       \
2456          "stw 3,12(1)\n\t"                                        \
2457          /* arg9 */                                               \
2458          "lwz 3,36(11)\n\t"                                       \
2459          "stw 3,8(1)\n\t"                                         \
2460          /* args1-8 */                                            \
2461          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2462          "lwz 4,8(11)\n\t"                                        \
2463          "lwz 5,12(11)\n\t"                                       \
2464          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2465          "lwz 7,20(11)\n\t"                                       \
2466          "lwz 8,24(11)\n\t"                                       \
2467          "lwz 9,28(11)\n\t"                                       \
2468          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2469          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2470          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2471          VALGRIND_RESTORE_STACK                                   \
2472          "mr %0,3"                                                \
2473          : /*out*/   "=r" (_res)                                  \
2474          : /*in*/    "r" (&_argvec[0])                            \
2475          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2476       );                                                          \
2477       lval = (__typeof__(lval)) _res;                             \
2478    } while (0)
2479
2480 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2481                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2482    do {                                                           \
2483       volatile OrigFn        _orig = (orig);                      \
2484       volatile unsigned long _argvec[13];                         \
2485       volatile unsigned long _res;                                \
2486       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2487       _argvec[1] = (unsigned long)arg1;                           \
2488       _argvec[2] = (unsigned long)arg2;                           \
2489       _argvec[3] = (unsigned long)arg3;                           \
2490       _argvec[4] = (unsigned long)arg4;                           \
2491       _argvec[5] = (unsigned long)arg5;                           \
2492       _argvec[6] = (unsigned long)arg6;                           \
2493       _argvec[7] = (unsigned long)arg7;                           \
2494       _argvec[8] = (unsigned long)arg8;                           \
2495       _argvec[9] = (unsigned long)arg9;                           \
2496       _argvec[10] = (unsigned long)arg10;                         \
2497       _argvec[11] = (unsigned long)arg11;                         \
2498       _argvec[12] = (unsigned long)arg12;                         \
2499       __asm__ volatile(                                           \
2500          VALGRIND_ALIGN_STACK                                     \
2501          "mr 11,%1\n\t"                                           \
2502          "addi 1,1,-32\n\t"                                       \
2503          /* arg12 */                                              \
2504          "lwz 3,48(11)\n\t"                                       \
2505          "stw 3,20(1)\n\t"                                        \
2506          /* arg11 */                                              \
2507          "lwz 3,44(11)\n\t"                                       \
2508          "stw 3,16(1)\n\t"                                        \
2509          /* arg10 */                                              \
2510          "lwz 3,40(11)\n\t"                                       \
2511          "stw 3,12(1)\n\t"                                        \
2512          /* arg9 */                                               \
2513          "lwz 3,36(11)\n\t"                                       \
2514          "stw 3,8(1)\n\t"                                         \
2515          /* args1-8 */                                            \
2516          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2517          "lwz 4,8(11)\n\t"                                        \
2518          "lwz 5,12(11)\n\t"                                       \
2519          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2520          "lwz 7,20(11)\n\t"                                       \
2521          "lwz 8,24(11)\n\t"                                       \
2522          "lwz 9,28(11)\n\t"                                       \
2523          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2524          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2525          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2526          VALGRIND_RESTORE_STACK                                   \
2527          "mr %0,3"                                                \
2528          : /*out*/   "=r" (_res)                                  \
2529          : /*in*/    "r" (&_argvec[0])                            \
2530          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2531       );                                                          \
2532       lval = (__typeof__(lval)) _res;                             \
2533    } while (0)
2534
2535 #endif /* PLAT_ppc32_linux */
2536
2537 /* ------------------------ ppc64-linux ------------------------ */
2538
2539 #if defined(PLAT_ppc64_linux)
2540
2541 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2542
2543 /* These regs are trashed by the hidden call. */
2544 #define __CALLER_SAVED_REGS                                       \
2545    "lr", "ctr", "xer",                                            \
2546    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2547    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2548    "r11", "r12", "r13"
2549
2550 /* Macros to save and align the stack before making a function
2551    call and restore it afterwards as gcc may not keep the stack
2552    pointer aligned if it doesn't realise calls are being made
2553    to other functions. */
2554
2555 #define VALGRIND_ALIGN_STACK               \
2556       "mr 28,1\n\t"                        \
2557       "rldicr 1,1,0,59\n\t"
2558 #define VALGRIND_RESTORE_STACK             \
2559       "mr 1,28\n\t"
2560
2561 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2562    long) == 8. */
2563
2564 #define CALL_FN_W_v(lval, orig)                                   \
2565    do {                                                           \
2566       volatile OrigFn        _orig = (orig);                      \
2567       volatile unsigned long _argvec[3+0];                        \
2568       volatile unsigned long _res;                                \
2569       /* _argvec[0] holds current r2 across the call */           \
2570       _argvec[1] = (unsigned long)_orig.r2;                       \
2571       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2572       __asm__ volatile(                                           \
2573          VALGRIND_ALIGN_STACK                                     \
2574          "mr 11,%1\n\t"                                           \
2575          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2576          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2577          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2578          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2579          "mr 11,%1\n\t"                                           \
2580          "mr %0,3\n\t"                                            \
2581          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2582          VALGRIND_RESTORE_STACK                                   \
2583          : /*out*/   "=r" (_res)                                  \
2584          : /*in*/    "r" (&_argvec[2])                            \
2585          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2586       );                                                          \
2587       lval = (__typeof__(lval)) _res;                             \
2588    } while (0)
2589
2590 #define CALL_FN_W_W(lval, orig, arg1)                             \
2591    do {                                                           \
2592       volatile OrigFn        _orig = (orig);                      \
2593       volatile unsigned long _argvec[3+1];                        \
2594       volatile unsigned long _res;                                \
2595       /* _argvec[0] holds current r2 across the call */           \
2596       _argvec[1]   = (unsigned long)_orig.r2;                     \
2597       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2598       _argvec[2+1] = (unsigned long)arg1;                         \
2599       __asm__ volatile(                                           \
2600          VALGRIND_ALIGN_STACK                                     \
2601          "mr 11,%1\n\t"                                           \
2602          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2603          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2604          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2605          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2606          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2607          "mr 11,%1\n\t"                                           \
2608          "mr %0,3\n\t"                                            \
2609          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2610          VALGRIND_RESTORE_STACK                                   \
2611          : /*out*/   "=r" (_res)                                  \
2612          : /*in*/    "r" (&_argvec[2])                            \
2613          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2614       );                                                          \
2615       lval = (__typeof__(lval)) _res;                             \
2616    } while (0)
2617
2618 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2619    do {                                                           \
2620       volatile OrigFn        _orig = (orig);                      \
2621       volatile unsigned long _argvec[3+2];                        \
2622       volatile unsigned long _res;                                \
2623       /* _argvec[0] holds current r2 across the call */           \
2624       _argvec[1]   = (unsigned long)_orig.r2;                     \
2625       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2626       _argvec[2+1] = (unsigned long)arg1;                         \
2627       _argvec[2+2] = (unsigned long)arg2;                         \
2628       __asm__ volatile(                                           \
2629          VALGRIND_ALIGN_STACK                                     \
2630          "mr 11,%1\n\t"                                           \
2631          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2632          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2633          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2634          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2635          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2636          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2637          "mr 11,%1\n\t"                                           \
2638          "mr %0,3\n\t"                                            \
2639          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2640          VALGRIND_RESTORE_STACK                                   \
2641          : /*out*/   "=r" (_res)                                  \
2642          : /*in*/    "r" (&_argvec[2])                            \
2643          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2644       );                                                          \
2645       lval = (__typeof__(lval)) _res;                             \
2646    } while (0)
2647
2648 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2649    do {                                                           \
2650       volatile OrigFn        _orig = (orig);                      \
2651       volatile unsigned long _argvec[3+3];                        \
2652       volatile unsigned long _res;                                \
2653       /* _argvec[0] holds current r2 across the call */           \
2654       _argvec[1]   = (unsigned long)_orig.r2;                     \
2655       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2656       _argvec[2+1] = (unsigned long)arg1;                         \
2657       _argvec[2+2] = (unsigned long)arg2;                         \
2658       _argvec[2+3] = (unsigned long)arg3;                         \
2659       __asm__ volatile(                                           \
2660          VALGRIND_ALIGN_STACK                                     \
2661          "mr 11,%1\n\t"                                           \
2662          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2663          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2664          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2665          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2666          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2667          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2668          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2669          "mr 11,%1\n\t"                                           \
2670          "mr %0,3\n\t"                                            \
2671          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2672          VALGRIND_RESTORE_STACK                                   \
2673          : /*out*/   "=r" (_res)                                  \
2674          : /*in*/    "r" (&_argvec[2])                            \
2675          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2676       );                                                          \
2677       lval = (__typeof__(lval)) _res;                             \
2678    } while (0)
2679
2680 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2681    do {                                                           \
2682       volatile OrigFn        _orig = (orig);                      \
2683       volatile unsigned long _argvec[3+4];                        \
2684       volatile unsigned long _res;                                \
2685       /* _argvec[0] holds current r2 across the call */           \
2686       _argvec[1]   = (unsigned long)_orig.r2;                     \
2687       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2688       _argvec[2+1] = (unsigned long)arg1;                         \
2689       _argvec[2+2] = (unsigned long)arg2;                         \
2690       _argvec[2+3] = (unsigned long)arg3;                         \
2691       _argvec[2+4] = (unsigned long)arg4;                         \
2692       __asm__ volatile(                                           \
2693          VALGRIND_ALIGN_STACK                                     \
2694          "mr 11,%1\n\t"                                           \
2695          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2696          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2697          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2698          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2699          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2700          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2701          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2702          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2703          "mr 11,%1\n\t"                                           \
2704          "mr %0,3\n\t"                                            \
2705          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2706          VALGRIND_RESTORE_STACK                                   \
2707          : /*out*/   "=r" (_res)                                  \
2708          : /*in*/    "r" (&_argvec[2])                            \
2709          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2710       );                                                          \
2711       lval = (__typeof__(lval)) _res;                             \
2712    } while (0)
2713
2714 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2715    do {                                                           \
2716       volatile OrigFn        _orig = (orig);                      \
2717       volatile unsigned long _argvec[3+5];                        \
2718       volatile unsigned long _res;                                \
2719       /* _argvec[0] holds current r2 across the call */           \
2720       _argvec[1]   = (unsigned long)_orig.r2;                     \
2721       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2722       _argvec[2+1] = (unsigned long)arg1;                         \
2723       _argvec[2+2] = (unsigned long)arg2;                         \
2724       _argvec[2+3] = (unsigned long)arg3;                         \
2725       _argvec[2+4] = (unsigned long)arg4;                         \
2726       _argvec[2+5] = (unsigned long)arg5;                         \
2727       __asm__ volatile(                                           \
2728          VALGRIND_ALIGN_STACK                                     \
2729          "mr 11,%1\n\t"                                           \
2730          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2731          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2732          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2733          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2734          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2735          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2736          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2737          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2738          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2739          "mr 11,%1\n\t"                                           \
2740          "mr %0,3\n\t"                                            \
2741          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2742          VALGRIND_RESTORE_STACK                                   \
2743          : /*out*/   "=r" (_res)                                  \
2744          : /*in*/    "r" (&_argvec[2])                            \
2745          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2746       );                                                          \
2747       lval = (__typeof__(lval)) _res;                             \
2748    } while (0)
2749
2750 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2751    do {                                                           \
2752       volatile OrigFn        _orig = (orig);                      \
2753       volatile unsigned long _argvec[3+6];                        \
2754       volatile unsigned long _res;                                \
2755       /* _argvec[0] holds current r2 across the call */           \
2756       _argvec[1]   = (unsigned long)_orig.r2;                     \
2757       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2758       _argvec[2+1] = (unsigned long)arg1;                         \
2759       _argvec[2+2] = (unsigned long)arg2;                         \
2760       _argvec[2+3] = (unsigned long)arg3;                         \
2761       _argvec[2+4] = (unsigned long)arg4;                         \
2762       _argvec[2+5] = (unsigned long)arg5;                         \
2763       _argvec[2+6] = (unsigned long)arg6;                         \
2764       __asm__ volatile(                                           \
2765          VALGRIND_ALIGN_STACK                                     \
2766          "mr 11,%1\n\t"                                           \
2767          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2768          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2769          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2770          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2771          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2772          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2773          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2774          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2775          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2776          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2777          "mr 11,%1\n\t"                                           \
2778          "mr %0,3\n\t"                                            \
2779          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2780          VALGRIND_RESTORE_STACK                                   \
2781          : /*out*/   "=r" (_res)                                  \
2782          : /*in*/    "r" (&_argvec[2])                            \
2783          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2784       );                                                          \
2785       lval = (__typeof__(lval)) _res;                             \
2786    } while (0)
2787
2788 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2789                                  arg7)                            \
2790    do {                                                           \
2791       volatile OrigFn        _orig = (orig);                      \
2792       volatile unsigned long _argvec[3+7];                        \
2793       volatile unsigned long _res;                                \
2794       /* _argvec[0] holds current r2 across the call */           \
2795       _argvec[1]   = (unsigned long)_orig.r2;                     \
2796       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2797       _argvec[2+1] = (unsigned long)arg1;                         \
2798       _argvec[2+2] = (unsigned long)arg2;                         \
2799       _argvec[2+3] = (unsigned long)arg3;                         \
2800       _argvec[2+4] = (unsigned long)arg4;                         \
2801       _argvec[2+5] = (unsigned long)arg5;                         \
2802       _argvec[2+6] = (unsigned long)arg6;                         \
2803       _argvec[2+7] = (unsigned long)arg7;                         \
2804       __asm__ volatile(                                           \
2805          VALGRIND_ALIGN_STACK                                     \
2806          "mr 11,%1\n\t"                                           \
2807          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2808          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2809          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2810          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2811          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2812          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2813          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2814          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2815          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2816          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2817          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2818          "mr 11,%1\n\t"                                           \
2819          "mr %0,3\n\t"                                            \
2820          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2821          VALGRIND_RESTORE_STACK                                   \
2822          : /*out*/   "=r" (_res)                                  \
2823          : /*in*/    "r" (&_argvec[2])                            \
2824          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2825       );                                                          \
2826       lval = (__typeof__(lval)) _res;                             \
2827    } while (0)
2828
2829 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2830                                  arg7,arg8)                       \
2831    do {                                                           \
2832       volatile OrigFn        _orig = (orig);                      \
2833       volatile unsigned long _argvec[3+8];                        \
2834       volatile unsigned long _res;                                \
2835       /* _argvec[0] holds current r2 across the call */           \
2836       _argvec[1]   = (unsigned long)_orig.r2;                     \
2837       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2838       _argvec[2+1] = (unsigned long)arg1;                         \
2839       _argvec[2+2] = (unsigned long)arg2;                         \
2840       _argvec[2+3] = (unsigned long)arg3;                         \
2841       _argvec[2+4] = (unsigned long)arg4;                         \
2842       _argvec[2+5] = (unsigned long)arg5;                         \
2843       _argvec[2+6] = (unsigned long)arg6;                         \
2844       _argvec[2+7] = (unsigned long)arg7;                         \
2845       _argvec[2+8] = (unsigned long)arg8;                         \
2846       __asm__ volatile(                                           \
2847          VALGRIND_ALIGN_STACK                                     \
2848          "mr 11,%1\n\t"                                           \
2849          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2850          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2851          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2852          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2853          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2854          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2855          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2856          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2857          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2858          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2859          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2860          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2861          "mr 11,%1\n\t"                                           \
2862          "mr %0,3\n\t"                                            \
2863          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2864          VALGRIND_RESTORE_STACK                                   \
2865          : /*out*/   "=r" (_res)                                  \
2866          : /*in*/    "r" (&_argvec[2])                            \
2867          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2868       );                                                          \
2869       lval = (__typeof__(lval)) _res;                             \
2870    } while (0)
2871
2872 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2873                                  arg7,arg8,arg9)                  \
2874    do {                                                           \
2875       volatile OrigFn        _orig = (orig);                      \
2876       volatile unsigned long _argvec[3+9];                        \
2877       volatile unsigned long _res;                                \
2878       /* _argvec[0] holds current r2 across the call */           \
2879       _argvec[1]   = (unsigned long)_orig.r2;                     \
2880       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2881       _argvec[2+1] = (unsigned long)arg1;                         \
2882       _argvec[2+2] = (unsigned long)arg2;                         \
2883       _argvec[2+3] = (unsigned long)arg3;                         \
2884       _argvec[2+4] = (unsigned long)arg4;                         \
2885       _argvec[2+5] = (unsigned long)arg5;                         \
2886       _argvec[2+6] = (unsigned long)arg6;                         \
2887       _argvec[2+7] = (unsigned long)arg7;                         \
2888       _argvec[2+8] = (unsigned long)arg8;                         \
2889       _argvec[2+9] = (unsigned long)arg9;                         \
2890       __asm__ volatile(                                           \
2891          VALGRIND_ALIGN_STACK                                     \
2892          "mr 11,%1\n\t"                                           \
2893          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2894          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2895          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2896          /* arg9 */                                               \
2897          "ld  3,72(11)\n\t"                                       \
2898          "std 3,112(1)\n\t"                                       \
2899          /* args1-8 */                                            \
2900          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2901          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2902          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2903          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2904          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2905          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2906          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2907          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2908          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2909          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2910          "mr 11,%1\n\t"                                           \
2911          "mr %0,3\n\t"                                            \
2912          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2913          VALGRIND_RESTORE_STACK                                   \
2914          : /*out*/   "=r" (_res)                                  \
2915          : /*in*/    "r" (&_argvec[2])                            \
2916          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2917       );                                                          \
2918       lval = (__typeof__(lval)) _res;                             \
2919    } while (0)
2920
2921 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2922                                   arg7,arg8,arg9,arg10)           \
2923    do {                                                           \
2924       volatile OrigFn        _orig = (orig);                      \
2925       volatile unsigned long _argvec[3+10];                       \
2926       volatile unsigned long _res;                                \
2927       /* _argvec[0] holds current r2 across the call */           \
2928       _argvec[1]   = (unsigned long)_orig.r2;                     \
2929       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2930       _argvec[2+1] = (unsigned long)arg1;                         \
2931       _argvec[2+2] = (unsigned long)arg2;                         \
2932       _argvec[2+3] = (unsigned long)arg3;                         \
2933       _argvec[2+4] = (unsigned long)arg4;                         \
2934       _argvec[2+5] = (unsigned long)arg5;                         \
2935       _argvec[2+6] = (unsigned long)arg6;                         \
2936       _argvec[2+7] = (unsigned long)arg7;                         \
2937       _argvec[2+8] = (unsigned long)arg8;                         \
2938       _argvec[2+9] = (unsigned long)arg9;                         \
2939       _argvec[2+10] = (unsigned long)arg10;                       \
2940       __asm__ volatile(                                           \
2941          VALGRIND_ALIGN_STACK                                     \
2942          "mr 11,%1\n\t"                                           \
2943          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2944          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2945          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2946          /* arg10 */                                              \
2947          "ld  3,80(11)\n\t"                                       \
2948          "std 3,120(1)\n\t"                                       \
2949          /* arg9 */                                               \
2950          "ld  3,72(11)\n\t"                                       \
2951          "std 3,112(1)\n\t"                                       \
2952          /* args1-8 */                                            \
2953          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2954          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2955          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2956          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2957          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2958          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2959          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2960          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2961          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2962          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2963          "mr 11,%1\n\t"                                           \
2964          "mr %0,3\n\t"                                            \
2965          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2966          VALGRIND_RESTORE_STACK                                   \
2967          : /*out*/   "=r" (_res)                                  \
2968          : /*in*/    "r" (&_argvec[2])                            \
2969          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2970       );                                                          \
2971       lval = (__typeof__(lval)) _res;                             \
2972    } while (0)
2973
2974 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2975                                   arg7,arg8,arg9,arg10,arg11)     \
2976    do {                                                           \
2977       volatile OrigFn        _orig = (orig);                      \
2978       volatile unsigned long _argvec[3+11];                       \
2979       volatile unsigned long _res;                                \
2980       /* _argvec[0] holds current r2 across the call */           \
2981       _argvec[1]   = (unsigned long)_orig.r2;                     \
2982       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2983       _argvec[2+1] = (unsigned long)arg1;                         \
2984       _argvec[2+2] = (unsigned long)arg2;                         \
2985       _argvec[2+3] = (unsigned long)arg3;                         \
2986       _argvec[2+4] = (unsigned long)arg4;                         \
2987       _argvec[2+5] = (unsigned long)arg5;                         \
2988       _argvec[2+6] = (unsigned long)arg6;                         \
2989       _argvec[2+7] = (unsigned long)arg7;                         \
2990       _argvec[2+8] = (unsigned long)arg8;                         \
2991       _argvec[2+9] = (unsigned long)arg9;                         \
2992       _argvec[2+10] = (unsigned long)arg10;                       \
2993       _argvec[2+11] = (unsigned long)arg11;                       \
2994       __asm__ volatile(                                           \
2995          VALGRIND_ALIGN_STACK                                     \
2996          "mr 11,%1\n\t"                                           \
2997          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2998          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2999          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3000          /* arg11 */                                              \
3001          "ld  3,88(11)\n\t"                                       \
3002          "std 3,128(1)\n\t"                                       \
3003          /* arg10 */                                              \
3004          "ld  3,80(11)\n\t"                                       \
3005          "std 3,120(1)\n\t"                                       \
3006          /* arg9 */                                               \
3007          "ld  3,72(11)\n\t"                                       \
3008          "std 3,112(1)\n\t"                                       \
3009          /* args1-8 */                                            \
3010          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3011          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3012          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3013          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3014          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3015          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3016          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3017          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3018          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3019          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3020          "mr 11,%1\n\t"                                           \
3021          "mr %0,3\n\t"                                            \
3022          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3023          VALGRIND_RESTORE_STACK                                   \
3024          : /*out*/   "=r" (_res)                                  \
3025          : /*in*/    "r" (&_argvec[2])                            \
3026          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3027       );                                                          \
3028       lval = (__typeof__(lval)) _res;                             \
3029    } while (0)
3030
3031 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3032                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3033    do {                                                           \
3034       volatile OrigFn        _orig = (orig);                      \
3035       volatile unsigned long _argvec[3+12];                       \
3036       volatile unsigned long _res;                                \
3037       /* _argvec[0] holds current r2 across the call */           \
3038       _argvec[1]   = (unsigned long)_orig.r2;                     \
3039       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3040       _argvec[2+1] = (unsigned long)arg1;                         \
3041       _argvec[2+2] = (unsigned long)arg2;                         \
3042       _argvec[2+3] = (unsigned long)arg3;                         \
3043       _argvec[2+4] = (unsigned long)arg4;                         \
3044       _argvec[2+5] = (unsigned long)arg5;                         \
3045       _argvec[2+6] = (unsigned long)arg6;                         \
3046       _argvec[2+7] = (unsigned long)arg7;                         \
3047       _argvec[2+8] = (unsigned long)arg8;                         \
3048       _argvec[2+9] = (unsigned long)arg9;                         \
3049       _argvec[2+10] = (unsigned long)arg10;                       \
3050       _argvec[2+11] = (unsigned long)arg11;                       \
3051       _argvec[2+12] = (unsigned long)arg12;                       \
3052       __asm__ volatile(                                           \
3053          VALGRIND_ALIGN_STACK                                     \
3054          "mr 11,%1\n\t"                                           \
3055          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3056          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3057          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3058          /* arg12 */                                              \
3059          "ld  3,96(11)\n\t"                                       \
3060          "std 3,136(1)\n\t"                                       \
3061          /* arg11 */                                              \
3062          "ld  3,88(11)\n\t"                                       \
3063          "std 3,128(1)\n\t"                                       \
3064          /* arg10 */                                              \
3065          "ld  3,80(11)\n\t"                                       \
3066          "std 3,120(1)\n\t"                                       \
3067          /* arg9 */                                               \
3068          "ld  3,72(11)\n\t"                                       \
3069          "std 3,112(1)\n\t"                                       \
3070          /* args1-8 */                                            \
3071          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3072          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3073          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3074          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3075          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3076          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3077          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3078          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3079          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3080          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3081          "mr 11,%1\n\t"                                           \
3082          "mr %0,3\n\t"                                            \
3083          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3084          VALGRIND_RESTORE_STACK                                   \
3085          : /*out*/   "=r" (_res)                                  \
3086          : /*in*/    "r" (&_argvec[2])                            \
3087          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3088       );                                                          \
3089       lval = (__typeof__(lval)) _res;                             \
3090    } while (0)
3091
3092 #endif /* PLAT_ppc64_linux */
3093
3094 /* ------------------------- arm-linux ------------------------- */
3095
3096 #if defined(PLAT_arm_linux)
3097
3098 /* These regs are trashed by the hidden call. */
3099 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
3100
3101 /* Macros to save and align the stack before making a function
3102    call and restore it afterwards as gcc may not keep the stack
3103    pointer aligned if it doesn't realise calls are being made
3104    to other functions. */
3105
3106 /* This is a bit tricky.  We store the original stack pointer in r10
3107    as it is callee-saves.  gcc doesn't allow the use of r11 for some
3108    reason.  Also, we can't directly "bic" the stack pointer in thumb
3109    mode since r13 isn't an allowed register number in that context.
3110    So use r4 as a temporary, since that is about to get trashed
3111    anyway, just after each use of this macro.  Side effect is we need
3112    to be very careful about any future changes, since
3113    VALGRIND_ALIGN_STACK simply assumes r4 is usable. */
3114 #define VALGRIND_ALIGN_STACK               \
3115       "mov r10, sp\n\t"                    \
3116       "mov r4,  sp\n\t"                    \
3117       "bic r4,  r4, #7\n\t"                \
3118       "mov sp,  r4\n\t"
3119 #define VALGRIND_RESTORE_STACK             \
3120       "mov sp,  r10\n\t"
3121
3122 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
3123    long) == 4. */
3124
3125 #define CALL_FN_W_v(lval, orig)                                   \
3126    do {                                                           \
3127       volatile OrigFn        _orig = (orig);                      \
3128       volatile unsigned long _argvec[1];                          \
3129       volatile unsigned long _res;                                \
3130       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3131       __asm__ volatile(                                           \
3132          VALGRIND_ALIGN_STACK                                     \
3133          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3134          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3135          VALGRIND_RESTORE_STACK                                   \
3136          "mov %0, r0\n"                                           \
3137          : /*out*/   "=r" (_res)                                  \
3138          : /*in*/    "0" (&_argvec[0])                            \
3139          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3140       );                                                          \
3141       lval = (__typeof__(lval)) _res;                             \
3142    } while (0)
3143
3144 #define CALL_FN_W_W(lval, orig, arg1)                             \
3145    do {                                                           \
3146       volatile OrigFn        _orig = (orig);                      \
3147       volatile unsigned long _argvec[2];                          \
3148       volatile unsigned long _res;                                \
3149       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3150       _argvec[1] = (unsigned long)(arg1);                         \
3151       __asm__ volatile(                                           \
3152          VALGRIND_ALIGN_STACK                                     \
3153          "ldr r0, [%1, #4] \n\t"                                  \
3154          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3155          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3156          VALGRIND_RESTORE_STACK                                   \
3157          "mov %0, r0\n"                                           \
3158          : /*out*/   "=r" (_res)                                  \
3159          : /*in*/    "0" (&_argvec[0])                            \
3160          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3161       );                                                          \
3162       lval = (__typeof__(lval)) _res;                             \
3163    } while (0)
3164
3165 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3166    do {                                                           \
3167       volatile OrigFn        _orig = (orig);                      \
3168       volatile unsigned long _argvec[3];                          \
3169       volatile unsigned long _res;                                \
3170       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3171       _argvec[1] = (unsigned long)(arg1);                         \
3172       _argvec[2] = (unsigned long)(arg2);                         \
3173       __asm__ volatile(                                           \
3174          VALGRIND_ALIGN_STACK                                     \
3175          "ldr r0, [%1, #4] \n\t"                                  \
3176          "ldr r1, [%1, #8] \n\t"                                  \
3177          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3178          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3179          VALGRIND_RESTORE_STACK                                   \
3180          "mov %0, r0\n"                                           \
3181          : /*out*/   "=r" (_res)                                  \
3182          : /*in*/    "0" (&_argvec[0])                            \
3183          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3184       );                                                          \
3185       lval = (__typeof__(lval)) _res;                             \
3186    } while (0)
3187
3188 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3189    do {                                                           \
3190       volatile OrigFn        _orig = (orig);                      \
3191       volatile unsigned long _argvec[4];                          \
3192       volatile unsigned long _res;                                \
3193       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3194       _argvec[1] = (unsigned long)(arg1);                         \
3195       _argvec[2] = (unsigned long)(arg2);                         \
3196       _argvec[3] = (unsigned long)(arg3);                         \
3197       __asm__ volatile(                                           \
3198          VALGRIND_ALIGN_STACK                                     \
3199          "ldr r0, [%1, #4] \n\t"                                  \
3200          "ldr r1, [%1, #8] \n\t"                                  \
3201          "ldr r2, [%1, #12] \n\t"                                 \
3202          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3203          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3204          VALGRIND_RESTORE_STACK                                   \
3205          "mov %0, r0\n"                                           \
3206          : /*out*/   "=r" (_res)                                  \
3207          : /*in*/    "0" (&_argvec[0])                            \
3208          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3209       );                                                          \
3210       lval = (__typeof__(lval)) _res;                             \
3211    } while (0)
3212
3213 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3214    do {                                                           \
3215       volatile OrigFn        _orig = (orig);                      \
3216       volatile unsigned long _argvec[5];                          \
3217       volatile unsigned long _res;                                \
3218       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3219       _argvec[1] = (unsigned long)(arg1);                         \
3220       _argvec[2] = (unsigned long)(arg2);                         \
3221       _argvec[3] = (unsigned long)(arg3);                         \
3222       _argvec[4] = (unsigned long)(arg4);                         \
3223       __asm__ volatile(                                           \
3224          VALGRIND_ALIGN_STACK                                     \
3225          "ldr r0, [%1, #4] \n\t"                                  \
3226          "ldr r1, [%1, #8] \n\t"                                  \
3227          "ldr r2, [%1, #12] \n\t"                                 \
3228          "ldr r3, [%1, #16] \n\t"                                 \
3229          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3230          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3231          VALGRIND_RESTORE_STACK                                   \
3232          "mov %0, r0"                                             \
3233          : /*out*/   "=r" (_res)                                  \
3234          : /*in*/    "0" (&_argvec[0])                            \
3235          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3236       );                                                          \
3237       lval = (__typeof__(lval)) _res;                             \
3238    } while (0)
3239
3240 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3241    do {                                                           \
3242       volatile OrigFn        _orig = (orig);                      \
3243       volatile unsigned long _argvec[6];                          \
3244       volatile unsigned long _res;                                \
3245       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3246       _argvec[1] = (unsigned long)(arg1);                         \
3247       _argvec[2] = (unsigned long)(arg2);                         \
3248       _argvec[3] = (unsigned long)(arg3);                         \
3249       _argvec[4] = (unsigned long)(arg4);                         \
3250       _argvec[5] = (unsigned long)(arg5);                         \
3251       __asm__ volatile(                                           \
3252          VALGRIND_ALIGN_STACK                                     \
3253          "sub sp, sp, #4 \n\t"                                    \
3254          "ldr r0, [%1, #20] \n\t"                                 \
3255          "push {r0} \n\t"                                         \
3256          "ldr r0, [%1, #4] \n\t"                                  \
3257          "ldr r1, [%1, #8] \n\t"                                  \
3258          "ldr r2, [%1, #12] \n\t"                                 \
3259          "ldr r3, [%1, #16] \n\t"                                 \
3260          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3261          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3262          VALGRIND_RESTORE_STACK                                   \
3263          "mov %0, r0"                                             \
3264          : /*out*/   "=r" (_res)                                  \
3265          : /*in*/    "0" (&_argvec[0])                            \
3266          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3267       );                                                          \
3268       lval = (__typeof__(lval)) _res;                             \
3269    } while (0)
3270
3271 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3272    do {                                                           \
3273       volatile OrigFn        _orig = (orig);                      \
3274       volatile unsigned long _argvec[7];                          \
3275       volatile unsigned long _res;                                \
3276       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3277       _argvec[1] = (unsigned long)(arg1);                         \
3278       _argvec[2] = (unsigned long)(arg2);                         \
3279       _argvec[3] = (unsigned long)(arg3);                         \
3280       _argvec[4] = (unsigned long)(arg4);                         \
3281       _argvec[5] = (unsigned long)(arg5);                         \
3282       _argvec[6] = (unsigned long)(arg6);                         \
3283       __asm__ volatile(                                           \
3284          VALGRIND_ALIGN_STACK                                     \
3285          "ldr r0, [%1, #20] \n\t"                                 \
3286          "ldr r1, [%1, #24] \n\t"                                 \
3287          "push {r0, r1} \n\t"                                     \
3288          "ldr r0, [%1, #4] \n\t"                                  \
3289          "ldr r1, [%1, #8] \n\t"                                  \
3290          "ldr r2, [%1, #12] \n\t"                                 \
3291          "ldr r3, [%1, #16] \n\t"                                 \
3292          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3293          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3294          VALGRIND_RESTORE_STACK                                   \
3295          "mov %0, r0"                                             \
3296          : /*out*/   "=r" (_res)                                  \
3297          : /*in*/    "0" (&_argvec[0])                            \
3298          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3299       );                                                          \
3300       lval = (__typeof__(lval)) _res;                             \
3301    } while (0)
3302
3303 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3304                                  arg7)                            \
3305    do {                                                           \
3306       volatile OrigFn        _orig = (orig);                      \
3307       volatile unsigned long _argvec[8];                          \
3308       volatile unsigned long _res;                                \
3309       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3310       _argvec[1] = (unsigned long)(arg1);                         \
3311       _argvec[2] = (unsigned long)(arg2);                         \
3312       _argvec[3] = (unsigned long)(arg3);                         \
3313       _argvec[4] = (unsigned long)(arg4);                         \
3314       _argvec[5] = (unsigned long)(arg5);                         \
3315       _argvec[6] = (unsigned long)(arg6);                         \
3316       _argvec[7] = (unsigned long)(arg7);                         \
3317       __asm__ volatile(                                           \
3318          VALGRIND_ALIGN_STACK                                     \
3319          "sub sp, sp, #4 \n\t"                                    \
3320          "ldr r0, [%1, #20] \n\t"                                 \
3321          "ldr r1, [%1, #24] \n\t"                                 \
3322          "ldr r2, [%1, #28] \n\t"                                 \
3323          "push {r0, r1, r2} \n\t"                                 \
3324          "ldr r0, [%1, #4] \n\t"                                  \
3325          "ldr r1, [%1, #8] \n\t"                                  \
3326          "ldr r2, [%1, #12] \n\t"                                 \
3327          "ldr r3, [%1, #16] \n\t"                                 \
3328          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3329          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3330          VALGRIND_RESTORE_STACK                                   \
3331          "mov %0, r0"                                             \
3332          : /*out*/   "=r" (_res)                                  \
3333          : /*in*/    "0" (&_argvec[0])                            \
3334          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3335       );                                                          \
3336       lval = (__typeof__(lval)) _res;                             \
3337    } while (0)
3338
3339 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3340                                  arg7,arg8)                       \
3341    do {                                                           \
3342       volatile OrigFn        _orig = (orig);                      \
3343       volatile unsigned long _argvec[9];                          \
3344       volatile unsigned long _res;                                \
3345       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3346       _argvec[1] = (unsigned long)(arg1);                         \
3347       _argvec[2] = (unsigned long)(arg2);                         \
3348       _argvec[3] = (unsigned long)(arg3);                         \
3349       _argvec[4] = (unsigned long)(arg4);                         \
3350       _argvec[5] = (unsigned long)(arg5);                         \
3351       _argvec[6] = (unsigned long)(arg6);                         \
3352       _argvec[7] = (unsigned long)(arg7);                         \
3353       _argvec[8] = (unsigned long)(arg8);                         \
3354       __asm__ volatile(                                           \
3355          VALGRIND_ALIGN_STACK                                     \
3356          "ldr r0, [%1, #20] \n\t"                                 \
3357          "ldr r1, [%1, #24] \n\t"                                 \
3358          "ldr r2, [%1, #28] \n\t"                                 \
3359          "ldr r3, [%1, #32] \n\t"                                 \
3360          "push {r0, r1, r2, r3} \n\t"                             \
3361          "ldr r0, [%1, #4] \n\t"                                  \
3362          "ldr r1, [%1, #8] \n\t"                                  \
3363          "ldr r2, [%1, #12] \n\t"                                 \
3364          "ldr r3, [%1, #16] \n\t"                                 \
3365          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3366          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3367          VALGRIND_RESTORE_STACK                                   \
3368          "mov %0, r0"                                             \
3369          : /*out*/   "=r" (_res)                                  \
3370          : /*in*/    "0" (&_argvec[0])                            \
3371          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3372       );                                                          \
3373       lval = (__typeof__(lval)) _res;                             \
3374    } while (0)
3375
3376 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3377                                  arg7,arg8,arg9)                  \
3378    do {                                                           \
3379       volatile OrigFn        _orig = (orig);                      \
3380       volatile unsigned long _argvec[10];                         \
3381       volatile unsigned long _res;                                \
3382       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3383       _argvec[1] = (unsigned long)(arg1);                         \
3384       _argvec[2] = (unsigned long)(arg2);                         \
3385       _argvec[3] = (unsigned long)(arg3);                         \
3386       _argvec[4] = (unsigned long)(arg4);                         \
3387       _argvec[5] = (unsigned long)(arg5);                         \
3388       _argvec[6] = (unsigned long)(arg6);                         \
3389       _argvec[7] = (unsigned long)(arg7);                         \
3390       _argvec[8] = (unsigned long)(arg8);                         \
3391       _argvec[9] = (unsigned long)(arg9);                         \
3392       __asm__ volatile(                                           \
3393          VALGRIND_ALIGN_STACK                                     \
3394          "sub sp, sp, #4 \n\t"                                    \
3395          "ldr r0, [%1, #20] \n\t"                                 \
3396          "ldr r1, [%1, #24] \n\t"                                 \
3397          "ldr r2, [%1, #28] \n\t"                                 \
3398          "ldr r3, [%1, #32] \n\t"                                 \
3399          "ldr r4, [%1, #36] \n\t"                                 \
3400          "push {r0, r1, r2, r3, r4} \n\t"                         \
3401          "ldr r0, [%1, #4] \n\t"                                  \
3402          "ldr r1, [%1, #8] \n\t"                                  \
3403          "ldr r2, [%1, #12] \n\t"                                 \
3404          "ldr r3, [%1, #16] \n\t"                                 \
3405          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3406          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3407          VALGRIND_RESTORE_STACK                                   \
3408          "mov %0, r0"                                             \
3409          : /*out*/   "=r" (_res)                                  \
3410          : /*in*/    "0" (&_argvec[0])                            \
3411          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3412       );                                                          \
3413       lval = (__typeof__(lval)) _res;                             \
3414    } while (0)
3415
3416 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3417                                   arg7,arg8,arg9,arg10)           \
3418    do {                                                           \
3419       volatile OrigFn        _orig = (orig);                      \
3420       volatile unsigned long _argvec[11];                         \
3421       volatile unsigned long _res;                                \
3422       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3423       _argvec[1] = (unsigned long)(arg1);                         \
3424       _argvec[2] = (unsigned long)(arg2);                         \
3425       _argvec[3] = (unsigned long)(arg3);                         \
3426       _argvec[4] = (unsigned long)(arg4);                         \
3427       _argvec[5] = (unsigned long)(arg5);                         \
3428       _argvec[6] = (unsigned long)(arg6);                         \
3429       _argvec[7] = (unsigned long)(arg7);                         \
3430       _argvec[8] = (unsigned long)(arg8);                         \
3431       _argvec[9] = (unsigned long)(arg9);                         \
3432       _argvec[10] = (unsigned long)(arg10);                       \
3433       __asm__ volatile(                                           \
3434          VALGRIND_ALIGN_STACK                                     \
3435          "ldr r0, [%1, #40] \n\t"                                 \
3436          "push {r0} \n\t"                                         \
3437          "ldr r0, [%1, #20] \n\t"                                 \
3438          "ldr r1, [%1, #24] \n\t"                                 \
3439          "ldr r2, [%1, #28] \n\t"                                 \
3440          "ldr r3, [%1, #32] \n\t"                                 \
3441          "ldr r4, [%1, #36] \n\t"                                 \
3442          "push {r0, r1, r2, r3, r4} \n\t"                         \
3443          "ldr r0, [%1, #4] \n\t"                                  \
3444          "ldr r1, [%1, #8] \n\t"                                  \
3445          "ldr r2, [%1, #12] \n\t"                                 \
3446          "ldr r3, [%1, #16] \n\t"                                 \
3447          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3448          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3449          VALGRIND_RESTORE_STACK                                   \
3450          "mov %0, r0"                                             \
3451          : /*out*/   "=r" (_res)                                  \
3452          : /*in*/    "0" (&_argvec[0])                            \
3453          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3454       );                                                          \
3455       lval = (__typeof__(lval)) _res;                             \
3456    } while (0)
3457
3458 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3459                                   arg6,arg7,arg8,arg9,arg10,      \
3460                                   arg11)                          \
3461    do {                                                           \
3462       volatile OrigFn        _orig = (orig);                      \
3463       volatile unsigned long _argvec[12];                         \
3464       volatile unsigned long _res;                                \
3465       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3466       _argvec[1] = (unsigned long)(arg1);                         \
3467       _argvec[2] = (unsigned long)(arg2);                         \
3468       _argvec[3] = (unsigned long)(arg3);                         \
3469       _argvec[4] = (unsigned long)(arg4);                         \
3470       _argvec[5] = (unsigned long)(arg5);                         \
3471       _argvec[6] = (unsigned long)(arg6);                         \
3472       _argvec[7] = (unsigned long)(arg7);                         \
3473       _argvec[8] = (unsigned long)(arg8);                         \
3474       _argvec[9] = (unsigned long)(arg9);                         \
3475       _argvec[10] = (unsigned long)(arg10);                       \
3476       _argvec[11] = (unsigned long)(arg11);                       \
3477       __asm__ volatile(                                           \
3478          VALGRIND_ALIGN_STACK                                     \
3479          "sub sp, sp, #4 \n\t"                                    \
3480          "ldr r0, [%1, #40] \n\t"                                 \
3481          "ldr r1, [%1, #44] \n\t"                                 \
3482          "push {r0, r1} \n\t"                                     \
3483          "ldr r0, [%1, #20] \n\t"                                 \
3484          "ldr r1, [%1, #24] \n\t"                                 \
3485          "ldr r2, [%1, #28] \n\t"                                 \
3486          "ldr r3, [%1, #32] \n\t"                                 \
3487          "ldr r4, [%1, #36] \n\t"                                 \
3488          "push {r0, r1, r2, r3, r4} \n\t"                         \
3489          "ldr r0, [%1, #4] \n\t"                                  \
3490          "ldr r1, [%1, #8] \n\t"                                  \
3491          "ldr r2, [%1, #12] \n\t"                                 \
3492          "ldr r3, [%1, #16] \n\t"                                 \
3493          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3494          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3495          VALGRIND_RESTORE_STACK                                   \
3496          "mov %0, r0"                                             \
3497          : /*out*/   "=r" (_res)                                  \
3498          : /*in*/    "0" (&_argvec[0])                            \
3499          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3500       );                                                          \
3501       lval = (__typeof__(lval)) _res;                             \
3502    } while (0)
3503
3504 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3505                                   arg6,arg7,arg8,arg9,arg10,      \
3506                                   arg11,arg12)                    \
3507    do {                                                           \
3508       volatile OrigFn        _orig = (orig);                      \
3509       volatile unsigned long _argvec[13];                         \
3510       volatile unsigned long _res;                                \
3511       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3512       _argvec[1] = (unsigned long)(arg1);                         \
3513       _argvec[2] = (unsigned long)(arg2);                         \
3514       _argvec[3] = (unsigned long)(arg3);                         \
3515       _argvec[4] = (unsigned long)(arg4);                         \
3516       _argvec[5] = (unsigned long)(arg5);                         \
3517       _argvec[6] = (unsigned long)(arg6);                         \
3518       _argvec[7] = (unsigned long)(arg7);                         \
3519       _argvec[8] = (unsigned long)(arg8);                         \
3520       _argvec[9] = (unsigned long)(arg9);                         \
3521       _argvec[10] = (unsigned long)(arg10);                       \
3522       _argvec[11] = (unsigned long)(arg11);                       \
3523       _argvec[12] = (unsigned long)(arg12);                       \
3524       __asm__ volatile(                                           \
3525          VALGRIND_ALIGN_STACK                                     \
3526          "ldr r0, [%1, #40] \n\t"                                 \
3527          "ldr r1, [%1, #44] \n\t"                                 \
3528          "ldr r2, [%1, #48] \n\t"                                 \
3529          "push {r0, r1, r2} \n\t"                                 \
3530          "ldr r0, [%1, #20] \n\t"                                 \
3531          "ldr r1, [%1, #24] \n\t"                                 \
3532          "ldr r2, [%1, #28] \n\t"                                 \
3533          "ldr r3, [%1, #32] \n\t"                                 \
3534          "ldr r4, [%1, #36] \n\t"                                 \
3535          "push {r0, r1, r2, r3, r4} \n\t"                         \
3536          "ldr r0, [%1, #4] \n\t"                                  \
3537          "ldr r1, [%1, #8] \n\t"                                  \
3538          "ldr r2, [%1, #12] \n\t"                                 \
3539          "ldr r3, [%1, #16] \n\t"                                 \
3540          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3541          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3542          VALGRIND_RESTORE_STACK                                   \
3543          "mov %0, r0"                                             \
3544          : /*out*/   "=r" (_res)                                  \
3545          : /*in*/    "0" (&_argvec[0])                            \
3546          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3547       );                                                          \
3548       lval = (__typeof__(lval)) _res;                             \
3549    } while (0)
3550
3551 #endif /* PLAT_arm_linux */
3552
3553 /* ------------------------ arm64-linux ------------------------ */
3554
3555 #if defined(PLAT_arm64_linux)
3556
3557 /* These regs are trashed by the hidden call. */
3558 #define __CALLER_SAVED_REGS \
3559      "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9",   \
3560      "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17",      \
3561      "x18", "x19", "x20", "x30",                                  \
3562      "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",  \
3563      "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",      \
3564      "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",      \
3565      "v26", "v27", "v28", "v29", "v30", "v31"
3566
3567 /* x21 is callee-saved, so we can use it to save and restore SP around
3568    the hidden call. */
3569 #define VALGRIND_ALIGN_STACK               \
3570       "mov x21, sp\n\t"                    \
3571       "bic sp, x21, #15\n\t"
3572 #define VALGRIND_RESTORE_STACK             \
3573       "mov sp,  x21\n\t"
3574
3575 /* These CALL_FN_ macros assume that on arm64-linux,
3576    sizeof(unsigned long) == 8. */
3577
3578 #define CALL_FN_W_v(lval, orig)                                   \
3579    do {                                                           \
3580       volatile OrigFn        _orig = (orig);                      \
3581       volatile unsigned long _argvec[1];                          \
3582       volatile unsigned long _res;                                \
3583       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3584       __asm__ volatile(                                           \
3585          VALGRIND_ALIGN_STACK                                     \
3586          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3587          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3588          VALGRIND_RESTORE_STACK                                   \
3589          "mov %0, x0\n"                                           \
3590          : /*out*/   "=r" (_res)                                  \
3591          : /*in*/    "0" (&_argvec[0])                            \
3592          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3593       );                                                          \
3594       lval = (__typeof__(lval)) _res;                             \
3595    } while (0)
3596
3597 #define CALL_FN_W_W(lval, orig, arg1)                             \
3598    do {                                                           \
3599       volatile OrigFn        _orig = (orig);                      \
3600       volatile unsigned long _argvec[2];                          \
3601       volatile unsigned long _res;                                \
3602       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3603       _argvec[1] = (unsigned long)(arg1);                         \
3604       __asm__ volatile(                                           \
3605          VALGRIND_ALIGN_STACK                                     \
3606          "ldr x0, [%1, #8] \n\t"                                  \
3607          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3608          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3609          VALGRIND_RESTORE_STACK                                   \
3610          "mov %0, x0\n"                                           \
3611          : /*out*/   "=r" (_res)                                  \
3612          : /*in*/    "0" (&_argvec[0])                            \
3613          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3614       );                                                          \
3615       lval = (__typeof__(lval)) _res;                             \
3616    } while (0)
3617
3618 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3619    do {                                                           \
3620       volatile OrigFn        _orig = (orig);                      \
3621       volatile unsigned long _argvec[3];                          \
3622       volatile unsigned long _res;                                \
3623       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3624       _argvec[1] = (unsigned long)(arg1);                         \
3625       _argvec[2] = (unsigned long)(arg2);                         \
3626       __asm__ volatile(                                           \
3627          VALGRIND_ALIGN_STACK                                     \
3628          "ldr x0, [%1, #8] \n\t"                                  \
3629          "ldr x1, [%1, #16] \n\t"                                 \
3630          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3631          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3632          VALGRIND_RESTORE_STACK                                   \
3633          "mov %0, x0\n"                                           \
3634          : /*out*/   "=r" (_res)                                  \
3635          : /*in*/    "0" (&_argvec[0])                            \
3636          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3637       );                                                          \
3638       lval = (__typeof__(lval)) _res;                             \
3639    } while (0)
3640
3641 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3642    do {                                                           \
3643       volatile OrigFn        _orig = (orig);                      \
3644       volatile unsigned long _argvec[4];                          \
3645       volatile unsigned long _res;                                \
3646       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3647       _argvec[1] = (unsigned long)(arg1);                         \
3648       _argvec[2] = (unsigned long)(arg2);                         \
3649       _argvec[3] = (unsigned long)(arg3);                         \
3650       __asm__ volatile(                                           \
3651          VALGRIND_ALIGN_STACK                                     \
3652          "ldr x0, [%1, #8] \n\t"                                  \
3653          "ldr x1, [%1, #16] \n\t"                                 \
3654          "ldr x2, [%1, #24] \n\t"                                 \
3655          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3656          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3657          VALGRIND_RESTORE_STACK                                   \
3658          "mov %0, x0\n"                                           \
3659          : /*out*/   "=r" (_res)                                  \
3660          : /*in*/    "0" (&_argvec[0])                            \
3661          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3662       );                                                          \
3663       lval = (__typeof__(lval)) _res;                             \
3664    } while (0)
3665
3666 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3667    do {                                                           \
3668       volatile OrigFn        _orig = (orig);                      \
3669       volatile unsigned long _argvec[5];                          \
3670       volatile unsigned long _res;                                \
3671       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3672       _argvec[1] = (unsigned long)(arg1);                         \
3673       _argvec[2] = (unsigned long)(arg2);                         \
3674       _argvec[3] = (unsigned long)(arg3);                         \
3675       _argvec[4] = (unsigned long)(arg4);                         \
3676       __asm__ volatile(                                           \
3677          VALGRIND_ALIGN_STACK                                     \
3678          "ldr x0, [%1, #8] \n\t"                                  \
3679          "ldr x1, [%1, #16] \n\t"                                 \
3680          "ldr x2, [%1, #24] \n\t"                                 \
3681          "ldr x3, [%1, #32] \n\t"                                 \
3682          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3683          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3684          VALGRIND_RESTORE_STACK                                   \
3685          "mov %0, x0"                                             \
3686          : /*out*/   "=r" (_res)                                  \
3687          : /*in*/    "0" (&_argvec[0])                            \
3688          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3689       );                                                          \
3690       lval = (__typeof__(lval)) _res;                             \
3691    } while (0)
3692
3693 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3694    do {                                                           \
3695       volatile OrigFn        _orig = (orig);                      \
3696       volatile unsigned long _argvec[6];                          \
3697       volatile unsigned long _res;                                \
3698       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3699       _argvec[1] = (unsigned long)(arg1);                         \
3700       _argvec[2] = (unsigned long)(arg2);                         \
3701       _argvec[3] = (unsigned long)(arg3);                         \
3702       _argvec[4] = (unsigned long)(arg4);                         \
3703       _argvec[5] = (unsigned long)(arg5);                         \
3704       __asm__ volatile(                                           \
3705          VALGRIND_ALIGN_STACK                                     \
3706          "ldr x0, [%1, #8] \n\t"                                  \
3707          "ldr x1, [%1, #16] \n\t"                                 \
3708          "ldr x2, [%1, #24] \n\t"                                 \
3709          "ldr x3, [%1, #32] \n\t"                                 \
3710          "ldr x4, [%1, #40] \n\t"                                 \
3711          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3712          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3713          VALGRIND_RESTORE_STACK                                   \
3714          "mov %0, x0"                                             \
3715          : /*out*/   "=r" (_res)                                  \
3716          : /*in*/    "0" (&_argvec[0])                            \
3717          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3718       );                                                          \
3719       lval = (__typeof__(lval)) _res;                             \
3720    } while (0)
3721
3722 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3723    do {                                                           \
3724       volatile OrigFn        _orig = (orig);                      \
3725       volatile unsigned long _argvec[7];                          \
3726       volatile unsigned long _res;                                \
3727       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3728       _argvec[1] = (unsigned long)(arg1);                         \
3729       _argvec[2] = (unsigned long)(arg2);                         \
3730       _argvec[3] = (unsigned long)(arg3);                         \
3731       _argvec[4] = (unsigned long)(arg4);                         \
3732       _argvec[5] = (unsigned long)(arg5);                         \
3733       _argvec[6] = (unsigned long)(arg6);                         \
3734       __asm__ volatile(                                           \
3735          VALGRIND_ALIGN_STACK                                     \
3736          "ldr x0, [%1, #8] \n\t"                                  \
3737          "ldr x1, [%1, #16] \n\t"                                 \
3738          "ldr x2, [%1, #24] \n\t"                                 \
3739          "ldr x3, [%1, #32] \n\t"                                 \
3740          "ldr x4, [%1, #40] \n\t"                                 \
3741          "ldr x5, [%1, #48] \n\t"                                 \
3742          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3743          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3744          VALGRIND_RESTORE_STACK                                   \
3745          "mov %0, x0"                                             \
3746          : /*out*/   "=r" (_res)                                  \
3747          : /*in*/    "0" (&_argvec[0])                            \
3748          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3749       );                                                          \
3750       lval = (__typeof__(lval)) _res;                             \
3751    } while (0)
3752
3753 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3754                                  arg7)                            \
3755    do {                                                           \
3756       volatile OrigFn        _orig = (orig);                      \
3757       volatile unsigned long _argvec[8];                          \
3758       volatile unsigned long _res;                                \
3759       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3760       _argvec[1] = (unsigned long)(arg1);                         \
3761       _argvec[2] = (unsigned long)(arg2);                         \
3762       _argvec[3] = (unsigned long)(arg3);                         \
3763       _argvec[4] = (unsigned long)(arg4);                         \
3764       _argvec[5] = (unsigned long)(arg5);                         \
3765       _argvec[6] = (unsigned long)(arg6);                         \
3766       _argvec[7] = (unsigned long)(arg7);                         \
3767       __asm__ volatile(                                           \
3768          VALGRIND_ALIGN_STACK                                     \
3769          "ldr x0, [%1, #8] \n\t"                                  \
3770          "ldr x1, [%1, #16] \n\t"                                 \
3771          "ldr x2, [%1, #24] \n\t"                                 \
3772          "ldr x3, [%1, #32] \n\t"                                 \
3773          "ldr x4, [%1, #40] \n\t"                                 \
3774          "ldr x5, [%1, #48] \n\t"                                 \
3775          "ldr x6, [%1, #56] \n\t"                                 \
3776          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3777          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3778          VALGRIND_RESTORE_STACK                                   \
3779          "mov %0, x0"                                             \
3780          : /*out*/   "=r" (_res)                                  \
3781          : /*in*/    "0" (&_argvec[0])                            \
3782          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3783       );                                                          \
3784       lval = (__typeof__(lval)) _res;                             \
3785    } while (0)
3786
3787 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3788                                  arg7,arg8)                       \
3789    do {                                                           \
3790       volatile OrigFn        _orig = (orig);                      \
3791       volatile unsigned long _argvec[9];                          \
3792       volatile unsigned long _res;                                \
3793       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3794       _argvec[1] = (unsigned long)(arg1);                         \
3795       _argvec[2] = (unsigned long)(arg2);                         \
3796       _argvec[3] = (unsigned long)(arg3);                         \
3797       _argvec[4] = (unsigned long)(arg4);                         \
3798       _argvec[5] = (unsigned long)(arg5);                         \
3799       _argvec[6] = (unsigned long)(arg6);                         \
3800       _argvec[7] = (unsigned long)(arg7);                         \
3801       _argvec[8] = (unsigned long)(arg8);                         \
3802       __asm__ volatile(                                           \
3803          VALGRIND_ALIGN_STACK                                     \
3804          "ldr x0, [%1, #8] \n\t"                                  \
3805          "ldr x1, [%1, #16] \n\t"                                 \
3806          "ldr x2, [%1, #24] \n\t"                                 \
3807          "ldr x3, [%1, #32] \n\t"                                 \
3808          "ldr x4, [%1, #40] \n\t"                                 \
3809          "ldr x5, [%1, #48] \n\t"                                 \
3810          "ldr x6, [%1, #56] \n\t"                                 \
3811          "ldr x7, [%1, #64] \n\t"                                 \
3812          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3813          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3814          VALGRIND_RESTORE_STACK                                   \
3815          "mov %0, x0"                                             \
3816          : /*out*/   "=r" (_res)                                  \
3817          : /*in*/    "0" (&_argvec[0])                            \
3818          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3819       );                                                          \
3820       lval = (__typeof__(lval)) _res;                             \
3821    } while (0)
3822
3823 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3824                                  arg7,arg8,arg9)                  \
3825    do {                                                           \
3826       volatile OrigFn        _orig = (orig);                      \
3827       volatile unsigned long _argvec[10];                         \
3828       volatile unsigned long _res;                                \
3829       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3830       _argvec[1] = (unsigned long)(arg1);                         \
3831       _argvec[2] = (unsigned long)(arg2);                         \
3832       _argvec[3] = (unsigned long)(arg3);                         \
3833       _argvec[4] = (unsigned long)(arg4);                         \
3834       _argvec[5] = (unsigned long)(arg5);                         \
3835       _argvec[6] = (unsigned long)(arg6);                         \
3836       _argvec[7] = (unsigned long)(arg7);                         \
3837       _argvec[8] = (unsigned long)(arg8);                         \
3838       _argvec[9] = (unsigned long)(arg9);                         \
3839       __asm__ volatile(                                           \
3840          VALGRIND_ALIGN_STACK                                     \
3841          "sub sp, sp, #0x20 \n\t"                                 \
3842          "ldr x0, [%1, #8] \n\t"                                  \
3843          "ldr x1, [%1, #16] \n\t"                                 \
3844          "ldr x2, [%1, #24] \n\t"                                 \
3845          "ldr x3, [%1, #32] \n\t"                                 \
3846          "ldr x4, [%1, #40] \n\t"                                 \
3847          "ldr x5, [%1, #48] \n\t"                                 \
3848          "ldr x6, [%1, #56] \n\t"                                 \
3849          "ldr x7, [%1, #64] \n\t"                                 \
3850          "ldr x8, [%1, #72] \n\t"                                 \
3851          "str x8, [sp, #0]  \n\t"                                 \
3852          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3853          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3854          VALGRIND_RESTORE_STACK                                   \
3855          "mov %0, x0"                                             \
3856          : /*out*/   "=r" (_res)                                  \
3857          : /*in*/    "0" (&_argvec[0])                            \
3858          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3859       );                                                          \
3860       lval = (__typeof__(lval)) _res;                             \
3861    } while (0)
3862
3863 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3864                                   arg7,arg8,arg9,arg10)           \
3865    do {                                                           \
3866       volatile OrigFn        _orig = (orig);                      \
3867       volatile unsigned long _argvec[11];                         \
3868       volatile unsigned long _res;                                \
3869       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3870       _argvec[1] = (unsigned long)(arg1);                         \
3871       _argvec[2] = (unsigned long)(arg2);                         \
3872       _argvec[3] = (unsigned long)(arg3);                         \
3873       _argvec[4] = (unsigned long)(arg4);                         \
3874       _argvec[5] = (unsigned long)(arg5);                         \
3875       _argvec[6] = (unsigned long)(arg6);                         \
3876       _argvec[7] = (unsigned long)(arg7);                         \
3877       _argvec[8] = (unsigned long)(arg8);                         \
3878       _argvec[9] = (unsigned long)(arg9);                         \
3879       _argvec[10] = (unsigned long)(arg10);                       \
3880       __asm__ volatile(                                           \
3881          VALGRIND_ALIGN_STACK                                     \
3882          "sub sp, sp, #0x20 \n\t"                                 \
3883          "ldr x0, [%1, #8] \n\t"                                  \
3884          "ldr x1, [%1, #16] \n\t"                                 \
3885          "ldr x2, [%1, #24] \n\t"                                 \
3886          "ldr x3, [%1, #32] \n\t"                                 \
3887          "ldr x4, [%1, #40] \n\t"                                 \
3888          "ldr x5, [%1, #48] \n\t"                                 \
3889          "ldr x6, [%1, #56] \n\t"                                 \
3890          "ldr x7, [%1, #64] \n\t"                                 \
3891          "ldr x8, [%1, #72] \n\t"                                 \
3892          "str x8, [sp, #0]  \n\t"                                 \
3893          "ldr x8, [%1, #80] \n\t"                                 \
3894          "str x8, [sp, #8]  \n\t"                                 \
3895          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3896          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3897          VALGRIND_RESTORE_STACK                                   \
3898          "mov %0, x0"                                             \
3899          : /*out*/   "=r" (_res)                                  \
3900          : /*in*/    "0" (&_argvec[0])                            \
3901          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3902       );                                                          \
3903       lval = (__typeof__(lval)) _res;                             \
3904    } while (0)
3905
3906 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3907                                   arg7,arg8,arg9,arg10,arg11)     \
3908    do {                                                           \
3909       volatile OrigFn        _orig = (orig);                      \
3910       volatile unsigned long _argvec[12];                         \
3911       volatile unsigned long _res;                                \
3912       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3913       _argvec[1] = (unsigned long)(arg1);                         \
3914       _argvec[2] = (unsigned long)(arg2);                         \
3915       _argvec[3] = (unsigned long)(arg3);                         \
3916       _argvec[4] = (unsigned long)(arg4);                         \
3917       _argvec[5] = (unsigned long)(arg5);                         \
3918       _argvec[6] = (unsigned long)(arg6);                         \
3919       _argvec[7] = (unsigned long)(arg7);                         \
3920       _argvec[8] = (unsigned long)(arg8);                         \
3921       _argvec[9] = (unsigned long)(arg9);                         \
3922       _argvec[10] = (unsigned long)(arg10);                       \
3923       _argvec[11] = (unsigned long)(arg11);                       \
3924       __asm__ volatile(                                           \
3925          VALGRIND_ALIGN_STACK                                     \
3926          "sub sp, sp, #0x30 \n\t"                                 \
3927          "ldr x0, [%1, #8] \n\t"                                  \
3928          "ldr x1, [%1, #16] \n\t"                                 \
3929          "ldr x2, [%1, #24] \n\t"                                 \
3930          "ldr x3, [%1, #32] \n\t"                                 \
3931          "ldr x4, [%1, #40] \n\t"                                 \
3932          "ldr x5, [%1, #48] \n\t"                                 \
3933          "ldr x6, [%1, #56] \n\t"                                 \
3934          "ldr x7, [%1, #64] \n\t"                                 \
3935          "ldr x8, [%1, #72] \n\t"                                 \
3936          "str x8, [sp, #0]  \n\t"                                 \
3937          "ldr x8, [%1, #80] \n\t"                                 \
3938          "str x8, [sp, #8]  \n\t"                                 \
3939          "ldr x8, [%1, #88] \n\t"                                 \
3940          "str x8, [sp, #16] \n\t"                                 \
3941          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3942          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3943          VALGRIND_RESTORE_STACK                                   \
3944          "mov %0, x0"                                             \
3945          : /*out*/   "=r" (_res)                                  \
3946          : /*in*/    "0" (&_argvec[0])                            \
3947          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3948       );                                                          \
3949       lval = (__typeof__(lval)) _res;                             \
3950    } while (0)
3951
3952 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3953                                   arg7,arg8,arg9,arg10,arg11,     \
3954                                   arg12)                          \
3955    do {                                                           \
3956       volatile OrigFn        _orig = (orig);                      \
3957       volatile unsigned long _argvec[13];                         \
3958       volatile unsigned long _res;                                \
3959       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3960       _argvec[1] = (unsigned long)(arg1);                         \
3961       _argvec[2] = (unsigned long)(arg2);                         \
3962       _argvec[3] = (unsigned long)(arg3);                         \
3963       _argvec[4] = (unsigned long)(arg4);                         \
3964       _argvec[5] = (unsigned long)(arg5);                         \
3965       _argvec[6] = (unsigned long)(arg6);                         \
3966       _argvec[7] = (unsigned long)(arg7);                         \
3967       _argvec[8] = (unsigned long)(arg8);                         \
3968       _argvec[9] = (unsigned long)(arg9);                         \
3969       _argvec[10] = (unsigned long)(arg10);                       \
3970       _argvec[11] = (unsigned long)(arg11);                       \
3971       _argvec[12] = (unsigned long)(arg12);                       \
3972       __asm__ volatile(                                           \
3973          VALGRIND_ALIGN_STACK                                     \
3974          "sub sp, sp, #0x30 \n\t"                                 \
3975          "ldr x0, [%1, #8] \n\t"                                  \
3976          "ldr x1, [%1, #16] \n\t"                                 \
3977          "ldr x2, [%1, #24] \n\t"                                 \
3978          "ldr x3, [%1, #32] \n\t"                                 \
3979          "ldr x4, [%1, #40] \n\t"                                 \
3980          "ldr x5, [%1, #48] \n\t"                                 \
3981          "ldr x6, [%1, #56] \n\t"                                 \
3982          "ldr x7, [%1, #64] \n\t"                                 \
3983          "ldr x8, [%1, #72] \n\t"                                 \
3984          "str x8, [sp, #0]  \n\t"                                 \
3985          "ldr x8, [%1, #80] \n\t"                                 \
3986          "str x8, [sp, #8]  \n\t"                                 \
3987          "ldr x8, [%1, #88] \n\t"                                 \
3988          "str x8, [sp, #16] \n\t"                                 \
3989          "ldr x8, [%1, #96] \n\t"                                 \
3990          "str x8, [sp, #24] \n\t"                                 \
3991          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
3992          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
3993          VALGRIND_RESTORE_STACK                                   \
3994          "mov %0, x0"                                             \
3995          : /*out*/   "=r" (_res)                                  \
3996          : /*in*/    "0" (&_argvec[0])                            \
3997          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
3998       );                                                          \
3999       lval = (__typeof__(lval)) _res;                             \
4000    } while (0)
4001
4002 #endif /* PLAT_arm64_linux */
4003
4004 /* ------------------------- s390x-linux ------------------------- */
4005
4006 #if defined(PLAT_s390x_linux)
4007
4008 /* Similar workaround as amd64 (see above), but we use r11 as frame
4009    pointer and save the old r11 in r7. r11 might be used for
4010    argvec, therefore we copy argvec in r1 since r1 is clobbered
4011    after the call anyway.  */
4012 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4013 #  define __FRAME_POINTER                                         \
4014       ,"d"(__builtin_dwarf_cfa())
4015 #  define VALGRIND_CFI_PROLOGUE                                   \
4016       ".cfi_remember_state\n\t"                                   \
4017       "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
4018       "lgr 7,11\n\t"                                              \
4019       "lgr 11,%2\n\t"                                             \
4020       ".cfi_def_cfa r11, 0\n\t"
4021 #  define VALGRIND_CFI_EPILOGUE                                   \
4022       "lgr 11, 7\n\t"                                             \
4023       ".cfi_restore_state\n\t"
4024 #else
4025 #  define __FRAME_POINTER
4026 #  define VALGRIND_CFI_PROLOGUE                                   \
4027       "lgr 1,%1\n\t"
4028 #  define VALGRIND_CFI_EPILOGUE
4029 #endif
4030
4031 /* Nb: On s390 the stack pointer is properly aligned *at all times*
4032    according to the s390 GCC maintainer. (The ABI specification is not
4033    precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and
4034    VALGRIND_RESTORE_STACK are not defined here. */
4035
4036 /* These regs are trashed by the hidden call. Note that we overwrite
4037    r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4038    function a proper return address. All others are ABI defined call
4039    clobbers. */
4040 #define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
4041                            "f0","f1","f2","f3","f4","f5","f6","f7"
4042
4043 /* Nb: Although r11 is modified in the asm snippets below (inside
4044    VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for
4045    two reasons:
4046    (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not
4047        modified
4048    (2) GCC will complain that r11 cannot appear inside a clobber section,
4049        when compiled with -O -fno-omit-frame-pointer
4050  */
4051
4052 #define CALL_FN_W_v(lval, orig)                                  \
4053    do {                                                          \
4054       volatile OrigFn        _orig = (orig);                     \
4055       volatile unsigned long  _argvec[1];                        \
4056       volatile unsigned long _res;                               \
4057       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4058       __asm__ volatile(                                          \
4059          VALGRIND_CFI_PROLOGUE                                   \
4060          "aghi 15,-160\n\t"                                      \
4061          "lg 1, 0(1)\n\t"  /* target->r1 */                      \
4062          VALGRIND_CALL_NOREDIR_R1                                \
4063          "lgr %0, 2\n\t"                                         \
4064          "aghi 15,160\n\t"                                       \
4065          VALGRIND_CFI_EPILOGUE                                   \
4066          : /*out*/   "=d" (_res)                                 \
4067          : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
4068          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4069       );                                                         \
4070       lval = (__typeof__(lval)) _res;                            \
4071    } while (0)
4072
4073 /* The call abi has the arguments in r2-r6 and stack */
4074 #define CALL_FN_W_W(lval, orig, arg1)                            \
4075    do {                                                          \
4076       volatile OrigFn        _orig = (orig);                     \
4077       volatile unsigned long _argvec[2];                         \
4078       volatile unsigned long _res;                               \
4079       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4080       _argvec[1] = (unsigned long)arg1;                          \
4081       __asm__ volatile(                                          \
4082          VALGRIND_CFI_PROLOGUE                                   \
4083          "aghi 15,-160\n\t"                                      \
4084          "lg 2, 8(1)\n\t"                                        \
4085          "lg 1, 0(1)\n\t"                                        \
4086          VALGRIND_CALL_NOREDIR_R1                                \
4087          "lgr %0, 2\n\t"                                         \
4088          "aghi 15,160\n\t"                                       \
4089          VALGRIND_CFI_EPILOGUE                                   \
4090          : /*out*/   "=d" (_res)                                 \
4091          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4092          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4093       );                                                         \
4094       lval = (__typeof__(lval)) _res;                            \
4095    } while (0)
4096
4097 #define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
4098    do {                                                          \
4099       volatile OrigFn        _orig = (orig);                     \
4100       volatile unsigned long _argvec[3];                         \
4101       volatile unsigned long _res;                               \
4102       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4103       _argvec[1] = (unsigned long)arg1;                          \
4104       _argvec[2] = (unsigned long)arg2;                          \
4105       __asm__ volatile(                                          \
4106          VALGRIND_CFI_PROLOGUE                                   \
4107          "aghi 15,-160\n\t"                                      \
4108          "lg 2, 8(1)\n\t"                                        \
4109          "lg 3,16(1)\n\t"                                        \
4110          "lg 1, 0(1)\n\t"                                        \
4111          VALGRIND_CALL_NOREDIR_R1                                \
4112          "lgr %0, 2\n\t"                                         \
4113          "aghi 15,160\n\t"                                       \
4114          VALGRIND_CFI_EPILOGUE                                   \
4115          : /*out*/   "=d" (_res)                                 \
4116          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4117          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4118       );                                                         \
4119       lval = (__typeof__(lval)) _res;                            \
4120    } while (0)
4121
4122 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
4123    do {                                                          \
4124       volatile OrigFn        _orig = (orig);                     \
4125       volatile unsigned long _argvec[4];                         \
4126       volatile unsigned long _res;                               \
4127       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4128       _argvec[1] = (unsigned long)arg1;                          \
4129       _argvec[2] = (unsigned long)arg2;                          \
4130       _argvec[3] = (unsigned long)arg3;                          \
4131       __asm__ volatile(                                          \
4132          VALGRIND_CFI_PROLOGUE                                   \
4133          "aghi 15,-160\n\t"                                      \
4134          "lg 2, 8(1)\n\t"                                        \
4135          "lg 3,16(1)\n\t"                                        \
4136          "lg 4,24(1)\n\t"                                        \
4137          "lg 1, 0(1)\n\t"                                        \
4138          VALGRIND_CALL_NOREDIR_R1                                \
4139          "lgr %0, 2\n\t"                                         \
4140          "aghi 15,160\n\t"                                       \
4141          VALGRIND_CFI_EPILOGUE                                   \
4142          : /*out*/   "=d" (_res)                                 \
4143          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4144          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4145       );                                                         \
4146       lval = (__typeof__(lval)) _res;                            \
4147    } while (0)
4148
4149 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
4150    do {                                                          \
4151       volatile OrigFn        _orig = (orig);                     \
4152       volatile unsigned long _argvec[5];                         \
4153       volatile unsigned long _res;                               \
4154       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4155       _argvec[1] = (unsigned long)arg1;                          \
4156       _argvec[2] = (unsigned long)arg2;                          \
4157       _argvec[3] = (unsigned long)arg3;                          \
4158       _argvec[4] = (unsigned long)arg4;                          \
4159       __asm__ volatile(                                          \
4160          VALGRIND_CFI_PROLOGUE                                   \
4161          "aghi 15,-160\n\t"                                      \
4162          "lg 2, 8(1)\n\t"                                        \
4163          "lg 3,16(1)\n\t"                                        \
4164          "lg 4,24(1)\n\t"                                        \
4165          "lg 5,32(1)\n\t"                                        \
4166          "lg 1, 0(1)\n\t"                                        \
4167          VALGRIND_CALL_NOREDIR_R1                                \
4168          "lgr %0, 2\n\t"                                         \
4169          "aghi 15,160\n\t"                                       \
4170          VALGRIND_CFI_EPILOGUE                                   \
4171          : /*out*/   "=d" (_res)                                 \
4172          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4173          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4174       );                                                         \
4175       lval = (__typeof__(lval)) _res;                            \
4176    } while (0)
4177
4178 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
4179    do {                                                          \
4180       volatile OrigFn        _orig = (orig);                     \
4181       volatile unsigned long _argvec[6];                         \
4182       volatile unsigned long _res;                               \
4183       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4184       _argvec[1] = (unsigned long)arg1;                          \
4185       _argvec[2] = (unsigned long)arg2;                          \
4186       _argvec[3] = (unsigned long)arg3;                          \
4187       _argvec[4] = (unsigned long)arg4;                          \
4188       _argvec[5] = (unsigned long)arg5;                          \
4189       __asm__ volatile(                                          \
4190          VALGRIND_CFI_PROLOGUE                                   \
4191          "aghi 15,-160\n\t"                                      \
4192          "lg 2, 8(1)\n\t"                                        \
4193          "lg 3,16(1)\n\t"                                        \
4194          "lg 4,24(1)\n\t"                                        \
4195          "lg 5,32(1)\n\t"                                        \
4196          "lg 6,40(1)\n\t"                                        \
4197          "lg 1, 0(1)\n\t"                                        \
4198          VALGRIND_CALL_NOREDIR_R1                                \
4199          "lgr %0, 2\n\t"                                         \
4200          "aghi 15,160\n\t"                                       \
4201          VALGRIND_CFI_EPILOGUE                                   \
4202          : /*out*/   "=d" (_res)                                 \
4203          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4204          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4205       );                                                         \
4206       lval = (__typeof__(lval)) _res;                            \
4207    } while (0)
4208
4209 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4210                      arg6)                                       \
4211    do {                                                          \
4212       volatile OrigFn        _orig = (orig);                     \
4213       volatile unsigned long _argvec[7];                         \
4214       volatile unsigned long _res;                               \
4215       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4216       _argvec[1] = (unsigned long)arg1;                          \
4217       _argvec[2] = (unsigned long)arg2;                          \
4218       _argvec[3] = (unsigned long)arg3;                          \
4219       _argvec[4] = (unsigned long)arg4;                          \
4220       _argvec[5] = (unsigned long)arg5;                          \
4221       _argvec[6] = (unsigned long)arg6;                          \
4222       __asm__ volatile(                                          \
4223          VALGRIND_CFI_PROLOGUE                                   \
4224          "aghi 15,-168\n\t"                                      \
4225          "lg 2, 8(1)\n\t"                                        \
4226          "lg 3,16(1)\n\t"                                        \
4227          "lg 4,24(1)\n\t"                                        \
4228          "lg 5,32(1)\n\t"                                        \
4229          "lg 6,40(1)\n\t"                                        \
4230          "mvc 160(8,15), 48(1)\n\t"                              \
4231          "lg 1, 0(1)\n\t"                                        \
4232          VALGRIND_CALL_NOREDIR_R1                                \
4233          "lgr %0, 2\n\t"                                         \
4234          "aghi 15,168\n\t"                                       \
4235          VALGRIND_CFI_EPILOGUE                                   \
4236          : /*out*/   "=d" (_res)                                 \
4237          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4238          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4239       );                                                         \
4240       lval = (__typeof__(lval)) _res;                            \
4241    } while (0)
4242
4243 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4244                      arg6, arg7)                                 \
4245    do {                                                          \
4246       volatile OrigFn        _orig = (orig);                     \
4247       volatile unsigned long _argvec[8];                         \
4248       volatile unsigned long _res;                               \
4249       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4250       _argvec[1] = (unsigned long)arg1;                          \
4251       _argvec[2] = (unsigned long)arg2;                          \
4252       _argvec[3] = (unsigned long)arg3;                          \
4253       _argvec[4] = (unsigned long)arg4;                          \
4254       _argvec[5] = (unsigned long)arg5;                          \
4255       _argvec[6] = (unsigned long)arg6;                          \
4256       _argvec[7] = (unsigned long)arg7;                          \
4257       __asm__ volatile(                                          \
4258          VALGRIND_CFI_PROLOGUE                                   \
4259          "aghi 15,-176\n\t"                                      \
4260          "lg 2, 8(1)\n\t"                                        \
4261          "lg 3,16(1)\n\t"                                        \
4262          "lg 4,24(1)\n\t"                                        \
4263          "lg 5,32(1)\n\t"                                        \
4264          "lg 6,40(1)\n\t"                                        \
4265          "mvc 160(8,15), 48(1)\n\t"                              \
4266          "mvc 168(8,15), 56(1)\n\t"                              \
4267          "lg 1, 0(1)\n\t"                                        \
4268          VALGRIND_CALL_NOREDIR_R1                                \
4269          "lgr %0, 2\n\t"                                         \
4270          "aghi 15,176\n\t"                                       \
4271          VALGRIND_CFI_EPILOGUE                                   \
4272          : /*out*/   "=d" (_res)                                 \
4273          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4274          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4275       );                                                         \
4276       lval = (__typeof__(lval)) _res;                            \
4277    } while (0)
4278
4279 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4280                      arg6, arg7 ,arg8)                           \
4281    do {                                                          \
4282       volatile OrigFn        _orig = (orig);                     \
4283       volatile unsigned long _argvec[9];                         \
4284       volatile unsigned long _res;                               \
4285       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4286       _argvec[1] = (unsigned long)arg1;                          \
4287       _argvec[2] = (unsigned long)arg2;                          \
4288       _argvec[3] = (unsigned long)arg3;                          \
4289       _argvec[4] = (unsigned long)arg4;                          \
4290       _argvec[5] = (unsigned long)arg5;                          \
4291       _argvec[6] = (unsigned long)arg6;                          \
4292       _argvec[7] = (unsigned long)arg7;                          \
4293       _argvec[8] = (unsigned long)arg8;                          \
4294       __asm__ volatile(                                          \
4295          VALGRIND_CFI_PROLOGUE                                   \
4296          "aghi 15,-184\n\t"                                      \
4297          "lg 2, 8(1)\n\t"                                        \
4298          "lg 3,16(1)\n\t"                                        \
4299          "lg 4,24(1)\n\t"                                        \
4300          "lg 5,32(1)\n\t"                                        \
4301          "lg 6,40(1)\n\t"                                        \
4302          "mvc 160(8,15), 48(1)\n\t"                              \
4303          "mvc 168(8,15), 56(1)\n\t"                              \
4304          "mvc 176(8,15), 64(1)\n\t"                              \
4305          "lg 1, 0(1)\n\t"                                        \
4306          VALGRIND_CALL_NOREDIR_R1                                \
4307          "lgr %0, 2\n\t"                                         \
4308          "aghi 15,184\n\t"                                       \
4309          VALGRIND_CFI_EPILOGUE                                   \
4310          : /*out*/   "=d" (_res)                                 \
4311          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4312          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4313       );                                                         \
4314       lval = (__typeof__(lval)) _res;                            \
4315    } while (0)
4316
4317 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4318                      arg6, arg7 ,arg8, arg9)                     \
4319    do {                                                          \
4320       volatile OrigFn        _orig = (orig);                     \
4321       volatile unsigned long _argvec[10];                        \
4322       volatile unsigned long _res;                               \
4323       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4324       _argvec[1] = (unsigned long)arg1;                          \
4325       _argvec[2] = (unsigned long)arg2;                          \
4326       _argvec[3] = (unsigned long)arg3;                          \
4327       _argvec[4] = (unsigned long)arg4;                          \
4328       _argvec[5] = (unsigned long)arg5;                          \
4329       _argvec[6] = (unsigned long)arg6;                          \
4330       _argvec[7] = (unsigned long)arg7;                          \
4331       _argvec[8] = (unsigned long)arg8;                          \
4332       _argvec[9] = (unsigned long)arg9;                          \
4333       __asm__ volatile(                                          \
4334          VALGRIND_CFI_PROLOGUE                                   \
4335          "aghi 15,-192\n\t"                                      \
4336          "lg 2, 8(1)\n\t"                                        \
4337          "lg 3,16(1)\n\t"                                        \
4338          "lg 4,24(1)\n\t"                                        \
4339          "lg 5,32(1)\n\t"                                        \
4340          "lg 6,40(1)\n\t"                                        \
4341          "mvc 160(8,15), 48(1)\n\t"                              \
4342          "mvc 168(8,15), 56(1)\n\t"                              \
4343          "mvc 176(8,15), 64(1)\n\t"                              \
4344          "mvc 184(8,15), 72(1)\n\t"                              \
4345          "lg 1, 0(1)\n\t"                                        \
4346          VALGRIND_CALL_NOREDIR_R1                                \
4347          "lgr %0, 2\n\t"                                         \
4348          "aghi 15,192\n\t"                                       \
4349          VALGRIND_CFI_EPILOGUE                                   \
4350          : /*out*/   "=d" (_res)                                 \
4351          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4352          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4353       );                                                         \
4354       lval = (__typeof__(lval)) _res;                            \
4355    } while (0)
4356
4357 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4358                      arg6, arg7 ,arg8, arg9, arg10)              \
4359    do {                                                          \
4360       volatile OrigFn        _orig = (orig);                     \
4361       volatile unsigned long _argvec[11];                        \
4362       volatile unsigned long _res;                               \
4363       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4364       _argvec[1] = (unsigned long)arg1;                          \
4365       _argvec[2] = (unsigned long)arg2;                          \
4366       _argvec[3] = (unsigned long)arg3;                          \
4367       _argvec[4] = (unsigned long)arg4;                          \
4368       _argvec[5] = (unsigned long)arg5;                          \
4369       _argvec[6] = (unsigned long)arg6;                          \
4370       _argvec[7] = (unsigned long)arg7;                          \
4371       _argvec[8] = (unsigned long)arg8;                          \
4372       _argvec[9] = (unsigned long)arg9;                          \
4373       _argvec[10] = (unsigned long)arg10;                        \
4374       __asm__ volatile(                                          \
4375          VALGRIND_CFI_PROLOGUE                                   \
4376          "aghi 15,-200\n\t"                                      \
4377          "lg 2, 8(1)\n\t"                                        \
4378          "lg 3,16(1)\n\t"                                        \
4379          "lg 4,24(1)\n\t"                                        \
4380          "lg 5,32(1)\n\t"                                        \
4381          "lg 6,40(1)\n\t"                                        \
4382          "mvc 160(8,15), 48(1)\n\t"                              \
4383          "mvc 168(8,15), 56(1)\n\t"                              \
4384          "mvc 176(8,15), 64(1)\n\t"                              \
4385          "mvc 184(8,15), 72(1)\n\t"                              \
4386          "mvc 192(8,15), 80(1)\n\t"                              \
4387          "lg 1, 0(1)\n\t"                                        \
4388          VALGRIND_CALL_NOREDIR_R1                                \
4389          "lgr %0, 2\n\t"                                         \
4390          "aghi 15,200\n\t"                                       \
4391          VALGRIND_CFI_EPILOGUE                                   \
4392          : /*out*/   "=d" (_res)                                 \
4393          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4394          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4395       );                                                         \
4396       lval = (__typeof__(lval)) _res;                            \
4397    } while (0)
4398
4399 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4400                      arg6, arg7 ,arg8, arg9, arg10, arg11)       \
4401    do {                                                          \
4402       volatile OrigFn        _orig = (orig);                     \
4403       volatile unsigned long _argvec[12];                        \
4404       volatile unsigned long _res;                               \
4405       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4406       _argvec[1] = (unsigned long)arg1;                          \
4407       _argvec[2] = (unsigned long)arg2;                          \
4408       _argvec[3] = (unsigned long)arg3;                          \
4409       _argvec[4] = (unsigned long)arg4;                          \
4410       _argvec[5] = (unsigned long)arg5;                          \
4411       _argvec[6] = (unsigned long)arg6;                          \
4412       _argvec[7] = (unsigned long)arg7;                          \
4413       _argvec[8] = (unsigned long)arg8;                          \
4414       _argvec[9] = (unsigned long)arg9;                          \
4415       _argvec[10] = (unsigned long)arg10;                        \
4416       _argvec[11] = (unsigned long)arg11;                        \
4417       __asm__ volatile(                                          \
4418          VALGRIND_CFI_PROLOGUE                                   \
4419          "aghi 15,-208\n\t"                                      \
4420          "lg 2, 8(1)\n\t"                                        \
4421          "lg 3,16(1)\n\t"                                        \
4422          "lg 4,24(1)\n\t"                                        \
4423          "lg 5,32(1)\n\t"                                        \
4424          "lg 6,40(1)\n\t"                                        \
4425          "mvc 160(8,15), 48(1)\n\t"                              \
4426          "mvc 168(8,15), 56(1)\n\t"                              \
4427          "mvc 176(8,15), 64(1)\n\t"                              \
4428          "mvc 184(8,15), 72(1)\n\t"                              \
4429          "mvc 192(8,15), 80(1)\n\t"                              \
4430          "mvc 200(8,15), 88(1)\n\t"                              \
4431          "lg 1, 0(1)\n\t"                                        \
4432          VALGRIND_CALL_NOREDIR_R1                                \
4433          "lgr %0, 2\n\t"                                         \
4434          "aghi 15,208\n\t"                                       \
4435          VALGRIND_CFI_EPILOGUE                                   \
4436          : /*out*/   "=d" (_res)                                 \
4437          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4438          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4439       );                                                         \
4440       lval = (__typeof__(lval)) _res;                            \
4441    } while (0)
4442
4443 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4444                      arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
4445    do {                                                          \
4446       volatile OrigFn        _orig = (orig);                     \
4447       volatile unsigned long _argvec[13];                        \
4448       volatile unsigned long _res;                               \
4449       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4450       _argvec[1] = (unsigned long)arg1;                          \
4451       _argvec[2] = (unsigned long)arg2;                          \
4452       _argvec[3] = (unsigned long)arg3;                          \
4453       _argvec[4] = (unsigned long)arg4;                          \
4454       _argvec[5] = (unsigned long)arg5;                          \
4455       _argvec[6] = (unsigned long)arg6;                          \
4456       _argvec[7] = (unsigned long)arg7;                          \
4457       _argvec[8] = (unsigned long)arg8;                          \
4458       _argvec[9] = (unsigned long)arg9;                          \
4459       _argvec[10] = (unsigned long)arg10;                        \
4460       _argvec[11] = (unsigned long)arg11;                        \
4461       _argvec[12] = (unsigned long)arg12;                        \
4462       __asm__ volatile(                                          \
4463          VALGRIND_CFI_PROLOGUE                                   \
4464          "aghi 15,-216\n\t"                                      \
4465          "lg 2, 8(1)\n\t"                                        \
4466          "lg 3,16(1)\n\t"                                        \
4467          "lg 4,24(1)\n\t"                                        \
4468          "lg 5,32(1)\n\t"                                        \
4469          "lg 6,40(1)\n\t"                                        \
4470          "mvc 160(8,15), 48(1)\n\t"                              \
4471          "mvc 168(8,15), 56(1)\n\t"                              \
4472          "mvc 176(8,15), 64(1)\n\t"                              \
4473          "mvc 184(8,15), 72(1)\n\t"                              \
4474          "mvc 192(8,15), 80(1)\n\t"                              \
4475          "mvc 200(8,15), 88(1)\n\t"                              \
4476          "mvc 208(8,15), 96(1)\n\t"                              \
4477          "lg 1, 0(1)\n\t"                                        \
4478          VALGRIND_CALL_NOREDIR_R1                                \
4479          "lgr %0, 2\n\t"                                         \
4480          "aghi 15,216\n\t"                                       \
4481          VALGRIND_CFI_EPILOGUE                                   \
4482          : /*out*/   "=d" (_res)                                 \
4483          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4484          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4485       );                                                         \
4486       lval = (__typeof__(lval)) _res;                            \
4487    } while (0)
4488
4489
4490 #endif /* PLAT_s390x_linux */
4491
4492 /* ------------------------- mips32-linux ----------------------- */
4493
4494 #if defined(PLAT_mips32_linux)
4495
4496 /* These regs are trashed by the hidden call. */
4497 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
4498 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
4499 "$25", "$31"
4500
4501 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
4502    long) == 4. */
4503
4504 #define CALL_FN_W_v(lval, orig)                                   \
4505    do {                                                           \
4506       volatile OrigFn        _orig = (orig);                      \
4507       volatile unsigned long _argvec[1];                          \
4508       volatile unsigned long _res;                                \
4509       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4510       __asm__ volatile(                                           \
4511          "subu $29, $29, 8 \n\t"                                  \
4512          "sw $28, 0($29) \n\t"                                    \
4513          "sw $31, 4($29) \n\t"                                    \
4514          "subu $29, $29, 16 \n\t"                                 \
4515          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4516          VALGRIND_CALL_NOREDIR_T9                                 \
4517          "addu $29, $29, 16\n\t"                                  \
4518          "lw $28, 0($29) \n\t"                                    \
4519          "lw $31, 4($29) \n\t"                                    \
4520          "addu $29, $29, 8 \n\t"                                  \
4521          "move %0, $2\n"                                          \
4522          : /*out*/   "=r" (_res)                                  \
4523          : /*in*/    "0" (&_argvec[0])                            \
4524          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4525       );                                                          \
4526       lval = (__typeof__(lval)) _res;                             \
4527    } while (0)
4528
4529 #define CALL_FN_W_W(lval, orig, arg1)                             \
4530    do {                                                           \
4531       volatile OrigFn        _orig = (orig);                      \
4532      volatile unsigned long _argvec[2];                           \
4533       volatile unsigned long _res;                                \
4534       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4535       _argvec[1] = (unsigned long)(arg1);                         \
4536       __asm__ volatile(                                           \
4537          "subu $29, $29, 8 \n\t"                                  \
4538          "sw $28, 0($29) \n\t"                                    \
4539          "sw $31, 4($29) \n\t"                                    \
4540          "subu $29, $29, 16 \n\t"                                 \
4541          "lw $4, 4(%1) \n\t"   /* arg1*/                          \
4542          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4543          VALGRIND_CALL_NOREDIR_T9                                 \
4544          "addu $29, $29, 16 \n\t"                                 \
4545          "lw $28, 0($29) \n\t"                                    \
4546          "lw $31, 4($29) \n\t"                                    \
4547          "addu $29, $29, 8 \n\t"                                  \
4548          "move %0, $2\n"                                          \
4549          : /*out*/   "=r" (_res)                                  \
4550          : /*in*/    "0" (&_argvec[0])                            \
4551          : /*trash*/ "memory",  __CALLER_SAVED_REGS               \
4552       );                                                          \
4553       lval = (__typeof__(lval)) _res;                             \
4554    } while (0)
4555
4556 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
4557    do {                                                           \
4558       volatile OrigFn        _orig = (orig);                      \
4559       volatile unsigned long _argvec[3];                          \
4560       volatile unsigned long _res;                                \
4561       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4562       _argvec[1] = (unsigned long)(arg1);                         \
4563       _argvec[2] = (unsigned long)(arg2);                         \
4564       __asm__ volatile(                                           \
4565          "subu $29, $29, 8 \n\t"                                  \
4566          "sw $28, 0($29) \n\t"                                    \
4567          "sw $31, 4($29) \n\t"                                    \
4568          "subu $29, $29, 16 \n\t"                                 \
4569          "lw $4, 4(%1) \n\t"                                      \
4570          "lw $5, 8(%1) \n\t"                                      \
4571          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4572          VALGRIND_CALL_NOREDIR_T9                                 \
4573          "addu $29, $29, 16 \n\t"                                 \
4574          "lw $28, 0($29) \n\t"                                    \
4575          "lw $31, 4($29) \n\t"                                    \
4576          "addu $29, $29, 8 \n\t"                                  \
4577          "move %0, $2\n"                                          \
4578          : /*out*/   "=r" (_res)                                  \
4579          : /*in*/    "0" (&_argvec[0])                            \
4580          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4581       );                                                          \
4582       lval = (__typeof__(lval)) _res;                             \
4583    } while (0)
4584
4585 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
4586    do {                                                           \
4587       volatile OrigFn        _orig = (orig);                      \
4588       volatile unsigned long _argvec[4];                          \
4589       volatile unsigned long _res;                                \
4590       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4591       _argvec[1] = (unsigned long)(arg1);                         \
4592       _argvec[2] = (unsigned long)(arg2);                         \
4593       _argvec[3] = (unsigned long)(arg3);                         \
4594       __asm__ volatile(                                           \
4595          "subu $29, $29, 8 \n\t"                                  \
4596          "sw $28, 0($29) \n\t"                                    \
4597          "sw $31, 4($29) \n\t"                                    \
4598          "subu $29, $29, 16 \n\t"                                 \
4599          "lw $4, 4(%1) \n\t"                                      \
4600          "lw $5, 8(%1) \n\t"                                      \
4601          "lw $6, 12(%1) \n\t"                                     \
4602          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4603          VALGRIND_CALL_NOREDIR_T9                                 \
4604          "addu $29, $29, 16 \n\t"                                 \
4605          "lw $28, 0($29) \n\t"                                    \
4606          "lw $31, 4($29) \n\t"                                    \
4607          "addu $29, $29, 8 \n\t"                                  \
4608          "move %0, $2\n"                                          \
4609          : /*out*/   "=r" (_res)                                  \
4610          : /*in*/    "0" (&_argvec[0])                            \
4611          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4612       );                                                          \
4613       lval = (__typeof__(lval)) _res;                             \
4614    } while (0)
4615
4616 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
4617    do {                                                           \
4618       volatile OrigFn        _orig = (orig);                      \
4619       volatile unsigned long _argvec[5];                          \
4620       volatile unsigned long _res;                                \
4621       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4622       _argvec[1] = (unsigned long)(arg1);                         \
4623       _argvec[2] = (unsigned long)(arg2);                         \
4624       _argvec[3] = (unsigned long)(arg3);                         \
4625       _argvec[4] = (unsigned long)(arg4);                         \
4626       __asm__ volatile(                                           \
4627          "subu $29, $29, 8 \n\t"                                  \
4628          "sw $28, 0($29) \n\t"                                    \
4629          "sw $31, 4($29) \n\t"                                    \
4630          "subu $29, $29, 16 \n\t"                                 \
4631          "lw $4, 4(%1) \n\t"                                      \
4632          "lw $5, 8(%1) \n\t"                                      \
4633          "lw $6, 12(%1) \n\t"                                     \
4634          "lw $7, 16(%1) \n\t"                                     \
4635          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4636          VALGRIND_CALL_NOREDIR_T9                                 \
4637          "addu $29, $29, 16 \n\t"                                 \
4638          "lw $28, 0($29) \n\t"                                    \
4639          "lw $31, 4($29) \n\t"                                    \
4640          "addu $29, $29, 8 \n\t"                                  \
4641          "move %0, $2\n"                                          \
4642          : /*out*/   "=r" (_res)                                  \
4643          : /*in*/    "0" (&_argvec[0])                            \
4644          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4645       );                                                          \
4646       lval = (__typeof__(lval)) _res;                             \
4647    } while (0)
4648
4649 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
4650    do {                                                           \
4651       volatile OrigFn        _orig = (orig);                      \
4652       volatile unsigned long _argvec[6];                          \
4653       volatile unsigned long _res;                                \
4654       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4655       _argvec[1] = (unsigned long)(arg1);                         \
4656       _argvec[2] = (unsigned long)(arg2);                         \
4657       _argvec[3] = (unsigned long)(arg3);                         \
4658       _argvec[4] = (unsigned long)(arg4);                         \
4659       _argvec[5] = (unsigned long)(arg5);                         \
4660       __asm__ volatile(                                           \
4661          "subu $29, $29, 8 \n\t"                                  \
4662          "sw $28, 0($29) \n\t"                                    \
4663          "sw $31, 4($29) \n\t"                                    \
4664          "lw $4, 20(%1) \n\t"                                     \
4665          "subu $29, $29, 24\n\t"                                  \
4666          "sw $4, 16($29) \n\t"                                    \
4667          "lw $4, 4(%1) \n\t"                                      \
4668          "lw $5, 8(%1) \n\t"                                      \
4669          "lw $6, 12(%1) \n\t"                                     \
4670          "lw $7, 16(%1) \n\t"                                     \
4671          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4672          VALGRIND_CALL_NOREDIR_T9                                 \
4673          "addu $29, $29, 24 \n\t"                                 \
4674          "lw $28, 0($29) \n\t"                                    \
4675          "lw $31, 4($29) \n\t"                                    \
4676          "addu $29, $29, 8 \n\t"                                  \
4677          "move %0, $2\n"                                          \
4678          : /*out*/   "=r" (_res)                                  \
4679          : /*in*/    "0" (&_argvec[0])                            \
4680          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4681       );                                                          \
4682       lval = (__typeof__(lval)) _res;                             \
4683    } while (0)
4684 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
4685    do {                                                           \
4686       volatile OrigFn        _orig = (orig);                      \
4687       volatile unsigned long _argvec[7];                          \
4688       volatile unsigned long _res;                                \
4689       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4690       _argvec[1] = (unsigned long)(arg1);                         \
4691       _argvec[2] = (unsigned long)(arg2);                         \
4692       _argvec[3] = (unsigned long)(arg3);                         \
4693       _argvec[4] = (unsigned long)(arg4);                         \
4694       _argvec[5] = (unsigned long)(arg5);                         \
4695       _argvec[6] = (unsigned long)(arg6);                         \
4696       __asm__ volatile(                                           \
4697          "subu $29, $29, 8 \n\t"                                  \
4698          "sw $28, 0($29) \n\t"                                    \
4699          "sw $31, 4($29) \n\t"                                    \
4700          "lw $4, 20(%1) \n\t"                                     \
4701          "subu $29, $29, 32\n\t"                                  \
4702          "sw $4, 16($29) \n\t"                                    \
4703          "lw $4, 24(%1) \n\t"                                     \
4704          "nop\n\t"                                                \
4705          "sw $4, 20($29) \n\t"                                    \
4706          "lw $4, 4(%1) \n\t"                                      \
4707          "lw $5, 8(%1) \n\t"                                      \
4708          "lw $6, 12(%1) \n\t"                                     \
4709          "lw $7, 16(%1) \n\t"                                     \
4710          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4711          VALGRIND_CALL_NOREDIR_T9                                 \
4712          "addu $29, $29, 32 \n\t"                                 \
4713          "lw $28, 0($29) \n\t"                                    \
4714          "lw $31, 4($29) \n\t"                                    \
4715          "addu $29, $29, 8 \n\t"                                  \
4716          "move %0, $2\n"                                          \
4717          : /*out*/   "=r" (_res)                                  \
4718          : /*in*/    "0" (&_argvec[0])                            \
4719          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4720       );                                                          \
4721       lval = (__typeof__(lval)) _res;                             \
4722    } while (0)
4723
4724 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4725                                  arg7)                            \
4726    do {                                                           \
4727       volatile OrigFn        _orig = (orig);                      \
4728       volatile unsigned long _argvec[8];                          \
4729       volatile unsigned long _res;                                \
4730       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4731       _argvec[1] = (unsigned long)(arg1);                         \
4732       _argvec[2] = (unsigned long)(arg2);                         \
4733       _argvec[3] = (unsigned long)(arg3);                         \
4734       _argvec[4] = (unsigned long)(arg4);                         \
4735       _argvec[5] = (unsigned long)(arg5);                         \
4736       _argvec[6] = (unsigned long)(arg6);                         \
4737       _argvec[7] = (unsigned long)(arg7);                         \
4738       __asm__ volatile(                                           \
4739          "subu $29, $29, 8 \n\t"                                  \
4740          "sw $28, 0($29) \n\t"                                    \
4741          "sw $31, 4($29) \n\t"                                    \
4742          "lw $4, 20(%1) \n\t"                                     \
4743          "subu $29, $29, 32\n\t"                                  \
4744          "sw $4, 16($29) \n\t"                                    \
4745          "lw $4, 24(%1) \n\t"                                     \
4746          "sw $4, 20($29) \n\t"                                    \
4747          "lw $4, 28(%1) \n\t"                                     \
4748          "sw $4, 24($29) \n\t"                                    \
4749          "lw $4, 4(%1) \n\t"                                      \
4750          "lw $5, 8(%1) \n\t"                                      \
4751          "lw $6, 12(%1) \n\t"                                     \
4752          "lw $7, 16(%1) \n\t"                                     \
4753          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4754          VALGRIND_CALL_NOREDIR_T9                                 \
4755          "addu $29, $29, 32 \n\t"                                 \
4756          "lw $28, 0($29) \n\t"                                    \
4757          "lw $31, 4($29) \n\t"                                    \
4758          "addu $29, $29, 8 \n\t"                                  \
4759          "move %0, $2\n"                                          \
4760          : /*out*/   "=r" (_res)                                  \
4761          : /*in*/    "0" (&_argvec[0])                            \
4762          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4763       );                                                          \
4764       lval = (__typeof__(lval)) _res;                             \
4765    } while (0)
4766
4767 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4768                                  arg7,arg8)                       \
4769    do {                                                           \
4770       volatile OrigFn        _orig = (orig);                      \
4771       volatile unsigned long _argvec[9];                          \
4772       volatile unsigned long _res;                                \
4773       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4774       _argvec[1] = (unsigned long)(arg1);                         \
4775       _argvec[2] = (unsigned long)(arg2);                         \
4776       _argvec[3] = (unsigned long)(arg3);                         \
4777       _argvec[4] = (unsigned long)(arg4);                         \
4778       _argvec[5] = (unsigned long)(arg5);                         \
4779       _argvec[6] = (unsigned long)(arg6);                         \
4780       _argvec[7] = (unsigned long)(arg7);                         \
4781       _argvec[8] = (unsigned long)(arg8);                         \
4782       __asm__ volatile(                                           \
4783          "subu $29, $29, 8 \n\t"                                  \
4784          "sw $28, 0($29) \n\t"                                    \
4785          "sw $31, 4($29) \n\t"                                    \
4786          "lw $4, 20(%1) \n\t"                                     \
4787          "subu $29, $29, 40\n\t"                                  \
4788          "sw $4, 16($29) \n\t"                                    \
4789          "lw $4, 24(%1) \n\t"                                     \
4790          "sw $4, 20($29) \n\t"                                    \
4791          "lw $4, 28(%1) \n\t"                                     \
4792          "sw $4, 24($29) \n\t"                                    \
4793          "lw $4, 32(%1) \n\t"                                     \
4794          "sw $4, 28($29) \n\t"                                    \
4795          "lw $4, 4(%1) \n\t"                                      \
4796          "lw $5, 8(%1) \n\t"                                      \
4797          "lw $6, 12(%1) \n\t"                                     \
4798          "lw $7, 16(%1) \n\t"                                     \
4799          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4800          VALGRIND_CALL_NOREDIR_T9                                 \
4801          "addu $29, $29, 40 \n\t"                                 \
4802          "lw $28, 0($29) \n\t"                                    \
4803          "lw $31, 4($29) \n\t"                                    \
4804          "addu $29, $29, 8 \n\t"                                  \
4805          "move %0, $2\n"                                          \
4806          : /*out*/   "=r" (_res)                                  \
4807          : /*in*/    "0" (&_argvec[0])                            \
4808          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4809       );                                                          \
4810       lval = (__typeof__(lval)) _res;                             \
4811    } while (0)
4812
4813 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4814                                  arg7,arg8,arg9)                  \
4815    do {                                                           \
4816       volatile OrigFn        _orig = (orig);                      \
4817       volatile unsigned long _argvec[10];                         \
4818       volatile unsigned long _res;                                \
4819       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4820       _argvec[1] = (unsigned long)(arg1);                         \
4821       _argvec[2] = (unsigned long)(arg2);                         \
4822       _argvec[3] = (unsigned long)(arg3);                         \
4823       _argvec[4] = (unsigned long)(arg4);                         \
4824       _argvec[5] = (unsigned long)(arg5);                         \
4825       _argvec[6] = (unsigned long)(arg6);                         \
4826       _argvec[7] = (unsigned long)(arg7);                         \
4827       _argvec[8] = (unsigned long)(arg8);                         \
4828       _argvec[9] = (unsigned long)(arg9);                         \
4829       __asm__ volatile(                                           \
4830          "subu $29, $29, 8 \n\t"                                  \
4831          "sw $28, 0($29) \n\t"                                    \
4832          "sw $31, 4($29) \n\t"                                    \
4833          "lw $4, 20(%1) \n\t"                                     \
4834          "subu $29, $29, 40\n\t"                                  \
4835          "sw $4, 16($29) \n\t"                                    \
4836          "lw $4, 24(%1) \n\t"                                     \
4837          "sw $4, 20($29) \n\t"                                    \
4838          "lw $4, 28(%1) \n\t"                                     \
4839          "sw $4, 24($29) \n\t"                                    \
4840          "lw $4, 32(%1) \n\t"                                     \
4841          "sw $4, 28($29) \n\t"                                    \
4842          "lw $4, 36(%1) \n\t"                                     \
4843          "sw $4, 32($29) \n\t"                                    \
4844          "lw $4, 4(%1) \n\t"                                      \
4845          "lw $5, 8(%1) \n\t"                                      \
4846          "lw $6, 12(%1) \n\t"                                     \
4847          "lw $7, 16(%1) \n\t"                                     \
4848          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4849          VALGRIND_CALL_NOREDIR_T9                                 \
4850          "addu $29, $29, 40 \n\t"                                 \
4851          "lw $28, 0($29) \n\t"                                    \
4852          "lw $31, 4($29) \n\t"                                    \
4853          "addu $29, $29, 8 \n\t"                                  \
4854          "move %0, $2\n"                                          \
4855          : /*out*/   "=r" (_res)                                  \
4856          : /*in*/    "0" (&_argvec[0])                            \
4857          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4858       );                                                          \
4859       lval = (__typeof__(lval)) _res;                             \
4860    } while (0)
4861
4862 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4863                                   arg7,arg8,arg9,arg10)           \
4864    do {                                                           \
4865       volatile OrigFn        _orig = (orig);                      \
4866       volatile unsigned long _argvec[11];                         \
4867       volatile unsigned long _res;                                \
4868       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4869       _argvec[1] = (unsigned long)(arg1);                         \
4870       _argvec[2] = (unsigned long)(arg2);                         \
4871       _argvec[3] = (unsigned long)(arg3);                         \
4872       _argvec[4] = (unsigned long)(arg4);                         \
4873       _argvec[5] = (unsigned long)(arg5);                         \
4874       _argvec[6] = (unsigned long)(arg6);                         \
4875       _argvec[7] = (unsigned long)(arg7);                         \
4876       _argvec[8] = (unsigned long)(arg8);                         \
4877       _argvec[9] = (unsigned long)(arg9);                         \
4878       _argvec[10] = (unsigned long)(arg10);                       \
4879       __asm__ volatile(                                           \
4880          "subu $29, $29, 8 \n\t"                                  \
4881          "sw $28, 0($29) \n\t"                                    \
4882          "sw $31, 4($29) \n\t"                                    \
4883          "lw $4, 20(%1) \n\t"                                     \
4884          "subu $29, $29, 48\n\t"                                  \
4885          "sw $4, 16($29) \n\t"                                    \
4886          "lw $4, 24(%1) \n\t"                                     \
4887          "sw $4, 20($29) \n\t"                                    \
4888          "lw $4, 28(%1) \n\t"                                     \
4889          "sw $4, 24($29) \n\t"                                    \
4890          "lw $4, 32(%1) \n\t"                                     \
4891          "sw $4, 28($29) \n\t"                                    \
4892          "lw $4, 36(%1) \n\t"                                     \
4893          "sw $4, 32($29) \n\t"                                    \
4894          "lw $4, 40(%1) \n\t"                                     \
4895          "sw $4, 36($29) \n\t"                                    \
4896          "lw $4, 4(%1) \n\t"                                      \
4897          "lw $5, 8(%1) \n\t"                                      \
4898          "lw $6, 12(%1) \n\t"                                     \
4899          "lw $7, 16(%1) \n\t"                                     \
4900          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4901          VALGRIND_CALL_NOREDIR_T9                                 \
4902          "addu $29, $29, 48 \n\t"                                 \
4903          "lw $28, 0($29) \n\t"                                    \
4904          "lw $31, 4($29) \n\t"                                    \
4905          "addu $29, $29, 8 \n\t"                                  \
4906          "move %0, $2\n"                                          \
4907          : /*out*/   "=r" (_res)                                  \
4908          : /*in*/    "0" (&_argvec[0])                            \
4909          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4910       );                                                          \
4911       lval = (__typeof__(lval)) _res;                             \
4912    } while (0)
4913
4914 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4915                                   arg6,arg7,arg8,arg9,arg10,      \
4916                                   arg11)                          \
4917    do {                                                           \
4918       volatile OrigFn        _orig = (orig);                      \
4919       volatile unsigned long _argvec[12];                         \
4920       volatile unsigned long _res;                                \
4921       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4922       _argvec[1] = (unsigned long)(arg1);                         \
4923       _argvec[2] = (unsigned long)(arg2);                         \
4924       _argvec[3] = (unsigned long)(arg3);                         \
4925       _argvec[4] = (unsigned long)(arg4);                         \
4926       _argvec[5] = (unsigned long)(arg5);                         \
4927       _argvec[6] = (unsigned long)(arg6);                         \
4928       _argvec[7] = (unsigned long)(arg7);                         \
4929       _argvec[8] = (unsigned long)(arg8);                         \
4930       _argvec[9] = (unsigned long)(arg9);                         \
4931       _argvec[10] = (unsigned long)(arg10);                       \
4932       _argvec[11] = (unsigned long)(arg11);                       \
4933       __asm__ volatile(                                           \
4934          "subu $29, $29, 8 \n\t"                                  \
4935          "sw $28, 0($29) \n\t"                                    \
4936          "sw $31, 4($29) \n\t"                                    \
4937          "lw $4, 20(%1) \n\t"                                     \
4938          "subu $29, $29, 48\n\t"                                  \
4939          "sw $4, 16($29) \n\t"                                    \
4940          "lw $4, 24(%1) \n\t"                                     \
4941          "sw $4, 20($29) \n\t"                                    \
4942          "lw $4, 28(%1) \n\t"                                     \
4943          "sw $4, 24($29) \n\t"                                    \
4944          "lw $4, 32(%1) \n\t"                                     \
4945          "sw $4, 28($29) \n\t"                                    \
4946          "lw $4, 36(%1) \n\t"                                     \
4947          "sw $4, 32($29) \n\t"                                    \
4948          "lw $4, 40(%1) \n\t"                                     \
4949          "sw $4, 36($29) \n\t"                                    \
4950          "lw $4, 44(%1) \n\t"                                     \
4951          "sw $4, 40($29) \n\t"                                    \
4952          "lw $4, 4(%1) \n\t"                                      \
4953          "lw $5, 8(%1) \n\t"                                      \
4954          "lw $6, 12(%1) \n\t"                                     \
4955          "lw $7, 16(%1) \n\t"                                     \
4956          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
4957          VALGRIND_CALL_NOREDIR_T9                                 \
4958          "addu $29, $29, 48 \n\t"                                 \
4959          "lw $28, 0($29) \n\t"                                    \
4960          "lw $31, 4($29) \n\t"                                    \
4961          "addu $29, $29, 8 \n\t"                                  \
4962          "move %0, $2\n"                                          \
4963          : /*out*/   "=r" (_res)                                  \
4964          : /*in*/    "0" (&_argvec[0])                            \
4965          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
4966       );                                                          \
4967       lval = (__typeof__(lval)) _res;                             \
4968    } while (0)
4969
4970 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4971                                   arg6,arg7,arg8,arg9,arg10,      \
4972                                   arg11,arg12)                    \
4973    do {                                                           \
4974       volatile OrigFn        _orig = (orig);                      \
4975       volatile unsigned long _argvec[13];                         \
4976       volatile unsigned long _res;                                \
4977       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4978       _argvec[1] = (unsigned long)(arg1);                         \
4979       _argvec[2] = (unsigned long)(arg2);                         \
4980       _argvec[3] = (unsigned long)(arg3);                         \
4981       _argvec[4] = (unsigned long)(arg4);                         \
4982       _argvec[5] = (unsigned long)(arg5);                         \
4983       _argvec[6] = (unsigned long)(arg6);                         \
4984       _argvec[7] = (unsigned long)(arg7);                         \
4985       _argvec[8] = (unsigned long)(arg8);                         \
4986       _argvec[9] = (unsigned long)(arg9);                         \
4987       _argvec[10] = (unsigned long)(arg10);                       \
4988       _argvec[11] = (unsigned long)(arg11);                       \
4989       _argvec[12] = (unsigned long)(arg12);                       \
4990       __asm__ volatile(                                           \
4991          "subu $29, $29, 8 \n\t"                                  \
4992          "sw $28, 0($29) \n\t"                                    \
4993          "sw $31, 4($29) \n\t"                                    \
4994          "lw $4, 20(%1) \n\t"                                     \
4995          "subu $29, $29, 56\n\t"                                  \
4996          "sw $4, 16($29) \n\t"                                    \
4997          "lw $4, 24(%1) \n\t"                                     \
4998          "sw $4, 20($29) \n\t"                                    \
4999          "lw $4, 28(%1) \n\t"                                     \
5000          "sw $4, 24($29) \n\t"                                    \
5001          "lw $4, 32(%1) \n\t"                                     \
5002          "sw $4, 28($29) \n\t"                                    \
5003          "lw $4, 36(%1) \n\t"                                     \
5004          "sw $4, 32($29) \n\t"                                    \
5005          "lw $4, 40(%1) \n\t"                                     \
5006          "sw $4, 36($29) \n\t"                                    \
5007          "lw $4, 44(%1) \n\t"                                     \
5008          "sw $4, 40($29) \n\t"                                    \
5009          "lw $4, 48(%1) \n\t"                                     \
5010          "sw $4, 44($29) \n\t"                                    \
5011          "lw $4, 4(%1) \n\t"                                      \
5012          "lw $5, 8(%1) \n\t"                                      \
5013          "lw $6, 12(%1) \n\t"                                     \
5014          "lw $7, 16(%1) \n\t"                                     \
5015          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5016          VALGRIND_CALL_NOREDIR_T9                                 \
5017          "addu $29, $29, 56 \n\t"                                 \
5018          "lw $28, 0($29) \n\t"                                    \
5019          "lw $31, 4($29) \n\t"                                    \
5020          "addu $29, $29, 8 \n\t"                                  \
5021          "move %0, $2\n"                                          \
5022          : /*out*/   "=r" (_res)                                  \
5023          : /*in*/    "r" (&_argvec[0])                            \
5024          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5025       );                                                          \
5026       lval = (__typeof__(lval)) _res;                             \
5027    } while (0)
5028
5029 #endif /* PLAT_mips32_linux */
5030
5031 /* ------------------------- mips64-linux ------------------------- */
5032
5033 #if defined(PLAT_mips64_linux)
5034
5035 /* These regs are trashed by the hidden call. */
5036 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
5037 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5038 "$25", "$31"
5039
5040 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5041    long) == 4. */
5042
5043 #define CALL_FN_W_v(lval, orig)                                   \
5044    do {                                                           \
5045       volatile OrigFn        _orig = (orig);                      \
5046       volatile unsigned long _argvec[1];                          \
5047       volatile unsigned long _res;                                \
5048       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5049       __asm__ volatile(                                           \
5050          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5051          VALGRIND_CALL_NOREDIR_T9                                 \
5052          "move %0, $2\n"                                          \
5053          : /*out*/   "=r" (_res)                                  \
5054          : /*in*/    "0" (&_argvec[0])                            \
5055          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5056       );                                                          \
5057       lval = (__typeof__(lval)) _res;                             \
5058    } while (0)
5059
5060 #define CALL_FN_W_W(lval, orig, arg1)                             \
5061    do {                                                           \
5062       volatile OrigFn        _orig = (orig);                      \
5063       volatile unsigned long _argvec[2];                          \
5064       volatile unsigned long _res;                                \
5065       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5066       _argvec[1] = (unsigned long)(arg1);                         \
5067       __asm__ volatile(                                           \
5068          "ld $4, 8(%1)\n\t"   /* arg1*/                           \
5069          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5070          VALGRIND_CALL_NOREDIR_T9                                 \
5071          "move %0, $2\n"                                          \
5072          : /*out*/   "=r" (_res)                                  \
5073          : /*in*/    "r" (&_argvec[0])                            \
5074          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5075       );                                                          \
5076       lval = (__typeof__(lval)) _res;                             \
5077    } while (0)
5078
5079 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
5080    do {                                                           \
5081       volatile OrigFn        _orig = (orig);                      \
5082       volatile unsigned long _argvec[3];                          \
5083       volatile unsigned long _res;                                \
5084       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5085       _argvec[1] = (unsigned long)(arg1);                         \
5086       _argvec[2] = (unsigned long)(arg2);                         \
5087       __asm__ volatile(                                           \
5088          "ld $4, 8(%1)\n\t"                                       \
5089          "ld $5, 16(%1)\n\t"                                      \
5090          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5091          VALGRIND_CALL_NOREDIR_T9                                 \
5092          "move %0, $2\n"                                          \
5093          : /*out*/   "=r" (_res)                                  \
5094          : /*in*/    "r" (&_argvec[0])                            \
5095          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5096       );                                                          \
5097       lval = (__typeof__(lval)) _res;                             \
5098    } while (0)
5099
5100 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
5101    do {                                                           \
5102       volatile OrigFn        _orig = (orig);                      \
5103       volatile unsigned long _argvec[4];                          \
5104       volatile unsigned long _res;                                \
5105       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5106       _argvec[1] = (unsigned long)(arg1);                         \
5107       _argvec[2] = (unsigned long)(arg2);                         \
5108       _argvec[3] = (unsigned long)(arg3);                         \
5109       __asm__ volatile(                                           \
5110          "ld $4, 8(%1)\n\t"                                       \
5111          "ld $5, 16(%1)\n\t"                                      \
5112          "ld $6, 24(%1)\n\t"                                      \
5113          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5114          VALGRIND_CALL_NOREDIR_T9                                 \
5115          "move %0, $2\n"                                          \
5116          : /*out*/   "=r" (_res)                                  \
5117          : /*in*/    "r" (&_argvec[0])                            \
5118          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5119       );                                                          \
5120       lval = (__typeof__(lval)) _res;                             \
5121    } while (0)
5122
5123 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
5124    do {                                                           \
5125       volatile OrigFn        _orig = (orig);                      \
5126       volatile unsigned long _argvec[5];                          \
5127       volatile unsigned long _res;                                \
5128       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5129       _argvec[1] = (unsigned long)(arg1);                         \
5130       _argvec[2] = (unsigned long)(arg2);                         \
5131       _argvec[3] = (unsigned long)(arg3);                         \
5132       _argvec[4] = (unsigned long)(arg4);                         \
5133       __asm__ volatile(                                           \
5134          "ld $4, 8(%1)\n\t"                                       \
5135          "ld $5, 16(%1)\n\t"                                      \
5136          "ld $6, 24(%1)\n\t"                                      \
5137          "ld $7, 32(%1)\n\t"                                      \
5138          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5139          VALGRIND_CALL_NOREDIR_T9                                 \
5140          "move %0, $2\n"                                          \
5141          : /*out*/   "=r" (_res)                                  \
5142          : /*in*/    "r" (&_argvec[0])                            \
5143          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5144       );                                                          \
5145       lval = (__typeof__(lval)) _res;                             \
5146    } while (0)
5147
5148 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
5149    do {                                                           \
5150       volatile OrigFn        _orig = (orig);                      \
5151       volatile unsigned long _argvec[6];                          \
5152       volatile unsigned long _res;                                \
5153       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5154       _argvec[1] = (unsigned long)(arg1);                         \
5155       _argvec[2] = (unsigned long)(arg2);                         \
5156       _argvec[3] = (unsigned long)(arg3);                         \
5157       _argvec[4] = (unsigned long)(arg4);                         \
5158       _argvec[5] = (unsigned long)(arg5);                         \
5159       __asm__ volatile(                                           \
5160          "ld $4, 8(%1)\n\t"                                       \
5161          "ld $5, 16(%1)\n\t"                                      \
5162          "ld $6, 24(%1)\n\t"                                      \
5163          "ld $7, 32(%1)\n\t"                                      \
5164          "ld $8, 40(%1)\n\t"                                      \
5165          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5166          VALGRIND_CALL_NOREDIR_T9                                 \
5167          "move %0, $2\n"                                          \
5168          : /*out*/   "=r" (_res)                                  \
5169          : /*in*/    "r" (&_argvec[0])                            \
5170          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5171       );                                                          \
5172       lval = (__typeof__(lval)) _res;                             \
5173    } while (0)
5174
5175 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
5176    do {                                                           \
5177       volatile OrigFn        _orig = (orig);                      \
5178       volatile unsigned long _argvec[7];                          \
5179       volatile unsigned long _res;                                \
5180       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5181       _argvec[1] = (unsigned long)(arg1);                         \
5182       _argvec[2] = (unsigned long)(arg2);                         \
5183       _argvec[3] = (unsigned long)(arg3);                         \
5184       _argvec[4] = (unsigned long)(arg4);                         \
5185       _argvec[5] = (unsigned long)(arg5);                         \
5186       _argvec[6] = (unsigned long)(arg6);                         \
5187       __asm__ volatile(                                           \
5188          "ld $4, 8(%1)\n\t"                                       \
5189          "ld $5, 16(%1)\n\t"                                      \
5190          "ld $6, 24(%1)\n\t"                                      \
5191          "ld $7, 32(%1)\n\t"                                      \
5192          "ld $8, 40(%1)\n\t"                                      \
5193          "ld $9, 48(%1)\n\t"                                      \
5194          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5195          VALGRIND_CALL_NOREDIR_T9                                 \
5196          "move %0, $2\n"                                          \
5197          : /*out*/   "=r" (_res)                                  \
5198          : /*in*/    "r" (&_argvec[0])                            \
5199          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5200       );                                                          \
5201       lval = (__typeof__(lval)) _res;                             \
5202    } while (0)
5203
5204 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5205                                  arg7)                            \
5206    do {                                                           \
5207       volatile OrigFn        _orig = (orig);                      \
5208       volatile unsigned long _argvec[8];                          \
5209       volatile unsigned long _res;                                \
5210       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5211       _argvec[1] = (unsigned long)(arg1);                         \
5212       _argvec[2] = (unsigned long)(arg2);                         \
5213       _argvec[3] = (unsigned long)(arg3);                         \
5214       _argvec[4] = (unsigned long)(arg4);                         \
5215       _argvec[5] = (unsigned long)(arg5);                         \
5216       _argvec[6] = (unsigned long)(arg6);                         \
5217       _argvec[7] = (unsigned long)(arg7);                         \
5218       __asm__ volatile(                                           \
5219          "ld $4, 8(%1)\n\t"                                       \
5220          "ld $5, 16(%1)\n\t"                                      \
5221          "ld $6, 24(%1)\n\t"                                      \
5222          "ld $7, 32(%1)\n\t"                                      \
5223          "ld $8, 40(%1)\n\t"                                      \
5224          "ld $9, 48(%1)\n\t"                                      \
5225          "ld $10, 56(%1)\n\t"                                     \
5226          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
5227          VALGRIND_CALL_NOREDIR_T9                                 \
5228          "move %0, $2\n"                                          \
5229          : /*out*/   "=r" (_res)                                  \
5230          : /*in*/    "r" (&_argvec[0])                            \
5231          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5232       );                                                          \
5233       lval = (__typeof__(lval)) _res;                             \
5234    } while (0)
5235
5236 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5237                                  arg7,arg8)                       \
5238    do {                                                           \
5239       volatile OrigFn        _orig = (orig);                      \
5240       volatile unsigned long _argvec[9];                          \
5241       volatile unsigned long _res;                                \
5242       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5243       _argvec[1] = (unsigned long)(arg1);                         \
5244       _argvec[2] = (unsigned long)(arg2);                         \
5245       _argvec[3] = (unsigned long)(arg3);                         \
5246       _argvec[4] = (unsigned long)(arg4);                         \
5247       _argvec[5] = (unsigned long)(arg5);                         \
5248       _argvec[6] = (unsigned long)(arg6);                         \
5249       _argvec[7] = (unsigned long)(arg7);                         \
5250       _argvec[8] = (unsigned long)(arg8);                         \
5251       __asm__ volatile(                                           \
5252          "ld $4, 8(%1)\n\t"                                       \
5253          "ld $5, 16(%1)\n\t"                                      \
5254          "ld $6, 24(%1)\n\t"                                      \
5255          "ld $7, 32(%1)\n\t"                                      \
5256          "ld $8, 40(%1)\n\t"                                      \
5257          "ld $9, 48(%1)\n\t"                                      \
5258          "ld $10, 56(%1)\n\t"                                     \
5259          "ld $11, 64(%1)\n\t"                                     \
5260          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
5261          VALGRIND_CALL_NOREDIR_T9                                 \
5262          "move %0, $2\n"                                          \
5263          : /*out*/   "=r" (_res)                                  \
5264          : /*in*/    "r" (&_argvec[0])                            \
5265          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5266       );                                                          \
5267       lval = (__typeof__(lval)) _res;                             \
5268    } while (0)
5269
5270 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5271                                  arg7,arg8,arg9)                  \
5272    do {                                                           \
5273       volatile OrigFn        _orig = (orig);                      \
5274       volatile unsigned long _argvec[10];                         \
5275       volatile unsigned long _res;                                \
5276       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5277       _argvec[1] = (unsigned long)(arg1);                         \
5278       _argvec[2] = (unsigned long)(arg2);                         \
5279       _argvec[3] = (unsigned long)(arg3);                         \
5280       _argvec[4] = (unsigned long)(arg4);                         \
5281       _argvec[5] = (unsigned long)(arg5);                         \
5282       _argvec[6] = (unsigned long)(arg6);                         \
5283       _argvec[7] = (unsigned long)(arg7);                         \
5284       _argvec[8] = (unsigned long)(arg8);                         \
5285       _argvec[9] = (unsigned long)(arg9);                         \
5286       __asm__ volatile(                                           \
5287          "dsubu $29, $29, 8\n\t"                                  \
5288          "ld $4, 72(%1)\n\t"                                      \
5289          "sd $4, 0($29)\n\t"                                      \
5290          "ld $4, 8(%1)\n\t"                                       \
5291          "ld $5, 16(%1)\n\t"                                      \
5292          "ld $6, 24(%1)\n\t"                                      \
5293          "ld $7, 32(%1)\n\t"                                      \
5294          "ld $8, 40(%1)\n\t"                                      \
5295          "ld $9, 48(%1)\n\t"                                      \
5296          "ld $10, 56(%1)\n\t"                                     \
5297          "ld $11, 64(%1)\n\t"                                     \
5298          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5299          VALGRIND_CALL_NOREDIR_T9                                 \
5300          "daddu $29, $29, 8\n\t"                                  \
5301          "move %0, $2\n"                                          \
5302          : /*out*/   "=r" (_res)                                  \
5303          : /*in*/    "r" (&_argvec[0])                            \
5304          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5305       );                                                          \
5306       lval = (__typeof__(lval)) _res;                             \
5307    } while (0)
5308
5309 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
5310                                   arg7,arg8,arg9,arg10)           \
5311    do {                                                           \
5312       volatile OrigFn        _orig = (orig);                      \
5313       volatile unsigned long _argvec[11];                         \
5314       volatile unsigned long _res;                                \
5315       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5316       _argvec[1] = (unsigned long)(arg1);                         \
5317       _argvec[2] = (unsigned long)(arg2);                         \
5318       _argvec[3] = (unsigned long)(arg3);                         \
5319       _argvec[4] = (unsigned long)(arg4);                         \
5320       _argvec[5] = (unsigned long)(arg5);                         \
5321       _argvec[6] = (unsigned long)(arg6);                         \
5322       _argvec[7] = (unsigned long)(arg7);                         \
5323       _argvec[8] = (unsigned long)(arg8);                         \
5324       _argvec[9] = (unsigned long)(arg9);                         \
5325       _argvec[10] = (unsigned long)(arg10);                       \
5326       __asm__ volatile(                                           \
5327          "dsubu $29, $29, 16\n\t"                                 \
5328          "ld $4, 72(%1)\n\t"                                      \
5329          "sd $4, 0($29)\n\t"                                      \
5330          "ld $4, 80(%1)\n\t"                                      \
5331          "sd $4, 8($29)\n\t"                                      \
5332          "ld $4, 8(%1)\n\t"                                       \
5333          "ld $5, 16(%1)\n\t"                                      \
5334          "ld $6, 24(%1)\n\t"                                      \
5335          "ld $7, 32(%1)\n\t"                                      \
5336          "ld $8, 40(%1)\n\t"                                      \
5337          "ld $9, 48(%1)\n\t"                                      \
5338          "ld $10, 56(%1)\n\t"                                     \
5339          "ld $11, 64(%1)\n\t"                                     \
5340          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5341          VALGRIND_CALL_NOREDIR_T9                                 \
5342          "daddu $29, $29, 16\n\t"                                 \
5343          "move %0, $2\n"                                          \
5344          : /*out*/   "=r" (_res)                                  \
5345          : /*in*/    "r" (&_argvec[0])                            \
5346          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5347       );                                                          \
5348       lval = (__typeof__(lval)) _res;                             \
5349    } while (0)
5350
5351 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5352                                   arg6,arg7,arg8,arg9,arg10,      \
5353                                   arg11)                          \
5354    do {                                                           \
5355       volatile OrigFn        _orig = (orig);                      \
5356       volatile unsigned long _argvec[12];                         \
5357       volatile unsigned long _res;                                \
5358       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5359       _argvec[1] = (unsigned long)(arg1);                         \
5360       _argvec[2] = (unsigned long)(arg2);                         \
5361       _argvec[3] = (unsigned long)(arg3);                         \
5362       _argvec[4] = (unsigned long)(arg4);                         \
5363       _argvec[5] = (unsigned long)(arg5);                         \
5364       _argvec[6] = (unsigned long)(arg6);                         \
5365       _argvec[7] = (unsigned long)(arg7);                         \
5366       _argvec[8] = (unsigned long)(arg8);                         \
5367       _argvec[9] = (unsigned long)(arg9);                         \
5368       _argvec[10] = (unsigned long)(arg10);                       \
5369       _argvec[11] = (unsigned long)(arg11);                       \
5370       __asm__ volatile(                                           \
5371          "dsubu $29, $29, 24\n\t"                                 \
5372          "ld $4, 72(%1)\n\t"                                      \
5373          "sd $4, 0($29)\n\t"                                      \
5374          "ld $4, 80(%1)\n\t"                                      \
5375          "sd $4, 8($29)\n\t"                                      \
5376          "ld $4, 88(%1)\n\t"                                      \
5377          "sd $4, 16($29)\n\t"                                     \
5378          "ld $4, 8(%1)\n\t"                                       \
5379          "ld $5, 16(%1)\n\t"                                      \
5380          "ld $6, 24(%1)\n\t"                                      \
5381          "ld $7, 32(%1)\n\t"                                      \
5382          "ld $8, 40(%1)\n\t"                                      \
5383          "ld $9, 48(%1)\n\t"                                      \
5384          "ld $10, 56(%1)\n\t"                                     \
5385          "ld $11, 64(%1)\n\t"                                     \
5386          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5387          VALGRIND_CALL_NOREDIR_T9                                 \
5388          "daddu $29, $29, 24\n\t"                                 \
5389          "move %0, $2\n"                                          \
5390          : /*out*/   "=r" (_res)                                  \
5391          : /*in*/    "r" (&_argvec[0])                            \
5392          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5393       );                                                          \
5394       lval = (__typeof__(lval)) _res;                             \
5395    } while (0)
5396
5397 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5398                                   arg6,arg7,arg8,arg9,arg10,      \
5399                                   arg11,arg12)                    \
5400    do {                                                           \
5401       volatile OrigFn        _orig = (orig);                      \
5402       volatile unsigned long _argvec[13];                         \
5403       volatile unsigned long _res;                                \
5404       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5405       _argvec[1] = (unsigned long)(arg1);                         \
5406       _argvec[2] = (unsigned long)(arg2);                         \
5407       _argvec[3] = (unsigned long)(arg3);                         \
5408       _argvec[4] = (unsigned long)(arg4);                         \
5409       _argvec[5] = (unsigned long)(arg5);                         \
5410       _argvec[6] = (unsigned long)(arg6);                         \
5411       _argvec[7] = (unsigned long)(arg7);                         \
5412       _argvec[8] = (unsigned long)(arg8);                         \
5413       _argvec[9] = (unsigned long)(arg9);                         \
5414       _argvec[10] = (unsigned long)(arg10);                       \
5415       _argvec[11] = (unsigned long)(arg11);                       \
5416       _argvec[12] = (unsigned long)(arg12);                       \
5417       __asm__ volatile(                                           \
5418          "dsubu $29, $29, 32\n\t"                                 \
5419          "ld $4, 72(%1)\n\t"                                      \
5420          "sd $4, 0($29)\n\t"                                      \
5421          "ld $4, 80(%1)\n\t"                                      \
5422          "sd $4, 8($29)\n\t"                                      \
5423          "ld $4, 88(%1)\n\t"                                      \
5424          "sd $4, 16($29)\n\t"                                     \
5425          "ld $4, 96(%1)\n\t"                                      \
5426          "sd $4, 24($29)\n\t"                                     \
5427          "ld $4, 8(%1)\n\t"                                       \
5428          "ld $5, 16(%1)\n\t"                                      \
5429          "ld $6, 24(%1)\n\t"                                      \
5430          "ld $7, 32(%1)\n\t"                                      \
5431          "ld $8, 40(%1)\n\t"                                      \
5432          "ld $9, 48(%1)\n\t"                                      \
5433          "ld $10, 56(%1)\n\t"                                     \
5434          "ld $11, 64(%1)\n\t"                                     \
5435          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5436          VALGRIND_CALL_NOREDIR_T9                                 \
5437          "daddu $29, $29, 32\n\t"                                 \
5438          "move %0, $2\n"                                          \
5439          : /*out*/   "=r" (_res)                                  \
5440          : /*in*/    "r" (&_argvec[0])                            \
5441          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5442       );                                                          \
5443       lval = (__typeof__(lval)) _res;                             \
5444    } while (0)
5445
5446 #endif /* PLAT_mips64_linux */
5447
5448
5449 /* ------------------------------------------------------------------ */
5450 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
5451 /*                                                                    */
5452 /* ------------------------------------------------------------------ */
5453
5454 /* Some request codes.  There are many more of these, but most are not
5455    exposed to end-user view.  These are the public ones, all of the
5456    form 0x1000 + small_number.
5457
5458    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
5459    ones start at 0x2000.
5460 */
5461
5462 /* These macros are used by tools -- they must be public, but don't
5463    embed them into other programs. */
5464 #define VG_USERREQ_TOOL_BASE(a,b) \
5465    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
5466 #define VG_IS_TOOL_USERREQ(a, b, v) \
5467    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
5468
5469 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
5470    This enum comprises an ABI exported by Valgrind to programs
5471    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
5472    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
5473 typedef
5474    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
5475           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
5476
5477           /* These allow any function to be called from the simulated
5478              CPU but run on the real CPU.  Nb: the first arg passed to
5479              the function is always the ThreadId of the running
5480              thread!  So CLIENT_CALL0 actually requires a 1 arg
5481              function, etc. */
5482           VG_USERREQ__CLIENT_CALL0 = 0x1101,
5483           VG_USERREQ__CLIENT_CALL1 = 0x1102,
5484           VG_USERREQ__CLIENT_CALL2 = 0x1103,
5485           VG_USERREQ__CLIENT_CALL3 = 0x1104,
5486
5487           /* Can be useful in regression testing suites -- eg. can
5488              send Valgrind's output to /dev/null and still count
5489              errors. */
5490           VG_USERREQ__COUNT_ERRORS = 0x1201,
5491
5492           /* Allows the client program and/or gdbserver to execute a monitor
5493              command. */
5494           VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
5495
5496           /* These are useful and can be interpreted by any tool that
5497              tracks malloc() et al, by using vg_replace_malloc.c. */
5498           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
5499           VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
5500           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
5501           /* Memory pool support. */
5502           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
5503           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
5504           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
5505           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
5506           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
5507           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
5508           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
5509           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
5510
5511           /* Allow printfs to valgrind log. */
5512           /* The first two pass the va_list argument by value, which
5513              assumes it is the same size as or smaller than a UWord,
5514              which generally isn't the case.  Hence are deprecated.
5515              The second two pass the vargs by reference and so are
5516              immune to this problem. */
5517           /* both :: char* fmt, va_list vargs (DEPRECATED) */
5518           VG_USERREQ__PRINTF           = 0x1401,
5519           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
5520           /* both :: char* fmt, va_list* vargs */
5521           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
5522           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
5523
5524           /* Stack support. */
5525           VG_USERREQ__STACK_REGISTER   = 0x1501,
5526           VG_USERREQ__STACK_DEREGISTER = 0x1502,
5527           VG_USERREQ__STACK_CHANGE     = 0x1503,
5528
5529           /* Wine support */
5530           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
5531
5532           /* Querying of debug info. */
5533           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701,
5534
5535           /* Disable/enable error reporting level.  Takes a single
5536              Word arg which is the delta to this thread's error
5537              disablement indicator.  Hence 1 disables or further
5538              disables errors, and -1 moves back towards enablement.
5539              Other values are not allowed. */
5540           VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801,
5541
5542           /* Initialise IR injection */
5543           VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901
5544    } Vg_ClientRequest;
5545
5546 #if !defined(__GNUC__)
5547 #  define __extension__ /* */
5548 #endif
5549
5550
5551 /* Returns the number of Valgrinds this code is running under.  That
5552    is, 0 if running natively, 1 if running under Valgrind, 2 if
5553    running under Valgrind which is running under another Valgrind,
5554    etc. */
5555 #define RUNNING_ON_VALGRIND                                           \
5556     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,         \
5557                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
5558                                     0, 0, 0, 0, 0)                    \
5559
5560
5561 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
5562    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
5563    since it provides a way to make sure valgrind will retranslate the
5564    invalidated area.  Returns no value. */
5565 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)              \
5566     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS,  \
5567                                     _qzz_addr, _qzz_len, 0, 0, 0)
5568
5569
5570 /* These requests are for getting Valgrind itself to print something.
5571    Possibly with a backtrace.  This is a really ugly hack.  The return value
5572    is the number of characters printed, excluding the "**<pid>** " part at the
5573    start and the backtrace (if present). */
5574
5575 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
5576 /* Modern GCC will optimize the static routine out if unused,
5577    and unused attribute will shut down warnings about it.  */
5578 static int VALGRIND_PRINTF(const char *format, ...)
5579    __attribute__((format(__printf__, 1, 2), __unused__));
5580 #endif
5581 static int
5582 #if defined(_MSC_VER)
5583 __inline
5584 #endif
5585 VALGRIND_PRINTF(const char *format, ...)
5586 {
5587 #if defined(NVALGRIND)
5588    return 0;
5589 #else /* NVALGRIND */
5590 #if defined(_MSC_VER) || defined(__MINGW64__)
5591    uintptr_t _qzz_res;
5592 #else
5593    unsigned long _qzz_res;
5594 #endif
5595    va_list vargs;
5596    va_start(vargs, format);
5597 #if defined(_MSC_VER) || defined(__MINGW64__)
5598    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5599                               VG_USERREQ__PRINTF_VALIST_BY_REF,
5600                               (uintptr_t)format,
5601                               (uintptr_t)&vargs,
5602                               0, 0, 0);
5603 #else
5604    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5605                               VG_USERREQ__PRINTF_VALIST_BY_REF,
5606                               (unsigned long)format,
5607                               (unsigned long)&vargs,
5608                               0, 0, 0);
5609 #endif
5610    va_end(vargs);
5611    return (int)_qzz_res;
5612 #endif /* NVALGRIND */
5613 }
5614
5615 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
5616 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
5617    __attribute__((format(__printf__, 1, 2), __unused__));
5618 #endif
5619 static int
5620 #if defined(_MSC_VER)
5621 __inline
5622 #endif
5623 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
5624 {
5625 #if defined(NVALGRIND)
5626    return 0;
5627 #else /* NVALGRIND */
5628 #if defined(_MSC_VER) || defined(__MINGW64__)
5629    uintptr_t _qzz_res;
5630 #else
5631    unsigned long _qzz_res;
5632 #endif
5633    va_list vargs;
5634    va_start(vargs, format);
5635 #if defined(_MSC_VER) || defined(__MINGW64__)
5636    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5637                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5638                               (uintptr_t)format,
5639                               (uintptr_t)&vargs,
5640                               0, 0, 0);
5641 #else
5642    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5643                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5644                               (unsigned long)format,
5645                               (unsigned long)&vargs,
5646                               0, 0, 0);
5647 #endif
5648    va_end(vargs);
5649    return (int)_qzz_res;
5650 #endif /* NVALGRIND */
5651 }
5652
5653
5654 /* These requests allow control to move from the simulated CPU to the
5655    real CPU, calling an arbitary function.
5656
5657    Note that the current ThreadId is inserted as the first argument.
5658    So this call:
5659
5660      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
5661
5662    requires f to have this signature:
5663
5664      Word f(Word tid, Word arg1, Word arg2)
5665
5666    where "Word" is a word-sized type.
5667
5668    Note that these client requests are not entirely reliable.  For example,
5669    if you call a function with them that subsequently calls printf(),
5670    there's a high chance Valgrind will crash.  Generally, your prospects of
5671    these working are made higher if the called function does not refer to
5672    any global variables, and does not refer to any libc or other functions
5673    (printf et al).  Any kind of entanglement with libc or dynamic linking is
5674    likely to have a bad outcome, for tricky reasons which we've grappled
5675    with a lot in the past.
5676 */
5677 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
5678     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,       \
5679                                     VG_USERREQ__CLIENT_CALL0,     \
5680                                     _qyy_fn,                      \
5681                                     0, 0, 0, 0)
5682
5683 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)                    \
5684     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
5685                                     VG_USERREQ__CLIENT_CALL1,          \
5686                                     _qyy_fn,                           \
5687                                     _qyy_arg1, 0, 0, 0)
5688
5689 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)         \
5690     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
5691                                     VG_USERREQ__CLIENT_CALL2,          \
5692                                     _qyy_fn,                           \
5693                                     _qyy_arg1, _qyy_arg2, 0, 0)
5694
5695 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
5696     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,             \
5697                                     VG_USERREQ__CLIENT_CALL3,           \
5698                                     _qyy_fn,                            \
5699                                     _qyy_arg1, _qyy_arg2,               \
5700                                     _qyy_arg3, 0)
5701
5702
5703 /* Counts the number of errors that have been recorded by a tool.  Nb:
5704    the tool must record the errors with VG_(maybe_record_error)() or
5705    VG_(unique_error)() for them to be counted. */
5706 #define VALGRIND_COUNT_ERRORS                                     \
5707     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(                    \
5708                                0 /* default return */,            \
5709                                VG_USERREQ__COUNT_ERRORS,          \
5710                                0, 0, 0, 0, 0)
5711
5712 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
5713    when heap blocks are allocated in order to give accurate results.  This
5714    happens automatically for the standard allocator functions such as
5715    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
5716    delete[], etc.
5717
5718    But if your program uses a custom allocator, this doesn't automatically
5719    happen, and Valgrind will not do as well.  For example, if you allocate
5720    superblocks with mmap() and then allocates chunks of the superblocks, all
5721    Valgrind's observations will be at the mmap() level and it won't know that
5722    the chunks should be considered separate entities.  In Memcheck's case,
5723    that means you probably won't get heap block overrun detection (because
5724    there won't be redzones marked as unaddressable) and you definitely won't
5725    get any leak detection.
5726
5727    The following client requests allow a custom allocator to be annotated so
5728    that it can be handled accurately by Valgrind.
5729
5730    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
5731    by a malloc()-like function.  For Memcheck (an illustrative case), this
5732    does two things:
5733
5734    - It records that the block has been allocated.  This means any addresses
5735      within the block mentioned in error messages will be
5736      identified as belonging to the block.  It also means that if the block
5737      isn't freed it will be detected by the leak checker.
5738
5739    - It marks the block as being addressable and undefined (if 'is_zeroed' is
5740      not set), or addressable and defined (if 'is_zeroed' is set).  This
5741      controls how accesses to the block by the program are handled.
5742
5743    'addr' is the start of the usable block (ie. after any
5744    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
5745    can apply redzones -- these are blocks of padding at the start and end of
5746    each block.  Adding redzones is recommended as it makes it much more likely
5747    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
5748    zeroed (or filled with another predictable value), as is the case for
5749    calloc().
5750
5751    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
5752    heap block -- that will be used by the client program -- is allocated.
5753    It's best to put it at the outermost level of the allocator if possible;
5754    for example, if you have a function my_alloc() which calls
5755    internal_alloc(), and the client request is put inside internal_alloc(),
5756    stack traces relating to the heap block will contain entries for both
5757    my_alloc() and internal_alloc(), which is probably not what you want.
5758
5759    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
5760    custom blocks from within a heap block, B, that has been allocated with
5761    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
5762    -- the custom blocks will take precedence.
5763
5764    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
5765    Memcheck, it does two things:
5766
5767    - It records that the block has been deallocated.  This assumes that the
5768      block was annotated as having been allocated via
5769      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
5770
5771    - It marks the block as being unaddressable.
5772
5773    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
5774    heap block is deallocated.
5775
5776    VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
5777    Memcheck, it does four things:
5778
5779    - It records that the size of a block has been changed.  This assumes that
5780      the block was annotated as having been allocated via
5781      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
5782
5783    - If the block shrunk, it marks the freed memory as being unaddressable.
5784
5785    - If the block grew, it marks the new area as undefined and defines a red
5786      zone past the end of the new block.
5787
5788    - The V-bits of the overlap between the old and the new block are preserved.
5789
5790    VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
5791    and before deallocation of the old block.
5792
5793    In many cases, these three client requests will not be enough to get your
5794    allocator working well with Memcheck.  More specifically, if your allocator
5795    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
5796    will be necessary to mark the memory as addressable just before the zeroing
5797    occurs, otherwise you'll get a lot of invalid write errors.  For example,
5798    you'll need to do this if your allocator recycles freed blocks, but it
5799    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
5800    Alternatively, if your allocator reuses freed blocks for allocator-internal
5801    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
5802
5803    Really, what's happening is a blurring of the lines between the client
5804    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
5805    memory should be considered unaddressable to the client program, but the
5806    allocator knows more than the rest of the client program and so may be able
5807    to safely access it.  Extra client requests are necessary for Valgrind to
5808    understand the distinction between the allocator and the rest of the
5809    program.
5810
5811    Ignored if addr == 0.
5812 */
5813 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)          \
5814     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK,       \
5815                                     addr, sizeB, rzB, is_zeroed, 0)
5816
5817 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
5818    Ignored if addr == 0.
5819 */
5820 #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)     \
5821     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK,    \
5822                                     addr, oldSizeB, newSizeB, rzB, 0)
5823
5824 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
5825    Ignored if addr == 0.
5826 */
5827 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                              \
5828     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK,         \
5829                                     addr, rzB, 0, 0, 0)
5830
5831 /* Create a memory pool. */
5832 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
5833     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,   \
5834                                     pool, rzB, is_zeroed, 0, 0)
5835
5836 /* Destroy a memory pool. */
5837 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
5838     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL,  \
5839                                     pool, 0, 0, 0, 0)
5840
5841 /* Associate a piece of memory with a memory pool. */
5842 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
5843     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC,    \
5844                                     pool, addr, size, 0, 0)
5845
5846 /* Disassociate a piece of memory from a memory pool. */
5847 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
5848     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE,     \
5849                                     pool, addr, 0, 0, 0)
5850
5851 /* Disassociate any pieces outside a particular range. */
5852 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
5853     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM,     \
5854                                     pool, addr, size, 0, 0)
5855
5856 /* Resize and/or move a piece associated with a memory pool. */
5857 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
5858     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL,     \
5859                                     poolA, poolB, 0, 0, 0)
5860
5861 /* Resize and/or move a piece associated with a memory pool. */
5862 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
5863     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE,   \
5864                                     pool, addrA, addrB, size, 0)
5865
5866 /* Return 1 if a mempool exists, else 0. */
5867 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
5868     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5869                                VG_USERREQ__MEMPOOL_EXISTS,        \
5870                                pool, 0, 0, 0, 0)
5871
5872 /* Mark a piece of memory as being a stack. Returns a stack id. */
5873 #define VALGRIND_STACK_REGISTER(start, end)                       \
5874     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5875                                VG_USERREQ__STACK_REGISTER,        \
5876                                start, end, 0, 0, 0)
5877
5878 /* Unmark the piece of memory associated with a stack id as being a
5879    stack. */
5880 #define VALGRIND_STACK_DEREGISTER(id)                             \
5881     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \
5882                                     id, 0, 0, 0, 0)
5883
5884 /* Change the start and end address of the stack id. */
5885 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
5886     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE,     \
5887                                     id, start, end, 0, 0)
5888
5889 /* Load PDB debug info for Wine PE image_map. */
5890 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)     \
5891     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \
5892                                     fd, ptr, total_size, delta, 0)
5893
5894 /* Map a code address to a source file name and line number.  buf64
5895    must point to a 64-byte buffer in the caller's address space.  The
5896    result will be dumped in there and is guaranteed to be zero
5897    terminated.  If no info is found, the first byte is set to zero. */
5898 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
5899     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5900                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
5901                                addr, buf64, 0, 0, 0)
5902
5903 /* Disable error reporting for this thread.  Behaves in a stack like
5904    way, so you can safely call this multiple times provided that
5905    VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times
5906    to re-enable reporting.  The first call of this macro disables
5907    reporting.  Subsequent calls have no effect except to increase the
5908    number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable
5909    reporting.  Child threads do not inherit this setting from their
5910    parents -- they are always created with reporting enabled. */
5911 #define VALGRIND_DISABLE_ERROR_REPORTING                                \
5912     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
5913                                     1, 0, 0, 0, 0)
5914
5915 /* Re-enable error reporting, as per comments on
5916    VALGRIND_DISABLE_ERROR_REPORTING. */
5917 #define VALGRIND_ENABLE_ERROR_REPORTING                                 \
5918     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
5919                                     -1, 0, 0, 0, 0)
5920
5921 /* Execute a monitor command from the client program.
5922    If a connection is opened with GDB, the output will be sent
5923    according to the output mode set for vgdb.
5924    If no connection is opened, output will go to the log output.
5925    Returns 1 if command not recognised, 0 otherwise. */
5926 #define VALGRIND_MONITOR_COMMAND(command)                               \
5927    VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \
5928                                    command, 0, 0, 0, 0)
5929
5930
5931 #undef PLAT_x86_darwin
5932 #undef PLAT_amd64_darwin
5933 #undef PLAT_x86_win32
5934 #undef PLAT_amd64_win64
5935 #undef PLAT_x86_linux
5936 #undef PLAT_amd64_linux
5937 #undef PLAT_ppc32_linux
5938 #undef PLAT_ppc64_linux
5939 #undef PLAT_arm_linux
5940 #undef PLAT_s390x_linux
5941 #undef PLAT_mips32_linux
5942 #undef PLAT_mips64_linux
5943
5944 #endif   /* __VALGRIND_H */