build/valgrind/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2010 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76
  77 /* ------------------------------------------------------------------ */
  78 /* VERSION NUMBER OF VALGRIND                                         */
  79 /* ------------------------------------------------------------------ */
  80
  81 /* Specify Valgrind's version number, so that user code can
  82    conditionally compile based on our version number.  Note that these
  83    were introduced at version 3.6 and so do not exist in version 3.5
  84    or earlier.  The recommended way to use them to check for "version
  85    X.Y or later" is (eg)
  86
  87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  88     && (__VALGRIND_MAJOR__ > 3                                   \
  89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  90 */
  91 #define __VALGRIND_MAJOR__    3
  92 #define __VALGRIND_MINOR__    6
  93
  94
  95 #include <stdarg.h>
  96
  97 /* Nb: this file might be included in a file compiled with -ansi.  So
  98    we can't use C++ style "//" comments nor the "asm" keyword (instead
  99    use "__asm__"). */
 100
 101 /* Derive some tags indicating what the target platform is.  Note
 102    that in this file we're using the compiler's CPP symbols for
 103    identifying architectures, which are different to the ones we use
 104    within the rest of Valgrind.  Note, __powerpc__ is active for both
 105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 106    latter (on Linux, that is).
 107
 108    Misc note: how to find out what's predefined in gcc by default:
 109    gcc -Wp,-dM somefile.c
 110 */
 111 #undef PLAT_ppc64_aix5
 112 #undef PLAT_ppc32_aix5
 113 #undef PLAT_x86_darwin
 114 #undef PLAT_amd64_darwin
 115 #undef PLAT_x86_win32
 116 #undef PLAT_x86_linux
 117 #undef PLAT_amd64_linux
 118 #undef PLAT_ppc32_linux
 119 #undef PLAT_ppc64_linux
 120 #undef PLAT_arm_linux
 121 #undef PLAT_s390x_linux
 122
 123
 124 #if defined(_AIX) && defined(__64BIT__)
 125 #  define PLAT_ppc64_aix5 1
 126 #elif defined(_AIX) && !defined(__64BIT__)
 127 #  define PLAT_ppc32_aix5 1
 128 #elif defined(__APPLE__) && defined(__i386__)
 129 #  define PLAT_x86_darwin 1
 130 #elif defined(__APPLE__) && defined(__x86_64__)
 131 #  define PLAT_amd64_darwin 1
 132 #elif defined(__MINGW32__) || defined(__CYGWIN32__) || defined(_WIN32) && defined(_M_IX86)
 133 #  define PLAT_x86_win32 1
 134 #elif defined(__linux__) && defined(__i386__)
 135 #  define PLAT_x86_linux 1
 136 #elif defined(__linux__) && defined(__x86_64__)
 137 #  define PLAT_amd64_linux 1
 138 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 139 #  define PLAT_ppc32_linux 1
 140 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 141 #  define PLAT_ppc64_linux 1
 142 #elif defined(__linux__) && defined(__arm__)
 143 #  define PLAT_arm_linux 1
 144 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
 145 #  define PLAT_s390x_linux 1
 146 #else
 147 /* If we're not compiling for our target platform, don't generate
 148    any inline asms.  */
 149 #  if !defined(NVALGRIND)
 150 #    define NVALGRIND 1
 151 #  endif
 152 #endif
 153
 154
 155 /* ------------------------------------------------------------------ */
 156 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 157 /* in here of use to end-users -- skip to the next section.           */
 158 /* ------------------------------------------------------------------ */
 159
 160 #if defined(NVALGRIND)
 161
 162 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 163    from the compiled code (analogous to NDEBUG's effects on
 164    assert()) */
 165 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 166         _zzq_rlval, _zzq_default, _zzq_request,                   \
 167         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 168    {                                                              \
 169       (_zzq_rlval) = (_zzq_default);                              \
 170    }
 171
 172 #else  /* ! NVALGRIND */
 173
 174 /* The following defines the magic code sequences which the JITter
 175    spots and handles magically.  Don't look too closely at them as
 176    they will rot your brain.
 177
 178    The assembly code sequences for all architectures is in this one
 179    file.  This is because this file must be stand-alone, and we don't
 180    want to have multiple files.
 181
 182    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 183    value gets put in the return slot, so that everything works when
 184    this is executed not under Valgrind.  Args are passed in a memory
 185    block, and so there's no intrinsic limit to the number that could
 186    be passed, but it's currently five.
 187
 188    The macro args are:
 189       _zzq_rlval    result lvalue
 190       _zzq_default  default value (result returned when running on real CPU)
 191       _zzq_request  request code
 192       _zzq_arg1..5  request params
 193
 194    The other two macros are used to support function wrapping, and are
 195    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 196    guest's NRADDR pseudo-register and whatever other information is
 197    needed to safely run the call original from the wrapper: on
 198    ppc64-linux, the R2 value at the divert point is also needed.  This
 199    information is abstracted into a user-visible type, OrigFn.
 200
 201    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 202    guest, but guarantees that the branch instruction will not be
 203    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 204    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 205    complete inline asm, since it needs to be combined with more magic
 206    inline asm stuff to be useful.
 207 */
 208
 209 /* ------------------------- x86-{linux,darwin} ---------------- */
 210
 211 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
 212     ||  (defined(PLAT_x86_win32) && defined(__GNUC__))
 213
 214 typedef
 215    struct {
 216       unsigned int nraddr; /* where's the code? */
 217    }
 218    OrigFn;
 219
 220 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 221                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 222                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 223
 224 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 225         _zzq_rlval, _zzq_default, _zzq_request,                   \
 226         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 227   { volatile unsigned int _zzq_args[6];                           \
 228     volatile unsigned int _zzq_result;                            \
 229     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 230     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 231     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 232     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 233     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 234     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 235     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 236                      /* %EDX = client_request ( %EAX ) */         \
 237                      "xchgl %%ebx,%%ebx"                          \
 238                      : "=d" (_zzq_result)                         \
 239                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 240                      : "cc", "memory"                             \
 241                     );                                            \
 242     _zzq_rlval = _zzq_result;                                     \
 243   }
 244
 245 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 246   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 247     volatile unsigned int __addr;                                 \
 248     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 249                      /* %EAX = guest_NRADDR */                    \
 250                      "xchgl %%ecx,%%ecx"                          \
 251                      : "=a" (__addr)                              \
 252                      :                                            \
 253                      : "cc", "memory"                             \
 254                     );                                            \
 255     _zzq_orig->nraddr = __addr;                                   \
 256   }
 257
 258 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 259                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 260                      /* call-noredir *%EAX */                     \
 261                      "xchgl %%edx,%%edx\n\t"
 262 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) */
 263
 264 /* ------------------------- x86-Win32 ------------------------- */
 265
 266 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
 267
 268 typedef
 269    struct {
 270       unsigned int nraddr; /* where's the code? */
 271    }
 272    OrigFn;
 273
 274 #if defined(_MSC_VER)
 275
 276 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 277                      __asm rol edi, 3  __asm rol edi, 13          \
 278                      __asm rol edi, 29 __asm rol edi, 19
 279
 280 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 281         _zzq_rlval, _zzq_default, _zzq_request,                   \
 282         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 283   { volatile uintptr_t _zzq_args[6];                              \
 284     volatile unsigned int _zzq_result;                            \
 285     _zzq_args[0] = (uintptr_t)(_zzq_request);                     \
 286     _zzq_args[1] = (uintptr_t)(_zzq_arg1);                        \
 287     _zzq_args[2] = (uintptr_t)(_zzq_arg2);                        \
 288     _zzq_args[3] = (uintptr_t)(_zzq_arg3);                        \
 289     _zzq_args[4] = (uintptr_t)(_zzq_arg4);                        \
 290     _zzq_args[5] = (uintptr_t)(_zzq_arg5);                        \
 291     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default  \
 292             __SPECIAL_INSTRUCTION_PREAMBLE                        \
 293             /* %EDX = client_request ( %EAX ) */                  \
 294             __asm xchg ebx,ebx                                    \
 295             __asm mov _zzq_result, edx                            \
 296     }                                                             \
 297     _zzq_rlval = _zzq_result;                                     \
 298   }
 299
 300 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 301   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 302     volatile unsigned int __addr;                                 \
 303     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
 304             /* %EAX = guest_NRADDR */                             \
 305             __asm xchg ecx,ecx                                    \
 306             __asm mov __addr, eax                                 \
 307     }                                                             \
 308     _zzq_orig->nraddr = __addr;                                   \
 309   }
 310
 311 #define VALGRIND_CALL_NOREDIR_EAX ERROR
 312
 313 #else
 314 #error Unsupported compiler.
 315 #endif
 316
 317 #endif /* PLAT_x86_win32 */
 318
 319 /* ------------------------ amd64-{linux,darwin} --------------- */
 320
 321 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 322
 323 typedef
 324    struct {
 325       unsigned long long int nraddr; /* where's the code? */
 326    }
 327    OrigFn;
 328
 329 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 330                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 331                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 332
 333 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 334         _zzq_rlval, _zzq_default, _zzq_request,                   \
 335         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 336   { volatile unsigned long long int _zzq_args[6];                 \
 337     volatile unsigned long long int _zzq_result;                  \
 338     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 339     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 340     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 341     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 342     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 343     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 344     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 345                      /* %RDX = client_request ( %RAX ) */         \
 346                      "xchgq %%rbx,%%rbx"                          \
 347                      : "=d" (_zzq_result)                         \
 348                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 349                      : "cc", "memory"                             \
 350                     );                                            \
 351     _zzq_rlval = _zzq_result;                                     \
 352   }
 353
 354 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 355   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 356     volatile unsigned long long int __addr;                       \
 357     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 358                      /* %RAX = guest_NRADDR */                    \
 359                      "xchgq %%rcx,%%rcx"                          \
 360                      : "=a" (__addr)                              \
 361                      :                                            \
 362                      : "cc", "memory"                             \
 363                     );                                            \
 364     _zzq_orig->nraddr = __addr;                                   \
 365   }
 366
 367 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 368                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 369                      /* call-noredir *%RAX */                     \
 370                      "xchgq %%rdx,%%rdx\n\t"
 371 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 372
 373 /* ------------------------ ppc32-linux ------------------------ */
 374
 375 #if defined(PLAT_ppc32_linux)
 376
 377 typedef
 378    struct {
 379       unsigned int nraddr; /* where's the code? */
 380    }
 381    OrigFn;
 382
 383 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 384                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 385                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 386
 387 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 388         _zzq_rlval, _zzq_default, _zzq_request,                   \
 389         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 390                                                                   \
 391   {          unsigned int  _zzq_args[6];                          \
 392              unsigned int  _zzq_result;                           \
 393              unsigned int* _zzq_ptr;                              \
 394     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 395     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 396     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 397     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 398     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 399     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 400     _zzq_ptr = _zzq_args;                                         \
 401     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 402                      "mr 4,%2\n\t" /*ptr*/                        \
 403                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 404                      /* %R3 = client_request ( %R4 ) */           \
 405                      "or 1,1,1\n\t"                               \
 406                      "mr %0,3"     /*result*/                     \
 407                      : "=b" (_zzq_result)                         \
 408                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 409                      : "cc", "memory", "r3", "r4");               \
 410     _zzq_rlval = _zzq_result;                                     \
 411   }
 412
 413 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 414   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 415     unsigned int __addr;                                          \
 416     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 417                      /* %R3 = guest_NRADDR */                     \
 418                      "or 2,2,2\n\t"                               \
 419                      "mr %0,3"                                    \
 420                      : "=b" (__addr)                              \
 421                      :                                            \
 422                      : "cc", "memory", "r3"                       \
 423                     );                                            \
 424     _zzq_orig->nraddr = __addr;                                   \
 425   }
 426
 427 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 428                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 429                      /* branch-and-link-to-noredir *%R11 */       \
 430                      "or 3,3,3\n\t"
 431 #endif /* PLAT_ppc32_linux */
 432
 433 /* ------------------------ ppc64-linux ------------------------ */
 434
 435 #if defined(PLAT_ppc64_linux)
 436
 437 typedef
 438    struct {
 439       unsigned long long int nraddr; /* where's the code? */
 440       unsigned long long int r2;  /* what tocptr do we need? */
 441    }
 442    OrigFn;
 443
 444 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 445                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 446                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 447
 448 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 449         _zzq_rlval, _zzq_default, _zzq_request,                   \
 450         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 451                                                                   \
 452   {          unsigned long long int  _zzq_args[6];                \
 453     register unsigned long long int  _zzq_result __asm__("r3");   \
 454     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 455     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 456     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 457     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 458     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 459     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 460     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 461     _zzq_ptr = _zzq_args;                                         \
 462     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 463                      /* %R3 = client_request ( %R4 ) */           \
 464                      "or 1,1,1"                                   \
 465                      : "=r" (_zzq_result)                         \
 466                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 467                      : "cc", "memory");                           \
 468     _zzq_rlval = _zzq_result;                                     \
 469   }
 470
 471 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 472   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 473     register unsigned long long int __addr __asm__("r3");         \
 474     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 475                      /* %R3 = guest_NRADDR */                     \
 476                      "or 2,2,2"                                   \
 477                      : "=r" (__addr)                              \
 478                      :                                            \
 479                      : "cc", "memory"                             \
 480                     );                                            \
 481     _zzq_orig->nraddr = __addr;                                   \
 482     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 483                      /* %R3 = guest_NRADDR_GPR2 */                \
 484                      "or 4,4,4"                                   \
 485                      : "=r" (__addr)                              \
 486                      :                                            \
 487                      : "cc", "memory"                             \
 488                     );                                            \
 489     _zzq_orig->r2 = __addr;                                       \
 490   }
 491
 492 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 493                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 494                      /* branch-and-link-to-noredir *%R11 */       \
 495                      "or 3,3,3\n\t"
 496
 497 #endif /* PLAT_ppc64_linux */
 498
 499 /* ------------------------- arm-linux ------------------------- */
 500
 501 #if defined(PLAT_arm_linux)
 502
 503 typedef
 504    struct {
 505       unsigned int nraddr; /* where's the code? */
 506    }
 507    OrigFn;
 508
 509 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 510             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 511             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 512
 513 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 514         _zzq_rlval, _zzq_default, _zzq_request,                   \
 515         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 516                                                                   \
 517   { volatile unsigned int  _zzq_args[6];                          \
 518     volatile unsigned int  _zzq_result;                           \
 519     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 520     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 521     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 522     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 523     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 524     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 525     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 526                      "mov r4, %2\n\t" /*ptr*/                     \
 527                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 528                      /* R3 = client_request ( R4 ) */             \
 529                      "orr r10, r10, r10\n\t"                      \
 530                      "mov %0, r3"     /*result*/                  \
 531                      : "=r" (_zzq_result)                         \
 532                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 533                      : "cc","memory", "r3", "r4");                \
 534     _zzq_rlval = _zzq_result;                                     \
 535   }
 536
 537 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 538   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 539     unsigned int __addr;                                          \
 540     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 541                      /* R3 = guest_NRADDR */                      \
 542                      "orr r11, r11, r11\n\t"                      \
 543                      "mov %0, r3"                                 \
 544                      : "=r" (__addr)                              \
 545                      :                                            \
 546                      : "cc", "memory", "r3"                       \
 547                     );                                            \
 548     _zzq_orig->nraddr = __addr;                                   \
 549   }
 550
 551 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 552                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 553                      /* branch-and-link-to-noredir *%R4 */        \
 554                      "orr r12, r12, r12\n\t"
 555
 556 #endif /* PLAT_arm_linux */
 557
 558 /* ------------------------ ppc32-aix5 ------------------------- */
 559
 560 #if defined(PLAT_ppc32_aix5)
 561
 562 typedef
 563    struct {
 564       unsigned int nraddr; /* where's the code? */
 565       unsigned int r2;  /* what tocptr do we need? */
 566    }
 567    OrigFn;
 568
 569 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 570                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 571                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 572
 573 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 574         _zzq_rlval, _zzq_default, _zzq_request,                   \
 575         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 576                                                                   \
 577   {          unsigned int  _zzq_args[7];                          \
 578     register unsigned int  _zzq_result;                           \
 579     register unsigned int* _zzq_ptr;                              \
 580     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 581     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 582     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 583     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 584     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 585     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 586     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 587     _zzq_ptr = _zzq_args;                                         \
 588     __asm__ volatile("mr 4,%1\n\t"                                \
 589                      "lwz 3, 24(4)\n\t"                           \
 590                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 591                      /* %R3 = client_request ( %R4 ) */           \
 592                      "or 1,1,1\n\t"                               \
 593                      "mr %0,3"                                    \
 594                      : "=b" (_zzq_result)                         \
 595                      : "b" (_zzq_ptr)                             \
 596                      : "r3", "r4", "cc", "memory");               \
 597     _zzq_rlval = _zzq_result;                                     \
 598   }
 599
 600 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 601   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 602     register unsigned int __addr;                                 \
 603     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 604                      /* %R3 = guest_NRADDR */                     \
 605                      "or 2,2,2\n\t"                               \
 606                      "mr %0,3"                                    \
 607                      : "=b" (__addr)                              \
 608                      :                                            \
 609                      : "r3", "cc", "memory"                       \
 610                     );                                            \
 611     _zzq_orig->nraddr = __addr;                                   \
 612     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 613                      /* %R3 = guest_NRADDR_GPR2 */                \
 614                      "or 4,4,4\n\t"                               \
 615                      "mr %0,3"                                    \
 616                      : "=b" (__addr)                              \
 617                      :                                            \
 618                      : "r3", "cc", "memory"                       \
 619                     );                                            \
 620     _zzq_orig->r2 = __addr;                                       \
 621   }
 622
 623 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 624                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 625                      /* branch-and-link-to-noredir *%R11 */       \
 626                      "or 3,3,3\n\t"
 627
 628 #endif /* PLAT_ppc32_aix5 */
 629
 630 /* ------------------------ ppc64-aix5 ------------------------- */
 631
 632 #if defined(PLAT_ppc64_aix5)
 633
 634 typedef
 635    struct {
 636       unsigned long long int nraddr; /* where's the code? */
 637       unsigned long long int r2;  /* what tocptr do we need? */
 638    }
 639    OrigFn;
 640
 641 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 642                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 643                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 644
 645 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 646         _zzq_rlval, _zzq_default, _zzq_request,                   \
 647         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 648                                                                   \
 649   {          unsigned long long int  _zzq_args[7];                \
 650     register unsigned long long int  _zzq_result;                 \
 651     register unsigned long long int* _zzq_ptr;                    \
 652     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 653     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 654     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 655     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 656     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 657     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 658     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 659     _zzq_ptr = _zzq_args;                                         \
 660     __asm__ volatile("mr 4,%1\n\t"                                \
 661                      "ld 3, 48(4)\n\t"                            \
 662                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 663                      /* %R3 = client_request ( %R4 ) */           \
 664                      "or 1,1,1\n\t"                               \
 665                      "mr %0,3"                                    \
 666                      : "=b" (_zzq_result)                         \
 667                      : "b" (_zzq_ptr)                             \
 668                      : "r3", "r4", "cc", "memory");               \
 669     _zzq_rlval = _zzq_result;                                     \
 670   }
 671
 672 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 673   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 674     register unsigned long long int __addr;                       \
 675     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 676                      /* %R3 = guest_NRADDR */                     \
 677                      "or 2,2,2\n\t"                               \
 678                      "mr %0,3"                                    \
 679                      : "=b" (__addr)                              \
 680                      :                                            \
 681                      : "r3", "cc", "memory"                       \
 682                     );                                            \
 683     _zzq_orig->nraddr = __addr;                                   \
 684     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 685                      /* %R3 = guest_NRADDR_GPR2 */                \
 686                      "or 4,4,4\n\t"                               \
 687                      "mr %0,3"                                    \
 688                      : "=b" (__addr)                              \
 689                      :                                            \
 690                      : "r3", "cc", "memory"                       \
 691                     );                                            \
 692     _zzq_orig->r2 = __addr;                                       \
 693   }
 694
 695 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 696                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 697                      /* branch-and-link-to-noredir *%R11 */       \
 698                      "or 3,3,3\n\t"
 699
 700 #endif /* PLAT_ppc64_aix5 */
 701
 702 /* ------------------------ s390x-linux ------------------------ */
 703
 704 #if defined(PLAT_s390x_linux)
 705
 706 typedef
 707   struct {
 708      unsigned long long int nraddr; /* where's the code? */
 709   }
 710   OrigFn;
 711
 712 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
 713  * code. This detection is implemented in platform specific toIR.c
 714  * (e.g. VEX/priv/guest_s390_decoder.c).
 715  */
 716 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
 717                      "lr 15,15\n\t"                              \
 718                      "lr 1,1\n\t"                                \
 719                      "lr 2,2\n\t"                                \
 720                      "lr 3,3\n\t"
 721
 722 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
 723 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
 724 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
 725
 726 #define VALGRIND_DO_CLIENT_REQUEST(                              \
 727        _zzq_rlval, _zzq_default, _zzq_request,                   \
 728        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 729  { volatile unsigned long long int _zzq_args[6];                 \
 730    volatile unsigned long long int _zzq_result;                  \
 731    _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 732    _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 733    _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 734    _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 735    _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 736    _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 737    __asm__ volatile(/* r2 = args */                              \
 738                     "lgr 2,%1\n\t"                               \
 739                     /* r3 = default */                           \
 740                     "lgr 3,%2\n\t"                               \
 741                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 742                     __CLIENT_REQUEST_CODE                        \
 743                     /* results = r3 */                           \
 744                     "lgr %0, 3\n\t"                              \
 745                     : "=d" (_zzq_result)                         \
 746                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 747                     : "cc", "2", "3", "memory"                   \
 748                    );                                            \
 749    _zzq_rlval = _zzq_result;                                     \
 750  }
 751
 752 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
 753  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 754    volatile unsigned long long int __addr;                       \
 755    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 756                     __GET_NR_CONTEXT_CODE                        \
 757                     "lgr %0, 3\n\t"                              \
 758                     : "=a" (__addr)                              \
 759                     :                                            \
 760                     : "cc", "3", "memory"                        \
 761                    );                                            \
 762    _zzq_orig->nraddr = __addr;                                   \
 763  }
 764
 765 #define VALGRIND_CALL_NOREDIR_R1                                 \
 766                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 767                     __CALL_NO_REDIR_CODE
 768
 769 #endif /* PLAT_s390x_linux */
 770
 771 /* Insert assembly code for other platforms here... */
 772
 773 #endif /* NVALGRIND */
 774
 775
 776 /* ------------------------------------------------------------------ */
 777 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 778 /* ugly.  It's the least-worst tradeoff I can think of.               */
 779 /* ------------------------------------------------------------------ */
 780
 781 /* This section defines magic (a.k.a appalling-hack) macros for doing
 782    guaranteed-no-redirection macros, so as to get from function
 783    wrappers to the functions they are wrapping.  The whole point is to
 784    construct standard call sequences, but to do the call itself with a
 785    special no-redirect call pseudo-instruction that the JIT
 786    understands and handles specially.  This section is long and
 787    repetitious, and I can't see a way to make it shorter.
 788
 789    The naming scheme is as follows:
 790
 791       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 792
 793    'W' stands for "word" and 'v' for "void".  Hence there are
 794    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 795    and for each, the possibility of returning a word-typed result, or
 796    no result.
 797 */
 798
 799 /* Use these to write the name of your wrapper.  NOTE: duplicates
 800    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 801
 802 /* Use an extra level of macroisation so as to ensure the soname/fnname
 803    args are fully macro-expanded before pasting them together. */
 804 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 805
 806 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 807    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 808
 809 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 810    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 811
 812 /* Use this macro from within a wrapper function to collect the
 813    context (address and possibly other info) of the original function.
 814    Once you have that you can then use it in one of the CALL_FN_
 815    macros.  The type of the argument _lval is OrigFn. */
 816 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 817
 818 /* Derivatives of the main macros below, for calling functions
 819    returning void. */
 820
 821 #define CALL_FN_v_v(fnptr)                                        \
 822    do { volatile unsigned long _junk;                             \
 823         CALL_FN_W_v(_junk,fnptr); } while (0)
 824
 825 #define CALL_FN_v_W(fnptr, arg1)                                  \
 826    do { volatile unsigned long _junk;                             \
 827         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 828
 829 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 830    do { volatile unsigned long _junk;                             \
 831         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 832
 833 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 834    do { volatile unsigned long _junk;                             \
 835         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 836
 837 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 838    do { volatile unsigned long _junk;                             \
 839         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 840
 841 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 842    do { volatile unsigned long _junk;                             \
 843         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 844
 845 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 846    do { volatile unsigned long _junk;                             \
 847         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 848
 849 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 850    do { volatile unsigned long _junk;                             \
 851         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 852
 853 /* ------------------------- x86-{linux,darwin} ---------------- */
 854
 855 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 856
 857 /* These regs are trashed by the hidden call.  No need to mention eax
 858    as gcc can already see that, plus causes gcc to bomb. */
 859 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 860
 861 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 862    long) == 4. */
 863
 864 #define CALL_FN_W_v(lval, orig)                                   \
 865    do {                                                           \
 866       volatile OrigFn        _orig = (orig);                      \
 867       volatile unsigned long _argvec[1];                          \
 868       volatile unsigned long _res;                                \
 869       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 870       __asm__ volatile(                                           \
 871          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 872          VALGRIND_CALL_NOREDIR_EAX                                \
 873          : /*out*/   "=a" (_res)                                  \
 874          : /*in*/    "a" (&_argvec[0])                            \
 875          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 876       );                                                          \
 877       lval = (__typeof__(lval)) _res;                             \
 878    } while (0)
 879
 880 #define CALL_FN_W_W(lval, orig, arg1)                             \
 881    do {                                                           \
 882       volatile OrigFn        _orig = (orig);                      \
 883       volatile unsigned long _argvec[2];                          \
 884       volatile unsigned long _res;                                \
 885       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 886       _argvec[1] = (unsigned long)(arg1);                         \
 887       __asm__ volatile(                                           \
 888          "subl $12, %%esp\n\t"                                    \
 889          "pushl 4(%%eax)\n\t"                                     \
 890          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 891          VALGRIND_CALL_NOREDIR_EAX                                \
 892          "addl $16, %%esp\n"                                      \
 893          : /*out*/   "=a" (_res)                                  \
 894          : /*in*/    "a" (&_argvec[0])                            \
 895          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 896       );                                                          \
 897       lval = (__typeof__(lval)) _res;                             \
 898    } while (0)
 899
 900 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 901    do {                                                           \
 902       volatile OrigFn        _orig = (orig);                      \
 903       volatile unsigned long _argvec[3];                          \
 904       volatile unsigned long _res;                                \
 905       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 906       _argvec[1] = (unsigned long)(arg1);                         \
 907       _argvec[2] = (unsigned long)(arg2);                         \
 908       __asm__ volatile(                                           \
 909          "subl $8, %%esp\n\t"                                     \
 910          "pushl 8(%%eax)\n\t"                                     \
 911          "pushl 4(%%eax)\n\t"                                     \
 912          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 913          VALGRIND_CALL_NOREDIR_EAX                                \
 914          "addl $16, %%esp\n"                                      \
 915          : /*out*/   "=a" (_res)                                  \
 916          : /*in*/    "a" (&_argvec[0])                            \
 917          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 918       );                                                          \
 919       lval = (__typeof__(lval)) _res;                             \
 920    } while (0)
 921
 922 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 923    do {                                                           \
 924       volatile OrigFn        _orig = (orig);                      \
 925       volatile unsigned long _argvec[4];                          \
 926       volatile unsigned long _res;                                \
 927       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 928       _argvec[1] = (unsigned long)(arg1);                         \
 929       _argvec[2] = (unsigned long)(arg2);                         \
 930       _argvec[3] = (unsigned long)(arg3);                         \
 931       __asm__ volatile(                                           \
 932          "subl $4, %%esp\n\t"                                     \
 933          "pushl 12(%%eax)\n\t"                                    \
 934          "pushl 8(%%eax)\n\t"                                     \
 935          "pushl 4(%%eax)\n\t"                                     \
 936          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 937          VALGRIND_CALL_NOREDIR_EAX                                \
 938          "addl $16, %%esp\n"                                      \
 939          : /*out*/   "=a" (_res)                                  \
 940          : /*in*/    "a" (&_argvec[0])                            \
 941          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 942       );                                                          \
 943       lval = (__typeof__(lval)) _res;                             \
 944    } while (0)
 945
 946 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 947    do {                                                           \
 948       volatile OrigFn        _orig = (orig);                      \
 949       volatile unsigned long _argvec[5];                          \
 950       volatile unsigned long _res;                                \
 951       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 952       _argvec[1] = (unsigned long)(arg1);                         \
 953       _argvec[2] = (unsigned long)(arg2);                         \
 954       _argvec[3] = (unsigned long)(arg3);                         \
 955       _argvec[4] = (unsigned long)(arg4);                         \
 956       __asm__ volatile(                                           \
 957          "pushl 16(%%eax)\n\t"                                    \
 958          "pushl 12(%%eax)\n\t"                                    \
 959          "pushl 8(%%eax)\n\t"                                     \
 960          "pushl 4(%%eax)\n\t"                                     \
 961          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 962          VALGRIND_CALL_NOREDIR_EAX                                \
 963          "addl $16, %%esp\n"                                      \
 964          : /*out*/   "=a" (_res)                                  \
 965          : /*in*/    "a" (&_argvec[0])                            \
 966          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 967       );                                                          \
 968       lval = (__typeof__(lval)) _res;                             \
 969    } while (0)
 970
 971 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 972    do {                                                           \
 973       volatile OrigFn        _orig = (orig);                      \
 974       volatile unsigned long _argvec[6];                          \
 975       volatile unsigned long _res;                                \
 976       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 977       _argvec[1] = (unsigned long)(arg1);                         \
 978       _argvec[2] = (unsigned long)(arg2);                         \
 979       _argvec[3] = (unsigned long)(arg3);                         \
 980       _argvec[4] = (unsigned long)(arg4);                         \
 981       _argvec[5] = (unsigned long)(arg5);                         \
 982       __asm__ volatile(                                           \
 983          "subl $12, %%esp\n\t"                                    \
 984          "pushl 20(%%eax)\n\t"                                    \
 985          "pushl 16(%%eax)\n\t"                                    \
 986          "pushl 12(%%eax)\n\t"                                    \
 987          "pushl 8(%%eax)\n\t"                                     \
 988          "pushl 4(%%eax)\n\t"                                     \
 989          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 990          VALGRIND_CALL_NOREDIR_EAX                                \
 991          "addl $32, %%esp\n"                                      \
 992          : /*out*/   "=a" (_res)                                  \
 993          : /*in*/    "a" (&_argvec[0])                            \
 994          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 995       );                                                          \
 996       lval = (__typeof__(lval)) _res;                             \
 997    } while (0)
 998
 999 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1000    do {                                                           \
1001       volatile OrigFn        _orig = (orig);                      \
1002       volatile unsigned long _argvec[7];                          \
1003       volatile unsigned long _res;                                \
1004       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1005       _argvec[1] = (unsigned long)(arg1);                         \
1006       _argvec[2] = (unsigned long)(arg2);                         \
1007       _argvec[3] = (unsigned long)(arg3);                         \
1008       _argvec[4] = (unsigned long)(arg4);                         \
1009       _argvec[5] = (unsigned long)(arg5);                         \
1010       _argvec[6] = (unsigned long)(arg6);                         \
1011       __asm__ volatile(                                           \
1012          "subl $8, %%esp\n\t"                                     \
1013          "pushl 24(%%eax)\n\t"                                    \
1014          "pushl 20(%%eax)\n\t"                                    \
1015          "pushl 16(%%eax)\n\t"                                    \
1016          "pushl 12(%%eax)\n\t"                                    \
1017          "pushl 8(%%eax)\n\t"                                     \
1018          "pushl 4(%%eax)\n\t"                                     \
1019          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1020          VALGRIND_CALL_NOREDIR_EAX                                \
1021          "addl $32, %%esp\n"                                      \
1022          : /*out*/   "=a" (_res)                                  \
1023          : /*in*/    "a" (&_argvec[0])                            \
1024          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1025       );                                                          \
1026       lval = (__typeof__(lval)) _res;                             \
1027    } while (0)
1028
1029 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1030                                  arg7)                            \
1031    do {                                                           \
1032       volatile OrigFn        _orig = (orig);                      \
1033       volatile unsigned long _argvec[8];                          \
1034       volatile unsigned long _res;                                \
1035       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1036       _argvec[1] = (unsigned long)(arg1);                         \
1037       _argvec[2] = (unsigned long)(arg2);                         \
1038       _argvec[3] = (unsigned long)(arg3);                         \
1039       _argvec[4] = (unsigned long)(arg4);                         \
1040       _argvec[5] = (unsigned long)(arg5);                         \
1041       _argvec[6] = (unsigned long)(arg6);                         \
1042       _argvec[7] = (unsigned long)(arg7);                         \
1043       __asm__ volatile(                                           \
1044          "subl $4, %%esp\n\t"                                     \
1045          "pushl 28(%%eax)\n\t"                                    \
1046          "pushl 24(%%eax)\n\t"                                    \
1047          "pushl 20(%%eax)\n\t"                                    \
1048          "pushl 16(%%eax)\n\t"                                    \
1049          "pushl 12(%%eax)\n\t"                                    \
1050          "pushl 8(%%eax)\n\t"                                     \
1051          "pushl 4(%%eax)\n\t"                                     \
1052          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1053          VALGRIND_CALL_NOREDIR_EAX                                \
1054          "addl $32, %%esp\n"                                      \
1055          : /*out*/   "=a" (_res)                                  \
1056          : /*in*/    "a" (&_argvec[0])                            \
1057          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1058       );                                                          \
1059       lval = (__typeof__(lval)) _res;                             \
1060    } while (0)
1061
1062 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1063                                  arg7,arg8)                       \
1064    do {                                                           \
1065       volatile OrigFn        _orig = (orig);                      \
1066       volatile unsigned long _argvec[9];                          \
1067       volatile unsigned long _res;                                \
1068       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1069       _argvec[1] = (unsigned long)(arg1);                         \
1070       _argvec[2] = (unsigned long)(arg2);                         \
1071       _argvec[3] = (unsigned long)(arg3);                         \
1072       _argvec[4] = (unsigned long)(arg4);                         \
1073       _argvec[5] = (unsigned long)(arg5);                         \
1074       _argvec[6] = (unsigned long)(arg6);                         \
1075       _argvec[7] = (unsigned long)(arg7);                         \
1076       _argvec[8] = (unsigned long)(arg8);                         \
1077       __asm__ volatile(                                           \
1078          "pushl 32(%%eax)\n\t"                                    \
1079          "pushl 28(%%eax)\n\t"                                    \
1080          "pushl 24(%%eax)\n\t"                                    \
1081          "pushl 20(%%eax)\n\t"                                    \
1082          "pushl 16(%%eax)\n\t"                                    \
1083          "pushl 12(%%eax)\n\t"                                    \
1084          "pushl 8(%%eax)\n\t"                                     \
1085          "pushl 4(%%eax)\n\t"                                     \
1086          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1087          VALGRIND_CALL_NOREDIR_EAX                                \
1088          "addl $32, %%esp\n"                                      \
1089          : /*out*/   "=a" (_res)                                  \
1090          : /*in*/    "a" (&_argvec[0])                            \
1091          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1092       );                                                          \
1093       lval = (__typeof__(lval)) _res;                             \
1094    } while (0)
1095
1096 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1097                                  arg7,arg8,arg9)                  \
1098    do {                                                           \
1099       volatile OrigFn        _orig = (orig);                      \
1100       volatile unsigned long _argvec[10];                         \
1101       volatile unsigned long _res;                                \
1102       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1103       _argvec[1] = (unsigned long)(arg1);                         \
1104       _argvec[2] = (unsigned long)(arg2);                         \
1105       _argvec[3] = (unsigned long)(arg3);                         \
1106       _argvec[4] = (unsigned long)(arg4);                         \
1107       _argvec[5] = (unsigned long)(arg5);                         \
1108       _argvec[6] = (unsigned long)(arg6);                         \
1109       _argvec[7] = (unsigned long)(arg7);                         \
1110       _argvec[8] = (unsigned long)(arg8);                         \
1111       _argvec[9] = (unsigned long)(arg9);                         \
1112       __asm__ volatile(                                           \
1113          "subl $12, %%esp\n\t"                                    \
1114          "pushl 36(%%eax)\n\t"                                    \
1115          "pushl 32(%%eax)\n\t"                                    \
1116          "pushl 28(%%eax)\n\t"                                    \
1117          "pushl 24(%%eax)\n\t"                                    \
1118          "pushl 20(%%eax)\n\t"                                    \
1119          "pushl 16(%%eax)\n\t"                                    \
1120          "pushl 12(%%eax)\n\t"                                    \
1121          "pushl 8(%%eax)\n\t"                                     \
1122          "pushl 4(%%eax)\n\t"                                     \
1123          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1124          VALGRIND_CALL_NOREDIR_EAX                                \
1125          "addl $48, %%esp\n"                                      \
1126          : /*out*/   "=a" (_res)                                  \
1127          : /*in*/    "a" (&_argvec[0])                            \
1128          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1129       );                                                          \
1130       lval = (__typeof__(lval)) _res;                             \
1131    } while (0)
1132
1133 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1134                                   arg7,arg8,arg9,arg10)           \
1135    do {                                                           \
1136       volatile OrigFn        _orig = (orig);                      \
1137       volatile unsigned long _argvec[11];                         \
1138       volatile unsigned long _res;                                \
1139       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1140       _argvec[1] = (unsigned long)(arg1);                         \
1141       _argvec[2] = (unsigned long)(arg2);                         \
1142       _argvec[3] = (unsigned long)(arg3);                         \
1143       _argvec[4] = (unsigned long)(arg4);                         \
1144       _argvec[5] = (unsigned long)(arg5);                         \
1145       _argvec[6] = (unsigned long)(arg6);                         \
1146       _argvec[7] = (unsigned long)(arg7);                         \
1147       _argvec[8] = (unsigned long)(arg8);                         \
1148       _argvec[9] = (unsigned long)(arg9);                         \
1149       _argvec[10] = (unsigned long)(arg10);                       \
1150       __asm__ volatile(                                           \
1151          "subl $8, %%esp\n\t"                                     \
1152          "pushl 40(%%eax)\n\t"                                    \
1153          "pushl 36(%%eax)\n\t"                                    \
1154          "pushl 32(%%eax)\n\t"                                    \
1155          "pushl 28(%%eax)\n\t"                                    \
1156          "pushl 24(%%eax)\n\t"                                    \
1157          "pushl 20(%%eax)\n\t"                                    \
1158          "pushl 16(%%eax)\n\t"                                    \
1159          "pushl 12(%%eax)\n\t"                                    \
1160          "pushl 8(%%eax)\n\t"                                     \
1161          "pushl 4(%%eax)\n\t"                                     \
1162          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1163          VALGRIND_CALL_NOREDIR_EAX                                \
1164          "addl $48, %%esp\n"                                      \
1165          : /*out*/   "=a" (_res)                                  \
1166          : /*in*/    "a" (&_argvec[0])                            \
1167          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1168       );                                                          \
1169       lval = (__typeof__(lval)) _res;                             \
1170    } while (0)
1171
1172 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1173                                   arg6,arg7,arg8,arg9,arg10,      \
1174                                   arg11)                          \
1175    do {                                                           \
1176       volatile OrigFn        _orig = (orig);                      \
1177       volatile unsigned long _argvec[12];                         \
1178       volatile unsigned long _res;                                \
1179       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1180       _argvec[1] = (unsigned long)(arg1);                         \
1181       _argvec[2] = (unsigned long)(arg2);                         \
1182       _argvec[3] = (unsigned long)(arg3);                         \
1183       _argvec[4] = (unsigned long)(arg4);                         \
1184       _argvec[5] = (unsigned long)(arg5);                         \
1185       _argvec[6] = (unsigned long)(arg6);                         \
1186       _argvec[7] = (unsigned long)(arg7);                         \
1187       _argvec[8] = (unsigned long)(arg8);                         \
1188       _argvec[9] = (unsigned long)(arg9);                         \
1189       _argvec[10] = (unsigned long)(arg10);                       \
1190       _argvec[11] = (unsigned long)(arg11);                       \
1191       __asm__ volatile(                                           \
1192          "subl $4, %%esp\n\t"                                     \
1193          "pushl 44(%%eax)\n\t"                                    \
1194          "pushl 40(%%eax)\n\t"                                    \
1195          "pushl 36(%%eax)\n\t"                                    \
1196          "pushl 32(%%eax)\n\t"                                    \
1197          "pushl 28(%%eax)\n\t"                                    \
1198          "pushl 24(%%eax)\n\t"                                    \
1199          "pushl 20(%%eax)\n\t"                                    \
1200          "pushl 16(%%eax)\n\t"                                    \
1201          "pushl 12(%%eax)\n\t"                                    \
1202          "pushl 8(%%eax)\n\t"                                     \
1203          "pushl 4(%%eax)\n\t"                                     \
1204          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1205          VALGRIND_CALL_NOREDIR_EAX                                \
1206          "addl $48, %%esp\n"                                      \
1207          : /*out*/   "=a" (_res)                                  \
1208          : /*in*/    "a" (&_argvec[0])                            \
1209          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1210       );                                                          \
1211       lval = (__typeof__(lval)) _res;                             \
1212    } while (0)
1213
1214 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1215                                   arg6,arg7,arg8,arg9,arg10,      \
1216                                   arg11,arg12)                    \
1217    do {                                                           \
1218       volatile OrigFn        _orig = (orig);                      \
1219       volatile unsigned long _argvec[13];                         \
1220       volatile unsigned long _res;                                \
1221       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1222       _argvec[1] = (unsigned long)(arg1);                         \
1223       _argvec[2] = (unsigned long)(arg2);                         \
1224       _argvec[3] = (unsigned long)(arg3);                         \
1225       _argvec[4] = (unsigned long)(arg4);                         \
1226       _argvec[5] = (unsigned long)(arg5);                         \
1227       _argvec[6] = (unsigned long)(arg6);                         \
1228       _argvec[7] = (unsigned long)(arg7);                         \
1229       _argvec[8] = (unsigned long)(arg8);                         \
1230       _argvec[9] = (unsigned long)(arg9);                         \
1231       _argvec[10] = (unsigned long)(arg10);                       \
1232       _argvec[11] = (unsigned long)(arg11);                       \
1233       _argvec[12] = (unsigned long)(arg12);                       \
1234       __asm__ volatile(                                           \
1235          "pushl 48(%%eax)\n\t"                                    \
1236          "pushl 44(%%eax)\n\t"                                    \
1237          "pushl 40(%%eax)\n\t"                                    \
1238          "pushl 36(%%eax)\n\t"                                    \
1239          "pushl 32(%%eax)\n\t"                                    \
1240          "pushl 28(%%eax)\n\t"                                    \
1241          "pushl 24(%%eax)\n\t"                                    \
1242          "pushl 20(%%eax)\n\t"                                    \
1243          "pushl 16(%%eax)\n\t"                                    \
1244          "pushl 12(%%eax)\n\t"                                    \
1245          "pushl 8(%%eax)\n\t"                                     \
1246          "pushl 4(%%eax)\n\t"                                     \
1247          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1248          VALGRIND_CALL_NOREDIR_EAX                                \
1249          "addl $48, %%esp\n"                                      \
1250          : /*out*/   "=a" (_res)                                  \
1251          : /*in*/    "a" (&_argvec[0])                            \
1252          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1253       );                                                          \
1254       lval = (__typeof__(lval)) _res;                             \
1255    } while (0)
1256
1257 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1258
1259 /* ------------------------ amd64-{linux,darwin} --------------- */
1260
1261 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1262
1263 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1264
1265 /* These regs are trashed by the hidden call. */
1266 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1267                             "rdi", "r8", "r9", "r10", "r11"
1268
1269 /* This is all pretty complex.  It's so as to make stack unwinding
1270    work reliably.  See bug 243270.  The basic problem is the sub and
1271    add of 128 of %rsp in all of the following macros.  If gcc believes
1272    the CFA is in %rsp, then unwinding may fail, because what's at the
1273    CFA is not what gcc "expected" when it constructs the CFIs for the
1274    places where the macros are instantiated.
1275
1276    But we can't just add a CFI annotation to increase the CFA offset
1277    by 128, to match the sub of 128 from %rsp, because we don't know
1278    whether gcc has chosen %rsp as the CFA at that point, or whether it
1279    has chosen some other register (eg, %rbp).  In the latter case,
1280    adding a CFI annotation to change the CFA offset is simply wrong.
1281
1282    So the solution is to get hold of the CFA using
1283    __builtin_dwarf_cfa(), put it in a known register, and add a
1284    CFI annotation to say what the register is.  We choose %rbp for
1285    this (perhaps perversely), because:
1286
1287    (1) %rbp is already subject to unwinding.  If a new register was
1288        chosen then the unwinder would have to unwind it in all stack
1289        traces, which is expensive, and
1290
1291    (2) %rbp is already subject to precise exception updates in the
1292        JIT.  If a new register was chosen, we'd have to have precise
1293        exceptions for it too, which reduces performance of the
1294        generated code.
1295
1296    However .. one extra complication.  We can't just whack the result
1297    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1298    list of trashed registers at the end of the inline assembly
1299    fragments; gcc won't allow %rbp to appear in that list.  Hence
1300    instead we need to stash %rbp in %r15 for the duration of the asm,
1301    and say that %r15 is trashed instead.  gcc seems happy to go with
1302    that.
1303
1304    Oh .. and this all needs to be conditionalised so that it is
1305    unchanged from before this commit, when compiled with older gccs
1306    that don't support __builtin_dwarf_cfa.  Furthermore, since
1307    this header file is freestanding, it has to be independent of
1308    config.h, and so the following conditionalisation cannot depend on
1309    configure time checks.
1310
1311    Although it's not clear from
1312    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1313    this expression excludes Darwin.
1314    .cfi directives in Darwin assembly appear to be completely
1315    different and I haven't investigated how they work.
1316
1317    For even more entertainment value, note we have to use the
1318    completely undocumented __builtin_dwarf_cfa(), which appears to
1319    really compute the CFA, whereas __builtin_frame_address(0) claims
1320    to but actually doesn't.  See
1321    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1322 */
1323 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1324 #  define __FRAME_POINTER                                         \
1325       ,"r"(__builtin_dwarf_cfa())
1326 #  define VALGRIND_CFI_PROLOGUE                                   \
1327       "movq %%rbp, %%r15\n\t"                                     \
1328       "movq %2, %%rbp\n\t"                                        \
1329       ".cfi_remember_state\n\t"                                   \
1330       ".cfi_def_cfa rbp, 0\n\t"
1331 #  define VALGRIND_CFI_EPILOGUE                                   \
1332       "movq %%r15, %%rbp\n\t"                                     \
1333       ".cfi_restore_state\n\t"
1334 #else
1335 #  define __FRAME_POINTER
1336 #  define VALGRIND_CFI_PROLOGUE
1337 #  define VALGRIND_CFI_EPILOGUE
1338 #endif
1339
1340
1341 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1342    long) == 8. */
1343
1344 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1345    macros.  In order not to trash the stack redzone, we need to drop
1346    %rsp by 128 before the hidden call, and restore afterwards.  The
1347    nastyness is that it is only by luck that the stack still appears
1348    to be unwindable during the hidden call - since then the behaviour
1349    of any routine using this macro does not match what the CFI data
1350    says.  Sigh.
1351
1352    Why is this important?  Imagine that a wrapper has a stack
1353    allocated local, and passes to the hidden call, a pointer to it.
1354    Because gcc does not know about the hidden call, it may allocate
1355    that local in the redzone.  Unfortunately the hidden call may then
1356    trash it before it comes to use it.  So we must step clear of the
1357    redzone, for the duration of the hidden call, to make it safe.
1358
1359    Probably the same problem afflicts the other redzone-style ABIs too
1360    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1361    self describing (none of this CFI nonsense) so at least messing
1362    with the stack pointer doesn't give a danger of non-unwindable
1363    stack. */
1364
1365 #define CALL_FN_W_v(lval, orig)                                   \
1366    do {                                                           \
1367       volatile OrigFn        _orig = (orig);                      \
1368       volatile unsigned long _argvec[1];                          \
1369       volatile unsigned long _res;                                \
1370       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1371       __asm__ volatile(                                           \
1372          VALGRIND_CFI_PROLOGUE                                    \
1373          "subq $128,%%rsp\n\t"                                    \
1374          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1375          VALGRIND_CALL_NOREDIR_RAX                                \
1376          "addq $128,%%rsp\n\t"                                    \
1377          VALGRIND_CFI_EPILOGUE                                    \
1378          : /*out*/   "=a" (_res)                                  \
1379          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1380          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1381       );                                                          \
1382       lval = (__typeof__(lval)) _res;                             \
1383    } while (0)
1384
1385 #define CALL_FN_W_W(lval, orig, arg1)                             \
1386    do {                                                           \
1387       volatile OrigFn        _orig = (orig);                      \
1388       volatile unsigned long _argvec[2];                          \
1389       volatile unsigned long _res;                                \
1390       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1391       _argvec[1] = (unsigned long)(arg1);                         \
1392       __asm__ volatile(                                           \
1393          VALGRIND_CFI_PROLOGUE                                    \
1394          "subq $128,%%rsp\n\t"                                    \
1395          "movq 8(%%rax), %%rdi\n\t"                               \
1396          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1397          VALGRIND_CALL_NOREDIR_RAX                                \
1398          "addq $128,%%rsp\n\t"                                    \
1399          VALGRIND_CFI_EPILOGUE                                    \
1400          : /*out*/   "=a" (_res)                                  \
1401          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1402          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1403       );                                                          \
1404       lval = (__typeof__(lval)) _res;                             \
1405    } while (0)
1406
1407 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1408    do {                                                           \
1409       volatile OrigFn        _orig = (orig);                      \
1410       volatile unsigned long _argvec[3];                          \
1411       volatile unsigned long _res;                                \
1412       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1413       _argvec[1] = (unsigned long)(arg1);                         \
1414       _argvec[2] = (unsigned long)(arg2);                         \
1415       __asm__ volatile(                                           \
1416          VALGRIND_CFI_PROLOGUE                                    \
1417          "subq $128,%%rsp\n\t"                                    \
1418          "movq 16(%%rax), %%rsi\n\t"                              \
1419          "movq 8(%%rax), %%rdi\n\t"                               \
1420          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1421          VALGRIND_CALL_NOREDIR_RAX                                \
1422          "addq $128,%%rsp\n\t"                                    \
1423          VALGRIND_CFI_EPILOGUE                                    \
1424          : /*out*/   "=a" (_res)                                  \
1425          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1426          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1427       );                                                          \
1428       lval = (__typeof__(lval)) _res;                             \
1429    } while (0)
1430
1431 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1432    do {                                                           \
1433       volatile OrigFn        _orig = (orig);                      \
1434       volatile unsigned long _argvec[4];                          \
1435       volatile unsigned long _res;                                \
1436       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1437       _argvec[1] = (unsigned long)(arg1);                         \
1438       _argvec[2] = (unsigned long)(arg2);                         \
1439       _argvec[3] = (unsigned long)(arg3);                         \
1440       __asm__ volatile(                                           \
1441          VALGRIND_CFI_PROLOGUE                                    \
1442          "subq $128,%%rsp\n\t"                                    \
1443          "movq 24(%%rax), %%rdx\n\t"                              \
1444          "movq 16(%%rax), %%rsi\n\t"                              \
1445          "movq 8(%%rax), %%rdi\n\t"                               \
1446          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1447          VALGRIND_CALL_NOREDIR_RAX                                \
1448          "addq $128,%%rsp\n\t"                                    \
1449          VALGRIND_CFI_EPILOGUE                                    \
1450          : /*out*/   "=a" (_res)                                  \
1451          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1452          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1453       );                                                          \
1454       lval = (__typeof__(lval)) _res;                             \
1455    } while (0)
1456
1457 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1458    do {                                                           \
1459       volatile OrigFn        _orig = (orig);                      \
1460       volatile unsigned long _argvec[5];                          \
1461       volatile unsigned long _res;                                \
1462       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1463       _argvec[1] = (unsigned long)(arg1);                         \
1464       _argvec[2] = (unsigned long)(arg2);                         \
1465       _argvec[3] = (unsigned long)(arg3);                         \
1466       _argvec[4] = (unsigned long)(arg4);                         \
1467       __asm__ volatile(                                           \
1468          VALGRIND_CFI_PROLOGUE                                    \
1469          "subq $128,%%rsp\n\t"                                    \
1470          "movq 32(%%rax), %%rcx\n\t"                              \
1471          "movq 24(%%rax), %%rdx\n\t"                              \
1472          "movq 16(%%rax), %%rsi\n\t"                              \
1473          "movq 8(%%rax), %%rdi\n\t"                               \
1474          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1475          VALGRIND_CALL_NOREDIR_RAX                                \
1476          "addq $128,%%rsp\n\t"                                    \
1477          VALGRIND_CFI_EPILOGUE                                    \
1478          : /*out*/   "=a" (_res)                                  \
1479          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1480          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1481       );                                                          \
1482       lval = (__typeof__(lval)) _res;                             \
1483    } while (0)
1484
1485 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1486    do {                                                           \
1487       volatile OrigFn        _orig = (orig);                      \
1488       volatile unsigned long _argvec[6];                          \
1489       volatile unsigned long _res;                                \
1490       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1491       _argvec[1] = (unsigned long)(arg1);                         \
1492       _argvec[2] = (unsigned long)(arg2);                         \
1493       _argvec[3] = (unsigned long)(arg3);                         \
1494       _argvec[4] = (unsigned long)(arg4);                         \
1495       _argvec[5] = (unsigned long)(arg5);                         \
1496       __asm__ volatile(                                           \
1497          VALGRIND_CFI_PROLOGUE                                    \
1498          "subq $128,%%rsp\n\t"                                    \
1499          "movq 40(%%rax), %%r8\n\t"                               \
1500          "movq 32(%%rax), %%rcx\n\t"                              \
1501          "movq 24(%%rax), %%rdx\n\t"                              \
1502          "movq 16(%%rax), %%rsi\n\t"                              \
1503          "movq 8(%%rax), %%rdi\n\t"                               \
1504          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1505          VALGRIND_CALL_NOREDIR_RAX                                \
1506          "addq $128,%%rsp\n\t"                                    \
1507          VALGRIND_CFI_EPILOGUE                                    \
1508          : /*out*/   "=a" (_res)                                  \
1509          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1510          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1511       );                                                          \
1512       lval = (__typeof__(lval)) _res;                             \
1513    } while (0)
1514
1515 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1516    do {                                                           \
1517       volatile OrigFn        _orig = (orig);                      \
1518       volatile unsigned long _argvec[7];                          \
1519       volatile unsigned long _res;                                \
1520       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1521       _argvec[1] = (unsigned long)(arg1);                         \
1522       _argvec[2] = (unsigned long)(arg2);                         \
1523       _argvec[3] = (unsigned long)(arg3);                         \
1524       _argvec[4] = (unsigned long)(arg4);                         \
1525       _argvec[5] = (unsigned long)(arg5);                         \
1526       _argvec[6] = (unsigned long)(arg6);                         \
1527       __asm__ volatile(                                           \
1528          VALGRIND_CFI_PROLOGUE                                    \
1529          "subq $128,%%rsp\n\t"                                    \
1530          "movq 48(%%rax), %%r9\n\t"                               \
1531          "movq 40(%%rax), %%r8\n\t"                               \
1532          "movq 32(%%rax), %%rcx\n\t"                              \
1533          "movq 24(%%rax), %%rdx\n\t"                              \
1534          "movq 16(%%rax), %%rsi\n\t"                              \
1535          "movq 8(%%rax), %%rdi\n\t"                               \
1536          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1537          VALGRIND_CALL_NOREDIR_RAX                                \
1538          "addq $128,%%rsp\n\t"                                    \
1539          VALGRIND_CFI_EPILOGUE                                    \
1540          : /*out*/   "=a" (_res)                                  \
1541          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1542          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1543       );                                                          \
1544       lval = (__typeof__(lval)) _res;                             \
1545    } while (0)
1546
1547 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1548                                  arg7)                            \
1549    do {                                                           \
1550       volatile OrigFn        _orig = (orig);                      \
1551       volatile unsigned long _argvec[8];                          \
1552       volatile unsigned long _res;                                \
1553       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1554       _argvec[1] = (unsigned long)(arg1);                         \
1555       _argvec[2] = (unsigned long)(arg2);                         \
1556       _argvec[3] = (unsigned long)(arg3);                         \
1557       _argvec[4] = (unsigned long)(arg4);                         \
1558       _argvec[5] = (unsigned long)(arg5);                         \
1559       _argvec[6] = (unsigned long)(arg6);                         \
1560       _argvec[7] = (unsigned long)(arg7);                         \
1561       __asm__ volatile(                                           \
1562          VALGRIND_CFI_PROLOGUE                                    \
1563          "subq $136,%%rsp\n\t"                                    \
1564          "pushq 56(%%rax)\n\t"                                    \
1565          "movq 48(%%rax), %%r9\n\t"                               \
1566          "movq 40(%%rax), %%r8\n\t"                               \
1567          "movq 32(%%rax), %%rcx\n\t"                              \
1568          "movq 24(%%rax), %%rdx\n\t"                              \
1569          "movq 16(%%rax), %%rsi\n\t"                              \
1570          "movq 8(%%rax), %%rdi\n\t"                               \
1571          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1572          VALGRIND_CALL_NOREDIR_RAX                                \
1573          "addq $8, %%rsp\n"                                       \
1574          "addq $136,%%rsp\n\t"                                    \
1575          VALGRIND_CFI_EPILOGUE                                    \
1576          : /*out*/   "=a" (_res)                                  \
1577          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1578          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1579       );                                                          \
1580       lval = (__typeof__(lval)) _res;                             \
1581    } while (0)
1582
1583 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1584                                  arg7,arg8)                       \
1585    do {                                                           \
1586       volatile OrigFn        _orig = (orig);                      \
1587       volatile unsigned long _argvec[9];                          \
1588       volatile unsigned long _res;                                \
1589       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1590       _argvec[1] = (unsigned long)(arg1);                         \
1591       _argvec[2] = (unsigned long)(arg2);                         \
1592       _argvec[3] = (unsigned long)(arg3);                         \
1593       _argvec[4] = (unsigned long)(arg4);                         \
1594       _argvec[5] = (unsigned long)(arg5);                         \
1595       _argvec[6] = (unsigned long)(arg6);                         \
1596       _argvec[7] = (unsigned long)(arg7);                         \
1597       _argvec[8] = (unsigned long)(arg8);                         \
1598       __asm__ volatile(                                           \
1599          VALGRIND_CFI_PROLOGUE                                    \
1600          "subq $128,%%rsp\n\t"                                    \
1601          "pushq 64(%%rax)\n\t"                                    \
1602          "pushq 56(%%rax)\n\t"                                    \
1603          "movq 48(%%rax), %%r9\n\t"                               \
1604          "movq 40(%%rax), %%r8\n\t"                               \
1605          "movq 32(%%rax), %%rcx\n\t"                              \
1606          "movq 24(%%rax), %%rdx\n\t"                              \
1607          "movq 16(%%rax), %%rsi\n\t"                              \
1608          "movq 8(%%rax), %%rdi\n\t"                               \
1609          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1610          VALGRIND_CALL_NOREDIR_RAX                                \
1611          "addq $16, %%rsp\n"                                      \
1612          "addq $128,%%rsp\n\t"                                    \
1613          VALGRIND_CFI_EPILOGUE                                    \
1614          : /*out*/   "=a" (_res)                                  \
1615          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1616          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1617       );                                                          \
1618       lval = (__typeof__(lval)) _res;                             \
1619    } while (0)
1620
1621 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1622                                  arg7,arg8,arg9)                  \
1623    do {                                                           \
1624       volatile OrigFn        _orig = (orig);                      \
1625       volatile unsigned long _argvec[10];                         \
1626       volatile unsigned long _res;                                \
1627       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1628       _argvec[1] = (unsigned long)(arg1);                         \
1629       _argvec[2] = (unsigned long)(arg2);                         \
1630       _argvec[3] = (unsigned long)(arg3);                         \
1631       _argvec[4] = (unsigned long)(arg4);                         \
1632       _argvec[5] = (unsigned long)(arg5);                         \
1633       _argvec[6] = (unsigned long)(arg6);                         \
1634       _argvec[7] = (unsigned long)(arg7);                         \
1635       _argvec[8] = (unsigned long)(arg8);                         \
1636       _argvec[9] = (unsigned long)(arg9);                         \
1637       __asm__ volatile(                                           \
1638          VALGRIND_CFI_PROLOGUE                                    \
1639          "subq $136,%%rsp\n\t"                                    \
1640          "pushq 72(%%rax)\n\t"                                    \
1641          "pushq 64(%%rax)\n\t"                                    \
1642          "pushq 56(%%rax)\n\t"                                    \
1643          "movq 48(%%rax), %%r9\n\t"                               \
1644          "movq 40(%%rax), %%r8\n\t"                               \
1645          "movq 32(%%rax), %%rcx\n\t"                              \
1646          "movq 24(%%rax), %%rdx\n\t"                              \
1647          "movq 16(%%rax), %%rsi\n\t"                              \
1648          "movq 8(%%rax), %%rdi\n\t"                               \
1649          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1650          VALGRIND_CALL_NOREDIR_RAX                                \
1651          "addq $24, %%rsp\n"                                      \
1652          "addq $136,%%rsp\n\t"                                    \
1653          VALGRIND_CFI_EPILOGUE                                    \
1654          : /*out*/   "=a" (_res)                                  \
1655          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1656          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1657       );                                                          \
1658       lval = (__typeof__(lval)) _res;                             \
1659    } while (0)
1660
1661 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1662                                   arg7,arg8,arg9,arg10)           \
1663    do {                                                           \
1664       volatile OrigFn        _orig = (orig);                      \
1665       volatile unsigned long _argvec[11];                         \
1666       volatile unsigned long _res;                                \
1667       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1668       _argvec[1] = (unsigned long)(arg1);                         \
1669       _argvec[2] = (unsigned long)(arg2);                         \
1670       _argvec[3] = (unsigned long)(arg3);                         \
1671       _argvec[4] = (unsigned long)(arg4);                         \
1672       _argvec[5] = (unsigned long)(arg5);                         \
1673       _argvec[6] = (unsigned long)(arg6);                         \
1674       _argvec[7] = (unsigned long)(arg7);                         \
1675       _argvec[8] = (unsigned long)(arg8);                         \
1676       _argvec[9] = (unsigned long)(arg9);                         \
1677       _argvec[10] = (unsigned long)(arg10);                       \
1678       __asm__ volatile(                                           \
1679          VALGRIND_CFI_PROLOGUE                                    \
1680          "subq $128,%%rsp\n\t"                                    \
1681          "pushq 80(%%rax)\n\t"                                    \
1682          "pushq 72(%%rax)\n\t"                                    \
1683          "pushq 64(%%rax)\n\t"                                    \
1684          "pushq 56(%%rax)\n\t"                                    \
1685          "movq 48(%%rax), %%r9\n\t"                               \
1686          "movq 40(%%rax), %%r8\n\t"                               \
1687          "movq 32(%%rax), %%rcx\n\t"                              \
1688          "movq 24(%%rax), %%rdx\n\t"                              \
1689          "movq 16(%%rax), %%rsi\n\t"                              \
1690          "movq 8(%%rax), %%rdi\n\t"                               \
1691          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1692          VALGRIND_CALL_NOREDIR_RAX                                \
1693          "addq $32, %%rsp\n"                                      \
1694          "addq $128,%%rsp\n\t"                                    \
1695          VALGRIND_CFI_EPILOGUE                                    \
1696          : /*out*/   "=a" (_res)                                  \
1697          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1698          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1699       );                                                          \
1700       lval = (__typeof__(lval)) _res;                             \
1701    } while (0)
1702
1703 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1704                                   arg7,arg8,arg9,arg10,arg11)     \
1705    do {                                                           \
1706       volatile OrigFn        _orig = (orig);                      \
1707       volatile unsigned long _argvec[12];                         \
1708       volatile unsigned long _res;                                \
1709       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1710       _argvec[1] = (unsigned long)(arg1);                         \
1711       _argvec[2] = (unsigned long)(arg2);                         \
1712       _argvec[3] = (unsigned long)(arg3);                         \
1713       _argvec[4] = (unsigned long)(arg4);                         \
1714       _argvec[5] = (unsigned long)(arg5);                         \
1715       _argvec[6] = (unsigned long)(arg6);                         \
1716       _argvec[7] = (unsigned long)(arg7);                         \
1717       _argvec[8] = (unsigned long)(arg8);                         \
1718       _argvec[9] = (unsigned long)(arg9);                         \
1719       _argvec[10] = (unsigned long)(arg10);                       \
1720       _argvec[11] = (unsigned long)(arg11);                       \
1721       __asm__ volatile(                                           \
1722          VALGRIND_CFI_PROLOGUE                                    \
1723          "subq $136,%%rsp\n\t"                                    \
1724          "pushq 88(%%rax)\n\t"                                    \
1725          "pushq 80(%%rax)\n\t"                                    \
1726          "pushq 72(%%rax)\n\t"                                    \
1727          "pushq 64(%%rax)\n\t"                                    \
1728          "pushq 56(%%rax)\n\t"                                    \
1729          "movq 48(%%rax), %%r9\n\t"                               \
1730          "movq 40(%%rax), %%r8\n\t"                               \
1731          "movq 32(%%rax), %%rcx\n\t"                              \
1732          "movq 24(%%rax), %%rdx\n\t"                              \
1733          "movq 16(%%rax), %%rsi\n\t"                              \
1734          "movq 8(%%rax), %%rdi\n\t"                               \
1735          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1736          VALGRIND_CALL_NOREDIR_RAX                                \
1737          "addq $40, %%rsp\n"                                      \
1738          "addq $136,%%rsp\n\t"                                    \
1739          VALGRIND_CFI_EPILOGUE                                    \
1740          : /*out*/   "=a" (_res)                                  \
1741          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1742          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1743       );                                                          \
1744       lval = (__typeof__(lval)) _res;                             \
1745    } while (0)
1746
1747 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1748                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1749    do {                                                           \
1750       volatile OrigFn        _orig = (orig);                      \
1751       volatile unsigned long _argvec[13];                         \
1752       volatile unsigned long _res;                                \
1753       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1754       _argvec[1] = (unsigned long)(arg1);                         \
1755       _argvec[2] = (unsigned long)(arg2);                         \
1756       _argvec[3] = (unsigned long)(arg3);                         \
1757       _argvec[4] = (unsigned long)(arg4);                         \
1758       _argvec[5] = (unsigned long)(arg5);                         \
1759       _argvec[6] = (unsigned long)(arg6);                         \
1760       _argvec[7] = (unsigned long)(arg7);                         \
1761       _argvec[8] = (unsigned long)(arg8);                         \
1762       _argvec[9] = (unsigned long)(arg9);                         \
1763       _argvec[10] = (unsigned long)(arg10);                       \
1764       _argvec[11] = (unsigned long)(arg11);                       \
1765       _argvec[12] = (unsigned long)(arg12);                       \
1766       __asm__ volatile(                                           \
1767          VALGRIND_CFI_PROLOGUE                                    \
1768          "subq $128,%%rsp\n\t"                                    \
1769          "pushq 96(%%rax)\n\t"                                    \
1770          "pushq 88(%%rax)\n\t"                                    \
1771          "pushq 80(%%rax)\n\t"                                    \
1772          "pushq 72(%%rax)\n\t"                                    \
1773          "pushq 64(%%rax)\n\t"                                    \
1774          "pushq 56(%%rax)\n\t"                                    \
1775          "movq 48(%%rax), %%r9\n\t"                               \
1776          "movq 40(%%rax), %%r8\n\t"                               \
1777          "movq 32(%%rax), %%rcx\n\t"                              \
1778          "movq 24(%%rax), %%rdx\n\t"                              \
1779          "movq 16(%%rax), %%rsi\n\t"                              \
1780          "movq 8(%%rax), %%rdi\n\t"                               \
1781          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1782          VALGRIND_CALL_NOREDIR_RAX                                \
1783          "addq $48, %%rsp\n"                                      \
1784          "addq $128,%%rsp\n\t"                                    \
1785          VALGRIND_CFI_EPILOGUE                                    \
1786          : /*out*/   "=a" (_res)                                  \
1787          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1788          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1789       );                                                          \
1790       lval = (__typeof__(lval)) _res;                             \
1791    } while (0)
1792
1793 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1794
1795 /* ------------------------ ppc32-linux ------------------------ */
1796
1797 #if defined(PLAT_ppc32_linux)
1798
1799 /* This is useful for finding out about the on-stack stuff:
1800
1801    extern int f9  ( int,int,int,int,int,int,int,int,int );
1802    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1803    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1804    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1805
1806    int g9 ( void ) {
1807       return f9(11,22,33,44,55,66,77,88,99);
1808    }
1809    int g10 ( void ) {
1810       return f10(11,22,33,44,55,66,77,88,99,110);
1811    }
1812    int g11 ( void ) {
1813       return f11(11,22,33,44,55,66,77,88,99,110,121);
1814    }
1815    int g12 ( void ) {
1816       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1817    }
1818 */
1819
1820 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1821
1822 /* These regs are trashed by the hidden call. */
1823 #define __CALLER_SAVED_REGS                                       \
1824    "lr", "ctr", "xer",                                            \
1825    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1826    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1827    "r11", "r12", "r13"
1828
1829 /* These CALL_FN_ macros assume that on ppc32-linux,
1830    sizeof(unsigned long) == 4. */
1831
1832 #define CALL_FN_W_v(lval, orig)                                   \
1833    do {                                                           \
1834       volatile OrigFn        _orig = (orig);                      \
1835       volatile unsigned long _argvec[1];                          \
1836       volatile unsigned long _res;                                \
1837       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1838       __asm__ volatile(                                           \
1839          "mr 11,%1\n\t"                                           \
1840          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1841          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1842          "mr %0,3"                                                \
1843          : /*out*/   "=r" (_res)                                  \
1844          : /*in*/    "r" (&_argvec[0])                            \
1845          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1846       );                                                          \
1847       lval = (__typeof__(lval)) _res;                             \
1848    } while (0)
1849
1850 #define CALL_FN_W_W(lval, orig, arg1)                             \
1851    do {                                                           \
1852       volatile OrigFn        _orig = (orig);                      \
1853       volatile unsigned long _argvec[2];                          \
1854       volatile unsigned long _res;                                \
1855       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1856       _argvec[1] = (unsigned long)arg1;                           \
1857       __asm__ volatile(                                           \
1858          "mr 11,%1\n\t"                                           \
1859          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1860          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1861          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1862          "mr %0,3"                                                \
1863          : /*out*/   "=r" (_res)                                  \
1864          : /*in*/    "r" (&_argvec[0])                            \
1865          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1866       );                                                          \
1867       lval = (__typeof__(lval)) _res;                             \
1868    } while (0)
1869
1870 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1871    do {                                                           \
1872       volatile OrigFn        _orig = (orig);                      \
1873       volatile unsigned long _argvec[3];                          \
1874       volatile unsigned long _res;                                \
1875       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1876       _argvec[1] = (unsigned long)arg1;                           \
1877       _argvec[2] = (unsigned long)arg2;                           \
1878       __asm__ volatile(                                           \
1879          "mr 11,%1\n\t"                                           \
1880          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1881          "lwz 4,8(11)\n\t"                                        \
1882          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1883          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1884          "mr %0,3"                                                \
1885          : /*out*/   "=r" (_res)                                  \
1886          : /*in*/    "r" (&_argvec[0])                            \
1887          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1888       );                                                          \
1889       lval = (__typeof__(lval)) _res;                             \
1890    } while (0)
1891
1892 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1893    do {                                                           \
1894       volatile OrigFn        _orig = (orig);                      \
1895       volatile unsigned long _argvec[4];                          \
1896       volatile unsigned long _res;                                \
1897       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1898       _argvec[1] = (unsigned long)arg1;                           \
1899       _argvec[2] = (unsigned long)arg2;                           \
1900       _argvec[3] = (unsigned long)arg3;                           \
1901       __asm__ volatile(                                           \
1902          "mr 11,%1\n\t"                                           \
1903          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1904          "lwz 4,8(11)\n\t"                                        \
1905          "lwz 5,12(11)\n\t"                                       \
1906          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1907          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1908          "mr %0,3"                                                \
1909          : /*out*/   "=r" (_res)                                  \
1910          : /*in*/    "r" (&_argvec[0])                            \
1911          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1912       );                                                          \
1913       lval = (__typeof__(lval)) _res;                             \
1914    } while (0)
1915
1916 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1917    do {                                                           \
1918       volatile OrigFn        _orig = (orig);                      \
1919       volatile unsigned long _argvec[5];                          \
1920       volatile unsigned long _res;                                \
1921       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1922       _argvec[1] = (unsigned long)arg1;                           \
1923       _argvec[2] = (unsigned long)arg2;                           \
1924       _argvec[3] = (unsigned long)arg3;                           \
1925       _argvec[4] = (unsigned long)arg4;                           \
1926       __asm__ volatile(                                           \
1927          "mr 11,%1\n\t"                                           \
1928          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1929          "lwz 4,8(11)\n\t"                                        \
1930          "lwz 5,12(11)\n\t"                                       \
1931          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1932          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1933          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1934          "mr %0,3"                                                \
1935          : /*out*/   "=r" (_res)                                  \
1936          : /*in*/    "r" (&_argvec[0])                            \
1937          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1938       );                                                          \
1939       lval = (__typeof__(lval)) _res;                             \
1940    } while (0)
1941
1942 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1943    do {                                                           \
1944       volatile OrigFn        _orig = (orig);                      \
1945       volatile unsigned long _argvec[6];                          \
1946       volatile unsigned long _res;                                \
1947       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1948       _argvec[1] = (unsigned long)arg1;                           \
1949       _argvec[2] = (unsigned long)arg2;                           \
1950       _argvec[3] = (unsigned long)arg3;                           \
1951       _argvec[4] = (unsigned long)arg4;                           \
1952       _argvec[5] = (unsigned long)arg5;                           \
1953       __asm__ volatile(                                           \
1954          "mr 11,%1\n\t"                                           \
1955          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1956          "lwz 4,8(11)\n\t"                                        \
1957          "lwz 5,12(11)\n\t"                                       \
1958          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1959          "lwz 7,20(11)\n\t"                                       \
1960          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1961          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1962          "mr %0,3"                                                \
1963          : /*out*/   "=r" (_res)                                  \
1964          : /*in*/    "r" (&_argvec[0])                            \
1965          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1966       );                                                          \
1967       lval = (__typeof__(lval)) _res;                             \
1968    } while (0)
1969
1970 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1971    do {                                                           \
1972       volatile OrigFn        _orig = (orig);                      \
1973       volatile unsigned long _argvec[7];                          \
1974       volatile unsigned long _res;                                \
1975       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1976       _argvec[1] = (unsigned long)arg1;                           \
1977       _argvec[2] = (unsigned long)arg2;                           \
1978       _argvec[3] = (unsigned long)arg3;                           \
1979       _argvec[4] = (unsigned long)arg4;                           \
1980       _argvec[5] = (unsigned long)arg5;                           \
1981       _argvec[6] = (unsigned long)arg6;                           \
1982       __asm__ volatile(                                           \
1983          "mr 11,%1\n\t"                                           \
1984          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1985          "lwz 4,8(11)\n\t"                                        \
1986          "lwz 5,12(11)\n\t"                                       \
1987          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1988          "lwz 7,20(11)\n\t"                                       \
1989          "lwz 8,24(11)\n\t"                                       \
1990          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1991          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1992          "mr %0,3"                                                \
1993          : /*out*/   "=r" (_res)                                  \
1994          : /*in*/    "r" (&_argvec[0])                            \
1995          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1996       );                                                          \
1997       lval = (__typeof__(lval)) _res;                             \
1998    } while (0)
1999
2000 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2001                                  arg7)                            \
2002    do {                                                           \
2003       volatile OrigFn        _orig = (orig);                      \
2004       volatile unsigned long _argvec[8];                          \
2005       volatile unsigned long _res;                                \
2006       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2007       _argvec[1] = (unsigned long)arg1;                           \
2008       _argvec[2] = (unsigned long)arg2;                           \
2009       _argvec[3] = (unsigned long)arg3;                           \
2010       _argvec[4] = (unsigned long)arg4;                           \
2011       _argvec[5] = (unsigned long)arg5;                           \
2012       _argvec[6] = (unsigned long)arg6;                           \
2013       _argvec[7] = (unsigned long)arg7;                           \
2014       __asm__ volatile(                                           \
2015          "mr 11,%1\n\t"                                           \
2016          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2017          "lwz 4,8(11)\n\t"                                        \
2018          "lwz 5,12(11)\n\t"                                       \
2019          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2020          "lwz 7,20(11)\n\t"                                       \
2021          "lwz 8,24(11)\n\t"                                       \
2022          "lwz 9,28(11)\n\t"                                       \
2023          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2024          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2025          "mr %0,3"                                                \
2026          : /*out*/   "=r" (_res)                                  \
2027          : /*in*/    "r" (&_argvec[0])                            \
2028          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2029       );                                                          \
2030       lval = (__typeof__(lval)) _res;                             \
2031    } while (0)
2032
2033 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2034                                  arg7,arg8)                       \
2035    do {                                                           \
2036       volatile OrigFn        _orig = (orig);                      \
2037       volatile unsigned long _argvec[9];                          \
2038       volatile unsigned long _res;                                \
2039       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2040       _argvec[1] = (unsigned long)arg1;                           \
2041       _argvec[2] = (unsigned long)arg2;                           \
2042       _argvec[3] = (unsigned long)arg3;                           \
2043       _argvec[4] = (unsigned long)arg4;                           \
2044       _argvec[5] = (unsigned long)arg5;                           \
2045       _argvec[6] = (unsigned long)arg6;                           \
2046       _argvec[7] = (unsigned long)arg7;                           \
2047       _argvec[8] = (unsigned long)arg8;                           \
2048       __asm__ volatile(                                           \
2049          "mr 11,%1\n\t"                                           \
2050          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2051          "lwz 4,8(11)\n\t"                                        \
2052          "lwz 5,12(11)\n\t"                                       \
2053          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2054          "lwz 7,20(11)\n\t"                                       \
2055          "lwz 8,24(11)\n\t"                                       \
2056          "lwz 9,28(11)\n\t"                                       \
2057          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2058          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2059          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2060          "mr %0,3"                                                \
2061          : /*out*/   "=r" (_res)                                  \
2062          : /*in*/    "r" (&_argvec[0])                            \
2063          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2064       );                                                          \
2065       lval = (__typeof__(lval)) _res;                             \
2066    } while (0)
2067
2068 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2069                                  arg7,arg8,arg9)                  \
2070    do {                                                           \
2071       volatile OrigFn        _orig = (orig);                      \
2072       volatile unsigned long _argvec[10];                         \
2073       volatile unsigned long _res;                                \
2074       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2075       _argvec[1] = (unsigned long)arg1;                           \
2076       _argvec[2] = (unsigned long)arg2;                           \
2077       _argvec[3] = (unsigned long)arg3;                           \
2078       _argvec[4] = (unsigned long)arg4;                           \
2079       _argvec[5] = (unsigned long)arg5;                           \
2080       _argvec[6] = (unsigned long)arg6;                           \
2081       _argvec[7] = (unsigned long)arg7;                           \
2082       _argvec[8] = (unsigned long)arg8;                           \
2083       _argvec[9] = (unsigned long)arg9;                           \
2084       __asm__ volatile(                                           \
2085          "mr 11,%1\n\t"                                           \
2086          "addi 1,1,-16\n\t"                                       \
2087          /* arg9 */                                               \
2088          "lwz 3,36(11)\n\t"                                       \
2089          "stw 3,8(1)\n\t"                                         \
2090          /* args1-8 */                                            \
2091          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2092          "lwz 4,8(11)\n\t"                                        \
2093          "lwz 5,12(11)\n\t"                                       \
2094          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2095          "lwz 7,20(11)\n\t"                                       \
2096          "lwz 8,24(11)\n\t"                                       \
2097          "lwz 9,28(11)\n\t"                                       \
2098          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2099          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2100          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2101          "addi 1,1,16\n\t"                                        \
2102          "mr %0,3"                                                \
2103          : /*out*/   "=r" (_res)                                  \
2104          : /*in*/    "r" (&_argvec[0])                            \
2105          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2106       );                                                          \
2107       lval = (__typeof__(lval)) _res;                             \
2108    } while (0)
2109
2110 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2111                                   arg7,arg8,arg9,arg10)           \
2112    do {                                                           \
2113       volatile OrigFn        _orig = (orig);                      \
2114       volatile unsigned long _argvec[11];                         \
2115       volatile unsigned long _res;                                \
2116       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2117       _argvec[1] = (unsigned long)arg1;                           \
2118       _argvec[2] = (unsigned long)arg2;                           \
2119       _argvec[3] = (unsigned long)arg3;                           \
2120       _argvec[4] = (unsigned long)arg4;                           \
2121       _argvec[5] = (unsigned long)arg5;                           \
2122       _argvec[6] = (unsigned long)arg6;                           \
2123       _argvec[7] = (unsigned long)arg7;                           \
2124       _argvec[8] = (unsigned long)arg8;                           \
2125       _argvec[9] = (unsigned long)arg9;                           \
2126       _argvec[10] = (unsigned long)arg10;                         \
2127       __asm__ volatile(                                           \
2128          "mr 11,%1\n\t"                                           \
2129          "addi 1,1,-16\n\t"                                       \
2130          /* arg10 */                                              \
2131          "lwz 3,40(11)\n\t"                                       \
2132          "stw 3,12(1)\n\t"                                        \
2133          /* arg9 */                                               \
2134          "lwz 3,36(11)\n\t"                                       \
2135          "stw 3,8(1)\n\t"                                         \
2136          /* args1-8 */                                            \
2137          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2138          "lwz 4,8(11)\n\t"                                        \
2139          "lwz 5,12(11)\n\t"                                       \
2140          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2141          "lwz 7,20(11)\n\t"                                       \
2142          "lwz 8,24(11)\n\t"                                       \
2143          "lwz 9,28(11)\n\t"                                       \
2144          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2145          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2146          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2147          "addi 1,1,16\n\t"                                        \
2148          "mr %0,3"                                                \
2149          : /*out*/   "=r" (_res)                                  \
2150          : /*in*/    "r" (&_argvec[0])                            \
2151          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2152       );                                                          \
2153       lval = (__typeof__(lval)) _res;                             \
2154    } while (0)
2155
2156 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2157                                   arg7,arg8,arg9,arg10,arg11)     \
2158    do {                                                           \
2159       volatile OrigFn        _orig = (orig);                      \
2160       volatile unsigned long _argvec[12];                         \
2161       volatile unsigned long _res;                                \
2162       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2163       _argvec[1] = (unsigned long)arg1;                           \
2164       _argvec[2] = (unsigned long)arg2;                           \
2165       _argvec[3] = (unsigned long)arg3;                           \
2166       _argvec[4] = (unsigned long)arg4;                           \
2167       _argvec[5] = (unsigned long)arg5;                           \
2168       _argvec[6] = (unsigned long)arg6;                           \
2169       _argvec[7] = (unsigned long)arg7;                           \
2170       _argvec[8] = (unsigned long)arg8;                           \
2171       _argvec[9] = (unsigned long)arg9;                           \
2172       _argvec[10] = (unsigned long)arg10;                         \
2173       _argvec[11] = (unsigned long)arg11;                         \
2174       __asm__ volatile(                                           \
2175          "mr 11,%1\n\t"                                           \
2176          "addi 1,1,-32\n\t"                                       \
2177          /* arg11 */                                              \
2178          "lwz 3,44(11)\n\t"                                       \
2179          "stw 3,16(1)\n\t"                                        \
2180          /* arg10 */                                              \
2181          "lwz 3,40(11)\n\t"                                       \
2182          "stw 3,12(1)\n\t"                                        \
2183          /* arg9 */                                               \
2184          "lwz 3,36(11)\n\t"                                       \
2185          "stw 3,8(1)\n\t"                                         \
2186          /* args1-8 */                                            \
2187          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2188          "lwz 4,8(11)\n\t"                                        \
2189          "lwz 5,12(11)\n\t"                                       \
2190          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2191          "lwz 7,20(11)\n\t"                                       \
2192          "lwz 8,24(11)\n\t"                                       \
2193          "lwz 9,28(11)\n\t"                                       \
2194          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2195          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2196          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2197          "addi 1,1,32\n\t"                                        \
2198          "mr %0,3"                                                \
2199          : /*out*/   "=r" (_res)                                  \
2200          : /*in*/    "r" (&_argvec[0])                            \
2201          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2202       );                                                          \
2203       lval = (__typeof__(lval)) _res;                             \
2204    } while (0)
2205
2206 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2207                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2208    do {                                                           \
2209       volatile OrigFn        _orig = (orig);                      \
2210       volatile unsigned long _argvec[13];                         \
2211       volatile unsigned long _res;                                \
2212       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2213       _argvec[1] = (unsigned long)arg1;                           \
2214       _argvec[2] = (unsigned long)arg2;                           \
2215       _argvec[3] = (unsigned long)arg3;                           \
2216       _argvec[4] = (unsigned long)arg4;                           \
2217       _argvec[5] = (unsigned long)arg5;                           \
2218       _argvec[6] = (unsigned long)arg6;                           \
2219       _argvec[7] = (unsigned long)arg7;                           \
2220       _argvec[8] = (unsigned long)arg8;                           \
2221       _argvec[9] = (unsigned long)arg9;                           \
2222       _argvec[10] = (unsigned long)arg10;                         \
2223       _argvec[11] = (unsigned long)arg11;                         \
2224       _argvec[12] = (unsigned long)arg12;                         \
2225       __asm__ volatile(                                           \
2226          "mr 11,%1\n\t"                                           \
2227          "addi 1,1,-32\n\t"                                       \
2228          /* arg12 */                                              \
2229          "lwz 3,48(11)\n\t"                                       \
2230          "stw 3,20(1)\n\t"                                        \
2231          /* arg11 */                                              \
2232          "lwz 3,44(11)\n\t"                                       \
2233          "stw 3,16(1)\n\t"                                        \
2234          /* arg10 */                                              \
2235          "lwz 3,40(11)\n\t"                                       \
2236          "stw 3,12(1)\n\t"                                        \
2237          /* arg9 */                                               \
2238          "lwz 3,36(11)\n\t"                                       \
2239          "stw 3,8(1)\n\t"                                         \
2240          /* args1-8 */                                            \
2241          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2242          "lwz 4,8(11)\n\t"                                        \
2243          "lwz 5,12(11)\n\t"                                       \
2244          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2245          "lwz 7,20(11)\n\t"                                       \
2246          "lwz 8,24(11)\n\t"                                       \
2247          "lwz 9,28(11)\n\t"                                       \
2248          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2249          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2250          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2251          "addi 1,1,32\n\t"                                        \
2252          "mr %0,3"                                                \
2253          : /*out*/   "=r" (_res)                                  \
2254          : /*in*/    "r" (&_argvec[0])                            \
2255          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2256       );                                                          \
2257       lval = (__typeof__(lval)) _res;                             \
2258    } while (0)
2259
2260 #endif /* PLAT_ppc32_linux */
2261
2262 /* ------------------------ ppc64-linux ------------------------ */
2263
2264 #if defined(PLAT_ppc64_linux)
2265
2266 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2267
2268 /* These regs are trashed by the hidden call. */
2269 #define __CALLER_SAVED_REGS                                       \
2270    "lr", "ctr", "xer",                                            \
2271    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2272    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2273    "r11", "r12", "r13"
2274
2275 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2276    long) == 8. */
2277
2278 #define CALL_FN_W_v(lval, orig)                                   \
2279    do {                                                           \
2280       volatile OrigFn        _orig = (orig);                      \
2281       volatile unsigned long _argvec[3+0];                        \
2282       volatile unsigned long _res;                                \
2283       /* _argvec[0] holds current r2 across the call */           \
2284       _argvec[1] = (unsigned long)_orig.r2;                       \
2285       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2286       __asm__ volatile(                                           \
2287          "mr 11,%1\n\t"                                           \
2288          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2289          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2290          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2291          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2292          "mr 11,%1\n\t"                                           \
2293          "mr %0,3\n\t"                                            \
2294          "ld 2,-16(11)" /* restore tocptr */                      \
2295          : /*out*/   "=r" (_res)                                  \
2296          : /*in*/    "r" (&_argvec[2])                            \
2297          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2298       );                                                          \
2299       lval = (__typeof__(lval)) _res;                             \
2300    } while (0)
2301
2302 #define CALL_FN_W_W(lval, orig, arg1)                             \
2303    do {                                                           \
2304       volatile OrigFn        _orig = (orig);                      \
2305       volatile unsigned long _argvec[3+1];                        \
2306       volatile unsigned long _res;                                \
2307       /* _argvec[0] holds current r2 across the call */           \
2308       _argvec[1]   = (unsigned long)_orig.r2;                     \
2309       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2310       _argvec[2+1] = (unsigned long)arg1;                         \
2311       __asm__ volatile(                                           \
2312          "mr 11,%1\n\t"                                           \
2313          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2314          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2315          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2316          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2317          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2318          "mr 11,%1\n\t"                                           \
2319          "mr %0,3\n\t"                                            \
2320          "ld 2,-16(11)" /* restore tocptr */                      \
2321          : /*out*/   "=r" (_res)                                  \
2322          : /*in*/    "r" (&_argvec[2])                            \
2323          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2324       );                                                          \
2325       lval = (__typeof__(lval)) _res;                             \
2326    } while (0)
2327
2328 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2329    do {                                                           \
2330       volatile OrigFn        _orig = (orig);                      \
2331       volatile unsigned long _argvec[3+2];                        \
2332       volatile unsigned long _res;                                \
2333       /* _argvec[0] holds current r2 across the call */           \
2334       _argvec[1]   = (unsigned long)_orig.r2;                     \
2335       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2336       _argvec[2+1] = (unsigned long)arg1;                         \
2337       _argvec[2+2] = (unsigned long)arg2;                         \
2338       __asm__ volatile(                                           \
2339          "mr 11,%1\n\t"                                           \
2340          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2341          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2342          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2343          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2344          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2345          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2346          "mr 11,%1\n\t"                                           \
2347          "mr %0,3\n\t"                                            \
2348          "ld 2,-16(11)" /* restore tocptr */                      \
2349          : /*out*/   "=r" (_res)                                  \
2350          : /*in*/    "r" (&_argvec[2])                            \
2351          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2352       );                                                          \
2353       lval = (__typeof__(lval)) _res;                             \
2354    } while (0)
2355
2356 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2357    do {                                                           \
2358       volatile OrigFn        _orig = (orig);                      \
2359       volatile unsigned long _argvec[3+3];                        \
2360       volatile unsigned long _res;                                \
2361       /* _argvec[0] holds current r2 across the call */           \
2362       _argvec[1]   = (unsigned long)_orig.r2;                     \
2363       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2364       _argvec[2+1] = (unsigned long)arg1;                         \
2365       _argvec[2+2] = (unsigned long)arg2;                         \
2366       _argvec[2+3] = (unsigned long)arg3;                         \
2367       __asm__ volatile(                                           \
2368          "mr 11,%1\n\t"                                           \
2369          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2370          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2371          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2372          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2373          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2374          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2375          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2376          "mr 11,%1\n\t"                                           \
2377          "mr %0,3\n\t"                                            \
2378          "ld 2,-16(11)" /* restore tocptr */                      \
2379          : /*out*/   "=r" (_res)                                  \
2380          : /*in*/    "r" (&_argvec[2])                            \
2381          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2382       );                                                          \
2383       lval = (__typeof__(lval)) _res;                             \
2384    } while (0)
2385
2386 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2387    do {                                                           \
2388       volatile OrigFn        _orig = (orig);                      \
2389       volatile unsigned long _argvec[3+4];                        \
2390       volatile unsigned long _res;                                \
2391       /* _argvec[0] holds current r2 across the call */           \
2392       _argvec[1]   = (unsigned long)_orig.r2;                     \
2393       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2394       _argvec[2+1] = (unsigned long)arg1;                         \
2395       _argvec[2+2] = (unsigned long)arg2;                         \
2396       _argvec[2+3] = (unsigned long)arg3;                         \
2397       _argvec[2+4] = (unsigned long)arg4;                         \
2398       __asm__ volatile(                                           \
2399          "mr 11,%1\n\t"                                           \
2400          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2401          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2402          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2403          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2404          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2405          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2406          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2407          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2408          "mr 11,%1\n\t"                                           \
2409          "mr %0,3\n\t"                                            \
2410          "ld 2,-16(11)" /* restore tocptr */                      \
2411          : /*out*/   "=r" (_res)                                  \
2412          : /*in*/    "r" (&_argvec[2])                            \
2413          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2414       );                                                          \
2415       lval = (__typeof__(lval)) _res;                             \
2416    } while (0)
2417
2418 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2419    do {                                                           \
2420       volatile OrigFn        _orig = (orig);                      \
2421       volatile unsigned long _argvec[3+5];                        \
2422       volatile unsigned long _res;                                \
2423       /* _argvec[0] holds current r2 across the call */           \
2424       _argvec[1]   = (unsigned long)_orig.r2;                     \
2425       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2426       _argvec[2+1] = (unsigned long)arg1;                         \
2427       _argvec[2+2] = (unsigned long)arg2;                         \
2428       _argvec[2+3] = (unsigned long)arg3;                         \
2429       _argvec[2+4] = (unsigned long)arg4;                         \
2430       _argvec[2+5] = (unsigned long)arg5;                         \
2431       __asm__ volatile(                                           \
2432          "mr 11,%1\n\t"                                           \
2433          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2434          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2435          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2436          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2437          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2438          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2439          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2440          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2441          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2442          "mr 11,%1\n\t"                                           \
2443          "mr %0,3\n\t"                                            \
2444          "ld 2,-16(11)" /* restore tocptr */                      \
2445          : /*out*/   "=r" (_res)                                  \
2446          : /*in*/    "r" (&_argvec[2])                            \
2447          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2448       );                                                          \
2449       lval = (__typeof__(lval)) _res;                             \
2450    } while (0)
2451
2452 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2453    do {                                                           \
2454       volatile OrigFn        _orig = (orig);                      \
2455       volatile unsigned long _argvec[3+6];                        \
2456       volatile unsigned long _res;                                \
2457       /* _argvec[0] holds current r2 across the call */           \
2458       _argvec[1]   = (unsigned long)_orig.r2;                     \
2459       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2460       _argvec[2+1] = (unsigned long)arg1;                         \
2461       _argvec[2+2] = (unsigned long)arg2;                         \
2462       _argvec[2+3] = (unsigned long)arg3;                         \
2463       _argvec[2+4] = (unsigned long)arg4;                         \
2464       _argvec[2+5] = (unsigned long)arg5;                         \
2465       _argvec[2+6] = (unsigned long)arg6;                         \
2466       __asm__ volatile(                                           \
2467          "mr 11,%1\n\t"                                           \
2468          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2469          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2470          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2471          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2472          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2473          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2474          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2475          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2476          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2477          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2478          "mr 11,%1\n\t"                                           \
2479          "mr %0,3\n\t"                                            \
2480          "ld 2,-16(11)" /* restore tocptr */                      \
2481          : /*out*/   "=r" (_res)                                  \
2482          : /*in*/    "r" (&_argvec[2])                            \
2483          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2484       );                                                          \
2485       lval = (__typeof__(lval)) _res;                             \
2486    } while (0)
2487
2488 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2489                                  arg7)                            \
2490    do {                                                           \
2491       volatile OrigFn        _orig = (orig);                      \
2492       volatile unsigned long _argvec[3+7];                        \
2493       volatile unsigned long _res;                                \
2494       /* _argvec[0] holds current r2 across the call */           \
2495       _argvec[1]   = (unsigned long)_orig.r2;                     \
2496       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2497       _argvec[2+1] = (unsigned long)arg1;                         \
2498       _argvec[2+2] = (unsigned long)arg2;                         \
2499       _argvec[2+3] = (unsigned long)arg3;                         \
2500       _argvec[2+4] = (unsigned long)arg4;                         \
2501       _argvec[2+5] = (unsigned long)arg5;                         \
2502       _argvec[2+6] = (unsigned long)arg6;                         \
2503       _argvec[2+7] = (unsigned long)arg7;                         \
2504       __asm__ volatile(                                           \
2505          "mr 11,%1\n\t"                                           \
2506          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2507          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2508          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2509          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2510          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2511          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2512          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2513          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2514          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2515          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2516          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2517          "mr 11,%1\n\t"                                           \
2518          "mr %0,3\n\t"                                            \
2519          "ld 2,-16(11)" /* restore tocptr */                      \
2520          : /*out*/   "=r" (_res)                                  \
2521          : /*in*/    "r" (&_argvec[2])                            \
2522          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2523       );                                                          \
2524       lval = (__typeof__(lval)) _res;                             \
2525    } while (0)
2526
2527 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2528                                  arg7,arg8)                       \
2529    do {                                                           \
2530       volatile OrigFn        _orig = (orig);                      \
2531       volatile unsigned long _argvec[3+8];                        \
2532       volatile unsigned long _res;                                \
2533       /* _argvec[0] holds current r2 across the call */           \
2534       _argvec[1]   = (unsigned long)_orig.r2;                     \
2535       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2536       _argvec[2+1] = (unsigned long)arg1;                         \
2537       _argvec[2+2] = (unsigned long)arg2;                         \
2538       _argvec[2+3] = (unsigned long)arg3;                         \
2539       _argvec[2+4] = (unsigned long)arg4;                         \
2540       _argvec[2+5] = (unsigned long)arg5;                         \
2541       _argvec[2+6] = (unsigned long)arg6;                         \
2542       _argvec[2+7] = (unsigned long)arg7;                         \
2543       _argvec[2+8] = (unsigned long)arg8;                         \
2544       __asm__ volatile(                                           \
2545          "mr 11,%1\n\t"                                           \
2546          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2547          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2548          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2549          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2550          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2551          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2552          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2553          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2554          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2555          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2556          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2557          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2558          "mr 11,%1\n\t"                                           \
2559          "mr %0,3\n\t"                                            \
2560          "ld 2,-16(11)" /* restore tocptr */                      \
2561          : /*out*/   "=r" (_res)                                  \
2562          : /*in*/    "r" (&_argvec[2])                            \
2563          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2564       );                                                          \
2565       lval = (__typeof__(lval)) _res;                             \
2566    } while (0)
2567
2568 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2569                                  arg7,arg8,arg9)                  \
2570    do {                                                           \
2571       volatile OrigFn        _orig = (orig);                      \
2572       volatile unsigned long _argvec[3+9];                        \
2573       volatile unsigned long _res;                                \
2574       /* _argvec[0] holds current r2 across the call */           \
2575       _argvec[1]   = (unsigned long)_orig.r2;                     \
2576       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2577       _argvec[2+1] = (unsigned long)arg1;                         \
2578       _argvec[2+2] = (unsigned long)arg2;                         \
2579       _argvec[2+3] = (unsigned long)arg3;                         \
2580       _argvec[2+4] = (unsigned long)arg4;                         \
2581       _argvec[2+5] = (unsigned long)arg5;                         \
2582       _argvec[2+6] = (unsigned long)arg6;                         \
2583       _argvec[2+7] = (unsigned long)arg7;                         \
2584       _argvec[2+8] = (unsigned long)arg8;                         \
2585       _argvec[2+9] = (unsigned long)arg9;                         \
2586       __asm__ volatile(                                           \
2587          "mr 11,%1\n\t"                                           \
2588          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2589          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2590          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2591          /* arg9 */                                               \
2592          "ld  3,72(11)\n\t"                                       \
2593          "std 3,112(1)\n\t"                                       \
2594          /* args1-8 */                                            \
2595          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2596          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2597          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2598          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2599          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2600          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2601          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2602          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2603          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2604          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2605          "mr 11,%1\n\t"                                           \
2606          "mr %0,3\n\t"                                            \
2607          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2608          "addi 1,1,128"     /* restore frame */                   \
2609          : /*out*/   "=r" (_res)                                  \
2610          : /*in*/    "r" (&_argvec[2])                            \
2611          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2612       );                                                          \
2613       lval = (__typeof__(lval)) _res;                             \
2614    } while (0)
2615
2616 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2617                                   arg7,arg8,arg9,arg10)           \
2618    do {                                                           \
2619       volatile OrigFn        _orig = (orig);                      \
2620       volatile unsigned long _argvec[3+10];                       \
2621       volatile unsigned long _res;                                \
2622       /* _argvec[0] holds current r2 across the call */           \
2623       _argvec[1]   = (unsigned long)_orig.r2;                     \
2624       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2625       _argvec[2+1] = (unsigned long)arg1;                         \
2626       _argvec[2+2] = (unsigned long)arg2;                         \
2627       _argvec[2+3] = (unsigned long)arg3;                         \
2628       _argvec[2+4] = (unsigned long)arg4;                         \
2629       _argvec[2+5] = (unsigned long)arg5;                         \
2630       _argvec[2+6] = (unsigned long)arg6;                         \
2631       _argvec[2+7] = (unsigned long)arg7;                         \
2632       _argvec[2+8] = (unsigned long)arg8;                         \
2633       _argvec[2+9] = (unsigned long)arg9;                         \
2634       _argvec[2+10] = (unsigned long)arg10;                       \
2635       __asm__ volatile(                                           \
2636          "mr 11,%1\n\t"                                           \
2637          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2638          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2639          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2640          /* arg10 */                                              \
2641          "ld  3,80(11)\n\t"                                       \
2642          "std 3,120(1)\n\t"                                       \
2643          /* arg9 */                                               \
2644          "ld  3,72(11)\n\t"                                       \
2645          "std 3,112(1)\n\t"                                       \
2646          /* args1-8 */                                            \
2647          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2648          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2649          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2650          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2651          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2652          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2653          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2654          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2655          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2656          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2657          "mr 11,%1\n\t"                                           \
2658          "mr %0,3\n\t"                                            \
2659          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2660          "addi 1,1,128"     /* restore frame */                   \
2661          : /*out*/   "=r" (_res)                                  \
2662          : /*in*/    "r" (&_argvec[2])                            \
2663          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2664       );                                                          \
2665       lval = (__typeof__(lval)) _res;                             \
2666    } while (0)
2667
2668 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2669                                   arg7,arg8,arg9,arg10,arg11)     \
2670    do {                                                           \
2671       volatile OrigFn        _orig = (orig);                      \
2672       volatile unsigned long _argvec[3+11];                       \
2673       volatile unsigned long _res;                                \
2674       /* _argvec[0] holds current r2 across the call */           \
2675       _argvec[1]   = (unsigned long)_orig.r2;                     \
2676       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2677       _argvec[2+1] = (unsigned long)arg1;                         \
2678       _argvec[2+2] = (unsigned long)arg2;                         \
2679       _argvec[2+3] = (unsigned long)arg3;                         \
2680       _argvec[2+4] = (unsigned long)arg4;                         \
2681       _argvec[2+5] = (unsigned long)arg5;                         \
2682       _argvec[2+6] = (unsigned long)arg6;                         \
2683       _argvec[2+7] = (unsigned long)arg7;                         \
2684       _argvec[2+8] = (unsigned long)arg8;                         \
2685       _argvec[2+9] = (unsigned long)arg9;                         \
2686       _argvec[2+10] = (unsigned long)arg10;                       \
2687       _argvec[2+11] = (unsigned long)arg11;                       \
2688       __asm__ volatile(                                           \
2689          "mr 11,%1\n\t"                                           \
2690          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2691          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2692          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2693          /* arg11 */                                              \
2694          "ld  3,88(11)\n\t"                                       \
2695          "std 3,128(1)\n\t"                                       \
2696          /* arg10 */                                              \
2697          "ld  3,80(11)\n\t"                                       \
2698          "std 3,120(1)\n\t"                                       \
2699          /* arg9 */                                               \
2700          "ld  3,72(11)\n\t"                                       \
2701          "std 3,112(1)\n\t"                                       \
2702          /* args1-8 */                                            \
2703          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2704          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2705          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2706          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2707          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2708          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2709          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2710          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2711          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2712          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2713          "mr 11,%1\n\t"                                           \
2714          "mr %0,3\n\t"                                            \
2715          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2716          "addi 1,1,144"     /* restore frame */                   \
2717          : /*out*/   "=r" (_res)                                  \
2718          : /*in*/    "r" (&_argvec[2])                            \
2719          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2720       );                                                          \
2721       lval = (__typeof__(lval)) _res;                             \
2722    } while (0)
2723
2724 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2725                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2726    do {                                                           \
2727       volatile OrigFn        _orig = (orig);                      \
2728       volatile unsigned long _argvec[3+12];                       \
2729       volatile unsigned long _res;                                \
2730       /* _argvec[0] holds current r2 across the call */           \
2731       _argvec[1]   = (unsigned long)_orig.r2;                     \
2732       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2733       _argvec[2+1] = (unsigned long)arg1;                         \
2734       _argvec[2+2] = (unsigned long)arg2;                         \
2735       _argvec[2+3] = (unsigned long)arg3;                         \
2736       _argvec[2+4] = (unsigned long)arg4;                         \
2737       _argvec[2+5] = (unsigned long)arg5;                         \
2738       _argvec[2+6] = (unsigned long)arg6;                         \
2739       _argvec[2+7] = (unsigned long)arg7;                         \
2740       _argvec[2+8] = (unsigned long)arg8;                         \
2741       _argvec[2+9] = (unsigned long)arg9;                         \
2742       _argvec[2+10] = (unsigned long)arg10;                       \
2743       _argvec[2+11] = (unsigned long)arg11;                       \
2744       _argvec[2+12] = (unsigned long)arg12;                       \
2745       __asm__ volatile(                                           \
2746          "mr 11,%1\n\t"                                           \
2747          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2748          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2749          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2750          /* arg12 */                                              \
2751          "ld  3,96(11)\n\t"                                       \
2752          "std 3,136(1)\n\t"                                       \
2753          /* arg11 */                                              \
2754          "ld  3,88(11)\n\t"                                       \
2755          "std 3,128(1)\n\t"                                       \
2756          /* arg10 */                                              \
2757          "ld  3,80(11)\n\t"                                       \
2758          "std 3,120(1)\n\t"                                       \
2759          /* arg9 */                                               \
2760          "ld  3,72(11)\n\t"                                       \
2761          "std 3,112(1)\n\t"                                       \
2762          /* args1-8 */                                            \
2763          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2764          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2765          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2766          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2767          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2768          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2769          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2770          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2771          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2772          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2773          "mr 11,%1\n\t"                                           \
2774          "mr %0,3\n\t"                                            \
2775          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2776          "addi 1,1,144"     /* restore frame */                   \
2777          : /*out*/   "=r" (_res)                                  \
2778          : /*in*/    "r" (&_argvec[2])                            \
2779          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2780       );                                                          \
2781       lval = (__typeof__(lval)) _res;                             \
2782    } while (0)
2783
2784 #endif /* PLAT_ppc64_linux */
2785
2786 /* ------------------------- arm-linux ------------------------- */
2787
2788 #if defined(PLAT_arm_linux)
2789
2790 /* These regs are trashed by the hidden call. */
2791 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
2792
2793 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
2794    long) == 4. */
2795
2796 #define CALL_FN_W_v(lval, orig)                                   \
2797    do {                                                           \
2798       volatile OrigFn        _orig = (orig);                      \
2799       volatile unsigned long _argvec[1];                          \
2800       volatile unsigned long _res;                                \
2801       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2802       __asm__ volatile(                                           \
2803          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2804          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2805          "mov %0, r0\n"                                           \
2806          : /*out*/   "=r" (_res)                                  \
2807          : /*in*/    "0" (&_argvec[0])                            \
2808          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2809       );                                                          \
2810       lval = (__typeof__(lval)) _res;                             \
2811    } while (0)
2812
2813 #define CALL_FN_W_W(lval, orig, arg1)                             \
2814    do {                                                           \
2815       volatile OrigFn        _orig = (orig);                      \
2816       volatile unsigned long _argvec[2];                          \
2817       volatile unsigned long _res;                                \
2818       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2819       _argvec[1] = (unsigned long)(arg1);                         \
2820       __asm__ volatile(                                           \
2821          "ldr r0, [%1, #4] \n\t"                                  \
2822          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2823          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2824          "mov %0, r0\n"                                           \
2825          : /*out*/   "=r" (_res)                                  \
2826          : /*in*/    "0" (&_argvec[0])                            \
2827          : /*trash*/ "cc", "memory",  __CALLER_SAVED_REGS         \
2828       );                                                          \
2829       lval = (__typeof__(lval)) _res;                             \
2830    } while (0)
2831
2832 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2833    do {                                                           \
2834       volatile OrigFn        _orig = (orig);                      \
2835       volatile unsigned long _argvec[3];                          \
2836       volatile unsigned long _res;                                \
2837       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2838       _argvec[1] = (unsigned long)(arg1);                         \
2839       _argvec[2] = (unsigned long)(arg2);                         \
2840       __asm__ volatile(                                           \
2841          "ldr r0, [%1, #4] \n\t"                                  \
2842          "ldr r1, [%1, #8] \n\t"                                  \
2843          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2844          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2845          "mov %0, r0\n"                                           \
2846          : /*out*/   "=r" (_res)                                  \
2847          : /*in*/    "0" (&_argvec[0])                            \
2848          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2849       );                                                          \
2850       lval = (__typeof__(lval)) _res;                             \
2851    } while (0)
2852
2853 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2854    do {                                                           \
2855       volatile OrigFn        _orig = (orig);                      \
2856       volatile unsigned long _argvec[4];                          \
2857       volatile unsigned long _res;                                \
2858       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2859       _argvec[1] = (unsigned long)(arg1);                         \
2860       _argvec[2] = (unsigned long)(arg2);                         \
2861       _argvec[3] = (unsigned long)(arg3);                         \
2862       __asm__ volatile(                                           \
2863          "ldr r0, [%1, #4] \n\t"                                  \
2864          "ldr r1, [%1, #8] \n\t"                                  \
2865          "ldr r2, [%1, #12] \n\t"                                 \
2866          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2867          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2868          "mov %0, r0\n"                                           \
2869          : /*out*/   "=r" (_res)                                  \
2870          : /*in*/    "0" (&_argvec[0])                            \
2871          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2872       );                                                          \
2873       lval = (__typeof__(lval)) _res;                             \
2874    } while (0)
2875
2876 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2877    do {                                                           \
2878       volatile OrigFn        _orig = (orig);                      \
2879       volatile unsigned long _argvec[5];                          \
2880       volatile unsigned long _res;                                \
2881       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2882       _argvec[1] = (unsigned long)(arg1);                         \
2883       _argvec[2] = (unsigned long)(arg2);                         \
2884       _argvec[3] = (unsigned long)(arg3);                         \
2885       _argvec[4] = (unsigned long)(arg4);                         \
2886       __asm__ volatile(                                           \
2887          "ldr r0, [%1, #4] \n\t"                                  \
2888          "ldr r1, [%1, #8] \n\t"                                  \
2889          "ldr r2, [%1, #12] \n\t"                                 \
2890          "ldr r3, [%1, #16] \n\t"                                 \
2891          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2892          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2893          "mov %0, r0"                                             \
2894          : /*out*/   "=r" (_res)                                  \
2895          : /*in*/    "0" (&_argvec[0])                            \
2896          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2897       );                                                          \
2898       lval = (__typeof__(lval)) _res;                             \
2899    } while (0)
2900
2901 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2902    do {                                                           \
2903       volatile OrigFn        _orig = (orig);                      \
2904       volatile unsigned long _argvec[6];                          \
2905       volatile unsigned long _res;                                \
2906       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2907       _argvec[1] = (unsigned long)(arg1);                         \
2908       _argvec[2] = (unsigned long)(arg2);                         \
2909       _argvec[3] = (unsigned long)(arg3);                         \
2910       _argvec[4] = (unsigned long)(arg4);                         \
2911       _argvec[5] = (unsigned long)(arg5);                         \
2912       __asm__ volatile(                                           \
2913          "ldr r0, [%1, #20] \n\t"                                 \
2914          "push {r0} \n\t"                                         \
2915          "ldr r0, [%1, #4] \n\t"                                  \
2916          "ldr r1, [%1, #8] \n\t"                                  \
2917          "ldr r2, [%1, #12] \n\t"                                 \
2918          "ldr r3, [%1, #16] \n\t"                                 \
2919          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2920          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2921          "add sp, sp, #4 \n\t"                                    \
2922          "mov %0, r0"                                             \
2923          : /*out*/   "=r" (_res)                                  \
2924          : /*in*/    "0" (&_argvec[0])                            \
2925          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2926       );                                                          \
2927       lval = (__typeof__(lval)) _res;                             \
2928    } while (0)
2929
2930 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2931    do {                                                           \
2932       volatile OrigFn        _orig = (orig);                      \
2933       volatile unsigned long _argvec[7];                          \
2934       volatile unsigned long _res;                                \
2935       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2936       _argvec[1] = (unsigned long)(arg1);                         \
2937       _argvec[2] = (unsigned long)(arg2);                         \
2938       _argvec[3] = (unsigned long)(arg3);                         \
2939       _argvec[4] = (unsigned long)(arg4);                         \
2940       _argvec[5] = (unsigned long)(arg5);                         \
2941       _argvec[6] = (unsigned long)(arg6);                         \
2942       __asm__ volatile(                                           \
2943          "ldr r0, [%1, #20] \n\t"                                 \
2944          "ldr r1, [%1, #24] \n\t"                                 \
2945          "push {r0, r1} \n\t"                                     \
2946          "ldr r0, [%1, #4] \n\t"                                  \
2947          "ldr r1, [%1, #8] \n\t"                                  \
2948          "ldr r2, [%1, #12] \n\t"                                 \
2949          "ldr r3, [%1, #16] \n\t"                                 \
2950          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2951          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2952          "add sp, sp, #8 \n\t"                                    \
2953          "mov %0, r0"                                             \
2954          : /*out*/   "=r" (_res)                                  \
2955          : /*in*/    "0" (&_argvec[0])                            \
2956          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2957       );                                                          \
2958       lval = (__typeof__(lval)) _res;                             \
2959    } while (0)
2960
2961 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2962                                  arg7)                            \
2963    do {                                                           \
2964       volatile OrigFn        _orig = (orig);                      \
2965       volatile unsigned long _argvec[8];                          \
2966       volatile unsigned long _res;                                \
2967       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2968       _argvec[1] = (unsigned long)(arg1);                         \
2969       _argvec[2] = (unsigned long)(arg2);                         \
2970       _argvec[3] = (unsigned long)(arg3);                         \
2971       _argvec[4] = (unsigned long)(arg4);                         \
2972       _argvec[5] = (unsigned long)(arg5);                         \
2973       _argvec[6] = (unsigned long)(arg6);                         \
2974       _argvec[7] = (unsigned long)(arg7);                         \
2975       __asm__ volatile(                                           \
2976          "ldr r0, [%1, #20] \n\t"                                 \
2977          "ldr r1, [%1, #24] \n\t"                                 \
2978          "ldr r2, [%1, #28] \n\t"                                 \
2979          "push {r0, r1, r2} \n\t"                                 \
2980          "ldr r0, [%1, #4] \n\t"                                  \
2981          "ldr r1, [%1, #8] \n\t"                                  \
2982          "ldr r2, [%1, #12] \n\t"                                 \
2983          "ldr r3, [%1, #16] \n\t"                                 \
2984          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2985          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2986          "add sp, sp, #12 \n\t"                                   \
2987          "mov %0, r0"                                             \
2988          : /*out*/   "=r" (_res)                                  \
2989          : /*in*/    "0" (&_argvec[0])                            \
2990          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2991       );                                                          \
2992       lval = (__typeof__(lval)) _res;                             \
2993    } while (0)
2994
2995 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2996                                  arg7,arg8)                       \
2997    do {                                                           \
2998       volatile OrigFn        _orig = (orig);                      \
2999       volatile unsigned long _argvec[9];                          \
3000       volatile unsigned long _res;                                \
3001       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3002       _argvec[1] = (unsigned long)(arg1);                         \
3003       _argvec[2] = (unsigned long)(arg2);                         \
3004       _argvec[3] = (unsigned long)(arg3);                         \
3005       _argvec[4] = (unsigned long)(arg4);                         \
3006       _argvec[5] = (unsigned long)(arg5);                         \
3007       _argvec[6] = (unsigned long)(arg6);                         \
3008       _argvec[7] = (unsigned long)(arg7);                         \
3009       _argvec[8] = (unsigned long)(arg8);                         \
3010       __asm__ volatile(                                           \
3011          "ldr r0, [%1, #20] \n\t"                                 \
3012          "ldr r1, [%1, #24] \n\t"                                 \
3013          "ldr r2, [%1, #28] \n\t"                                 \
3014          "ldr r3, [%1, #32] \n\t"                                 \
3015          "push {r0, r1, r2, r3} \n\t"                             \
3016          "ldr r0, [%1, #4] \n\t"                                  \
3017          "ldr r1, [%1, #8] \n\t"                                  \
3018          "ldr r2, [%1, #12] \n\t"                                 \
3019          "ldr r3, [%1, #16] \n\t"                                 \
3020          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3021          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3022          "add sp, sp, #16 \n\t"                                   \
3023          "mov %0, r0"                                             \
3024          : /*out*/   "=r" (_res)                                  \
3025          : /*in*/    "0" (&_argvec[0])                            \
3026          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3027       );                                                          \
3028       lval = (__typeof__(lval)) _res;                             \
3029    } while (0)
3030
3031 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3032                                  arg7,arg8,arg9)                  \
3033    do {                                                           \
3034       volatile OrigFn        _orig = (orig);                      \
3035       volatile unsigned long _argvec[10];                         \
3036       volatile unsigned long _res;                                \
3037       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3038       _argvec[1] = (unsigned long)(arg1);                         \
3039       _argvec[2] = (unsigned long)(arg2);                         \
3040       _argvec[3] = (unsigned long)(arg3);                         \
3041       _argvec[4] = (unsigned long)(arg4);                         \
3042       _argvec[5] = (unsigned long)(arg5);                         \
3043       _argvec[6] = (unsigned long)(arg6);                         \
3044       _argvec[7] = (unsigned long)(arg7);                         \
3045       _argvec[8] = (unsigned long)(arg8);                         \
3046       _argvec[9] = (unsigned long)(arg9);                         \
3047       __asm__ volatile(                                           \
3048          "ldr r0, [%1, #20] \n\t"                                 \
3049          "ldr r1, [%1, #24] \n\t"                                 \
3050          "ldr r2, [%1, #28] \n\t"                                 \
3051          "ldr r3, [%1, #32] \n\t"                                 \
3052          "ldr r4, [%1, #36] \n\t"                                 \
3053          "push {r0, r1, r2, r3, r4} \n\t"                         \
3054          "ldr r0, [%1, #4] \n\t"                                  \
3055          "ldr r1, [%1, #8] \n\t"                                  \
3056          "ldr r2, [%1, #12] \n\t"                                 \
3057          "ldr r3, [%1, #16] \n\t"                                 \
3058          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3059          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3060          "add sp, sp, #20 \n\t"                                   \
3061          "mov %0, r0"                                             \
3062          : /*out*/   "=r" (_res)                                  \
3063          : /*in*/    "0" (&_argvec[0])                            \
3064          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3065       );                                                          \
3066       lval = (__typeof__(lval)) _res;                             \
3067    } while (0)
3068
3069 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3070                                   arg7,arg8,arg9,arg10)           \
3071    do {                                                           \
3072       volatile OrigFn        _orig = (orig);                      \
3073       volatile unsigned long _argvec[11];                         \
3074       volatile unsigned long _res;                                \
3075       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3076       _argvec[1] = (unsigned long)(arg1);                         \
3077       _argvec[2] = (unsigned long)(arg2);                         \
3078       _argvec[3] = (unsigned long)(arg3);                         \
3079       _argvec[4] = (unsigned long)(arg4);                         \
3080       _argvec[5] = (unsigned long)(arg5);                         \
3081       _argvec[6] = (unsigned long)(arg6);                         \
3082       _argvec[7] = (unsigned long)(arg7);                         \
3083       _argvec[8] = (unsigned long)(arg8);                         \
3084       _argvec[9] = (unsigned long)(arg9);                         \
3085       _argvec[10] = (unsigned long)(arg10);                       \
3086       __asm__ volatile(                                           \
3087          "ldr r0, [%1, #40] \n\t"                                 \
3088          "push {r0} \n\t"                                         \
3089          "ldr r0, [%1, #20] \n\t"                                 \
3090          "ldr r1, [%1, #24] \n\t"                                 \
3091          "ldr r2, [%1, #28] \n\t"                                 \
3092          "ldr r3, [%1, #32] \n\t"                                 \
3093          "ldr r4, [%1, #36] \n\t"                                 \
3094          "push {r0, r1, r2, r3, r4} \n\t"                         \
3095          "ldr r0, [%1, #4] \n\t"                                  \
3096          "ldr r1, [%1, #8] \n\t"                                  \
3097          "ldr r2, [%1, #12] \n\t"                                 \
3098          "ldr r3, [%1, #16] \n\t"                                 \
3099          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3100          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3101          "add sp, sp, #24 \n\t"                                   \
3102          "mov %0, r0"                                             \
3103          : /*out*/   "=r" (_res)                                  \
3104          : /*in*/    "0" (&_argvec[0])                            \
3105          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3106       );                                                          \
3107       lval = (__typeof__(lval)) _res;                             \
3108    } while (0)
3109
3110 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3111                                   arg6,arg7,arg8,arg9,arg10,      \
3112                                   arg11)                          \
3113    do {                                                           \
3114       volatile OrigFn        _orig = (orig);                      \
3115       volatile unsigned long _argvec[12];                         \
3116       volatile unsigned long _res;                                \
3117       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3118       _argvec[1] = (unsigned long)(arg1);                         \
3119       _argvec[2] = (unsigned long)(arg2);                         \
3120       _argvec[3] = (unsigned long)(arg3);                         \
3121       _argvec[4] = (unsigned long)(arg4);                         \
3122       _argvec[5] = (unsigned long)(arg5);                         \
3123       _argvec[6] = (unsigned long)(arg6);                         \
3124       _argvec[7] = (unsigned long)(arg7);                         \
3125       _argvec[8] = (unsigned long)(arg8);                         \
3126       _argvec[9] = (unsigned long)(arg9);                         \
3127       _argvec[10] = (unsigned long)(arg10);                       \
3128       _argvec[11] = (unsigned long)(arg11);                       \
3129       __asm__ volatile(                                           \
3130          "ldr r0, [%1, #40] \n\t"                                 \
3131          "ldr r1, [%1, #44] \n\t"                                 \
3132          "push {r0, r1} \n\t"                                     \
3133          "ldr r0, [%1, #20] \n\t"                                 \
3134          "ldr r1, [%1, #24] \n\t"                                 \
3135          "ldr r2, [%1, #28] \n\t"                                 \
3136          "ldr r3, [%1, #32] \n\t"                                 \
3137          "ldr r4, [%1, #36] \n\t"                                 \
3138          "push {r0, r1, r2, r3, r4} \n\t"                         \
3139          "ldr r0, [%1, #4] \n\t"                                  \
3140          "ldr r1, [%1, #8] \n\t"                                  \
3141          "ldr r2, [%1, #12] \n\t"                                 \
3142          "ldr r3, [%1, #16] \n\t"                                 \
3143          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3144          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3145          "add sp, sp, #28 \n\t"                                   \
3146          "mov %0, r0"                                             \
3147          : /*out*/   "=r" (_res)                                  \
3148          : /*in*/    "0" (&_argvec[0])                            \
3149          : /*trash*/ "cc", "memory",__CALLER_SAVED_REGS           \
3150       );                                                          \
3151       lval = (__typeof__(lval)) _res;                             \
3152    } while (0)
3153
3154 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3155                                   arg6,arg7,arg8,arg9,arg10,      \
3156                                   arg11,arg12)                    \
3157    do {                                                           \
3158       volatile OrigFn        _orig = (orig);                      \
3159       volatile unsigned long _argvec[13];                         \
3160       volatile unsigned long _res;                                \
3161       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3162       _argvec[1] = (unsigned long)(arg1);                         \
3163       _argvec[2] = (unsigned long)(arg2);                         \
3164       _argvec[3] = (unsigned long)(arg3);                         \
3165       _argvec[4] = (unsigned long)(arg4);                         \
3166       _argvec[5] = (unsigned long)(arg5);                         \
3167       _argvec[6] = (unsigned long)(arg6);                         \
3168       _argvec[7] = (unsigned long)(arg7);                         \
3169       _argvec[8] = (unsigned long)(arg8);                         \
3170       _argvec[9] = (unsigned long)(arg9);                         \
3171       _argvec[10] = (unsigned long)(arg10);                       \
3172       _argvec[11] = (unsigned long)(arg11);                       \
3173       _argvec[12] = (unsigned long)(arg12);                       \
3174       __asm__ volatile(                                           \
3175          "ldr r0, [%1, #40] \n\t"                                 \
3176          "ldr r1, [%1, #44] \n\t"                                 \
3177          "ldr r2, [%1, #48] \n\t"                                 \
3178          "push {r0, r1, r2} \n\t"                                 \
3179          "ldr r0, [%1, #20] \n\t"                                 \
3180          "ldr r1, [%1, #24] \n\t"                                 \
3181          "ldr r2, [%1, #28] \n\t"                                 \
3182          "ldr r3, [%1, #32] \n\t"                                 \
3183          "ldr r4, [%1, #36] \n\t"                                 \
3184          "push {r0, r1, r2, r3, r4} \n\t"                         \
3185          "ldr r0, [%1, #4] \n\t"                                  \
3186          "ldr r1, [%1, #8] \n\t"                                  \
3187          "ldr r2, [%1, #12] \n\t"                                 \
3188          "ldr r3, [%1, #16] \n\t"                                 \
3189          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3190          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3191          "add sp, sp, #32 \n\t"                                   \
3192          "mov %0, r0"                                             \
3193          : /*out*/   "=r" (_res)                                  \
3194          : /*in*/    "0" (&_argvec[0])                            \
3195          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3196       );                                                          \
3197       lval = (__typeof__(lval)) _res;                             \
3198    } while (0)
3199
3200 #endif /* PLAT_arm_linux */
3201
3202 /* ------------------------ ppc32-aix5 ------------------------- */
3203
3204 #if defined(PLAT_ppc32_aix5)
3205
3206 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3207
3208 /* These regs are trashed by the hidden call. */
3209 #define __CALLER_SAVED_REGS                                       \
3210    "lr", "ctr", "xer",                                            \
3211    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3212    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3213    "r11", "r12", "r13"
3214
3215 /* Expand the stack frame, copying enough info that unwinding
3216    still works.  Trashes r3. */
3217
3218 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3219          "addi 1,1,-" #_n_fr "\n\t"                               \
3220          "lwz  3," #_n_fr "(1)\n\t"                               \
3221          "stw  3,0(1)\n\t"
3222
3223 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3224          "addi 1,1," #_n_fr "\n\t"
3225
3226 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
3227    long) == 4. */
3228
3229 #define CALL_FN_W_v(lval, orig)                                   \
3230    do {                                                           \
3231       volatile OrigFn        _orig = (orig);                      \
3232       volatile unsigned long _argvec[3+0];                        \
3233       volatile unsigned long _res;                                \
3234       /* _argvec[0] holds current r2 across the call */           \
3235       _argvec[1] = (unsigned long)_orig.r2;                       \
3236       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3237       __asm__ volatile(                                           \
3238          "mr 11,%1\n\t"                                           \
3239          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3240          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3241          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3242          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3243          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3244          "mr 11,%1\n\t"                                           \
3245          "mr %0,3\n\t"                                            \
3246          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3247          VG_CONTRACT_FRAME_BY(512)                                \
3248          : /*out*/   "=r" (_res)                                  \
3249          : /*in*/    "r" (&_argvec[2])                            \
3250          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3251       );                                                          \
3252       lval = (__typeof__(lval)) _res;                             \
3253    } while (0)
3254
3255 #define CALL_FN_W_W(lval, orig, arg1)                             \
3256    do {                                                           \
3257       volatile OrigFn        _orig = (orig);                      \
3258       volatile unsigned long _argvec[3+1];                        \
3259       volatile unsigned long _res;                                \
3260       /* _argvec[0] holds current r2 across the call */           \
3261       _argvec[1]   = (unsigned long)_orig.r2;                     \
3262       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3263       _argvec[2+1] = (unsigned long)arg1;                         \
3264       __asm__ volatile(                                           \
3265          "mr 11,%1\n\t"                                           \
3266          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3267          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3268          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3269          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3270          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3271          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3272          "mr 11,%1\n\t"                                           \
3273          "mr %0,3\n\t"                                            \
3274          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3275          VG_CONTRACT_FRAME_BY(512)                                \
3276          : /*out*/   "=r" (_res)                                  \
3277          : /*in*/    "r" (&_argvec[2])                            \
3278          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3279       );                                                          \
3280       lval = (__typeof__(lval)) _res;                             \
3281    } while (0)
3282
3283 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3284    do {                                                           \
3285       volatile OrigFn        _orig = (orig);                      \
3286       volatile unsigned long _argvec[3+2];                        \
3287       volatile unsigned long _res;                                \
3288       /* _argvec[0] holds current r2 across the call */           \
3289       _argvec[1]   = (unsigned long)_orig.r2;                     \
3290       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3291       _argvec[2+1] = (unsigned long)arg1;                         \
3292       _argvec[2+2] = (unsigned long)arg2;                         \
3293       __asm__ volatile(                                           \
3294          "mr 11,%1\n\t"                                           \
3295          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3296          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3297          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3298          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3299          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3300          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3301          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3302          "mr 11,%1\n\t"                                           \
3303          "mr %0,3\n\t"                                            \
3304          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3305          VG_CONTRACT_FRAME_BY(512)                                \
3306          : /*out*/   "=r" (_res)                                  \
3307          : /*in*/    "r" (&_argvec[2])                            \
3308          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3309       );                                                          \
3310       lval = (__typeof__(lval)) _res;                             \
3311    } while (0)
3312
3313 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3314    do {                                                           \
3315       volatile OrigFn        _orig = (orig);                      \
3316       volatile unsigned long _argvec[3+3];                        \
3317       volatile unsigned long _res;                                \
3318       /* _argvec[0] holds current r2 across the call */           \
3319       _argvec[1]   = (unsigned long)_orig.r2;                     \
3320       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3321       _argvec[2+1] = (unsigned long)arg1;                         \
3322       _argvec[2+2] = (unsigned long)arg2;                         \
3323       _argvec[2+3] = (unsigned long)arg3;                         \
3324       __asm__ volatile(                                           \
3325          "mr 11,%1\n\t"                                           \
3326          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3327          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3328          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3329          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3330          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3331          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3332          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3333          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3334          "mr 11,%1\n\t"                                           \
3335          "mr %0,3\n\t"                                            \
3336          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3337          VG_CONTRACT_FRAME_BY(512)                                \
3338          : /*out*/   "=r" (_res)                                  \
3339          : /*in*/    "r" (&_argvec[2])                            \
3340          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3341       );                                                          \
3342       lval = (__typeof__(lval)) _res;                             \
3343    } while (0)
3344
3345 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3346    do {                                                           \
3347       volatile OrigFn        _orig = (orig);                      \
3348       volatile unsigned long _argvec[3+4];                        \
3349       volatile unsigned long _res;                                \
3350       /* _argvec[0] holds current r2 across the call */           \
3351       _argvec[1]   = (unsigned long)_orig.r2;                     \
3352       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3353       _argvec[2+1] = (unsigned long)arg1;                         \
3354       _argvec[2+2] = (unsigned long)arg2;                         \
3355       _argvec[2+3] = (unsigned long)arg3;                         \
3356       _argvec[2+4] = (unsigned long)arg4;                         \
3357       __asm__ volatile(                                           \
3358          "mr 11,%1\n\t"                                           \
3359          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3360          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3361          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3362          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3363          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3364          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3365          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3366          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3367          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3368          "mr 11,%1\n\t"                                           \
3369          "mr %0,3\n\t"                                            \
3370          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3371          VG_CONTRACT_FRAME_BY(512)                                \
3372          : /*out*/   "=r" (_res)                                  \
3373          : /*in*/    "r" (&_argvec[2])                            \
3374          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3375       );                                                          \
3376       lval = (__typeof__(lval)) _res;                             \
3377    } while (0)
3378
3379 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3380    do {                                                           \
3381       volatile OrigFn        _orig = (orig);                      \
3382       volatile unsigned long _argvec[3+5];                        \
3383       volatile unsigned long _res;                                \
3384       /* _argvec[0] holds current r2 across the call */           \
3385       _argvec[1]   = (unsigned long)_orig.r2;                     \
3386       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3387       _argvec[2+1] = (unsigned long)arg1;                         \
3388       _argvec[2+2] = (unsigned long)arg2;                         \
3389       _argvec[2+3] = (unsigned long)arg3;                         \
3390       _argvec[2+4] = (unsigned long)arg4;                         \
3391       _argvec[2+5] = (unsigned long)arg5;                         \
3392       __asm__ volatile(                                           \
3393          "mr 11,%1\n\t"                                           \
3394          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3395          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3396          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3397          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3398          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
3399          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3400          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3401          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3402          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3403          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3404          "mr 11,%1\n\t"                                           \
3405          "mr %0,3\n\t"                                            \
3406          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3407          VG_CONTRACT_FRAME_BY(512)                                \
3408          : /*out*/   "=r" (_res)                                  \
3409          : /*in*/    "r" (&_argvec[2])                            \
3410          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3411       );                                                          \
3412       lval = (__typeof__(lval)) _res;                             \
3413    } while (0)
3414
3415 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3416    do {                                                           \
3417       volatile OrigFn        _orig = (orig);                      \
3418       volatile unsigned long _argvec[3+6];                        \
3419       volatile unsigned long _res;                                \
3420       /* _argvec[0] holds current r2 across the call */           \
3421       _argvec[1]   = (unsigned long)_orig.r2;                     \
3422       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3423       _argvec[2+1] = (unsigned long)arg1;                         \
3424       _argvec[2+2] = (unsigned long)arg2;                         \
3425       _argvec[2+3] = (unsigned long)arg3;                         \
3426       _argvec[2+4] = (unsigned long)arg4;                         \
3427       _argvec[2+5] = (unsigned long)arg5;                         \
3428       _argvec[2+6] = (unsigned long)arg6;                         \
3429       __asm__ volatile(                                           \
3430          "mr 11,%1\n\t"                                           \
3431          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3432          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3433          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3434          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3435          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3436          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3437          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3438          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3439          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3440          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3441          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3442          "mr 11,%1\n\t"                                           \
3443          "mr %0,3\n\t"                                            \
3444          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3445          VG_CONTRACT_FRAME_BY(512)                                \
3446          : /*out*/   "=r" (_res)                                  \
3447          : /*in*/    "r" (&_argvec[2])                            \
3448          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3449       );                                                          \
3450       lval = (__typeof__(lval)) _res;                             \
3451    } while (0)
3452
3453 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3454                                  arg7)                            \
3455    do {                                                           \
3456       volatile OrigFn        _orig = (orig);                      \
3457       volatile unsigned long _argvec[3+7];                        \
3458       volatile unsigned long _res;                                \
3459       /* _argvec[0] holds current r2 across the call */           \
3460       _argvec[1]   = (unsigned long)_orig.r2;                     \
3461       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3462       _argvec[2+1] = (unsigned long)arg1;                         \
3463       _argvec[2+2] = (unsigned long)arg2;                         \
3464       _argvec[2+3] = (unsigned long)arg3;                         \
3465       _argvec[2+4] = (unsigned long)arg4;                         \
3466       _argvec[2+5] = (unsigned long)arg5;                         \
3467       _argvec[2+6] = (unsigned long)arg6;                         \
3468       _argvec[2+7] = (unsigned long)arg7;                         \
3469       __asm__ volatile(                                           \
3470          "mr 11,%1\n\t"                                           \
3471          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3472          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3473          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3474          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3475          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3476          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3477          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3478          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3479          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3480          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3481          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3482          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3483          "mr 11,%1\n\t"                                           \
3484          "mr %0,3\n\t"                                            \
3485          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3486          VG_CONTRACT_FRAME_BY(512)                                \
3487          : /*out*/   "=r" (_res)                                  \
3488          : /*in*/    "r" (&_argvec[2])                            \
3489          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3490       );                                                          \
3491       lval = (__typeof__(lval)) _res;                             \
3492    } while (0)
3493
3494 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3495                                  arg7,arg8)                       \
3496    do {                                                           \
3497       volatile OrigFn        _orig = (orig);                      \
3498       volatile unsigned long _argvec[3+8];                        \
3499       volatile unsigned long _res;                                \
3500       /* _argvec[0] holds current r2 across the call */           \
3501       _argvec[1]   = (unsigned long)_orig.r2;                     \
3502       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3503       _argvec[2+1] = (unsigned long)arg1;                         \
3504       _argvec[2+2] = (unsigned long)arg2;                         \
3505       _argvec[2+3] = (unsigned long)arg3;                         \
3506       _argvec[2+4] = (unsigned long)arg4;                         \
3507       _argvec[2+5] = (unsigned long)arg5;                         \
3508       _argvec[2+6] = (unsigned long)arg6;                         \
3509       _argvec[2+7] = (unsigned long)arg7;                         \
3510       _argvec[2+8] = (unsigned long)arg8;                         \
3511       __asm__ volatile(                                           \
3512          "mr 11,%1\n\t"                                           \
3513          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3514          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3515          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3516          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3517          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3518          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3519          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3520          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3521          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3522          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3523          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3524          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3525          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3526          "mr 11,%1\n\t"                                           \
3527          "mr %0,3\n\t"                                            \
3528          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3529          VG_CONTRACT_FRAME_BY(512)                                \
3530          : /*out*/   "=r" (_res)                                  \
3531          : /*in*/    "r" (&_argvec[2])                            \
3532          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3533       );                                                          \
3534       lval = (__typeof__(lval)) _res;                             \
3535    } while (0)
3536
3537 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3538                                  arg7,arg8,arg9)                  \
3539    do {                                                           \
3540       volatile OrigFn        _orig = (orig);                      \
3541       volatile unsigned long _argvec[3+9];                        \
3542       volatile unsigned long _res;                                \
3543       /* _argvec[0] holds current r2 across the call */           \
3544       _argvec[1]   = (unsigned long)_orig.r2;                     \
3545       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3546       _argvec[2+1] = (unsigned long)arg1;                         \
3547       _argvec[2+2] = (unsigned long)arg2;                         \
3548       _argvec[2+3] = (unsigned long)arg3;                         \
3549       _argvec[2+4] = (unsigned long)arg4;                         \
3550       _argvec[2+5] = (unsigned long)arg5;                         \
3551       _argvec[2+6] = (unsigned long)arg6;                         \
3552       _argvec[2+7] = (unsigned long)arg7;                         \
3553       _argvec[2+8] = (unsigned long)arg8;                         \
3554       _argvec[2+9] = (unsigned long)arg9;                         \
3555       __asm__ volatile(                                           \
3556          "mr 11,%1\n\t"                                           \
3557          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3558          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3559          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3560          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3561          /* arg9 */                                               \
3562          "lwz 3,36(11)\n\t"                                       \
3563          "stw 3,56(1)\n\t"                                        \
3564          /* args1-8 */                                            \
3565          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3566          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3567          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3568          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3569          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3570          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3571          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3572          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3573          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3574          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3575          "mr 11,%1\n\t"                                           \
3576          "mr %0,3\n\t"                                            \
3577          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3578          VG_CONTRACT_FRAME_BY(64)                                 \
3579          VG_CONTRACT_FRAME_BY(512)                                \
3580          : /*out*/   "=r" (_res)                                  \
3581          : /*in*/    "r" (&_argvec[2])                            \
3582          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3583       );                                                          \
3584       lval = (__typeof__(lval)) _res;                             \
3585    } while (0)
3586
3587 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3588                                   arg7,arg8,arg9,arg10)           \
3589    do {                                                           \
3590       volatile OrigFn        _orig = (orig);                      \
3591       volatile unsigned long _argvec[3+10];                       \
3592       volatile unsigned long _res;                                \
3593       /* _argvec[0] holds current r2 across the call */           \
3594       _argvec[1]   = (unsigned long)_orig.r2;                     \
3595       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3596       _argvec[2+1] = (unsigned long)arg1;                         \
3597       _argvec[2+2] = (unsigned long)arg2;                         \
3598       _argvec[2+3] = (unsigned long)arg3;                         \
3599       _argvec[2+4] = (unsigned long)arg4;                         \
3600       _argvec[2+5] = (unsigned long)arg5;                         \
3601       _argvec[2+6] = (unsigned long)arg6;                         \
3602       _argvec[2+7] = (unsigned long)arg7;                         \
3603       _argvec[2+8] = (unsigned long)arg8;                         \
3604       _argvec[2+9] = (unsigned long)arg9;                         \
3605       _argvec[2+10] = (unsigned long)arg10;                       \
3606       __asm__ volatile(                                           \
3607          "mr 11,%1\n\t"                                           \
3608          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3609          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3610          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3611          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3612          /* arg10 */                                              \
3613          "lwz 3,40(11)\n\t"                                       \
3614          "stw 3,60(1)\n\t"                                        \
3615          /* arg9 */                                               \
3616          "lwz 3,36(11)\n\t"                                       \
3617          "stw 3,56(1)\n\t"                                        \
3618          /* args1-8 */                                            \
3619          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3620          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3621          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3622          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3623          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3624          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3625          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3626          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3627          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3628          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3629          "mr 11,%1\n\t"                                           \
3630          "mr %0,3\n\t"                                            \
3631          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3632          VG_CONTRACT_FRAME_BY(64)                                 \
3633          VG_CONTRACT_FRAME_BY(512)                                \
3634          : /*out*/   "=r" (_res)                                  \
3635          : /*in*/    "r" (&_argvec[2])                            \
3636          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3637       );                                                          \
3638       lval = (__typeof__(lval)) _res;                             \
3639    } while (0)
3640
3641 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3642                                   arg7,arg8,arg9,arg10,arg11)     \
3643    do {                                                           \
3644       volatile OrigFn        _orig = (orig);                      \
3645       volatile unsigned long _argvec[3+11];                       \
3646       volatile unsigned long _res;                                \
3647       /* _argvec[0] holds current r2 across the call */           \
3648       _argvec[1]   = (unsigned long)_orig.r2;                     \
3649       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3650       _argvec[2+1] = (unsigned long)arg1;                         \
3651       _argvec[2+2] = (unsigned long)arg2;                         \
3652       _argvec[2+3] = (unsigned long)arg3;                         \
3653       _argvec[2+4] = (unsigned long)arg4;                         \
3654       _argvec[2+5] = (unsigned long)arg5;                         \
3655       _argvec[2+6] = (unsigned long)arg6;                         \
3656       _argvec[2+7] = (unsigned long)arg7;                         \
3657       _argvec[2+8] = (unsigned long)arg8;                         \
3658       _argvec[2+9] = (unsigned long)arg9;                         \
3659       _argvec[2+10] = (unsigned long)arg10;                       \
3660       _argvec[2+11] = (unsigned long)arg11;                       \
3661       __asm__ volatile(                                           \
3662          "mr 11,%1\n\t"                                           \
3663          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3664          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3665          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3666          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3667          /* arg11 */                                              \
3668          "lwz 3,44(11)\n\t"                                       \
3669          "stw 3,64(1)\n\t"                                        \
3670          /* arg10 */                                              \
3671          "lwz 3,40(11)\n\t"                                       \
3672          "stw 3,60(1)\n\t"                                        \
3673          /* arg9 */                                               \
3674          "lwz 3,36(11)\n\t"                                       \
3675          "stw 3,56(1)\n\t"                                        \
3676          /* args1-8 */                                            \
3677          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3678          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3679          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3680          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3681          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3682          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3683          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3684          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3685          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3686          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3687          "mr 11,%1\n\t"                                           \
3688          "mr %0,3\n\t"                                            \
3689          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3690          VG_CONTRACT_FRAME_BY(72)                                 \
3691          VG_CONTRACT_FRAME_BY(512)                                \
3692          : /*out*/   "=r" (_res)                                  \
3693          : /*in*/    "r" (&_argvec[2])                            \
3694          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3695       );                                                          \
3696       lval = (__typeof__(lval)) _res;                             \
3697    } while (0)
3698
3699 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3700                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3701    do {                                                           \
3702       volatile OrigFn        _orig = (orig);                      \
3703       volatile unsigned long _argvec[3+12];                       \
3704       volatile unsigned long _res;                                \
3705       /* _argvec[0] holds current r2 across the call */           \
3706       _argvec[1]   = (unsigned long)_orig.r2;                     \
3707       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3708       _argvec[2+1] = (unsigned long)arg1;                         \
3709       _argvec[2+2] = (unsigned long)arg2;                         \
3710       _argvec[2+3] = (unsigned long)arg3;                         \
3711       _argvec[2+4] = (unsigned long)arg4;                         \
3712       _argvec[2+5] = (unsigned long)arg5;                         \
3713       _argvec[2+6] = (unsigned long)arg6;                         \
3714       _argvec[2+7] = (unsigned long)arg7;                         \
3715       _argvec[2+8] = (unsigned long)arg8;                         \
3716       _argvec[2+9] = (unsigned long)arg9;                         \
3717       _argvec[2+10] = (unsigned long)arg10;                       \
3718       _argvec[2+11] = (unsigned long)arg11;                       \
3719       _argvec[2+12] = (unsigned long)arg12;                       \
3720       __asm__ volatile(                                           \
3721          "mr 11,%1\n\t"                                           \
3722          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3723          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3724          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3725          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3726          /* arg12 */                                              \
3727          "lwz 3,48(11)\n\t"                                       \
3728          "stw 3,68(1)\n\t"                                        \
3729          /* arg11 */                                              \
3730          "lwz 3,44(11)\n\t"                                       \
3731          "stw 3,64(1)\n\t"                                        \
3732          /* arg10 */                                              \
3733          "lwz 3,40(11)\n\t"                                       \
3734          "stw 3,60(1)\n\t"                                        \
3735          /* arg9 */                                               \
3736          "lwz 3,36(11)\n\t"                                       \
3737          "stw 3,56(1)\n\t"                                        \
3738          /* args1-8 */                                            \
3739          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3740          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3741          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3742          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3743          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3744          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3745          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3746          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3747          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3748          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3749          "mr 11,%1\n\t"                                           \
3750          "mr %0,3\n\t"                                            \
3751          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3752          VG_CONTRACT_FRAME_BY(72)                                 \
3753          VG_CONTRACT_FRAME_BY(512)                                \
3754          : /*out*/   "=r" (_res)                                  \
3755          : /*in*/    "r" (&_argvec[2])                            \
3756          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3757       );                                                          \
3758       lval = (__typeof__(lval)) _res;                             \
3759    } while (0)
3760
3761 #endif /* PLAT_ppc32_aix5 */
3762
3763 /* ------------------------ ppc64-aix5 ------------------------- */
3764
3765 #if defined(PLAT_ppc64_aix5)
3766
3767 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3768
3769 /* These regs are trashed by the hidden call. */
3770 #define __CALLER_SAVED_REGS                                       \
3771    "lr", "ctr", "xer",                                            \
3772    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3773    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3774    "r11", "r12", "r13"
3775
3776 /* Expand the stack frame, copying enough info that unwinding
3777    still works.  Trashes r3. */
3778
3779 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3780          "addi 1,1,-" #_n_fr "\n\t"                               \
3781          "ld   3," #_n_fr "(1)\n\t"                               \
3782          "std  3,0(1)\n\t"
3783
3784 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3785          "addi 1,1," #_n_fr "\n\t"
3786
3787 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3788    long) == 8. */
3789
3790 #define CALL_FN_W_v(lval, orig)                                   \
3791    do {                                                           \
3792       volatile OrigFn        _orig = (orig);                      \
3793       volatile unsigned long _argvec[3+0];                        \
3794       volatile unsigned long _res;                                \
3795       /* _argvec[0] holds current r2 across the call */           \
3796       _argvec[1] = (unsigned long)_orig.r2;                       \
3797       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3798       __asm__ volatile(                                           \
3799          "mr 11,%1\n\t"                                           \
3800          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3801          "std  2,-16(11)\n\t" /* save tocptr */                   \
3802          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3803          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3804          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3805          "mr 11,%1\n\t"                                           \
3806          "mr %0,3\n\t"                                            \
3807          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3808          VG_CONTRACT_FRAME_BY(512)                                \
3809          : /*out*/   "=r" (_res)                                  \
3810          : /*in*/    "r" (&_argvec[2])                            \
3811          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3812       );                                                          \
3813       lval = (__typeof__(lval)) _res;                             \
3814    } while (0)
3815
3816 #define CALL_FN_W_W(lval, orig, arg1)                             \
3817    do {                                                           \
3818       volatile OrigFn        _orig = (orig);                      \
3819       volatile unsigned long _argvec[3+1];                        \
3820       volatile unsigned long _res;                                \
3821       /* _argvec[0] holds current r2 across the call */           \
3822       _argvec[1]   = (unsigned long)_orig.r2;                     \
3823       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3824       _argvec[2+1] = (unsigned long)arg1;                         \
3825       __asm__ volatile(                                           \
3826          "mr 11,%1\n\t"                                           \
3827          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3828          "std  2,-16(11)\n\t" /* save tocptr */                   \
3829          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3830          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3831          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3832          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3833          "mr 11,%1\n\t"                                           \
3834          "mr %0,3\n\t"                                            \
3835          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3836          VG_CONTRACT_FRAME_BY(512)                                \
3837          : /*out*/   "=r" (_res)                                  \
3838          : /*in*/    "r" (&_argvec[2])                            \
3839          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3840       );                                                          \
3841       lval = (__typeof__(lval)) _res;                             \
3842    } while (0)
3843
3844 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3845    do {                                                           \
3846       volatile OrigFn        _orig = (orig);                      \
3847       volatile unsigned long _argvec[3+2];                        \
3848       volatile unsigned long _res;                                \
3849       /* _argvec[0] holds current r2 across the call */           \
3850       _argvec[1]   = (unsigned long)_orig.r2;                     \
3851       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3852       _argvec[2+1] = (unsigned long)arg1;                         \
3853       _argvec[2+2] = (unsigned long)arg2;                         \
3854       __asm__ volatile(                                           \
3855          "mr 11,%1\n\t"                                           \
3856          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3857          "std  2,-16(11)\n\t" /* save tocptr */                   \
3858          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3859          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3860          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3861          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3862          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3863          "mr 11,%1\n\t"                                           \
3864          "mr %0,3\n\t"                                            \
3865          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3866          VG_CONTRACT_FRAME_BY(512)                                \
3867          : /*out*/   "=r" (_res)                                  \
3868          : /*in*/    "r" (&_argvec[2])                            \
3869          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3870       );                                                          \
3871       lval = (__typeof__(lval)) _res;                             \
3872    } while (0)
3873
3874 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3875    do {                                                           \
3876       volatile OrigFn        _orig = (orig);                      \
3877       volatile unsigned long _argvec[3+3];                        \
3878       volatile unsigned long _res;                                \
3879       /* _argvec[0] holds current r2 across the call */           \
3880       _argvec[1]   = (unsigned long)_orig.r2;                     \
3881       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3882       _argvec[2+1] = (unsigned long)arg1;                         \
3883       _argvec[2+2] = (unsigned long)arg2;                         \
3884       _argvec[2+3] = (unsigned long)arg3;                         \
3885       __asm__ volatile(                                           \
3886          "mr 11,%1\n\t"                                           \
3887          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3888          "std  2,-16(11)\n\t" /* save tocptr */                   \
3889          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3890          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3891          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3892          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3893          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3894          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3895          "mr 11,%1\n\t"                                           \
3896          "mr %0,3\n\t"                                            \
3897          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3898          VG_CONTRACT_FRAME_BY(512)                                \
3899          : /*out*/   "=r" (_res)                                  \
3900          : /*in*/    "r" (&_argvec[2])                            \
3901          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3902       );                                                          \
3903       lval = (__typeof__(lval)) _res;                             \
3904    } while (0)
3905
3906 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3907    do {                                                           \
3908       volatile OrigFn        _orig = (orig);                      \
3909       volatile unsigned long _argvec[3+4];                        \
3910       volatile unsigned long _res;                                \
3911       /* _argvec[0] holds current r2 across the call */           \
3912       _argvec[1]   = (unsigned long)_orig.r2;                     \
3913       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3914       _argvec[2+1] = (unsigned long)arg1;                         \
3915       _argvec[2+2] = (unsigned long)arg2;                         \
3916       _argvec[2+3] = (unsigned long)arg3;                         \
3917       _argvec[2+4] = (unsigned long)arg4;                         \
3918       __asm__ volatile(                                           \
3919          "mr 11,%1\n\t"                                           \
3920          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3921          "std  2,-16(11)\n\t" /* save tocptr */                   \
3922          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3923          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3924          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3925          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3926          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3927          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3928          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3929          "mr 11,%1\n\t"                                           \
3930          "mr %0,3\n\t"                                            \
3931          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3932          VG_CONTRACT_FRAME_BY(512)                                \
3933          : /*out*/   "=r" (_res)                                  \
3934          : /*in*/    "r" (&_argvec[2])                            \
3935          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3936       );                                                          \
3937       lval = (__typeof__(lval)) _res;                             \
3938    } while (0)
3939
3940 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3941    do {                                                           \
3942       volatile OrigFn        _orig = (orig);                      \
3943       volatile unsigned long _argvec[3+5];                        \
3944       volatile unsigned long _res;                                \
3945       /* _argvec[0] holds current r2 across the call */           \
3946       _argvec[1]   = (unsigned long)_orig.r2;                     \
3947       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3948       _argvec[2+1] = (unsigned long)arg1;                         \
3949       _argvec[2+2] = (unsigned long)arg2;                         \
3950       _argvec[2+3] = (unsigned long)arg3;                         \
3951       _argvec[2+4] = (unsigned long)arg4;                         \
3952       _argvec[2+5] = (unsigned long)arg5;                         \
3953       __asm__ volatile(                                           \
3954          "mr 11,%1\n\t"                                           \
3955          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3956          "std  2,-16(11)\n\t" /* save tocptr */                   \
3957          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3958          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3959          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3960          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3961          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3962          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3963          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3964          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3965          "mr 11,%1\n\t"                                           \
3966          "mr %0,3\n\t"                                            \
3967          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3968          VG_CONTRACT_FRAME_BY(512)                                \
3969          : /*out*/   "=r" (_res)                                  \
3970          : /*in*/    "r" (&_argvec[2])                            \
3971          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3972       );                                                          \
3973       lval = (__typeof__(lval)) _res;                             \
3974    } while (0)
3975
3976 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3977    do {                                                           \
3978       volatile OrigFn        _orig = (orig);                      \
3979       volatile unsigned long _argvec[3+6];                        \
3980       volatile unsigned long _res;                                \
3981       /* _argvec[0] holds current r2 across the call */           \
3982       _argvec[1]   = (unsigned long)_orig.r2;                     \
3983       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3984       _argvec[2+1] = (unsigned long)arg1;                         \
3985       _argvec[2+2] = (unsigned long)arg2;                         \
3986       _argvec[2+3] = (unsigned long)arg3;                         \
3987       _argvec[2+4] = (unsigned long)arg4;                         \
3988       _argvec[2+5] = (unsigned long)arg5;                         \
3989       _argvec[2+6] = (unsigned long)arg6;                         \
3990       __asm__ volatile(                                           \
3991          "mr 11,%1\n\t"                                           \
3992          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3993          "std  2,-16(11)\n\t" /* save tocptr */                   \
3994          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3995          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3996          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3997          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3998          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3999          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4000          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4001          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4002          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4003          "mr 11,%1\n\t"                                           \
4004          "mr %0,3\n\t"                                            \
4005          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4006          VG_CONTRACT_FRAME_BY(512)                                \
4007          : /*out*/   "=r" (_res)                                  \
4008          : /*in*/    "r" (&_argvec[2])                            \
4009          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4010       );                                                          \
4011       lval = (__typeof__(lval)) _res;                             \
4012    } while (0)
4013
4014 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4015                                  arg7)                            \
4016    do {                                                           \
4017       volatile OrigFn        _orig = (orig);                      \
4018       volatile unsigned long _argvec[3+7];                        \
4019       volatile unsigned long _res;                                \
4020       /* _argvec[0] holds current r2 across the call */           \
4021       _argvec[1]   = (unsigned long)_orig.r2;                     \
4022       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4023       _argvec[2+1] = (unsigned long)arg1;                         \
4024       _argvec[2+2] = (unsigned long)arg2;                         \
4025       _argvec[2+3] = (unsigned long)arg3;                         \
4026       _argvec[2+4] = (unsigned long)arg4;                         \
4027       _argvec[2+5] = (unsigned long)arg5;                         \
4028       _argvec[2+6] = (unsigned long)arg6;                         \
4029       _argvec[2+7] = (unsigned long)arg7;                         \
4030       __asm__ volatile(                                           \
4031          "mr 11,%1\n\t"                                           \
4032          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4033          "std  2,-16(11)\n\t" /* save tocptr */                   \
4034          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4035          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4036          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4037          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4038          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4039          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4040          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4041          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4042          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4043          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4044          "mr 11,%1\n\t"                                           \
4045          "mr %0,3\n\t"                                            \
4046          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4047          VG_CONTRACT_FRAME_BY(512)                                \
4048          : /*out*/   "=r" (_res)                                  \
4049          : /*in*/    "r" (&_argvec[2])                            \
4050          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4051       );                                                          \
4052       lval = (__typeof__(lval)) _res;                             \
4053    } while (0)
4054
4055 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4056                                  arg7,arg8)                       \
4057    do {                                                           \
4058       volatile OrigFn        _orig = (orig);                      \
4059       volatile unsigned long _argvec[3+8];                        \
4060       volatile unsigned long _res;                                \
4061       /* _argvec[0] holds current r2 across the call */           \
4062       _argvec[1]   = (unsigned long)_orig.r2;                     \
4063       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4064       _argvec[2+1] = (unsigned long)arg1;                         \
4065       _argvec[2+2] = (unsigned long)arg2;                         \
4066       _argvec[2+3] = (unsigned long)arg3;                         \
4067       _argvec[2+4] = (unsigned long)arg4;                         \
4068       _argvec[2+5] = (unsigned long)arg5;                         \
4069       _argvec[2+6] = (unsigned long)arg6;                         \
4070       _argvec[2+7] = (unsigned long)arg7;                         \
4071       _argvec[2+8] = (unsigned long)arg8;                         \
4072       __asm__ volatile(                                           \
4073          "mr 11,%1\n\t"                                           \
4074          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4075          "std  2,-16(11)\n\t" /* save tocptr */                   \
4076          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4077          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4078          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4079          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4080          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4081          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4082          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4083          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4084          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4085          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4086          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4087          "mr 11,%1\n\t"                                           \
4088          "mr %0,3\n\t"                                            \
4089          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4090          VG_CONTRACT_FRAME_BY(512)                                \
4091          : /*out*/   "=r" (_res)                                  \
4092          : /*in*/    "r" (&_argvec[2])                            \
4093          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4094       );                                                          \
4095       lval = (__typeof__(lval)) _res;                             \
4096    } while (0)
4097
4098 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4099                                  arg7,arg8,arg9)                  \
4100    do {                                                           \
4101       volatile OrigFn        _orig = (orig);                      \
4102       volatile unsigned long _argvec[3+9];                        \
4103       volatile unsigned long _res;                                \
4104       /* _argvec[0] holds current r2 across the call */           \
4105       _argvec[1]   = (unsigned long)_orig.r2;                     \
4106       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4107       _argvec[2+1] = (unsigned long)arg1;                         \
4108       _argvec[2+2] = (unsigned long)arg2;                         \
4109       _argvec[2+3] = (unsigned long)arg3;                         \
4110       _argvec[2+4] = (unsigned long)arg4;                         \
4111       _argvec[2+5] = (unsigned long)arg5;                         \
4112       _argvec[2+6] = (unsigned long)arg6;                         \
4113       _argvec[2+7] = (unsigned long)arg7;                         \
4114       _argvec[2+8] = (unsigned long)arg8;                         \
4115       _argvec[2+9] = (unsigned long)arg9;                         \
4116       __asm__ volatile(                                           \
4117          "mr 11,%1\n\t"                                           \
4118          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4119          "std  2,-16(11)\n\t" /* save tocptr */                   \
4120          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4121          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4122          /* arg9 */                                               \
4123          "ld  3,72(11)\n\t"                                       \
4124          "std 3,112(1)\n\t"                                       \
4125          /* args1-8 */                                            \
4126          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4127          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4128          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4129          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4130          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4131          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4132          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4133          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4134          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4135          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4136          "mr 11,%1\n\t"                                           \
4137          "mr %0,3\n\t"                                            \
4138          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4139          VG_CONTRACT_FRAME_BY(128)                                \
4140          VG_CONTRACT_FRAME_BY(512)                                \
4141          : /*out*/   "=r" (_res)                                  \
4142          : /*in*/    "r" (&_argvec[2])                            \
4143          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4144       );                                                          \
4145       lval = (__typeof__(lval)) _res;                             \
4146    } while (0)
4147
4148 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4149                                   arg7,arg8,arg9,arg10)           \
4150    do {                                                           \
4151       volatile OrigFn        _orig = (orig);                      \
4152       volatile unsigned long _argvec[3+10];                       \
4153       volatile unsigned long _res;                                \
4154       /* _argvec[0] holds current r2 across the call */           \
4155       _argvec[1]   = (unsigned long)_orig.r2;                     \
4156       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4157       _argvec[2+1] = (unsigned long)arg1;                         \
4158       _argvec[2+2] = (unsigned long)arg2;                         \
4159       _argvec[2+3] = (unsigned long)arg3;                         \
4160       _argvec[2+4] = (unsigned long)arg4;                         \
4161       _argvec[2+5] = (unsigned long)arg5;                         \
4162       _argvec[2+6] = (unsigned long)arg6;                         \
4163       _argvec[2+7] = (unsigned long)arg7;                         \
4164       _argvec[2+8] = (unsigned long)arg8;                         \
4165       _argvec[2+9] = (unsigned long)arg9;                         \
4166       _argvec[2+10] = (unsigned long)arg10;                       \
4167       __asm__ volatile(                                           \
4168          "mr 11,%1\n\t"                                           \
4169          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4170          "std  2,-16(11)\n\t" /* save tocptr */                   \
4171          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4172          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4173          /* arg10 */                                              \
4174          "ld  3,80(11)\n\t"                                       \
4175          "std 3,120(1)\n\t"                                       \
4176          /* arg9 */                                               \
4177          "ld  3,72(11)\n\t"                                       \
4178          "std 3,112(1)\n\t"                                       \
4179          /* args1-8 */                                            \
4180          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4181          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4182          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4183          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4184          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4185          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4186          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4187          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4188          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4189          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4190          "mr 11,%1\n\t"                                           \
4191          "mr %0,3\n\t"                                            \
4192          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4193          VG_CONTRACT_FRAME_BY(128)                                \
4194          VG_CONTRACT_FRAME_BY(512)                                \
4195          : /*out*/   "=r" (_res)                                  \
4196          : /*in*/    "r" (&_argvec[2])                            \
4197          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4198       );                                                          \
4199       lval = (__typeof__(lval)) _res;                             \
4200    } while (0)
4201
4202 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4203                                   arg7,arg8,arg9,arg10,arg11)     \
4204    do {                                                           \
4205       volatile OrigFn        _orig = (orig);                      \
4206       volatile unsigned long _argvec[3+11];                       \
4207       volatile unsigned long _res;                                \
4208       /* _argvec[0] holds current r2 across the call */           \
4209       _argvec[1]   = (unsigned long)_orig.r2;                     \
4210       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4211       _argvec[2+1] = (unsigned long)arg1;                         \
4212       _argvec[2+2] = (unsigned long)arg2;                         \
4213       _argvec[2+3] = (unsigned long)arg3;                         \
4214       _argvec[2+4] = (unsigned long)arg4;                         \
4215       _argvec[2+5] = (unsigned long)arg5;                         \
4216       _argvec[2+6] = (unsigned long)arg6;                         \
4217       _argvec[2+7] = (unsigned long)arg7;                         \
4218       _argvec[2+8] = (unsigned long)arg8;                         \
4219       _argvec[2+9] = (unsigned long)arg9;                         \
4220       _argvec[2+10] = (unsigned long)arg10;                       \
4221       _argvec[2+11] = (unsigned long)arg11;                       \
4222       __asm__ volatile(                                           \
4223          "mr 11,%1\n\t"                                           \
4224          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4225          "std  2,-16(11)\n\t" /* save tocptr */                   \
4226          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4227          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4228          /* arg11 */                                              \
4229          "ld  3,88(11)\n\t"                                       \
4230          "std 3,128(1)\n\t"                                       \
4231          /* arg10 */                                              \
4232          "ld  3,80(11)\n\t"                                       \
4233          "std 3,120(1)\n\t"                                       \
4234          /* arg9 */                                               \
4235          "ld  3,72(11)\n\t"                                       \
4236          "std 3,112(1)\n\t"                                       \
4237          /* args1-8 */                                            \
4238          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4239          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4240          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4241          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4242          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4243          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4244          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4245          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4246          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4247          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4248          "mr 11,%1\n\t"                                           \
4249          "mr %0,3\n\t"                                            \
4250          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4251          VG_CONTRACT_FRAME_BY(144)                                \
4252          VG_CONTRACT_FRAME_BY(512)                                \
4253          : /*out*/   "=r" (_res)                                  \
4254          : /*in*/    "r" (&_argvec[2])                            \
4255          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4256       );                                                          \
4257       lval = (__typeof__(lval)) _res;                             \
4258    } while (0)
4259
4260 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4261                                 arg7,arg8,arg9,arg10,arg11,arg12) \
4262    do {                                                           \
4263       volatile OrigFn        _orig = (orig);                      \
4264       volatile unsigned long _argvec[3+12];                       \
4265       volatile unsigned long _res;                                \
4266       /* _argvec[0] holds current r2 across the call */           \
4267       _argvec[1]   = (unsigned long)_orig.r2;                     \
4268       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4269       _argvec[2+1] = (unsigned long)arg1;                         \
4270       _argvec[2+2] = (unsigned long)arg2;                         \
4271       _argvec[2+3] = (unsigned long)arg3;                         \
4272       _argvec[2+4] = (unsigned long)arg4;                         \
4273       _argvec[2+5] = (unsigned long)arg5;                         \
4274       _argvec[2+6] = (unsigned long)arg6;                         \
4275       _argvec[2+7] = (unsigned long)arg7;                         \
4276       _argvec[2+8] = (unsigned long)arg8;                         \
4277       _argvec[2+9] = (unsigned long)arg9;                         \
4278       _argvec[2+10] = (unsigned long)arg10;                       \
4279       _argvec[2+11] = (unsigned long)arg11;                       \
4280       _argvec[2+12] = (unsigned long)arg12;                       \
4281       __asm__ volatile(                                           \
4282          "mr 11,%1\n\t"                                           \
4283          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4284          "std  2,-16(11)\n\t" /* save tocptr */                   \
4285          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4286          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4287          /* arg12 */                                              \
4288          "ld  3,96(11)\n\t"                                       \
4289          "std 3,136(1)\n\t"                                       \
4290          /* arg11 */                                              \
4291          "ld  3,88(11)\n\t"                                       \
4292          "std 3,128(1)\n\t"                                       \
4293          /* arg10 */                                              \
4294          "ld  3,80(11)\n\t"                                       \
4295          "std 3,120(1)\n\t"                                       \
4296          /* arg9 */                                               \
4297          "ld  3,72(11)\n\t"                                       \
4298          "std 3,112(1)\n\t"                                       \
4299          /* args1-8 */                                            \
4300          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4301          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4302          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4303          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4304          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4305          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4306          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4307          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4308          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4309          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4310          "mr 11,%1\n\t"                                           \
4311          "mr %0,3\n\t"                                            \
4312          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4313          VG_CONTRACT_FRAME_BY(144)                                \
4314          VG_CONTRACT_FRAME_BY(512)                                \
4315          : /*out*/   "=r" (_res)                                  \
4316          : /*in*/    "r" (&_argvec[2])                            \
4317          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4318       );                                                          \
4319       lval = (__typeof__(lval)) _res;                             \
4320    } while (0)
4321
4322 #endif /* PLAT_ppc64_aix5 */
4323
4324 /* ------------------------- s390x-linux ------------------------- */
4325
4326 #if defined(PLAT_s390x_linux)
4327
4328 /* Similar craziness as x86 (see above), but we use r11 as frame
4329    pointer and save the old r11 in r7. r11 might be used for
4330    argvec, therefore we copy argvec in r1 since r1 is clobbered
4331    after the call anyway.  */
4332 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4333 #  define __FRAME_POINTER                                         \
4334       ,"d"(__builtin_dwarf_cfa())
4335 #  define VALGRIND_CFI_PROLOGUE                                   \
4336       ".cfi_remember_state\n\t"                                   \
4337       "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
4338       "lgr 7,11\n\t"                                              \
4339       "lgr 11,%2\n\t"                                             \
4340       ".cfi_def_cfa r11, 0\n\t"
4341 #  define VALGRIND_CFI_EPILOGUE                                   \
4342       "lgr 11, 7\n\t"                                             \
4343       ".cfi_restore_state\n\t"
4344 #else
4345 #  define __FRAME_POINTER
4346 #  define VALGRIND_CFI_PROLOGUE                                   \
4347       "lgr 1,%1\n\t"
4348 #  define VALGRIND_CFI_EPILOGUE
4349 #endif
4350
4351
4352
4353
4354 /* These regs are trashed by the hidden call. Note that we overwrite
4355    r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4356    function a proper return address. All others are ABI defined call
4357    clobbers. */
4358 #define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
4359                            "f0","f1","f2","f3","f4","f5","f6","f7"
4360
4361
4362 #define CALL_FN_W_v(lval, orig)                                  \
4363    do {                                                          \
4364       volatile OrigFn        _orig = (orig);                     \
4365       volatile unsigned long  _argvec[1];                        \
4366       volatile unsigned long _res;                               \
4367       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4368       __asm__ volatile(                                          \
4369          VALGRIND_CFI_PROLOGUE                                   \
4370          "aghi 15,-160\n\t"                                      \
4371          "lg 1, 0(1)\n\t"  /* target->r1 */                      \
4372          VALGRIND_CALL_NOREDIR_R1                                \
4373          "lgr %0, 2\n\t"                                         \
4374          "aghi 15,160\n\t"                                       \
4375          VALGRIND_CFI_EPILOGUE                                   \
4376          : /*out*/   "=d" (_res)                                 \
4377          : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
4378          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4379       );                                                         \
4380       lval = (__typeof__(lval)) _res;                            \
4381    } while (0)
4382
4383 /* The call abi has the arguments in r2-r6 and stack */
4384 #define CALL_FN_W_W(lval, orig, arg1)                            \
4385    do {                                                          \
4386       volatile OrigFn        _orig = (orig);                     \
4387       volatile unsigned long _argvec[2];                         \
4388       volatile unsigned long _res;                               \
4389       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4390       _argvec[1] = (unsigned long)arg1;                          \
4391       __asm__ volatile(                                          \
4392          VALGRIND_CFI_PROLOGUE                                   \
4393          "aghi 15,-160\n\t"                                      \
4394          "lg 2, 8(1)\n\t"                                        \
4395          "lg 1, 0(1)\n\t"                                        \
4396          VALGRIND_CALL_NOREDIR_R1                                \
4397          "lgr %0, 2\n\t"                                         \
4398          "aghi 15,160\n\t"                                       \
4399          VALGRIND_CFI_EPILOGUE                                   \
4400          : /*out*/   "=d" (_res)                                 \
4401          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4402          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4403       );                                                         \
4404       lval = (__typeof__(lval)) _res;                            \
4405    } while (0)
4406
4407 #define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
4408    do {                                                          \
4409       volatile OrigFn        _orig = (orig);                     \
4410       volatile unsigned long _argvec[3];                         \
4411       volatile unsigned long _res;                               \
4412       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4413       _argvec[1] = (unsigned long)arg1;                          \
4414       _argvec[2] = (unsigned long)arg2;                          \
4415       __asm__ volatile(                                          \
4416          VALGRIND_CFI_PROLOGUE                                   \
4417          "aghi 15,-160\n\t"                                      \
4418          "lg 2, 8(1)\n\t"                                        \
4419          "lg 3,16(1)\n\t"                                        \
4420          "lg 1, 0(1)\n\t"                                        \
4421          VALGRIND_CALL_NOREDIR_R1                                \
4422          "lgr %0, 2\n\t"                                         \
4423          "aghi 15,160\n\t"                                       \
4424          VALGRIND_CFI_EPILOGUE                                   \
4425          : /*out*/   "=d" (_res)                                 \
4426          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4427          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4428       );                                                         \
4429       lval = (__typeof__(lval)) _res;                            \
4430    } while (0)
4431
4432 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
4433    do {                                                          \
4434       volatile OrigFn        _orig = (orig);                     \
4435       volatile unsigned long _argvec[4];                         \
4436       volatile unsigned long _res;                               \
4437       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4438       _argvec[1] = (unsigned long)arg1;                          \
4439       _argvec[2] = (unsigned long)arg2;                          \
4440       _argvec[3] = (unsigned long)arg3;                          \
4441       __asm__ volatile(                                          \
4442          VALGRIND_CFI_PROLOGUE                                   \
4443          "aghi 15,-160\n\t"                                      \
4444          "lg 2, 8(1)\n\t"                                        \
4445          "lg 3,16(1)\n\t"                                        \
4446          "lg 4,24(1)\n\t"                                        \
4447          "lg 1, 0(1)\n\t"                                        \
4448          VALGRIND_CALL_NOREDIR_R1                                \
4449          "lgr %0, 2\n\t"                                         \
4450          "aghi 15,160\n\t"                                       \
4451          VALGRIND_CFI_EPILOGUE                                   \
4452          : /*out*/   "=d" (_res)                                 \
4453          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4454          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4455       );                                                         \
4456       lval = (__typeof__(lval)) _res;                            \
4457    } while (0)
4458
4459 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
4460    do {                                                          \
4461       volatile OrigFn        _orig = (orig);                     \
4462       volatile unsigned long _argvec[5];                         \
4463       volatile unsigned long _res;                               \
4464       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4465       _argvec[1] = (unsigned long)arg1;                          \
4466       _argvec[2] = (unsigned long)arg2;                          \
4467       _argvec[3] = (unsigned long)arg3;                          \
4468       _argvec[4] = (unsigned long)arg4;                          \
4469       __asm__ volatile(                                          \
4470          VALGRIND_CFI_PROLOGUE                                   \
4471          "aghi 15,-160\n\t"                                      \
4472          "lg 2, 8(1)\n\t"                                        \
4473          "lg 3,16(1)\n\t"                                        \
4474          "lg 4,24(1)\n\t"                                        \
4475          "lg 5,32(1)\n\t"                                        \
4476          "lg 1, 0(1)\n\t"                                        \
4477          VALGRIND_CALL_NOREDIR_R1                                \
4478          "lgr %0, 2\n\t"                                         \
4479          "aghi 15,160\n\t"                                       \
4480          VALGRIND_CFI_EPILOGUE                                   \
4481          : /*out*/   "=d" (_res)                                 \
4482          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4483          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4484       );                                                         \
4485       lval = (__typeof__(lval)) _res;                            \
4486    } while (0)
4487
4488 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
4489    do {                                                          \
4490       volatile OrigFn        _orig = (orig);                     \
4491       volatile unsigned long _argvec[6];                         \
4492       volatile unsigned long _res;                               \
4493       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4494       _argvec[1] = (unsigned long)arg1;                          \
4495       _argvec[2] = (unsigned long)arg2;                          \
4496       _argvec[3] = (unsigned long)arg3;                          \
4497       _argvec[4] = (unsigned long)arg4;                          \
4498       _argvec[5] = (unsigned long)arg5;                          \
4499       __asm__ volatile(                                          \
4500          VALGRIND_CFI_PROLOGUE                                   \
4501          "aghi 15,-160\n\t"                                      \
4502          "lg 2, 8(1)\n\t"                                        \
4503          "lg 3,16(1)\n\t"                                        \
4504          "lg 4,24(1)\n\t"                                        \
4505          "lg 5,32(1)\n\t"                                        \
4506          "lg 6,40(1)\n\t"                                        \
4507          "lg 1, 0(1)\n\t"                                        \
4508          VALGRIND_CALL_NOREDIR_R1                                \
4509          "lgr %0, 2\n\t"                                         \
4510          "aghi 15,160\n\t"                                       \
4511          VALGRIND_CFI_EPILOGUE                                   \
4512          : /*out*/   "=d" (_res)                                 \
4513          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4514          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4515       );                                                         \
4516       lval = (__typeof__(lval)) _res;                            \
4517    } while (0)
4518
4519 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4520                      arg6)                                       \
4521    do {                                                          \
4522       volatile OrigFn        _orig = (orig);                     \
4523       volatile unsigned long _argvec[7];                         \
4524       volatile unsigned long _res;                               \
4525       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4526       _argvec[1] = (unsigned long)arg1;                          \
4527       _argvec[2] = (unsigned long)arg2;                          \
4528       _argvec[3] = (unsigned long)arg3;                          \
4529       _argvec[4] = (unsigned long)arg4;                          \
4530       _argvec[5] = (unsigned long)arg5;                          \
4531       _argvec[6] = (unsigned long)arg6;                          \
4532       __asm__ volatile(                                          \
4533          VALGRIND_CFI_PROLOGUE                                   \
4534          "aghi 15,-168\n\t"                                      \
4535          "lg 2, 8(1)\n\t"                                        \
4536          "lg 3,16(1)\n\t"                                        \
4537          "lg 4,24(1)\n\t"                                        \
4538          "lg 5,32(1)\n\t"                                        \
4539          "lg 6,40(1)\n\t"                                        \
4540          "mvc 160(8,15), 48(1)\n\t"                              \
4541          "lg 1, 0(1)\n\t"                                        \
4542          VALGRIND_CALL_NOREDIR_R1                                \
4543          "lgr %0, 2\n\t"                                         \
4544          "aghi 15,168\n\t"                                       \
4545          VALGRIND_CFI_EPILOGUE                                   \
4546          : /*out*/   "=d" (_res)                                 \
4547          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4548          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4549       );                                                         \
4550       lval = (__typeof__(lval)) _res;                            \
4551    } while (0)
4552
4553 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4554                      arg6, arg7)                                 \
4555    do {                                                          \
4556       volatile OrigFn        _orig = (orig);                     \
4557       volatile unsigned long _argvec[8];                         \
4558       volatile unsigned long _res;                               \
4559       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4560       _argvec[1] = (unsigned long)arg1;                          \
4561       _argvec[2] = (unsigned long)arg2;                          \
4562       _argvec[3] = (unsigned long)arg3;                          \
4563       _argvec[4] = (unsigned long)arg4;                          \
4564       _argvec[5] = (unsigned long)arg5;                          \
4565       _argvec[6] = (unsigned long)arg6;                          \
4566       _argvec[7] = (unsigned long)arg7;                          \
4567       __asm__ volatile(                                          \
4568          VALGRIND_CFI_PROLOGUE                                   \
4569          "aghi 15,-176\n\t"                                      \
4570          "lg 2, 8(1)\n\t"                                        \
4571          "lg 3,16(1)\n\t"                                        \
4572          "lg 4,24(1)\n\t"                                        \
4573          "lg 5,32(1)\n\t"                                        \
4574          "lg 6,40(1)\n\t"                                        \
4575          "mvc 160(8,15), 48(1)\n\t"                              \
4576          "mvc 168(8,15), 56(1)\n\t"                              \
4577          "lg 1, 0(1)\n\t"                                        \
4578          VALGRIND_CALL_NOREDIR_R1                                \
4579          "lgr %0, 2\n\t"                                         \
4580          "aghi 15,176\n\t"                                       \
4581          VALGRIND_CFI_EPILOGUE                                   \
4582          : /*out*/   "=d" (_res)                                 \
4583          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4584          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4585       );                                                         \
4586       lval = (__typeof__(lval)) _res;                            \
4587    } while (0)
4588
4589 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4590                      arg6, arg7 ,arg8)                           \
4591    do {                                                          \
4592       volatile OrigFn        _orig = (orig);                     \
4593       volatile unsigned long _argvec[9];                         \
4594       volatile unsigned long _res;                               \
4595       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4596       _argvec[1] = (unsigned long)arg1;                          \
4597       _argvec[2] = (unsigned long)arg2;                          \
4598       _argvec[3] = (unsigned long)arg3;                          \
4599       _argvec[4] = (unsigned long)arg4;                          \
4600       _argvec[5] = (unsigned long)arg5;                          \
4601       _argvec[6] = (unsigned long)arg6;                          \
4602       _argvec[7] = (unsigned long)arg7;                          \
4603       _argvec[8] = (unsigned long)arg8;                          \
4604       __asm__ volatile(                                          \
4605          VALGRIND_CFI_PROLOGUE                                   \
4606          "aghi 15,-184\n\t"                                      \
4607          "lg 2, 8(1)\n\t"                                        \
4608          "lg 3,16(1)\n\t"                                        \
4609          "lg 4,24(1)\n\t"                                        \
4610          "lg 5,32(1)\n\t"                                        \
4611          "lg 6,40(1)\n\t"                                        \
4612          "mvc 160(8,15), 48(1)\n\t"                              \
4613          "mvc 168(8,15), 56(1)\n\t"                              \
4614          "mvc 176(8,15), 64(1)\n\t"                              \
4615          "lg 1, 0(1)\n\t"                                        \
4616          VALGRIND_CALL_NOREDIR_R1                                \
4617          "lgr %0, 2\n\t"                                         \
4618          "aghi 15,184\n\t"                                       \
4619          VALGRIND_CFI_EPILOGUE                                   \
4620          : /*out*/   "=d" (_res)                                 \
4621          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4622          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4623       );                                                         \
4624       lval = (__typeof__(lval)) _res;                            \
4625    } while (0)
4626
4627 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4628                      arg6, arg7 ,arg8, arg9)                     \
4629    do {                                                          \
4630       volatile OrigFn        _orig = (orig);                     \
4631       volatile unsigned long _argvec[10];                        \
4632       volatile unsigned long _res;                               \
4633       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4634       _argvec[1] = (unsigned long)arg1;                          \
4635       _argvec[2] = (unsigned long)arg2;                          \
4636       _argvec[3] = (unsigned long)arg3;                          \
4637       _argvec[4] = (unsigned long)arg4;                          \
4638       _argvec[5] = (unsigned long)arg5;                          \
4639       _argvec[6] = (unsigned long)arg6;                          \
4640       _argvec[7] = (unsigned long)arg7;                          \
4641       _argvec[8] = (unsigned long)arg8;                          \
4642       _argvec[9] = (unsigned long)arg9;                          \
4643       __asm__ volatile(                                          \
4644          VALGRIND_CFI_PROLOGUE                                   \
4645          "aghi 15,-192\n\t"                                      \
4646          "lg 2, 8(1)\n\t"                                        \
4647          "lg 3,16(1)\n\t"                                        \
4648          "lg 4,24(1)\n\t"                                        \
4649          "lg 5,32(1)\n\t"                                        \
4650          "lg 6,40(1)\n\t"                                        \
4651          "mvc 160(8,15), 48(1)\n\t"                              \
4652          "mvc 168(8,15), 56(1)\n\t"                              \
4653          "mvc 176(8,15), 64(1)\n\t"                              \
4654          "mvc 184(8,15), 72(1)\n\t"                              \
4655          "lg 1, 0(1)\n\t"                                        \
4656          VALGRIND_CALL_NOREDIR_R1                                \
4657          "lgr %0, 2\n\t"                                         \
4658          "aghi 15,192\n\t"                                       \
4659          VALGRIND_CFI_EPILOGUE                                   \
4660          : /*out*/   "=d" (_res)                                 \
4661          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4662          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4663       );                                                         \
4664       lval = (__typeof__(lval)) _res;                            \
4665    } while (0)
4666
4667 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4668                      arg6, arg7 ,arg8, arg9, arg10)              \
4669    do {                                                          \
4670       volatile OrigFn        _orig = (orig);                     \
4671       volatile unsigned long _argvec[11];                        \
4672       volatile unsigned long _res;                               \
4673       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4674       _argvec[1] = (unsigned long)arg1;                          \
4675       _argvec[2] = (unsigned long)arg2;                          \
4676       _argvec[3] = (unsigned long)arg3;                          \
4677       _argvec[4] = (unsigned long)arg4;                          \
4678       _argvec[5] = (unsigned long)arg5;                          \
4679       _argvec[6] = (unsigned long)arg6;                          \
4680       _argvec[7] = (unsigned long)arg7;                          \
4681       _argvec[8] = (unsigned long)arg8;                          \
4682       _argvec[9] = (unsigned long)arg9;                          \
4683       _argvec[10] = (unsigned long)arg10;                        \
4684       __asm__ volatile(                                          \
4685          VALGRIND_CFI_PROLOGUE                                   \
4686          "aghi 15,-200\n\t"                                      \
4687          "lg 2, 8(1)\n\t"                                        \
4688          "lg 3,16(1)\n\t"                                        \
4689          "lg 4,24(1)\n\t"                                        \
4690          "lg 5,32(1)\n\t"                                        \
4691          "lg 6,40(1)\n\t"                                        \
4692          "mvc 160(8,15), 48(1)\n\t"                              \
4693          "mvc 168(8,15), 56(1)\n\t"                              \
4694          "mvc 176(8,15), 64(1)\n\t"                              \
4695          "mvc 184(8,15), 72(1)\n\t"                              \
4696          "mvc 192(8,15), 80(1)\n\t"                              \
4697          "lg 1, 0(1)\n\t"                                        \
4698          VALGRIND_CALL_NOREDIR_R1                                \
4699          "lgr %0, 2\n\t"                                         \
4700          "aghi 15,200\n\t"                                       \
4701          VALGRIND_CFI_EPILOGUE                                   \
4702          : /*out*/   "=d" (_res)                                 \
4703          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4704          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4705       );                                                         \
4706       lval = (__typeof__(lval)) _res;                            \
4707    } while (0)
4708
4709 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4710                      arg6, arg7 ,arg8, arg9, arg10, arg11)       \
4711    do {                                                          \
4712       volatile OrigFn        _orig = (orig);                     \
4713       volatile unsigned long _argvec[12];                        \
4714       volatile unsigned long _res;                               \
4715       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4716       _argvec[1] = (unsigned long)arg1;                          \
4717       _argvec[2] = (unsigned long)arg2;                          \
4718       _argvec[3] = (unsigned long)arg3;                          \
4719       _argvec[4] = (unsigned long)arg4;                          \
4720       _argvec[5] = (unsigned long)arg5;                          \
4721       _argvec[6] = (unsigned long)arg6;                          \
4722       _argvec[7] = (unsigned long)arg7;                          \
4723       _argvec[8] = (unsigned long)arg8;                          \
4724       _argvec[9] = (unsigned long)arg9;                          \
4725       _argvec[10] = (unsigned long)arg10;                        \
4726       _argvec[11] = (unsigned long)arg11;                        \
4727       __asm__ volatile(                                          \
4728          VALGRIND_CFI_PROLOGUE                                   \
4729          "aghi 15,-208\n\t"                                      \
4730          "lg 2, 8(1)\n\t"                                        \
4731          "lg 3,16(1)\n\t"                                        \
4732          "lg 4,24(1)\n\t"                                        \
4733          "lg 5,32(1)\n\t"                                        \
4734          "lg 6,40(1)\n\t"                                        \
4735          "mvc 160(8,15), 48(1)\n\t"                              \
4736          "mvc 168(8,15), 56(1)\n\t"                              \
4737          "mvc 176(8,15), 64(1)\n\t"                              \
4738          "mvc 184(8,15), 72(1)\n\t"                              \
4739          "mvc 192(8,15), 80(1)\n\t"                              \
4740          "mvc 200(8,15), 88(1)\n\t"                              \
4741          "lg 1, 0(1)\n\t"                                        \
4742          VALGRIND_CALL_NOREDIR_R1                                \
4743          "lgr %0, 2\n\t"                                         \
4744          "aghi 15,208\n\t"                                       \
4745          VALGRIND_CFI_EPILOGUE                                   \
4746          : /*out*/   "=d" (_res)                                 \
4747          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4748          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4749       );                                                         \
4750       lval = (__typeof__(lval)) _res;                            \
4751    } while (0)
4752
4753 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4754                      arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
4755    do {                                                          \
4756       volatile OrigFn        _orig = (orig);                     \
4757       volatile unsigned long _argvec[13];                        \
4758       volatile unsigned long _res;                               \
4759       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4760       _argvec[1] = (unsigned long)arg1;                          \
4761       _argvec[2] = (unsigned long)arg2;                          \
4762       _argvec[3] = (unsigned long)arg3;                          \
4763       _argvec[4] = (unsigned long)arg4;                          \
4764       _argvec[5] = (unsigned long)arg5;                          \
4765       _argvec[6] = (unsigned long)arg6;                          \
4766       _argvec[7] = (unsigned long)arg7;                          \
4767       _argvec[8] = (unsigned long)arg8;                          \
4768       _argvec[9] = (unsigned long)arg9;                          \
4769       _argvec[10] = (unsigned long)arg10;                        \
4770       _argvec[11] = (unsigned long)arg11;                        \
4771       _argvec[12] = (unsigned long)arg12;                        \
4772       __asm__ volatile(                                          \
4773          VALGRIND_CFI_PROLOGUE                                   \
4774          "aghi 15,-216\n\t"                                      \
4775          "lg 2, 8(1)\n\t"                                        \
4776          "lg 3,16(1)\n\t"                                        \
4777          "lg 4,24(1)\n\t"                                        \
4778          "lg 5,32(1)\n\t"                                        \
4779          "lg 6,40(1)\n\t"                                        \
4780          "mvc 160(8,15), 48(1)\n\t"                              \
4781          "mvc 168(8,15), 56(1)\n\t"                              \
4782          "mvc 176(8,15), 64(1)\n\t"                              \
4783          "mvc 184(8,15), 72(1)\n\t"                              \
4784          "mvc 192(8,15), 80(1)\n\t"                              \
4785          "mvc 200(8,15), 88(1)\n\t"                              \
4786          "mvc 208(8,15), 96(1)\n\t"                              \
4787          "lg 1, 0(1)\n\t"                                        \
4788          VALGRIND_CALL_NOREDIR_R1                                \
4789          "lgr %0, 2\n\t"                                         \
4790          "aghi 15,216\n\t"                                       \
4791          VALGRIND_CFI_EPILOGUE                                   \
4792          : /*out*/   "=d" (_res)                                 \
4793          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4794          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4795       );                                                         \
4796       lval = (__typeof__(lval)) _res;                            \
4797    } while (0)
4798
4799
4800 #endif /* PLAT_s390x_linux */
4801
4802
4803 /* ------------------------------------------------------------------ */
4804 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
4805 /*                                                                    */
4806 /* ------------------------------------------------------------------ */
4807
4808 /* Some request codes.  There are many more of these, but most are not
4809    exposed to end-user view.  These are the public ones, all of the
4810    form 0x1000 + small_number.
4811
4812    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
4813    ones start at 0x2000.
4814 */
4815
4816 /* These macros are used by tools -- they must be public, but don't
4817    embed them into other programs. */
4818 #define VG_USERREQ_TOOL_BASE(a,b) \
4819    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
4820 #define VG_IS_TOOL_USERREQ(a, b, v) \
4821    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
4822
4823 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
4824    This enum comprises an ABI exported by Valgrind to programs
4825    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
4826    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
4827 typedef
4828    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
4829           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
4830
4831           /* These allow any function to be called from the simulated
4832              CPU but run on the real CPU.  Nb: the first arg passed to
4833              the function is always the ThreadId of the running
4834              thread!  So CLIENT_CALL0 actually requires a 1 arg
4835              function, etc. */
4836           VG_USERREQ__CLIENT_CALL0 = 0x1101,
4837           VG_USERREQ__CLIENT_CALL1 = 0x1102,
4838           VG_USERREQ__CLIENT_CALL2 = 0x1103,
4839           VG_USERREQ__CLIENT_CALL3 = 0x1104,
4840
4841           /* Can be useful in regression testing suites -- eg. can
4842              send Valgrind's output to /dev/null and still count
4843              errors. */
4844           VG_USERREQ__COUNT_ERRORS = 0x1201,
4845
4846           /* These are useful and can be interpreted by any tool that
4847              tracks malloc() et al, by using vg_replace_malloc.c. */
4848           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
4849           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
4850           /* Memory pool support. */
4851           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
4852           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
4853           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
4854           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
4855           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
4856           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
4857           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
4858           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
4859
4860           /* Allow printfs to valgrind log. */
4861           /* The first two pass the va_list argument by value, which
4862              assumes it is the same size as or smaller than a UWord,
4863              which generally isn't the case.  Hence are deprecated.
4864              The second two pass the vargs by reference and so are
4865              immune to this problem. */
4866           /* both :: char* fmt, va_list vargs (DEPRECATED) */
4867           VG_USERREQ__PRINTF           = 0x1401,
4868           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
4869           /* both :: char* fmt, va_list* vargs */
4870           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
4871           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
4872
4873           /* Stack support. */
4874           VG_USERREQ__STACK_REGISTER   = 0x1501,
4875           VG_USERREQ__STACK_DEREGISTER = 0x1502,
4876           VG_USERREQ__STACK_CHANGE     = 0x1503,
4877
4878           /* Wine support */
4879           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
4880
4881           /* Querying of debug info. */
4882           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701
4883    } Vg_ClientRequest;
4884
4885 #if !defined(__GNUC__)
4886 #  define __extension__ /* */
4887 #endif
4888
4889
4890 /*
4891  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
4892  * client request and whose value equals the client request result.
4893  */
4894
4895 #if defined(NVALGRIND)
4896
4897 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                               \
4898         _zzq_default, _zzq_request,                                    \
4899         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)         \
4900    (_zzq_default)
4901
4902 #else /*defined(NVALGRIND)*/
4903
4904 #if defined(_MSC_VER)
4905
4906 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                                \
4907         _zzq_default, _zzq_request,                                     \
4908         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)          \
4909    (vg_VALGRIND_DO_CLIENT_REQUEST_EXPR((uintptr_t)(_zzq_default),       \
4910         (_zzq_request), (uintptr_t)(_zzq_arg1), (uintptr_t)(_zzq_arg2), \
4911         (uintptr_t)(_zzq_arg3), (uintptr_t)(_zzq_arg4),                 \
4912         (uintptr_t)(_zzq_arg5)))
4913
4914 static __inline unsigned
4915 vg_VALGRIND_DO_CLIENT_REQUEST_EXPR(uintptr_t _zzq_default,
4916                                    unsigned _zzq_request, uintptr_t _zzq_arg1,
4917                                    uintptr_t _zzq_arg2, uintptr_t _zzq_arg3,
4918                                    uintptr_t _zzq_arg4, uintptr_t _zzq_arg5)
4919 {
4920     unsigned _zzq_rlval;
4921     VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, _zzq_request,
4922                       _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5);
4923     return _zzq_rlval;
4924 }
4925
4926 #else /*defined(_MSC_VER)*/
4927
4928 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                               \
4929         _zzq_default, _zzq_request,                                    \
4930         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)         \
4931    (__extension__({unsigned int _zzq_rlval;                            \
4932     VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, _zzq_request, \
4933                 _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
4934     _zzq_rlval;                                                        \
4935    }))
4936
4937 #endif /*defined(_MSC_VER)*/
4938
4939 #endif /*defined(NVALGRIND)*/
4940
4941
4942 /* Returns the number of Valgrinds this code is running under.  That
4943    is, 0 if running natively, 1 if running under Valgrind, 2 if
4944    running under Valgrind which is running under another Valgrind,
4945    etc. */
4946 #define RUNNING_ON_VALGRIND                                           \
4947     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,                   \
4948                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
4949                                     0, 0, 0, 0, 0)                    \
4950
4951
4952 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
4953    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
4954    since it provides a way to make sure valgrind will retranslate the
4955    invalidated area.  Returns no value. */
4956 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
4957    {unsigned int _qzz_res VG_UNUSED;                                        \
4958     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4959                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
4960                                _qzz_addr, _qzz_len, 0, 0, 0);     \
4961    }
4962
4963
4964 /* These requests are for getting Valgrind itself to print something.
4965    Possibly with a backtrace.  This is a really ugly hack.  The return value
4966    is the number of characters printed, excluding the "**<pid>** " part at the
4967    start and the backtrace (if present). */
4968
4969 #if defined(NVALGRIND)
4970
4971 #  define VALGRIND_PRINTF(...)
4972 #  define VALGRIND_PRINTF_BACKTRACE(...)
4973
4974 #else /* NVALGRIND */
4975
4976 #if !defined(_MSC_VER)
4977 /* Modern GCC will optimize the static routine out if unused,
4978    and unused attribute will shut down warnings about it.  */
4979 static int VALGRIND_PRINTF(const char *format, ...)
4980    __attribute__((format(__printf__, 1, 2), __unused__));
4981 #endif
4982 static int
4983 #if defined(_MSC_VER)
4984 __inline
4985 #endif
4986 VALGRIND_PRINTF(const char *format, ...)
4987 {
4988    unsigned long _qzz_res;
4989    va_list vargs;
4990    va_start(vargs, format);
4991 #if defined(_MSC_VER)
4992    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4993                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4994                               (uintptr_t)format,
4995                               (uintptr_t)&vargs,
4996                               0, 0, 0);
4997 #else
4998    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4999                               VG_USERREQ__PRINTF_VALIST_BY_REF,
5000                               (unsigned long)format,
5001                               (unsigned long)&vargs,
5002                               0, 0, 0);
5003 #endif
5004    va_end(vargs);
5005    return (int)_qzz_res;
5006 }
5007
5008 #if !defined(_MSC_VER)
5009 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
5010    __attribute__((format(__printf__, 1, 2), __unused__));
5011 #endif
5012 static int
5013 #if defined(_MSC_VER)
5014 __inline
5015 #endif
5016 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
5017 {
5018    unsigned long _qzz_res;
5019    va_list vargs;
5020    va_start(vargs, format);
5021 #if defined(_MSC_VER)
5022    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
5023                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5024                               (uintptr_t)format,
5025                               (uintptr_t)&vargs,
5026                               0, 0, 0);
5027 #else
5028    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
5029                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5030                               (unsigned long)format,
5031                               (unsigned long)&vargs,
5032                               0, 0, 0);
5033 #endif
5034    va_end(vargs);
5035    return (int)_qzz_res;
5036 }
5037
5038 #endif /* NVALGRIND */
5039
5040
5041 /* These requests allow control to move from the simulated CPU to the
5042    real CPU, calling an arbitary function.
5043
5044    Note that the current ThreadId is inserted as the first argument.
5045    So this call:
5046
5047      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
5048
5049    requires f to have this signature:
5050
5051      Word f(Word tid, Word arg1, Word arg2)
5052
5053    where "Word" is a word-sized type.
5054
5055    Note that these client requests are not entirely reliable.  For example,
5056    if you call a function with them that subsequently calls printf(),
5057    there's a high chance Valgrind will crash.  Generally, your prospects of
5058    these working are made higher if the called function does not refer to
5059    any global variables, and does not refer to any libc or other functions
5060    (printf et al).  Any kind of entanglement with libc or dynamic linking is
5061    likely to have a bad outcome, for tricky reasons which we've grappled
5062    with a lot in the past.
5063 */
5064 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
5065    __extension__                                                  \
5066    ({unsigned long _qyy_res;                                      \
5067     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
5068                                VG_USERREQ__CLIENT_CALL0,          \
5069                                _qyy_fn,                           \
5070                                0, 0, 0, 0);                       \
5071     _qyy_res;                                                     \
5072    })
5073
5074 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
5075    __extension__                                                  \
5076    ({unsigned long _qyy_res;                                      \
5077     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
5078                                VG_USERREQ__CLIENT_CALL1,          \
5079                                _qyy_fn,                           \
5080                                _qyy_arg1, 0, 0, 0);               \
5081     _qyy_res;                                                     \
5082    })
5083
5084 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
5085    __extension__                                                  \
5086    ({unsigned long _qyy_res;                                      \
5087     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
5088                                VG_USERREQ__CLIENT_CALL2,          \
5089                                _qyy_fn,                           \
5090                                _qyy_arg1, _qyy_arg2, 0, 0);       \
5091     _qyy_res;                                                     \
5092    })
5093
5094 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
5095    __extension__                                                  \
5096    ({unsigned long _qyy_res;                                      \
5097     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
5098                                VG_USERREQ__CLIENT_CALL3,          \
5099                                _qyy_fn,                           \
5100                                _qyy_arg1, _qyy_arg2,              \
5101                                _qyy_arg3, 0);                     \
5102     _qyy_res;                                                     \
5103    })
5104
5105
5106 /* Counts the number of errors that have been recorded by a tool.  Nb:
5107    the tool must record the errors with VG_(maybe_record_error)() or
5108    VG_(unique_error)() for them to be counted. */
5109 #define VALGRIND_COUNT_ERRORS                                     \
5110    __extension__                                                  \
5111    ({unsigned int _qyy_res;                                       \
5112     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
5113                                VG_USERREQ__COUNT_ERRORS,          \
5114                                0, 0, 0, 0, 0);                    \
5115     _qyy_res;                                                     \
5116    })
5117
5118 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
5119    when heap blocks are allocated in order to give accurate results.  This
5120    happens automatically for the standard allocator functions such as
5121    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
5122    delete[], etc.
5123
5124    But if your program uses a custom allocator, this doesn't automatically
5125    happen, and Valgrind will not do as well.  For example, if you allocate
5126    superblocks with mmap() and then allocates chunks of the superblocks, all
5127    Valgrind's observations will be at the mmap() level and it won't know that
5128    the chunks should be considered separate entities.  In Memcheck's case,
5129    that means you probably won't get heap block overrun detection (because
5130    there won't be redzones marked as unaddressable) and you definitely won't
5131    get any leak detection.
5132
5133    The following client requests allow a custom allocator to be annotated so
5134    that it can be handled accurately by Valgrind.
5135
5136    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
5137    by a malloc()-like function.  For Memcheck (an illustrative case), this
5138    does two things:
5139
5140    - It records that the block has been allocated.  This means any addresses
5141      within the block mentioned in error messages will be
5142      identified as belonging to the block.  It also means that if the block
5143      isn't freed it will be detected by the leak checker.
5144
5145    - It marks the block as being addressable and undefined (if 'is_zeroed' is
5146      not set), or addressable and defined (if 'is_zeroed' is set).  This
5147      controls how accesses to the block by the program are handled.
5148
5149    'addr' is the start of the usable block (ie. after any
5150    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
5151    can apply redzones -- these are blocks of padding at the start and end of
5152    each block.  Adding redzones is recommended as it makes it much more likely
5153    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
5154    zeroed (or filled with another predictable value), as is the case for
5155    calloc().
5156
5157    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
5158    heap block -- that will be used by the client program -- is allocated.
5159    It's best to put it at the outermost level of the allocator if possible;
5160    for example, if you have a function my_alloc() which calls
5161    internal_alloc(), and the client request is put inside internal_alloc(),
5162    stack traces relating to the heap block will contain entries for both
5163    my_alloc() and internal_alloc(), which is probably not what you want.
5164
5165    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
5166    custom blocks from within a heap block, B, that has been allocated with
5167    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
5168    -- the custom blocks will take precedence.
5169
5170    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
5171    Memcheck, it does two things:
5172
5173    - It records that the block has been deallocated.  This assumes that the
5174      block was annotated as having been allocated via
5175      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
5176
5177    - It marks the block as being unaddressable.
5178
5179    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
5180    heap block is deallocated.
5181
5182    In many cases, these two client requests will not be enough to get your
5183    allocator working well with Memcheck.  More specifically, if your allocator
5184    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
5185    will be necessary to mark the memory as addressable just before the zeroing
5186    occurs, otherwise you'll get a lot of invalid write errors.  For example,
5187    you'll need to do this if your allocator recycles freed blocks, but it
5188    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
5189    Alternatively, if your allocator reuses freed blocks for allocator-internal
5190    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
5191
5192    Really, what's happening is a blurring of the lines between the client
5193    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
5194    memory should be considered unaddressable to the client program, but the
5195    allocator knows more than the rest of the client program and so may be able
5196    to safely access it.  Extra client requests are necessary for Valgrind to
5197    understand the distinction between the allocator and the rest of the
5198    program.
5199
5200    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
5201    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
5202
5203    Ignored if addr == 0.
5204 */
5205 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
5206    {unsigned int _qzz_res VG_UNUSED;                                        \
5207     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5208                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
5209                                addr, sizeB, rzB, is_zeroed, 0);   \
5210    }
5211
5212 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
5213    Ignored if addr == 0.
5214 */
5215 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
5216    {unsigned int _qzz_res VG_UNUSED;                                        \
5217     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5218                                VG_USERREQ__FREELIKE_BLOCK,        \
5219                                addr, rzB, 0, 0, 0);               \
5220    }
5221
5222 /* Create a memory pool. */
5223 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
5224    {unsigned int _qzz_res VG_UNUSED;                              \
5225     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5226                                VG_USERREQ__CREATE_MEMPOOL,        \
5227                                pool, rzB, is_zeroed, 0, 0);       \
5228    }
5229
5230 /* Destroy a memory pool. */
5231 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
5232    {unsigned int _qzz_res VG_UNUSED;                                        \
5233     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5234                                VG_USERREQ__DESTROY_MEMPOOL,       \
5235                                pool, 0, 0, 0, 0);                 \
5236    }
5237
5238 /* Associate a piece of memory with a memory pool. */
5239 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
5240    {unsigned int _qzz_res VG_UNUSED;                              \
5241     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5242                                VG_USERREQ__MEMPOOL_ALLOC,         \
5243                                pool, addr, size, 0, 0);           \
5244    }
5245
5246 /* Disassociate a piece of memory from a memory pool. */
5247 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
5248    {unsigned int _qzz_res VG_UNUSED;                              \
5249     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5250                                VG_USERREQ__MEMPOOL_FREE,          \
5251                                pool, addr, 0, 0, 0);              \
5252    }
5253
5254 /* Disassociate any pieces outside a particular range. */
5255 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
5256    {unsigned int _qzz_res VG_UNUSED;                              \
5257     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5258                                VG_USERREQ__MEMPOOL_TRIM,          \
5259                                pool, addr, size, 0, 0);           \
5260    }
5261
5262 /* Resize and/or move a piece associated with a memory pool. */
5263 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
5264    {unsigned int _qzz_res VG_UNUSED;                              \
5265     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5266                                VG_USERREQ__MOVE_MEMPOOL,          \
5267                                poolA, poolB, 0, 0, 0);            \
5268    }
5269
5270 /* Resize and/or move a piece associated with a memory pool. */
5271 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
5272    {unsigned int _qzz_res VG_UNUSED;                              \
5273     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5274                                VG_USERREQ__MEMPOOL_CHANGE,        \
5275                                pool, addrA, addrB, size, 0);      \
5276    }
5277
5278 /* Return 1 if a mempool exists, else 0. */
5279 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
5280    __extension__                                                  \
5281    ({unsigned int _qzz_res;                                       \
5282     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5283                                VG_USERREQ__MEMPOOL_EXISTS,        \
5284                                pool, 0, 0, 0, 0);                 \
5285     _qzz_res;                                                     \
5286    })
5287
5288 /* Mark a piece of memory as being a stack. Returns a stack id. */
5289 #define VALGRIND_STACK_REGISTER(start, end)                       \
5290    __extension__                                                  \
5291    ({unsigned int _qzz_res;                                       \
5292     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5293                                VG_USERREQ__STACK_REGISTER,        \
5294                                start, end, 0, 0, 0);              \
5295     _qzz_res;                                                     \
5296    })
5297
5298 /* Unmark the piece of memory associated with a stack id as being a
5299    stack. */
5300 #define VALGRIND_STACK_DEREGISTER(id)                             \
5301    {unsigned int _qzz_res VG_UNUSED;                              \
5302     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5303                                VG_USERREQ__STACK_DEREGISTER,      \
5304                                id, 0, 0, 0, 0);                   \
5305    }
5306
5307 /* Change the start and end address of the stack id. */
5308 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
5309    {unsigned int _qzz_res VG_UNUSED;                              \
5310     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5311                                VG_USERREQ__STACK_CHANGE,          \
5312                                id, start, end, 0, 0);             \
5313    }
5314
5315 /* Load PDB debug info for Wine PE image_map. */
5316 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
5317    {unsigned int _qzz_res VG_UNUSED;                              \
5318     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5319                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
5320                                fd, ptr, total_size, delta, 0);    \
5321    }
5322
5323 /* Map a code address to a source file name and line number.  buf64
5324    must point to a 64-byte buffer in the caller's address space.  The
5325    result will be dumped in there and is guaranteed to be zero
5326    terminated.  If no info is found, the first byte is set to zero. */
5327 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
5328    {unsigned int _qzz_res VG_UNUSED;                              \
5329     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
5330                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
5331                                addr, buf64, 0, 0, 0);             \
5332    }
5333
5334
5335 #undef PLAT_x86_linux
5336 #undef PLAT_amd64_linux
5337 #undef PLAT_ppc32_linux
5338 #undef PLAT_ppc64_linux
5339 #undef PLAT_arm_linux
5340 #undef PLAT_s390x_linux
5341 #undef PLAT_ppc32_aix5
5342 #undef PLAT_ppc64_aix5
5343
5344 #endif   /* __VALGRIND_H */