arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37
  38 #ifndef CONFIG_PPC_64K_PAGES
  39
  40 /*****************************************************************************
  41  *                                                                           *
  42  *           4K SW & 4K HW pages implementation                              *
  43  *                                                                           *
  44  *****************************************************************************/
  45
  46
  47 /*
  48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  49  *               pte_t *ptep, unsigned long trap, int local, int ssize)
  50  *
  51  * Adds a 4K page to the hash table in a segment of 4K pages only
  52  */
  53
  54 _GLOBAL(__hash_page_4K)
  55         mflr    r0
  56         std     r0,16(r1)
  57         stdu    r1,-STACKFRAMESIZE(r1)
  58         /* Save all params that we need after a function call */
  59         std     r6,STK_PARAM(R6)(r1)
  60         std     r8,STK_PARAM(R8)(r1)
  61         std     r9,STK_PARAM(R9)(r1)
  62
  63         /* Save non-volatile registers.
  64          * r31 will hold "old PTE"
  65          * r30 is "new PTE"
  66          * r29 is vpn
  67          * r28 is a hash value
  68          * r27 is hashtab mask (maybe dynamic patched instead ?)
  69          */
  70         std     r27,STK_REG(R27)(r1)
  71         std     r28,STK_REG(R28)(r1)
  72         std     r29,STK_REG(R29)(r1)
  73         std     r30,STK_REG(R30)(r1)
  74         std     r31,STK_REG(R31)(r1)
  75
  76         /* Step 1:
  77          *
  78          * Check permissions, atomically mark the linux PTE busy
  79          * and hashed.
  80          */
  81 1:
  82         ldarx   r31,0,r6
  83         /* Check access rights (access & ~(pte_val(*ptep))) */
  84         andc.   r0,r4,r31
  85         bne-    htab_wrong_access
  86         /* Check if PTE is busy */
  87         andi.   r0,r31,_PAGE_BUSY
  88         /* If so, just bail out and refault if needed. Someone else
  89          * is changing this PTE anyway and might hash it.
  90          */
  91         bne-    htab_bail_ok
  92
  93         /* Prepare new PTE value (turn access RW into DIRTY, then
  94          * add BUSY,HASHPTE and ACCESSED)
  95          */
  96         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  97         or      r30,r30,r31
  98         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
  99         /* Write the linux PTE atomically (setting busy) */
 100         stdcx.  r30,0,r6
 101         bne-    1b
 102         isync
 103
 104         /* Step 2:
 105          *
 106          * Insert/Update the HPTE in the hash table. At this point,
 107          * r4 (access) is re-useable, we use it for the new HPTE flags
 108          */
 109
 110 BEGIN_FTR_SECTION
 111         cmpdi   r9,0                    /* check segment size */
 112         bne     3f
 113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 114         /* Calc vpn and put it in r29 */
 115         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 116         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 117         or      r29,r28,r29
 118         /*
 119          * Calculate hash value for primary slot and store it in r28
 120          * r3 = va, r5 = vsid
 121          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 122          */
 123         rldicl  r0,r3,64-12,48
 124         xor     r28,r5,r0               /* hash */
 125         b       4f
 126
 127 3:      /* Calc vpn and put it in r29 */
 128         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 129         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 130         or      r29,r28,r29
 131
 132         /*
 133          * calculate hash value for primary slot and
 134          * store it in r28 for 1T segment
 135          * r3 = va, r5 = vsid
 136          */
 137         sldi    r28,r5,25               /* vsid << 25 */
 138         /* r0 =  (va >> 12) & ((1ul << (40 - 12)) -1) */
 139         rldicl  r0,r3,64-12,36
 140         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 141         xor     r28,r28,r0              /* hash */
 142
 143         /* Convert linux PTE bits into HW equivalents */
 144 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 145         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 146         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 147         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 148         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 149         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 150         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 151         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 152
 153         /* We eventually do the icache sync here (maybe inline that
 154          * code rather than call a C function...)
 155          */
 156 BEGIN_FTR_SECTION
 157         mr      r4,r30
 158         mr      r5,r7
 159         bl      .hash_page_do_lazy_icache
 160 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 161
 162         /* At this point, r3 contains new PP bits, save them in
 163          * place of "access" in the param area (sic)
 164          */
 165         std     r3,STK_PARAM(R4)(r1)
 166
 167         /* Get htab_hash_mask */
 168         ld      r4,htab_hash_mask@got(2)
 169         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 170
 171         /* Check if we may already be in the hashtable, in this case, we
 172          * go to out-of-line code to try to modify the HPTE
 173          */
 174         andi.   r0,r31,_PAGE_HASHPTE
 175         bne     htab_modify_pte
 176
 177 htab_insert_pte:
 178         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 179          * add _PAGE_HASHPTE
 180          */
 181         lis     r0,_PAGE_HPTEFLAGS@h
 182         ori     r0,r0,_PAGE_HPTEFLAGS@l
 183         andc    r30,r30,r0
 184         ori     r30,r30,_PAGE_HASHPTE
 185
 186         /* physical address r5 */
 187         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 188         sldi    r5,r5,PAGE_SHIFT
 189
 190         /* Calculate primary group hash */
 191         and     r0,r28,r27
 192         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 193
 194         /* Call ppc_md.hpte_insert */
 195         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 196         mr      r4,r29                  /* Retrieve vpn */
 197         li      r7,0                    /* !bolted, !secondary */
 198         li      r8,MMU_PAGE_4K          /* page size */
 199         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 200 _GLOBAL(htab_call_hpte_insert1)
 201         bl      .                       /* Patched by htab_finish_init() */
 202         cmpdi   0,r3,0
 203         bge     htab_pte_insert_ok      /* Insertion successful */
 204         cmpdi   0,r3,-2                 /* Critical failure */
 205         beq-    htab_pte_insert_failure
 206
 207         /* Now try secondary slot */
 208
 209         /* physical address r5 */
 210         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 211         sldi    r5,r5,PAGE_SHIFT
 212
 213         /* Calculate secondary group hash */
 214         andc    r0,r27,r28
 215         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 216
 217         /* Call ppc_md.hpte_insert */
 218         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 219         mr      r4,r29                  /* Retrieve vpn */
 220         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 221         li      r8,MMU_PAGE_4K          /* page size */
 222         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 223 _GLOBAL(htab_call_hpte_insert2)
 224         bl      .                       /* Patched by htab_finish_init() */
 225         cmpdi   0,r3,0
 226         bge+    htab_pte_insert_ok      /* Insertion successful */
 227         cmpdi   0,r3,-2                 /* Critical failure */
 228         beq-    htab_pte_insert_failure
 229
 230         /* Both are full, we need to evict something */
 231         mftb    r0
 232         /* Pick a random group based on TB */
 233         andi.   r0,r0,1
 234         mr      r5,r28
 235         bne     2f
 236         not     r5,r5
 237 2:      and     r0,r5,r27
 238         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 239         /* Call ppc_md.hpte_remove */
 240 _GLOBAL(htab_call_hpte_remove)
 241         bl      .                       /* Patched by htab_finish_init() */
 242
 243         /* Try all again */
 244         b       htab_insert_pte
 245
 246 htab_bail_ok:
 247         li      r3,0
 248         b       htab_bail
 249
 250 htab_pte_insert_ok:
 251         /* Insert slot number & secondary bit in PTE */
 252         rldimi  r30,r3,12,63-15
 253
 254         /* Write out the PTE with a normal write
 255          * (maybe add eieio may be good still ?)
 256          */
 257 htab_write_out_pte:
 258         ld      r6,STK_PARAM(R6)(r1)
 259         std     r30,0(r6)
 260         li      r3, 0
 261 htab_bail:
 262         ld      r27,STK_REG(R27)(r1)
 263         ld      r28,STK_REG(R28)(r1)
 264         ld      r29,STK_REG(R29)(r1)
 265         ld      r30,STK_REG(R30)(r1)
 266         ld      r31,STK_REG(R31)(r1)
 267         addi    r1,r1,STACKFRAMESIZE
 268         ld      r0,16(r1)
 269         mtlr    r0
 270         blr
 271
 272 htab_modify_pte:
 273         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 274         mr      r4,r3
 275         rlwinm  r3,r31,32-12,29,31
 276
 277         /* Secondary group ? if yes, get a inverted hash value */
 278         mr      r5,r28
 279         andi.   r0,r31,_PAGE_SECONDARY
 280         beq     1f
 281         not     r5,r5
 282 1:
 283         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 284         and     r0,r5,r27
 285         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 286         add     r3,r0,r3        /* add slot idx */
 287
 288         /* Call ppc_md.hpte_updatepp */
 289         mr      r5,r29                  /* vpn */
 290         li      r6,MMU_PAGE_4K          /* page size */
 291         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 292         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 293 _GLOBAL(htab_call_hpte_updatepp)
 294         bl      .                       /* Patched by htab_finish_init() */
 295
 296         /* if we failed because typically the HPTE wasn't really here
 297          * we try an insertion.
 298          */
 299         cmpdi   0,r3,-1
 300         beq-    htab_insert_pte
 301
 302         /* Clear the BUSY bit and Write out the PTE */
 303         li      r0,_PAGE_BUSY
 304         andc    r30,r30,r0
 305         b       htab_write_out_pte
 306
 307 htab_wrong_access:
 308         /* Bail out clearing reservation */
 309         stdcx.  r31,0,r6
 310         li      r3,1
 311         b       htab_bail
 312
 313 htab_pte_insert_failure:
 314         /* Bail out restoring old PTE */
 315         ld      r6,STK_PARAM(R6)(r1)
 316         std     r31,0(r6)
 317         li      r3,-1
 318         b       htab_bail
 319
 320
 321 #else /* CONFIG_PPC_64K_PAGES */
 322
 323
 324 /*****************************************************************************
 325  *                                                                           *
 326  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 327  *                                                                           *
 328  *****************************************************************************/
 329
 330 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 331  *               pte_t *ptep, unsigned long trap, int local, int ssize,
 332  *               int subpg_prot)
 333  */
 334
 335 /*
 336  * For now, we do NOT implement Admixed pages
 337  */
 338 _GLOBAL(__hash_page_4K)
 339         mflr    r0
 340         std     r0,16(r1)
 341         stdu    r1,-STACKFRAMESIZE(r1)
 342         /* Save all params that we need after a function call */
 343         std     r6,STK_PARAM(R6)(r1)
 344         std     r8,STK_PARAM(R8)(r1)
 345         std     r9,STK_PARAM(R9)(r1)
 346
 347         /* Save non-volatile registers.
 348          * r31 will hold "old PTE"
 349          * r30 is "new PTE"
 350          * r29 is vpn
 351          * r28 is a hash value
 352          * r27 is hashtab mask (maybe dynamic patched instead ?)
 353          * r26 is the hidx mask
 354          * r25 is the index in combo page
 355          */
 356         std     r25,STK_REG(R25)(r1)
 357         std     r26,STK_REG(R26)(r1)
 358         std     r27,STK_REG(R27)(r1)
 359         std     r28,STK_REG(R28)(r1)
 360         std     r29,STK_REG(R29)(r1)
 361         std     r30,STK_REG(R30)(r1)
 362         std     r31,STK_REG(R31)(r1)
 363
 364         /* Step 1:
 365          *
 366          * Check permissions, atomically mark the linux PTE busy
 367          * and hashed.
 368          */
 369 1:
 370         ldarx   r31,0,r6
 371         /* Check access rights (access & ~(pte_val(*ptep))) */
 372         andc.   r0,r4,r31
 373         bne-    htab_wrong_access
 374         /* Check if PTE is busy */
 375         andi.   r0,r31,_PAGE_BUSY
 376         /* If so, just bail out and refault if needed. Someone else
 377          * is changing this PTE anyway and might hash it.
 378          */
 379         bne-    htab_bail_ok
 380         /* Prepare new PTE value (turn access RW into DIRTY, then
 381          * add BUSY and ACCESSED)
 382          */
 383         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 384         or      r30,r30,r31
 385         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 386         oris    r30,r30,_PAGE_COMBO@h
 387         /* Write the linux PTE atomically (setting busy) */
 388         stdcx.  r30,0,r6
 389         bne-    1b
 390         isync
 391
 392         /* Step 2:
 393          *
 394          * Insert/Update the HPTE in the hash table. At this point,
 395          * r4 (access) is re-useable, we use it for the new HPTE flags
 396          */
 397
 398         /* Load the hidx index */
 399         rldicl  r25,r3,64-12,60
 400
 401 BEGIN_FTR_SECTION
 402         cmpdi   r9,0                    /* check segment size */
 403         bne     3f
 404 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 405         /* Calc vpn and put it in r29 */
 406         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 407         /*
 408          * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
 409          * srdi  r28,r3,VPN_SHIFT
 410          */
 411         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 412         or      r29,r28,r29
 413         /*
 414          * Calculate hash value for primary slot and store it in r28
 415          * r3 = va, r5 = vsid
 416          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 417          */
 418         rldicl  r0,r3,64-12,48
 419         xor     r28,r5,r0               /* hash */
 420         b       4f
 421
 422 3:      /* Calc vpn and put it in r29 */
 423         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 424         /*
 425          * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
 426          * srdi r28,r3,VPN_SHIFT
 427          */
 428         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 429         or      r29,r28,r29
 430
 431         /*
 432          * Calculate hash value for primary slot and
 433          * store it in r28  for 1T segment
 434          * r3 = va, r5 = vsid
 435          */
 436         sldi    r28,r5,25               /* vsid << 25 */
 437         /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
 438         rldicl  r0,r3,64-12,36
 439         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 440         xor     r28,r28,r0              /* hash */
 441
 442         /* Convert linux PTE bits into HW equivalents */
 443 4:
 444 #ifdef CONFIG_PPC_SUBPAGE_PROT
 445         andc    r10,r30,r10
 446         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 447         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 448 #else
 449         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 450         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 451 #endif
 452         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 453         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 454         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 455         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 456         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 457         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 458
 459         /* We eventually do the icache sync here (maybe inline that
 460          * code rather than call a C function...)
 461          */
 462 BEGIN_FTR_SECTION
 463         mr      r4,r30
 464         mr      r5,r7
 465         bl      .hash_page_do_lazy_icache
 466 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 467
 468         /* At this point, r3 contains new PP bits, save them in
 469          * place of "access" in the param area (sic)
 470          */
 471         std     r3,STK_PARAM(R4)(r1)
 472
 473         /* Get htab_hash_mask */
 474         ld      r4,htab_hash_mask@got(2)
 475         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 476
 477         /* Check if we may already be in the hashtable, in this case, we
 478          * go to out-of-line code to try to modify the HPTE. We look for
 479          * the bit at (1 >> (index + 32))
 480          */
 481         rldicl. r0,r31,64-12,48
 482         li      r26,0                   /* Default hidx */
 483         beq     htab_insert_pte
 484
 485         /*
 486          * Check if the pte was already inserted into the hash table
 487          * as a 64k HW page, and invalidate the 64k HPTE if so.
 488          */
 489         andis.  r0,r31,_PAGE_COMBO@h
 490         beq     htab_inval_old_hpte
 491
 492         ld      r6,STK_PARAM(R6)(r1)
 493         ori     r26,r6,0x8000           /* Load the hidx mask */
 494         ld      r26,0(r26)
 495         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 496         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 497         bne     htab_modify_pte
 498
 499 htab_insert_pte:
 500         /* real page number in r5, PTE RPN value + index */
 501         andis.  r0,r31,_PAGE_4K_PFN@h
 502         srdi    r5,r31,PTE_RPN_SHIFT
 503         bne-    htab_special_pfn
 504         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 505         add     r5,r5,r25
 506 htab_special_pfn:
 507         sldi    r5,r5,HW_PAGE_SHIFT
 508
 509         /* Calculate primary group hash */
 510         and     r0,r28,r27
 511         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 512
 513         /* Call ppc_md.hpte_insert */
 514         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 515         mr      r4,r29                  /* Retrieve vpn */
 516         li      r7,0                    /* !bolted, !secondary */
 517         li      r8,MMU_PAGE_4K          /* page size */
 518         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 519 _GLOBAL(htab_call_hpte_insert1)
 520         bl      .                       /* patched by htab_finish_init() */
 521         cmpdi   0,r3,0
 522         bge     htab_pte_insert_ok      /* Insertion successful */
 523         cmpdi   0,r3,-2                 /* Critical failure */
 524         beq-    htab_pte_insert_failure
 525
 526         /* Now try secondary slot */
 527
 528         /* real page number in r5, PTE RPN value + index */
 529         andis.  r0,r31,_PAGE_4K_PFN@h
 530         srdi    r5,r31,PTE_RPN_SHIFT
 531         bne-    3f
 532         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 533         add     r5,r5,r25
 534 3:      sldi    r5,r5,HW_PAGE_SHIFT
 535
 536         /* Calculate secondary group hash */
 537         andc    r0,r27,r28
 538         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 539
 540         /* Call ppc_md.hpte_insert */
 541         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 542         mr      r4,r29                  /* Retrieve vpn */
 543         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 544         li      r8,MMU_PAGE_4K          /* page size */
 545         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 546 _GLOBAL(htab_call_hpte_insert2)
 547         bl      .                       /* patched by htab_finish_init() */
 548         cmpdi   0,r3,0
 549         bge+    htab_pte_insert_ok      /* Insertion successful */
 550         cmpdi   0,r3,-2                 /* Critical failure */
 551         beq-    htab_pte_insert_failure
 552
 553         /* Both are full, we need to evict something */
 554         mftb    r0
 555         /* Pick a random group based on TB */
 556         andi.   r0,r0,1
 557         mr      r5,r28
 558         bne     2f
 559         not     r5,r5
 560 2:      and     r0,r5,r27
 561         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 562         /* Call ppc_md.hpte_remove */
 563 _GLOBAL(htab_call_hpte_remove)
 564         bl      .                       /* patched by htab_finish_init() */
 565
 566         /* Try all again */
 567         b       htab_insert_pte
 568
 569         /*
 570          * Call out to C code to invalidate an 64k HW HPTE that is
 571          * useless now that the segment has been switched to 4k pages.
 572          */
 573 htab_inval_old_hpte:
 574         mr      r3,r29                  /* vpn */
 575         mr      r4,r31                  /* PTE.pte */
 576         li      r5,0                    /* PTE.hidx */
 577         li      r6,MMU_PAGE_64K         /* psize */
 578         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
 579         ld      r8,STK_PARAM(R8)(r1)    /* local */
 580         bl      .flush_hash_page
 581         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 582         lis     r0,_PAGE_HPTE_SUB@h
 583         ori     r0,r0,_PAGE_HPTE_SUB@l
 584         andc    r30,r30,r0
 585         b       htab_insert_pte
 586
 587 htab_bail_ok:
 588         li      r3,0
 589         b       htab_bail
 590
 591 htab_pte_insert_ok:
 592         /* Insert slot number & secondary bit in PTE second half,
 593          * clear _PAGE_BUSY and set approriate HPTE slot bit
 594          */
 595         ld      r6,STK_PARAM(R6)(r1)
 596         li      r0,_PAGE_BUSY
 597         andc    r30,r30,r0
 598         /* HPTE SUB bit */
 599         li      r0,1
 600         subfic  r5,r25,27               /* Must match bit position in */
 601         sld     r0,r0,r5                /* pgtable.h */
 602         or      r30,r30,r0
 603         /* hindx */
 604         sldi    r5,r25,2
 605         sld     r3,r3,r5
 606         li      r4,0xf
 607         sld     r4,r4,r5
 608         andc    r26,r26,r4
 609         or      r26,r26,r3
 610         ori     r5,r6,0x8000
 611         std     r26,0(r5)
 612         lwsync
 613         std     r30,0(r6)
 614         li      r3, 0
 615 htab_bail:
 616         ld      r25,STK_REG(R25)(r1)
 617         ld      r26,STK_REG(R26)(r1)
 618         ld      r27,STK_REG(R27)(r1)
 619         ld      r28,STK_REG(R28)(r1)
 620         ld      r29,STK_REG(R29)(r1)
 621         ld      r30,STK_REG(R30)(r1)
 622         ld      r31,STK_REG(R31)(r1)
 623         addi    r1,r1,STACKFRAMESIZE
 624         ld      r0,16(r1)
 625         mtlr    r0
 626         blr
 627
 628 htab_modify_pte:
 629         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 630         mr      r4,r3
 631         sldi    r5,r25,2
 632         srd     r3,r26,r5
 633
 634         /* Secondary group ? if yes, get a inverted hash value */
 635         mr      r5,r28
 636         andi.   r0,r3,0x8 /* page secondary ? */
 637         beq     1f
 638         not     r5,r5
 639 1:      andi.   r3,r3,0x7 /* extract idx alone */
 640
 641         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 642         and     r0,r5,r27
 643         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 644         add     r3,r0,r3        /* add slot idx */
 645
 646         /* Call ppc_md.hpte_updatepp */
 647         mr      r5,r29                  /* vpn */
 648         li      r6,MMU_PAGE_4K          /* page size */
 649         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 650         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 651 _GLOBAL(htab_call_hpte_updatepp)
 652         bl      .                       /* patched by htab_finish_init() */
 653
 654         /* if we failed because typically the HPTE wasn't really here
 655          * we try an insertion.
 656          */
 657         cmpdi   0,r3,-1
 658         beq-    htab_insert_pte
 659
 660         /* Clear the BUSY bit and Write out the PTE */
 661         li      r0,_PAGE_BUSY
 662         andc    r30,r30,r0
 663         ld      r6,STK_PARAM(R6)(r1)
 664         std     r30,0(r6)
 665         li      r3,0
 666         b       htab_bail
 667
 668 htab_wrong_access:
 669         /* Bail out clearing reservation */
 670         stdcx.  r31,0,r6
 671         li      r3,1
 672         b       htab_bail
 673
 674 htab_pte_insert_failure:
 675         /* Bail out restoring old PTE */
 676         ld      r6,STK_PARAM(R6)(r1)
 677         std     r31,0(r6)
 678         li      r3,-1
 679         b       htab_bail
 680
 681 #endif /* CONFIG_PPC_64K_PAGES */
 682
 683 #ifdef CONFIG_PPC_HAS_HASH_64K
 684
 685 /*****************************************************************************
 686  *                                                                           *
 687  *           64K SW & 64K HW in a 64K segment pages implementation           *
 688  *                                                                           *
 689  *****************************************************************************/
 690
 691 _GLOBAL(__hash_page_64K)
 692         mflr    r0
 693         std     r0,16(r1)
 694         stdu    r1,-STACKFRAMESIZE(r1)
 695         /* Save all params that we need after a function call */
 696         std     r6,STK_PARAM(R6)(r1)
 697         std     r8,STK_PARAM(R8)(r1)
 698         std     r9,STK_PARAM(R9)(r1)
 699
 700         /* Save non-volatile registers.
 701          * r31 will hold "old PTE"
 702          * r30 is "new PTE"
 703          * r29 is vpn
 704          * r28 is a hash value
 705          * r27 is hashtab mask (maybe dynamic patched instead ?)
 706          */
 707         std     r27,STK_REG(R27)(r1)
 708         std     r28,STK_REG(R28)(r1)
 709         std     r29,STK_REG(R29)(r1)
 710         std     r30,STK_REG(R30)(r1)
 711         std     r31,STK_REG(R31)(r1)
 712
 713         /* Step 1:
 714          *
 715          * Check permissions, atomically mark the linux PTE busy
 716          * and hashed.
 717          */
 718 1:
 719         ldarx   r31,0,r6
 720         /* Check access rights (access & ~(pte_val(*ptep))) */
 721         andc.   r0,r4,r31
 722         bne-    ht64_wrong_access
 723         /* Check if PTE is busy */
 724         andi.   r0,r31,_PAGE_BUSY
 725         /* If so, just bail out and refault if needed. Someone else
 726          * is changing this PTE anyway and might hash it.
 727          */
 728         bne-    ht64_bail_ok
 729 BEGIN_FTR_SECTION
 730         /* Check if PTE has the cache-inhibit bit set */
 731         andi.   r0,r31,_PAGE_NO_CACHE
 732         /* If so, bail out and refault as a 4k page */
 733         bne-    ht64_bail_ok
 734 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
 735         /* Prepare new PTE value (turn access RW into DIRTY, then
 736          * add BUSY and ACCESSED)
 737          */
 738         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 739         or      r30,r30,r31
 740         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 741         /* Write the linux PTE atomically (setting busy) */
 742         stdcx.  r30,0,r6
 743         bne-    1b
 744         isync
 745
 746         /* Step 2:
 747          *
 748          * Insert/Update the HPTE in the hash table. At this point,
 749          * r4 (access) is re-useable, we use it for the new HPTE flags
 750          */
 751
 752 BEGIN_FTR_SECTION
 753         cmpdi   r9,0                    /* check segment size */
 754         bne     3f
 755 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 756         /* Calc vpn and put it in r29 */
 757         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 758         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 759         or      r29,r28,r29
 760
 761         /* Calculate hash value for primary slot and store it in r28
 762          * r3 = va, r5 = vsid
 763          * r0 = (va >> 16) & ((1ul << (28 - 16)) -1)
 764          */
 765         rldicl  r0,r3,64-16,52
 766         xor     r28,r5,r0               /* hash */
 767         b       4f
 768
 769 3:      /* Calc vpn and put it in r29 */
 770         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 771         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 772         or      r29,r28,r29
 773         /*
 774          * calculate hash value for primary slot and
 775          * store it in r28 for 1T segment
 776          * r3 = va, r5 = vsid
 777          */
 778         sldi    r28,r5,25               /* vsid << 25 */
 779         /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */
 780         rldicl  r0,r3,64-16,40
 781         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 782         xor     r28,r28,r0              /* hash */
 783
 784         /* Convert linux PTE bits into HW equivalents */
 785 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 786         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 787         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 788         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 789         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 790         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 791         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 792         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 793
 794         /* We eventually do the icache sync here (maybe inline that
 795          * code rather than call a C function...)
 796          */
 797 BEGIN_FTR_SECTION
 798         mr      r4,r30
 799         mr      r5,r7
 800         bl      .hash_page_do_lazy_icache
 801 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 802
 803         /* At this point, r3 contains new PP bits, save them in
 804          * place of "access" in the param area (sic)
 805          */
 806         std     r3,STK_PARAM(R4)(r1)
 807
 808         /* Get htab_hash_mask */
 809         ld      r4,htab_hash_mask@got(2)
 810         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 811
 812         /* Check if we may already be in the hashtable, in this case, we
 813          * go to out-of-line code to try to modify the HPTE
 814          */
 815         rldicl. r0,r31,64-12,48
 816         bne     ht64_modify_pte
 817
 818 ht64_insert_pte:
 819         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 820          * add _PAGE_HPTE_SUB0
 821          */
 822         lis     r0,_PAGE_HPTEFLAGS@h
 823         ori     r0,r0,_PAGE_HPTEFLAGS@l
 824         andc    r30,r30,r0
 825 #ifdef CONFIG_PPC_64K_PAGES
 826         oris    r30,r30,_PAGE_HPTE_SUB0@h
 827 #else
 828         ori     r30,r30,_PAGE_HASHPTE
 829 #endif
 830         /* Phyical address in r5 */
 831         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 832         sldi    r5,r5,PAGE_SHIFT
 833
 834         /* Calculate primary group hash */
 835         and     r0,r28,r27
 836         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 837
 838         /* Call ppc_md.hpte_insert */
 839         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 840         mr      r4,r29                  /* Retrieve vpn */
 841         li      r7,0                    /* !bolted, !secondary */
 842         li      r8,MMU_PAGE_64K
 843         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 844 _GLOBAL(ht64_call_hpte_insert1)
 845         bl      .                       /* patched by htab_finish_init() */
 846         cmpdi   0,r3,0
 847         bge     ht64_pte_insert_ok      /* Insertion successful */
 848         cmpdi   0,r3,-2                 /* Critical failure */
 849         beq-    ht64_pte_insert_failure
 850
 851         /* Now try secondary slot */
 852
 853         /* Phyical address in r5 */
 854         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 855         sldi    r5,r5,PAGE_SHIFT
 856
 857         /* Calculate secondary group hash */
 858         andc    r0,r27,r28
 859         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 860
 861         /* Call ppc_md.hpte_insert */
 862         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 863         mr      r4,r29                  /* Retrieve vpn */
 864         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 865         li      r8,MMU_PAGE_64K
 866         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
 867 _GLOBAL(ht64_call_hpte_insert2)
 868         bl      .                       /* patched by htab_finish_init() */
 869         cmpdi   0,r3,0
 870         bge+    ht64_pte_insert_ok      /* Insertion successful */
 871         cmpdi   0,r3,-2                 /* Critical failure */
 872         beq-    ht64_pte_insert_failure
 873
 874         /* Both are full, we need to evict something */
 875         mftb    r0
 876         /* Pick a random group based on TB */
 877         andi.   r0,r0,1
 878         mr      r5,r28
 879         bne     2f
 880         not     r5,r5
 881 2:      and     r0,r5,r27
 882         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 883         /* Call ppc_md.hpte_remove */
 884 _GLOBAL(ht64_call_hpte_remove)
 885         bl      .                       /* patched by htab_finish_init() */
 886
 887         /* Try all again */
 888         b       ht64_insert_pte
 889
 890 ht64_bail_ok:
 891         li      r3,0
 892         b       ht64_bail
 893
 894 ht64_pte_insert_ok:
 895         /* Insert slot number & secondary bit in PTE */
 896         rldimi  r30,r3,12,63-15
 897
 898         /* Write out the PTE with a normal write
 899          * (maybe add eieio may be good still ?)
 900          */
 901 ht64_write_out_pte:
 902         ld      r6,STK_PARAM(R6)(r1)
 903         std     r30,0(r6)
 904         li      r3, 0
 905 ht64_bail:
 906         ld      r27,STK_REG(R27)(r1)
 907         ld      r28,STK_REG(R28)(r1)
 908         ld      r29,STK_REG(R29)(r1)
 909         ld      r30,STK_REG(R30)(r1)
 910         ld      r31,STK_REG(R31)(r1)
 911         addi    r1,r1,STACKFRAMESIZE
 912         ld      r0,16(r1)
 913         mtlr    r0
 914         blr
 915
 916 ht64_modify_pte:
 917         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 918         mr      r4,r3
 919         rlwinm  r3,r31,32-12,29,31
 920
 921         /* Secondary group ? if yes, get a inverted hash value */
 922         mr      r5,r28
 923         andi.   r0,r31,_PAGE_F_SECOND
 924         beq     1f
 925         not     r5,r5
 926 1:
 927         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 928         and     r0,r5,r27
 929         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 930         add     r3,r0,r3        /* add slot idx */
 931
 932         /* Call ppc_md.hpte_updatepp */
 933         mr      r5,r29                  /* vpn */
 934         li      r6,MMU_PAGE_64K
 935         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
 936         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
 937 _GLOBAL(ht64_call_hpte_updatepp)
 938         bl      .                       /* patched by htab_finish_init() */
 939
 940         /* if we failed because typically the HPTE wasn't really here
 941          * we try an insertion.
 942          */
 943         cmpdi   0,r3,-1
 944         beq-    ht64_insert_pte
 945
 946         /* Clear the BUSY bit and Write out the PTE */
 947         li      r0,_PAGE_BUSY
 948         andc    r30,r30,r0
 949         b       ht64_write_out_pte
 950
 951 ht64_wrong_access:
 952         /* Bail out clearing reservation */
 953         stdcx.  r31,0,r6
 954         li      r3,1
 955         b       ht64_bail
 956
 957 ht64_pte_insert_failure:
 958         /* Bail out restoring old PTE */
 959         ld      r6,STK_PARAM(R6)(r1)
 960         std     r31,0(r6)
 961         li      r3,-1
 962         b       ht64_bail
 963
 964
 965 #endif /* CONFIG_PPC_HAS_HASH_64K */
 966
 967
 968 /*****************************************************************************
 969  *                                                                           *
 970  *           Huge pages implementation is in hugetlbpage.c                   *
 971  *                                                                           *
 972  *****************************************************************************/