arch/arm/mm/mmu.c

   1 /*
   2  *  linux/arch/arm/mm/mmu.c
   3  *
   4  *  Copyright (C) 1995-2005 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/module.h>
  11 #include <linux/kernel.h>
  12 #include <linux/errno.h>
  13 #include <linux/init.h>
  14 #include <linux/mman.h>
  15 #include <linux/nodemask.h>
  16 #include <linux/memblock.h>
  17 #include <linux/sort.h>
  18 #include <linux/fs.h>
  19
  20 #include <asm/cputype.h>
  21 #include <asm/sections.h>
  22 #include <asm/cachetype.h>
  23 #include <asm/setup.h>
  24 #include <asm/sizes.h>
  25 #include <asm/smp_plat.h>
  26 #include <asm/tlb.h>
  27 #include <asm/highmem.h>
  28
  29 #include <asm/mach/arch.h>
  30 #include <asm/mach/map.h>
  31
  32 #include "mm.h"
  33
  34 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
  35 extern void s5p_reserve_bootmem(void);          /* Temporary code for Bootmem */
  36
  37 /*
  38  * empty_zero_page is a special page that is used for
  39  * zero-initialized data and COW.
  40  */
  41 struct page *empty_zero_page;
  42 EXPORT_SYMBOL(empty_zero_page);
  43
  44 /*
  45  * The pmd table for the upper-most set of pages.
  46  */
  47 pmd_t *top_pmd;
  48
  49 #define CPOLICY_UNCACHED        0
  50 #define CPOLICY_BUFFERED        1
  51 #define CPOLICY_WRITETHROUGH    2
  52 #define CPOLICY_WRITEBACK       3
  53 #define CPOLICY_WRITEALLOC      4
  54
  55 static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
  56 static unsigned int ecc_mask __initdata = 0;
  57 pgprot_t pgprot_user;
  58 pgprot_t pgprot_kernel;
  59
  60 EXPORT_SYMBOL(pgprot_user);
  61 EXPORT_SYMBOL(pgprot_kernel);
  62
  63 struct cachepolicy {
  64         const char      policy[16];
  65         unsigned int    cr_mask;
  66         unsigned int    pmd;
  67         unsigned int    pte;
  68 };
  69
  70 static struct cachepolicy cache_policies[] __initdata = {
  71         {
  72                 .policy         = "uncached",
  73                 .cr_mask        = CR_W|CR_C,
  74                 .pmd            = PMD_SECT_UNCACHED,
  75                 .pte            = L_PTE_MT_UNCACHED,
  76         }, {
  77                 .policy         = "buffered",
  78                 .cr_mask        = CR_C,
  79                 .pmd            = PMD_SECT_BUFFERED,
  80                 .pte            = L_PTE_MT_BUFFERABLE,
  81         }, {
  82                 .policy         = "writethrough",
  83                 .cr_mask        = 0,
  84                 .pmd            = PMD_SECT_WT,
  85                 .pte            = L_PTE_MT_WRITETHROUGH,
  86         }, {
  87                 .policy         = "writeback",
  88                 .cr_mask        = 0,
  89                 .pmd            = PMD_SECT_WB,
  90                 .pte            = L_PTE_MT_WRITEBACK,
  91         }, {
  92                 .policy         = "writealloc",
  93                 .cr_mask        = 0,
  94                 .pmd            = PMD_SECT_WBWA,
  95                 .pte            = L_PTE_MT_WRITEALLOC,
  96         }
  97 };
  98
  99 /*
 100  * These are useful for identifying cache coherency
 101  * problems by allowing the cache or the cache and
 102  * writebuffer to be turned off.  (Note: the write
 103  * buffer should not be on and the cache off).
 104  */
 105 static int __init early_cachepolicy(char *p)
 106 {
 107         int i;
 108
 109         for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
 110                 int len = strlen(cache_policies[i].policy);
 111
 112                 if (memcmp(p, cache_policies[i].policy, len) == 0) {
 113                         cachepolicy = i;
 114                         cr_alignment &= ~cache_policies[i].cr_mask;
 115                         cr_no_alignment &= ~cache_policies[i].cr_mask;
 116                         break;
 117                 }
 118         }
 119         if (i == ARRAY_SIZE(cache_policies))
 120                 printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n");
 121         /*
 122          * This restriction is partly to do with the way we boot; it is
 123          * unpredictable to have memory mapped using two different sets of
 124          * memory attributes (shared, type, and cache attribs).  We can not
 125          * change these attributes once the initial assembly has setup the
 126          * page tables.
 127          */
 128         if (cpu_architecture() >= CPU_ARCH_ARMv6) {
 129                 printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n");
 130                 cachepolicy = CPOLICY_WRITEBACK;
 131         }
 132         flush_cache_all();
 133         set_cr(cr_alignment);
 134         return 0;
 135 }
 136 early_param("cachepolicy", early_cachepolicy);
 137
 138 static int __init early_nocache(char *__unused)
 139 {
 140         char *p = "buffered";
 141         printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
 142         early_cachepolicy(p);
 143         return 0;
 144 }
 145 early_param("nocache", early_nocache);
 146
 147 static int __init early_nowrite(char *__unused)
 148 {
 149         char *p = "uncached";
 150         printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
 151         early_cachepolicy(p);
 152         return 0;
 153 }
 154 early_param("nowb", early_nowrite);
 155
 156 static int __init early_ecc(char *p)
 157 {
 158         if (memcmp(p, "on", 2) == 0)
 159                 ecc_mask = PMD_PROTECTION;
 160         else if (memcmp(p, "off", 3) == 0)
 161                 ecc_mask = 0;
 162         return 0;
 163 }
 164 early_param("ecc", early_ecc);
 165
 166 static int __init noalign_setup(char *__unused)
 167 {
 168         cr_alignment &= ~CR_A;
 169         cr_no_alignment &= ~CR_A;
 170         set_cr(cr_alignment);
 171         return 1;
 172 }
 173 __setup("noalign", noalign_setup);
 174
 175 #ifndef CONFIG_SMP
 176 void adjust_cr(unsigned long mask, unsigned long set)
 177 {
 178         unsigned long flags;
 179
 180         mask &= ~CR_A;
 181
 182         set &= mask;
 183
 184         local_irq_save(flags);
 185
 186         cr_no_alignment = (cr_no_alignment & ~mask) | set;
 187         cr_alignment = (cr_alignment & ~mask) | set;
 188
 189         set_cr((get_cr() & ~mask) | set);
 190
 191         local_irq_restore(flags);
 192 }
 193 #endif
 194
 195 #define PROT_PTE_DEVICE         L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_WRITE
 196 #define PROT_SECT_DEVICE        PMD_TYPE_SECT|PMD_SECT_AP_WRITE
 197
 198 static struct mem_type mem_types[] = {
 199         [MT_DEVICE] = {           /* Strongly ordered / ARMv6 shared device */
 200                 .prot_pte       = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
 201                                   L_PTE_SHARED,
 202                 .prot_l1        = PMD_TYPE_TABLE,
 203                 .prot_sect      = PROT_SECT_DEVICE | PMD_SECT_S,
 204                 .domain         = DOMAIN_IO,
 205         },
 206         [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
 207                 .prot_pte       = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
 208                 .prot_l1        = PMD_TYPE_TABLE,
 209                 .prot_sect      = PROT_SECT_DEVICE,
 210                 .domain         = DOMAIN_IO,
 211         },
 212         [MT_DEVICE_CACHED] = {    /* ioremap_cached */
 213                 .prot_pte       = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
 214                 .prot_l1        = PMD_TYPE_TABLE,
 215                 .prot_sect      = PROT_SECT_DEVICE | PMD_SECT_WB,
 216                 .domain         = DOMAIN_IO,
 217         },
 218         [MT_DEVICE_WC] = {      /* ioremap_wc */
 219                 .prot_pte       = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
 220                 .prot_l1        = PMD_TYPE_TABLE,
 221                 .prot_sect      = PROT_SECT_DEVICE,
 222                 .domain         = DOMAIN_IO,
 223         },
 224         [MT_UNCACHED] = {
 225                 .prot_pte       = PROT_PTE_DEVICE,
 226                 .prot_l1        = PMD_TYPE_TABLE,
 227                 .prot_sect      = PMD_TYPE_SECT | PMD_SECT_XN,
 228                 .domain         = DOMAIN_IO,
 229         },
 230         [MT_CACHECLEAN] = {
 231                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
 232                 .domain    = DOMAIN_KERNEL,
 233         },
 234         [MT_MINICLEAN] = {
 235                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
 236                 .domain    = DOMAIN_KERNEL,
 237         },
 238         [MT_LOW_VECTORS] = {
 239                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 240                                 L_PTE_EXEC,
 241                 .prot_l1   = PMD_TYPE_TABLE,
 242                 .domain    = DOMAIN_USER,
 243         },
 244         [MT_HIGH_VECTORS] = {
 245                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 246                                 L_PTE_USER | L_PTE_EXEC,
 247                 .prot_l1   = PMD_TYPE_TABLE,
 248                 .domain    = DOMAIN_USER,
 249         },
 250         [MT_MEMORY] = {
 251                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 252                                 L_PTE_WRITE | L_PTE_EXEC,
 253                 .prot_l1   = PMD_TYPE_TABLE,
 254                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 255                 .domain    = DOMAIN_KERNEL,
 256         },
 257         [MT_ROM] = {
 258                 .prot_sect = PMD_TYPE_SECT,
 259                 .domain    = DOMAIN_KERNEL,
 260         },
 261         [MT_MEMORY_NONCACHED] = {
 262                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 263                                 L_PTE_WRITE | L_PTE_EXEC | L_PTE_MT_BUFFERABLE,
 264                 .prot_l1   = PMD_TYPE_TABLE,
 265                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 266                 .domain    = DOMAIN_KERNEL,
 267         },
 268         [MT_MEMORY_DTCM] = {
 269                 .prot_pte       = L_PTE_PRESENT | L_PTE_YOUNG |
 270                                   L_PTE_DIRTY | L_PTE_WRITE,
 271                 .prot_l1        = PMD_TYPE_TABLE,
 272                 .prot_sect      = PMD_TYPE_SECT | PMD_SECT_XN,
 273                 .domain         = DOMAIN_KERNEL,
 274         },
 275         [MT_MEMORY_ITCM] = {
 276                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 277                                 L_PTE_USER | L_PTE_EXEC,
 278                 .prot_l1   = PMD_TYPE_TABLE,
 279                 .domain    = DOMAIN_IO,
 280         },
 281 };
 282
 283 const struct mem_type *get_mem_type(unsigned int type)
 284 {
 285         return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
 286 }
 287 EXPORT_SYMBOL(get_mem_type);
 288
 289 /*
 290  * Adjust the PMD section entries according to the CPU in use.
 291  */
 292 static void __init build_mem_type_table(void)
 293 {
 294         struct cachepolicy *cp;
 295         unsigned int cr = get_cr();
 296         unsigned int user_pgprot, kern_pgprot, vecs_pgprot;
 297         int cpu_arch = cpu_architecture();
 298         int i;
 299
 300         if (cpu_arch < CPU_ARCH_ARMv6) {
 301 #if defined(CONFIG_CPU_DCACHE_DISABLE)
 302                 if (cachepolicy > CPOLICY_BUFFERED)
 303                         cachepolicy = CPOLICY_BUFFERED;
 304 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
 305                 if (cachepolicy > CPOLICY_WRITETHROUGH)
 306                         cachepolicy = CPOLICY_WRITETHROUGH;
 307 #endif
 308         }
 309         if (cpu_arch < CPU_ARCH_ARMv5) {
 310                 if (cachepolicy >= CPOLICY_WRITEALLOC)
 311                         cachepolicy = CPOLICY_WRITEBACK;
 312                 ecc_mask = 0;
 313         }
 314 #ifdef CONFIG_SMP
 315         cachepolicy = CPOLICY_WRITEALLOC;
 316 #endif
 317
 318         /*
 319          * Strip out features not present on earlier architectures.
 320          * Pre-ARMv5 CPUs don't have TEX bits.  Pre-ARMv6 CPUs or those
 321          * without extended page tables don't have the 'Shared' bit.
 322          */
 323         if (cpu_arch < CPU_ARCH_ARMv5)
 324                 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 325                         mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
 326         if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
 327                 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 328                         mem_types[i].prot_sect &= ~PMD_SECT_S;
 329
 330         /*
 331          * ARMv5 and lower, bit 4 must be set for page tables (was: cache
 332          * "update-able on write" bit on ARM610).  However, Xscale and
 333          * Xscale3 require this bit to be cleared.
 334          */
 335         if (cpu_is_xscale() || cpu_is_xsc3()) {
 336                 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 337                         mem_types[i].prot_sect &= ~PMD_BIT4;
 338                         mem_types[i].prot_l1 &= ~PMD_BIT4;
 339                 }
 340         } else if (cpu_arch < CPU_ARCH_ARMv6) {
 341                 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 342                         if (mem_types[i].prot_l1)
 343                                 mem_types[i].prot_l1 |= PMD_BIT4;
 344                         if (mem_types[i].prot_sect)
 345                                 mem_types[i].prot_sect |= PMD_BIT4;
 346                 }
 347         }
 348
 349         /*
 350          * Mark the device areas according to the CPU/architecture.
 351          */
 352         if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
 353                 if (!cpu_is_xsc3()) {
 354                         /*
 355                          * Mark device regions on ARMv6+ as execute-never
 356                          * to prevent speculative instruction fetches.
 357                          */
 358                         mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
 359                         mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
 360                         mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
 361                         mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
 362                 }
 363                 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
 364                         /*
 365                          * For ARMv7 with TEX remapping,
 366                          * - shared device is SXCB=1100
 367                          * - nonshared device is SXCB=0100
 368                          * - write combine device mem is SXCB=0001
 369                          * (Uncached Normal memory)
 370                          */
 371                         mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
 372                         mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
 373                         mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
 374                 } else if (cpu_is_xsc3()) {
 375                         /*
 376                          * For Xscale3,
 377                          * - shared device is TEXCB=00101
 378                          * - nonshared device is TEXCB=01000
 379                          * - write combine device mem is TEXCB=00100
 380                          * (Inner/Outer Uncacheable in xsc3 parlance)
 381                          */
 382                         mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
 383                         mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
 384                         mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
 385                 } else {
 386                         /*
 387                          * For ARMv6 and ARMv7 without TEX remapping,
 388                          * - shared device is TEXCB=00001
 389                          * - nonshared device is TEXCB=01000
 390                          * - write combine device mem is TEXCB=00100
 391                          * (Uncached Normal in ARMv6 parlance).
 392                          */
 393                         mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
 394                         mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
 395                         mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
 396                 }
 397         } else {
 398                 /*
 399                  * On others, write combining is "Uncached/Buffered"
 400                  */
 401                 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
 402         }
 403
 404         /*
 405          * Now deal with the memory-type mappings
 406          */
 407         cp = &cache_policies[cachepolicy];
 408         vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
 409
 410 #ifndef CONFIG_SMP
 411         /*
 412          * Only use write-through for non-SMP systems
 413          */
 414         if (cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH)
 415                 vecs_pgprot = cache_policies[CPOLICY_WRITETHROUGH].pte;
 416 #endif
 417
 418         /*
 419          * Enable CPU-specific coherency if supported.
 420          * (Only available on XSC3 at the moment.)
 421          */
 422         if (arch_is_coherent() && cpu_is_xsc3()) {
 423                 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
 424                 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
 425                 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
 426                 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
 427         }
 428         /*
 429          * ARMv6 and above have extended page tables.
 430          */
 431         if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
 432                 /*
 433                  * Mark cache clean areas and XIP ROM read only
 434                  * from SVC mode and no access from userspace.
 435                  */
 436                 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 437                 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 438                 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 439
 440 #ifdef CONFIG_SMP
 441                 /*
 442                  * Mark memory with the "shared" attribute for SMP systems
 443                  */
 444                 user_pgprot |= L_PTE_SHARED;
 445                 kern_pgprot |= L_PTE_SHARED;
 446                 vecs_pgprot |= L_PTE_SHARED;
 447                 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
 448                 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
 449                 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
 450                 mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
 451                 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
 452                 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
 453                 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
 454                 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
 455 #endif
 456         }
 457
 458         /*
 459          * Non-cacheable Normal - intended for memory areas that must
 460          * not cause dirty cache line writebacks when used
 461          */
 462         if (cpu_arch >= CPU_ARCH_ARMv6) {
 463                 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
 464                         /* Non-cacheable Normal is XCB = 001 */
 465                         mem_types[MT_MEMORY_NONCACHED].prot_sect |=
 466                                 PMD_SECT_BUFFERED;
 467                 } else {
 468                         /* For both ARMv6 and non-TEX-remapping ARMv7 */
 469                         mem_types[MT_MEMORY_NONCACHED].prot_sect |=
 470                                 PMD_SECT_TEX(1);
 471                 }
 472         } else {
 473                 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
 474         }
 475
 476         for (i = 0; i < 16; i++) {
 477                 unsigned long v = pgprot_val(protection_map[i]);
 478                 protection_map[i] = __pgprot(v | user_pgprot);
 479         }
 480
 481         mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
 482         mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
 483
 484         pgprot_user   = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
 485         pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
 486                                  L_PTE_DIRTY | L_PTE_WRITE | kern_pgprot);
 487
 488         mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
 489         mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
 490         mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
 491         mem_types[MT_MEMORY].prot_pte |= kern_pgprot;
 492         mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask;
 493         mem_types[MT_ROM].prot_sect |= cp->pmd;
 494
 495         switch (cp->pmd) {
 496         case PMD_SECT_WT:
 497                 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
 498                 break;
 499         case PMD_SECT_WB:
 500         case PMD_SECT_WBWA:
 501                 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
 502                 break;
 503         }
 504         printk("Memory policy: ECC %sabled, Data cache %s\n",
 505                 ecc_mask ? "en" : "dis", cp->policy);
 506
 507         for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 508                 struct mem_type *t = &mem_types[i];
 509                 if (t->prot_l1)
 510                         t->prot_l1 |= PMD_DOMAIN(t->domain);
 511                 if (t->prot_sect)
 512                         t->prot_sect |= PMD_DOMAIN(t->domain);
 513         }
 514 }
 515
 516 #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
 517 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 518                               unsigned long size, pgprot_t vma_prot)
 519 {
 520         if (!pfn_valid(pfn))
 521                 return pgprot_noncached(vma_prot);
 522         else if (file->f_flags & O_SYNC)
 523                 return pgprot_writecombine(vma_prot);
 524         return vma_prot;
 525 }
 526 EXPORT_SYMBOL(phys_mem_access_prot);
 527 #endif
 528
 529 #define vectors_base()  (vectors_high() ? 0xffff0000 : 0)
 530
 531 static void __init *early_alloc(unsigned long sz)
 532 {
 533         void *ptr = __va(memblock_alloc(sz, sz));
 534         memset(ptr, 0, sz);
 535         return ptr;
 536 }
 537
 538 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
 539 {
 540         if (pmd_none(*pmd)) {
 541                 pte_t *pte = early_alloc(2 * PTRS_PER_PTE * sizeof(pte_t));
 542                 __pmd_populate(pmd, __pa(pte) | prot);
 543         }
 544         BUG_ON(pmd_bad(*pmd));
 545         return pte_offset_kernel(pmd, addr);
 546 }
 547
 548 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
 549                                   unsigned long end, unsigned long pfn,
 550                                   const struct mem_type *type)
 551 {
 552         pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
 553         do {
 554                 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
 555                 pfn++;
 556         } while (pte++, addr += PAGE_SIZE, addr != end);
 557 }
 558
 559 static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
 560                                       unsigned long end, unsigned long phys,
 561                                       const struct mem_type *type)
 562 {
 563         pmd_t *pmd = pmd_offset(pgd, addr);
 564
 565         /*
 566          * Try a section mapping - end, addr and phys must all be aligned
 567          * to a section boundary.  Note that PMDs refer to the individual
 568          * L1 entries, whereas PGDs refer to a group of L1 entries making
 569          * up one logical pointer to an L2 table.
 570          */
 571         if (((addr | end | phys) & ~SECTION_MASK) == 0) {
 572                 pmd_t *p = pmd;
 573
 574                 if (addr & SECTION_SIZE)
 575                         pmd++;
 576
 577                 do {
 578                         *pmd = __pmd(phys | type->prot_sect);
 579                         phys += SECTION_SIZE;
 580                 } while (pmd++, addr += SECTION_SIZE, addr != end);
 581
 582                 flush_pmd_entry(p);
 583         } else {
 584                 /*
 585                  * No need to loop; pte's aren't interested in the
 586                  * individual L1 entries.
 587                  */
 588                 alloc_init_pte(pmd, addr, end, __phys_to_pfn(phys), type);
 589         }
 590 }
 591
 592 static void __init create_36bit_mapping(struct map_desc *md,
 593                                         const struct mem_type *type)
 594 {
 595         unsigned long phys, addr, length, end;
 596         pgd_t *pgd;
 597
 598         addr = md->virtual;
 599         phys = (unsigned long)__pfn_to_phys(md->pfn);
 600         length = PAGE_ALIGN(md->length);
 601
 602         if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
 603                 printk(KERN_ERR "MM: CPU does not support supersection "
 604                        "mapping for 0x%08llx at 0x%08lx\n",
 605                        __pfn_to_phys((u64)md->pfn), addr);
 606                 return;
 607         }
 608
 609         /* N.B. ARMv6 supersections are only defined to work with domain 0.
 610          *      Since domain assignments can in fact be arbitrary, the
 611          *      'domain == 0' check below is required to insure that ARMv6
 612          *      supersections are only allocated for domain 0 regardless
 613          *      of the actual domain assignments in use.
 614          */
 615         if (type->domain) {
 616                 printk(KERN_ERR "MM: invalid domain in supersection "
 617                        "mapping for 0x%08llx at 0x%08lx\n",
 618                        __pfn_to_phys((u64)md->pfn), addr);
 619                 return;
 620         }
 621
 622         if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
 623                 printk(KERN_ERR "MM: cannot create mapping for "
 624                        "0x%08llx at 0x%08lx invalid alignment\n",
 625                        __pfn_to_phys((u64)md->pfn), addr);
 626                 return;
 627         }
 628
 629         /*
 630          * Shift bits [35:32] of address into bits [23:20] of PMD
 631          * (See ARMv6 spec).
 632          */
 633         phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
 634
 635         pgd = pgd_offset_k(addr);
 636         end = addr + length;
 637         do {
 638                 pmd_t *pmd = pmd_offset(pgd, addr);
 639                 int i;
 640
 641                 for (i = 0; i < 16; i++)
 642                         *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
 643
 644                 addr += SUPERSECTION_SIZE;
 645                 phys += SUPERSECTION_SIZE;
 646                 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
 647         } while (addr != end);
 648 }
 649
 650 /*
 651  * Create the page directory entries and any necessary
 652  * page tables for the mapping specified by `md'.  We
 653  * are able to cope here with varying sizes and address
 654  * offsets, and we take full advantage of sections and
 655  * supersections.
 656  */
 657 static void __init create_mapping(struct map_desc *md)
 658 {
 659         unsigned long phys, addr, length, end;
 660         const struct mem_type *type;
 661         pgd_t *pgd;
 662
 663         if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
 664                 printk(KERN_WARNING "BUG: not creating mapping for "
 665                        "0x%08llx at 0x%08lx in user region\n",
 666                        __pfn_to_phys((u64)md->pfn), md->virtual);
 667                 return;
 668         }
 669
 670         if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
 671             md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) {
 672                 printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx "
 673                        "overlaps vmalloc space\n",
 674                        __pfn_to_phys((u64)md->pfn), md->virtual);
 675         }
 676
 677         type = &mem_types[md->type];
 678
 679         /*
 680          * Catch 36-bit addresses
 681          */
 682         if (md->pfn >= 0x100000) {
 683                 create_36bit_mapping(md, type);
 684                 return;
 685         }
 686
 687         addr = md->virtual & PAGE_MASK;
 688         phys = (unsigned long)__pfn_to_phys(md->pfn);
 689         length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
 690
 691         if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
 692                 printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not "
 693                        "be mapped using pages, ignoring.\n",
 694                        __pfn_to_phys(md->pfn), addr);
 695                 return;
 696         }
 697
 698         pgd = pgd_offset_k(addr);
 699         end = addr + length;
 700         do {
 701                 unsigned long next = pgd_addr_end(addr, end);
 702
 703                 alloc_init_section(pgd, addr, next, phys, type);
 704
 705                 phys += next - addr;
 706                 addr = next;
 707         } while (pgd++, addr != end);
 708 }
 709
 710 /*
 711  * Create the architecture specific mappings
 712  */
 713 void __init iotable_init(struct map_desc *io_desc, int nr)
 714 {
 715         int i;
 716
 717         for (i = 0; i < nr; i++)
 718                 create_mapping(io_desc + i);
 719 }
 720
 721 static void * __initdata vmalloc_min = (void *)(VMALLOC_END - SZ_128M);
 722
 723 /*
 724  * vmalloc=size forces the vmalloc area to be exactly 'size'
 725  * bytes. This can be used to increase (or decrease) the vmalloc
 726  * area - the default is 128m.
 727  */
 728 static int __init early_vmalloc(char *arg)
 729 {
 730         unsigned long vmalloc_reserve = memparse(arg, NULL);
 731
 732         if (vmalloc_reserve < SZ_16M) {
 733                 vmalloc_reserve = SZ_16M;
 734                 printk(KERN_WARNING
 735                         "vmalloc area too small, limiting to %luMB\n",
 736                         vmalloc_reserve >> 20);
 737         }
 738
 739         if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
 740                 vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
 741                 printk(KERN_WARNING
 742                         "vmalloc area is too big, limiting to %luMB\n",
 743                         vmalloc_reserve >> 20);
 744         }
 745
 746         vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
 747         return 0;
 748 }
 749 early_param("vmalloc", early_vmalloc);
 750
 751 phys_addr_t lowmem_end_addr;
 752
 753 static void __init sanity_check_meminfo(void)
 754 {
 755         int i, j, highmem = 0;
 756
 757
 758         for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
 759                 struct membank *bank = &meminfo.bank[j];
 760                 *bank = meminfo.bank[i];
 761
 762 #ifdef CONFIG_HIGHMEM
 763                 if (__va(bank->start) > vmalloc_min ||
 764                     __va(bank->start) < (void *)PAGE_OFFSET)
 765                         highmem = 1;
 766
 767                 bank->highmem = highmem;
 768
 769                 /*
 770                  * Split those memory banks which are partially overlapping
 771                  * the vmalloc area greatly simplifying things later.
 772                  */
 773                 if (__va(bank->start) < vmalloc_min &&
 774                     bank->size > vmalloc_min - __va(bank->start)) {
 775                         if (meminfo.nr_banks >= NR_BANKS) {
 776                                 printk(KERN_CRIT "NR_BANKS too low, "
 777                                                  "ignoring high memory\n");
 778                         } else {
 779                                 memmove(bank + 1, bank,
 780                                         (meminfo.nr_banks - i) * sizeof(*bank));
 781                                 meminfo.nr_banks++;
 782                                 i++;
 783                                 bank[1].size -= vmalloc_min - __va(bank->start);
 784                                 bank[1].start = __pa(vmalloc_min - 1) + 1;
 785                                 bank[1].highmem = highmem = 1;
 786                                 j++;
 787                         }
 788                         bank->size = vmalloc_min - __va(bank->start);
 789                 }
 790 #else
 791                 bank->highmem = highmem;
 792
 793                 /*
 794                  * Check whether this memory bank would entirely overlap
 795                  * the vmalloc area.
 796                  */
 797                 if (__va(bank->start) >= vmalloc_min ||
 798                     __va(bank->start) < (void *)PAGE_OFFSET) {
 799                         printk(KERN_NOTICE "Ignoring RAM at %.8lx-%.8lx "
 800                                "(vmalloc region overlap).\n",
 801                                bank->start, bank->start + bank->size - 1);
 802                         continue;
 803                 }
 804
 805                 /*
 806                  * Check whether this memory bank would partially overlap
 807                  * the vmalloc area.
 808                  */
 809                 if (__va(bank->start + bank->size) > vmalloc_min ||
 810                     __va(bank->start + bank->size) < __va(bank->start)) {
 811                         unsigned long newsize = vmalloc_min - __va(bank->start);
 812                         printk(KERN_NOTICE "Truncating RAM at %.8lx-%.8lx "
 813                                "to -%.8lx (vmalloc region overlap).\n",
 814                                bank->start, bank->start + bank->size - 1,
 815                                bank->start + newsize - 1);
 816                         bank->size = newsize;
 817                 }
 818 #endif
 819                 j++;
 820         }
 821 #ifdef CONFIG_HIGHMEM
 822         if (highmem) {
 823                 const char *reason = NULL;
 824
 825                 if (cache_is_vipt_aliasing()) {
 826                         /*
 827                          * Interactions between kmap and other mappings
 828                          * make highmem support with aliasing VIPT caches
 829                          * rather difficult.
 830                          */
 831                         reason = "with VIPT aliasing cache";
 832 #ifdef CONFIG_SMP
 833                 } else if (tlb_ops_need_broadcast()) {
 834                         /*
 835                          * kmap_high needs to occasionally flush TLB entries,
 836                          * however, if the TLB entries need to be broadcast
 837                          * we may deadlock:
 838                          *  kmap_high(irqs off)->flush_all_zero_pkmaps->
 839                          *  flush_tlb_kernel_range->smp_call_function_many
 840                          *   (must not be called with irqs off)
 841                          */
 842                         reason = "without hardware TLB ops broadcasting";
 843 #endif
 844                 }
 845                 if (reason) {
 846                         printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
 847                                 reason);
 848                         while (j > 0 && meminfo.bank[j - 1].highmem)
 849                                 j--;
 850                 }
 851         }
 852 #endif
 853         meminfo.nr_banks = j;
 854 }
 855
 856 static inline void prepare_page_table(void)
 857 {
 858         unsigned long addr;
 859
 860         /*
 861          * Clear out all the mappings below the kernel image.
 862          */
 863         for (addr = 0; addr < MODULES_VADDR; addr += PGDIR_SIZE)
 864                 pmd_clear(pmd_off_k(addr));
 865
 866 #ifdef CONFIG_XIP_KERNEL
 867         /* The XIP kernel is mapped in the module area -- skip over it */
 868         addr = ((unsigned long)_etext + PGDIR_SIZE - 1) & PGDIR_MASK;
 869 #endif
 870         for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE)
 871                 pmd_clear(pmd_off_k(addr));
 872
 873         /*
 874          * Clear out all the kernel space mappings, except for the first
 875          * memory bank, up to the end of the vmalloc region.
 876          */
 877         for (addr = __phys_to_virt(bank_phys_end(&meminfo.bank[0]));
 878              addr < VMALLOC_END; addr += PGDIR_SIZE)
 879                 pmd_clear(pmd_off_k(addr));
 880 }
 881
 882 /*
 883  * Reserve the special regions of memory
 884  */
 885 void __init arm_mm_memblock_reserve(void)
 886 {
 887         /*
 888          * Reserve the page tables.  These are already in use,
 889          * and can only be in node 0.
 890          */
 891         memblock_reserve(__pa(swapper_pg_dir), PTRS_PER_PGD * sizeof(pgd_t));
 892
 893 #ifdef CONFIG_SA1111
 894         /*
 895          * Because of the SA1111 DMA bug, we want to preserve our
 896          * precious DMA-able memory...
 897          */
 898         memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
 899 #endif
 900         lowmem_end_addr = __pa(vmalloc_min - 1) + 1;
 901 }
 902
 903 /*
 904  * Set up device the mappings.  Since we clear out the page tables for all
 905  * mappings above VMALLOC_END, we will remove any debug device mappings.
 906  * This means you have to be careful how you debug this function, or any
 907  * called function.  This means you can't use any function or debugging
 908  * method which may touch any device, otherwise the kernel _will_ crash.
 909  */
 910 static void __init devicemaps_init(struct machine_desc *mdesc)
 911 {
 912         struct map_desc map;
 913         unsigned long addr;
 914         void *vectors;
 915
 916         /*
 917          * Allocate the vector page early.
 918          */
 919         vectors = early_alloc(PAGE_SIZE);
 920
 921         for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE)
 922                 pmd_clear(pmd_off_k(addr));
 923
 924         /*
 925          * Map the kernel if it is XIP.
 926          * It is always first in the modulearea.
 927          */
 928 #ifdef CONFIG_XIP_KERNEL
 929         map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
 930         map.virtual = MODULES_VADDR;
 931         map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
 932         map.type = MT_ROM;
 933         create_mapping(&map);
 934 #endif
 935
 936         /*
 937          * Map the cache flushing regions.
 938          */
 939 #ifdef FLUSH_BASE
 940         map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
 941         map.virtual = FLUSH_BASE;
 942         map.length = SZ_1M;
 943         map.type = MT_CACHECLEAN;
 944         create_mapping(&map);
 945 #endif
 946 #ifdef FLUSH_BASE_MINICACHE
 947         map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
 948         map.virtual = FLUSH_BASE_MINICACHE;
 949         map.length = SZ_1M;
 950         map.type = MT_MINICLEAN;
 951         create_mapping(&map);
 952 #endif
 953
 954         /*
 955          * Create a mapping for the machine vectors at the high-vectors
 956          * location (0xffff0000).  If we aren't using high-vectors, also
 957          * create a mapping at the low-vectors virtual address.
 958          */
 959         map.pfn = __phys_to_pfn(virt_to_phys(vectors));
 960         map.virtual = 0xffff0000;
 961         map.length = PAGE_SIZE;
 962         map.type = MT_HIGH_VECTORS;
 963         create_mapping(&map);
 964
 965         if (!vectors_high()) {
 966                 map.virtual = 0;
 967                 map.type = MT_LOW_VECTORS;
 968                 create_mapping(&map);
 969         }
 970
 971         /*
 972          * Ask the machine support to map in the statically mapped devices.
 973          */
 974         if (mdesc->map_io)
 975                 mdesc->map_io();
 976
 977         /*
 978          * Finally flush the caches and tlb to ensure that we're in a
 979          * consistent state wrt the writebuffer.  This also ensures that
 980          * any write-allocated cache lines in the vector page are written
 981          * back.  After this point, we can start to touch devices again.
 982          */
 983         local_flush_tlb_all();
 984         flush_cache_all();
 985 }
 986
 987 static void __init kmap_init(void)
 988 {
 989 #ifdef CONFIG_HIGHMEM
 990         pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
 991                 PKMAP_BASE, _PAGE_KERNEL_TABLE);
 992 #endif
 993 }
 994
 995 static inline void map_memory_bank(struct membank *bank)
 996 {
 997         struct map_desc map;
 998
 999         map.pfn = bank_pfn_start(bank);
1000         map.virtual = __phys_to_virt(bank_phys_start(bank));
1001         map.length = bank_phys_size(bank);
1002         map.type = MT_MEMORY;
1003
1004         create_mapping(&map);
1005 }
1006
1007 static void __init map_lowmem(void)
1008 {
1009         struct meminfo *mi = &meminfo;
1010         int i;
1011
1012         /* Map all the lowmem memory banks. */
1013         for (i = 0; i < mi->nr_banks; i++) {
1014                 struct membank *bank = &mi->bank[i];
1015
1016                 if (!bank->highmem)
1017                         map_memory_bank(bank);
1018         }
1019 }
1020
1021 static int __init meminfo_cmp(const void *_a, const void *_b)
1022 {
1023         const struct membank *a = _a, *b = _b;
1024         long cmp = bank_pfn_start(a) - bank_pfn_start(b);
1025         return cmp < 0 ? -1 : cmp > 0 ? 1 : 0;
1026 }
1027
1028 /*
1029  * paging_init() sets up the page tables, initialises the zone memory
1030  * maps, and sets up the zero page, bad page and bad page tables.
1031  */
1032 void __init paging_init(struct machine_desc *mdesc)
1033 {
1034         void *zero_page;
1035
1036         sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL);
1037
1038         build_mem_type_table();
1039         sanity_check_meminfo();
1040         prepare_page_table();
1041         map_lowmem();
1042         devicemaps_init(mdesc);
1043         kmap_init();
1044
1045         top_pmd = pmd_off_k(0xffff0000);
1046
1047         /* allocate the zero page. */
1048         zero_page = early_alloc(PAGE_SIZE);
1049
1050         bootmem_init();
1051
1052         empty_zero_page = virt_to_page(zero_page);
1053         __flush_dcache_page(NULL, empty_zero_page);
1054 }
1055
1056 /*
1057  * In order to soft-boot, we need to insert a 1:1 mapping in place of
1058  * the user-mode pages.  This will then ensure that we have predictable
1059  * results when turning the mmu off
1060  */
1061 void setup_mm_for_reboot(char mode)
1062 {
1063         unsigned long base_pmdval;
1064         pgd_t *pgd;
1065         int i;
1066
1067         /*
1068          * We need to access to user-mode page tables here. For kernel threads
1069          * we don't have any user-mode mappings so we use the context that we
1070          * "borrowed".
1071          */
1072         pgd = current->active_mm->pgd;
1073
1074         base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT;
1075         if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
1076                 base_pmdval |= PMD_BIT4;
1077
1078         for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++, pgd++) {
1079                 unsigned long pmdval = (i << PGDIR_SHIFT) | base_pmdval;
1080                 pmd_t *pmd;
1081
1082                 pmd = pmd_off(pgd, i << PGDIR_SHIFT);
1083                 pmd[0] = __pmd(pmdval);
1084                 pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1)));
1085                 flush_pmd_entry(pmd);
1086         }
1087
1088         local_flush_tlb_all();
1089 }