emgd/gmm/gtt.c

   1 /*
   2  *-----------------------------------------------------------------------------
   3  * Filename: gtt.c
   4  * $Revision: 1.17 $
   5  *-----------------------------------------------------------------------------
   6  * Copyright (c) 2002-2010, Intel Corporation.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a copy
   9  * of this software and associated documentation files (the "Software"), to deal
  10  * in the Software without restriction, including without limitation the rights
  11  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12  * copies of the Software, and to permit persons to whom the Software is
  13  * furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice shall be included in
  16  * all copies or substantial portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24  * THE SOFTWARE.
  25  *
  26  *-----------------------------------------------------------------------------
  27  * Description:
  28  *  Manage the GTT.
  29  *-----------------------------------------------------------------------------
  30  */
  31 #include <igd_debug.h>
  32 #include <drmP.h>
  33 #include <memlist.h>
  34 #include <io.h>
  35 #include <memory.h>
  36 #include <asm/cacheflush.h>
  37 #include <linux/version.h>
  38
  39 #define PFX "EMGD: "
  40
  41 #define SCR1    0x71410 /* scratch register set by vbios indicating status*/
  42 #define SCR2    0x71418 /* scratch register set by vbios indicating amount of stolen memory */
  43 #define FW_ID   0xE1DF0000 /* firmware identifier */
  44 #define ST_BIT  0x00000004 /* bit2- stolen memory bit */
  45 #define PSB_PTE_VALID  0x0001
  46
  47 void emgd_free_pages(gmm_mem_buffer_t *mem);
  48
  49 static DEFINE_MUTEX(client_sem);
  50 static DEFINE_MUTEX(gtt_sem);
  51
  52 struct client_list_struct {
  53         struct list_head list;
  54         struct vm_area_struct *vma;
  55         pid_t pid;
  56 };
  57
  58 static LIST_HEAD(client_list);
  59
  60 static void ipi_handler(void *null) {
  61         //flush_agp_cache(); /* from agp.h */
  62         wbinvd();
  63 }
  64
  65 static void emgd_cache_flush(void) {
  66         if (on_each_cpu(ipi_handler, NULL, 1) != 0)
  67                 panic(PFX "timed out waiting for the other CPUs!\n");
  68 }
  69
  70 static void tlb_flush(void) {
  71         /* If needed, this would flush the SGX/MSVDX mmu TLB's */
  72 }
  73
  74 /* FIXME - TURNS OUT THAT THIS FUNCTION IS CURRENTLY A NOOP, BECAUSE
  75  * client_list NEVER HAS ANYTHING ADDED TO IT.
  76  */
  77 static void invalidate_vma(unsigned long pg_offset, unsigned long bus_addr) {
  78         int zap;
  79         struct list_head *tmp;
  80         struct client_list_struct *entry;
  81         unsigned long addr_start=0;
  82         unsigned long addr_end=0;
  83         unsigned long addr_offset=0;
  84         unsigned long vaddr;
  85         pgd_t *pgd;
  86         pud_t *pud;
  87         pmd_t *pmd;
  88         pte_t *pte;
  89
  90         mutex_lock(&client_sem);
  91         list_for_each(tmp, &client_list) {
  92                 entry = list_entry(tmp, struct client_list_struct, list);
  93
  94                 /*
  95                  * We need to handle invalidating VMA's that are only mapping
  96                  * a portion of the virtual aperture.  Calculate what if
  97                  * any invalidated pages need to be zapped
  98                  */
  99                 addr_start = (entry->vma->vm_pgoff << PAGE_SHIFT)
 100                         - bus_addr;
 101                 addr_end = addr_start + (entry->vma->vm_end - entry->vma->vm_start);
 102                 addr_offset = pg_offset << PAGE_SHIFT;
 103
 104                 vaddr = entry->vma->vm_start + (addr_offset - addr_start);
 105
 106                 zap=0;
 107                 pgd=NULL;
 108                 pud=NULL;
 109                 pmd=NULL;
 110                 pte=NULL;
 111
 112                 /*
 113                  * Look up page table entries for all VMAs that currently
 114                  * have the virtual aperture mapped -- to see if the page
 115                  * has ever faulted
 116                  */
 117                 pgd = pgd_offset(entry->vma->vm_mm, vaddr);
 118                 if (!pgd_none(*pgd)) {
 119                         pud = pud_offset(pgd, vaddr);
 120                         if (!pud_none(*pud)) {
 121                                 pmd = pmd_offset(pud, vaddr);
 122                                 if (!pmd_none(*pmd)) {
 123                                         pte = pte_offset_map(pmd, vaddr);
 124                                         if (!pte_none(*pte)) {
 125                                                 zap=1;
 126                                         }
 127                                 }
 128                         }
 129                 }
 130                 /*
 131                  * Only zap a page if it falls within the mapped region
 132                  * and it has previously faulted
 133                  */
 134                 if (zap && (addr_offset >= addr_start) &&
 135                                 (addr_offset < addr_end)) {
 136
 137
 138                         if (!page_mapcount(pte_page(*pte))) {
 139                                 printk(KERN_ERR "[EMGD] ERROR No mapcount\n");
 140                                 printk(KERN_ALERT "[EMGD] ZR %p %08lX %d %d %p\n",
 141                                                 pte_page(*pte),
 142                                                 pte_page(*pte)->flags, page_count(pte_page(*pte)),
 143                                                 page_mapcount(pte_page(*pte)), pte_page(*pte)->mapping);
 144                         } else {
 145                                 atomic_add_negative(-1, &pte_page(*pte)->_mapcount);
 146                                 put_page(pte_page(*pte));
 147 #if  (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34))
 148                                 dec_mm_counter(entry->vma->vm_mm, file_rss);
 149 #else
 150                                 dec_mm_counter(entry->vma->vm_mm, MM_FILEPAGES);
 151 #endif
 152                         }
 153
 154                         pte_clear(entry->vma->vm_mm, vaddr, pte);
 155                 }
 156
 157                 if(pte) {
 158                         pte_unmap(pte);
 159                 }
 160         }
 161         mutex_unlock(&client_sem);
 162 }
 163
 164
 165
 166 /*
 167  * Allocate pages from the kernel and store in a page list.
 168  */
 169 gmm_mem_buffer_t *emgd_alloc_pages(unsigned long num_pages, int type) {
 170         gmm_mem_buffer_t *mem;
 171         size_t list_size;
 172         struct page *page;
 173         int i;
 174         int order;
 175
 176         mem = (gmm_mem_buffer_t *)kzalloc(sizeof(gmm_mem_buffer_t), GFP_KERNEL);
 177         if (mem == NULL) {
 178                 return NULL;
 179         }
 180
 181         /* First allocate page array */
 182         list_size = num_pages * sizeof(struct page *);
 183         mem->vmalloc_flag = false;
 184
 185         if (list_size <= (2 * PAGE_SIZE)) {
 186                 mem->pages = kmalloc(list_size, GFP_KERNEL | __GFP_NORETRY);
 187         }
 188
 189         if (mem->pages == NULL) {
 190                 mem->pages = vmalloc(list_size);
 191                 mem->vmalloc_flag = true;
 192         }
 193
 194         if (mem->pages == NULL) {
 195                 printk(KERN_ERR "[EMGD] Failed to allocate memory info struct.\n");
 196                 kfree(mem);
 197                 return NULL;
 198         }
 199
 200         /*
 201          * If we need phyical contiguous memory, then do things differently.
 202          *   Call alloc_pages(GFP_KERNEL, pages) to allocate all the pages.
 203          *   The page structure returned is just the first page of the group.
 204          *   ? is it a virtual address ?
 205          *
 206          *   mem->pages[0] = virt_to_phys(page)
 207          *   mem->pages[1] = mem->pages[0] + PAGE_SIZE
 208          */
 209
 210         if ((type == 1) || (type == 0)) {
 211         /* Next allocate the pages */
 212                 for (i = 0; i < num_pages; i++) {
 213                         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 214                         if (page == NULL) {
 215                                 /* Error! */
 216                                 printk(KERN_ERR "[EMGD] Memory allocation failure!\n");
 217                                 if (mem->vmalloc_flag) {
 218                                         vfree(mem->pages);
 219                                 } else {
 220                                         kfree(mem->pages);
 221                                 }
 222                                 kfree(mem);
 223                                 return NULL;
 224                         }
 225
 226                         /* Make sure this page isn't cached */
 227                         if (set_memory_uc((unsigned long) page_address(page), 1) < 0) {
 228                                 printk(KERN_ERR "[EMGD] Unable to set page attributes for newly"
 229                                         " allocated graphics memory.\n");
 230                                 /* Rely on the fact that we've kept up the data structures: */
 231                                 emgd_free_pages(mem);
 232                                 /* XXX - THIS IS WHAT SOME OLD IEGD CODE DID--A GOOD IDEA??? */
 233                                 set_memory_wb((unsigned long) page_address(page), 1);
 234                                 __free_page(page);
 235                                 return NULL;
 236                         }
 237
 238                         get_page(page);
 239                         mem->pages[i] = page;
 240                         mem->page_count++;
 241                 }
 242         } else {
 243                 if (num_pages == 1) {
 244                         order = 0;
 245                 } else if (num_pages == 4) {
 246                         order = 2;
 247                 } else if (num_pages == 8) {
 248                         order = 3;
 249                 } else {
 250                         printk(KERN_ERR "[EMGD] Page count is not valid for physical "
 251                                 "allocation.\n");
 252                         if (mem->vmalloc_flag) {
 253                                 vfree(mem->pages);
 254                         } else {
 255                                 kfree(mem->pages);
 256                         }
 257                         kfree(mem);
 258                         return NULL;
 259                 }
 260
 261                 page = alloc_pages(GFP_KERNEL, order);
 262                 if (page == NULL) {
 263                         /* Error! */
 264                         printk(KERN_ERR "[EMGD] Memory allocation failure!\n");
 265                         if (mem->vmalloc_flag) {
 266                                 vfree(mem->pages);
 267                         } else {
 268                                 kfree(mem->pages);
 269                         }
 270                         kfree(mem);
 271                         return NULL;
 272                 } else {
 273                         /* Make sure these pages aren't cached */
 274                         if (set_memory_uc((unsigned long) page_address(page),
 275                                 num_pages) < 0) {
 276                                 printk(KERN_ERR "[EMGD] Unable to set page attributes for newly"
 277                                         " allocated physical graphics memory.\n");
 278                                 /* XXX - THIS IS WHAT SOME OLD IEGD CODE DID--A GOOD IDEA??? */
 279                                 set_memory_wb((unsigned long) page_address(page), num_pages);
 280                                 __free_pages(page, num_pages);
 281                                 if (mem->vmalloc_flag) {
 282                                         vfree(mem->pages);
 283                                 } else {
 284                                         kfree(mem->pages);
 285                                 }
 286                                 kfree(mem);
 287                                 return NULL;
 288                         }
 289
 290                         get_page(page);
 291                         mem->pages[0] = page;
 292                         if (num_pages > 1) {
 293                                 for (i = 1; i < num_pages; i++) {
 294                                         mem->pages[i] = mem->pages[i-1] + 1;
 295                                 }
 296                         }
 297                         mem->physical = page_to_phys(mem->pages[0]);
 298                         mem->page_count = num_pages;
 299                 }
 300         }
 301
 302         return mem;
 303 }
 304
 305 /*
 306  * Free memory pages.
 307  */
 308 void emgd_free_pages(gmm_mem_buffer_t *mem) {
 309         int i;
 310         struct page *page;
 311
 312         for (i = 0; i < mem->page_count; i++) {
 313                 page = mem->pages[i];
 314                 /* XXX - THIS IS WHAT SOME OLD IEGD CODE DID--A GOOD IDEA??? */
 315                 set_memory_wb((unsigned long) page_address(page), 1);
 316                 put_page(page);
 317                 __free_page(page);
 318                 mem->pages[i] = NULL;
 319         }
 320
 321         if (mem->vmalloc_flag) {
 322                 vfree(mem->pages);
 323         } else {
 324                 kfree(mem->pages);
 325         }
 326
 327         kfree(mem);
 328 }
 329
 330
 331 /*
 332  * Need a function to populate the GTT with the pages.
 333  *
 334  * The caller provides the offset into the GTT where the memory needs
 335  * to go.  This simply needs to allocate the pages and insert them
 336  * into the GTT.
 337  */
 338 void emgd_gtt_insert(igd_context_t *context,
 339                 gmm_mem_buffer_t *mem,
 340                 unsigned long offset)
 341 {
 342         struct page *page;
 343         unsigned long pte;
 344         unsigned long pg_off;
 345         int i, j;
 346
 347         pg_off = offset >> PAGE_SHIFT;
 348
 349         /* Check that the offset is within the gtt's range */
 350         if ((pg_off + mem->page_count) > context->device_context.gatt_pages) {
 351                 printk(KERN_ERR "[EMGD] Attempt to insert a offset beyond of GTT range.\n");
 352                 return;
 353         }
 354
 355         /* Flush before inserting pages into the GTT */
 356         emgd_cache_flush();
 357         tlb_flush();
 358
 359
 360         /* Insert the pages into the GTT */
 361         mutex_lock(&gtt_sem);
 362         for (i = 0, j = pg_off; i < mem->page_count; i++, j++) {
 363                 page = mem->pages[i];
 364
 365                 /* Mark the page as valid */
 366                 pte = page_to_phys(page) | PSB_PTE_VALID;
 367                 writel(pte, (context->device_context.virt_gttadr + j));
 368                 readl(context->device_context.virt_gttadr + j);
 369
 370         }
 371
 372         mutex_unlock(&gtt_sem);
 373
 374         (void)readl(context->device_context.virt_gttadr + j - 1);
 375
 376         /* Invalidate VMA's */
 377         invalidate_vma(j,
 378                 (context->device_context.gmch_ctl | PCI_BASE_ADDRESS_MEM_MASK));
 379
 380         /* Flush */
 381         emgd_cache_flush();
 382         tlb_flush();
 383
 384         return;
 385 }
 386
 387 /*
 388  * Need a function to remove pages from the GTT (and replace with the
 389  * scratch page?) and free the pages.
 390  */
 391
 392 void emgd_gtt_remove(igd_context_t *context,
 393                 gmm_mem_buffer_t *mem,
 394                 unsigned long offset)
 395 {
 396         struct page *page;
 397         unsigned long pte;
 398         int i;
 399         unsigned long pg_start;
 400
 401         pg_start = offset >> PAGE_SHIFT;
 402
 403         /* Flush before inserting pages into the GTT */
 404         emgd_cache_flush();
 405         tlb_flush();
 406
 407         mutex_lock(&gtt_sem);
 408
 409         page = context->device_context.scratch_page;
 410         pte = page_to_phys(page) | PSB_PTE_VALID;
 411
 412         /* Insert the scratch page into the GTT */
 413         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 414                 /* FIXME: Apparently we don't really need to copy stolen memory pages.
 415                  * If so, what should we do about the following code?  Is it correct to
 416                  * do nothing?
 417                  */
 418                 if (i < context->device_context.stolen_pages) {
 419                         /* This is stolen memory.... */
 420                 } else {
 421                         writel(pte, context->device_context.virt_gttadr + i);
 422                         (void)readl(context->device_context.virt_gttadr + i);
 423                 }
 424
 425         }
 426
 427         mutex_unlock(&gtt_sem);
 428
 429         /* Invaidate VMA's */
 430         invalidate_vma(i,
 431                         (context->device_context.gmch_ctl | PCI_BASE_ADDRESS_MEM_MASK));
 432
 433         /* Flush */
 434         emgd_cache_flush();
 435         tlb_flush();
 436 }