pvr/services4/srvkm/devices/sgx/mmu.c

   1 /**********************************************************************
   2  Copyright (c) Imagination Technologies Ltd.
   3
   4  Permission is hereby granted, free of charge, to any person obtaining a copy
   5  of this software and associated documentation files (the "Software"), to deal
   6  in the Software without restriction, including without limitation the rights
   7  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8  copies of the Software, and to permit persons to whom the Software is
   9  furnished to do so, subject to the following conditions:
  10
  11  The above copyright notice and this permission notice shall be included in
  12  all copies or substantial portions of the Software.
  13
  14  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20  THE SOFTWARE.
  21  ******************************************************************************/
  22
  23 #include "sgxdefs.h"
  24 #include "sgxmmu.h"
  25 #include "services_headers.h"
  26 #include "buffer_manager.h"
  27 #include "hash.h"
  28 #include "ra.h"
  29 #include "pdump_km.h"
  30 #include "sgxapi_km.h"
  31 #include "sgxinfo.h"
  32 #include "sgxinfokm.h"
  33 #include "mmu.h"
  34 #include "sgxconfig.h"
  35
  36 #define UINT32_MAX_VALUE        0xFFFFFFFFUL
  37
  38 #define SGX_MAX_PD_ENTRIES      (1<<(SGX_FEATURE_ADDRESS_SPACE_SIZE - SGX_MMU_PT_SHIFT - SGX_MMU_PAGE_SHIFT))
  39
  40 typedef struct _MMU_PT_INFO_
  41 {
  42
  43         IMG_VOID *hPTPageOSMemHandle;
  44         IMG_CPU_VIRTADDR PTPageCpuVAddr;
  45         IMG_UINT32 ui32ValidPTECount;
  46 } MMU_PT_INFO;
  47
  48 struct _MMU_CONTEXT_
  49 {
  50
  51         PVRSRV_DEVICE_NODE *psDeviceNode;
  52
  53
  54         IMG_CPU_VIRTADDR pvPDCpuVAddr;
  55         IMG_DEV_PHYADDR sPDDevPAddr;
  56
  57         IMG_VOID *hPDOSMemHandle;
  58
  59
  60         MMU_PT_INFO *apsPTInfoList[SGX_MAX_PD_ENTRIES];
  61
  62         PVRSRV_SGXDEV_INFO *psDevInfo;
  63
  64 #if defined(PDUMP)
  65         IMG_UINT32 ui32PDumpMMUContextID;
  66 #endif
  67
  68         struct _MMU_CONTEXT_ *psNext;
  69 };
  70
  71 struct _MMU_HEAP_
  72 {
  73
  74         MMU_CONTEXT                     *psMMUContext;
  75
  76
  77
  78
  79         IMG_UINT32                      ui32PDBaseIndex;
  80
  81         IMG_UINT32                      ui32PageTableCount;
  82
  83         IMG_UINT32                      ui32PTETotal;
  84
  85         IMG_UINT32                      ui32PDEPageSizeCtrl;
  86
  87
  88
  89
  90         IMG_UINT32                      ui32DataPageSize;
  91
  92         IMG_UINT32                      ui32DataPageBitWidth;
  93
  94         IMG_UINT32                      ui32DataPageMask;
  95
  96
  97
  98
  99         IMG_UINT32                      ui32PTShift;
 100
 101         IMG_UINT32                      ui32PTBitWidth;
 102
 103         IMG_UINT32                      ui32PTMask;
 104
 105         IMG_UINT32                      ui32PTSize;
 106
 107         IMG_UINT32                      ui32PTECount;
 108
 109
 110
 111
 112         IMG_UINT32                      ui32PDShift;
 113
 114         IMG_UINT32                      ui32PDBitWidth;
 115
 116         IMG_UINT32                      ui32PDMask;
 117
 118
 119
 120         RA_ARENA *psVMArena;
 121         DEV_ARENA_DESCRIPTOR *psDevArena;
 122 };
 123
 124
 125
 126 #if defined (SUPPORT_SGX_MMU_DUMMY_PAGE)
 127 #define DUMMY_DATA_PAGE_SIGNATURE       0xDEADBEEF
 128 #endif
 129
 130 #if defined(PDUMP)
 131 static IMG_VOID
 132 MMU_PDumpPageTables     (MMU_HEAP *pMMUHeap,
 133                                          IMG_DEV_VIRTADDR DevVAddr,
 134                                          IMG_SIZE_T uSize,
 135                                          IMG_BOOL bForUnmap,
 136                                          IMG_HANDLE hUniqueTag);
 137 #endif
 138
 139 #define PAGE_TEST                                       0
 140 #if PAGE_TEST
 141 static IMG_VOID PageTest(IMG_VOID* pMem, IMG_DEV_PHYADDR sDevPAddr);
 142 #endif
 143
 144 #define PT_DEBUG 0
 145 #if PT_DEBUG
 146 static IMG_VOID DumpPT(MMU_PT_INFO *psPTInfoList)
 147 {
 148         IMG_UINT32 *p = (IMG_UINT32*)psPTInfoList->PTPageCpuVAddr;
 149         IMG_UINT32 i;
 150
 151
 152         for(i = 0; i < 1024; i += 8)
 153         {
 154                 PVR_DPF((PVR_DBG_WARNING,
 155                                  "%.8lx %.8lx %.8lx %.8lx %.8lx %.8lx %.8lx %.8lx\n",
 156                                  p[i + 0], p[i + 1], p[i + 2], p[i + 3],
 157                                  p[i + 4], p[i + 5], p[i + 6], p[i + 7]));
 158         }
 159 }
 160
 161 static IMG_VOID CheckPT(MMU_PT_INFO *psPTInfoList)
 162 {
 163         IMG_UINT32 *p = (IMG_UINT32*) psPTInfoList->PTPageCpuVAddr;
 164         IMG_UINT32 i, ui32Count = 0;
 165
 166
 167         for(i = 0; i < 1024; i++)
 168                 if(p[i] & SGX_MMU_PTE_VALID)
 169                         ui32Count++;
 170
 171         if(psPTInfoList->ui32ValidPTECount != ui32Count)
 172         {
 173                 PVR_DPF((PVR_DBG_WARNING, "ui32ValidPTECount: %lu ui32Count: %lu\n",
 174                                  psPTInfoList->ui32ValidPTECount, ui32Count));
 175                 DumpPT(psPTInfoList);
 176                 BUG();
 177         }
 178 }
 179 #else
 180 static INLINE IMG_VOID DumpPT(MMU_PT_INFO *psPTInfoList)
 181 {
 182         PVR_UNREFERENCED_PARAMETER(psPTInfoList);
 183 }
 184
 185 static INLINE IMG_VOID CheckPT(MMU_PT_INFO *psPTInfoList)
 186 {
 187         PVR_UNREFERENCED_PARAMETER(psPTInfoList);
 188 }
 189 #endif
 190
 191 #ifdef SUPPORT_SGX_MMU_BYPASS
 192 IMG_VOID
 193 EnableHostAccess (MMU_CONTEXT *psMMUContext)
 194 {
 195         IMG_UINT32 ui32RegVal;
 196         IMG_VOID *pvRegsBaseKM = psMMUContext->psDevInfo->pvRegsBaseKM;
 197
 198
 199
 200
 201         ui32RegVal = OSReadHWReg(pvRegsBaseKM, EUR_CR_BIF_CTRL);
 202
 203         OSWriteHWReg(pvRegsBaseKM,
 204                                 EUR_CR_BIF_CTRL,
 205                                 ui32RegVal | EUR_CR_BIF_CTRL_MMU_BYPASS_HOST_MASK);
 206
 207         PDUMPREG(EUR_CR_BIF_CTRL, EUR_CR_BIF_CTRL_MMU_BYPASS_HOST_MASK);
 208 }
 209
 210 IMG_VOID
 211 DisableHostAccess (MMU_CONTEXT *psMMUContext)
 212 {
 213         IMG_UINT32 ui32RegVal;
 214         IMG_VOID *pvRegsBaseKM = psMMUContext->psDevInfo->pvRegsBaseKM;
 215
 216
 217
 218
 219
 220         OSWriteHWReg(pvRegsBaseKM,
 221                                 EUR_CR_BIF_CTRL,
 222                                 ui32RegVal & ~EUR_CR_BIF_CTRL_MMU_BYPASS_HOST_MASK);
 223
 224         PDUMPREG(EUR_CR_BIF_CTRL, 0);
 225 }
 226 #endif
 227
 228
 229 IMG_VOID MMU_InvalidateSystemLevelCache(PVRSRV_SGXDEV_INFO *psDevInfo)
 230 {
 231         #if defined(SGX_FEATURE_MP)
 232         psDevInfo->ui32CacheControl |= SGXMKIF_CC_INVAL_BIF_SL;
 233         #else
 234
 235         PVR_UNREFERENCED_PARAMETER(psDevInfo);
 236         #endif
 237 }
 238
 239
 240 IMG_VOID MMU_InvalidateDirectoryCache(PVRSRV_SGXDEV_INFO *psDevInfo)
 241 {
 242         psDevInfo->ui32CacheControl |= SGXMKIF_CC_INVAL_BIF_PD;
 243         #if defined(SGX_FEATURE_SYSTEM_CACHE)
 244         MMU_InvalidateSystemLevelCache(psDevInfo);
 245         #endif
 246 }
 247
 248
 249 IMG_VOID MMU_InvalidatePageTableCache(PVRSRV_SGXDEV_INFO *psDevInfo)
 250 {
 251         psDevInfo->ui32CacheControl |= SGXMKIF_CC_INVAL_BIF_PT;
 252         #if defined(SGX_FEATURE_SYSTEM_CACHE)
 253         MMU_InvalidateSystemLevelCache(psDevInfo);
 254         #endif
 255 }
 256
 257
 258 static IMG_BOOL
 259 _AllocPageTableMemory (MMU_HEAP *pMMUHeap,
 260                                                 MMU_PT_INFO *psPTInfoList,
 261                                                 IMG_DEV_PHYADDR *psDevPAddr)
 262 {
 263         IMG_DEV_PHYADDR sDevPAddr;
 264         IMG_CPU_PHYADDR sCpuPAddr;
 265
 266
 267
 268
 269         if(pMMUHeap->psDevArena->psDeviceMemoryHeapInfo->psLocalDevMemArena == IMG_NULL)
 270         {
 271
 272                 if (OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
 273                                                    pMMUHeap->ui32PTSize,
 274                                                    SGX_MMU_PAGE_SIZE,
 275                                                    (IMG_VOID **)&psPTInfoList->PTPageCpuVAddr,
 276                                                    &psPTInfoList->hPTPageOSMemHandle) != PVRSRV_OK)
 277                 {
 278                         PVR_DPF((PVR_DBG_ERROR, "_AllocPageTableMemory: ERROR call to OSAllocPages failed"));
 279                         return IMG_FALSE;
 280                 }
 281
 282
 283                 if(psPTInfoList->PTPageCpuVAddr)
 284                 {
 285                         sCpuPAddr = OSMapLinToCPUPhys(psPTInfoList->PTPageCpuVAddr);
 286                 }
 287                 else
 288                 {
 289
 290                         sCpuPAddr = OSMemHandleToCpuPAddr(psPTInfoList->hPTPageOSMemHandle, 0);
 291                 }
 292
 293                 sDevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
 294         }
 295         else
 296         {
 297                 IMG_SYS_PHYADDR sSysPAddr;
 298
 299
 300
 301
 302
 303                 if(RA_Alloc(pMMUHeap->psDevArena->psDeviceMemoryHeapInfo->psLocalDevMemArena,
 304                                         SGX_MMU_PAGE_SIZE,
 305                                         IMG_NULL,
 306                                         IMG_NULL,
 307                                         0,
 308                                         SGX_MMU_PAGE_SIZE,
 309                                         0,
 310                                         &(sSysPAddr.uiAddr))!= IMG_TRUE)
 311                 {
 312                         PVR_DPF((PVR_DBG_ERROR, "_AllocPageTableMemory: ERROR call to RA_Alloc failed"));
 313                         return IMG_FALSE;
 314                 }
 315
 316
 317                 sCpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
 318
 319                 psPTInfoList->PTPageCpuVAddr = OSMapPhysToLin(sCpuPAddr,
 320                                                                                                         SGX_MMU_PAGE_SIZE,
 321                                                                                                         PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
 322                                                                                                         &psPTInfoList->hPTPageOSMemHandle);
 323                 if(!psPTInfoList->PTPageCpuVAddr)
 324                 {
 325                         PVR_DPF((PVR_DBG_ERROR, "_AllocPageTableMemory: ERROR failed to map page tables"));
 326                         return IMG_FALSE;
 327                 }
 328
 329
 330                 sDevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
 331
 332                 #if PAGE_TEST
 333                 PageTest(psPTInfoList->PTPageCpuVAddr, sDevPAddr);
 334                 #endif
 335         }
 336
 337 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 338         {
 339                 IMG_UINT32 *pui32Tmp;
 340                 IMG_UINT32 i;
 341
 342                 pui32Tmp = (IMG_UINT32*)psPTInfoList->PTPageCpuVAddr;
 343
 344                 for(i=0; i<pMMUHeap->ui32PTECount; i++)
 345                 {
 346                         pui32Tmp[i] = (pMMUHeap->psMMUContext->psDevInfo->sDummyDataDevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
 347                                                 | SGX_MMU_PTE_VALID;
 348                 }
 349         }
 350 #else
 351
 352         OSMemSet(psPTInfoList->PTPageCpuVAddr, 0, pMMUHeap->ui32PTSize);
 353 #endif
 354
 355
 356         PDUMPMALLOCPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psPTInfoList->PTPageCpuVAddr, pMMUHeap->ui32PTSize, PDUMP_PT_UNIQUETAG);
 357
 358         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, psPTInfoList->PTPageCpuVAddr, pMMUHeap->ui32PTSize, 0, IMG_TRUE, PDUMP_PT_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 359
 360
 361         *psDevPAddr = sDevPAddr;
 362
 363         return IMG_TRUE;
 364 }
 365
 366
 367 static IMG_VOID
 368 _FreePageTableMemory (MMU_HEAP *pMMUHeap, MMU_PT_INFO *psPTInfoList)
 369 {
 370
 371
 372
 373
 374         if(pMMUHeap->psDevArena->psDeviceMemoryHeapInfo->psLocalDevMemArena == IMG_NULL)
 375         {
 376
 377                 OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
 378                                           pMMUHeap->ui32PTSize,
 379                                           psPTInfoList->PTPageCpuVAddr,
 380                                           psPTInfoList->hPTPageOSMemHandle);
 381         }
 382         else
 383         {
 384                 IMG_SYS_PHYADDR sSysPAddr;
 385                 IMG_CPU_PHYADDR sCpuPAddr;
 386
 387
 388                 sCpuPAddr = OSMapLinToCPUPhys(psPTInfoList->PTPageCpuVAddr);
 389                 sSysPAddr = SysCpuPAddrToSysPAddr (sCpuPAddr);
 390
 391
 392
 393                 OSUnMapPhysToLin(psPTInfoList->PTPageCpuVAddr,
 394                          SGX_MMU_PAGE_SIZE,
 395                          PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
 396                          psPTInfoList->hPTPageOSMemHandle);
 397
 398
 399
 400
 401                 RA_Free (pMMUHeap->psDevArena->psDeviceMemoryHeapInfo->psLocalDevMemArena, sSysPAddr.uiAddr, IMG_FALSE);
 402         }
 403 }
 404
 405
 406
 407 static IMG_VOID
 408 _DeferredFreePageTable (MMU_HEAP *pMMUHeap, IMG_UINT32 ui32PTIndex, IMG_BOOL bOSFreePT)
 409 {
 410         IMG_UINT32 *pui32PDEntry;
 411         IMG_UINT32 i;
 412         IMG_UINT32 ui32PDIndex;
 413         SYS_DATA *psSysData;
 414         MMU_PT_INFO **ppsPTInfoList;
 415
 416         SysAcquireData(&psSysData);
 417
 418
 419         ui32PDIndex = pMMUHeap->psDevArena->BaseDevVAddr.uiAddr >> pMMUHeap->ui32PDShift;
 420
 421
 422         ppsPTInfoList = &pMMUHeap->psMMUContext->apsPTInfoList[ui32PDIndex];
 423
 424         {
 425 #if PT_DEBUG
 426                 if(ppsPTInfoList[ui32PTIndex] && ppsPTInfoList[ui32PTIndex]->ui32ValidPTECount > 0)
 427                 {
 428                         DumpPT(ppsPTInfoList[ui32PTIndex]);
 429
 430                 }
 431 #endif
 432
 433
 434                 PVR_ASSERT(ppsPTInfoList[ui32PTIndex] == IMG_NULL || ppsPTInfoList[ui32PTIndex]->ui32ValidPTECount == 0);
 435         }
 436
 437
 438         PDUMPCOMMENT("Free page table (page count == %08X)", pMMUHeap->ui32PageTableCount);
 439         if(ppsPTInfoList[ui32PTIndex] && ppsPTInfoList[ui32PTIndex]->PTPageCpuVAddr)
 440         {
 441                 PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, ppsPTInfoList[ui32PTIndex]->PTPageCpuVAddr, pMMUHeap->ui32PTSize, PDUMP_PT_UNIQUETAG);
 442         }
 443
 444         switch(pMMUHeap->psDevArena->DevMemHeapType)
 445         {
 446                 case DEVICE_MEMORY_HEAP_SHARED :
 447                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED :
 448                 {
 449
 450                         MMU_CONTEXT *psMMUContext = (MMU_CONTEXT*)pMMUHeap->psMMUContext->psDevInfo->pvMMUContextList;
 451
 452                         while(psMMUContext)
 453                         {
 454
 455                                 pui32PDEntry = (IMG_UINT32*)psMMUContext->pvPDCpuVAddr;
 456                                 pui32PDEntry += ui32PDIndex;
 457
 458 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 459
 460                                 pui32PDEntry[ui32PTIndex] = (psMMUContext->psDevInfo->sDummyPTDevPAddr.uiAddr
 461                                                                                         >>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
 462                                                                                         | SGX_MMU_PDE_PAGE_SIZE_4K
 463                                                                                         | SGX_MMU_PDE_VALID;
 464 #else
 465
 466                                 if(bOSFreePT)
 467                                 {
 468                                         pui32PDEntry[ui32PTIndex] = 0;
 469                                 }
 470 #endif
 471
 472
 473                                 PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID*)&pui32PDEntry[ui32PTIndex], sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PT_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 474
 475
 476                                 psMMUContext = psMMUContext->psNext;
 477                         }
 478                         break;
 479                 }
 480                 case DEVICE_MEMORY_HEAP_PERCONTEXT :
 481                 case DEVICE_MEMORY_HEAP_KERNEL :
 482                 {
 483
 484                         pui32PDEntry = (IMG_UINT32*)pMMUHeap->psMMUContext->pvPDCpuVAddr;
 485                         pui32PDEntry += ui32PDIndex;
 486
 487 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 488
 489                         pui32PDEntry[ui32PTIndex] = (pMMUHeap->psMMUContext->psDevInfo->sDummyPTDevPAddr.uiAddr
 490                                                                                 >>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
 491                                                                                 | SGX_MMU_PDE_PAGE_SIZE_4K
 492                                                                                 | SGX_MMU_PDE_VALID;
 493 #else
 494
 495                         if(bOSFreePT)
 496                         {
 497                                 pui32PDEntry[ui32PTIndex] = 0;
 498                         }
 499 #endif
 500
 501
 502                         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID*)&pui32PDEntry[ui32PTIndex], sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 503                         break;
 504                 }
 505                 default:
 506                 {
 507                         PVR_DPF((PVR_DBG_ERROR, "_DeferredFreePagetable: ERROR invalid heap type"));
 508                         return;
 509                 }
 510         }
 511
 512
 513         if(ppsPTInfoList[ui32PTIndex] != IMG_NULL)
 514         {
 515                 if(ppsPTInfoList[ui32PTIndex]->PTPageCpuVAddr != IMG_NULL)
 516                 {
 517                         IMG_PUINT32 pui32Tmp;
 518
 519                         pui32Tmp = (IMG_UINT32*)ppsPTInfoList[ui32PTIndex]->PTPageCpuVAddr;
 520
 521
 522                         for(i=0;
 523                                 (i<pMMUHeap->ui32PTETotal) && (i<pMMUHeap->ui32PTECount);
 524                                  i++)
 525                         {
 526                                 pui32Tmp[i] = 0;
 527                         }
 528
 529
 530
 531                         if(bOSFreePT)
 532                         {
 533                                 _FreePageTableMemory(pMMUHeap, ppsPTInfoList[ui32PTIndex]);
 534                         }
 535
 536
 537
 538
 539                         pMMUHeap->ui32PTETotal -= i;
 540                 }
 541                 else
 542                 {
 543
 544                         pMMUHeap->ui32PTETotal -= pMMUHeap->ui32PTECount;
 545                 }
 546
 547                 if(bOSFreePT)
 548                 {
 549
 550                         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
 551                                                 sizeof(MMU_PT_INFO),
 552                                                 ppsPTInfoList[ui32PTIndex],
 553                                                 IMG_NULL);
 554                         ppsPTInfoList[ui32PTIndex] = IMG_NULL;
 555                 }
 556         }
 557         else
 558         {
 559
 560                 pMMUHeap->ui32PTETotal -= pMMUHeap->ui32PTECount;
 561         }
 562
 563         PDUMPCOMMENT("Finished free page table (page count == %08X)", pMMUHeap->ui32PageTableCount);
 564 }
 565
 566 static IMG_VOID
 567 _DeferredFreePageTables (MMU_HEAP *pMMUHeap)
 568 {
 569         IMG_UINT32 i;
 570
 571         for(i=0; i<pMMUHeap->ui32PageTableCount; i++)
 572         {
 573                 _DeferredFreePageTable(pMMUHeap, i, IMG_TRUE);
 574         }
 575         MMU_InvalidateDirectoryCache(pMMUHeap->psMMUContext->psDevInfo);
 576 }
 577
 578
 579 static IMG_BOOL
 580 _DeferredAllocPagetables(MMU_HEAP *pMMUHeap, IMG_DEV_VIRTADDR DevVAddr, IMG_UINT32 ui32Size)
 581 {
 582         IMG_UINT32 ui32PageTableCount;
 583         IMG_UINT32 ui32PDIndex;
 584         IMG_UINT32 i;
 585         IMG_UINT32 *pui32PDEntry;
 586         MMU_PT_INFO **ppsPTInfoList;
 587         SYS_DATA *psSysData;
 588         IMG_DEV_VIRTADDR sHighDevVAddr;
 589
 590
 591 #if SGX_FEATURE_ADDRESS_SPACE_SIZE < 32
 592         PVR_ASSERT(DevVAddr.uiAddr < (1<<SGX_FEATURE_ADDRESS_SPACE_SIZE));
 593 #endif
 594
 595
 596         SysAcquireData(&psSysData);
 597
 598
 599         ui32PDIndex = DevVAddr.uiAddr >> pMMUHeap->ui32PDShift;
 600
 601
 602
 603         if((UINT32_MAX_VALUE - DevVAddr.uiAddr)
 604                 < (ui32Size + pMMUHeap->ui32DataPageMask + pMMUHeap->ui32PTMask))
 605         {
 606
 607                 sHighDevVAddr.uiAddr = UINT32_MAX_VALUE;
 608         }
 609         else
 610         {
 611                 sHighDevVAddr.uiAddr = DevVAddr.uiAddr
 612                                                                 + ui32Size
 613                                                                 + pMMUHeap->ui32DataPageMask
 614                                                                 + pMMUHeap->ui32PTMask;
 615         }
 616
 617         ui32PageTableCount = sHighDevVAddr.uiAddr >> pMMUHeap->ui32PDShift;
 618
 619         ui32PageTableCount -= ui32PDIndex;
 620
 621
 622         pui32PDEntry = (IMG_UINT32*)pMMUHeap->psMMUContext->pvPDCpuVAddr;
 623         pui32PDEntry += ui32PDIndex;
 624
 625
 626         ppsPTInfoList = &pMMUHeap->psMMUContext->apsPTInfoList[ui32PDIndex];
 627
 628         PDUMPCOMMENT("Alloc page table (page count == %08X)", ui32PageTableCount);
 629         PDUMPCOMMENT("Page directory mods (page count == %08X)", ui32PageTableCount);
 630
 631
 632         for(i=0; i<ui32PageTableCount; i++)
 633         {
 634                 if(ppsPTInfoList[i] == IMG_NULL)
 635                 {
 636                         OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 637                                                  sizeof (MMU_PT_INFO),
 638                                                  (IMG_VOID **)&ppsPTInfoList[i], IMG_NULL,
 639                                                  "MMU Page Table Info");
 640                         if (ppsPTInfoList[i] == IMG_NULL)
 641                         {
 642                                 PVR_DPF((PVR_DBG_ERROR, "_DeferredAllocPagetables: ERROR call to OSAllocMem failed"));
 643                                 return IMG_FALSE;
 644                         }
 645                         OSMemSet (ppsPTInfoList[i], 0, sizeof(MMU_PT_INFO));
 646                 }
 647
 648                 if(ppsPTInfoList[i]->hPTPageOSMemHandle == IMG_NULL
 649                 && ppsPTInfoList[i]->PTPageCpuVAddr == IMG_NULL)
 650                 {
 651                         IMG_DEV_PHYADDR sDevPAddr;
 652 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 653                         IMG_UINT32 *pui32Tmp;
 654                         IMG_UINT32 j;
 655 #else
 656
 657                         PVR_ASSERT(pui32PDEntry[i] == 0);
 658 #endif
 659
 660                         if(_AllocPageTableMemory (pMMUHeap, ppsPTInfoList[i], &sDevPAddr) != IMG_TRUE)
 661                         {
 662                                 PVR_DPF((PVR_DBG_ERROR, "_DeferredAllocPagetables: ERROR call to _AllocPageTableMemory failed"));
 663                                 return IMG_FALSE;
 664                         }
 665
 666                         switch(pMMUHeap->psDevArena->DevMemHeapType)
 667                         {
 668                                 case DEVICE_MEMORY_HEAP_SHARED :
 669                                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED :
 670                                 {
 671
 672                                         MMU_CONTEXT *psMMUContext = (MMU_CONTEXT*)pMMUHeap->psMMUContext->psDevInfo->pvMMUContextList;
 673
 674                                         while(psMMUContext)
 675                                         {
 676
 677                                                 pui32PDEntry = (IMG_UINT32*)psMMUContext->pvPDCpuVAddr;
 678                                                 pui32PDEntry += ui32PDIndex;
 679
 680
 681                                                 pui32PDEntry[i] = (sDevPAddr.uiAddr>>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
 682                                                                                 | pMMUHeap->ui32PDEPageSizeCtrl
 683                                                                                 | SGX_MMU_PDE_VALID;
 684
 685
 686                                                 PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID*)&pui32PDEntry[i], sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 687
 688
 689                                                 psMMUContext = psMMUContext->psNext;
 690                                         }
 691                                         break;
 692                                 }
 693                                 case DEVICE_MEMORY_HEAP_PERCONTEXT :
 694                                 case DEVICE_MEMORY_HEAP_KERNEL :
 695                                 {
 696
 697                                         pui32PDEntry[i] = (sDevPAddr.uiAddr>>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
 698                                                                         | pMMUHeap->ui32PDEPageSizeCtrl
 699                                                                         | SGX_MMU_PDE_VALID;
 700
 701
 702                                         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID*)&pui32PDEntry[i], sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 703                                         break;
 704                                 }
 705                                 default:
 706                                 {
 707                                         PVR_DPF((PVR_DBG_ERROR, "_DeferredAllocPagetables: ERROR invalid heap type"));
 708                                         return IMG_FALSE;
 709                                 }
 710                         }
 711
 712 #if !defined(SGX_FEATURE_MULTIPLE_MEM_CONTEXTS)
 713
 714
 715
 716
 717                         MMU_InvalidateDirectoryCache(pMMUHeap->psMMUContext->psDevInfo);
 718 #endif
 719                 }
 720                 else
 721                 {
 722
 723                         PVR_ASSERT(pui32PDEntry[i] != 0);
 724                 }
 725         }
 726
 727         #if defined(SGX_FEATURE_SYSTEM_CACHE)
 728         MMU_InvalidateSystemLevelCache(pMMUHeap->psMMUContext->psDevInfo);
 729         #endif
 730
 731         return IMG_TRUE;
 732 }
 733
 734
 735 PVRSRV_ERROR
 736 MMU_Initialise (PVRSRV_DEVICE_NODE *psDeviceNode, MMU_CONTEXT **ppsMMUContext, IMG_DEV_PHYADDR *psPDDevPAddr)
 737 {
 738         IMG_UINT32 *pui32Tmp;
 739         IMG_UINT32 i;
 740         IMG_CPU_VIRTADDR pvPDCpuVAddr;
 741         IMG_DEV_PHYADDR sPDDevPAddr;
 742         IMG_CPU_PHYADDR sCpuPAddr;
 743         MMU_CONTEXT *psMMUContext;
 744         IMG_HANDLE hPDOSMemHandle;
 745         SYS_DATA *psSysData;
 746         PVRSRV_SGXDEV_INFO *psDevInfo;
 747
 748         PVR_DPF ((PVR_DBG_MESSAGE, "MMU_Initialise"));
 749
 750         SysAcquireData(&psSysData);
 751
 752         OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 753                                  sizeof (MMU_CONTEXT),
 754                                  (IMG_VOID **)&psMMUContext, IMG_NULL,
 755                                  "MMU Context");
 756         if (psMMUContext == IMG_NULL)
 757         {
 758                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to OSAllocMem failed"));
 759                 return PVRSRV_ERROR_GENERIC;
 760         }
 761         OSMemSet (psMMUContext, 0, sizeof(MMU_CONTEXT));
 762
 763
 764         psDevInfo = (PVRSRV_SGXDEV_INFO*)psDeviceNode->pvDevice;
 765         psMMUContext->psDevInfo = psDevInfo;
 766
 767
 768         psMMUContext->psDeviceNode = psDeviceNode;
 769
 770
 771         if(psDeviceNode->psLocalDevMemArena == IMG_NULL)
 772         {
 773                 if (OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
 774                                                         SGX_MMU_PAGE_SIZE,
 775                                                         SGX_MMU_PAGE_SIZE,
 776                                                         &pvPDCpuVAddr,
 777                                                         &hPDOSMemHandle) != PVRSRV_OK)
 778                 {
 779                         PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to OSAllocPages failed"));
 780                         return PVRSRV_ERROR_GENERIC;
 781                 }
 782
 783                 if(pvPDCpuVAddr)
 784                 {
 785                         sCpuPAddr = OSMapLinToCPUPhys(pvPDCpuVAddr);
 786                 }
 787                 else
 788                 {
 789
 790                         sCpuPAddr = OSMemHandleToCpuPAddr(hPDOSMemHandle, 0);
 791                 }
 792                 sPDDevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
 793
 794                 #if PAGE_TEST
 795                 PageTest(pvPDCpuVAddr, sPDDevPAddr);
 796                 #endif
 797
 798 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 799
 800                 if(!psDevInfo->pvMMUContextList)
 801                 {
 802
 803                         if (OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
 804                                                                 SGX_MMU_PAGE_SIZE,
 805                                                                 SGX_MMU_PAGE_SIZE,
 806                                                                 &psDevInfo->pvDummyPTPageCpuVAddr,
 807                                                                 &psDevInfo->hDummyPTPageOSMemHandle) != PVRSRV_OK)
 808                         {
 809                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to OSAllocPages failed"));
 810                                 return PVRSRV_ERROR_GENERIC;
 811                         }
 812
 813                         if(psDevInfo->pvDummyPTPageCpuVAddr)
 814                         {
 815                                 sCpuPAddr = OSMapLinToCPUPhys(psDevInfo->pvDummyPTPageCpuVAddr);
 816                         }
 817                         else
 818                         {
 819
 820                                 sCpuPAddr = OSMemHandleToCpuPAddr(psDevInfo->hDummyPTPageOSMemHandle, 0);
 821                         }
 822                         psDevInfo->sDummyPTDevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
 823
 824
 825                         if (OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
 826                                                                 SGX_MMU_PAGE_SIZE,
 827                                                                 SGX_MMU_PAGE_SIZE,
 828                                                                 &psDevInfo->pvDummyDataPageCpuVAddr,
 829                                                                 &psDevInfo->hDummyDataPageOSMemHandle) != PVRSRV_OK)
 830                         {
 831                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to OSAllocPages failed"));
 832                                 return PVRSRV_ERROR_GENERIC;
 833                         }
 834
 835                         if(psDevInfo->pvDummyDataPageCpuVAddr)
 836                         {
 837                                 sCpuPAddr = OSMapLinToCPUPhys(psDevInfo->pvDummyDataPageCpuVAddr);
 838                         }
 839                         else
 840                         {
 841                                 sCpuPAddr = OSMemHandleToCpuPAddr(psDevInfo->hDummyDataPageOSMemHandle, 0);
 842                         }
 843                         psDevInfo->sDummyDataDevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
 844                 }
 845 #endif
 846         }
 847         else
 848         {
 849                 IMG_SYS_PHYADDR sSysPAddr;
 850
 851
 852                 if(RA_Alloc(psDeviceNode->psLocalDevMemArena,
 853                                         SGX_MMU_PAGE_SIZE,
 854                                         IMG_NULL,
 855                                         IMG_NULL,
 856                                         0,
 857                                         SGX_MMU_PAGE_SIZE,
 858                                         0,
 859                                         &(sSysPAddr.uiAddr))!= IMG_TRUE)
 860                 {
 861                         PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to RA_Alloc failed"));
 862                         return PVRSRV_ERROR_GENERIC;
 863                 }
 864
 865
 866                 sCpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
 867                 sPDDevPAddr = SysSysPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sSysPAddr);
 868                 pvPDCpuVAddr = OSMapPhysToLin(sCpuPAddr,
 869                                                                                 SGX_MMU_PAGE_SIZE,
 870                                                                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
 871                                                                                 &hPDOSMemHandle);
 872                 if(!pvPDCpuVAddr)
 873                 {
 874                         PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR failed to map page tables"));
 875                         return PVRSRV_ERROR_GENERIC;
 876                 }
 877
 878                 #if PAGE_TEST
 879                 PageTest(pvPDCpuVAddr, sPDDevPAddr);
 880                 #endif
 881
 882 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 883
 884                 if(!psDevInfo->pvMMUContextList)
 885                 {
 886
 887                         if(RA_Alloc(psDeviceNode->psLocalDevMemArena,
 888                                                 SGX_MMU_PAGE_SIZE,
 889                                                 IMG_NULL,
 890                                                 IMG_NULL,
 891                                                 0,
 892                                                 SGX_MMU_PAGE_SIZE,
 893                                                 0,
 894                                                 &(sSysPAddr.uiAddr))!= IMG_TRUE)
 895                         {
 896                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to RA_Alloc failed"));
 897                                 return PVRSRV_ERROR_GENERIC;
 898                         }
 899
 900
 901                         sCpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
 902                         psDevInfo->sDummyPTDevPAddr = SysSysPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sSysPAddr);
 903                         psDevInfo->pvDummyPTPageCpuVAddr = OSMapPhysToLin(sCpuPAddr,
 904                                                                                                                                 SGX_MMU_PAGE_SIZE,
 905                                                                                                                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
 906                                                                                                                                 &psDevInfo->hDummyPTPageOSMemHandle);
 907                         if(!psDevInfo->pvDummyPTPageCpuVAddr)
 908                         {
 909                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR failed to map page tables"));
 910                                 return PVRSRV_ERROR_GENERIC;
 911                         }
 912
 913
 914                         if(RA_Alloc(psDeviceNode->psLocalDevMemArena,
 915                                                 SGX_MMU_PAGE_SIZE,
 916                                                 IMG_NULL,
 917                                                 IMG_NULL,
 918                                                 0,
 919                                                 SGX_MMU_PAGE_SIZE,
 920                                                 0,
 921                                                 &(sSysPAddr.uiAddr))!= IMG_TRUE)
 922                         {
 923                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to RA_Alloc failed"));
 924                                 return PVRSRV_ERROR_GENERIC;
 925                         }
 926
 927
 928                         sCpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
 929                         psDevInfo->sDummyDataDevPAddr = SysSysPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sSysPAddr);
 930                         psDevInfo->pvDummyDataPageCpuVAddr = OSMapPhysToLin(sCpuPAddr,
 931                                                                                                                                 SGX_MMU_PAGE_SIZE,
 932                                                                                                                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
 933                                                                                                                                 &psDevInfo->hDummyDataPageOSMemHandle);
 934                         if(!psDevInfo->pvDummyDataPageCpuVAddr)
 935                         {
 936                                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR failed to map page tables"));
 937                                 return PVRSRV_ERROR_GENERIC;
 938                         }
 939                 }
 940 #endif
 941         }
 942
 943
 944         PDUMPCOMMENT("Alloc page directory");
 945 #ifdef SUPPORT_SGX_MMU_BYPASS
 946         EnableHostAccess(psMMUContext);
 947 #endif
 948
 949         PDUMPMALLOCPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, pvPDCpuVAddr, SGX_MMU_PAGE_SIZE, PDUMP_PD_UNIQUETAG);
 950
 951         if (pvPDCpuVAddr)
 952         {
 953                 pui32Tmp = (IMG_UINT32 *)pvPDCpuVAddr;
 954         }
 955         else
 956         {
 957                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: pvPDCpuVAddr invalid"));
 958                 return PVRSRV_ERROR_GENERIC;
 959         }
 960
 961 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
 962
 963         for(i=0; i<SGX_MMU_PD_SIZE; i++)
 964         {
 965                 pui32Tmp[i] = (psDevInfo->sDummyPTDevPAddr.uiAddr>>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
 966                                         | SGX_MMU_PDE_PAGE_SIZE_4K
 967                                         | SGX_MMU_PDE_VALID;
 968         }
 969
 970         if(!psDevInfo->pvMMUContextList)
 971         {
 972
 973
 974
 975                 pui32Tmp = (IMG_UINT32 *)psDevInfo->pvDummyPTPageCpuVAddr;
 976                 for(i=0; i<SGX_MMU_PT_SIZE; i++)
 977                 {
 978                         pui32Tmp[i] = (psDevInfo->sDummyDataDevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
 979                                                 | SGX_MMU_PTE_VALID;
 980                 }
 981
 982                 PDUMPCOMMENT("Dummy Page table contents");
 983                 PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pvDummyPTPageCpuVAddr, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 984
 985
 986
 987                 pui32Tmp = (IMG_UINT32 *)psDevInfo->pvDummyDataPageCpuVAddr;
 988                 for(i=0; i<(SGX_MMU_PAGE_SIZE/4); i++)
 989                 {
 990                         pui32Tmp[i] = DUMMY_DATA_PAGE_SIGNATURE;
 991                 }
 992
 993                 PDUMPCOMMENT("Dummy Data Page contents");
 994                 PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pvDummyDataPageCpuVAddr, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
 995         }
 996 #else
 997
 998         for(i=0; i<SGX_MMU_PD_SIZE; i++)
 999         {
1000
1001                 pui32Tmp[i] = 0;
1002         }
1003 #endif
1004
1005
1006         PDUMPCOMMENT("Page directory contents");
1007         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pvPDCpuVAddr, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
1008
1009
1010 #if defined(PDUMP)
1011         if(PDumpSetMMUContext(PVRSRV_DEVICE_TYPE_SGX,
1012                                                         "SGXMEM",
1013                                                         &psMMUContext->ui32PDumpMMUContextID,
1014                                                         2,
1015                                                         PDUMP_PT_UNIQUETAG,
1016                                                         pvPDCpuVAddr) != PVRSRV_OK)
1017         {
1018                 PVR_DPF((PVR_DBG_ERROR, "MMU_Initialise: ERROR call to PDumpSetMMUContext failed"));
1019                 return PVRSRV_ERROR_GENERIC;
1020         }
1021 #endif
1022
1023
1024         psMMUContext->pvPDCpuVAddr = pvPDCpuVAddr;
1025         psMMUContext->sPDDevPAddr = sPDDevPAddr;
1026         psMMUContext->hPDOSMemHandle = hPDOSMemHandle;
1027
1028
1029         *ppsMMUContext = psMMUContext;
1030
1031
1032         *psPDDevPAddr = sPDDevPAddr;
1033
1034
1035         psMMUContext->psNext = (MMU_CONTEXT*)psDevInfo->pvMMUContextList;
1036         psDevInfo->pvMMUContextList = (IMG_VOID*)psMMUContext;
1037
1038 #ifdef SUPPORT_SGX_MMU_BYPASS
1039         DisableHostAccess(psMMUContext);
1040 #endif
1041
1042         return PVRSRV_OK;
1043 }
1044
1045 IMG_VOID
1046 MMU_Finalise (MMU_CONTEXT *psMMUContext)
1047 {
1048         IMG_UINT32 *pui32Tmp, i;
1049         SYS_DATA *psSysData;
1050         MMU_CONTEXT **ppsMMUContext;
1051 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1052         PVRSRV_SGXDEV_INFO *psDevInfo = (PVRSRV_SGXDEV_INFO*)psMMUContext->psDevInfo;
1053         MMU_CONTEXT *psMMUContextList = (MMU_CONTEXT*)psDevInfo->pvMMUContextList;
1054 #endif
1055
1056         SysAcquireData(&psSysData);
1057
1058
1059         PDUMPCLEARMMUCONTEXT(PVRSRV_DEVICE_TYPE_SGX, "SGXMEM", psMMUContext->ui32PDumpMMUContextID, 2);
1060
1061
1062         PDUMPCOMMENT("Free page directory");
1063         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psMMUContext->pvPDCpuVAddr, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
1064 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1065         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pvDummyPTPageCpuVAddr, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
1066         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pvDummyDataPageCpuVAddr, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
1067 #endif
1068
1069         pui32Tmp = (IMG_UINT32 *)psMMUContext->pvPDCpuVAddr;
1070
1071
1072         for(i=0; i<SGX_MMU_PD_SIZE; i++)
1073         {
1074
1075                 pui32Tmp[i] = 0;
1076         }
1077
1078
1079
1080
1081
1082         if(psMMUContext->psDeviceNode->psLocalDevMemArena == IMG_NULL)
1083         {
1084                 OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
1085                                                 SGX_MMU_PAGE_SIZE,
1086                                                 psMMUContext->pvPDCpuVAddr,
1087                                                 psMMUContext->hPDOSMemHandle);
1088
1089 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1090
1091                 if(!psMMUContextList->psNext)
1092                 {
1093                         OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
1094                                                         SGX_MMU_PAGE_SIZE,
1095                                                         psDevInfo->pvDummyPTPageCpuVAddr,
1096                                                         psDevInfo->hDummyPTPageOSMemHandle);
1097                         OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
1098                                                         SGX_MMU_PAGE_SIZE,
1099                                                         psDevInfo->pvDummyDataPageCpuVAddr,
1100                                                         psDevInfo->hDummyDataPageOSMemHandle);
1101                 }
1102 #endif
1103         }
1104         else
1105         {
1106                 IMG_SYS_PHYADDR sSysPAddr;
1107                 IMG_CPU_PHYADDR sCpuPAddr;
1108
1109
1110                 sCpuPAddr = OSMapLinToCPUPhys(psMMUContext->pvPDCpuVAddr);
1111                 sSysPAddr = SysCpuPAddrToSysPAddr(sCpuPAddr);
1112
1113
1114                 OSUnMapPhysToLin(psMMUContext->pvPDCpuVAddr,
1115                                                         SGX_MMU_PAGE_SIZE,
1116                             PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
1117                                                         psMMUContext->hPDOSMemHandle);
1118
1119                 RA_Free (psMMUContext->psDeviceNode->psLocalDevMemArena, sSysPAddr.uiAddr, IMG_FALSE);
1120
1121 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1122
1123                 if(!psMMUContextList->psNext)
1124                 {
1125
1126                         sCpuPAddr = OSMapLinToCPUPhys(psDevInfo->pvDummyPTPageCpuVAddr);
1127                         sSysPAddr = SysCpuPAddrToSysPAddr(sCpuPAddr);
1128
1129
1130                         OSUnMapPhysToLin(psDevInfo->pvDummyPTPageCpuVAddr,
1131                                                                 SGX_MMU_PAGE_SIZE,
1132                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
1133                                                                 psDevInfo->hDummyPTPageOSMemHandle);
1134
1135                         RA_Free (psMMUContext->psDeviceNode->psLocalDevMemArena, sSysPAddr.uiAddr, IMG_FALSE);
1136
1137
1138                         sCpuPAddr = OSMapLinToCPUPhys(psDevInfo->pvDummyDataPageCpuVAddr);
1139                         sSysPAddr = SysCpuPAddrToSysPAddr(sCpuPAddr);
1140
1141
1142                         OSUnMapPhysToLin(psDevInfo->pvDummyDataPageCpuVAddr,
1143                                                                 SGX_MMU_PAGE_SIZE,
1144                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
1145                                                                 psDevInfo->hDummyDataPageOSMemHandle);
1146
1147                         RA_Free (psMMUContext->psDeviceNode->psLocalDevMemArena, sSysPAddr.uiAddr, IMG_FALSE);
1148                 }
1149 #endif
1150         }
1151
1152         PVR_DPF ((PVR_DBG_MESSAGE, "MMU_Finalise"));
1153
1154
1155         ppsMMUContext = (MMU_CONTEXT**)&psMMUContext->psDevInfo->pvMMUContextList;
1156         while(*ppsMMUContext)
1157         {
1158                 if(*ppsMMUContext == psMMUContext)
1159                 {
1160
1161                         *ppsMMUContext = psMMUContext->psNext;
1162                         break;
1163                 }
1164
1165
1166                 ppsMMUContext = &((*ppsMMUContext)->psNext);
1167         }
1168
1169
1170         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(MMU_CONTEXT), psMMUContext, IMG_NULL);
1171
1172 }
1173
1174
1175 IMG_VOID
1176 MMU_InsertHeap(MMU_CONTEXT *psMMUContext, MMU_HEAP *psMMUHeap)
1177 {
1178         IMG_UINT32 *pui32PDCpuVAddr = (IMG_UINT32 *) psMMUContext->pvPDCpuVAddr;
1179         IMG_UINT32 *pui32KernelPDCpuVAddr = (IMG_UINT32 *) psMMUHeap->psMMUContext->pvPDCpuVAddr;
1180         IMG_UINT32 ui32PDEntry;
1181 #if !defined(SGX_FEATURE_MULTIPLE_MEM_CONTEXTS)
1182         IMG_BOOL bInvalidateDirectoryCache = IMG_FALSE;
1183 #endif
1184
1185
1186         pui32PDCpuVAddr += psMMUHeap->psDevArena->BaseDevVAddr.uiAddr >> psMMUHeap->ui32PDShift;
1187         pui32KernelPDCpuVAddr += psMMUHeap->psDevArena->BaseDevVAddr.uiAddr >> psMMUHeap->ui32PDShift;
1188
1189
1190
1191
1192         PDUMPCOMMENT("Page directory shared heap range copy");
1193 #ifdef SUPPORT_SGX_MMU_BYPASS
1194         EnableHostAccess(psMMUContext);
1195 #endif
1196
1197         for (ui32PDEntry = 0; ui32PDEntry < psMMUHeap->ui32PageTableCount; ui32PDEntry++)
1198         {
1199 #if !defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1200
1201                 PVR_ASSERT(pui32PDCpuVAddr[ui32PDEntry] == 0);
1202 #endif
1203
1204
1205                 pui32PDCpuVAddr[ui32PDEntry] = pui32KernelPDCpuVAddr[ui32PDEntry];
1206                 if (pui32PDCpuVAddr[ui32PDEntry])
1207                 {
1208                         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID *) &pui32PDCpuVAddr[ui32PDEntry], sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
1209
1210 #if !defined(SGX_FEATURE_MULTIPLE_MEM_CONTEXTS)
1211                         bInvalidateDirectoryCache = IMG_TRUE;
1212 #endif
1213                 }
1214         }
1215
1216 #ifdef SUPPORT_SGX_MMU_BYPASS
1217         DisableHostAccess(psMMUContext);
1218 #endif
1219
1220 #if !defined(SGX_FEATURE_MULTIPLE_MEM_CONTEXTS)
1221         if (bInvalidateDirectoryCache)
1222         {
1223
1224
1225
1226
1227                 MMU_InvalidateDirectoryCache(psMMUContext->psDevInfo);
1228         }
1229 #endif
1230 }
1231
1232
1233 static IMG_VOID
1234 MMU_UnmapPagesAndFreePTs (MMU_HEAP *psMMUHeap,
1235                                                   IMG_DEV_VIRTADDR sDevVAddr,
1236                                                   IMG_UINT32 ui32PageCount,
1237                                                   IMG_HANDLE hUniqueTag)
1238 {
1239         IMG_DEV_VIRTADDR        sTmpDevVAddr;
1240         IMG_UINT32                      i;
1241         IMG_UINT32                      ui32PDIndex;
1242         IMG_UINT32                      ui32PTIndex;
1243         IMG_UINT32                      *pui32Tmp;
1244         IMG_BOOL                        bInvalidateDirectoryCache = IMG_FALSE;
1245
1246 #if !defined (PDUMP)
1247         PVR_UNREFERENCED_PARAMETER(hUniqueTag);
1248 #endif
1249
1250         sTmpDevVAddr = sDevVAddr;
1251
1252         for(i=0; i<ui32PageCount; i++)
1253         {
1254                 MMU_PT_INFO **ppsPTInfoList;
1255
1256
1257                 ui32PDIndex = sTmpDevVAddr.uiAddr >> psMMUHeap->ui32PDShift;
1258
1259
1260                 ppsPTInfoList = &psMMUHeap->psMMUContext->apsPTInfoList[ui32PDIndex];
1261
1262                 {
1263
1264                         ui32PTIndex = (sTmpDevVAddr.uiAddr & psMMUHeap->ui32PTMask) >> psMMUHeap->ui32PTShift;
1265
1266
1267                         if (!ppsPTInfoList[0])
1268                         {
1269                                 PVR_DPF((PVR_DBG_MESSAGE, "MMU_UnmapPagesAndFreePTs: Invalid PT for alloc at VAddr:0x%08lX (VaddrIni:0x%08lX AllocPage:%u) PDIdx:%u PTIdx:%u",sTmpDevVAddr.uiAddr, sDevVAddr.uiAddr,i, ui32PDIndex, ui32PTIndex ));
1270
1271
1272                                 sTmpDevVAddr.uiAddr += psMMUHeap->ui32DataPageSize;
1273
1274
1275                                 continue;
1276                         }
1277
1278
1279                         pui32Tmp = (IMG_UINT32*)ppsPTInfoList[0]->PTPageCpuVAddr;
1280
1281
1282                         if (!pui32Tmp)
1283                         {
1284                                 continue;
1285                         }
1286
1287                         CheckPT(ppsPTInfoList[0]);
1288
1289
1290                         if (pui32Tmp[ui32PTIndex] & SGX_MMU_PTE_VALID)
1291                         {
1292                                 ppsPTInfoList[0]->ui32ValidPTECount--;
1293                         }
1294                         else
1295                         {
1296                                 PVR_DPF((PVR_DBG_MESSAGE, "MMU_UnmapPagesAndFreePTs: Page is already invalid for alloc at VAddr:0x%08lX (VAddrIni:0x%08lX AllocPage:%u) PDIdx:%u PTIdx:%u",sTmpDevVAddr.uiAddr, sDevVAddr.uiAddr,i, ui32PDIndex, ui32PTIndex ));
1297                         }
1298
1299
1300                         PVR_ASSERT((IMG_INT32)ppsPTInfoList[0]->ui32ValidPTECount >= 0);
1301
1302 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1303
1304                         pui32Tmp[ui32PTIndex] = (psMMUHeap->psMMUContext->psDevInfo->sDummyDataDevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
1305                                                                         | SGX_MMU_PTE_VALID;
1306 #else
1307
1308                         pui32Tmp[ui32PTIndex] = 0;
1309 #endif
1310
1311                         CheckPT(ppsPTInfoList[0]);
1312                 }
1313
1314
1315
1316                 if (ppsPTInfoList[0] && ppsPTInfoList[0]->ui32ValidPTECount == 0)
1317                 {
1318                         _DeferredFreePageTable(psMMUHeap, ui32PDIndex - psMMUHeap->ui32PDBaseIndex, IMG_TRUE);
1319                         bInvalidateDirectoryCache = IMG_TRUE;
1320                 }
1321
1322
1323                 sTmpDevVAddr.uiAddr += psMMUHeap->ui32DataPageSize;
1324         }
1325
1326         if(bInvalidateDirectoryCache)
1327         {
1328                 MMU_InvalidateDirectoryCache(psMMUHeap->psMMUContext->psDevInfo);
1329         }
1330         else
1331         {
1332                 MMU_InvalidatePageTableCache(psMMUHeap->psMMUContext->psDevInfo);
1333         }
1334
1335 #if defined(PDUMP)
1336         MMU_PDumpPageTables(psMMUHeap,
1337                                                 sDevVAddr,
1338                                                 psMMUHeap->ui32DataPageSize * ui32PageCount,
1339                                                 IMG_TRUE,
1340                                                 hUniqueTag);
1341 #endif
1342 }
1343
1344
1345 IMG_VOID MMU_FreePageTables(IMG_PVOID pvMMUHeap,
1346                             IMG_SIZE_T ui32Start,
1347                             IMG_SIZE_T ui32End,
1348                             IMG_HANDLE hUniqueTag)
1349 {
1350         MMU_HEAP *pMMUHeap = (MMU_HEAP*)pvMMUHeap;
1351         IMG_DEV_VIRTADDR Start;
1352
1353         Start.uiAddr = ui32Start;
1354
1355         MMU_UnmapPagesAndFreePTs(pMMUHeap, Start, (ui32End - ui32Start) >> pMMUHeap->ui32PTShift, hUniqueTag);
1356 }
1357
1358 MMU_HEAP *
1359 MMU_Create (MMU_CONTEXT *psMMUContext,
1360                         DEV_ARENA_DESCRIPTOR *psDevArena,
1361                         RA_ARENA **ppsVMArena)
1362 {
1363         MMU_HEAP *pMMUHeap;
1364         IMG_UINT32 ui32ScaleSize;
1365
1366         PVR_ASSERT (psDevArena != IMG_NULL);
1367
1368         if (psDevArena == IMG_NULL)
1369         {
1370                 PVR_DPF((PVR_DBG_ERROR, "MMU_Create: invalid parameter"));
1371                 return IMG_NULL;
1372         }
1373
1374         OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
1375                                  sizeof (MMU_HEAP),
1376                                  (IMG_VOID **)&pMMUHeap, IMG_NULL,
1377                                  "MMU Heap");
1378         if (pMMUHeap == IMG_NULL)
1379         {
1380                 PVR_DPF((PVR_DBG_ERROR, "MMU_Create: ERROR call to OSAllocMem failed"));
1381                 return IMG_NULL;
1382         }
1383
1384         pMMUHeap->psMMUContext = psMMUContext;
1385         pMMUHeap->psDevArena = psDevArena;
1386
1387
1388
1389
1390         switch(pMMUHeap->psDevArena->ui32DataPageSize)
1391         {
1392                 case 0x1000:
1393                         ui32ScaleSize = 0;
1394                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_4K;
1395                         break;
1396 #if defined(SGX_FEATURE_VARIABLE_MMU_PAGE_SIZE)
1397                 case 0x4000:
1398                         ui32ScaleSize = 2;
1399                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_16K;
1400                         break;
1401                 case 0x10000:
1402                         ui32ScaleSize = 4;
1403                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_64K;
1404                         break;
1405                 case 0x40000:
1406                         ui32ScaleSize = 6;
1407                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_256K;
1408                         break;
1409                 case 0x100000:
1410                         ui32ScaleSize = 8;
1411                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_1M;
1412                         break;
1413                 case 0x400000:
1414                         ui32ScaleSize = 10;
1415                         pMMUHeap->ui32PDEPageSizeCtrl = SGX_MMU_PDE_PAGE_SIZE_4M;
1416                         break;
1417 #endif
1418                 default:
1419                         PVR_DPF((PVR_DBG_ERROR, "MMU_Create: invalid data page size"));
1420                         goto ErrorFreeHeap;
1421         }
1422
1423
1424         pMMUHeap->ui32DataPageSize = psDevArena->ui32DataPageSize;
1425         pMMUHeap->ui32DataPageBitWidth = SGX_MMU_PAGE_SHIFT + ui32ScaleSize;
1426         pMMUHeap->ui32DataPageMask = pMMUHeap->ui32DataPageSize - 1;
1427
1428         pMMUHeap->ui32PTShift = pMMUHeap->ui32DataPageBitWidth;
1429         pMMUHeap->ui32PTBitWidth = SGX_MMU_PT_SHIFT - ui32ScaleSize;
1430         pMMUHeap->ui32PTMask = SGX_MMU_PT_MASK & (SGX_MMU_PT_MASK<<ui32ScaleSize);
1431         pMMUHeap->ui32PTSize = (1UL<<pMMUHeap->ui32PTBitWidth) * sizeof(IMG_UINT32);
1432
1433         if(pMMUHeap->ui32PTSize < 4 * sizeof(IMG_UINT32))
1434         {
1435                 pMMUHeap->ui32PTSize = 4 * sizeof(IMG_UINT32);
1436         }
1437         pMMUHeap->ui32PTECount = pMMUHeap->ui32PTSize >> 2;
1438
1439
1440         pMMUHeap->ui32PDShift = pMMUHeap->ui32PTBitWidth + pMMUHeap->ui32PTShift;
1441         pMMUHeap->ui32PDBitWidth = SGX_FEATURE_ADDRESS_SPACE_SIZE - pMMUHeap->ui32PTBitWidth - pMMUHeap->ui32DataPageBitWidth;
1442         pMMUHeap->ui32PDMask = SGX_MMU_PD_MASK & (SGX_MMU_PD_MASK>>(32-SGX_FEATURE_ADDRESS_SPACE_SIZE));
1443
1444
1445
1446
1447
1448         if(psDevArena->BaseDevVAddr.uiAddr > (pMMUHeap->ui32DataPageMask | pMMUHeap->ui32PTMask))
1449         {
1450
1451
1452
1453                 PVR_ASSERT ((psDevArena->BaseDevVAddr.uiAddr
1454                                                 & (pMMUHeap->ui32DataPageMask
1455                                                         | pMMUHeap->ui32PTMask)) == 0);
1456         }
1457
1458
1459         pMMUHeap->ui32PTETotal = pMMUHeap->psDevArena->ui32Size >> pMMUHeap->ui32PTShift;
1460
1461
1462         pMMUHeap->ui32PDBaseIndex = (pMMUHeap->psDevArena->BaseDevVAddr.uiAddr & pMMUHeap->ui32PDMask) >> pMMUHeap->ui32PDShift;
1463
1464
1465
1466
1467         pMMUHeap->ui32PageTableCount = (pMMUHeap->ui32PTETotal + pMMUHeap->ui32PTECount - 1)
1468                                                                                 >> pMMUHeap->ui32PTBitWidth;
1469
1470
1471         pMMUHeap->psVMArena = RA_Create(psDevArena->pszName,
1472                                                                         psDevArena->BaseDevVAddr.uiAddr,
1473                                                                         psDevArena->ui32Size,
1474                                                                         IMG_NULL,
1475                                                                         pMMUHeap->ui32DataPageSize,
1476                                                                         IMG_NULL,
1477                                                                         IMG_NULL,
1478                                                                         MMU_FreePageTables,
1479                                                                         pMMUHeap);
1480
1481         if (pMMUHeap->psVMArena == IMG_NULL)
1482         {
1483                 PVR_DPF((PVR_DBG_ERROR, "MMU_Create: ERROR call to RA_Create failed"));
1484                 goto ErrorFreePagetables;
1485         }
1486
1487 #if 0
1488
1489         if(psDevArena->ui32HeapID == SGX_TILED_HEAP_ID)
1490         {
1491                 IMG_UINT32 ui32RegVal;
1492                 IMG_UINT32 ui32XTileStride;
1493
1494
1495
1496
1497
1498
1499                 ui32XTileStride = 2;
1500
1501                 ui32RegVal = (EUR_CR_BIF_TILE0_MIN_ADDRESS_MASK
1502                                                 & ((psDevArena->BaseDevVAddr.uiAddr>>20)
1503                                                 << EUR_CR_BIF_TILE0_MIN_ADDRESS_SHIFT))
1504                                         |(EUR_CR_BIF_TILE0_MAX_ADDRESS_MASK
1505                                                 & (((psDevArena->BaseDevVAddr.uiAddr+psDevArena->ui32Size)>>20)
1506                                                 << EUR_CR_BIF_TILE0_MAX_ADDRESS_SHIFT))
1507                                         |(EUR_CR_BIF_TILE0_CFG_MASK
1508                                                 & (((ui32XTileStride<<1)|8) << EUR_CR_BIF_TILE0_CFG_SHIFT));
1509                 PDUMPREG(EUR_CR_BIF_TILE0, ui32RegVal);
1510         }
1511 #endif
1512
1513
1514
1515         *ppsVMArena = pMMUHeap->psVMArena;
1516
1517         return pMMUHeap;
1518
1519
1520 ErrorFreePagetables:
1521         _DeferredFreePageTables (pMMUHeap);
1522
1523 ErrorFreeHeap:
1524         OSFreeMem (PVRSRV_OS_PAGEABLE_HEAP, sizeof(MMU_HEAP), pMMUHeap, IMG_NULL);
1525
1526
1527         return IMG_NULL;
1528 }
1529
1530 IMG_VOID
1531 MMU_Delete (MMU_HEAP *pMMUHeap)
1532 {
1533         if (pMMUHeap != IMG_NULL)
1534         {
1535                 PVR_DPF ((PVR_DBG_MESSAGE, "MMU_Delete"));
1536
1537                 if(pMMUHeap->psVMArena)
1538                 {
1539                         RA_Delete (pMMUHeap->psVMArena);
1540                 }
1541
1542 #ifdef SUPPORT_SGX_MMU_BYPASS
1543                 EnableHostAccess(pMMUHeap->psMMUContext);
1544 #endif
1545                 _DeferredFreePageTables (pMMUHeap);
1546 #ifdef SUPPORT_SGX_MMU_BYPASS
1547                 DisableHostAccess(pMMUHeap->psMMUContext);
1548 #endif
1549
1550                 OSFreeMem (PVRSRV_OS_PAGEABLE_HEAP, sizeof(MMU_HEAP), pMMUHeap, IMG_NULL);
1551
1552         }
1553 }
1554
1555 IMG_BOOL
1556 MMU_Alloc (MMU_HEAP *pMMUHeap,
1557                    IMG_SIZE_T uSize,
1558                    IMG_SIZE_T *pActualSize,
1559                    IMG_UINT32 uFlags,
1560                    IMG_UINT32 uDevVAddrAlignment,
1561                    IMG_DEV_VIRTADDR *psDevVAddr)
1562 {
1563         IMG_BOOL bStatus;
1564
1565         PVR_DPF ((PVR_DBG_MESSAGE,
1566                 "MMU_Alloc: uSize=0x%x, flags=0x%x, align=0x%x",
1567                 uSize, uFlags, uDevVAddrAlignment));
1568
1569
1570
1571         if((uFlags & PVRSRV_MEM_USER_SUPPLIED_DEVVADDR) == 0)
1572         {
1573                 IMG_UINTPTR_T uiAddr;
1574
1575                 bStatus = RA_Alloc (pMMUHeap->psVMArena,
1576                                                         uSize,
1577                                                         pActualSize,
1578                                                         IMG_NULL,
1579                                                         0,
1580                                                         uDevVAddrAlignment,
1581                                                         0,
1582                                                         &uiAddr);
1583                 if(!bStatus)
1584                 {
1585                         PVR_DPF((PVR_DBG_ERROR,"MMU_Alloc: RA_Alloc of VMArena failed"));
1586                         return bStatus;
1587                 }
1588
1589                 psDevVAddr->uiAddr = IMG_CAST_TO_DEVVADDR_UINT(uiAddr);
1590         }
1591
1592         #ifdef SUPPORT_SGX_MMU_BYPASS
1593         EnableHostAccess(pMMUHeap->psMMUContext);
1594         #endif
1595
1596
1597         bStatus = _DeferredAllocPagetables(pMMUHeap, *psDevVAddr, uSize);
1598
1599         #ifdef SUPPORT_SGX_MMU_BYPASS
1600         DisableHostAccess(pMMUHeap->psMMUContext);
1601         #endif
1602
1603         if (!bStatus)
1604         {
1605                 PVR_DPF((PVR_DBG_ERROR,"MMU_Alloc: _DeferredAllocPagetables failed"));
1606                 if((uFlags & PVRSRV_MEM_USER_SUPPLIED_DEVVADDR) == 0)
1607                 {
1608
1609                         RA_Free (pMMUHeap->psVMArena, psDevVAddr->uiAddr, IMG_FALSE);
1610                 }
1611         }
1612
1613         return bStatus;
1614 }
1615
1616 IMG_VOID
1617 MMU_Free (MMU_HEAP *pMMUHeap, IMG_DEV_VIRTADDR DevVAddr, IMG_UINT32 ui32Size)
1618 {
1619         PVR_ASSERT (pMMUHeap != IMG_NULL);
1620
1621         if (pMMUHeap == IMG_NULL)
1622         {
1623                 PVR_DPF((PVR_DBG_ERROR, "MMU_Free: invalid parameter"));
1624                 return;
1625         }
1626
1627         PVR_DPF ((PVR_DBG_MESSAGE,
1628                 "MMU_Free: mmu=%08X, dev_vaddr=%08X", pMMUHeap, DevVAddr.uiAddr));
1629
1630         if((DevVAddr.uiAddr >= pMMUHeap->psDevArena->BaseDevVAddr.uiAddr) &&
1631                 (DevVAddr.uiAddr + ui32Size <= pMMUHeap->psDevArena->BaseDevVAddr.uiAddr + pMMUHeap->psDevArena->ui32Size))
1632         {
1633                 RA_Free (pMMUHeap->psVMArena, DevVAddr.uiAddr, IMG_TRUE);
1634                 return;
1635         }
1636
1637         PVR_DPF((PVR_DBG_ERROR,"MMU_Free: Couldn't find DevVAddr %08X in a DevArena",DevVAddr.uiAddr));
1638 }
1639
1640 IMG_VOID
1641 MMU_Enable (MMU_HEAP *pMMUHeap)
1642 {
1643         PVR_UNREFERENCED_PARAMETER(pMMUHeap);
1644
1645 }
1646
1647 IMG_VOID
1648 MMU_Disable (MMU_HEAP *pMMUHeap)
1649 {
1650         PVR_UNREFERENCED_PARAMETER(pMMUHeap);
1651
1652 }
1653
1654 #if defined(PDUMP)
1655 static IMG_VOID
1656 MMU_PDumpPageTables     (MMU_HEAP *pMMUHeap,
1657                                          IMG_DEV_VIRTADDR DevVAddr,
1658                                          IMG_SIZE_T uSize,
1659                                          IMG_BOOL bForUnmap,
1660                                          IMG_HANDLE hUniqueTag)
1661 {
1662         IMG_UINT32      ui32NumPTEntries;
1663         IMG_UINT32      ui32PTIndex;
1664         IMG_UINT32      *pui32PTEntry;
1665
1666         MMU_PT_INFO **ppsPTInfoList;
1667         IMG_UINT32 ui32PDIndex;
1668         IMG_UINT32 ui32PTDumpCount;
1669
1670
1671         ui32NumPTEntries = (uSize + pMMUHeap->ui32DataPageMask) >> pMMUHeap->ui32PTShift;
1672
1673
1674         ui32PDIndex = DevVAddr.uiAddr >> pMMUHeap->ui32PDShift;
1675
1676
1677         ppsPTInfoList = &pMMUHeap->psMMUContext->apsPTInfoList[ui32PDIndex];
1678
1679
1680         ui32PTIndex = (DevVAddr.uiAddr & pMMUHeap->ui32PTMask) >> pMMUHeap->ui32PTShift;
1681
1682
1683         PDUMPCOMMENT("Page table mods (num entries == %08X) %s", ui32NumPTEntries, bForUnmap ? "(for unmap)" : "");
1684
1685
1686         while(ui32NumPTEntries > 0)
1687         {
1688                 MMU_PT_INFO* psPTInfo = *ppsPTInfoList++;
1689
1690                 if(ui32NumPTEntries <= pMMUHeap->ui32PTECount - ui32PTIndex)
1691                 {
1692                         ui32PTDumpCount = ui32NumPTEntries;
1693                 }
1694                 else
1695                 {
1696                         ui32PTDumpCount = pMMUHeap->ui32PTECount - ui32PTIndex;
1697                 }
1698
1699                 if (psPTInfo)
1700                 {
1701                         pui32PTEntry = (IMG_UINT32*)psPTInfo->PTPageCpuVAddr;
1702                         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, (IMG_VOID *) &pui32PTEntry[ui32PTIndex], ui32PTDumpCount * sizeof(IMG_UINT32), 0, IMG_FALSE, PDUMP_PT_UNIQUETAG, hUniqueTag);
1703                 }
1704
1705
1706                 ui32NumPTEntries -= ui32PTDumpCount;
1707
1708
1709                 ui32PTIndex = 0;
1710         }
1711
1712         PDUMPCOMMENT("Finished page table mods %s", bForUnmap ? "(for unmap)" : "");
1713 }
1714 #endif
1715
1716
1717 static IMG_VOID
1718 MMU_MapPage (MMU_HEAP *pMMUHeap,
1719                          IMG_DEV_VIRTADDR DevVAddr,
1720                          IMG_DEV_PHYADDR DevPAddr,
1721                          IMG_UINT32 ui32MemFlags)
1722 {
1723         IMG_UINT32 ui32Index;
1724         IMG_UINT32 *pui32Tmp;
1725         IMG_UINT32 ui32MMUFlags = 0;
1726         MMU_PT_INFO **ppsPTInfoList;
1727
1728
1729         PVR_ASSERT((DevPAddr.uiAddr & pMMUHeap->ui32DataPageMask) == 0);
1730
1731
1732
1733         if(((PVRSRV_MEM_READ|PVRSRV_MEM_WRITE) & ui32MemFlags) == (PVRSRV_MEM_READ|PVRSRV_MEM_WRITE))
1734         {
1735
1736                 ui32MMUFlags = 0;
1737         }
1738         else if(PVRSRV_MEM_READ & ui32MemFlags)
1739         {
1740
1741                 ui32MMUFlags |= SGX_MMU_PTE_READONLY;
1742         }
1743         else if(PVRSRV_MEM_WRITE & ui32MemFlags)
1744         {
1745
1746                 ui32MMUFlags |= SGX_MMU_PTE_WRITEONLY;
1747         }
1748
1749
1750         if(PVRSRV_MEM_CACHE_CONSISTENT & ui32MemFlags)
1751         {
1752                 ui32MMUFlags |= SGX_MMU_PTE_CACHECONSISTENT;
1753         }
1754
1755 #if !defined(FIX_HW_BRN_25503)
1756
1757         if(PVRSRV_MEM_EDM_PROTECT & ui32MemFlags)
1758         {
1759                 ui32MMUFlags |= SGX_MMU_PTE_EDMPROTECT;
1760         }
1761 #endif
1762
1763
1764
1765
1766
1767         ui32Index = DevVAddr.uiAddr >> pMMUHeap->ui32PDShift;
1768
1769
1770         ppsPTInfoList = &pMMUHeap->psMMUContext->apsPTInfoList[ui32Index];
1771
1772         CheckPT(ppsPTInfoList[0]);
1773
1774
1775         ui32Index = (DevVAddr.uiAddr & pMMUHeap->ui32PTMask) >> pMMUHeap->ui32PTShift;
1776
1777
1778         pui32Tmp = (IMG_UINT32*)ppsPTInfoList[0]->PTPageCpuVAddr;
1779
1780 #if !defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
1781
1782         if (pui32Tmp[ui32Index] & SGX_MMU_PTE_VALID)
1783         {
1784                 PVR_DPF((PVR_DBG_ERROR, "MMU_MapPage: Page is already valid for alloc at VAddr:0x%08lX PDIdx:%u PTIdx:%u",
1785                                                                 DevVAddr.uiAddr,
1786                                                                 DevVAddr.uiAddr >> pMMUHeap->ui32PDShift,
1787                                                                 ui32Index ));
1788                 PVR_DPF((PVR_DBG_ERROR, "MMU_MapPage: Page table entry value: 0x%08lX", pui32Tmp[ui32Index]));
1789                 PVR_DPF((PVR_DBG_ERROR, "MMU_MapPage: Physical page to map: 0x%08lX", DevPAddr.uiAddr));
1790         }
1791
1792         PVR_ASSERT((pui32Tmp[ui32Index] & SGX_MMU_PTE_VALID) == 0);
1793 #endif
1794
1795
1796         ppsPTInfoList[0]->ui32ValidPTECount++;
1797
1798
1799         pui32Tmp[ui32Index] = ((DevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
1800                                                 & ((~pMMUHeap->ui32DataPageMask)>>SGX_MMU_PTE_ADDR_ALIGNSHIFT))
1801                                                 | SGX_MMU_PTE_VALID
1802                                                 | ui32MMUFlags;
1803
1804         CheckPT(ppsPTInfoList[0]);
1805 }
1806
1807
1808 IMG_VOID
1809 MMU_MapScatter (MMU_HEAP *pMMUHeap,
1810                                 IMG_DEV_VIRTADDR DevVAddr,
1811                                 IMG_SYS_PHYADDR *psSysAddr,
1812                                 IMG_SIZE_T uSize,
1813                                 IMG_UINT32 ui32MemFlags,
1814                                 IMG_HANDLE hUniqueTag)
1815 {
1816 #if defined(PDUMP)
1817         IMG_DEV_VIRTADDR MapBaseDevVAddr;
1818 #endif
1819         IMG_UINT32 uCount, i;
1820         IMG_DEV_PHYADDR DevPAddr;
1821
1822         PVR_ASSERT (pMMUHeap != IMG_NULL);
1823
1824 #if defined(PDUMP)
1825         MapBaseDevVAddr = DevVAddr;
1826 #else
1827         PVR_UNREFERENCED_PARAMETER(hUniqueTag);
1828 #endif
1829
1830         for (i=0, uCount=0; uCount<uSize; i++, uCount+=pMMUHeap->ui32DataPageSize)
1831         {
1832                 IMG_SYS_PHYADDR sSysAddr;
1833
1834                 sSysAddr = psSysAddr[i];
1835
1836
1837
1838                 PVR_ASSERT((sSysAddr.uiAddr & pMMUHeap->ui32DataPageMask) == 0);
1839
1840                 DevPAddr = SysSysPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sSysAddr);
1841
1842                 MMU_MapPage (pMMUHeap, DevVAddr, DevPAddr, ui32MemFlags);
1843                 DevVAddr.uiAddr += pMMUHeap->ui32DataPageSize;
1844
1845                 PVR_DPF ((PVR_DBG_MESSAGE,
1846                                  "MMU_MapScatter: devVAddr=%08X, SysAddr=%08X, size=0x%x/0x%x",
1847                                   DevVAddr.uiAddr, sSysAddr.uiAddr, uCount, uSize));
1848         }
1849
1850 #if defined(PDUMP)
1851         MMU_PDumpPageTables (pMMUHeap, MapBaseDevVAddr, uSize, IMG_FALSE, hUniqueTag);
1852 #endif
1853 }
1854
1855 IMG_VOID
1856 MMU_MapPages (MMU_HEAP *pMMUHeap,
1857                           IMG_DEV_VIRTADDR DevVAddr,
1858                           IMG_SYS_PHYADDR SysPAddr,
1859                           IMG_SIZE_T uSize,
1860                           IMG_UINT32 ui32MemFlags,
1861                           IMG_HANDLE hUniqueTag)
1862 {
1863         IMG_DEV_PHYADDR DevPAddr;
1864 #if defined(PDUMP)
1865         IMG_DEV_VIRTADDR MapBaseDevVAddr;
1866 #endif
1867         IMG_UINT32 uCount;
1868         IMG_UINT32 ui32VAdvance;
1869         IMG_UINT32 ui32PAdvance;
1870
1871         PVR_ASSERT (pMMUHeap != IMG_NULL);
1872
1873         PVR_DPF ((PVR_DBG_MESSAGE,
1874                   "MMU_MapPages: mmu=%08X, devVAddr=%08X, SysPAddr=%08X, size=0x%x",
1875                   pMMUHeap, DevVAddr.uiAddr, SysPAddr.uiAddr, uSize));
1876
1877
1878         ui32VAdvance = pMMUHeap->ui32DataPageSize;
1879         ui32PAdvance = pMMUHeap->ui32DataPageSize;
1880
1881 #if defined(PDUMP)
1882         MapBaseDevVAddr = DevVAddr;
1883 #else
1884         PVR_UNREFERENCED_PARAMETER(hUniqueTag);
1885 #endif
1886
1887         DevPAddr = SysSysPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, SysPAddr);
1888
1889
1890         PVR_ASSERT((DevPAddr.uiAddr & pMMUHeap->ui32DataPageMask) == 0);
1891
1892 #if defined(FIX_HW_BRN_23281)
1893         if(ui32MemFlags & PVRSRV_MEM_INTERLEAVED)
1894         {
1895                 ui32VAdvance *= 2;
1896         }
1897 #endif
1898
1899
1900
1901
1902         if(ui32MemFlags & PVRSRV_MEM_DUMMY)
1903         {
1904                 ui32PAdvance = 0;
1905         }
1906
1907         for (uCount=0; uCount<uSize; uCount+=ui32VAdvance)
1908         {
1909                 MMU_MapPage (pMMUHeap, DevVAddr, DevPAddr, ui32MemFlags);
1910                 DevVAddr.uiAddr += ui32VAdvance;
1911                 DevPAddr.uiAddr += ui32PAdvance;
1912         }
1913
1914 #if defined(PDUMP)
1915         MMU_PDumpPageTables (pMMUHeap, MapBaseDevVAddr, uSize, IMG_FALSE, hUniqueTag);
1916 #endif
1917 }
1918
1919 IMG_VOID
1920 MMU_MapShadow (MMU_HEAP          *pMMUHeap,
1921                            IMG_DEV_VIRTADDR   MapBaseDevVAddr,
1922                            IMG_SIZE_T         uByteSize,
1923                            IMG_CPU_VIRTADDR   CpuVAddr,
1924                            IMG_HANDLE         hOSMemHandle,
1925                            IMG_DEV_VIRTADDR  *pDevVAddr,
1926                            IMG_UINT32         ui32MemFlags,
1927                            IMG_HANDLE         hUniqueTag)
1928 {
1929         IMG_UINT32                      i;
1930         IMG_UINT32                      uOffset = 0;
1931         IMG_DEV_VIRTADDR        MapDevVAddr;
1932         IMG_UINT32                      ui32VAdvance;
1933         IMG_UINT32                      ui32PAdvance;
1934
1935 #if !defined (PDUMP)
1936         PVR_UNREFERENCED_PARAMETER(hUniqueTag);
1937 #endif
1938
1939         PVR_DPF ((PVR_DBG_MESSAGE,
1940                         "MMU_MapShadow: %08X, 0x%x, %08X",
1941                         MapBaseDevVAddr.uiAddr,
1942                         uByteSize,
1943                         CpuVAddr));
1944
1945
1946         ui32VAdvance = pMMUHeap->ui32DataPageSize;
1947         ui32PAdvance = pMMUHeap->ui32DataPageSize;
1948
1949
1950         PVR_ASSERT(((IMG_UINT32)CpuVAddr & (SGX_MMU_PAGE_SIZE - 1)) == 0);
1951         PVR_ASSERT(((IMG_UINT32)uByteSize & pMMUHeap->ui32DataPageMask) == 0);
1952         pDevVAddr->uiAddr = MapBaseDevVAddr.uiAddr;
1953
1954 #if defined(FIX_HW_BRN_23281)
1955         if(ui32MemFlags & PVRSRV_MEM_INTERLEAVED)
1956         {
1957                 ui32VAdvance *= 2;
1958         }
1959 #endif
1960
1961
1962
1963
1964         if(ui32MemFlags & PVRSRV_MEM_DUMMY)
1965         {
1966                 ui32PAdvance = 0;
1967         }
1968
1969
1970         MapDevVAddr = MapBaseDevVAddr;
1971         for (i=0; i<uByteSize; i+=ui32VAdvance)
1972         {
1973                 IMG_CPU_PHYADDR CpuPAddr;
1974                 IMG_DEV_PHYADDR DevPAddr;
1975
1976                 if(CpuVAddr)
1977                 {
1978                         CpuPAddr = OSMapLinToCPUPhys ((IMG_VOID *)((IMG_UINT32)CpuVAddr + uOffset));
1979                 }
1980                 else
1981                 {
1982                         CpuPAddr = OSMemHandleToCpuPAddr(hOSMemHandle, uOffset);
1983                 }
1984                 DevPAddr = SysCpuPAddrToDevPAddr (PVRSRV_DEVICE_TYPE_SGX, CpuPAddr);
1985
1986
1987                 PVR_ASSERT((DevPAddr.uiAddr & pMMUHeap->ui32DataPageMask) == 0);
1988
1989                 PVR_DPF ((PVR_DBG_MESSAGE,
1990                                 "0x%x: CpuVAddr=%08X, CpuPAddr=%08X, DevVAddr=%08X, DevPAddr=%08X",
1991                                 uOffset,
1992                                 (IMG_UINTPTR_T)CpuVAddr + uOffset,
1993                                 CpuPAddr.uiAddr,
1994                                 MapDevVAddr.uiAddr,
1995                                 DevPAddr.uiAddr));
1996
1997                 MMU_MapPage (pMMUHeap, MapDevVAddr, DevPAddr, ui32MemFlags);
1998
1999
2000                 MapDevVAddr.uiAddr += ui32VAdvance;
2001                 uOffset += ui32PAdvance;
2002         }
2003
2004 #if defined(PDUMP)
2005         MMU_PDumpPageTables (pMMUHeap, MapBaseDevVAddr, uByteSize, IMG_FALSE, hUniqueTag);
2006 #endif
2007 }
2008
2009
2010 IMG_VOID
2011 MMU_UnmapPages (MMU_HEAP *psMMUHeap,
2012                                 IMG_DEV_VIRTADDR sDevVAddr,
2013                                 IMG_UINT32 ui32PageCount,
2014                                 IMG_HANDLE hUniqueTag)
2015 {
2016         IMG_UINT32                      uPageSize = psMMUHeap->ui32DataPageSize;
2017         IMG_DEV_VIRTADDR        sTmpDevVAddr;
2018         IMG_UINT32                      i;
2019         IMG_UINT32                      ui32PDIndex;
2020         IMG_UINT32                      ui32PTIndex;
2021         IMG_UINT32                      *pui32Tmp;
2022
2023 #if !defined (PDUMP)
2024         PVR_UNREFERENCED_PARAMETER(hUniqueTag);
2025 #endif
2026
2027
2028         sTmpDevVAddr = sDevVAddr;
2029
2030         for(i=0; i<ui32PageCount; i++)
2031         {
2032                 MMU_PT_INFO **ppsPTInfoList;
2033
2034
2035                 ui32PDIndex = sTmpDevVAddr.uiAddr >> psMMUHeap->ui32PDShift;
2036
2037
2038                 ppsPTInfoList = &psMMUHeap->psMMUContext->apsPTInfoList[ui32PDIndex];
2039
2040
2041                 ui32PTIndex = (sTmpDevVAddr.uiAddr & psMMUHeap->ui32PTMask) >> psMMUHeap->ui32PTShift;
2042
2043
2044                 if (!ppsPTInfoList[0])
2045                 {
2046                         PVR_DPF((PVR_DBG_ERROR, "MMU_UnmapPages: ERROR Invalid PT for alloc at VAddr:0x%08lX (VaddrIni:0x%08lX AllocPage:%u) PDIdx:%u PTIdx:%u",
2047                                                                         sTmpDevVAddr.uiAddr,
2048                                                                         sDevVAddr.uiAddr,
2049                                                                         i,
2050                                                                         ui32PDIndex,
2051                                                                         ui32PTIndex));
2052
2053
2054                         sTmpDevVAddr.uiAddr += uPageSize;
2055
2056
2057                         continue;
2058                 }
2059
2060                 CheckPT(ppsPTInfoList[0]);
2061
2062
2063                 pui32Tmp = (IMG_UINT32*)ppsPTInfoList[0]->PTPageCpuVAddr;
2064
2065
2066                 if (pui32Tmp[ui32PTIndex] & SGX_MMU_PTE_VALID)
2067                 {
2068                         ppsPTInfoList[0]->ui32ValidPTECount--;
2069                 }
2070                 else
2071                 {
2072                         PVR_DPF((PVR_DBG_ERROR, "MMU_UnmapPages: Page is already invalid for alloc at VAddr:0x%08lX (VAddrIni:0x%08lX AllocPage:%u) PDIdx:%u PTIdx:%u",
2073                                                                         sTmpDevVAddr.uiAddr,
2074                                                                         sDevVAddr.uiAddr,
2075                                                                         i,
2076                                                                         ui32PDIndex,
2077                                                                         ui32PTIndex));
2078                         PVR_DPF((PVR_DBG_ERROR, "MMU_UnmapPages: Page table entry value: 0x%08lX", pui32Tmp[ui32PTIndex]));
2079                 }
2080
2081
2082                 PVR_ASSERT((IMG_INT32)ppsPTInfoList[0]->ui32ValidPTECount >= 0);
2083
2084 #if defined(SUPPORT_SGX_MMU_DUMMY_PAGE)
2085
2086                 pui32Tmp[ui32PTIndex] = (psMMUHeap->psMMUContext->psDevInfo->sDummyDataDevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
2087                                                                 | SGX_MMU_PTE_VALID;
2088 #else
2089
2090                 pui32Tmp[ui32PTIndex] = 0;
2091 #endif
2092
2093                 CheckPT(ppsPTInfoList[0]);
2094
2095
2096                 sTmpDevVAddr.uiAddr += uPageSize;
2097         }
2098
2099         MMU_InvalidatePageTableCache(psMMUHeap->psMMUContext->psDevInfo);
2100
2101 #if defined(PDUMP)
2102         MMU_PDumpPageTables (psMMUHeap, sDevVAddr, uPageSize*ui32PageCount, IMG_TRUE, hUniqueTag);
2103 #endif
2104 }
2105
2106
2107 IMG_DEV_PHYADDR
2108 MMU_GetPhysPageAddr(MMU_HEAP *pMMUHeap, IMG_DEV_VIRTADDR sDevVPageAddr)
2109 {
2110         IMG_UINT32 *pui32PageTable;
2111         IMG_UINT32 ui32Index;
2112         IMG_DEV_PHYADDR sDevPAddr;
2113         MMU_PT_INFO **ppsPTInfoList;
2114
2115
2116         ui32Index = sDevVPageAddr.uiAddr >> pMMUHeap->ui32PDShift;
2117
2118
2119         ppsPTInfoList = &pMMUHeap->psMMUContext->apsPTInfoList[ui32Index];
2120         if (!ppsPTInfoList[0])
2121         {
2122                 PVR_DPF((PVR_DBG_ERROR,"MMU_GetPhysPageAddr: Not mapped in at 0x%08x", sDevVPageAddr.uiAddr));
2123                 sDevPAddr.uiAddr = 0;
2124                 return sDevPAddr;
2125         }
2126
2127
2128         ui32Index = (sDevVPageAddr.uiAddr & pMMUHeap->ui32PTMask) >> pMMUHeap->ui32PTShift;
2129
2130
2131         pui32PageTable = (IMG_UINT32*)ppsPTInfoList[0]->PTPageCpuVAddr;
2132
2133
2134         sDevPAddr.uiAddr = pui32PageTable[ui32Index];
2135
2136
2137         sDevPAddr.uiAddr &= ~(pMMUHeap->ui32DataPageMask>>SGX_MMU_PTE_ADDR_ALIGNSHIFT);
2138
2139
2140         sDevPAddr.uiAddr <<= SGX_MMU_PTE_ADDR_ALIGNSHIFT;
2141
2142         return sDevPAddr;
2143 }
2144
2145
2146 IMG_DEV_PHYADDR MMU_GetPDDevPAddr(MMU_CONTEXT *pMMUContext)
2147 {
2148         return (pMMUContext->sPDDevPAddr);
2149 }
2150
2151
2152 IMG_EXPORT
2153 PVRSRV_ERROR SGXGetPhysPageAddrKM (IMG_HANDLE hDevMemHeap,
2154                                                                    IMG_DEV_VIRTADDR sDevVAddr,
2155                                                                    IMG_DEV_PHYADDR *pDevPAddr,
2156                                                                    IMG_CPU_PHYADDR *pCpuPAddr)
2157 {
2158         MMU_HEAP *pMMUHeap;
2159         IMG_DEV_PHYADDR DevPAddr;
2160
2161
2162
2163         pMMUHeap = (MMU_HEAP*)BM_GetMMUHeap(hDevMemHeap);
2164
2165         DevPAddr = MMU_GetPhysPageAddr(pMMUHeap, sDevVAddr);
2166         pCpuPAddr->uiAddr = DevPAddr.uiAddr;
2167         pDevPAddr->uiAddr = DevPAddr.uiAddr;
2168
2169         return (pDevPAddr->uiAddr != 0) ? PVRSRV_OK : PVRSRV_ERROR_INVALID_PARAMS;
2170 }
2171
2172
2173 PVRSRV_ERROR SGXGetMMUPDAddrKM(IMG_HANDLE               hDevCookie,
2174                                                                 IMG_HANDLE              hDevMemContext,
2175                                                                 IMG_DEV_PHYADDR *psPDDevPAddr)
2176 {
2177         if (!hDevCookie || !hDevMemContext || !psPDDevPAddr)
2178         {
2179                 return PVRSRV_ERROR_INVALID_PARAMS;
2180         }
2181
2182
2183         *psPDDevPAddr = ((BM_CONTEXT*)hDevMemContext)->psMMUContext->sPDDevPAddr;
2184
2185         return PVRSRV_OK;
2186 }
2187
2188 PVRSRV_ERROR MMU_BIFResetPDAlloc(PVRSRV_SGXDEV_INFO *psDevInfo)
2189 {
2190         PVRSRV_ERROR eError;
2191         SYS_DATA *psSysData;
2192         RA_ARENA *psLocalDevMemArena;
2193         IMG_HANDLE hOSMemHandle = IMG_NULL;
2194         IMG_BYTE *pui8MemBlock = IMG_NULL;
2195         IMG_SYS_PHYADDR sMemBlockSysPAddr;
2196         IMG_CPU_PHYADDR sMemBlockCpuPAddr;
2197
2198         SysAcquireData(&psSysData);
2199
2200         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2201
2202
2203         if(psLocalDevMemArena == IMG_NULL)
2204         {
2205
2206                 eError = OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2207                                                       3 * SGX_MMU_PAGE_SIZE,
2208                                                       SGX_MMU_PAGE_SIZE,
2209                                                       (IMG_VOID **)&pui8MemBlock,
2210                                                       &hOSMemHandle);
2211                 if (eError != PVRSRV_OK)
2212                 {
2213                         PVR_DPF((PVR_DBG_ERROR, "MMU_BIFResetPDAlloc: ERROR call to OSAllocPages failed"));
2214                         return eError;
2215                 }
2216
2217
2218                 if(pui8MemBlock)
2219                 {
2220                         sMemBlockCpuPAddr = OSMapLinToCPUPhys(pui8MemBlock);
2221                 }
2222                 else
2223                 {
2224
2225                         sMemBlockCpuPAddr = OSMemHandleToCpuPAddr(hOSMemHandle, 0);
2226                 }
2227         }
2228         else
2229         {
2230
2231
2232                 if(RA_Alloc(psLocalDevMemArena,
2233                                         3 * SGX_MMU_PAGE_SIZE,
2234                                         IMG_NULL,
2235                                         IMG_NULL,
2236                                         0,
2237                                         SGX_MMU_PAGE_SIZE,
2238                                         0,
2239                                         &(sMemBlockSysPAddr.uiAddr)) != IMG_TRUE)
2240                 {
2241                         PVR_DPF((PVR_DBG_ERROR, "MMU_BIFResetPDAlloc: ERROR call to RA_Alloc failed"));
2242                         return PVRSRV_ERROR_OUT_OF_MEMORY;
2243                 }
2244
2245
2246                 sMemBlockCpuPAddr = SysSysPAddrToCpuPAddr(sMemBlockSysPAddr);
2247                 pui8MemBlock = OSMapPhysToLin(sMemBlockCpuPAddr,
2248                                                                           SGX_MMU_PAGE_SIZE * 3,
2249                                                                           PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2250                                                                           &hOSMemHandle);
2251                 if(!pui8MemBlock)
2252                 {
2253                         PVR_DPF((PVR_DBG_ERROR, "MMU_BIFResetPDAlloc: ERROR failed to map page tables"));
2254                         return PVRSRV_ERROR_BAD_MAPPING;
2255                 }
2256         }
2257
2258         psDevInfo->hBIFResetPDOSMemHandle = hOSMemHandle;
2259         psDevInfo->sBIFResetPDDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sMemBlockCpuPAddr);
2260         psDevInfo->sBIFResetPTDevPAddr.uiAddr = psDevInfo->sBIFResetPDDevPAddr.uiAddr + SGX_MMU_PAGE_SIZE;
2261         psDevInfo->sBIFResetPageDevPAddr.uiAddr = psDevInfo->sBIFResetPTDevPAddr.uiAddr + SGX_MMU_PAGE_SIZE;
2262
2263
2264         psDevInfo->pui32BIFResetPD = (IMG_UINT32 *)pui8MemBlock;
2265         psDevInfo->pui32BIFResetPT = (IMG_UINT32 *)(pui8MemBlock + SGX_MMU_PAGE_SIZE);
2266
2267
2268         OSMemSet(psDevInfo->pui32BIFResetPD, 0, SGX_MMU_PAGE_SIZE);
2269         OSMemSet(psDevInfo->pui32BIFResetPT, 0, SGX_MMU_PAGE_SIZE);
2270
2271         OSMemSet(pui8MemBlock + (2 * SGX_MMU_PAGE_SIZE), 0xDB, SGX_MMU_PAGE_SIZE);
2272
2273         return PVRSRV_OK;
2274 }
2275
2276 IMG_VOID MMU_BIFResetPDFree(PVRSRV_SGXDEV_INFO *psDevInfo)
2277 {
2278         SYS_DATA *psSysData;
2279         RA_ARENA *psLocalDevMemArena;
2280         IMG_SYS_PHYADDR sPDSysPAddr;
2281
2282         SysAcquireData(&psSysData);
2283
2284         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2285
2286
2287         if(psLocalDevMemArena == IMG_NULL)
2288         {
2289                 OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2290                                         3 * SGX_MMU_PAGE_SIZE,
2291                                         psDevInfo->pui32BIFResetPD,
2292                                         psDevInfo->hBIFResetPDOSMemHandle);
2293         }
2294         else
2295         {
2296                 OSUnMapPhysToLin(psDevInfo->pui32BIFResetPD,
2297                          3 * SGX_MMU_PAGE_SIZE,
2298                          PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2299                          psDevInfo->hBIFResetPDOSMemHandle);
2300
2301                 sPDSysPAddr = SysDevPAddrToSysPAddr(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->sBIFResetPDDevPAddr);
2302                 RA_Free(psLocalDevMemArena, sPDSysPAddr.uiAddr, IMG_FALSE);
2303         }
2304 }
2305
2306
2307 #if defined(FIX_HW_BRN_22997) && defined(FIX_HW_BRN_23030) && defined(SGX_FEATURE_HOST_PORT)
2308 PVRSRV_ERROR WorkaroundBRN22997Alloc(PVRSRV_SGXDEV_INFO *psDevInfo)
2309 {
2310         PVRSRV_ERROR eError;
2311         SYS_DATA *psSysData;
2312         RA_ARENA *psLocalDevMemArena;
2313         IMG_HANDLE hPTPageOSMemHandle = IMG_NULL;
2314         IMG_HANDLE hPDPageOSMemHandle = IMG_NULL;
2315         IMG_UINT32 *pui32PD = IMG_NULL;
2316         IMG_UINT32 *pui32PT = IMG_NULL;
2317         IMG_CPU_PHYADDR sCpuPAddr;
2318         IMG_DEV_PHYADDR sPTDevPAddr;
2319         IMG_DEV_PHYADDR sPDDevPAddr;
2320
2321         SysAcquireData(&psSysData);
2322
2323         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2324
2325
2326         if(psLocalDevMemArena == IMG_NULL)
2327         {
2328
2329                 eError = OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2330                                                    SGX_MMU_PAGE_SIZE,
2331                                                    SGX_MMU_PAGE_SIZE,
2332                                                    (IMG_VOID **)&pui32PT,
2333                                                    &hPTPageOSMemHandle);
2334                 if (eError != PVRSRV_OK)
2335                 {
2336                         PVR_DPF((PVR_DBG_ERROR, "WorkaroundBRN22997: ERROR call to OSAllocPages failed"));
2337                         return eError;
2338                 }
2339
2340                 eError = OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2341                                                    SGX_MMU_PAGE_SIZE,
2342                                                    SGX_MMU_PAGE_SIZE,
2343                                                    (IMG_VOID **)&pui32PD,
2344                                                    &hPDPageOSMemHandle);
2345                 if (eError != PVRSRV_OK)
2346                 {
2347                         PVR_DPF((PVR_DBG_ERROR, "WorkaroundBRN22997: ERROR call to OSAllocPages failed"));
2348                         return eError;
2349                 }
2350
2351
2352                 if(pui32PT)
2353         {
2354             sCpuPAddr = OSMapLinToCPUPhys(pui32PT);
2355         }
2356         else
2357         {
2358
2359             sCpuPAddr = OSMemHandleToCpuPAddr(hPTPageOSMemHandle, 0);
2360         }
2361                 sPTDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
2362
2363                 if(pui32PD)
2364         {
2365             sCpuPAddr = OSMapLinToCPUPhys(pui32PD);
2366         }
2367         else
2368         {
2369
2370             sCpuPAddr = OSMemHandleToCpuPAddr(hPDPageOSMemHandle, 0);
2371         }
2372                 sPDDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
2373
2374         }
2375         else
2376         {
2377
2378
2379                 if(RA_Alloc(psLocalDevMemArena,
2380                                         SGX_MMU_PAGE_SIZE * 2,
2381                                         IMG_NULL,
2382                                         IMG_NULL,
2383                                         0,
2384                                         SGX_MMU_PAGE_SIZE,
2385                                         0,
2386                                         &(psDevInfo->sBRN22997SysPAddr.uiAddr))!= IMG_TRUE)
2387                 {
2388                         PVR_DPF((PVR_DBG_ERROR, "WorkaroundBRN22997: ERROR call to RA_Alloc failed"));
2389                         return PVRSRV_ERROR_OUT_OF_MEMORY;
2390                 }
2391
2392
2393                 sCpuPAddr = SysSysPAddrToCpuPAddr(psDevInfo->sBRN22997SysPAddr);
2394                 pui32PT = OSMapPhysToLin(sCpuPAddr,
2395                                                                 SGX_MMU_PAGE_SIZE * 2,
2396                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2397                                                                 &hPTPageOSMemHandle);
2398                 if(!pui32PT)
2399                 {
2400                         PVR_DPF((PVR_DBG_ERROR, "WorkaroundBRN22997: ERROR failed to map page tables"));
2401                         return PVRSRV_ERROR_BAD_MAPPING;
2402                 }
2403
2404
2405                 sPTDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
2406
2407                 pui32PD = pui32PT + 1024;
2408                 sPDDevPAddr.uiAddr = sPTDevPAddr.uiAddr + 4096;
2409         }
2410
2411         OSMemSet(pui32PD, 0, SGX_MMU_PAGE_SIZE);
2412         OSMemSet(pui32PT, 0, SGX_MMU_PAGE_SIZE);
2413
2414
2415         PDUMPMALLOCPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, PDUMP_PD_UNIQUETAG);
2416         PDUMPMALLOCPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
2417         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
2418         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PT_UNIQUETAG, PDUMP_PD_UNIQUETAG);
2419
2420         psDevInfo->hBRN22997PTPageOSMemHandle = hPTPageOSMemHandle;
2421         psDevInfo->hBRN22997PDPageOSMemHandle = hPDPageOSMemHandle;
2422         psDevInfo->sBRN22997PTDevPAddr = sPTDevPAddr;
2423         psDevInfo->sBRN22997PDDevPAddr = sPDDevPAddr;
2424         psDevInfo->pui32BRN22997PD = pui32PD;
2425         psDevInfo->pui32BRN22997PT = pui32PT;
2426
2427         return PVRSRV_OK;
2428 }
2429
2430
2431 IMG_VOID WorkaroundBRN22997ReadHostPort(PVRSRV_SGXDEV_INFO *psDevInfo)
2432 {
2433         IMG_UINT32 *pui32PD = psDevInfo->pui32BRN22997PD;
2434         IMG_UINT32 *pui32PT = psDevInfo->pui32BRN22997PT;
2435         IMG_UINT32 ui32PDIndex;
2436         IMG_UINT32 ui32PTIndex;
2437         IMG_DEV_VIRTADDR sDevVAddr;
2438         volatile IMG_UINT32 *pui32HostPort;
2439         IMG_UINT32 ui32BIFCtrl;
2440
2441
2442
2443
2444         pui32HostPort = (volatile IMG_UINT32*)(((IMG_UINT8*)psDevInfo->pvHostPortBaseKM) + SYS_SGX_HOSTPORT_BRN23030_OFFSET);
2445
2446
2447         sDevVAddr.uiAddr = SYS_SGX_HOSTPORT_BASE_DEVVADDR + SYS_SGX_HOSTPORT_BRN23030_OFFSET;
2448
2449         ui32PDIndex = (sDevVAddr.uiAddr & SGX_MMU_PD_MASK) >> (SGX_MMU_PAGE_SHIFT + SGX_MMU_PT_SHIFT);
2450         ui32PTIndex = (sDevVAddr.uiAddr & SGX_MMU_PT_MASK) >> SGX_MMU_PAGE_SHIFT;
2451
2452
2453         pui32PD[ui32PDIndex] = (psDevInfo->sBRN22997PTDevPAddr.uiAddr>>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
2454                                                         | SGX_MMU_PDE_VALID;
2455
2456         pui32PT[ui32PTIndex] = (psDevInfo->sBRN22997PTDevPAddr.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
2457                                                         | SGX_MMU_PTE_VALID;
2458
2459         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
2460         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PT_UNIQUETAG, PDUMP_PD_UNIQUETAG);
2461
2462
2463         OSWriteHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_DIR_LIST_BASE0,
2464                                  psDevInfo->sBRN22997PDDevPAddr.uiAddr);
2465         PDUMPPDREG(EUR_CR_BIF_DIR_LIST_BASE0, psDevInfo->sBRN22997PDDevPAddr.uiAddr, PDUMP_PD_UNIQUETAG);
2466
2467
2468         ui32BIFCtrl = OSReadHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_CTRL);
2469         OSWriteHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_CTRL, ui32BIFCtrl | EUR_CR_BIF_CTRL_INVALDC_MASK);
2470         PDUMPREG(EUR_CR_BIF_CTRL, ui32BIFCtrl | EUR_CR_BIF_CTRL_INVALDC_MASK);
2471         OSWriteHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_CTRL, ui32BIFCtrl);
2472         PDUMPREG(EUR_CR_BIF_CTRL, ui32BIFCtrl);
2473
2474
2475         if (pui32HostPort)
2476         {
2477
2478                 IMG_UINT32 ui32Tmp;
2479                 ui32Tmp = *pui32HostPort;
2480         }
2481         else
2482         {
2483                 PVR_DPF((PVR_DBG_ERROR,"Host Port not present for BRN22997 workaround"));
2484         }
2485
2486
2487
2488
2489
2490
2491
2492         PDUMPCOMMENT("RDW :SGXMEM:v4:%08lX\r\n", sDevVAddr.uiAddr);
2493
2494     PDUMPCOMMENT("SAB :SGXMEM:v4:%08lX 4 0 hostport.bin", sDevVAddr.uiAddr);
2495
2496
2497         pui32PD[ui32PDIndex] = 0;
2498         pui32PT[ui32PTIndex] = 0;
2499
2500
2501         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
2502         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PT_UNIQUETAG, PDUMP_PD_UNIQUETAG);
2503
2504         OSWriteHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_CTRL, ui32BIFCtrl | EUR_CR_BIF_CTRL_INVALDC_MASK);
2505         PDUMPREG(EUR_CR_BIF_CTRL, ui32BIFCtrl | EUR_CR_BIF_CTRL_INVALDC_MASK);
2506         OSWriteHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_BIF_CTRL, ui32BIFCtrl);
2507         PDUMPREG(EUR_CR_BIF_CTRL, ui32BIFCtrl);
2508 }
2509
2510
2511 IMG_VOID WorkaroundBRN22997Free(PVRSRV_SGXDEV_INFO *psDevInfo)
2512 {
2513         SYS_DATA *psSysData;
2514         RA_ARENA *psLocalDevMemArena;
2515
2516         SysAcquireData(&psSysData);
2517
2518         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2519
2520         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pui32BRN22997PD, SGX_MMU_PAGE_SIZE, PDUMP_PD_UNIQUETAG);
2521         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pui32BRN22997PT, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
2522
2523
2524         if(psLocalDevMemArena == IMG_NULL)
2525         {
2526                 if (psDevInfo->pui32BRN22997PD != IMG_NULL)
2527                 {
2528                         OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2529                                                   SGX_MMU_PAGE_SIZE,
2530                                                   psDevInfo->pui32BRN22997PD,
2531                                                   psDevInfo->hBRN22997PDPageOSMemHandle);
2532                 }
2533
2534                 if (psDevInfo->pui32BRN22997PT != IMG_NULL)
2535                 {
2536                         OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2537                                                   SGX_MMU_PAGE_SIZE,
2538                                                   psDevInfo->pui32BRN22997PT,
2539                                                   psDevInfo->hBRN22997PTPageOSMemHandle);
2540                 }
2541         }
2542         else
2543         {
2544                 if (psDevInfo->pui32BRN22997PT != IMG_NULL)
2545                 {
2546                         OSUnMapPhysToLin(psDevInfo->pui32BRN22997PT,
2547                                  SGX_MMU_PAGE_SIZE * 2,
2548                                  PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2549                                  psDevInfo->hBRN22997PTPageOSMemHandle);
2550
2551
2552                         RA_Free(psLocalDevMemArena, psDevInfo->sBRN22997SysPAddr.uiAddr, IMG_FALSE);
2553                 }
2554         }
2555 }
2556 #endif
2557
2558
2559 #if defined(SUPPORT_EXTERNAL_SYSTEM_CACHE)
2560 PVRSRV_ERROR MMU_MapExtSystemCacheRegs(PVRSRV_DEVICE_NODE *psDeviceNode)
2561 {
2562         PVRSRV_ERROR eError;
2563         SYS_DATA *psSysData;
2564         RA_ARENA *psLocalDevMemArena;
2565         IMG_HANDLE hPTPageOSMemHandle = IMG_NULL;
2566         IMG_UINT32 *pui32PD;
2567         IMG_UINT32 *pui32PT = IMG_NULL;
2568         IMG_CPU_PHYADDR sCpuPAddr;
2569         IMG_DEV_PHYADDR sPTDevPAddr;
2570         PVRSRV_SGXDEV_INFO *psDevInfo;
2571         IMG_UINT32 ui32PDIndex;
2572         IMG_UINT32 ui32PTIndex;
2573
2574         psDevInfo = (PVRSRV_SGXDEV_INFO*)psDeviceNode->pvDevice;
2575         pui32PD = (IMG_UINT32*)psDeviceNode->sDevMemoryInfo.pBMKernelContext->psMMUContext->pvPDCpuVAddr;
2576
2577         SysAcquireData(&psSysData);
2578
2579         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2580
2581
2582         if(psLocalDevMemArena == IMG_NULL)
2583         {
2584
2585                 eError = OSAllocPages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2586                                                    SGX_MMU_PAGE_SIZE,
2587                                                    SGX_MMU_PAGE_SIZE,
2588                                                    (IMG_VOID **)&pui32PT,
2589                                                    &hPTPageOSMemHandle);
2590                 if (eError != PVRSRV_OK)
2591                 {
2592                         PVR_DPF((PVR_DBG_ERROR, "MMU_MapExtSystemCacheRegs: ERROR call to OSAllocPages failed"));
2593                         return eError;
2594                 }
2595
2596
2597                 if(pui32PT)
2598         {
2599             sCpuPAddr = OSMapLinToCPUPhys(pui32PT);
2600         }
2601         else
2602         {
2603
2604             sCpuPAddr = OSMemHandleToCpuPAddr(hPTPageOSMemHandle, 0);
2605         }
2606                 sPTDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
2607         }
2608         else
2609         {
2610                 IMG_SYS_PHYADDR sSysPAddr;
2611
2612
2613                 if(RA_Alloc(psLocalDevMemArena,
2614                                         SGX_MMU_PAGE_SIZE,
2615                                         IMG_NULL,
2616                                         IMG_NULL,
2617                                         0,
2618                                         SGX_MMU_PAGE_SIZE,
2619                                         0,
2620                                         &(sSysPAddr.uiAddr))!= IMG_TRUE)
2621                 {
2622                         PVR_DPF((PVR_DBG_ERROR, "MMU_MapExtSystemCacheRegs: ERROR call to RA_Alloc failed"));
2623                         return PVRSRV_ERROR_OUT_OF_MEMORY;
2624                 }
2625
2626
2627                 sCpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
2628                 pui32PT = OSMapPhysToLin(sCpuPAddr,
2629                                                                 SGX_MMU_PAGE_SIZE,
2630                                 PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2631                                                                 &hPTPageOSMemHandle);
2632                 if(!pui32PT)
2633                 {
2634                         PVR_DPF((PVR_DBG_ERROR, "MMU_MapExtSystemCacheRegs: ERROR failed to map page tables"));
2635                         return PVRSRV_ERROR_BAD_MAPPING;
2636                 }
2637
2638
2639                 sPTDevPAddr = SysCpuPAddrToDevPAddr(PVRSRV_DEVICE_TYPE_SGX, sCpuPAddr);
2640
2641
2642                 psDevInfo->sExtSystemCacheRegsPTSysPAddr = sSysPAddr;
2643         }
2644
2645         OSMemSet(pui32PT, 0, SGX_MMU_PAGE_SIZE);
2646
2647         ui32PDIndex = (SGX_EXT_SYSTEM_CACHE_REGS_DEVVADDR_BASE & SGX_MMU_PD_MASK) >> (SGX_MMU_PAGE_SHIFT + SGX_MMU_PT_SHIFT);
2648         ui32PTIndex = (SGX_EXT_SYSTEM_CACHE_REGS_DEVVADDR_BASE & SGX_MMU_PT_MASK) >> SGX_MMU_PAGE_SHIFT;
2649
2650
2651         pui32PD[ui32PDIndex] = (sPTDevPAddr.uiAddr>>SGX_MMU_PDE_ADDR_ALIGNSHIFT)
2652                                                         | SGX_MMU_PDE_VALID;
2653
2654         pui32PT[ui32PTIndex] = (psDevInfo->sExtSysCacheRegsDevPBase.uiAddr>>SGX_MMU_PTE_ADDR_ALIGNSHIFT)
2655                                                         | SGX_MMU_PTE_VALID;
2656
2657
2658         PDUMPMALLOCPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
2659         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
2660         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PT, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PT_UNIQUETAG, PDUMP_PD_UNIQUETAG);
2661
2662
2663         psDevInfo->pui32ExtSystemCacheRegsPT = pui32PT;
2664         psDevInfo->hExtSystemCacheRegsPTPageOSMemHandle = hPTPageOSMemHandle;
2665
2666         return PVRSRV_OK;
2667 }
2668
2669
2670 PVRSRV_ERROR MMU_UnmapExtSystemCacheRegs(PVRSRV_DEVICE_NODE *psDeviceNode)
2671 {
2672         SYS_DATA *psSysData;
2673         RA_ARENA *psLocalDevMemArena;
2674         PVRSRV_SGXDEV_INFO *psDevInfo;
2675         IMG_UINT32 ui32PDIndex;
2676         IMG_UINT32 *pui32PD;
2677
2678         psDevInfo = (PVRSRV_SGXDEV_INFO*)psDeviceNode->pvDevice;
2679         pui32PD = (IMG_UINT32*)psDeviceNode->sDevMemoryInfo.pBMKernelContext->psMMUContext->pvPDCpuVAddr;
2680
2681         SysAcquireData(&psSysData);
2682
2683         psLocalDevMemArena = psSysData->apsLocalDevMemArena[0];
2684
2685
2686         ui32PDIndex = (SGX_EXT_SYSTEM_CACHE_REGS_DEVVADDR_BASE & SGX_MMU_PD_MASK) >> (SGX_MMU_PAGE_SHIFT + SGX_MMU_PT_SHIFT);
2687         pui32PD[ui32PDIndex] = 0;
2688
2689         PDUMPMEM2(PVRSRV_DEVICE_TYPE_SGX, pui32PD, SGX_MMU_PAGE_SIZE, 0, IMG_TRUE, PDUMP_PD_UNIQUETAG, PDUMP_PT_UNIQUETAG);
2690         PDUMPFREEPAGETABLE(PVRSRV_DEVICE_TYPE_SGX, psDevInfo->pui32ExtSystemCacheRegsPT, SGX_MMU_PAGE_SIZE, PDUMP_PT_UNIQUETAG);
2691
2692
2693         if(psLocalDevMemArena == IMG_NULL)
2694         {
2695                 if (psDevInfo->pui32ExtSystemCacheRegsPT != IMG_NULL)
2696                 {
2697                         OSFreePages(PVRSRV_HAP_WRITECOMBINE | PVRSRV_HAP_KERNEL_ONLY,
2698                                                   SGX_MMU_PAGE_SIZE,
2699                                                   psDevInfo->pui32ExtSystemCacheRegsPT,
2700                                                   psDevInfo->hExtSystemCacheRegsPTPageOSMemHandle);
2701                 }
2702         }
2703         else
2704         {
2705                 if (psDevInfo->pui32ExtSystemCacheRegsPT != IMG_NULL)
2706                 {
2707                         OSUnMapPhysToLin(psDevInfo->pui32ExtSystemCacheRegsPT,
2708                                  SGX_MMU_PAGE_SIZE,
2709                                  PVRSRV_HAP_WRITECOMBINE|PVRSRV_HAP_KERNEL_ONLY,
2710                                  psDevInfo->hExtSystemCacheRegsPTPageOSMemHandle);
2711
2712                         RA_Free(psLocalDevMemArena, psDevInfo->sExtSystemCacheRegsPTSysPAddr.uiAddr, IMG_FALSE);
2713                 }
2714         }
2715
2716         return PVRSRV_OK;
2717 }
2718 #endif
2719
2720
2721 #if PAGE_TEST
2722 static IMG_VOID PageTest(IMG_VOID* pMem, IMG_DEV_PHYADDR sDevPAddr)
2723 {
2724         volatile IMG_UINT32 ui32WriteData;
2725         volatile IMG_UINT32 ui32ReadData;
2726         volatile IMG_UINT32 *pMem32 = (volatile IMG_UINT32 *)pMem;
2727         IMG_INT n;
2728         IMG_BOOL bOK=IMG_TRUE;
2729
2730         ui32WriteData = 0xffffffff;
2731
2732         for (n=0; n<1024; n++)
2733         {
2734                 pMem32[n] = ui32WriteData;
2735                 ui32ReadData = pMem32[n];
2736
2737                 if (ui32WriteData != ui32ReadData)
2738                 {
2739
2740                         PVR_DPF ((PVR_DBG_ERROR, "Error - memory page test failed at device phys address 0x%08X", sDevPAddr.uiAddr + (n<<2) ));
2741                         PVR_DBG_BREAK;
2742                         bOK = IMG_FALSE;
2743                 }
2744         }
2745
2746         ui32WriteData = 0;
2747
2748         for (n=0; n<1024; n++)
2749         {
2750                 pMem32[n] = ui32WriteData;
2751                 ui32ReadData = pMem32[n];
2752
2753                 if (ui32WriteData != ui32ReadData)
2754                 {
2755
2756                         PVR_DPF ((PVR_DBG_ERROR, "Error - memory page test failed at device phys address 0x%08X", sDevPAddr.uiAddr + (n<<2) ));
2757                         PVR_DBG_BREAK;
2758                         bOK = IMG_FALSE;
2759                 }
2760         }
2761
2762         if (bOK)
2763         {
2764                 PVR_DPF ((PVR_DBG_VERBOSE, "MMU Page 0x%08X is OK", sDevPAddr.uiAddr));
2765         }
2766         else
2767         {
2768                 PVR_DPF ((PVR_DBG_VERBOSE, "MMU Page 0x%08X *** FAILED ***", sDevPAddr.uiAddr));
2769         }
2770 }
2771 #endif
2772