drivers/gpu/drm/img/img-rogue/services/server/common/devicemem_history_server.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          Devicemem history functions
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Devicemem history functions
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43
  44 #include "allocmem.h"
  45 #include "img_defs.h"
  46 #include "pmr.h"
  47 #include "pvrsrv.h"
  48 #include "pvrsrv_device.h"
  49 #include "pvr_debug.h"
  50 #include "devicemem_server.h"
  51 #include "lock.h"
  52 #include "devicemem_history_server.h"
  53 #include "pdump_km.h"
  54 #include "di_server.h"
  55
  56 #define ALLOCATION_LIST_NUM_ENTRIES 10000
  57
  58 /* data type to hold an allocation index.
  59  * we make it 16 bits wide if possible
  60  */
  61 #if ALLOCATION_LIST_NUM_ENTRIES <= 0xFFFF
  62 typedef uint16_t ALLOC_INDEX_T;
  63 #else
  64 typedef uint32_t ALLOC_INDEX_T;
  65 #endif
  66
  67 /* a record describing a single allocation known to DeviceMemHistory.
  68  * this is an element in a doubly linked list of allocations
  69  */
  70 typedef struct _RECORD_ALLOCATION_
  71 {
  72         /* time when this RECORD_ALLOCATION was created/initialised */
  73         IMG_UINT64 ui64CreationTime;
  74         /* serial number of the PMR relating to this allocation */
  75         IMG_UINT64 ui64Serial;
  76         /* base DevVAddr of this allocation */
  77         IMG_DEV_VIRTADDR sDevVAddr;
  78         /* size in bytes of this allocation */
  79         IMG_DEVMEM_SIZE_T uiSize;
  80         /* Log2 page size of this allocation's GPU pages */
  81         IMG_UINT32 ui32Log2PageSize;
  82         /* Process ID (PID) this allocation belongs to */
  83         IMG_PID uiPID;
  84         /* index of previous allocation in the list */
  85         ALLOC_INDEX_T ui32Prev;
  86         /* index of next allocation in the list */
  87         ALLOC_INDEX_T ui32Next;
  88         /* annotation/name of this allocation */
  89         IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN];
  90 } RECORD_ALLOCATION;
  91
  92 /* each command in the circular buffer is prefixed with an 8-bit value
  93  * denoting the command type
  94  */
  95 typedef enum _COMMAND_TYPE_
  96 {
  97         COMMAND_TYPE_NONE,
  98         COMMAND_TYPE_TIMESTAMP,
  99         COMMAND_TYPE_MAP_ALL,
 100         COMMAND_TYPE_UNMAP_ALL,
 101         COMMAND_TYPE_MAP_RANGE,
 102         COMMAND_TYPE_UNMAP_RANGE,
 103         /* sentinel value */
 104         COMMAND_TYPE_COUNT,
 105 } COMMAND_TYPE;
 106
 107 /* Timestamp command:
 108  * This command is inserted into the circular buffer to provide an updated
 109  * timestamp.
 110  * The nanosecond-accuracy timestamp is packed into a 56-bit integer, in order
 111  * for the whole command to fit into 8 bytes.
 112  */
 113 typedef struct _COMMAND_TIMESTAMP_
 114 {
 115         IMG_UINT8 aui8TimeNs[7];
 116 } COMMAND_TIMESTAMP;
 117
 118 /* MAP_ALL command:
 119  * This command denotes the allocation at the given index was wholly mapped
 120  * in to the GPU MMU
 121  */
 122 typedef struct _COMMAND_MAP_ALL_
 123 {
 124         ALLOC_INDEX_T uiAllocIndex;
 125 } COMMAND_MAP_ALL;
 126
 127 /* UNMAP_ALL command:
 128  * This command denotes the allocation at the given index was wholly unmapped
 129  * from the GPU MMU
 130  * Note: COMMAND_MAP_ALL and COMMAND_UNMAP_ALL commands have the same layout.
 131  */
 132 typedef COMMAND_MAP_ALL COMMAND_UNMAP_ALL;
 133
 134 /* packing attributes for the MAP_RANGE command */
 135 #define MAP_RANGE_MAX_START ((1 << 18) - 1)
 136 #define MAP_RANGE_MAX_RANGE ((1 << 12) - 1)
 137
 138 /* MAP_RANGE command:
 139  * Denotes a range of pages within the given allocation being mapped.
 140  * The range is expressed as [Page Index] + [Page Count]
 141  * This information is packed into a 40-bit integer, in order to make
 142  * the command size 8 bytes.
 143  */
 144
 145 typedef struct _COMMAND_MAP_RANGE_
 146 {
 147         IMG_UINT8 aui8Data[5];
 148         ALLOC_INDEX_T uiAllocIndex;
 149 } COMMAND_MAP_RANGE;
 150
 151 /* UNMAP_RANGE command:
 152  * Denotes a range of pages within the given allocation being mapped.
 153  * The range is expressed as [Page Index] + [Page Count]
 154  * This information is packed into a 40-bit integer, in order to make
 155  * the command size 8 bytes.
 156  * Note: COMMAND_MAP_RANGE and COMMAND_UNMAP_RANGE commands have the same layout.
 157  */
 158 typedef COMMAND_MAP_RANGE COMMAND_UNMAP_RANGE;
 159
 160 /* wrapper structure for a command */
 161 typedef struct _COMMAND_WRAPPER_
 162 {
 163         IMG_UINT8 ui8Type;
 164         union {
 165                 COMMAND_TIMESTAMP sTimeStamp;
 166                 COMMAND_MAP_ALL sMapAll;
 167                 COMMAND_UNMAP_ALL sUnmapAll;
 168                 COMMAND_MAP_RANGE sMapRange;
 169                 COMMAND_UNMAP_RANGE sUnmapRange;
 170         } u;
 171 } COMMAND_WRAPPER;
 172
 173 /* target size for the circular buffer of commands */
 174 #define CIRCULAR_BUFFER_SIZE_KB 2048
 175 /* turn the circular buffer target size into a number of commands */
 176 #define CIRCULAR_BUFFER_NUM_COMMANDS ((CIRCULAR_BUFFER_SIZE_KB * 1024) / sizeof(COMMAND_WRAPPER))
 177
 178 /* index value denoting the end of a list */
 179 #define END_OF_LIST 0xFFFFFFFF
 180 #define ALLOC_INDEX_TO_PTR(idx) (&(gsDevicememHistoryData.sRecords.pasAllocations[idx]))
 181 #define CHECK_ALLOC_INDEX(idx) (idx < ALLOCATION_LIST_NUM_ENTRIES)
 182
 183 /* wrapper structure for the allocation records and the commands circular buffer */
 184 typedef struct _RECORDS_
 185 {
 186         RECORD_ALLOCATION *pasAllocations;
 187         IMG_UINT32 ui32AllocationsListHead;
 188
 189         IMG_UINT32 ui32Head;
 190         IMG_UINT32 ui32Tail;
 191         COMMAND_WRAPPER *pasCircularBuffer;
 192 } RECORDS;
 193
 194 typedef struct _DEVICEMEM_HISTORY_DATA_
 195 {
 196         /* DI entry */
 197         DI_ENTRY *psDIEntry;
 198
 199         RECORDS sRecords;
 200         POS_LOCK hLock;
 201 } DEVICEMEM_HISTORY_DATA;
 202
 203 static DEVICEMEM_HISTORY_DATA gsDevicememHistoryData;
 204
 205 /* gsDevicememHistoryData is static, hLock is NULL unless
 206  * EnablePageFaultDebug is set and DevicememHistoryInitKM()
 207  * was called.
 208  */
 209 static void DevicememHistoryLock(void)
 210 {
 211         if (gsDevicememHistoryData.hLock)
 212         {
 213                 OSLockAcquire(gsDevicememHistoryData.hLock);
 214         }
 215 }
 216
 217 static void DevicememHistoryUnlock(void)
 218 {
 219         if (gsDevicememHistoryData.hLock)
 220         {
 221                 OSLockRelease(gsDevicememHistoryData.hLock);
 222         }
 223 }
 224
 225 /* given a time stamp, calculate the age in nanoseconds */
 226 static IMG_UINT64 _CalculateAge(IMG_UINT64 ui64Now,
 227                                                 IMG_UINT64 ui64Then,
 228                                                 IMG_UINT64 ui64Max)
 229 {
 230         if (ui64Now >= ui64Then)
 231         {
 232                 /* no clock wrap */
 233                 return ui64Now - ui64Then;
 234         }
 235         else
 236         {
 237                 /* clock has wrapped */
 238                 return (ui64Max - ui64Then) + ui64Now + 1;
 239         }
 240 }
 241
 242 /* AcquireCBSlot:
 243  * Acquire the next slot in the circular buffer and
 244  * move the circular buffer head along by one
 245  * Returns a pointer to the acquired slot.
 246  */
 247 static COMMAND_WRAPPER *AcquireCBSlot(void)
 248 {
 249         COMMAND_WRAPPER *psSlot;
 250
 251         psSlot = &gsDevicememHistoryData.sRecords.pasCircularBuffer[gsDevicememHistoryData.sRecords.ui32Head];
 252
 253         gsDevicememHistoryData.sRecords.ui32Head =
 254                 (gsDevicememHistoryData.sRecords.ui32Head + 1)
 255                                 % CIRCULAR_BUFFER_NUM_COMMANDS;
 256
 257         return psSlot;
 258 }
 259
 260 /* TimeStampPack:
 261  * Packs the given timestamp value into the COMMAND_TIMESTAMP structure.
 262  * This takes a 64-bit nanosecond timestamp and packs it in to a 56-bit
 263  * integer in the COMMAND_TIMESTAMP command.
 264  */
 265 static void TimeStampPack(COMMAND_TIMESTAMP *psTimeStamp, IMG_UINT64 ui64Now)
 266 {
 267         IMG_UINT32 i;
 268
 269         for (i = 0; i < ARRAY_SIZE(psTimeStamp->aui8TimeNs); i++)
 270         {
 271                 psTimeStamp->aui8TimeNs[i] = ui64Now & 0xFF;
 272                 ui64Now >>= 8;
 273         }
 274 }
 275
 276 /* packing a 64-bit nanosecond into a 7-byte integer loses the
 277  * top 8 bits of data. This must be taken into account when
 278  * comparing a full timestamp against an unpacked timestamp
 279  */
 280 #define TIME_STAMP_MASK ((1LLU << 56) - 1)
 281 #define DO_TIME_STAMP_MASK(ns64) (ns64 & TIME_STAMP_MASK)
 282
 283 /* TimeStampUnpack:
 284  * Unpack the timestamp value from the given COMMAND_TIMESTAMP command
 285  */
 286 static IMG_UINT64 TimeStampUnpack(COMMAND_TIMESTAMP *psTimeStamp)
 287 {
 288         IMG_UINT64 ui64TimeNs = 0;
 289         IMG_UINT32 i;
 290
 291         for (i = ARRAY_SIZE(psTimeStamp->aui8TimeNs); i > 0; i--)
 292         {
 293                 ui64TimeNs <<= 8;
 294                 ui64TimeNs |= (IMG_UINT64) psTimeStamp->aui8TimeNs[i - 1];
 295         }
 296
 297         return ui64TimeNs;
 298 }
 299
 300 #if defined(PDUMP)
 301
 302 static void EmitPDumpAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
 303                                 IMG_UINT32 ui32AllocationIndex,
 304                                 RECORD_ALLOCATION *psAlloc)
 305 {
 306         PDUMPCOMMENT(psDeviceNode,
 307                         "[SrvPFD] Allocation: %u"
 308                         " Addr: " IMG_DEV_VIRTADDR_FMTSPEC
 309                         " Size: " IMG_DEVMEM_SIZE_FMTSPEC
 310                         " Page size: %u"
 311                         " PID: %u"
 312                         " Process: %s"
 313                         " Name: %s",
 314                         ui32AllocationIndex,
 315                         psAlloc->sDevVAddr.uiAddr,
 316                         psAlloc->uiSize,
 317                         1U << psAlloc->ui32Log2PageSize,
 318                         psAlloc->uiPID,
 319                         OSGetCurrentClientProcessNameKM(),
 320                         psAlloc->szName);
 321 }
 322
 323 static void EmitPDumpMapUnmapAll(PVRSRV_DEVICE_NODE *psDeviceNode,
 324                                  COMMAND_TYPE eType,
 325                                  IMG_UINT32 ui32AllocationIndex)
 326 {
 327         const IMG_CHAR *pszOpName;
 328
 329         switch (eType)
 330         {
 331                 case COMMAND_TYPE_MAP_ALL:
 332                         pszOpName = "MAP_ALL";
 333                         break;
 334                 case COMMAND_TYPE_UNMAP_ALL:
 335                         pszOpName = "UNMAP_ALL";
 336                         break;
 337                 default:
 338                         PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapAll: Invalid type: %u",
 339                                                                                 eType));
 340                         return;
 341
 342         }
 343
 344         PDUMPCOMMENT(psDeviceNode,
 345                      "[SrvPFD] Op: %s Allocation: %u",
 346                      pszOpName,
 347                      ui32AllocationIndex);
 348 }
 349
 350 static void EmitPDumpMapUnmapRange(PVRSRV_DEVICE_NODE *psDeviceNode,
 351                                         COMMAND_TYPE eType,
 352                                         IMG_UINT32 ui32AllocationIndex,
 353                                         IMG_UINT32 ui32StartPage,
 354                                         IMG_UINT32 ui32Count)
 355 {
 356         const IMG_CHAR *pszOpName;
 357
 358         switch (eType)
 359         {
 360                 case COMMAND_TYPE_MAP_RANGE:
 361                         pszOpName = "MAP_RANGE";
 362                         break;
 363                 case COMMAND_TYPE_UNMAP_RANGE:
 364                         pszOpName = "UNMAP_RANGE";
 365                         break;
 366                 default:
 367                         PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapRange: Invalid type: %u",
 368                                                                                 eType));
 369                         return;
 370         }
 371
 372         PDUMPCOMMENT(psDeviceNode,
 373                  "[SrvPFD] Op: %s Allocation: %u Start Page: %u Count: %u",
 374                  pszOpName,
 375                  ui32AllocationIndex,
 376                  ui32StartPage,
 377                  ui32Count);
 378 }
 379
 380 #endif
 381
 382 /* InsertTimeStampCommand:
 383  * Insert a timestamp command into the circular buffer.
 384  */
 385 static void InsertTimeStampCommand(IMG_UINT64 ui64Now)
 386 {
 387         COMMAND_WRAPPER *psCommand;
 388
 389         psCommand = AcquireCBSlot();
 390
 391         psCommand->ui8Type = COMMAND_TYPE_TIMESTAMP;
 392
 393         TimeStampPack(&psCommand->u.sTimeStamp, ui64Now);
 394 }
 395
 396 /* InsertMapAllCommand:
 397  * Insert a "MAP_ALL" command for the given allocation into the circular buffer
 398  */
 399 static void InsertMapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
 400                                 IMG_UINT32 ui32AllocIndex)
 401 {
 402         COMMAND_WRAPPER *psCommand;
 403
 404         psCommand = AcquireCBSlot();
 405
 406         psCommand->ui8Type = COMMAND_TYPE_MAP_ALL;
 407         psCommand->u.sMapAll.uiAllocIndex = ui32AllocIndex;
 408
 409 #if defined(PDUMP)
 410         EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_MAP_ALL, ui32AllocIndex);
 411 #else
 412         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 413 #endif
 414 }
 415
 416 /* InsertUnmapAllCommand:
 417  * Insert a "UNMAP_ALL" command for the given allocation into the circular buffer
 418  */
 419 static void InsertUnmapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
 420                                   IMG_UINT32 ui32AllocIndex)
 421 {
 422         COMMAND_WRAPPER *psCommand;
 423
 424         psCommand = AcquireCBSlot();
 425
 426         psCommand->ui8Type = COMMAND_TYPE_UNMAP_ALL;
 427         psCommand->u.sUnmapAll.uiAllocIndex = ui32AllocIndex;
 428
 429 #if defined(PDUMP)
 430         EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_UNMAP_ALL, ui32AllocIndex);
 431 #else
 432         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 433 #endif
 434 }
 435
 436 /* MapRangePack:
 437  * Pack the given StartPage and Count values into the 40-bit representation
 438  * in the MAP_RANGE command.
 439  */
 440 static void MapRangePack(COMMAND_MAP_RANGE *psMapRange,
 441                                                 IMG_UINT32 ui32StartPage,
 442                                                 IMG_UINT32 ui32Count)
 443 {
 444         IMG_UINT64 ui64Data;
 445         IMG_UINT32 i;
 446
 447         /* we must encode the data into 40 bits:
 448          *   18 bits for the start page index
 449          *   12 bits for the range
 450         */
 451         PVR_ASSERT(ui32StartPage <= MAP_RANGE_MAX_START);
 452         PVR_ASSERT(ui32Count <= MAP_RANGE_MAX_RANGE);
 453
 454         ui64Data = (((IMG_UINT64) ui32StartPage) << 12) | ui32Count;
 455
 456         for (i = 0; i < ARRAY_SIZE(psMapRange->aui8Data); i++)
 457         {
 458                 psMapRange->aui8Data[i] = ui64Data & 0xFF;
 459                 ui64Data >>= 8;
 460         }
 461 }
 462
 463 /* MapRangePack:
 464  * Unpack the StartPage and Count values from the 40-bit representation
 465  * in the MAP_RANGE command.
 466  */
 467 static void MapRangeUnpack(COMMAND_MAP_RANGE *psMapRange,
 468                                                 IMG_UINT32 *pui32StartPage,
 469                                                 IMG_UINT32 *pui32Count)
 470 {
 471         IMG_UINT64 ui64Data = 0;
 472         IMG_UINT32 i;
 473
 474         for (i = ARRAY_SIZE(psMapRange->aui8Data); i > 0; i--)
 475         {
 476                 ui64Data <<= 8;
 477                 ui64Data |= (IMG_UINT64) psMapRange->aui8Data[i - 1];
 478         }
 479
 480         *pui32StartPage = (ui64Data >> 12);
 481         *pui32Count = ui64Data & ((1 << 12) - 1);
 482 }
 483
 484 /* InsertMapRangeCommand:
 485  * Insert a MAP_RANGE command into the circular buffer with the given
 486  * StartPage and Count values.
 487  */
 488 static void InsertMapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
 489                                                 IMG_UINT32 ui32AllocIndex,
 490                                                 IMG_UINT32 ui32StartPage,
 491                                                 IMG_UINT32 ui32Count)
 492 {
 493         COMMAND_WRAPPER *psCommand;
 494
 495         psCommand = AcquireCBSlot();
 496
 497         psCommand->ui8Type = COMMAND_TYPE_MAP_RANGE;
 498         psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
 499
 500         MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
 501
 502 #if defined(PDUMP)
 503         EmitPDumpMapUnmapRange(psDeviceNode,
 504                                COMMAND_TYPE_MAP_RANGE,
 505                                ui32AllocIndex,
 506                                ui32StartPage,
 507                                ui32Count);
 508 #else
 509         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 510 #endif
 511 }
 512
 513 /* InsertUnmapRangeCommand:
 514  * Insert a UNMAP_RANGE command into the circular buffer with the given
 515  * StartPage and Count values.
 516  */
 517 static void InsertUnmapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
 518                                                 IMG_UINT32 ui32AllocIndex,
 519                                                 IMG_UINT32 ui32StartPage,
 520                                                 IMG_UINT32 ui32Count)
 521 {
 522         COMMAND_WRAPPER *psCommand;
 523
 524         psCommand = AcquireCBSlot();
 525
 526         psCommand->ui8Type = COMMAND_TYPE_UNMAP_RANGE;
 527         psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
 528
 529         MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
 530
 531 #if defined(PDUMP)
 532         EmitPDumpMapUnmapRange(psDeviceNode,
 533                                COMMAND_TYPE_UNMAP_RANGE,
 534                                ui32AllocIndex,
 535                                ui32StartPage,
 536                                ui32Count);
 537 #else
 538         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 539 #endif
 540 }
 541
 542 /* InsertAllocationToList:
 543  * Helper function for the allocation list.
 544  * Inserts the given allocation at the head of the list, whose current head is
 545  * pointed to by pui32ListHead
 546  */
 547 static void InsertAllocationToList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
 548 {
 549         RECORD_ALLOCATION *psAlloc;
 550
 551         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 552
 553         if (*pui32ListHead == END_OF_LIST)
 554         {
 555                 /* list is currently empty, so just replace it */
 556                 *pui32ListHead = ui32Alloc;
 557                 psAlloc->ui32Next = psAlloc->ui32Prev = *pui32ListHead;
 558         }
 559         else
 560         {
 561                 RECORD_ALLOCATION *psHeadAlloc;
 562                 RECORD_ALLOCATION *psTailAlloc;
 563
 564                 psHeadAlloc = ALLOC_INDEX_TO_PTR(*pui32ListHead);
 565                 psTailAlloc = ALLOC_INDEX_TO_PTR(psHeadAlloc->ui32Prev);
 566
 567                 /* make the new alloc point forwards to the previous head */
 568                 psAlloc->ui32Next = *pui32ListHead;
 569                 /* make the new alloc point backwards to the previous tail */
 570                 psAlloc->ui32Prev = psHeadAlloc->ui32Prev;
 571
 572                 /* the head is now our new alloc */
 573                 *pui32ListHead = ui32Alloc;
 574
 575                 /* the old head now points back to the new head */
 576                 psHeadAlloc->ui32Prev = *pui32ListHead;
 577
 578                 /* the tail now points forward to the new head */
 579                 psTailAlloc->ui32Next = ui32Alloc;
 580         }
 581 }
 582
 583 static void InsertAllocationToBusyList(IMG_UINT32 ui32Alloc)
 584 {
 585         InsertAllocationToList(&gsDevicememHistoryData.sRecords.ui32AllocationsListHead, ui32Alloc);
 586 }
 587
 588 /* RemoveAllocationFromList:
 589  * Helper function for the allocation list.
 590  * Removes the given allocation from the list, whose head is
 591  * pointed to by pui32ListHead
 592  */
 593 static void RemoveAllocationFromList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
 594 {
 595         RECORD_ALLOCATION *psAlloc;
 596
 597         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 598
 599         /* if this is the only element in the list then just make the list empty */
 600         if ((*pui32ListHead == ui32Alloc) && (psAlloc->ui32Next == ui32Alloc))
 601         {
 602                 *pui32ListHead = END_OF_LIST;
 603         }
 604         else
 605         {
 606                 RECORD_ALLOCATION *psPrev, *psNext;
 607
 608                 psPrev = ALLOC_INDEX_TO_PTR(psAlloc->ui32Prev);
 609                 psNext = ALLOC_INDEX_TO_PTR(psAlloc->ui32Next);
 610
 611                 /* remove the allocation from the list */
 612                 psPrev->ui32Next = psAlloc->ui32Next;
 613                 psNext->ui32Prev = psAlloc->ui32Prev;
 614
 615                 /* if this allocation is the head then update the head */
 616                 if (*pui32ListHead == ui32Alloc)
 617                 {
 618                         *pui32ListHead = psAlloc->ui32Prev;
 619                 }
 620         }
 621 }
 622
 623 static void RemoveAllocationFromBusyList(IMG_UINT32 ui32Alloc)
 624 {
 625         RemoveAllocationFromList(&gsDevicememHistoryData.sRecords.ui32AllocationsListHead, ui32Alloc);
 626 }
 627
 628 /* TouchBusyAllocation:
 629  * Move the given allocation to the head of the list
 630  */
 631 static void TouchBusyAllocation(IMG_UINT32 ui32Alloc)
 632 {
 633         RemoveAllocationFromBusyList(ui32Alloc);
 634         InsertAllocationToBusyList(ui32Alloc);
 635 }
 636
 637 /* GetOldestBusyAllocation:
 638  * Returns the index of the oldest allocation in the MRU list
 639  */
 640 static IMG_UINT32 GetOldestBusyAllocation(void)
 641 {
 642         IMG_UINT32 ui32Alloc;
 643         RECORD_ALLOCATION *psAlloc;
 644
 645         ui32Alloc = gsDevicememHistoryData.sRecords.ui32AllocationsListHead;
 646
 647         if (ui32Alloc == END_OF_LIST)
 648         {
 649                 return END_OF_LIST;
 650         }
 651
 652         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 653
 654         return psAlloc->ui32Prev;
 655 }
 656
 657 static IMG_UINT32 GetFreeAllocation(void)
 658 {
 659         IMG_UINT32 ui32Alloc;
 660
 661         ui32Alloc = GetOldestBusyAllocation();
 662
 663         return ui32Alloc;
 664 }
 665
 666
 667 /* InitialiseAllocation:
 668  * Initialise the given allocation structure with the given properties
 669  */
 670 static void InitialiseAllocation(RECORD_ALLOCATION *psAlloc,
 671                                                         const IMG_CHAR *pszName,
 672                                                         IMG_UINT64 ui64Serial,
 673                                                         IMG_PID uiPID,
 674                                                         IMG_DEV_VIRTADDR sDevVAddr,
 675                                                         IMG_DEVMEM_SIZE_T uiSize,
 676                                                         IMG_UINT32 ui32Log2PageSize)
 677 {
 678         OSStringLCopy(psAlloc->szName, pszName, sizeof(psAlloc->szName));
 679         psAlloc->ui64Serial = ui64Serial;
 680         psAlloc->uiPID = uiPID;
 681         psAlloc->sDevVAddr = sDevVAddr;
 682         psAlloc->uiSize = uiSize;
 683         psAlloc->ui32Log2PageSize = ui32Log2PageSize;
 684         psAlloc->ui64CreationTime = OSClockns64();
 685 }
 686
 687 /* CreateAllocation:
 688  * Creates a new allocation with the given properties then outputs the
 689  * index of the allocation
 690  */
 691 static PVRSRV_ERROR CreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
 692                                                         const IMG_CHAR *pszName,
 693                                                         IMG_UINT64 ui64Serial,
 694                                                         IMG_PID uiPID,
 695                                                         IMG_DEV_VIRTADDR sDevVAddr,
 696                                                         IMG_DEVMEM_SIZE_T uiSize,
 697                                                         IMG_UINT32 ui32Log2PageSize,
 698                                                         IMG_BOOL bAutoPurge,
 699                                                         IMG_UINT32 *puiAllocationIndex)
 700 {
 701         IMG_UINT32 ui32Alloc;
 702         RECORD_ALLOCATION *psAlloc;
 703
 704         ui32Alloc = GetFreeAllocation();
 705
 706         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
 707
 708         InitialiseAllocation(ALLOC_INDEX_TO_PTR(ui32Alloc),
 709                                                 pszName,
 710                                                 ui64Serial,
 711                                                 uiPID,
 712                                                 sDevVAddr,
 713                                                 uiSize,
 714                                                 ui32Log2PageSize);
 715
 716         /* put the newly initialised allocation at the front of the MRU list */
 717         TouchBusyAllocation(ui32Alloc);
 718
 719         *puiAllocationIndex = ui32Alloc;
 720
 721 #if defined(PDUMP)
 722         EmitPDumpAllocation(psDeviceNode, ui32Alloc, psAlloc);
 723 #else
 724         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 725 #endif
 726
 727         return PVRSRV_OK;
 728 }
 729
 730 /* MatchAllocation:
 731  * Tests if the allocation at the given index matches the supplied properties.
 732  * Returns IMG_TRUE if it is a match, otherwise IMG_FALSE.
 733  */
 734 static IMG_BOOL MatchAllocation(IMG_UINT32 ui32AllocationIndex,
 735                                                 IMG_UINT64 ui64Serial,
 736                                                 IMG_DEV_VIRTADDR sDevVAddr,
 737                                                 IMG_DEVMEM_SIZE_T uiSize,
 738                                                 const IMG_CHAR *pszName,
 739                                                 IMG_UINT32 ui32Log2PageSize,
 740                                                 IMG_PID uiPID)
 741 {
 742         RECORD_ALLOCATION *psAlloc;
 743
 744         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocationIndex);
 745
 746         return (psAlloc->ui64Serial == ui64Serial) &&
 747                (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
 748                (psAlloc->uiSize == uiSize) &&
 749                (psAlloc->ui32Log2PageSize == ui32Log2PageSize) &&
 750                (OSStringNCompare(psAlloc->szName, pszName, DEVMEM_ANNOTATION_MAX_LEN) == 0);
 751 }
 752
 753 /* FindOrCreateAllocation:
 754  * Convenience function.
 755  * Given a set of allocation properties (serial, DevVAddr, size, name, etc),
 756  * this function will look for an existing record of this allocation and
 757  * create the allocation if there is no existing record
 758  */
 759 static PVRSRV_ERROR FindOrCreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
 760                                                         IMG_UINT32 ui32AllocationIndexHint,
 761                                                         IMG_UINT64 ui64Serial,
 762                                                         IMG_DEV_VIRTADDR sDevVAddr,
 763                                                         IMG_DEVMEM_SIZE_T uiSize,
 764                                                         const char *pszName,
 765                                                         IMG_UINT32 ui32Log2PageSize,
 766                                                         IMG_PID uiPID,
 767                                                         IMG_BOOL bSparse,
 768                                                         IMG_UINT32 *pui32AllocationIndexOut,
 769                                                         IMG_BOOL *pbCreated)
 770 {
 771         IMG_UINT32 ui32AllocationIndex;
 772         PVRSRV_ERROR eError;
 773
 774         if (ui32AllocationIndexHint != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE)
 775         {
 776                 IMG_BOOL bHaveAllocation;
 777
 778                 /* first, try to match against the index given by the client.
 779                  * if the caller provided a hint but the allocation record is no longer
 780                  * there, it must have been purged, so go ahead and create a new allocation
 781                  */
 782                 bHaveAllocation = MatchAllocation(ui32AllocationIndexHint,
 783                                                                 ui64Serial,
 784                                                                 sDevVAddr,
 785                                                                 uiSize,
 786                                                                 pszName,
 787                                                                 ui32Log2PageSize,
 788                                                                 uiPID);
 789                 if (bHaveAllocation)
 790                 {
 791                         *pbCreated = IMG_FALSE;
 792                         *pui32AllocationIndexOut = ui32AllocationIndexHint;
 793                         return PVRSRV_OK;
 794                 }
 795         }
 796
 797         /* if there is no record of the allocation then we
 798          * create it now
 799          */
 800         eError = CreateAllocation(psDeviceNode,
 801                                         pszName,
 802                                         ui64Serial,
 803                                         uiPID,
 804                                         sDevVAddr,
 805                                         uiSize,
 806                                         ui32Log2PageSize,
 807                                         IMG_TRUE,
 808                                         &ui32AllocationIndex);
 809
 810         if (eError == PVRSRV_OK)
 811         {
 812                 *pui32AllocationIndexOut = ui32AllocationIndex;
 813                 *pbCreated = IMG_TRUE;
 814         }
 815         else
 816         {
 817                 PVR_DPF((PVR_DBG_ERROR,
 818                         "%s: Failed to create record for allocation %s",
 819                                                                 __func__,
 820                                                                 pszName));
 821         }
 822
 823         return eError;
 824 }
 825
 826 /* GenerateMapUnmapCommandsForSparsePMR:
 827  * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the PMR's
 828  * current mapping table
 829  *
 830  * PMR: The PMR whose mapping table to read.
 831  * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
 832  * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
 833  *
 834  * This function goes through every page in the PMR's mapping table and looks for
 835  * virtually contiguous ranges to record as being mapped or unmapped.
 836  */
 837 static void GenerateMapUnmapCommandsForSparsePMR(PMR *psPMR,
 838                                                         IMG_UINT32 ui32AllocIndex,
 839                                                         IMG_BOOL bMap)
 840 {
 841         PMR_MAPPING_TABLE *psMappingTable;
 842         IMG_UINT32 ui32DonePages = 0;
 843         IMG_UINT32 ui32NumPages;
 844         IMG_UINT32 i;
 845         IMG_BOOL bInARun = IMG_FALSE;
 846         IMG_UINT32 ui32CurrentStart = 0;
 847         IMG_UINT32 ui32RunCount = 0;
 848
 849         psMappingTable = PMR_GetMappingTable(psPMR);
 850         ui32NumPages = psMappingTable->ui32NumPhysChunks;
 851
 852         if (ui32NumPages == 0)
 853         {
 854                 /* nothing to do */
 855                 return;
 856         }
 857
 858         for (i = 0; i < psMappingTable->ui32NumVirtChunks; i++)
 859         {
 860                 if (psMappingTable->aui32Translation[i] != TRANSLATION_INVALID)
 861                 {
 862                         if (!bInARun)
 863                         {
 864                                 bInARun = IMG_TRUE;
 865                                 ui32CurrentStart = i;
 866                                 ui32RunCount = 1;
 867                         }
 868                         else
 869                         {
 870                                 ui32RunCount++;
 871                         }
 872                 }
 873
 874                 if (bInARun)
 875                 {
 876                         /* test if we need to end this current run and generate the command,
 877                          * either because the next page is not virtually contiguous
 878                          * to the current page, we have reached the maximum range,
 879                          * or this is the last page in the mapping table
 880                          */
 881                         if ((psMappingTable->aui32Translation[i] == TRANSLATION_INVALID) ||
 882                                 (ui32RunCount == MAP_RANGE_MAX_RANGE) ||
 883                                 (i == (psMappingTable->ui32NumVirtChunks - 1)))
 884                         {
 885                                 if (bMap)
 886                                 {
 887                                         InsertMapRangeCommand(PMR_DeviceNode(psPMR),
 888                                                               ui32AllocIndex,
 889                                                               ui32CurrentStart,
 890                                                               ui32RunCount);
 891                                 }
 892                                 else
 893                                 {
 894                                         InsertUnmapRangeCommand(PMR_DeviceNode(psPMR),
 895                                                                 ui32AllocIndex,
 896                                                                 ui32CurrentStart,
 897                                                                 ui32RunCount);
 898                                 }
 899
 900                                 ui32DonePages += ui32RunCount;
 901
 902                                 if (ui32DonePages == ui32NumPages)
 903                                 {
 904                                          break;
 905                                 }
 906
 907                                 bInARun = IMG_FALSE;
 908                         }
 909                 }
 910         }
 911
 912 }
 913
 914 /* GenerateMapUnmapCommandsForChangeList:
 915  * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the
 916  * list of page change (page map or page unmap) indices given.
 917  *
 918  * ui32NumPages: Number of pages which have changed.
 919  * pui32PageList: List of indices of the pages which have changed.
 920  * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
 921  * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
 922  *
 923  * This function goes through every page in the list and looks for
 924  * virtually contiguous ranges to record as being mapped or unmapped.
 925  */
 926 static void GenerateMapUnmapCommandsForChangeList(PVRSRV_DEVICE_NODE *psDeviceNode,
 927                                                         IMG_UINT32 ui32NumPages,
 928                                                         IMG_UINT32 *pui32PageList,
 929                                                         IMG_UINT32 ui32AllocIndex,
 930                                                         IMG_BOOL bMap)
 931 {
 932         IMG_UINT32 i;
 933         IMG_BOOL bInARun = IMG_FALSE;
 934         IMG_UINT32 ui32CurrentStart = 0;
 935         IMG_UINT32 ui32RunCount = 0;
 936
 937         for (i = 0; i < ui32NumPages; i++)
 938         {
 939                 if (!bInARun)
 940                 {
 941                         bInARun = IMG_TRUE;
 942                         ui32CurrentStart = pui32PageList[i];
 943                 }
 944
 945                 ui32RunCount++;
 946
 947                  /* we flush if:
 948                  * - the next page in the list is not one greater than the current page
 949                  * - this is the last page in the list
 950                  * - we have reached the maximum range size
 951                  */
 952                 if ((i == (ui32NumPages - 1)) ||
 953                         ((pui32PageList[i] + 1) != pui32PageList[i + 1]) ||
 954                         (ui32RunCount == MAP_RANGE_MAX_RANGE))
 955                 {
 956                         if (bMap)
 957                         {
 958                                 InsertMapRangeCommand(psDeviceNode,
 959                                                                         ui32AllocIndex,
 960                                                                         ui32CurrentStart,
 961                                                                         ui32RunCount);
 962                         }
 963                         else
 964                         {
 965                                 InsertUnmapRangeCommand(psDeviceNode,
 966                                                                         ui32AllocIndex,
 967                                                                         ui32CurrentStart,
 968                                                                         ui32RunCount);
 969                         }
 970
 971                         bInARun = IMG_FALSE;
 972                         ui32RunCount = 0;
 973                 }
 974         }
 975 }
 976
 977 /* DevicememHistoryMapKM:
 978  * Entry point for when an allocation is mapped into the MMU GPU
 979  *
 980  * psPMR: The PMR to which the allocation belongs.
 981  * ui32Offset: The offset within the PMR at which the allocation begins.
 982  * sDevVAddr: The DevVAddr at which the allocation begins.
 983  * szName: Annotation/name for the allocation.
 984  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
 985  * ui32AllocationIndex: Allocation index as provided by the client.
 986  *                      We will use this as a short-cut to find the allocation
 987  *                      in our records.
 988  * pui32AllocationIndexOut: An updated allocation index for the client.
 989  *                          This may be a new value if we just created the
 990  *                          allocation record.
 991  */
 992 PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
 993                                                         IMG_UINT32 ui32Offset,
 994                                                         IMG_DEV_VIRTADDR sDevVAddr,
 995                                                         IMG_DEVMEM_SIZE_T uiSize,
 996                                                         const char szName[DEVMEM_ANNOTATION_MAX_LEN],
 997                                                         IMG_UINT32 ui32Log2PageSize,
 998                                                         IMG_UINT32 ui32AllocationIndex,
 999                                                         IMG_UINT32 *pui32AllocationIndexOut)
1000 {
1001         IMG_BOOL bSparse = PMR_IsSparse(psPMR);
1002         IMG_UINT64 ui64Serial;
1003         IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
1004         PVRSRV_ERROR eError;
1005         IMG_BOOL bCreated;
1006
1007         if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1008                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1009         {
1010                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1011                                                                 __func__,
1012                                                                 ui32AllocationIndex));
1013                 return PVRSRV_ERROR_INVALID_PARAMS;
1014         }
1015
1016         PMRGetUID(psPMR, &ui64Serial);
1017
1018         DevicememHistoryLock();
1019
1020         eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
1021                                                 ui32AllocationIndex,
1022                                                 ui64Serial,
1023                                                 sDevVAddr,
1024                                                 uiSize,
1025                                                 szName,
1026                                                 ui32Log2PageSize,
1027                                                 uiPID,
1028                                                 bSparse,
1029                                                 &ui32AllocationIndex,
1030                                                 &bCreated);
1031
1032         if ((eError == PVRSRV_OK) && !bCreated)
1033         {
1034                 /* touch the allocation so it goes to the head of our MRU list */
1035                 TouchBusyAllocation(ui32AllocationIndex);
1036         }
1037         else if (eError != PVRSRV_OK)
1038         {
1039                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1040                                                                         __func__,
1041                                                                         szName,
1042                                                                         PVRSRVGETERRORSTRING(eError)));
1043                 goto out_unlock;
1044         }
1045
1046         if (!bSparse)
1047         {
1048                 InsertMapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
1049         }
1050         else
1051         {
1052                 GenerateMapUnmapCommandsForSparsePMR(psPMR,
1053                                                                 ui32AllocationIndex,
1054                                                                 IMG_TRUE);
1055         }
1056
1057         InsertTimeStampCommand(OSClockns64());
1058
1059         *pui32AllocationIndexOut = ui32AllocationIndex;
1060
1061 out_unlock:
1062         DevicememHistoryUnlock();
1063
1064         return eError;
1065 }
1066
1067 static void VRangeInsertMapUnmapCommands(PVRSRV_DEVICE_NODE *psDeviceNode,
1068                                                         IMG_BOOL bMap,
1069                                                         IMG_UINT32 ui32AllocationIndex,
1070                                                         IMG_DEV_VIRTADDR sBaseDevVAddr,
1071                                                         IMG_UINT32 ui32StartPage,
1072                                                         IMG_UINT32 ui32NumPages,
1073                                                         const IMG_CHAR *pszName)
1074 {
1075         while (ui32NumPages > 0)
1076         {
1077                 IMG_UINT32 ui32PagesToAdd;
1078
1079                 ui32PagesToAdd = MIN(ui32NumPages, MAP_RANGE_MAX_RANGE);
1080
1081                 if (ui32StartPage > MAP_RANGE_MAX_START)
1082                 {
1083                         PVR_DPF((PVR_DBG_WARNING, "Cannot record %s range beginning at page "
1084                                                                         "%u on allocation %s",
1085                                                                         bMap ? "map" : "unmap",
1086                                                                         ui32StartPage,
1087                                                                         pszName));
1088                         return;
1089                 }
1090
1091                 if (bMap)
1092                 {
1093                         InsertMapRangeCommand(psDeviceNode,
1094                                                                 ui32AllocationIndex,
1095                                                                 ui32StartPage,
1096                                                                 ui32PagesToAdd);
1097                 }
1098                 else
1099                 {
1100                         InsertUnmapRangeCommand(psDeviceNode,
1101                                                                 ui32AllocationIndex,
1102                                                                 ui32StartPage,
1103                                                                 ui32PagesToAdd);
1104                 }
1105
1106                 ui32StartPage += ui32PagesToAdd;
1107                 ui32NumPages -= ui32PagesToAdd;
1108         }
1109 }
1110
1111 PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
1112                                                 PVRSRV_DEVICE_NODE *psDeviceNode,
1113                                                 IMG_DEV_VIRTADDR sBaseDevVAddr,
1114                                                 IMG_UINT32 ui32StartPage,
1115                                                 IMG_UINT32 ui32NumPages,
1116                                                 IMG_DEVMEM_SIZE_T uiAllocSize,
1117                                                 const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
1118                                                 IMG_UINT32 ui32Log2PageSize,
1119                                                 IMG_UINT32 ui32AllocationIndex,
1120                                                 IMG_UINT32 *pui32AllocationIndexOut)
1121 {
1122         IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
1123         PVRSRV_ERROR eError;
1124         IMG_BOOL bCreated;
1125
1126         PVR_UNREFERENCED_PARAMETER(psConnection);
1127
1128         if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1129                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1130         {
1131                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1132                                                                 __func__,
1133                                                         ui32AllocationIndex));
1134                 return PVRSRV_ERROR_INVALID_PARAMS;
1135         }
1136
1137         DevicememHistoryLock();
1138
1139         eError = FindOrCreateAllocation(psDeviceNode,
1140                                                 ui32AllocationIndex,
1141                                                 0,
1142                                                 sBaseDevVAddr,
1143                                                 uiAllocSize,
1144                                                 szName,
1145                                                 ui32Log2PageSize,
1146                                                 uiPID,
1147                                                 IMG_FALSE,
1148                                                 &ui32AllocationIndex,
1149                                                 &bCreated);
1150
1151         if ((eError == PVRSRV_OK) && !bCreated)
1152         {
1153                 /* touch the allocation so it goes to the head of our MRU list */
1154                 TouchBusyAllocation(ui32AllocationIndex);
1155         }
1156         else if (eError != PVRSRV_OK)
1157         {
1158                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1159                                                                         __func__,
1160                                                                         szName,
1161                                                                         PVRSRVGETERRORSTRING(eError)));
1162                 goto out_unlock;
1163         }
1164
1165         VRangeInsertMapUnmapCommands(psDeviceNode,
1166                                                 IMG_TRUE,
1167                                                 ui32AllocationIndex,
1168                                                 sBaseDevVAddr,
1169                                                 ui32StartPage,
1170                                                 ui32NumPages,
1171                                                 szName);
1172
1173         *pui32AllocationIndexOut = ui32AllocationIndex;
1174
1175 out_unlock:
1176         DevicememHistoryUnlock();
1177
1178         return eError;
1179
1180 }
1181
1182 PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
1183                                                 PVRSRV_DEVICE_NODE *psDeviceNode,
1184                                                 IMG_DEV_VIRTADDR sBaseDevVAddr,
1185                                                 IMG_UINT32 ui32StartPage,
1186                                                 IMG_UINT32 ui32NumPages,
1187                                                 IMG_DEVMEM_SIZE_T uiAllocSize,
1188                                                 const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
1189                                                 IMG_UINT32 ui32Log2PageSize,
1190                                                 IMG_UINT32 ui32AllocationIndex,
1191                                                 IMG_UINT32 *pui32AllocationIndexOut)
1192 {
1193         IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
1194         PVRSRV_ERROR eError;
1195         IMG_BOOL bCreated;
1196
1197         PVR_UNREFERENCED_PARAMETER(psConnection);
1198
1199         if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1200                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1201         {
1202                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1203                                                                 __func__,
1204                                                         ui32AllocationIndex));
1205                 return PVRSRV_ERROR_INVALID_PARAMS;
1206         }
1207
1208         DevicememHistoryLock();
1209
1210         eError = FindOrCreateAllocation(psDeviceNode,
1211                                                 ui32AllocationIndex,
1212                                                 0,
1213                                                 sBaseDevVAddr,
1214                                                 uiAllocSize,
1215                                                 szName,
1216                                                 ui32Log2PageSize,
1217                                                 uiPID,
1218                                                 IMG_FALSE,
1219                                                 &ui32AllocationIndex,
1220                                                 &bCreated);
1221
1222         if ((eError == PVRSRV_OK) && !bCreated)
1223         {
1224                 /* touch the allocation so it goes to the head of our MRU list */
1225                 TouchBusyAllocation(ui32AllocationIndex);
1226         }
1227         else if (eError != PVRSRV_OK)
1228         {
1229                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1230                                                                         __func__,
1231                                                                         szName,
1232                                                                         PVRSRVGETERRORSTRING(eError)));
1233                 goto out_unlock;
1234         }
1235
1236         VRangeInsertMapUnmapCommands(psDeviceNode,
1237                                                 IMG_FALSE,
1238                                                 ui32AllocationIndex,
1239                                                 sBaseDevVAddr,
1240                                                 ui32StartPage,
1241                                                 ui32NumPages,
1242                                                 szName);
1243
1244         *pui32AllocationIndexOut = ui32AllocationIndex;
1245
1246 out_unlock:
1247         DevicememHistoryUnlock();
1248
1249         return eError;
1250 }
1251
1252
1253
1254 /* DevicememHistoryUnmapKM:
1255  * Entry point for when an allocation is unmapped from the MMU GPU
1256  *
1257  * psPMR: The PMR to which the allocation belongs.
1258  * ui32Offset: The offset within the PMR at which the allocation begins.
1259  * sDevVAddr: The DevVAddr at which the allocation begins.
1260  * szName: Annotation/name for the allocation.
1261  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
1262  * ui32AllocationIndex: Allocation index as provided by the client.
1263  *                      We will use this as a short-cut to find the allocation
1264  *                      in our records.
1265  * pui32AllocationIndexOut: An updated allocation index for the client.
1266  *                          This may be a new value if we just created the
1267  *                          allocation record.
1268  */
1269 PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
1270                                                         IMG_UINT32 ui32Offset,
1271                                                         IMG_DEV_VIRTADDR sDevVAddr,
1272                                                         IMG_DEVMEM_SIZE_T uiSize,
1273                                                         const char szName[DEVMEM_ANNOTATION_MAX_LEN],
1274                                                         IMG_UINT32 ui32Log2PageSize,
1275                                                         IMG_UINT32 ui32AllocationIndex,
1276                                                         IMG_UINT32 *pui32AllocationIndexOut)
1277 {
1278         IMG_BOOL bSparse = PMR_IsSparse(psPMR);
1279         IMG_UINT64 ui64Serial;
1280         IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
1281         PVRSRV_ERROR eError;
1282         IMG_BOOL bCreated;
1283
1284         if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1285                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1286         {
1287                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1288                                                                 __func__,
1289                                                                 ui32AllocationIndex));
1290                 return PVRSRV_ERROR_INVALID_PARAMS;
1291         }
1292
1293         PMRGetUID(psPMR, &ui64Serial);
1294
1295         DevicememHistoryLock();
1296
1297         eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
1298                                                 ui32AllocationIndex,
1299                                                 ui64Serial,
1300                                                 sDevVAddr,
1301                                                 uiSize,
1302                                                 szName,
1303                                                 ui32Log2PageSize,
1304                                                 uiPID,
1305                                                 bSparse,
1306                                                 &ui32AllocationIndex,
1307                                                 &bCreated);
1308
1309         if ((eError == PVRSRV_OK) && !bCreated)
1310         {
1311                 /* touch the allocation so it goes to the head of our MRU list */
1312                 TouchBusyAllocation(ui32AllocationIndex);
1313         }
1314         else if (eError != PVRSRV_OK)
1315         {
1316                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1317                                                                         __func__,
1318                                                                         szName,
1319                                                                         PVRSRVGETERRORSTRING(eError)));
1320                 goto out_unlock;
1321         }
1322
1323         if (!bSparse)
1324         {
1325                 InsertUnmapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
1326         }
1327         else
1328         {
1329                 GenerateMapUnmapCommandsForSparsePMR(psPMR,
1330                                                                 ui32AllocationIndex,
1331                                                                 IMG_FALSE);
1332         }
1333
1334         InsertTimeStampCommand(OSClockns64());
1335
1336         *pui32AllocationIndexOut = ui32AllocationIndex;
1337
1338 out_unlock:
1339         DevicememHistoryUnlock();
1340
1341         return eError;
1342 }
1343
1344 /* DevicememHistorySparseChangeKM:
1345  * Entry point for when a sparse allocation is changed, such that some of the
1346  * pages within the sparse allocation are mapped or unmapped.
1347  *
1348  * psPMR: The PMR to which the allocation belongs.
1349  * ui32Offset: The offset within the PMR at which the allocation begins.
1350  * sDevVAddr: The DevVAddr at which the allocation begins.
1351  * szName: Annotation/name for the allocation.
1352  * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
1353  * ui32AllocPageCount: Number of pages which have been mapped.
1354  * paui32AllocPageIndices: Indices of pages which have been mapped.
1355  * ui32FreePageCount: Number of pages which have been unmapped.
1356  * paui32FreePageIndices: Indices of pages which have been unmapped.
1357  * ui32AllocationIndex: Allocation index as provided by the client.
1358  *                      We will use this as a short-cut to find the allocation
1359  *                      in our records.
1360  * pui32AllocationIndexOut: An updated allocation index for the client.
1361  *                          This may be a new value if we just created the
1362  *                          allocation record.
1363  */
1364 PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
1365                                                         IMG_UINT32 ui32Offset,
1366                                                         IMG_DEV_VIRTADDR sDevVAddr,
1367                                                         IMG_DEVMEM_SIZE_T uiSize,
1368                                                         const char szName[DEVMEM_ANNOTATION_MAX_LEN],
1369                                                         IMG_UINT32 ui32Log2PageSize,
1370                                                         IMG_UINT32 ui32AllocPageCount,
1371                                                         IMG_UINT32 *paui32AllocPageIndices,
1372                                                         IMG_UINT32 ui32FreePageCount,
1373                                                         IMG_UINT32 *paui32FreePageIndices,
1374                                                         IMG_UINT32 ui32AllocationIndex,
1375                                                         IMG_UINT32 *pui32AllocationIndexOut)
1376 {
1377         IMG_UINT64 ui64Serial;
1378         IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
1379         PVRSRV_ERROR eError;
1380         IMG_BOOL bCreated;
1381
1382         if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
1383                 !CHECK_ALLOC_INDEX(ui32AllocationIndex))
1384         {
1385                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
1386                                                                 __func__,
1387                                                                 ui32AllocationIndex));
1388                 return PVRSRV_ERROR_INVALID_PARAMS;
1389         }
1390
1391         PMRGetUID(psPMR, &ui64Serial);
1392
1393         DevicememHistoryLock();
1394
1395         eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
1396                                                 ui32AllocationIndex,
1397                                                 ui64Serial,
1398                                                 sDevVAddr,
1399                                                 uiSize,
1400                                                 szName,
1401                                                 ui32Log2PageSize,
1402                                                 uiPID,
1403                                                 IMG_TRUE /* bSparse */,
1404                                                 &ui32AllocationIndex,
1405                                                 &bCreated);
1406
1407         if ((eError == PVRSRV_OK) && !bCreated)
1408         {
1409                 /* touch the allocation so it goes to the head of our MRU list */
1410                 TouchBusyAllocation(ui32AllocationIndex);
1411         }
1412         else if (eError != PVRSRV_OK)
1413         {
1414                 PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
1415                                                                         __func__,
1416                                                                         szName,
1417                                                                         PVRSRVGETERRORSTRING(eError)));
1418                 goto out_unlock;
1419         }
1420
1421         GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
1422                                                         ui32AllocPageCount,
1423                                                         paui32AllocPageIndices,
1424                                                         ui32AllocationIndex,
1425                                                         IMG_TRUE);
1426
1427         GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
1428                                                         ui32FreePageCount,
1429                                                         paui32FreePageIndices,
1430                                                         ui32AllocationIndex,
1431                                                         IMG_FALSE);
1432
1433         InsertTimeStampCommand(OSClockns64());
1434
1435         *pui32AllocationIndexOut = ui32AllocationIndex;
1436
1437 out_unlock:
1438         DevicememHistoryUnlock();
1439
1440         return eError;
1441
1442 }
1443
1444 /* CircularBufferIterateStart:
1445  * Initialise local state for iterating over the circular buffer
1446  */
1447 static void CircularBufferIterateStart(IMG_UINT32 *pui32Head, IMG_UINT32 *pui32Iter)
1448 {
1449         *pui32Head = gsDevicememHistoryData.sRecords.ui32Head;
1450
1451         if (*pui32Head != 0)
1452         {
1453                 *pui32Iter = *pui32Head - 1;
1454         }
1455         else
1456         {
1457                 *pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
1458         }
1459 }
1460
1461 /* CircularBufferIteratePrevious:
1462  * Iterate to the previous item in the circular buffer.
1463  * This is called repeatedly to iterate over the whole circular buffer.
1464  */
1465 static COMMAND_WRAPPER *CircularBufferIteratePrevious(IMG_UINT32 ui32Head,
1466                                                         IMG_UINT32 *pui32Iter,
1467                                                         COMMAND_TYPE *peType,
1468                                                         IMG_BOOL *pbLast)
1469 {
1470         IMG_UINT8 *pui8Header;
1471         COMMAND_WRAPPER *psOut = NULL;
1472
1473         psOut = gsDevicememHistoryData.sRecords.pasCircularBuffer + *pui32Iter;
1474
1475         pui8Header = (void *) psOut;
1476
1477         /* Check the command looks valid.
1478          * this condition should never happen, but check for it anyway
1479          * and try to handle it
1480          */
1481         if (*pui8Header >= COMMAND_TYPE_COUNT)
1482         {
1483                 /* invalid header detected. Circular buffer corrupted? */
1484                 PVR_DPF((PVR_DBG_ERROR, "CircularBufferIteratePrevious: "
1485                                                         "Invalid header: %u",
1486                                                         *pui8Header));
1487                 *pbLast = IMG_TRUE;
1488                 return NULL;
1489         }
1490
1491         *peType = *pui8Header;
1492
1493         if (*pui32Iter != 0)
1494         {
1495                 (*pui32Iter)--;
1496         }
1497         else
1498         {
1499                 *pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
1500         }
1501
1502
1503         /* inform the caller this is the last command if either we have reached
1504          * the head (where we started) or if we have reached an empty command,
1505          * which means we have covered all populated entries
1506          */
1507         if ((*pui32Iter == ui32Head) || (*peType == COMMAND_TYPE_NONE))
1508         {
1509                 /* this is the final iteration */
1510                 *pbLast = IMG_TRUE;
1511         }
1512
1513         return psOut;
1514 }
1515
1516 /* MapUnmapCommandGetInfo:
1517  * Helper function to get the address and mapping information from a MAP_ALL, UNMAP_ALL,
1518  * MAP_RANGE or UNMAP_RANGE command
1519  */
1520 static void MapUnmapCommandGetInfo(COMMAND_WRAPPER *psCommand,
1521                                         COMMAND_TYPE eType,
1522                                         IMG_DEV_VIRTADDR *psDevVAddrStart,
1523                                         IMG_DEV_VIRTADDR *psDevVAddrEnd,
1524                                         IMG_BOOL *pbMap,
1525                                         IMG_UINT32 *pui32AllocIndex)
1526 {
1527         if ((eType == COMMAND_TYPE_MAP_ALL) || ((eType == COMMAND_TYPE_UNMAP_ALL)))
1528         {
1529                 COMMAND_MAP_ALL *psMapAll = &psCommand->u.sMapAll;
1530                 RECORD_ALLOCATION *psAlloc;
1531
1532                 *pbMap = (eType == COMMAND_TYPE_MAP_ALL);
1533                 *pui32AllocIndex = psMapAll->uiAllocIndex;
1534
1535                 psAlloc = ALLOC_INDEX_TO_PTR(psMapAll->uiAllocIndex);
1536
1537                 *psDevVAddrStart = psAlloc->sDevVAddr;
1538                 psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr + psAlloc->uiSize - 1;
1539         }
1540         else if ((eType == COMMAND_TYPE_MAP_RANGE) || ((eType == COMMAND_TYPE_UNMAP_RANGE)))
1541         {
1542                 COMMAND_MAP_RANGE *psMapRange = &psCommand->u.sMapRange;
1543                 RECORD_ALLOCATION *psAlloc;
1544                 IMG_UINT32 ui32StartPage, ui32Count;
1545
1546                 *pbMap = (eType == COMMAND_TYPE_MAP_RANGE);
1547                 *pui32AllocIndex = psMapRange->uiAllocIndex;
1548
1549                 psAlloc = ALLOC_INDEX_TO_PTR(psMapRange->uiAllocIndex);
1550
1551                 MapRangeUnpack(psMapRange, &ui32StartPage, &ui32Count);
1552
1553                 psDevVAddrStart->uiAddr = psAlloc->sDevVAddr.uiAddr +
1554                                 ((1ULL << psAlloc->ui32Log2PageSize) * ui32StartPage);
1555
1556                 psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr +
1557                                 ((1ULL << psAlloc->ui32Log2PageSize) * ui32Count) - 1;
1558         }
1559         else
1560         {
1561                 PVR_DPF((PVR_DBG_ERROR, "%s: Invalid command type: %u",
1562                                                                 __func__,
1563                                                                 eType));
1564         }
1565 }
1566
1567 /* DevicememHistoryQuery:
1568  * Entry point for rgxdebug to look up addresses relating to a page fault
1569  */
1570 IMG_BOOL DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
1571                                DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
1572                                IMG_UINT32 ui32PageSizeBytes,
1573                                IMG_BOOL bMatchAnyAllocInPage)
1574 {
1575         IMG_UINT32 ui32Head, ui32Iter;
1576         COMMAND_TYPE eType = COMMAND_TYPE_NONE;
1577         COMMAND_WRAPPER *psCommand = NULL;
1578         IMG_BOOL bLast = IMG_FALSE;
1579         IMG_UINT64 ui64StartTime = OSClockns64();
1580         IMG_UINT64 ui64TimeNs = 0;
1581
1582         /* initialise the results count for the caller */
1583         psQueryOut->ui32NumResults = 0;
1584
1585         DevicememHistoryLock();
1586
1587         /* if the search is constrained to a particular PID then we
1588          * first search the list of allocations to see if this
1589          * PID is known to us
1590          */
1591         if (psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY)
1592         {
1593                 IMG_UINT32 ui32Alloc;
1594                 ui32Alloc = gsDevicememHistoryData.sRecords.ui32AllocationsListHead;
1595
1596                 while (ui32Alloc != END_OF_LIST)
1597                 {
1598                         RECORD_ALLOCATION *psAlloc;
1599
1600                         psAlloc = ALLOC_INDEX_TO_PTR(ui32Alloc);
1601
1602                         if (psAlloc->uiPID == psQueryIn->uiPID)
1603                         {
1604                                 goto found_pid;
1605                         }
1606
1607                         if (ui32Alloc == gsDevicememHistoryData.sRecords.ui32AllocationsListHead)
1608                         {
1609                                 /* gone through whole list */
1610                                 break;
1611                         }
1612                 }
1613
1614                 /* PID not found, so we do not have any suitable data for this
1615                  * page fault
1616                  */
1617                 goto out_unlock;
1618         }
1619
1620 found_pid:
1621
1622         CircularBufferIterateStart(&ui32Head, &ui32Iter);
1623
1624         while (!bLast)
1625         {
1626                 psCommand = CircularBufferIteratePrevious(ui32Head, &ui32Iter, &eType, &bLast);
1627
1628                 if (eType == COMMAND_TYPE_TIMESTAMP)
1629                 {
1630                         ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
1631                         continue;
1632                 }
1633
1634                 if ((eType == COMMAND_TYPE_MAP_ALL) ||
1635                         (eType == COMMAND_TYPE_UNMAP_ALL) ||
1636                         (eType == COMMAND_TYPE_MAP_RANGE) ||
1637                         (eType == COMMAND_TYPE_UNMAP_RANGE))
1638                 {
1639                         RECORD_ALLOCATION *psAlloc;
1640                         IMG_DEV_VIRTADDR sAllocStartAddrOrig, sAllocEndAddrOrig;
1641                         IMG_DEV_VIRTADDR sAllocStartAddr, sAllocEndAddr;
1642                         IMG_BOOL bMap;
1643                         IMG_UINT32 ui32AllocIndex;
1644
1645                         MapUnmapCommandGetInfo(psCommand,
1646                                                         eType,
1647                                                         &sAllocStartAddrOrig,
1648                                                         &sAllocEndAddrOrig,
1649                                                         &bMap,
1650                                                         &ui32AllocIndex);
1651
1652                         sAllocStartAddr = sAllocStartAddrOrig;
1653                         sAllocEndAddr = sAllocEndAddrOrig;
1654
1655                         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocIndex);
1656
1657                         /* skip this command if we need to search within
1658                          * a particular PID, and this allocation is not from
1659                          * that PID
1660                          */
1661                         if ((psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY) &&
1662                                 (psAlloc->uiPID != psQueryIn->uiPID))
1663                         {
1664                                 continue;
1665                         }
1666
1667                         /* if the allocation was created after this event, then this
1668                          * event must be for an old/removed allocation, so skip it
1669                          */
1670                         if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
1671                         {
1672                                 continue;
1673                         }
1674
1675                         /* if the caller wants us to match any allocation in the
1676                          * same page as the allocation then tweak the real start/end
1677                          * addresses of the allocation here
1678                          */
1679                         if (bMatchAnyAllocInPage)
1680                         {
1681                                 sAllocStartAddr.uiAddr = sAllocStartAddr.uiAddr & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
1682                                 sAllocEndAddr.uiAddr = (sAllocEndAddr.uiAddr + ui32PageSizeBytes - 1) & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
1683                         }
1684
1685                         if ((psQueryIn->sDevVAddr.uiAddr >= sAllocStartAddr.uiAddr) &&
1686                                 (psQueryIn->sDevVAddr.uiAddr <  sAllocEndAddr.uiAddr))
1687                         {
1688                                 DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult = &psQueryOut->sResults[psQueryOut->ui32NumResults];
1689
1690                                 OSStringLCopy(psResult->szString, psAlloc->szName, sizeof(psResult->szString));
1691                                 psResult->sBaseDevVAddr = psAlloc->sDevVAddr;
1692                                 psResult->uiSize = psAlloc->uiSize;
1693                                 psResult->bMap = bMap;
1694                                 psResult->ui64Age = _CalculateAge(ui64StartTime, ui64TimeNs, TIME_STAMP_MASK);
1695                                 psResult->ui64When = ui64TimeNs;
1696                                 /* write the responsible PID in the placeholder */
1697                                 psResult->sProcessInfo.uiPID = psAlloc->uiPID;
1698
1699                                 if ((eType == COMMAND_TYPE_MAP_ALL) || (eType == COMMAND_TYPE_UNMAP_ALL))
1700                                 {
1701                                         psResult->bRange = IMG_FALSE;
1702                                         psResult->bAll = IMG_TRUE;
1703                                 }
1704                                 else
1705                                 {
1706                                         psResult->bRange = IMG_TRUE;
1707                                         MapRangeUnpack(&psCommand->u.sMapRange,
1708                                                                                 &psResult->ui32StartPage,
1709                                                                                 &psResult->ui32PageCount);
1710                                         psResult->bAll = (psResult->ui32PageCount * (1U << psAlloc->ui32Log2PageSize))
1711                                                                                         == psAlloc->uiSize;
1712                                         psResult->sMapStartAddr = sAllocStartAddrOrig;
1713                                         psResult->sMapEndAddr = sAllocEndAddrOrig;
1714                                 }
1715
1716                                 psQueryOut->ui32NumResults++;
1717
1718                                 if (psQueryOut->ui32NumResults == DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS)
1719                                 {
1720                                         break;
1721                                 }
1722                         }
1723                 }
1724         }
1725
1726 out_unlock:
1727         DevicememHistoryUnlock();
1728
1729         return psQueryOut->ui32NumResults > 0;
1730 }
1731
1732 static void DeviceMemHistoryFmt(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN],
1733                                                         IMG_PID uiPID,
1734                                                         const IMG_CHAR *pszName,
1735                                                         const IMG_CHAR *pszAction,
1736                                                         IMG_DEV_VIRTADDR sDevVAddrStart,
1737                                                         IMG_DEV_VIRTADDR sDevVAddrEnd,
1738                                                         IMG_UINT64 ui64TimeNs)
1739 {
1740
1741         OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
1742                                 /* PID NAME MAP/UNMAP MIN-MAX SIZE AbsUS AgeUS*/
1743                                 "%04u %-40s %-10s "
1744                                 IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC " "
1745                                 "0x%08" IMG_UINT64_FMTSPECX " "
1746                                 "%013" IMG_UINT64_FMTSPEC, /* 13 digits is over 2 hours of ns */
1747                                 uiPID,
1748                                 pszName,
1749                                 pszAction,
1750                                 sDevVAddrStart.uiAddr,
1751                                 sDevVAddrEnd.uiAddr,
1752                                 sDevVAddrEnd.uiAddr - sDevVAddrStart.uiAddr + 1,
1753                                 ui64TimeNs);
1754 }
1755
1756 static void DeviceMemHistoryFmtHeader(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN])
1757 {
1758         OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
1759                                 "%-4s %-40s %-6s   %10s   %10s   %8s %13s",
1760                                 "PID",
1761                                 "NAME",
1762                                 "ACTION",
1763                                 "ADDR MIN",
1764                                 "ADDR MAX",
1765                                 "SIZE",
1766                                 "ABS NS");
1767 }
1768
1769 static const char *CommandTypeToString(COMMAND_TYPE eType)
1770 {
1771         switch (eType)
1772         {
1773                 case COMMAND_TYPE_MAP_ALL:
1774                         return "MapAll";
1775                 case COMMAND_TYPE_UNMAP_ALL:
1776                         return "UnmapAll";
1777                 case COMMAND_TYPE_MAP_RANGE:
1778                         return "MapRange";
1779                 case COMMAND_TYPE_UNMAP_RANGE:
1780                         return "UnmapRange";
1781                 case COMMAND_TYPE_TIMESTAMP:
1782                         return "TimeStamp";
1783                 default:
1784                         return "???";
1785         }
1786 }
1787
1788 static void DevicememHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
1789 {
1790         IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
1791         IMG_UINT32 ui32Iter;
1792         IMG_UINT32 ui32Head;
1793         IMG_BOOL bLast = IMG_FALSE;
1794         IMG_UINT64 ui64TimeNs = 0;
1795         IMG_UINT64 ui64StartTime = OSClockns64();
1796
1797         DeviceMemHistoryFmtHeader(szBuffer);
1798         DIPrintf(psEntry, "%s\n", szBuffer);
1799
1800         CircularBufferIterateStart(&ui32Head, &ui32Iter);
1801
1802         while (!bLast)
1803         {
1804                 COMMAND_WRAPPER *psCommand;
1805                 COMMAND_TYPE eType = COMMAND_TYPE_NONE;
1806
1807                 psCommand = CircularBufferIteratePrevious(ui32Head, &ui32Iter, &eType,
1808                                                           &bLast);
1809
1810                 if (eType == COMMAND_TYPE_TIMESTAMP)
1811                 {
1812                         ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
1813                         continue;
1814                 }
1815
1816
1817                 if ((eType == COMMAND_TYPE_MAP_ALL) ||
1818                         (eType == COMMAND_TYPE_UNMAP_ALL) ||
1819                         (eType == COMMAND_TYPE_MAP_RANGE) ||
1820                         (eType == COMMAND_TYPE_UNMAP_RANGE))
1821                 {
1822                         RECORD_ALLOCATION *psAlloc;
1823                         IMG_DEV_VIRTADDR sDevVAddrStart, sDevVAddrEnd;
1824                         IMG_BOOL bMap;
1825                         IMG_UINT32 ui32AllocIndex;
1826
1827                         MapUnmapCommandGetInfo(psCommand,
1828                                                eType,
1829                                                &sDevVAddrStart,
1830                                                &sDevVAddrEnd,
1831                                                &bMap,
1832                                                &ui32AllocIndex);
1833
1834                         psAlloc = ALLOC_INDEX_TO_PTR(ui32AllocIndex);
1835
1836                         if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
1837                         {
1838                                 /* if this event relates to an allocation we
1839                                  * are no longer tracking then do not print it
1840                                  */
1841                                 continue;
1842                         }
1843
1844                         DeviceMemHistoryFmt(szBuffer,
1845                                             psAlloc->uiPID,
1846                                             psAlloc->szName,
1847                                             CommandTypeToString(eType),
1848                                             sDevVAddrStart,
1849                                             sDevVAddrEnd,
1850                                             ui64TimeNs);
1851
1852                         DIPrintf(psEntry, "%s\n", szBuffer);
1853                 }
1854         }
1855
1856         DIPrintf(psEntry, "\nTimestamp reference: %013" IMG_UINT64_FMTSPEC "\n",
1857                  ui64StartTime);
1858 }
1859
1860 static int DevicememHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry,
1861                                            void *pvData)
1862 {
1863         PVR_UNREFERENCED_PARAMETER(pvData);
1864
1865         DevicememHistoryLock();
1866         DevicememHistoryPrintAll(psEntry);
1867         DevicememHistoryUnlock();
1868
1869         return 0;
1870 }
1871
1872 static PVRSRV_ERROR CreateRecords(void)
1873 {
1874         gsDevicememHistoryData.sRecords.pasAllocations =
1875                         OSAllocMem(sizeof(RECORD_ALLOCATION) * ALLOCATION_LIST_NUM_ENTRIES);
1876
1877         PVR_RETURN_IF_NOMEM(gsDevicememHistoryData.sRecords.pasAllocations);
1878
1879         /* Allocated and initialise the circular buffer with zeros so every
1880          * command is initialised as a command of type COMMAND_TYPE_NONE. */
1881         gsDevicememHistoryData.sRecords.pasCircularBuffer =
1882                         OSAllocZMem(sizeof(COMMAND_WRAPPER) * CIRCULAR_BUFFER_NUM_COMMANDS);
1883
1884         if (gsDevicememHistoryData.sRecords.pasCircularBuffer == NULL)
1885         {
1886                 OSFreeMem(gsDevicememHistoryData.sRecords.pasAllocations);
1887                 return PVRSRV_ERROR_OUT_OF_MEMORY;
1888         }
1889
1890         return PVRSRV_OK;
1891 }
1892
1893 static void DestroyRecords(void)
1894 {
1895         OSFreeMem(gsDevicememHistoryData.sRecords.pasCircularBuffer);
1896         OSFreeMem(gsDevicememHistoryData.sRecords.pasAllocations);
1897 }
1898
1899 static void InitialiseRecords(void)
1900 {
1901         IMG_UINT32 i;
1902
1903         /* initialise the allocations list */
1904
1905         gsDevicememHistoryData.sRecords.pasAllocations[0].ui32Prev = ALLOCATION_LIST_NUM_ENTRIES - 1;
1906         gsDevicememHistoryData.sRecords.pasAllocations[0].ui32Next = 1;
1907
1908         for (i = 1; i < ALLOCATION_LIST_NUM_ENTRIES; i++)
1909         {
1910                 gsDevicememHistoryData.sRecords.pasAllocations[i].ui32Prev = i - 1;
1911                 gsDevicememHistoryData.sRecords.pasAllocations[i].ui32Next = i + 1;
1912         }
1913
1914         gsDevicememHistoryData.sRecords.pasAllocations[ALLOCATION_LIST_NUM_ENTRIES - 1].ui32Next = 0;
1915
1916         gsDevicememHistoryData.sRecords.ui32AllocationsListHead = 0;
1917 }
1918
1919 PVRSRV_ERROR DevicememHistoryInitKM(void)
1920 {
1921         PVRSRV_ERROR eError;
1922         DI_ITERATOR_CB sIterator = {.pfnShow = DevicememHistoryPrintAllWrapper};
1923
1924         eError = OSLockCreate(&gsDevicememHistoryData.hLock);
1925         PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", err_lock);
1926
1927         eError = CreateRecords();
1928         PVR_LOG_GOTO_IF_ERROR(eError, "CreateRecords", err_allocations);
1929
1930         InitialiseRecords();
1931
1932         eError = DICreateEntry("devicemem_history", NULL, &sIterator, NULL,
1933                                DI_ENTRY_TYPE_GENERIC,
1934                                &gsDevicememHistoryData.psDIEntry);
1935         PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", err_di_creation);
1936
1937         return PVRSRV_OK;
1938
1939 err_di_creation:
1940         DestroyRecords();
1941 err_allocations:
1942         OSLockDestroy(gsDevicememHistoryData.hLock);
1943         gsDevicememHistoryData.hLock = NULL;
1944 err_lock:
1945         return eError;
1946 }
1947
1948 void DevicememHistoryDeInitKM(void)
1949 {
1950         if (gsDevicememHistoryData.psDIEntry != NULL)
1951         {
1952                 DIDestroyEntry(gsDevicememHistoryData.psDIEntry);
1953         }
1954
1955         DestroyRecords();
1956
1957         if (gsDevicememHistoryData.hLock != NULL)
1958         {
1959                 OSLockDestroy(gsDevicememHistoryData.hLock);
1960                 gsDevicememHistoryData.hLock = NULL;
1961         }
1962 }