drivers/gpu/drm/radeon/cik_sdma.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <drm/drmP.h>
  26 #include "radeon.h"
  27 #include "radeon_asic.h"
  28 #include "cikd.h"
  29
  30 /* sdma */
  31 #define CIK_SDMA_UCODE_SIZE 1050
  32 #define CIK_SDMA_UCODE_VERSION 64
  33
  34 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev);
  35
  36 /*
  37  * sDMA - System DMA
  38  * Starting with CIK, the GPU has new asynchronous
  39  * DMA engines.  These engines are used for compute
  40  * and gfx.  There are two DMA engines (SDMA0, SDMA1)
  41  * and each one supports 1 ring buffer used for gfx
  42  * and 2 queues used for compute.
  43  *
  44  * The programming model is very similar to the CP
  45  * (ring buffer, IBs, etc.), but sDMA has it's own
  46  * packet format that is different from the PM4 format
  47  * used by the CP. sDMA supports copying data, writing
  48  * embedded data, solid fills, and a number of other
  49  * things.  It also has support for tiling/detiling of
  50  * buffers.
  51  */
  52
  53 /**
  54  * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
  55  *
  56  * @rdev: radeon_device pointer
  57  * @ib: IB object to schedule
  58  *
  59  * Schedule an IB in the DMA ring (CIK).
  60  */
  61 void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
  62                               struct radeon_ib *ib)
  63 {
  64         struct radeon_ring *ring = &rdev->ring[ib->ring];
  65         u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
  66
  67         if (rdev->wb.enabled) {
  68                 u32 next_rptr = ring->wptr + 5;
  69                 while ((next_rptr & 7) != 4)
  70                         next_rptr++;
  71                 next_rptr += 4;
  72                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
  73                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
  74                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
  75                 radeon_ring_write(ring, 1); /* number of DWs to follow */
  76                 radeon_ring_write(ring, next_rptr);
  77         }
  78
  79         /* IB packet must end on a 8 DW boundary */
  80         while ((ring->wptr & 7) != 4)
  81                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
  82         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
  83         radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
  84         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
  85         radeon_ring_write(ring, ib->length_dw);
  86
  87 }
  88
  89 /**
  90  * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
  91  *
  92  * @rdev: radeon_device pointer
  93  * @fence: radeon fence object
  94  *
  95  * Add a DMA fence packet to the ring to write
  96  * the fence seq number and DMA trap packet to generate
  97  * an interrupt if needed (CIK).
  98  */
  99 void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
 100                               struct radeon_fence *fence)
 101 {
 102         struct radeon_ring *ring = &rdev->ring[fence->ring];
 103         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
 104         u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
 105                           SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
 106         u32 ref_and_mask;
 107
 108         if (fence->ring == R600_RING_TYPE_DMA_INDEX)
 109                 ref_and_mask = SDMA0;
 110         else
 111                 ref_and_mask = SDMA1;
 112
 113         /* write the fence */
 114         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
 115         radeon_ring_write(ring, addr & 0xffffffff);
 116         radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
 117         radeon_ring_write(ring, fence->seq);
 118         /* generate an interrupt */
 119         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
 120         /* flush HDP */
 121         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
 122         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
 123         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
 124         radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
 125         radeon_ring_write(ring, ref_and_mask); /* MASK */
 126         radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
 127 }
 128
 129 /**
 130  * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
 131  *
 132  * @rdev: radeon_device pointer
 133  * @ring: radeon_ring structure holding ring information
 134  * @semaphore: radeon semaphore object
 135  * @emit_wait: wait or signal semaphore
 136  *
 137  * Add a DMA semaphore packet to the ring wait on or signal
 138  * other rings (CIK).
 139  */
 140 void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
 141                                   struct radeon_ring *ring,
 142                                   struct radeon_semaphore *semaphore,
 143                                   bool emit_wait)
 144 {
 145         u64 addr = semaphore->gpu_addr;
 146         u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
 147
 148         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
 149         radeon_ring_write(ring, addr & 0xfffffff8);
 150         radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
 151 }
 152
 153 /**
 154  * cik_sdma_gfx_stop - stop the gfx async dma engines
 155  *
 156  * @rdev: radeon_device pointer
 157  *
 158  * Stop the gfx async dma ring buffers (CIK).
 159  */
 160 static void cik_sdma_gfx_stop(struct radeon_device *rdev)
 161 {
 162         u32 rb_cntl, reg_offset;
 163         int i;
 164
 165         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
 166
 167         for (i = 0; i < 2; i++) {
 168                 if (i == 0)
 169                         reg_offset = SDMA0_REGISTER_OFFSET;
 170                 else
 171                         reg_offset = SDMA1_REGISTER_OFFSET;
 172                 rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
 173                 rb_cntl &= ~SDMA_RB_ENABLE;
 174                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
 175                 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
 176         }
 177         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
 178         rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
 179 }
 180
 181 /**
 182  * cik_sdma_rlc_stop - stop the compute async dma engines
 183  *
 184  * @rdev: radeon_device pointer
 185  *
 186  * Stop the compute async dma queues (CIK).
 187  */
 188 static void cik_sdma_rlc_stop(struct radeon_device *rdev)
 189 {
 190         /* XXX todo */
 191 }
 192
 193 /**
 194  * cik_sdma_enable - stop the async dma engines
 195  *
 196  * @rdev: radeon_device pointer
 197  * @enable: enable/disable the DMA MEs.
 198  *
 199  * Halt or unhalt the async dma engines (CIK).
 200  */
 201 void cik_sdma_enable(struct radeon_device *rdev, bool enable)
 202 {
 203         u32 me_cntl, reg_offset;
 204         int i;
 205
 206         for (i = 0; i < 2; i++) {
 207                 if (i == 0)
 208                         reg_offset = SDMA0_REGISTER_OFFSET;
 209                 else
 210                         reg_offset = SDMA1_REGISTER_OFFSET;
 211                 me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
 212                 if (enable)
 213                         me_cntl &= ~SDMA_HALT;
 214                 else
 215                         me_cntl |= SDMA_HALT;
 216                 WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
 217         }
 218 }
 219
 220 /**
 221  * cik_sdma_gfx_resume - setup and start the async dma engines
 222  *
 223  * @rdev: radeon_device pointer
 224  *
 225  * Set up the gfx DMA ring buffers and enable them (CIK).
 226  * Returns 0 for success, error for failure.
 227  */
 228 static int cik_sdma_gfx_resume(struct radeon_device *rdev)
 229 {
 230         struct radeon_ring *ring;
 231         u32 rb_cntl, ib_cntl;
 232         u32 rb_bufsz;
 233         u32 reg_offset, wb_offset;
 234         int i, r;
 235
 236         for (i = 0; i < 2; i++) {
 237                 if (i == 0) {
 238                         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
 239                         reg_offset = SDMA0_REGISTER_OFFSET;
 240                         wb_offset = R600_WB_DMA_RPTR_OFFSET;
 241                 } else {
 242                         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
 243                         reg_offset = SDMA1_REGISTER_OFFSET;
 244                         wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
 245                 }
 246
 247                 WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
 248                 WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
 249
 250                 /* Set ring buffer size in dwords */
 251                 rb_bufsz = order_base_2(ring->ring_size / 4);
 252                 rb_cntl = rb_bufsz << 1;
 253 #ifdef __BIG_ENDIAN
 254                 rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
 255 #endif
 256                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
 257
 258                 /* Initialize the ring buffer's read and write pointers */
 259                 WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
 260                 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
 261
 262                 /* set the wb address whether it's enabled or not */
 263                 WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
 264                        upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
 265                 WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
 266                        ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
 267
 268                 if (rdev->wb.enabled)
 269                         rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
 270
 271                 WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
 272                 WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
 273
 274                 ring->wptr = 0;
 275                 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
 276
 277                 ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
 278
 279                 /* enable DMA RB */
 280                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
 281
 282                 ib_cntl = SDMA_IB_ENABLE;
 283 #ifdef __BIG_ENDIAN
 284                 ib_cntl |= SDMA_IB_SWAP_ENABLE;
 285 #endif
 286                 /* enable DMA IBs */
 287                 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
 288
 289                 ring->ready = true;
 290
 291                 r = radeon_ring_test(rdev, ring->idx, ring);
 292                 if (r) {
 293                         ring->ready = false;
 294                         return r;
 295                 }
 296         }
 297
 298         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
 299
 300         return 0;
 301 }
 302
 303 /**
 304  * cik_sdma_rlc_resume - setup and start the async dma engines
 305  *
 306  * @rdev: radeon_device pointer
 307  *
 308  * Set up the compute DMA queues and enable them (CIK).
 309  * Returns 0 for success, error for failure.
 310  */
 311 static int cik_sdma_rlc_resume(struct radeon_device *rdev)
 312 {
 313         /* XXX todo */
 314         return 0;
 315 }
 316
 317 /**
 318  * cik_sdma_load_microcode - load the sDMA ME ucode
 319  *
 320  * @rdev: radeon_device pointer
 321  *
 322  * Loads the sDMA0/1 ucode.
 323  * Returns 0 for success, -EINVAL if the ucode is not available.
 324  */
 325 static int cik_sdma_load_microcode(struct radeon_device *rdev)
 326 {
 327         const __be32 *fw_data;
 328         int i;
 329
 330         if (!rdev->sdma_fw)
 331                 return -EINVAL;
 332
 333         /* stop the gfx rings and rlc compute queues */
 334         cik_sdma_gfx_stop(rdev);
 335         cik_sdma_rlc_stop(rdev);
 336
 337         /* halt the MEs */
 338         cik_sdma_enable(rdev, false);
 339
 340         /* sdma0 */
 341         fw_data = (const __be32 *)rdev->sdma_fw->data;
 342         WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
 343         for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
 344                 WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
 345         WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
 346
 347         /* sdma1 */
 348         fw_data = (const __be32 *)rdev->sdma_fw->data;
 349         WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
 350         for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
 351                 WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
 352         WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
 353
 354         WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
 355         WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
 356         return 0;
 357 }
 358
 359 /**
 360  * cik_sdma_resume - setup and start the async dma engines
 361  *
 362  * @rdev: radeon_device pointer
 363  *
 364  * Set up the DMA engines and enable them (CIK).
 365  * Returns 0 for success, error for failure.
 366  */
 367 int cik_sdma_resume(struct radeon_device *rdev)
 368 {
 369         int r;
 370
 371         /* Reset dma */
 372         WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
 373         RREG32(SRBM_SOFT_RESET);
 374         udelay(50);
 375         WREG32(SRBM_SOFT_RESET, 0);
 376         RREG32(SRBM_SOFT_RESET);
 377
 378         r = cik_sdma_load_microcode(rdev);
 379         if (r)
 380                 return r;
 381
 382         /* unhalt the MEs */
 383         cik_sdma_enable(rdev, true);
 384
 385         /* start the gfx rings and rlc compute queues */
 386         r = cik_sdma_gfx_resume(rdev);
 387         if (r)
 388                 return r;
 389         r = cik_sdma_rlc_resume(rdev);
 390         if (r)
 391                 return r;
 392
 393         return 0;
 394 }
 395
 396 /**
 397  * cik_sdma_fini - tear down the async dma engines
 398  *
 399  * @rdev: radeon_device pointer
 400  *
 401  * Stop the async dma engines and free the rings (CIK).
 402  */
 403 void cik_sdma_fini(struct radeon_device *rdev)
 404 {
 405         /* stop the gfx rings and rlc compute queues */
 406         cik_sdma_gfx_stop(rdev);
 407         cik_sdma_rlc_stop(rdev);
 408         /* halt the MEs */
 409         cik_sdma_enable(rdev, false);
 410         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
 411         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
 412         /* XXX - compute dma queue tear down */
 413 }
 414
 415 /**
 416  * cik_copy_dma - copy pages using the DMA engine
 417  *
 418  * @rdev: radeon_device pointer
 419  * @src_offset: src GPU address
 420  * @dst_offset: dst GPU address
 421  * @num_gpu_pages: number of GPU pages to xfer
 422  * @fence: radeon fence object
 423  *
 424  * Copy GPU paging using the DMA engine (CIK).
 425  * Used by the radeon ttm implementation to move pages if
 426  * registered as the asic copy callback.
 427  */
 428 int cik_copy_dma(struct radeon_device *rdev,
 429                  uint64_t src_offset, uint64_t dst_offset,
 430                  unsigned num_gpu_pages,
 431                  struct radeon_fence **fence)
 432 {
 433         struct radeon_semaphore *sem = NULL;
 434         int ring_index = rdev->asic->copy.dma_ring_index;
 435         struct radeon_ring *ring = &rdev->ring[ring_index];
 436         u32 size_in_bytes, cur_size_in_bytes;
 437         int i, num_loops;
 438         int r = 0;
 439
 440         r = radeon_semaphore_create(rdev, &sem);
 441         if (r) {
 442                 DRM_ERROR("radeon: moving bo (%d).\n", r);
 443                 return r;
 444         }
 445
 446         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
 447         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
 448         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
 449         if (r) {
 450                 DRM_ERROR("radeon: moving bo (%d).\n", r);
 451                 radeon_semaphore_free(rdev, &sem, NULL);
 452                 return r;
 453         }
 454
 455         if (radeon_fence_need_sync(*fence, ring->idx)) {
 456                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
 457                                             ring->idx);
 458                 radeon_fence_note_sync(*fence, ring->idx);
 459         } else {
 460                 radeon_semaphore_free(rdev, &sem, NULL);
 461         }
 462
 463         for (i = 0; i < num_loops; i++) {
 464                 cur_size_in_bytes = size_in_bytes;
 465                 if (cur_size_in_bytes > 0x1fffff)
 466                         cur_size_in_bytes = 0x1fffff;
 467                 size_in_bytes -= cur_size_in_bytes;
 468                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
 469                 radeon_ring_write(ring, cur_size_in_bytes);
 470                 radeon_ring_write(ring, 0); /* src/dst endian swap */
 471                 radeon_ring_write(ring, src_offset & 0xffffffff);
 472                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
 473                 radeon_ring_write(ring, dst_offset & 0xffffffff);
 474                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
 475                 src_offset += cur_size_in_bytes;
 476                 dst_offset += cur_size_in_bytes;
 477         }
 478
 479         r = radeon_fence_emit(rdev, fence, ring->idx);
 480         if (r) {
 481                 radeon_ring_unlock_undo(rdev, ring);
 482                 return r;
 483         }
 484
 485         radeon_ring_unlock_commit(rdev, ring);
 486         radeon_semaphore_free(rdev, &sem, *fence);
 487
 488         return r;
 489 }
 490
 491 /**
 492  * cik_sdma_ring_test - simple async dma engine test
 493  *
 494  * @rdev: radeon_device pointer
 495  * @ring: radeon_ring structure holding ring information
 496  *
 497  * Test the DMA engine by writing using it to write an
 498  * value to memory. (CIK).
 499  * Returns 0 for success, error for failure.
 500  */
 501 int cik_sdma_ring_test(struct radeon_device *rdev,
 502                        struct radeon_ring *ring)
 503 {
 504         unsigned i;
 505         int r;
 506         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
 507         u32 tmp;
 508
 509         if (!ptr) {
 510                 DRM_ERROR("invalid vram scratch pointer\n");
 511                 return -EINVAL;
 512         }
 513
 514         tmp = 0xCAFEDEAD;
 515         writel(tmp, ptr);
 516
 517         r = radeon_ring_lock(rdev, ring, 4);
 518         if (r) {
 519                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
 520                 return r;
 521         }
 522         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
 523         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
 524         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
 525         radeon_ring_write(ring, 1); /* number of DWs to follow */
 526         radeon_ring_write(ring, 0xDEADBEEF);
 527         radeon_ring_unlock_commit(rdev, ring);
 528
 529         for (i = 0; i < rdev->usec_timeout; i++) {
 530                 tmp = readl(ptr);
 531                 if (tmp == 0xDEADBEEF)
 532                         break;
 533                 DRM_UDELAY(1);
 534         }
 535
 536         if (i < rdev->usec_timeout) {
 537                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 538         } else {
 539                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
 540                           ring->idx, tmp);
 541                 r = -EINVAL;
 542         }
 543         return r;
 544 }
 545
 546 /**
 547  * cik_sdma_ib_test - test an IB on the DMA engine
 548  *
 549  * @rdev: radeon_device pointer
 550  * @ring: radeon_ring structure holding ring information
 551  *
 552  * Test a simple IB in the DMA ring (CIK).
 553  * Returns 0 on success, error on failure.
 554  */
 555 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 556 {
 557         struct radeon_ib ib;
 558         unsigned i;
 559         int r;
 560         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
 561         u32 tmp = 0;
 562
 563         if (!ptr) {
 564                 DRM_ERROR("invalid vram scratch pointer\n");
 565                 return -EINVAL;
 566         }
 567
 568         tmp = 0xCAFEDEAD;
 569         writel(tmp, ptr);
 570
 571         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
 572         if (r) {
 573                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
 574                 return r;
 575         }
 576
 577         ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
 578         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
 579         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
 580         ib.ptr[3] = 1;
 581         ib.ptr[4] = 0xDEADBEEF;
 582         ib.length_dw = 5;
 583
 584         r = radeon_ib_schedule(rdev, &ib, NULL);
 585         if (r) {
 586                 radeon_ib_free(rdev, &ib);
 587                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
 588                 return r;
 589         }
 590         r = radeon_fence_wait(ib.fence, false);
 591         if (r) {
 592                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
 593                 return r;
 594         }
 595         for (i = 0; i < rdev->usec_timeout; i++) {
 596                 tmp = readl(ptr);
 597                 if (tmp == 0xDEADBEEF)
 598                         break;
 599                 DRM_UDELAY(1);
 600         }
 601         if (i < rdev->usec_timeout) {
 602                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
 603         } else {
 604                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
 605                 r = -EINVAL;
 606         }
 607         radeon_ib_free(rdev, &ib);
 608         return r;
 609 }
 610
 611 /**
 612  * cik_sdma_is_lockup - Check if the DMA engine is locked up
 613  *
 614  * @rdev: radeon_device pointer
 615  * @ring: radeon_ring structure holding ring information
 616  *
 617  * Check if the async DMA engine is locked up (CIK).
 618  * Returns true if the engine appears to be locked up, false if not.
 619  */
 620 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 621 {
 622         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
 623         u32 mask;
 624
 625         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
 626                 mask = RADEON_RESET_DMA;
 627         else
 628                 mask = RADEON_RESET_DMA1;
 629
 630         if (!(reset_mask & mask)) {
 631                 radeon_ring_lockup_update(ring);
 632                 return false;
 633         }
 634         /* force ring activities */
 635         radeon_ring_force_activity(rdev, ring);
 636         return radeon_ring_test_lockup(rdev, ring);
 637 }
 638
 639 /**
 640  * cik_sdma_vm_set_page - update the page tables using sDMA
 641  *
 642  * @rdev: radeon_device pointer
 643  * @ib: indirect buffer to fill with commands
 644  * @pe: addr of the page entry
 645  * @addr: dst addr to write into pe
 646  * @count: number of page entries to update
 647  * @incr: increase next addr by incr bytes
 648  * @flags: access flags
 649  *
 650  * Update the page tables using sDMA (CIK).
 651  */
 652 void cik_sdma_vm_set_page(struct radeon_device *rdev,
 653                           struct radeon_ib *ib,
 654                           uint64_t pe,
 655                           uint64_t addr, unsigned count,
 656                           uint32_t incr, uint32_t flags)
 657 {
 658         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
 659         uint64_t value;
 660         unsigned ndw;
 661
 662         if (flags & RADEON_VM_PAGE_SYSTEM) {
 663                 while (count) {
 664                         ndw = count * 2;
 665                         if (ndw > 0xFFFFE)
 666                                 ndw = 0xFFFFE;
 667
 668                         /* for non-physically contiguous pages (system) */
 669                         ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
 670                         ib->ptr[ib->length_dw++] = pe;
 671                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 672                         ib->ptr[ib->length_dw++] = ndw;
 673                         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
 674                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
 675                                         value = radeon_vm_map_gart(rdev, addr);
 676                                         value &= 0xFFFFFFFFFFFFF000ULL;
 677                                 } else if (flags & RADEON_VM_PAGE_VALID) {
 678                                         value = addr;
 679                                 } else {
 680                                         value = 0;
 681                                 }
 682                                 addr += incr;
 683                                 value |= r600_flags;
 684                                 ib->ptr[ib->length_dw++] = value;
 685                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
 686                         }
 687                 }
 688         } else {
 689                 while (count) {
 690                         ndw = count;
 691                         if (ndw > 0x7FFFF)
 692                                 ndw = 0x7FFFF;
 693
 694                         if (flags & RADEON_VM_PAGE_VALID)
 695                                 value = addr;
 696                         else
 697                                 value = 0;
 698                         /* for physically contiguous pages (vram) */
 699                         ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
 700                         ib->ptr[ib->length_dw++] = pe; /* dst addr */
 701                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 702                         ib->ptr[ib->length_dw++] = r600_flags; /* mask */
 703                         ib->ptr[ib->length_dw++] = 0;
 704                         ib->ptr[ib->length_dw++] = value; /* value */
 705                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
 706                         ib->ptr[ib->length_dw++] = incr; /* increment size */
 707                         ib->ptr[ib->length_dw++] = 0;
 708                         ib->ptr[ib->length_dw++] = ndw; /* number of entries */
 709                         pe += ndw * 8;
 710                         addr += ndw * incr;
 711                         count -= ndw;
 712                 }
 713         }
 714         while (ib->length_dw & 0x7)
 715                 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
 716 }
 717
 718 /**
 719  * cik_dma_vm_flush - cik vm flush using sDMA
 720  *
 721  * @rdev: radeon_device pointer
 722  *
 723  * Update the page table base and flush the VM TLB
 724  * using sDMA (CIK).
 725  */
 726 void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 727 {
 728         struct radeon_ring *ring = &rdev->ring[ridx];
 729         u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
 730                           SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
 731         u32 ref_and_mask;
 732
 733         if (vm == NULL)
 734                 return;
 735
 736         if (ridx == R600_RING_TYPE_DMA_INDEX)
 737                 ref_and_mask = SDMA0;
 738         else
 739                 ref_and_mask = SDMA1;
 740
 741         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 742         if (vm->id < 8) {
 743                 radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
 744         } else {
 745                 radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
 746         }
 747         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
 748
 749         /* update SH_MEM_* regs */
 750         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 751         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
 752         radeon_ring_write(ring, VMID(vm->id));
 753
 754         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 755         radeon_ring_write(ring, SH_MEM_BASES >> 2);
 756         radeon_ring_write(ring, 0);
 757
 758         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 759         radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
 760         radeon_ring_write(ring, 0);
 761
 762         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 763         radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
 764         radeon_ring_write(ring, 1);
 765
 766         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 767         radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
 768         radeon_ring_write(ring, 0);
 769
 770         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 771         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
 772         radeon_ring_write(ring, VMID(0));
 773
 774         /* flush HDP */
 775         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
 776         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
 777         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
 778         radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
 779         radeon_ring_write(ring, ref_and_mask); /* MASK */
 780         radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
 781
 782         /* flush TLB */
 783         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 784         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
 785         radeon_ring_write(ring, 1 << vm->id);
 786 }
 787