linux-core/radeon_buffer.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2007 Dave Airlie
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  18  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  19  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  20  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * The above copyright notice and this permission notice (including the
  23  * next paragraph) shall be included in all copies or substantial portions
  24  * of the Software.
  25  *
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Dave Airlie <airlied@linux.ie>
  30  */
  31
  32 #include "drmP.h"
  33 #include "radeon_drm.h"
  34 #include "radeon_drv.h"
  35
  36 struct drm_ttm_backend *radeon_create_ttm_backend_entry(struct drm_device * dev)
  37 {
  38         drm_radeon_private_t *dev_priv = dev->dev_private;
  39
  40         if (dev_priv->flags & RADEON_IS_AGP)
  41                 return drm_agp_init_ttm(dev);
  42         else
  43                 return ati_pcigart_init_ttm(dev, &dev_priv->gart_info, radeon_gart_flush);
  44 }
  45
  46 int radeon_fence_types(struct drm_buffer_object *bo, uint32_t * class, uint32_t * type)
  47 {
  48         *class = 0;
  49         *type = 1;
  50         return 0;
  51 }
  52
  53 int radeon_invalidate_caches(struct drm_device * dev, uint64_t flags)
  54 {
  55         drm_radeon_private_t *dev_priv = dev->dev_private;
  56         RING_LOCALS;
  57
  58         if (!dev_priv->cp_running)
  59                 return 0;
  60
  61         BEGIN_RING(6);
  62         RADEON_PURGE_CACHE();
  63         RADEON_PURGE_ZCACHE();
  64         RADEON_WAIT_UNTIL_3D_IDLE();
  65         ADVANCE_RING();
  66         COMMIT_RING();
  67         return 0;
  68 }
  69
  70 int radeon_init_mem_type(struct drm_device * dev, uint32_t type,
  71                          struct drm_mem_type_manager * man)
  72 {
  73         drm_radeon_private_t *dev_priv = dev->dev_private;
  74
  75         switch (type) {
  76         case DRM_BO_MEM_LOCAL:
  77                 man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
  78                     _DRM_FLAG_MEMTYPE_CACHED;
  79                 man->drm_bus_maptype = 0;
  80                 break;
  81         case DRM_BO_MEM_VRAM:
  82                 man->flags =  _DRM_FLAG_MEMTYPE_FIXED | _DRM_FLAG_MEMTYPE_MAPPABLE | _DRM_FLAG_NEEDS_IOREMAP;
  83                 man->io_addr = NULL;
  84                 man->drm_bus_maptype = _DRM_FRAME_BUFFER;
  85                 man->io_offset = drm_get_resource_start(dev, 0);
  86                 man->io_size = drm_get_resource_len(dev, 0);
  87                 break;
  88         case DRM_BO_MEM_TT:
  89                 if (dev_priv->flags & RADEON_IS_AGP) {
  90                         if (!(drm_core_has_AGP(dev) && dev->agp)) {
  91                                 DRM_ERROR("AGP is not enabled for memory type %u\n",
  92                                           (unsigned)type);
  93                                 return -EINVAL;
  94                         }
  95                         man->io_offset = dev->agp->agp_info.aper_base;
  96                         man->io_size = dev->agp->agp_info.aper_size * 1024 * 1024;
  97                         man->io_addr = NULL;
  98                         man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
  99                                 _DRM_FLAG_MEMTYPE_CSELECT | _DRM_FLAG_NEEDS_IOREMAP;
 100                         man->drm_bus_maptype = _DRM_AGP;
 101                 } else {
 102                         man->io_offset = dev_priv->gart_vm_start;
 103                         man->io_size = dev_priv->gart_size;
 104                         man->io_addr = NULL;
 105                         man->flags = _DRM_FLAG_MEMTYPE_CSELECT | _DRM_FLAG_MEMTYPE_MAPPABLE | _DRM_FLAG_MEMTYPE_CMA;
 106                         man->drm_bus_maptype = _DRM_SCATTER_GATHER;
 107                 }
 108                 break;
 109         default:
 110                 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
 111                 return -EINVAL;
 112         }
 113         return 0;
 114 }
 115
 116 void radeon_emit_copy_blit(struct drm_device * dev,
 117                            uint32_t src_offset,
 118                            uint32_t dst_offset,
 119                            uint32_t pages)
 120 {
 121         uint32_t cur_pages;
 122         uint32_t stride_bytes = PAGE_SIZE;
 123         drm_radeon_private_t *dev_priv = dev->dev_private;
 124         uint32_t format, pitch;
 125         const uint32_t clip = (0x1fff) | (0x1fff << 16);
 126         uint32_t stride_pixels;
 127         RING_LOCALS;
 128
 129         if (!dev_priv)
 130                 return;
 131
 132         /* 32-bit copy format */
 133         format = RADEON_COLOR_FORMAT_ARGB8888;
 134
 135         /* radeon limited to 16k stride */
 136         stride_bytes &= 0x3fff;
 137         /* radeon pitch is /64 */
 138         pitch = stride_bytes / 64;
 139
 140         stride_pixels = stride_bytes / 4;
 141
 142         while(pages > 0) {
 143                 cur_pages = pages;
 144                 if (cur_pages > 8191)
 145                         cur_pages = 8191;
 146                 pages -= cur_pages;
 147
 148                 /* pages are in Y direction - height
 149                    page width in X direction - width */
 150                 BEGIN_RING(10);
 151                 OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 8));
 152                 OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
 153                          RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 154                          RADEON_GMC_SRC_CLIPPING | RADEON_GMC_DST_CLIPPING |
 155                          RADEON_GMC_BRUSH_NONE |
 156                          (format << 8) |
 157                          RADEON_GMC_SRC_DATATYPE_COLOR |
 158                          RADEON_ROP3_S |
 159                          RADEON_DP_SRC_SOURCE_MEMORY |
 160                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
 161                 OUT_RING((pitch << 22) | (src_offset >> 10));
 162                 OUT_RING((pitch << 22) | (dst_offset >> 10));
 163                 OUT_RING(clip); // SRC _SC BOT_RITE
 164                 OUT_RING(0);   // SC_TOP_LEFT
 165                 OUT_RING(clip); // SC_BOT_RITE
 166
 167                 OUT_RING(pages);
 168                 OUT_RING(pages); /* x - y */
 169                 OUT_RING(cur_pages | (stride_pixels << 16));
 170                 ADVANCE_RING();
 171         }
 172
 173         BEGIN_RING(4);
 174         OUT_RING(CP_PACKET0(RADEON_RB2D_DSTCACHE_CTLSTAT, 0));
 175         OUT_RING(RADEON_RB2D_DC_FLUSH_ALL);
 176         RADEON_WAIT_UNTIL_2D_IDLE();
 177         ADVANCE_RING();
 178
 179         COMMIT_RING();
 180         return;
 181 }
 182
 183 int radeon_move_blit(struct drm_buffer_object * bo,
 184                             int evict, int no_wait, struct drm_bo_mem_reg *new_mem,
 185                             struct drm_bo_mem_reg *old_mem)
 186 {
 187         struct drm_device *dev = bo->dev;
 188         drm_radeon_private_t *dev_priv = dev->dev_private;
 189         uint32_t old_start, new_start;
 190
 191         old_start = old_mem->mm_node->start << PAGE_SHIFT;
 192         new_start = new_mem->mm_node->start << PAGE_SHIFT;
 193
 194         if (old_mem->mem_type == DRM_BO_MEM_VRAM)
 195                 old_start += dev_priv->fb_location;
 196         if (old_mem->mem_type == DRM_BO_MEM_TT)
 197                 old_start += dev_priv->gart_vm_start;
 198
 199         if (new_mem->mem_type == DRM_BO_MEM_VRAM)
 200                 new_start += dev_priv->fb_location;
 201         if (new_mem->mem_type == DRM_BO_MEM_TT)
 202                 new_start += dev_priv->gart_vm_start;
 203
 204         radeon_emit_copy_blit(bo->dev,
 205                               old_start,
 206                               new_start,
 207                               new_mem->num_pages);
 208
 209         /* invalidate the chip caches */
 210
 211         return drm_bo_move_accel_cleanup(bo, evict, no_wait, 0,
 212                                          DRM_FENCE_TYPE_EXE, 0,
 213                                          new_mem);
 214 }
 215
 216 void radeon_emit_solid_fill(struct drm_device * dev,
 217                             uint32_t dst_offset,
 218                             uint32_t pages, uint8_t value)
 219 {
 220         uint32_t cur_pages;
 221         uint32_t stride_bytes = PAGE_SIZE;
 222         drm_radeon_private_t *dev_priv = dev->dev_private;
 223         uint32_t format, pitch;
 224         const uint32_t clip = (0x1fff) | (0x1fff << 16);
 225         uint32_t stride_pixels;
 226         RING_LOCALS;
 227
 228         if (!dev_priv)
 229                 return;
 230
 231         /* 32-bit copy format */
 232         format = RADEON_COLOR_FORMAT_ARGB8888;
 233
 234         /* radeon limited to 16k stride */
 235         stride_bytes &= 0x3fff;
 236         /* radeon pitch is /64 */
 237         pitch = stride_bytes / 64;
 238
 239         stride_pixels = stride_bytes / 4;
 240
 241         while(pages > 0) {
 242                 cur_pages = pages;
 243                 if (cur_pages > 8191)
 244                         cur_pages = 8191;
 245                 pages -= cur_pages;
 246
 247                 /* pages are in Y direction - height
 248                    page width in X direction - width */
 249                 BEGIN_RING(8);
 250                 OUT_RING(CP_PACKET3(RADEON_CNTL_PAINT_MULTI, 6));
 251                 OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 252                          RADEON_GMC_DST_CLIPPING |
 253                          RADEON_GMC_BRUSH_SOLID_COLOR |
 254                          (format << 8) |
 255                          RADEON_ROP3_S |
 256                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
 257                 OUT_RING((pitch << 22) | (dst_offset >> 10)); // PITCH
 258                 OUT_RING(0);   // SC_TOP_LEFT // DST CLIPPING
 259                 OUT_RING(clip); // SC_BOT_RITE
 260
 261                 OUT_RING(0); // COLOR
 262
 263                 OUT_RING(pages); /* x - y */
 264                 OUT_RING(cur_pages | (stride_pixels << 16));
 265                 ADVANCE_RING();
 266         }
 267
 268         BEGIN_RING(4);
 269         OUT_RING(CP_PACKET0(RADEON_RB2D_DSTCACHE_CTLSTAT, 0));
 270         OUT_RING(RADEON_RB2D_DC_FLUSH_ALL);
 271         RADEON_WAIT_UNTIL_2D_IDLE();
 272         ADVANCE_RING();
 273
 274         COMMIT_RING();
 275         return;
 276 }
 277
 278 int radeon_move_zero_fill(struct drm_buffer_object * bo,
 279                           int evict, int no_wait, struct drm_bo_mem_reg *new_mem)
 280 {
 281         struct drm_device *dev = bo->dev;
 282         drm_radeon_private_t *dev_priv = dev->dev_private;
 283         uint32_t new_start;
 284
 285         new_start = new_mem->mm_node->start << PAGE_SHIFT;
 286
 287         if (new_mem->mem_type == DRM_BO_MEM_VRAM)
 288                 new_start += dev_priv->fb_location;
 289
 290         radeon_emit_solid_fill(bo->dev,
 291                                new_start,
 292                                new_mem->num_pages, 0);
 293
 294         /* invalidate the chip caches */
 295
 296         return drm_bo_move_accel_cleanup(bo, 1, no_wait, 0,
 297                                          DRM_FENCE_TYPE_EXE, 0,
 298                                          new_mem);
 299 }
 300
 301 static int radeon_move_flip(struct drm_buffer_object * bo,
 302                             int evict, int no_wait, struct drm_bo_mem_reg * new_mem)
 303 {
 304         struct drm_device *dev = bo->dev;
 305         struct drm_bo_mem_reg tmp_mem;
 306         int ret;
 307
 308         tmp_mem = *new_mem;
 309
 310         /* if we are flipping into LOCAL memory we have no TTM so create one */
 311         if (new_mem->mem_type == DRM_BO_MEM_LOCAL) {
 312                 tmp_mem.mm_node = NULL;
 313                 tmp_mem.proposed_flags = DRM_BO_FLAG_MEM_TT;
 314
 315                 ret = drm_bo_mem_space(bo, &tmp_mem, no_wait);
 316                 if (ret)
 317                         return ret;
 318
 319                 ret = drm_ttm_bind(bo->ttm, &tmp_mem);
 320                 if (ret)
 321                         goto out_cleanup;
 322         }
 323
 324         ret = radeon_move_blit(bo, 1, no_wait, &tmp_mem, &bo->mem);
 325         if (ret)
 326                 goto out_cleanup;
 327
 328         if (new_mem->mem_type == DRM_BO_MEM_LOCAL) {
 329                 ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem);
 330         } else {
 331                 tmp_mem.mm_node = NULL;
 332                 new_mem->mm_node = NULL;
 333         }
 334
 335 out_cleanup:
 336         if (tmp_mem.mm_node) {
 337                 mutex_lock(&dev->struct_mutex);
 338                 if (tmp_mem.mm_node != bo->pinned_node)
 339                         drm_mm_put_block(tmp_mem.mm_node);
 340                 tmp_mem.mm_node = NULL;
 341                 mutex_unlock(&dev->struct_mutex);
 342         }
 343         return ret;
 344 }
 345
 346 static int radeon_move_vram(struct drm_buffer_object * bo,
 347                             int evict, int no_wait, struct drm_bo_mem_reg * new_mem)
 348 {
 349         struct drm_device *dev = bo->dev;
 350         struct drm_bo_mem_reg tmp_mem;
 351         struct drm_bo_mem_reg *old_mem = &bo->mem;
 352         int ret;
 353         bool was_local = false;
 354
 355         /* old - LOCAL memory node bo->mem
 356            tmp - TT type memory node
 357            new - VRAM memory node */
 358
 359         tmp_mem = *old_mem;
 360         tmp_mem.mm_node = NULL;
 361
 362         if (old_mem->mem_type == DRM_BO_MEM_LOCAL) {
 363                 tmp_mem.proposed_flags = DRM_BO_FLAG_MEM_TT;
 364
 365                 ret = drm_bo_mem_space(bo, &tmp_mem, no_wait);
 366                 if (ret)
 367                         return ret;
 368         }
 369
 370         if (!bo->ttm) {
 371                 ret = drm_bo_add_ttm(bo);
 372                 if (ret)
 373                         goto out_cleanup;
 374         }
 375
 376         if (old_mem->mem_type == DRM_BO_MEM_LOCAL) {
 377                 ret = drm_bo_move_ttm(bo, evict, no_wait, &tmp_mem);
 378                 if (ret)
 379                         return ret;
 380         }
 381
 382         ret = radeon_move_blit(bo, 1, no_wait, new_mem, &bo->mem);
 383         if (ret)
 384                 goto out_cleanup;
 385
 386 out_cleanup:
 387         if (tmp_mem.mm_node) {
 388                 mutex_lock(&dev->struct_mutex);
 389                 if (tmp_mem.mm_node != bo->pinned_node)
 390                         drm_mm_put_block(tmp_mem.mm_node);
 391                 tmp_mem.mm_node = NULL;
 392                 mutex_unlock(&dev->struct_mutex);
 393         }
 394         return ret;
 395 }
 396
 397 int radeon_move(struct drm_buffer_object * bo,
 398                 int evict, int no_wait, struct drm_bo_mem_reg *new_mem)
 399 {
 400         struct drm_device *dev = bo->dev;
 401         struct drm_bo_mem_reg *old_mem = &bo->mem;
 402         drm_radeon_private_t *dev_priv = dev->dev_private;
 403
 404         if (!dev_priv->cp_running)
 405                 goto fallback;
 406
 407         if (bo->mem.flags & DRM_BO_FLAG_CLEAN) /* need to implement solid fill */
 408         {
 409                 if (radeon_move_zero_fill(bo, evict, no_wait, new_mem))
 410                         return drm_bo_move_zero(bo, evict, no_wait, new_mem);
 411                 return 0;
 412         }
 413
 414         if (new_mem->mem_type == DRM_BO_MEM_VRAM) {
 415                 if (radeon_move_vram(bo, evict, no_wait, new_mem))
 416                         goto fallback;
 417         } else {
 418                 if (radeon_move_flip(bo, evict, no_wait, new_mem))
 419                         goto fallback;
 420         }
 421         return 0;
 422 fallback:
 423         if (bo->mem.flags & DRM_BO_FLAG_CLEAN)
 424                 return drm_bo_move_zero(bo, evict, no_wait, new_mem);
 425         else
 426                 return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
 427 }
 428
 429
 430 /*
 431  * i915_evict_flags:
 432  *
 433  * @bo: the buffer object to be evicted
 434  *
 435  * Return the bo flags for a buffer which is not mapped to the hardware.
 436  * These will be placed in proposed_flags so that when the move is
 437  * finished, they'll end up in bo->mem.flags
 438  */
 439 uint64_t radeon_evict_flags(struct drm_buffer_object *bo)
 440 {
 441         switch (bo->mem.mem_type) {
 442         case DRM_BO_MEM_LOCAL:
 443         case DRM_BO_MEM_TT:
 444                 return DRM_BO_FLAG_MEM_LOCAL;
 445         default:
 446                 return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MEM_LOCAL;
 447         }
 448 }