shared-core/radeon_ms_bo.c

   1 /*
   2  * Copyright 2007 Dave Airlie
   3  * Copyright 2007 Jérôme Glisse
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  18  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  19  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  20  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * The above copyright notice and this permission notice (including the
  23  * next paragraph) shall be included in all copies or substantial portions
  24  * of the Software.
  25  */
  26 /*
  27  * Authors:
  28  *    Dave Airlie <airlied@linux.ie>
  29  *    Jerome Glisse <glisse@freedesktop.org>
  30  */
  31 #include "drmP.h"
  32 #include "drm.h"
  33
  34 #include "radeon_ms.h"
  35
  36 #define GMC_SRC_PITCH_OFFSET_CNTL   (1    <<  0)
  37 #define GMC_DST_PITCH_OFFSET_CNTL   (1    <<  1)
  38 #define GMC_BRUSH_NONE              (15   <<  4)
  39 #define GMC_SRC_DATATYPE_COLOR      (3    << 12)
  40 #define ROP3_S                      0x00cc0000
  41 #define DP_SRC_SOURCE_MEMORY        (2    << 24)
  42 #define GMC_CLR_CMP_CNTL_DIS        (1    << 28)
  43 #define GMC_WR_MSK_DIS              (1    << 30)
  44
  45 void radeon_ms_bo_copy_blit(struct drm_device *dev,
  46                             uint32_t src_offset,
  47                             uint32_t dst_offset,
  48                             uint32_t pages)
  49 {
  50         struct drm_radeon_private *dev_priv = dev->dev_private;
  51         uint32_t num_pages, stride, c;
  52         uint32_t offset_inc = 0;
  53         uint32_t cmd[7];
  54
  55         if (!dev_priv) {
  56                 return;
  57         }
  58
  59         /* radeon limited to 16320=255*64 bytes per row so copy at
  60          * most 2 pages */
  61         num_pages = 2;
  62         stride = ((num_pages * PAGE_SIZE) / 64) & 0xff;
  63         while(pages > 0) {
  64                 if (num_pages > pages) {
  65                         num_pages = pages;
  66                         stride = ((num_pages * PAGE_SIZE) / 64) & 0xff;
  67                 }
  68                 c = pages / num_pages;
  69                 if (c >= 8192) {
  70                         c = 8191;
  71                 }
  72                 cmd[0] = CP_PACKET3(PACKET3_OPCODE_BITBLT, 5);
  73                 cmd[1] = GMC_SRC_PITCH_OFFSET_CNTL |
  74                          GMC_DST_PITCH_OFFSET_CNTL |
  75                          GMC_BRUSH_NONE |
  76                          (0x6 << 8) |
  77                          GMC_SRC_DATATYPE_COLOR |
  78                          ROP3_S |
  79                          DP_SRC_SOURCE_MEMORY |
  80                          GMC_CLR_CMP_CNTL_DIS |
  81                          GMC_WR_MSK_DIS;
  82                 cmd[2] = (stride << 22) | (src_offset >> 10);
  83                 cmd[3] = (stride << 22) | (dst_offset >> 10);
  84                 cmd[4] = (0 << 16) | 0;
  85                 cmd[5] = (0 << 16) | 0;
  86                 cmd[6] = ((stride * 16) << 16) | c;
  87                 radeon_ms_ring_emit(dev, cmd, 7);
  88                 offset_inc = num_pages * c * PAGE_SIZE;
  89                 src_offset += offset_inc;
  90                 dst_offset += offset_inc;
  91                 pages -= num_pages * c;
  92         }
  93         /* wait for 2d engine to go busy so wait_until stall */
  94         for (c = 0; c < dev_priv->usec_timeout; c++) {
  95                 uint32_t status = MMIO_R(RBBM_STATUS);
  96                 if ((RBBM_STATUS__E2_BUSY & status) ||
  97                     (RBBM_STATUS__CBA2D_BUSY & status)) {
  98                         DRM_INFO("[radeon_ms] RBBM_STATUS 0x%08X\n", status);
  99                         break;
 100                 }
 101                 DRM_UDELAY(1);
 102         }
 103         /* Sync everything up */
 104         cmd[0] = CP_PACKET0(WAIT_UNTIL, 0);
 105         cmd[1] = WAIT_UNTIL__WAIT_2D_IDLECLEAN |
 106                  WAIT_UNTIL__WAIT_HOST_IDLECLEAN;
 107         radeon_ms_ring_emit(dev, cmd, 2);
 108         return;
 109 }
 110
 111 static int radeon_ms_bo_move_blit(struct drm_buffer_object *bo,
 112                                   int evict, int no_wait,
 113                                   struct drm_bo_mem_reg *new_mem)
 114 {
 115         struct drm_device *dev = bo->dev;
 116         struct drm_bo_mem_reg *old_mem = &bo->mem;
 117         uint32_t gpu_src_addr;
 118         uint32_t gpu_dst_addr;
 119         int ret;
 120
 121         ret = radeon_ms_bo_get_gpu_addr(dev, old_mem, &gpu_src_addr);
 122         if (ret) {
 123                 return ret;
 124         }
 125         ret = radeon_ms_bo_get_gpu_addr(dev, new_mem, &gpu_dst_addr);
 126         if (ret) {
 127                 return ret;
 128         }
 129
 130         radeon_ms_bo_copy_blit(bo->dev,
 131                                gpu_src_addr,
 132                                gpu_dst_addr,
 133                                new_mem->num_pages);
 134
 135         ret = drm_bo_move_accel_cleanup(bo, evict, no_wait, 0,
 136                                          DRM_FENCE_TYPE_EXE |
 137                                          DRM_AMD_FENCE_TYPE_R |
 138                                          DRM_AMD_FENCE_TYPE_W,
 139                                          DRM_AMD_FENCE_FLAG_FLUSH,
 140                                          new_mem);
 141         return ret;
 142 }
 143
 144 static int radeon_ms_bo_move_flip(struct drm_buffer_object *bo,
 145                                   int evict, int no_wait,
 146                                   struct drm_bo_mem_reg *new_mem)
 147 {
 148         struct drm_device *dev = bo->dev;
 149         struct drm_bo_mem_reg tmp_mem;
 150         int ret;
 151
 152         tmp_mem = *new_mem;
 153         tmp_mem.mm_node = NULL;
 154         tmp_mem.flags = DRM_BO_FLAG_MEM_TT |
 155                        DRM_BO_FLAG_CACHED |
 156                        DRM_BO_FLAG_FORCE_CACHING;
 157         ret = drm_bo_mem_space(bo, &tmp_mem, no_wait);
 158         if (ret) {
 159                 return ret;
 160         }
 161
 162         ret = drm_ttm_bind(bo->ttm, &tmp_mem);
 163         if (ret) {
 164                 goto out_cleanup;
 165         }
 166         ret = radeon_ms_bo_move_blit(bo, 1, no_wait, &tmp_mem);
 167         if (ret) {
 168                 goto out_cleanup;
 169         }
 170         ret = drm_bo_move_ttm(bo, evict, no_wait, new_mem);
 171 out_cleanup:
 172         if (tmp_mem.mm_node) {
 173                 mutex_lock(&dev->struct_mutex);
 174                 if (tmp_mem.mm_node != bo->pinned_node)
 175                         drm_mm_put_block(tmp_mem.mm_node);
 176                 tmp_mem.mm_node = NULL;
 177                 mutex_unlock(&dev->struct_mutex);
 178         }
 179         return ret;
 180 }
 181
 182 int radeon_ms_bo_get_gpu_addr(struct drm_device *dev,
 183                               struct drm_bo_mem_reg *mem,
 184                               uint32_t *gpu_addr)
 185 {
 186         struct drm_radeon_private *dev_priv = dev->dev_private;
 187
 188         *gpu_addr = mem->mm_node->start << PAGE_SHIFT;
 189         switch (mem->flags & DRM_BO_MASK_MEM) {
 190         case DRM_BO_FLAG_MEM_TT:
 191                 *gpu_addr +=  dev_priv->gpu_gart_start;
 192                 DRM_INFO("[radeon_ms] GPU TT: 0x%08X\n", *gpu_addr);
 193                 break;
 194         case DRM_BO_FLAG_MEM_VRAM:
 195                 *gpu_addr +=  dev_priv->gpu_vram_start;
 196                 DRM_INFO("[radeon_ms] GPU VRAM: 0x%08X\n", *gpu_addr);
 197                 break;
 198         default:
 199                 DRM_ERROR("[radeon_ms] memory not accessible by GPU\n");
 200                 return -EINVAL;
 201         }
 202         return 0;
 203 }
 204
 205 int radeon_ms_bo_move(struct drm_buffer_object *bo, int evict,
 206                    int no_wait, struct drm_bo_mem_reg *new_mem)
 207 {
 208         struct drm_bo_mem_reg *old_mem = &bo->mem;
 209         if (old_mem->mem_type == DRM_BO_MEM_LOCAL) {
 210                 return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
 211         } else if (new_mem->mem_type == DRM_BO_MEM_LOCAL) {
 212                 if (radeon_ms_bo_move_flip(bo, evict, no_wait, new_mem))
 213                         return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
 214         } else {
 215                 if (radeon_ms_bo_move_blit(bo, evict, no_wait, new_mem))
 216                         return drm_bo_move_memcpy(bo, evict, no_wait, new_mem);
 217         }
 218         return 0;
 219 }
 220
 221 struct drm_ttm_backend *radeon_ms_create_ttm_backend(struct drm_device * dev)
 222 {
 223         struct drm_radeon_private *dev_priv = dev->dev_private;
 224
 225         if (dev_priv && dev_priv->create_ttm)
 226                 return dev_priv->create_ttm(dev);
 227         return NULL;
 228 }
 229
 230 uint64_t radeon_ms_evict_flags(struct drm_buffer_object *bo)
 231 {
 232         switch (bo->mem.mem_type) {
 233         case DRM_BO_MEM_LOCAL:
 234         case DRM_BO_MEM_TT:
 235                 return DRM_BO_FLAG_MEM_LOCAL;
 236         case DRM_BO_MEM_VRAM:
 237                 if (bo->mem.num_pages > 128)
 238                         return DRM_BO_MEM_TT;
 239                 else
 240                         return DRM_BO_MEM_LOCAL;
 241         default:
 242                 return DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_CACHED;
 243         }
 244 }
 245
 246 int radeon_ms_init_mem_type(struct drm_device * dev, uint32_t type,
 247                             struct drm_mem_type_manager * man)
 248 {
 249         struct drm_radeon_private *dev_priv = dev->dev_private;
 250
 251         switch (type) {
 252         case DRM_BO_MEM_LOCAL:
 253                 man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
 254                     _DRM_FLAG_MEMTYPE_CACHED;
 255                 man->drm_bus_maptype = 0;
 256                 break;
 257         case DRM_BO_MEM_VRAM:
 258                 man->flags =  _DRM_FLAG_MEMTYPE_FIXED |
 259                               _DRM_FLAG_MEMTYPE_MAPPABLE |
 260                               _DRM_FLAG_NEEDS_IOREMAP;
 261                 man->io_addr = NULL;
 262                 man->drm_bus_maptype = _DRM_FRAME_BUFFER;
 263                 man->io_offset = dev_priv->vram.offset;
 264                 man->io_size = dev_priv->vram.size;
 265                 break;
 266         case DRM_BO_MEM_TT:
 267                 if (!dev_priv->bus_ready) {
 268                         DRM_ERROR("Bus isn't initialized while "
 269                                   "intializing TT memory type\n");
 270                         return -EINVAL;
 271                 }
 272                 switch(dev_priv->bus_type) {
 273                 case RADEON_AGP:
 274                         if (!(drm_core_has_AGP(dev) && dev->agp)) {
 275                                 DRM_ERROR("AGP is not enabled for memory "
 276                                           "type %u\n", (unsigned)type);
 277                                 return -EINVAL;
 278                         }
 279                         man->io_offset = dev->agp->agp_info.aper_base;
 280                         man->io_size = dev->agp->agp_info.aper_size *
 281                                        1024 * 1024;
 282                         man->io_addr = NULL;
 283                         man->flags = _DRM_FLAG_MEMTYPE_MAPPABLE |
 284                                      _DRM_FLAG_MEMTYPE_CSELECT |
 285                                      _DRM_FLAG_NEEDS_IOREMAP;
 286                         man->drm_bus_maptype = _DRM_AGP;
 287                         man->gpu_offset = 0;
 288                         break;
 289                 default:
 290                         man->io_offset = dev_priv->gpu_gart_start;
 291                         man->io_size = dev_priv->gpu_gart_size;
 292                         man->io_addr = NULL;
 293                         man->flags = _DRM_FLAG_MEMTYPE_CSELECT |
 294                                      _DRM_FLAG_MEMTYPE_MAPPABLE |
 295                                      _DRM_FLAG_MEMTYPE_CMA;
 296                         man->drm_bus_maptype = _DRM_SCATTER_GATHER;
 297                         break;
 298                 }
 299                 break;
 300         default:
 301                 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
 302                 return -EINVAL;
 303         }
 304         return 0;
 305 }
 306
 307 int radeon_ms_invalidate_caches(struct drm_device * dev, uint64_t flags)
 308 {
 309         struct drm_radeon_private *dev_priv = dev->dev_private;
 310
 311         dev_priv->flush_cache(dev);
 312         return 0;
 313 }
 314
 315 void radeon_ms_ttm_flush(struct drm_ttm *ttm)
 316 {
 317         if (!ttm)
 318                 return;
 319
 320         DRM_MEMORYBARRIER();
 321 }