src/gallium/winsys/r600/drm/r600_hw_context.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Jerome Glisse
  25  */
  26 #include <errno.h>
  27 #include <stdint.h>
  28 #include <string.h>
  29 #include <stdlib.h>
  30 #include <assert.h>
  31 #include <pipe/p_compiler.h>
  32 #include <util/u_inlines.h>
  33 #include <util/u_memory.h>
  34 #include <pipebuffer/pb_bufmgr.h>
  35 #include "xf86drm.h"
  36 #include "radeon_drm.h"
  37 #include "r600_priv.h"
  38 #include "bof.h"
  39 #include "r600d.h"
  40
  41 #define GROUP_FORCE_NEW_BLOCK   0
  42
  43 /* Get backends mask */
  44 void r600_get_backend_mask(struct r600_context *ctx)
  45 {
  46         struct r600_bo * buffer;
  47         u32 * results;
  48         unsigned num_backends = r600_get_num_backends(ctx->radeon);
  49         unsigned i, mask = 0;
  50
  51         /* if backend_map query is supported by the kernel */
  52         if (ctx->radeon->backend_map_valid) {
  53                 unsigned num_tile_pipes = r600_get_num_tile_pipes(ctx->radeon);
  54                 unsigned backend_map = r600_get_backend_map(ctx->radeon);
  55                 unsigned item_width, item_mask;
  56
  57                 if (ctx->radeon->chip_class >= EVERGREEN) {
  58                         item_width = 4;
  59                         item_mask = 0x7;
  60                 } else {
  61                         item_width = 2;
  62                         item_mask = 0x3;
  63                 }
  64
  65                 while(num_tile_pipes--) {
  66                         i = backend_map & item_mask;
  67                         mask |= (1<<i);
  68                         backend_map >>= item_width;
  69                 }
  70                 if (mask != 0) {
  71                         ctx->backend_mask = mask;
  72                         return;
  73                 }
  74         }
  75
  76         /* otherwise backup path for older kernels */
  77
  78         /* create buffer for event data */
  79         buffer = r600_bo(ctx->radeon, ctx->max_db*16, 1, 0,
  80                                 PIPE_USAGE_STAGING);
  81         if (!buffer)
  82                 goto err;
  83
  84         /* initialize buffer with zeroes */
  85         results = r600_bo_map(ctx->radeon, buffer, PB_USAGE_CPU_WRITE, NULL);
  86         if (results) {
  87                 memset(results, 0, ctx->max_db * 4 * 4);
  88                 r600_bo_unmap(ctx->radeon, buffer);
  89
  90                 /* emit EVENT_WRITE for ZPASS_DONE */
  91                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
  92                 ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
  93                 ctx->pm4[ctx->pm4_cdwords++] = 0;
  94                 ctx->pm4[ctx->pm4_cdwords++] = 0;
  95
  96                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
  97                 ctx->pm4[ctx->pm4_cdwords++] = 0;
  98                 r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], buffer);
  99
 100                 /* execute */
 101                 r600_context_flush(ctx);
 102
 103                 /* analyze results */
 104                 results = r600_bo_map(ctx->radeon, buffer, PB_USAGE_CPU_READ, NULL);
 105                 if (results) {
 106                         for(i = 0; i < ctx->max_db; i++) {
 107                                 /* at least highest bit will be set if backend is used */
 108                                 if (results[i*4 + 1])
 109                                         mask |= (1<<i);
 110                         }
 111                         r600_bo_unmap(ctx->radeon, buffer);
 112                 }
 113         }
 114
 115         r600_bo_reference(ctx->radeon, &buffer, NULL);
 116
 117         if (mask != 0) {
 118                 ctx->backend_mask = mask;
 119                 return;
 120         }
 121
 122 err:
 123         /* fallback to old method - set num_backends lower bits to 1 */
 124         ctx->backend_mask = (~((u32)0))>>(32-num_backends);
 125         return;
 126 }
 127
 128 static inline void r600_context_ps_partial_flush(struct r600_context *ctx)
 129 {
 130         if (!(ctx->flags & R600_CONTEXT_DRAW_PENDING))
 131                 return;
 132
 133         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 134         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
 135
 136         ctx->flags &= ~R600_CONTEXT_DRAW_PENDING;
 137 }
 138
 139 void r600_init_cs(struct r600_context *ctx)
 140 {
 141         /* R6xx requires this packet at the start of each command buffer */
 142         if (ctx->radeon->family < CHIP_RV770) {
 143                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_START_3D_CMDBUF, 0, 0);
 144                 ctx->pm4[ctx->pm4_cdwords++] = 0x00000000;
 145         }
 146         /* All asics require this one */
 147         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_CONTEXT_CONTROL, 1, 0);
 148         ctx->pm4[ctx->pm4_cdwords++] = 0x80000000;
 149         ctx->pm4[ctx->pm4_cdwords++] = 0x80000000;
 150
 151         ctx->init_dwords = ctx->pm4_cdwords;
 152 }
 153
 154 static void INLINE r600_context_update_fenced_list(struct r600_context *ctx)
 155 {
 156         for (int i = 0; i < ctx->creloc; i++) {
 157                 if (!LIST_IS_EMPTY(&ctx->bo[i]->fencedlist))
 158                         LIST_DELINIT(&ctx->bo[i]->fencedlist);
 159                 LIST_ADDTAIL(&ctx->bo[i]->fencedlist, &ctx->fenced_bo);
 160                 ctx->bo[i]->fence = ctx->radeon->fence;
 161                 ctx->bo[i]->ctx = ctx;
 162         }
 163 }
 164
 165 static void INLINE r600_context_fence_wraparound(struct r600_context *ctx, unsigned fence)
 166 {
 167         struct radeon_bo *bo = NULL;
 168         struct radeon_bo *tmp;
 169
 170         LIST_FOR_EACH_ENTRY_SAFE(bo, tmp, &ctx->fenced_bo, fencedlist) {
 171                 if (bo->fence <= *ctx->radeon->cfence) {
 172                         LIST_DELINIT(&bo->fencedlist);
 173                         bo->fence = 0;
 174                 } else {
 175                         bo->fence = fence;
 176                 }
 177         }
 178 }
 179
 180 static void r600_init_block(struct r600_context *ctx,
 181                             struct r600_block *block,
 182                             const struct r600_reg *reg, int index, int nreg,
 183                             unsigned opcode, unsigned offset_base)
 184 {
 185         int i = index;
 186         int j, n = nreg;
 187
 188         /* initialize block */
 189         if (opcode == PKT3_SET_RESOURCE) {
 190                 block->flags = BLOCK_FLAG_RESOURCE;
 191                 block->status |= R600_BLOCK_STATUS_RESOURCE_DIRTY; /* dirty all blocks at start */
 192         } else {
 193                 block->flags = 0;
 194                 block->status |= R600_BLOCK_STATUS_DIRTY; /* dirty all blocks at start */
 195         }
 196         block->start_offset = reg[i].offset;
 197         block->pm4[block->pm4_ndwords++] = PKT3(opcode, n, 0);
 198         block->pm4[block->pm4_ndwords++] = (block->start_offset - offset_base) >> 2;
 199         block->reg = &block->pm4[block->pm4_ndwords];
 200         block->pm4_ndwords += n;
 201         block->nreg = n;
 202         block->nreg_dirty = n;
 203         LIST_INITHEAD(&block->list);
 204         LIST_INITHEAD(&block->enable_list);
 205
 206         for (j = 0; j < n; j++) {
 207                 if (reg[i+j].flags & REG_FLAG_DIRTY_ALWAYS) {
 208                         block->flags |= REG_FLAG_DIRTY_ALWAYS;
 209                 }
 210                 if (reg[i+j].flags & REG_FLAG_ENABLE_ALWAYS) {
 211                         if (!(block->status & R600_BLOCK_STATUS_ENABLED)) {
 212                                 block->status |= R600_BLOCK_STATUS_ENABLED;
 213                                 LIST_ADDTAIL(&block->enable_list, &ctx->enable_list);
 214                                 LIST_ADDTAIL(&block->list,&ctx->dirty);
 215                         }
 216                 }
 217                 if (reg[i+j].flags & REG_FLAG_FLUSH_CHANGE) {
 218                         block->flags |= REG_FLAG_FLUSH_CHANGE;
 219                 }
 220
 221                 if (reg[i+j].flags & REG_FLAG_NEED_BO) {
 222                         block->nbo++;
 223                         assert(block->nbo < R600_BLOCK_MAX_BO);
 224                         block->pm4_bo_index[j] = block->nbo;
 225                         block->pm4[block->pm4_ndwords++] = PKT3(PKT3_NOP, 0, 0);
 226                         block->pm4[block->pm4_ndwords++] = 0x00000000;
 227                         if (reg[i+j].flags & REG_FLAG_RV6XX_SBU) {
 228                                 block->reloc[block->nbo].flush_flags = 0;
 229                                 block->reloc[block->nbo].flush_mask = 0;
 230                         } else {
 231                                 block->reloc[block->nbo].flush_flags = reg[i+j].flush_flags;
 232                                 block->reloc[block->nbo].flush_mask = reg[i+j].flush_mask;
 233                         }
 234                         block->reloc[block->nbo].bo_pm4_index = block->pm4_ndwords - 1;
 235                 }
 236                 if ((ctx->radeon->family > CHIP_R600) &&
 237                     (ctx->radeon->family < CHIP_RV770) && reg[i+j].flags & REG_FLAG_RV6XX_SBU) {
 238                         block->pm4[block->pm4_ndwords++] = PKT3(PKT3_SURFACE_BASE_UPDATE, 0, 0);
 239                         block->pm4[block->pm4_ndwords++] = reg[i+j].flush_flags;
 240                 }
 241         }
 242         for (j = 0; j < n; j++) {
 243                 if (reg[i+j].flush_flags) {
 244                         block->pm4_flush_ndwords += 7;
 245                 }
 246         }
 247         /* check that we stay in limit */
 248         assert(block->pm4_ndwords < R600_BLOCK_MAX_REG);
 249 }
 250
 251 int r600_context_add_block(struct r600_context *ctx, const struct r600_reg *reg, unsigned nreg,
 252                            unsigned opcode, unsigned offset_base)
 253 {
 254         struct r600_block *block;
 255         struct r600_range *range;
 256         int offset;
 257
 258         for (unsigned i = 0, n = 0; i < nreg; i += n) {
 259                 /* ignore new block balise */
 260                 if (reg[i].offset == GROUP_FORCE_NEW_BLOCK) {
 261                         n = 1;
 262                         continue;
 263                 }
 264
 265                 /* ignore regs not on R600 on R600 */
 266                 if ((reg[i].flags & REG_FLAG_NOT_R600) && ctx->radeon->family == CHIP_R600) {
 267                         n = 1;
 268                         continue;
 269                 }
 270
 271                 /* register that need relocation are in their own group */
 272                 /* find number of consecutive registers */
 273                 n = 0;
 274                 offset = reg[i].offset;
 275                 while (reg[i + n].offset == offset) {
 276                         n++;
 277                         offset += 4;
 278                         if ((n + i) >= nreg)
 279                                 break;
 280                         if (n >= (R600_BLOCK_MAX_REG - 2))
 281                                 break;
 282                 }
 283
 284                 /* allocate new block */
 285                 block = calloc(1, sizeof(struct r600_block));
 286                 if (block == NULL) {
 287                         return -ENOMEM;
 288                 }
 289                 ctx->nblocks++;
 290                 for (int j = 0; j < n; j++) {
 291                         range = &ctx->range[CTX_RANGE_ID(reg[i + j].offset)];
 292                         /* create block table if it doesn't exist */
 293                         if (!range->blocks)
 294                                 range->blocks = calloc(1 << HASH_SHIFT, sizeof(void *));
 295                         if (!range->blocks)
 296                                 return -1;
 297
 298                         range->blocks[CTX_BLOCK_ID(reg[i + j].offset)] = block;
 299                 }
 300
 301                 r600_init_block(ctx, block, reg, i, n, opcode, offset_base);
 302
 303         }
 304         return 0;
 305 }
 306
 307 /* R600/R700 configuration */
 308 static const struct r600_reg r600_config_reg_list[] = {
 309         {R_008958_VGT_PRIMITIVE_TYPE, 0, 0, 0},
 310         {R_008C00_SQ_CONFIG, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 311         {R_008C04_SQ_GPR_RESOURCE_MGMT_1, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 312         {R_008C08_SQ_GPR_RESOURCE_MGMT_2, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 313         {R_008C0C_SQ_THREAD_RESOURCE_MGMT, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 314         {R_008C10_SQ_STACK_RESOURCE_MGMT_1, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 315         {R_008C14_SQ_STACK_RESOURCE_MGMT_2, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 316         {R_008D8C_SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 317         {R_009508_TA_CNTL_AUX, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 318         {R_009714_VC_ENHANCE, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 319         {R_009830_DB_DEBUG, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 320         {R_009838_DB_WATERMARKS, REG_FLAG_ENABLE_ALWAYS | REG_FLAG_FLUSH_CHANGE, 0, 0},
 321 };
 322
 323 static const struct r600_reg r600_ctl_const_list[] = {
 324         {R_03CFF0_SQ_VTX_BASE_VTX_LOC, 0, 0, 0},
 325         {R_03CFF4_SQ_VTX_START_INST_LOC, 0, 0, 0},
 326 };
 327
 328 static const struct r600_reg r600_context_reg_list[] = {
 329         {R_028350_SX_MISC, 0, 0, 0},
 330         {R_0286C8_SPI_THREAD_GROUPING, 0, 0, 0},
 331         {R_0288A8_SQ_ESGS_RING_ITEMSIZE, 0, 0, 0},
 332         {R_0288AC_SQ_GSVS_RING_ITEMSIZE, 0, 0, 0},
 333         {R_0288B0_SQ_ESTMP_RING_ITEMSIZE, 0, 0, 0},
 334         {R_0288B4_SQ_GSTMP_RING_ITEMSIZE, 0, 0, 0},
 335         {R_0288B8_SQ_VSTMP_RING_ITEMSIZE, 0, 0, 0},
 336         {R_0288BC_SQ_PSTMP_RING_ITEMSIZE, 0, 0, 0},
 337         {R_0288C0_SQ_FBUF_RING_ITEMSIZE, 0, 0, 0},
 338         {R_0288C4_SQ_REDUC_RING_ITEMSIZE, 0, 0, 0},
 339         {R_0288C8_SQ_GS_VERT_ITEMSIZE, 0, 0, 0},
 340         {R_028A10_VGT_OUTPUT_PATH_CNTL, 0, 0, 0},
 341         {R_028A14_VGT_HOS_CNTL, 0, 0, 0},
 342         {R_028A18_VGT_HOS_MAX_TESS_LEVEL, 0, 0, 0},
 343         {R_028A1C_VGT_HOS_MIN_TESS_LEVEL, 0, 0, 0},
 344         {R_028A20_VGT_HOS_REUSE_DEPTH, 0, 0, 0},
 345         {R_028A24_VGT_GROUP_PRIM_TYPE, 0, 0, 0},
 346         {R_028A28_VGT_GROUP_FIRST_DECR, 0, 0, 0},
 347         {R_028A2C_VGT_GROUP_DECR, 0, 0, 0},
 348         {R_028A30_VGT_GROUP_VECT_0_CNTL, 0, 0, 0},
 349         {R_028A34_VGT_GROUP_VECT_1_CNTL, 0, 0, 0},
 350         {R_028A38_VGT_GROUP_VECT_0_FMT_CNTL, 0, 0, 0},
 351         {R_028A3C_VGT_GROUP_VECT_1_FMT_CNTL, 0, 0, 0},
 352         {R_028A40_VGT_GS_MODE, 0, 0, 0},
 353         {R_028A4C_PA_SC_MODE_CNTL, 0, 0, 0},
 354         {R_028AB0_VGT_STRMOUT_EN, 0, 0, 0},
 355         {R_028AB4_VGT_REUSE_OFF, 0, 0, 0},
 356         {R_028AB8_VGT_VTX_CNT_EN, 0, 0, 0},
 357         {R_028B20_VGT_STRMOUT_BUFFER_EN, 0, 0, 0},
 358         {R_028028_DB_STENCIL_CLEAR, 0, 0, 0},
 359         {R_02802C_DB_DEPTH_CLEAR, 0, 0, 0},
 360         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 361         {R_028040_CB_COLOR0_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(0), 0},
 362         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 363         {R_0280A0_CB_COLOR0_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 364         {R_028060_CB_COLOR0_SIZE, 0, 0, 0},
 365         {R_028080_CB_COLOR0_VIEW, 0, 0, 0},
 366         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 367         {R_0280E0_CB_COLOR0_FRAG, REG_FLAG_NEED_BO, 0, 0},
 368         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 369         {R_0280C0_CB_COLOR0_TILE, REG_FLAG_NEED_BO, 0, 0},
 370         {R_028100_CB_COLOR0_MASK, 0, 0, 0},
 371         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 372         {R_028044_CB_COLOR1_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(1), 0},
 373         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 374         {R_0280A4_CB_COLOR1_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 375         {R_028064_CB_COLOR1_SIZE, 0, 0, 0},
 376         {R_028084_CB_COLOR1_VIEW, 0, 0, 0},
 377         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 378         {R_0280E4_CB_COLOR1_FRAG, REG_FLAG_NEED_BO, 0, 0},
 379         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 380         {R_0280C4_CB_COLOR1_TILE, REG_FLAG_NEED_BO, 0, 0},
 381         {R_028104_CB_COLOR1_MASK, 0, 0, 0},
 382         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 383         {R_028048_CB_COLOR2_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(2), 0},
 384         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 385         {R_0280A8_CB_COLOR2_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 386         {R_028068_CB_COLOR2_SIZE, 0, 0, 0},
 387         {R_028088_CB_COLOR2_VIEW, 0, 0, 0},
 388         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 389         {R_0280E8_CB_COLOR2_FRAG, REG_FLAG_NEED_BO, 0, 0},
 390         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 391         {R_0280C8_CB_COLOR2_TILE, REG_FLAG_NEED_BO, 0, 0},
 392         {R_028108_CB_COLOR2_MASK, 0, 0, 0},
 393         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 394         {R_02804C_CB_COLOR3_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(3), 0},
 395         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 396         {R_0280AC_CB_COLOR3_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 397         {R_02806C_CB_COLOR3_SIZE, 0, 0, 0},
 398         {R_02808C_CB_COLOR3_VIEW, 0, 0, 0},
 399         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 400         {R_0280EC_CB_COLOR3_FRAG, REG_FLAG_NEED_BO, 0, 0},
 401         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 402         {R_0280CC_CB_COLOR3_TILE, REG_FLAG_NEED_BO, 0, 0},
 403         {R_02810C_CB_COLOR3_MASK, 0, 0, 0},
 404         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 405         {R_028050_CB_COLOR4_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(4), 0},
 406         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 407         {R_0280B0_CB_COLOR4_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 408         {R_028070_CB_COLOR4_SIZE, 0, 0, 0},
 409         {R_028090_CB_COLOR4_VIEW, 0, 0, 0},
 410         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 411         {R_0280F0_CB_COLOR4_FRAG, REG_FLAG_NEED_BO, 0, 0},
 412         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 413         {R_0280D0_CB_COLOR4_TILE, REG_FLAG_NEED_BO, 0, 0},
 414         {R_028110_CB_COLOR4_MASK, 0, 0, 0},
 415         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 416         {R_028054_CB_COLOR5_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(5), 0},
 417         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 418         {R_0280B4_CB_COLOR5_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 419         {R_028074_CB_COLOR5_SIZE, 0, 0, 0},
 420         {R_028094_CB_COLOR5_VIEW, 0, 0, 0},
 421         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 422         {R_0280F4_CB_COLOR5_FRAG, REG_FLAG_NEED_BO, 0, 0},
 423         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 424         {R_0280D4_CB_COLOR5_TILE, REG_FLAG_NEED_BO, 0, 0},
 425         {R_028114_CB_COLOR5_MASK, 0, 0, 0},
 426         {R_028058_CB_COLOR6_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(6), 0},
 427         {R_0280B8_CB_COLOR6_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 428         {R_028078_CB_COLOR6_SIZE, 0, 0, 0},
 429         {R_028098_CB_COLOR6_VIEW, 0, 0, 0},
 430         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 431         {R_0280F8_CB_COLOR6_FRAG, REG_FLAG_NEED_BO, 0, 0},
 432         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 433         {R_0280D8_CB_COLOR6_TILE, REG_FLAG_NEED_BO, 0, 0},
 434         {R_028118_CB_COLOR6_MASK, 0, 0, 0},
 435         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 436         {R_02805C_CB_COLOR7_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_COLOR(7), 0},
 437         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 438         {R_0280BC_CB_COLOR7_INFO, REG_FLAG_NEED_BO, 0, 0xFFFFFFFF},
 439         {R_02807C_CB_COLOR7_SIZE, 0, 0, 0},
 440         {R_02809C_CB_COLOR7_VIEW, 0, 0, 0},
 441         {R_0280FC_CB_COLOR7_FRAG, REG_FLAG_NEED_BO, 0, 0},
 442         {R_0280DC_CB_COLOR7_TILE, REG_FLAG_NEED_BO, 0, 0},
 443         {R_02811C_CB_COLOR7_MASK, 0, 0, 0},
 444         {R_028120_CB_CLEAR_RED, 0, 0, 0},
 445         {R_028124_CB_CLEAR_GREEN, 0, 0, 0},
 446         {R_028128_CB_CLEAR_BLUE, 0, 0, 0},
 447         {R_02812C_CB_CLEAR_ALPHA, 0, 0, 0},
 448         {R_028140_ALU_CONST_BUFFER_SIZE_PS_0, REG_FLAG_DIRTY_ALWAYS, 0, 0},
 449         {R_028180_ALU_CONST_BUFFER_SIZE_VS_0, REG_FLAG_DIRTY_ALWAYS, 0, 0},
 450         {R_028940_ALU_CONST_CACHE_PS_0, REG_FLAG_NEED_BO, S_0085F0_SH_ACTION_ENA(1), 0xFFFFFFFF},
 451         {R_028980_ALU_CONST_CACHE_VS_0, REG_FLAG_NEED_BO, S_0085F0_SH_ACTION_ENA(1), 0xFFFFFFFF},
 452         {R_02823C_CB_SHADER_MASK, 0, 0, 0},
 453         {R_028238_CB_TARGET_MASK, 0, 0, 0},
 454         {R_028410_SX_ALPHA_TEST_CONTROL, 0, 0, 0},
 455         {R_028414_CB_BLEND_RED, 0, 0, 0},
 456         {R_028418_CB_BLEND_GREEN, 0, 0, 0},
 457         {R_02841C_CB_BLEND_BLUE, 0, 0, 0},
 458         {R_028420_CB_BLEND_ALPHA, 0, 0, 0},
 459         {R_028424_CB_FOG_RED, 0, 0, 0},
 460         {R_028428_CB_FOG_GREEN, 0, 0, 0},
 461         {R_02842C_CB_FOG_BLUE, 0, 0, 0},
 462         {R_028430_DB_STENCILREFMASK, 0, 0, 0},
 463         {R_028434_DB_STENCILREFMASK_BF, 0, 0, 0},
 464         {R_028438_SX_ALPHA_REF, 0, 0, 0},
 465         {R_0286DC_SPI_FOG_CNTL, 0, 0, 0},
 466         {R_0286E0_SPI_FOG_FUNC_SCALE, 0, 0, 0},
 467         {R_0286E4_SPI_FOG_FUNC_BIAS, 0, 0, 0},
 468         {R_028780_CB_BLEND0_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 469         {R_028784_CB_BLEND1_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 470         {R_028788_CB_BLEND2_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 471         {R_02878C_CB_BLEND3_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 472         {R_028790_CB_BLEND4_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 473         {R_028794_CB_BLEND5_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 474         {R_028798_CB_BLEND6_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 475         {R_02879C_CB_BLEND7_CONTROL, REG_FLAG_NOT_R600, 0, 0},
 476         {R_0287A0_CB_SHADER_CONTROL, 0, 0, 0},
 477         {R_028800_DB_DEPTH_CONTROL, 0, 0, 0},
 478         {R_028804_CB_BLEND_CONTROL, 0, 0, 0},
 479         {R_028808_CB_COLOR_CONTROL, 0, 0, 0},
 480         {R_02880C_DB_SHADER_CONTROL, 0, 0, 0},
 481         {R_028C04_PA_SC_AA_CONFIG, 0, 0, 0},
 482         {R_028C1C_PA_SC_AA_SAMPLE_LOCS_MCTX, 0, 0, 0},
 483         {R_028C20_PA_SC_AA_SAMPLE_LOCS_8S_WD1_MCTX, 0, 0, 0},
 484         {R_028C30_CB_CLRCMP_CONTROL, 0, 0, 0},
 485         {R_028C34_CB_CLRCMP_SRC, 0, 0, 0},
 486         {R_028C38_CB_CLRCMP_DST, 0, 0, 0},
 487         {R_028C3C_CB_CLRCMP_MSK, 0, 0, 0},
 488         {R_028C48_PA_SC_AA_MASK, 0, 0, 0},
 489         {R_028D2C_DB_SRESULTS_COMPARE_STATE1, 0, 0, 0},
 490         {R_028D44_DB_ALPHA_TO_MASK, 0, 0, 0},
 491         {R_02800C_DB_DEPTH_BASE, REG_FLAG_NEED_BO|REG_FLAG_RV6XX_SBU, SURFACE_BASE_UPDATE_DEPTH, 0},
 492         {R_028000_DB_DEPTH_SIZE, 0, 0, 0},
 493         {R_028004_DB_DEPTH_VIEW, 0, 0, 0},
 494         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 495         {R_028010_DB_DEPTH_INFO, REG_FLAG_NEED_BO, 0, 0},
 496         {R_028D0C_DB_RENDER_CONTROL, 0, 0, 0},
 497         {R_028D10_DB_RENDER_OVERRIDE, 0, 0, 0},
 498         {R_028D24_DB_HTILE_SURFACE, 0, 0, 0},
 499         {R_028D30_DB_PRELOAD_CONTROL, 0, 0, 0},
 500         {R_028D34_DB_PREFETCH_LIMIT, 0, 0, 0},
 501         {R_028030_PA_SC_SCREEN_SCISSOR_TL, 0, 0, 0},
 502         {R_028034_PA_SC_SCREEN_SCISSOR_BR, 0, 0, 0},
 503         {R_028200_PA_SC_WINDOW_OFFSET, 0, 0, 0},
 504         {R_028204_PA_SC_WINDOW_SCISSOR_TL, 0, 0, 0},
 505         {R_028208_PA_SC_WINDOW_SCISSOR_BR, 0, 0, 0},
 506         {R_02820C_PA_SC_CLIPRECT_RULE, 0, 0, 0},
 507         {R_028210_PA_SC_CLIPRECT_0_TL, 0, 0, 0},
 508         {R_028214_PA_SC_CLIPRECT_0_BR, 0, 0, 0},
 509         {R_028218_PA_SC_CLIPRECT_1_TL, 0, 0, 0},
 510         {R_02821C_PA_SC_CLIPRECT_1_BR, 0, 0, 0},
 511         {R_028220_PA_SC_CLIPRECT_2_TL, 0, 0, 0},
 512         {R_028224_PA_SC_CLIPRECT_2_BR, 0, 0, 0},
 513         {R_028228_PA_SC_CLIPRECT_3_TL, 0, 0, 0},
 514         {R_02822C_PA_SC_CLIPRECT_3_BR, 0, 0, 0},
 515         {R_028230_PA_SC_EDGERULE, 0, 0, 0},
 516         {R_028240_PA_SC_GENERIC_SCISSOR_TL, 0, 0, 0},
 517         {R_028244_PA_SC_GENERIC_SCISSOR_BR, 0, 0, 0},
 518         {R_028250_PA_SC_VPORT_SCISSOR_0_TL, 0, 0, 0},
 519         {R_028254_PA_SC_VPORT_SCISSOR_0_BR, 0, 0, 0},
 520         {R_0282D0_PA_SC_VPORT_ZMIN_0, 0, 0, 0},
 521         {R_0282D4_PA_SC_VPORT_ZMAX_0, 0, 0, 0},
 522         {R_02843C_PA_CL_VPORT_XSCALE_0, 0, 0, 0},
 523         {R_028440_PA_CL_VPORT_XOFFSET_0, 0, 0, 0},
 524         {R_028444_PA_CL_VPORT_YSCALE_0, 0, 0, 0},
 525         {R_028448_PA_CL_VPORT_YOFFSET_0, 0, 0, 0},
 526         {R_02844C_PA_CL_VPORT_ZSCALE_0, 0, 0, 0},
 527         {R_028450_PA_CL_VPORT_ZOFFSET_0, 0, 0, 0},
 528         {R_0286D4_SPI_INTERP_CONTROL_0, 0, 0, 0},
 529         {R_028810_PA_CL_CLIP_CNTL, 0, 0, 0},
 530         {R_028814_PA_SU_SC_MODE_CNTL, 0, 0, 0},
 531         {R_028818_PA_CL_VTE_CNTL, 0, 0, 0},
 532         {R_02881C_PA_CL_VS_OUT_CNTL, 0, 0, 0},
 533         {R_028820_PA_CL_NANINF_CNTL, 0, 0, 0},
 534         {R_028A00_PA_SU_POINT_SIZE, 0, 0, 0},
 535         {R_028A04_PA_SU_POINT_MINMAX, 0, 0, 0},
 536         {R_028A08_PA_SU_LINE_CNTL, 0, 0, 0},
 537         {R_028A0C_PA_SC_LINE_STIPPLE, 0, 0, 0},
 538         {R_028A48_PA_SC_MPASS_PS_CNTL, 0, 0, 0},
 539         {R_028C00_PA_SC_LINE_CNTL, 0, 0, 0},
 540         {R_028C08_PA_SU_VTX_CNTL, 0, 0, 0},
 541         {R_028C0C_PA_CL_GB_VERT_CLIP_ADJ, 0, 0, 0},
 542         {R_028C10_PA_CL_GB_VERT_DISC_ADJ, 0, 0, 0},
 543         {R_028C14_PA_CL_GB_HORZ_CLIP_ADJ, 0, 0, 0},
 544         {R_028C18_PA_CL_GB_HORZ_DISC_ADJ, 0, 0, 0},
 545         {R_028DF8_PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0, 0, 0},
 546         {R_028DFC_PA_SU_POLY_OFFSET_CLAMP, 0, 0, 0},
 547         {R_028E00_PA_SU_POLY_OFFSET_FRONT_SCALE, 0, 0, 0},
 548         {R_028E04_PA_SU_POLY_OFFSET_FRONT_OFFSET, 0, 0, 0},
 549         {R_028E08_PA_SU_POLY_OFFSET_BACK_SCALE, 0, 0, 0},
 550         {R_028E0C_PA_SU_POLY_OFFSET_BACK_OFFSET, 0, 0, 0},
 551         {R_028E20_PA_CL_UCP0_X, 0, 0, 0},
 552         {R_028E24_PA_CL_UCP0_Y, 0, 0, 0},
 553         {R_028E28_PA_CL_UCP0_Z, 0, 0, 0},
 554         {R_028E2C_PA_CL_UCP0_W, 0, 0, 0},
 555         {R_028E30_PA_CL_UCP1_X, 0, 0, 0},
 556         {R_028E34_PA_CL_UCP1_Y, 0, 0, 0},
 557         {R_028E38_PA_CL_UCP1_Z, 0, 0, 0},
 558         {R_028E3C_PA_CL_UCP1_W, 0, 0, 0},
 559         {R_028E40_PA_CL_UCP2_X, 0, 0, 0},
 560         {R_028E44_PA_CL_UCP2_Y, 0, 0, 0},
 561         {R_028E48_PA_CL_UCP2_Z, 0, 0, 0},
 562         {R_028E4C_PA_CL_UCP2_W, 0, 0, 0},
 563         {R_028E50_PA_CL_UCP3_X, 0, 0, 0},
 564         {R_028E54_PA_CL_UCP3_Y, 0, 0, 0},
 565         {R_028E58_PA_CL_UCP3_Z, 0, 0, 0},
 566         {R_028E5C_PA_CL_UCP3_W, 0, 0, 0},
 567         {R_028E60_PA_CL_UCP4_X, 0, 0, 0},
 568         {R_028E64_PA_CL_UCP4_Y, 0, 0, 0},
 569         {R_028E68_PA_CL_UCP4_Z, 0, 0, 0},
 570         {R_028E6C_PA_CL_UCP4_W, 0, 0, 0},
 571         {R_028E70_PA_CL_UCP5_X, 0, 0, 0},
 572         {R_028E74_PA_CL_UCP5_Y, 0, 0, 0},
 573         {R_028E78_PA_CL_UCP5_Z, 0, 0, 0},
 574         {R_028E7C_PA_CL_UCP5_W, 0, 0, 0},
 575         {R_028380_SQ_VTX_SEMANTIC_0, 0, 0, 0},
 576         {R_028384_SQ_VTX_SEMANTIC_1, 0, 0, 0},
 577         {R_028388_SQ_VTX_SEMANTIC_2, 0, 0, 0},
 578         {R_02838C_SQ_VTX_SEMANTIC_3, 0, 0, 0},
 579         {R_028390_SQ_VTX_SEMANTIC_4, 0, 0, 0},
 580         {R_028394_SQ_VTX_SEMANTIC_5, 0, 0, 0},
 581         {R_028398_SQ_VTX_SEMANTIC_6, 0, 0, 0},
 582         {R_02839C_SQ_VTX_SEMANTIC_7, 0, 0, 0},
 583         {R_0283A0_SQ_VTX_SEMANTIC_8, 0, 0, 0},
 584         {R_0283A4_SQ_VTX_SEMANTIC_9, 0, 0, 0},
 585         {R_0283A8_SQ_VTX_SEMANTIC_10, 0, 0, 0},
 586         {R_0283AC_SQ_VTX_SEMANTIC_11, 0, 0, 0},
 587         {R_0283B0_SQ_VTX_SEMANTIC_12, 0, 0, 0},
 588         {R_0283B4_SQ_VTX_SEMANTIC_13, 0, 0, 0},
 589         {R_0283B8_SQ_VTX_SEMANTIC_14, 0, 0, 0},
 590         {R_0283BC_SQ_VTX_SEMANTIC_15, 0, 0, 0},
 591         {R_0283C0_SQ_VTX_SEMANTIC_16, 0, 0, 0},
 592         {R_0283C4_SQ_VTX_SEMANTIC_17, 0, 0, 0},
 593         {R_0283C8_SQ_VTX_SEMANTIC_18, 0, 0, 0},
 594         {R_0283CC_SQ_VTX_SEMANTIC_19, 0, 0, 0},
 595         {R_0283D0_SQ_VTX_SEMANTIC_20, 0, 0, 0},
 596         {R_0283D4_SQ_VTX_SEMANTIC_21, 0, 0, 0},
 597         {R_0283D8_SQ_VTX_SEMANTIC_22, 0, 0, 0},
 598         {R_0283DC_SQ_VTX_SEMANTIC_23, 0, 0, 0},
 599         {R_0283E0_SQ_VTX_SEMANTIC_24, 0, 0, 0},
 600         {R_0283E4_SQ_VTX_SEMANTIC_25, 0, 0, 0},
 601         {R_0283E8_SQ_VTX_SEMANTIC_26, 0, 0, 0},
 602         {R_0283EC_SQ_VTX_SEMANTIC_27, 0, 0, 0},
 603         {R_0283F0_SQ_VTX_SEMANTIC_28, 0, 0, 0},
 604         {R_0283F4_SQ_VTX_SEMANTIC_29, 0, 0, 0},
 605         {R_0283F8_SQ_VTX_SEMANTIC_30, 0, 0, 0},
 606         {R_0283FC_SQ_VTX_SEMANTIC_31, 0, 0, 0},
 607         {R_028614_SPI_VS_OUT_ID_0, 0, 0, 0},
 608         {R_028618_SPI_VS_OUT_ID_1, 0, 0, 0},
 609         {R_02861C_SPI_VS_OUT_ID_2, 0, 0, 0},
 610         {R_028620_SPI_VS_OUT_ID_3, 0, 0, 0},
 611         {R_028624_SPI_VS_OUT_ID_4, 0, 0, 0},
 612         {R_028628_SPI_VS_OUT_ID_5, 0, 0, 0},
 613         {R_02862C_SPI_VS_OUT_ID_6, 0, 0, 0},
 614         {R_028630_SPI_VS_OUT_ID_7, 0, 0, 0},
 615         {R_028634_SPI_VS_OUT_ID_8, 0, 0, 0},
 616         {R_028638_SPI_VS_OUT_ID_9, 0, 0, 0},
 617         {R_0286C4_SPI_VS_OUT_CONFIG, 0, 0, 0},
 618         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 619         {R_028858_SQ_PGM_START_VS, REG_FLAG_NEED_BO, S_0085F0_SH_ACTION_ENA(1), 0xFFFFFFFF},
 620         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 621         {R_028868_SQ_PGM_RESOURCES_VS, 0, 0, 0},
 622         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 623         {R_028894_SQ_PGM_START_FS, REG_FLAG_NEED_BO, S_0085F0_SH_ACTION_ENA(1), 0xFFFFFFFF},
 624         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 625         {R_0288A4_SQ_PGM_RESOURCES_FS, 0, 0, 0},
 626         {R_0288D0_SQ_PGM_CF_OFFSET_VS, 0, 0, 0},
 627         {R_0288DC_SQ_PGM_CF_OFFSET_FS, 0, 0, 0},
 628         {R_028644_SPI_PS_INPUT_CNTL_0, 0, 0, 0},
 629         {R_028648_SPI_PS_INPUT_CNTL_1, 0, 0, 0},
 630         {R_02864C_SPI_PS_INPUT_CNTL_2, 0, 0, 0},
 631         {R_028650_SPI_PS_INPUT_CNTL_3, 0, 0, 0},
 632         {R_028654_SPI_PS_INPUT_CNTL_4, 0, 0, 0},
 633         {R_028658_SPI_PS_INPUT_CNTL_5, 0, 0, 0},
 634         {R_02865C_SPI_PS_INPUT_CNTL_6, 0, 0, 0},
 635         {R_028660_SPI_PS_INPUT_CNTL_7, 0, 0, 0},
 636         {R_028664_SPI_PS_INPUT_CNTL_8, 0, 0, 0},
 637         {R_028668_SPI_PS_INPUT_CNTL_9, 0, 0, 0},
 638         {R_02866C_SPI_PS_INPUT_CNTL_10, 0, 0, 0},
 639         {R_028670_SPI_PS_INPUT_CNTL_11, 0, 0, 0},
 640         {R_028674_SPI_PS_INPUT_CNTL_12, 0, 0, 0},
 641         {R_028678_SPI_PS_INPUT_CNTL_13, 0, 0, 0},
 642         {R_02867C_SPI_PS_INPUT_CNTL_14, 0, 0, 0},
 643         {R_028680_SPI_PS_INPUT_CNTL_15, 0, 0, 0},
 644         {R_028684_SPI_PS_INPUT_CNTL_16, 0, 0, 0},
 645         {R_028688_SPI_PS_INPUT_CNTL_17, 0, 0, 0},
 646         {R_02868C_SPI_PS_INPUT_CNTL_18, 0, 0, 0},
 647         {R_028690_SPI_PS_INPUT_CNTL_19, 0, 0, 0},
 648         {R_028694_SPI_PS_INPUT_CNTL_20, 0, 0, 0},
 649         {R_028698_SPI_PS_INPUT_CNTL_21, 0, 0, 0},
 650         {R_02869C_SPI_PS_INPUT_CNTL_22, 0, 0, 0},
 651         {R_0286A0_SPI_PS_INPUT_CNTL_23, 0, 0, 0},
 652         {R_0286A4_SPI_PS_INPUT_CNTL_24, 0, 0, 0},
 653         {R_0286A8_SPI_PS_INPUT_CNTL_25, 0, 0, 0},
 654         {R_0286AC_SPI_PS_INPUT_CNTL_26, 0, 0, 0},
 655         {R_0286B0_SPI_PS_INPUT_CNTL_27, 0, 0, 0},
 656         {R_0286B4_SPI_PS_INPUT_CNTL_28, 0, 0, 0},
 657         {R_0286B8_SPI_PS_INPUT_CNTL_29, 0, 0, 0},
 658         {R_0286BC_SPI_PS_INPUT_CNTL_30, 0, 0, 0},
 659         {R_0286C0_SPI_PS_INPUT_CNTL_31, 0, 0, 0},
 660         {R_0286CC_SPI_PS_IN_CONTROL_0, 0, 0, 0},
 661         {R_0286D0_SPI_PS_IN_CONTROL_1, 0, 0, 0},
 662         {R_0286D8_SPI_INPUT_Z, 0, 0, 0},
 663         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 664         {R_028840_SQ_PGM_START_PS, REG_FLAG_NEED_BO, S_0085F0_SH_ACTION_ENA(1), 0xFFFFFFFF},
 665         {GROUP_FORCE_NEW_BLOCK, 0, 0, 0},
 666         {R_028850_SQ_PGM_RESOURCES_PS, 0, 0, 0},
 667         {R_028854_SQ_PGM_EXPORTS_PS, 0, 0, 0},
 668         {R_0288CC_SQ_PGM_CF_OFFSET_PS, 0, 0, 0},
 669         {R_028400_VGT_MAX_VTX_INDX, 0, 0, 0},
 670         {R_028404_VGT_MIN_VTX_INDX, 0, 0, 0},
 671         {R_028408_VGT_INDX_OFFSET, 0, 0, 0},
 672         {R_02840C_VGT_MULTI_PRIM_IB_RESET_INDX, 0, 0, 0},
 673         {R_028A84_VGT_PRIMITIVEID_EN, 0, 0, 0},
 674         {R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, 0, 0, 0},
 675         {R_028AA0_VGT_INSTANCE_STEP_RATE_0, 0, 0, 0},
 676         {R_028AA4_VGT_INSTANCE_STEP_RATE_1, 0, 0, 0},
 677 };
 678
 679 /* SHADER RESOURCE R600/R700 */
 680 int r600_resource_init(struct r600_context *ctx, struct r600_range *range, unsigned offset, unsigned nblocks, unsigned stride, struct r600_reg *reg, int nreg, unsigned offset_base)
 681 {
 682         int i;
 683         struct r600_block *block;
 684         range->blocks = calloc(nblocks, sizeof(struct r600_block *));
 685         if (range->blocks == NULL)
 686                 return -ENOMEM;
 687
 688         reg[0].offset += offset;
 689         for (i = 0; i < nblocks; i++) {
 690                 block = calloc(1, sizeof(struct r600_block));
 691                 if (block == NULL) {
 692                         return -ENOMEM;
 693                 }
 694                 ctx->nblocks++;
 695                 range->blocks[i] = block;
 696                 r600_init_block(ctx, block, reg, 0, nreg, PKT3_SET_RESOURCE, offset_base);
 697
 698                 reg[0].offset += stride;
 699         }
 700         return 0;
 701 }
 702
 703
 704 static int r600_resource_range_init(struct r600_context *ctx, struct r600_range *range, unsigned offset, unsigned nblocks, unsigned stride)
 705 {
 706         struct r600_reg r600_shader_resource[] = {
 707                 {R_038000_RESOURCE0_WORD0, REG_FLAG_NEED_BO, S_0085F0_TC_ACTION_ENA(1) | S_0085F0_VC_ACTION_ENA(1), 0xFFFFFFFF},
 708                 {R_038004_RESOURCE0_WORD1, REG_FLAG_NEED_BO, S_0085F0_TC_ACTION_ENA(1) | S_0085F0_VC_ACTION_ENA(1), 0xFFFFFFFF},
 709                 {R_038008_RESOURCE0_WORD2, 0, 0, 0},
 710                 {R_03800C_RESOURCE0_WORD3, 0, 0, 0},
 711                 {R_038010_RESOURCE0_WORD4, 0, 0, 0},
 712                 {R_038014_RESOURCE0_WORD5, 0, 0, 0},
 713                 {R_038018_RESOURCE0_WORD6, 0, 0, 0},
 714         };
 715         unsigned nreg = Elements(r600_shader_resource);
 716
 717         return r600_resource_init(ctx, range, offset, nblocks, stride, r600_shader_resource, nreg, R600_RESOURCE_OFFSET);
 718 }
 719
 720 /* SHADER SAMPLER R600/R700 */
 721 static int r600_state_sampler_init(struct r600_context *ctx, u32 offset)
 722 {
 723         struct r600_reg r600_shader_sampler[] = {
 724                 {R_03C000_SQ_TEX_SAMPLER_WORD0_0, 0, 0, 0},
 725                 {R_03C004_SQ_TEX_SAMPLER_WORD1_0, 0, 0, 0},
 726                 {R_03C008_SQ_TEX_SAMPLER_WORD2_0, 0, 0, 0},
 727         };
 728         unsigned nreg = Elements(r600_shader_sampler);
 729
 730         for (int i = 0; i < nreg; i++) {
 731                 r600_shader_sampler[i].offset += offset;
 732         }
 733         return r600_context_add_block(ctx, r600_shader_sampler, nreg, PKT3_SET_SAMPLER, R600_SAMPLER_OFFSET);
 734 }
 735
 736 /* SHADER SAMPLER BORDER R600/R700 */
 737 static int r600_state_sampler_border_init(struct r600_context *ctx, u32 offset)
 738 {
 739         struct r600_reg r600_shader_sampler_border[] = {
 740                 {R_00A400_TD_PS_SAMPLER0_BORDER_RED, 0, 0, 0},
 741                 {R_00A404_TD_PS_SAMPLER0_BORDER_GREEN, 0, 0, 0},
 742                 {R_00A408_TD_PS_SAMPLER0_BORDER_BLUE, 0, 0, 0},
 743                 {R_00A40C_TD_PS_SAMPLER0_BORDER_ALPHA, 0, 0, 0},
 744         };
 745         unsigned nreg = Elements(r600_shader_sampler_border);
 746
 747         for (int i = 0; i < nreg; i++) {
 748                 r600_shader_sampler_border[i].offset += offset;
 749         }
 750         return r600_context_add_block(ctx, r600_shader_sampler_border, nreg, PKT3_SET_CONFIG_REG, R600_CONFIG_REG_OFFSET);
 751 }
 752
 753 static int r600_loop_const_init(struct r600_context *ctx, u32 offset)
 754 {
 755         unsigned nreg = 32;
 756         struct r600_reg r600_loop_consts[32];
 757         int i;
 758
 759         for (i = 0; i < nreg; i++) {
 760                 r600_loop_consts[i].offset = R600_LOOP_CONST_OFFSET + ((offset + i) * 4);
 761                 r600_loop_consts[i].flags = REG_FLAG_DIRTY_ALWAYS;
 762                 r600_loop_consts[i].flush_flags = 0;
 763                 r600_loop_consts[i].flush_mask = 0;
 764         }
 765         return r600_context_add_block(ctx, r600_loop_consts, nreg, PKT3_SET_LOOP_CONST, R600_LOOP_CONST_OFFSET);
 766 }
 767
 768 static void r600_context_clear_fenced_bo(struct r600_context *ctx)
 769 {
 770         struct radeon_bo *bo, *tmp;
 771
 772         LIST_FOR_EACH_ENTRY_SAFE(bo, tmp, &ctx->fenced_bo, fencedlist) {
 773                 LIST_DELINIT(&bo->fencedlist);
 774                 bo->fence = 0;
 775                 bo->ctx = NULL;
 776         }
 777 }
 778
 779 static void r600_free_resource_range(struct r600_context *ctx, struct r600_range *range, int nblocks)
 780 {
 781         struct r600_block *block;
 782         int i;
 783         for (i = 0; i < nblocks; i++) {
 784                 block = range->blocks[i];
 785                 if (block) {
 786                         for (int k = 1; k <= block->nbo; k++)
 787                                 r600_bo_reference(ctx->radeon, &block->reloc[k].bo, NULL);
 788                         free(block);
 789                 }
 790         }
 791         free(range->blocks);
 792
 793 }
 794
 795 /* initialize */
 796 void r600_context_fini(struct r600_context *ctx)
 797 {
 798         struct r600_block *block;
 799         struct r600_range *range;
 800
 801         for (int i = 0; i < NUM_RANGES; i++) {
 802                 if (!ctx->range[i].blocks)
 803                         continue;
 804                 for (int j = 0; j < (1 << HASH_SHIFT); j++) {
 805                         block = ctx->range[i].blocks[j];
 806                         if (block) {
 807                                 for (int k = 0, offset = block->start_offset; k < block->nreg; k++, offset += 4) {
 808                                         range = &ctx->range[CTX_RANGE_ID(offset)];
 809                                         range->blocks[CTX_BLOCK_ID(offset)] = NULL;
 810                                 }
 811                                 for (int k = 1; k <= block->nbo; k++) {
 812                                         r600_bo_reference(ctx->radeon, &block->reloc[k].bo, NULL);
 813                                 }
 814                                 free(block);
 815                         }
 816                 }
 817                 free(ctx->range[i].blocks);
 818         }
 819         r600_free_resource_range(ctx, &ctx->ps_resources, ctx->num_ps_resources);
 820         r600_free_resource_range(ctx, &ctx->vs_resources, ctx->num_vs_resources);
 821         r600_free_resource_range(ctx, &ctx->fs_resources, ctx->num_fs_resources);
 822         free(ctx->range);
 823         free(ctx->blocks);
 824         free(ctx->reloc);
 825         free(ctx->bo);
 826         free(ctx->pm4);
 827
 828         r600_context_clear_fenced_bo(ctx);
 829         memset(ctx, 0, sizeof(struct r600_context));
 830 }
 831
 832 static void r600_add_resource_block(struct r600_context *ctx, struct r600_range *range, int num_blocks, int *index)
 833 {
 834         int c = *index;
 835         for (int j = 0; j < num_blocks; j++) {
 836                 if (!range->blocks[j])
 837                         continue;
 838
 839                 ctx->blocks[c++] = range->blocks[j];
 840         }
 841         *index = c;
 842 }
 843
 844 int r600_setup_block_table(struct r600_context *ctx)
 845 {
 846         /* setup block table */
 847         int c = 0;
 848         ctx->blocks = calloc(ctx->nblocks, sizeof(void*));
 849         if (!ctx->blocks)
 850                 return -ENOMEM;
 851         for (int i = 0; i < NUM_RANGES; i++) {
 852                 if (!ctx->range[i].blocks)
 853                         continue;
 854                 for (int j = 0, add; j < (1 << HASH_SHIFT); j++) {
 855                         if (!ctx->range[i].blocks[j])
 856                                 continue;
 857
 858                         add = 1;
 859                         for (int k = 0; k < c; k++) {
 860                                 if (ctx->blocks[k] == ctx->range[i].blocks[j]) {
 861                                         add = 0;
 862                                         break;
 863                                 }
 864                         }
 865                         if (add) {
 866                                 assert(c < ctx->nblocks);
 867                                 ctx->blocks[c++] = ctx->range[i].blocks[j];
 868                                 j += (ctx->range[i].blocks[j]->nreg) - 1;
 869                         }
 870                 }
 871         }
 872
 873         r600_add_resource_block(ctx, &ctx->ps_resources, ctx->num_ps_resources, &c);
 874         r600_add_resource_block(ctx, &ctx->vs_resources, ctx->num_vs_resources, &c);
 875         r600_add_resource_block(ctx, &ctx->fs_resources, ctx->num_fs_resources, &c);
 876         return 0;
 877 }
 878
 879 int r600_context_init(struct r600_context *ctx, struct radeon *radeon)
 880 {
 881         int r;
 882
 883         memset(ctx, 0, sizeof(struct r600_context));
 884         ctx->radeon = radeon;
 885         LIST_INITHEAD(&ctx->query_list);
 886
 887         /* init dirty list */
 888         LIST_INITHEAD(&ctx->dirty);
 889         LIST_INITHEAD(&ctx->resource_dirty);
 890         LIST_INITHEAD(&ctx->enable_list);
 891
 892         ctx->range = calloc(NUM_RANGES, sizeof(struct r600_range));
 893         if (!ctx->range) {
 894                 r = -ENOMEM;
 895                 goto out_err;
 896         }
 897
 898         /* add blocks */
 899         r = r600_context_add_block(ctx, r600_config_reg_list,
 900                                    Elements(r600_config_reg_list), PKT3_SET_CONFIG_REG, R600_CONFIG_REG_OFFSET);
 901         if (r)
 902                 goto out_err;
 903         r = r600_context_add_block(ctx, r600_context_reg_list,
 904                                    Elements(r600_context_reg_list), PKT3_SET_CONTEXT_REG, R600_CONTEXT_REG_OFFSET);
 905         if (r)
 906                 goto out_err;
 907         r = r600_context_add_block(ctx, r600_ctl_const_list,
 908                                    Elements(r600_ctl_const_list), PKT3_SET_CTL_CONST, R600_CTL_CONST_OFFSET);
 909         if (r)
 910                 goto out_err;
 911
 912         /* PS SAMPLER BORDER */
 913         for (int j = 0, offset = 0; j < 18; j++, offset += 0x10) {
 914                 r = r600_state_sampler_border_init(ctx, offset);
 915                 if (r)
 916                         goto out_err;
 917         }
 918
 919         /* VS SAMPLER BORDER */
 920         for (int j = 0, offset = 0x200; j < 18; j++, offset += 0x10) {
 921                 r = r600_state_sampler_border_init(ctx, offset);
 922                 if (r)
 923                         goto out_err;
 924         }
 925         /* PS SAMPLER */
 926         for (int j = 0, offset = 0; j < 18; j++, offset += 0xC) {
 927                 r = r600_state_sampler_init(ctx, offset);
 928                 if (r)
 929                         goto out_err;
 930         }
 931         /* VS SAMPLER */
 932         for (int j = 0, offset = 0xD8; j < 18; j++, offset += 0xC) {
 933                 r = r600_state_sampler_init(ctx, offset);
 934                 if (r)
 935                         goto out_err;
 936         }
 937
 938         ctx->num_ps_resources = 160;
 939         ctx->num_vs_resources = 160;
 940         ctx->num_fs_resources = 16;
 941         r = r600_resource_range_init(ctx, &ctx->ps_resources, 0, 160, 0x1c);
 942         if (r)
 943                 goto out_err;
 944         r = r600_resource_range_init(ctx, &ctx->vs_resources, 0x1180, 160, 0x1c);
 945         if (r)
 946                 goto out_err;
 947         r = r600_resource_range_init(ctx, &ctx->fs_resources, 0x2300, 16, 0x1c);
 948         if (r)
 949                 goto out_err;
 950
 951         /* PS loop const */
 952         r600_loop_const_init(ctx, 0);
 953         /* VS loop const */
 954         r600_loop_const_init(ctx, 32);
 955
 956         r = r600_setup_block_table(ctx);
 957         if (r)
 958                 goto out_err;
 959
 960         /* allocate cs variables */
 961         ctx->nreloc = RADEON_CTX_MAX_PM4;
 962         ctx->reloc = calloc(ctx->nreloc, sizeof(struct r600_reloc));
 963         if (ctx->reloc == NULL) {
 964                 r = -ENOMEM;
 965                 goto out_err;
 966         }
 967         ctx->bo = calloc(ctx->nreloc, sizeof(void *));
 968         if (ctx->bo == NULL) {
 969                 r = -ENOMEM;
 970                 goto out_err;
 971         }
 972         ctx->pm4_ndwords = RADEON_CTX_MAX_PM4;
 973         ctx->pm4 = calloc(ctx->pm4_ndwords, 4);
 974         if (ctx->pm4 == NULL) {
 975                 r = -ENOMEM;
 976                 goto out_err;
 977         }
 978
 979         r600_init_cs(ctx);
 980         /* save 16dwords space for fence mecanism */
 981         ctx->pm4_ndwords -= 16;
 982
 983         LIST_INITHEAD(&ctx->fenced_bo);
 984
 985         ctx->max_db = 4;
 986
 987         r600_get_backend_mask(ctx);
 988
 989         return 0;
 990 out_err:
 991         r600_context_fini(ctx);
 992         return r;
 993 }
 994
 995 /* Flushes all surfaces */
 996 void r600_context_flush_all(struct r600_context *ctx, unsigned flush_flags)
 997 {
 998         unsigned ndwords = 5;
 999
1000         if ((ctx->pm4_dirty_cdwords + ndwords + ctx->pm4_cdwords) > ctx->pm4_ndwords) {
1001                 /* need to flush */
1002                 r600_context_flush(ctx);
1003         }
1004
1005         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_SURFACE_SYNC, 3, ctx->predicate_drawing);
1006         ctx->pm4[ctx->pm4_cdwords++] = flush_flags;     /* CP_COHER_CNTL */
1007         ctx->pm4[ctx->pm4_cdwords++] = 0xffffffff;      /* CP_COHER_SIZE */
1008         ctx->pm4[ctx->pm4_cdwords++] = 0;               /* CP_COHER_BASE */
1009         ctx->pm4[ctx->pm4_cdwords++] = 0x0000000A;      /* POLL_INTERVAL */
1010 }
1011
1012 void r600_context_bo_flush(struct r600_context *ctx, unsigned flush_flags,
1013                                 unsigned flush_mask, struct r600_bo *rbo)
1014 {
1015         struct radeon_bo *bo;
1016
1017         bo = rbo->bo;
1018         /* if bo has already been flushed */
1019         if (!(~bo->last_flush & flush_flags)) {
1020                 bo->last_flush &= flush_mask;
1021                 return;
1022         }
1023
1024         if ((ctx->radeon->family < CHIP_RV770) &&
1025             (G_0085F0_CB_ACTION_ENA(flush_flags) ||
1026              G_0085F0_DB_ACTION_ENA(flush_flags))) {
1027                 if (ctx->flags & R600_CONTEXT_CHECK_EVENT_FLUSH) {
1028                         /* the rv670 seems to fail fbo-generatemipmap unless we flush the CB1 dest base ena */
1029                         if ((bo->binding & BO_BOUND_TEXTURE) &&
1030                             (flush_flags & S_0085F0_CB_ACTION_ENA(1))) {
1031                                 if ((ctx->radeon->family == CHIP_RV670) ||
1032                                     (ctx->radeon->family == CHIP_RS780) ||
1033                                     (ctx->radeon->family == CHIP_RS880)) {
1034                                         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_SURFACE_SYNC, 3, ctx->predicate_drawing);
1035                                         ctx->pm4[ctx->pm4_cdwords++] = S_0085F0_CB1_DEST_BASE_ENA(1);     /* CP_COHER_CNTL */
1036                                         ctx->pm4[ctx->pm4_cdwords++] = 0xffffffff;      /* CP_COHER_SIZE */
1037                                         ctx->pm4[ctx->pm4_cdwords++] = 0;               /* CP_COHER_BASE */
1038                                         ctx->pm4[ctx->pm4_cdwords++] = 0x0000000A;      /* POLL_INTERVAL */
1039                                 }
1040                         }
1041
1042                         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 0, ctx->predicate_drawing);
1043                         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0);
1044                         ctx->flags &= ~R600_CONTEXT_CHECK_EVENT_FLUSH;
1045                 }
1046         } else {
1047                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_SURFACE_SYNC, 3, ctx->predicate_drawing);
1048                 ctx->pm4[ctx->pm4_cdwords++] = flush_flags;
1049                 ctx->pm4[ctx->pm4_cdwords++] = (bo->size + 255) >> 8;
1050                 ctx->pm4[ctx->pm4_cdwords++] = 0x00000000;
1051                 ctx->pm4[ctx->pm4_cdwords++] = 0x0000000A;
1052                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, ctx->predicate_drawing);
1053                 ctx->pm4[ctx->pm4_cdwords++] = bo->reloc_id;
1054         }
1055         bo->last_flush = (bo->last_flush | flush_flags) & flush_mask;
1056 }
1057
1058 void r600_context_get_reloc(struct r600_context *ctx, struct r600_bo *rbo)
1059 {
1060         struct radeon_bo *bo = rbo->bo;
1061         bo->reloc = &ctx->reloc[ctx->creloc];
1062         bo->reloc_id = ctx->creloc * sizeof(struct r600_reloc) / 4;
1063         ctx->reloc[ctx->creloc].handle = bo->handle;
1064         ctx->reloc[ctx->creloc].read_domain = rbo->domains & (RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM);
1065         ctx->reloc[ctx->creloc].write_domain = rbo->domains & (RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM);
1066         ctx->reloc[ctx->creloc].flags = 0;
1067         radeon_bo_reference(ctx->radeon, &ctx->bo[ctx->creloc], bo);
1068         rbo->fence = ctx->radeon->fence;
1069         ctx->creloc++;
1070 }
1071
1072 void r600_context_reg(struct r600_context *ctx,
1073                       unsigned offset, unsigned value,
1074                       unsigned mask)
1075 {
1076         struct r600_range *range;
1077         struct r600_block *block;
1078         unsigned id;
1079         unsigned new_val;
1080         int dirty;
1081
1082         range = &ctx->range[CTX_RANGE_ID(offset)];
1083         block = range->blocks[CTX_BLOCK_ID(offset)];
1084         id = (offset - block->start_offset) >> 2;
1085
1086         dirty = block->status & R600_BLOCK_STATUS_DIRTY;
1087
1088         new_val = block->reg[id];
1089         new_val &= ~mask;
1090         new_val |= value;
1091         if (new_val != block->reg[id]) {
1092                 dirty |= R600_BLOCK_STATUS_DIRTY;
1093                 block->reg[id] = new_val;
1094         }
1095         if (dirty)
1096                 r600_context_dirty_block(ctx, block, dirty, id);
1097 }
1098
1099 void r600_context_dirty_block(struct r600_context *ctx,
1100                               struct r600_block *block,
1101                               int dirty, int index)
1102 {
1103         if ((index + 1) > block->nreg_dirty)
1104                 block->nreg_dirty = index + 1;
1105
1106         if ((dirty != (block->status & R600_BLOCK_STATUS_DIRTY)) || !(block->status & R600_BLOCK_STATUS_ENABLED)) {
1107                 block->status |= R600_BLOCK_STATUS_DIRTY;
1108                 ctx->pm4_dirty_cdwords += block->pm4_ndwords + block->pm4_flush_ndwords;
1109                 if (!(block->status & R600_BLOCK_STATUS_ENABLED)) {
1110                         block->status |= R600_BLOCK_STATUS_ENABLED;
1111                         LIST_ADDTAIL(&block->enable_list, &ctx->enable_list);
1112                 }
1113                 LIST_ADDTAIL(&block->list,&ctx->dirty);
1114
1115                 if (block->flags & REG_FLAG_FLUSH_CHANGE) {
1116                         r600_context_ps_partial_flush(ctx);
1117                 }
1118         }
1119 }
1120
1121 void r600_context_pipe_state_set(struct r600_context *ctx, struct r600_pipe_state *state)
1122 {
1123         struct r600_block *block;
1124         unsigned new_val;
1125         int dirty;
1126         for (int i = 0; i < state->nregs; i++) {
1127                 unsigned id, reloc_id;
1128                 struct r600_pipe_reg *reg = &state->regs[i];
1129
1130                 block = reg->block;
1131                 id = reg->id;
1132
1133                 dirty = block->status & R600_BLOCK_STATUS_DIRTY;
1134
1135                 new_val = block->reg[id];
1136                 new_val &= ~reg->mask;
1137                 new_val |= reg->value;
1138                 if (new_val != block->reg[id]) {
1139                         block->reg[id] = new_val;
1140                         dirty |= R600_BLOCK_STATUS_DIRTY;
1141                 }
1142                 if (block->flags & REG_FLAG_DIRTY_ALWAYS)
1143                         dirty |= R600_BLOCK_STATUS_DIRTY;
1144                 if (block->pm4_bo_index[id]) {
1145                         /* find relocation */
1146                         reloc_id = block->pm4_bo_index[id];
1147                         r600_bo_reference(ctx->radeon, &block->reloc[reloc_id].bo, reg->bo);
1148                         reg->bo->fence = ctx->radeon->fence;
1149                         /* always force dirty for relocs for now */
1150                         dirty |= R600_BLOCK_STATUS_DIRTY;
1151                 }
1152
1153                 if (dirty)
1154                         r600_context_dirty_block(ctx, block, dirty, id);
1155         }
1156 }
1157
1158 static void r600_context_dirty_resource_block(struct r600_context *ctx,
1159                                               struct r600_block *block,
1160                                               int dirty, int index)
1161 {
1162         block->nreg_dirty = index + 1;
1163
1164         if ((dirty != (block->status & R600_BLOCK_STATUS_RESOURCE_DIRTY)) || !(block->status & R600_BLOCK_STATUS_ENABLED)) {
1165                 block->status |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1166                 ctx->pm4_dirty_cdwords += block->pm4_ndwords + block->pm4_flush_ndwords;
1167                 if (!(block->status & R600_BLOCK_STATUS_ENABLED)) {
1168                         block->status |= R600_BLOCK_STATUS_ENABLED;
1169                         LIST_ADDTAIL(&block->enable_list, &ctx->enable_list);
1170                 }
1171                 LIST_ADDTAIL(&block->list,&ctx->resource_dirty);
1172         }
1173 }
1174
1175 void r600_context_pipe_state_set_resource(struct r600_context *ctx, struct r600_pipe_resource_state *state, struct r600_block *block)
1176 {
1177         int dirty;
1178         int num_regs = ctx->radeon->chip_class >= EVERGREEN ? 8 : 7;
1179         boolean is_vertex;
1180
1181         if (state == NULL) {
1182                 block->status &= ~(R600_BLOCK_STATUS_ENABLED | R600_BLOCK_STATUS_RESOURCE_DIRTY);
1183                 if (block->reloc[1].bo)
1184                         block->reloc[1].bo->bo->binding &= ~BO_BOUND_TEXTURE;
1185
1186                 r600_bo_reference(ctx->radeon, &block->reloc[1].bo, NULL);
1187                 r600_bo_reference(ctx->radeon, &block->reloc[2].bo, NULL);
1188                 LIST_DELINIT(&block->list);
1189                 LIST_DELINIT(&block->enable_list);
1190                 return;
1191         }
1192
1193         is_vertex = ((state->val[num_regs-1] & 0xc0000000) == 0xc0000000);
1194         dirty = block->status & R600_BLOCK_STATUS_RESOURCE_DIRTY;
1195
1196         if (memcmp(block->reg, state->val, num_regs*4)) {
1197                 memcpy(block->reg, state->val, num_regs * 4);
1198                 dirty |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1199         }
1200
1201         /* if no BOs on block, force dirty */
1202         if (!block->reloc[1].bo || !block->reloc[2].bo)
1203                 dirty |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1204
1205         if (!dirty) {
1206                 if (is_vertex) {
1207                         if (block->reloc[1].bo->bo->handle != state->bo[0]->bo->handle)
1208                                 dirty |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1209                 } else {
1210                         if ((block->reloc[1].bo->bo->handle != state->bo[0]->bo->handle) ||
1211                             (block->reloc[2].bo->bo->handle != state->bo[1]->bo->handle))
1212                                 dirty |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1213                 }
1214         }
1215         if (!dirty) {
1216                 if (is_vertex)
1217                         state->bo[0]->fence = ctx->radeon->fence;
1218                 else {
1219                         state->bo[0]->fence = ctx->radeon->fence;
1220                         state->bo[1]->fence = ctx->radeon->fence;
1221                 }
1222         } else {
1223                 if (is_vertex) {
1224                         /* VERTEX RESOURCE, we preted there is 2 bo to relocate so
1225                          * we have single case btw VERTEX & TEXTURE resource
1226                          */
1227                         r600_bo_reference(ctx->radeon, &block->reloc[1].bo, state->bo[0]);
1228                         r600_bo_reference(ctx->radeon, &block->reloc[2].bo, NULL);
1229                         state->bo[0]->fence = ctx->radeon->fence;
1230                 } else {
1231                         /* TEXTURE RESOURCE */
1232                         r600_bo_reference(ctx->radeon, &block->reloc[1].bo, state->bo[0]);
1233                         r600_bo_reference(ctx->radeon, &block->reloc[2].bo, state->bo[1]);
1234                         state->bo[0]->fence = ctx->radeon->fence;
1235                         state->bo[1]->fence = ctx->radeon->fence;
1236                         state->bo[0]->bo->binding |= BO_BOUND_TEXTURE;
1237                 }
1238         }
1239         if (dirty) {
1240                 if (is_vertex)
1241                         block->status |= R600_BLOCK_STATUS_RESOURCE_VERTEX;
1242                 else
1243                         block->status &= ~R600_BLOCK_STATUS_RESOURCE_VERTEX;
1244
1245                 r600_context_dirty_resource_block(ctx, block, dirty, num_regs - 1);
1246         }
1247 }
1248
1249 void r600_context_pipe_state_set_ps_resource(struct r600_context *ctx, struct r600_pipe_resource_state *state, unsigned rid)
1250 {
1251         struct r600_block *block = ctx->ps_resources.blocks[rid];
1252
1253         r600_context_pipe_state_set_resource(ctx, state, block);
1254 }
1255
1256 void r600_context_pipe_state_set_vs_resource(struct r600_context *ctx, struct r600_pipe_resource_state *state, unsigned rid)
1257 {
1258         struct r600_block *block = ctx->vs_resources.blocks[rid];
1259
1260         r600_context_pipe_state_set_resource(ctx, state, block);
1261 }
1262
1263 void r600_context_pipe_state_set_fs_resource(struct r600_context *ctx, struct r600_pipe_resource_state *state, unsigned rid)
1264 {
1265         struct r600_block *block = ctx->fs_resources.blocks[rid];
1266
1267         r600_context_pipe_state_set_resource(ctx, state, block);
1268 }
1269
1270 static inline void r600_context_pipe_state_set_sampler(struct r600_context *ctx, struct r600_pipe_state *state, unsigned offset)
1271 {
1272         struct r600_range *range;
1273         struct r600_block *block;
1274         int i;
1275         int dirty;
1276
1277         range = &ctx->range[CTX_RANGE_ID(offset)];
1278         block = range->blocks[CTX_BLOCK_ID(offset)];
1279         if (state == NULL) {
1280                 block->status &= ~(R600_BLOCK_STATUS_ENABLED | R600_BLOCK_STATUS_DIRTY);
1281                 LIST_DELINIT(&block->list);
1282                 LIST_DELINIT(&block->enable_list);
1283                 return;
1284         }
1285         dirty = block->status & R600_BLOCK_STATUS_DIRTY;
1286         for (i = 0; i < 3; i++) {
1287                 if (block->reg[i] != state->regs[i].value) {
1288                         block->reg[i] = state->regs[i].value;
1289                         dirty |= R600_BLOCK_STATUS_DIRTY;
1290                 }
1291         }
1292
1293         if (dirty)
1294                 r600_context_dirty_block(ctx, block, dirty, 2);
1295 }
1296
1297
1298 static inline void r600_context_pipe_state_set_sampler_border(struct r600_context *ctx, struct r600_pipe_state *state, unsigned offset)
1299 {
1300         struct r600_range *range;
1301         struct r600_block *block;
1302         int i;
1303         int dirty;
1304
1305         range = &ctx->range[CTX_RANGE_ID(offset)];
1306         block = range->blocks[CTX_BLOCK_ID(offset)];
1307         if (state == NULL) {
1308                 block->status &= ~(R600_BLOCK_STATUS_ENABLED | R600_BLOCK_STATUS_DIRTY);
1309                 LIST_DELINIT(&block->list);
1310                 LIST_DELINIT(&block->enable_list);
1311                 return;
1312         }
1313         if (state->nregs <= 3) {
1314                 return;
1315         }
1316         dirty = block->status & R600_BLOCK_STATUS_DIRTY;
1317         for (i = 0; i < 4; i++) {
1318                 if (block->reg[i] != state->regs[i + 3].value) {
1319                         block->reg[i] = state->regs[i + 3].value;
1320                         dirty |= R600_BLOCK_STATUS_DIRTY;
1321                 }
1322         }
1323
1324         /* We have to flush the shaders before we change the border color
1325          * registers, or previous draw commands that haven't completed yet
1326          * will end up using the new border color. */
1327         if (dirty & R600_BLOCK_STATUS_DIRTY)
1328                 r600_context_ps_partial_flush(ctx);
1329         if (dirty)
1330                 r600_context_dirty_block(ctx, block, dirty, 3);
1331 }
1332
1333 void r600_context_pipe_state_set_ps_sampler(struct r600_context *ctx, struct r600_pipe_state *state, unsigned id)
1334 {
1335         unsigned offset;
1336
1337         offset = 0x0003C000 + id * 0xc;
1338         r600_context_pipe_state_set_sampler(ctx, state, offset);
1339         offset = 0x0000A400 + id * 0x10;
1340         r600_context_pipe_state_set_sampler_border(ctx, state, offset);
1341 }
1342
1343 void r600_context_pipe_state_set_vs_sampler(struct r600_context *ctx, struct r600_pipe_state *state, unsigned id)
1344 {
1345         unsigned offset;
1346
1347         offset = 0x0003C0D8 + id * 0xc;
1348         r600_context_pipe_state_set_sampler(ctx, state, offset);
1349         offset = 0x0000A600 + id * 0x10;
1350         r600_context_pipe_state_set_sampler_border(ctx, state, offset);
1351 }
1352
1353 struct r600_bo *r600_context_reg_bo(struct r600_context *ctx, unsigned offset)
1354 {
1355         struct r600_range *range;
1356         struct r600_block *block;
1357         unsigned id;
1358
1359         range = &ctx->range[CTX_RANGE_ID(offset)];
1360         block = range->blocks[CTX_BLOCK_ID(offset)];
1361         offset -= block->start_offset;
1362         id = block->pm4_bo_index[offset >> 2];
1363         if (block->reloc[id].bo) {
1364                 return block->reloc[id].bo;
1365         }
1366         return NULL;
1367 }
1368
1369 void r600_context_block_emit_dirty(struct r600_context *ctx, struct r600_block *block)
1370 {
1371         int id;
1372         int optional = block->nbo == 0 && !(block->flags & REG_FLAG_DIRTY_ALWAYS);
1373         int cp_dwords = block->pm4_ndwords, start_dword = 0;
1374         int new_dwords = 0;
1375         int nbo = block->nbo;
1376
1377         if (block->nreg_dirty == 0 && optional) {
1378                 goto out;
1379         }
1380
1381         if (nbo) {
1382                 ctx->flags |= R600_CONTEXT_CHECK_EVENT_FLUSH;
1383
1384                 for (int j = 0; j < block->nreg; j++) {
1385                         if (block->pm4_bo_index[j]) {
1386                                 /* find relocation */
1387                                 id = block->pm4_bo_index[j];
1388                                 r600_context_bo_reloc(ctx,
1389                                                       &block->pm4[block->reloc[id].bo_pm4_index],
1390                                                       block->reloc[id].bo);
1391                                 r600_context_bo_flush(ctx,
1392                                                       block->reloc[id].flush_flags,
1393                                                       block->reloc[id].flush_mask,
1394                                                       block->reloc[id].bo);
1395                                 nbo--;
1396                                 if (nbo == 0)
1397                                         break;
1398                         }
1399                 }
1400                 ctx->flags &= ~R600_CONTEXT_CHECK_EVENT_FLUSH;
1401         }
1402
1403         optional &= (block->nreg_dirty != block->nreg);
1404         if (optional) {
1405                 new_dwords = block->nreg_dirty;
1406                 start_dword = ctx->pm4_cdwords;
1407                 cp_dwords = new_dwords + 2;
1408         }
1409         memcpy(&ctx->pm4[ctx->pm4_cdwords], block->pm4, cp_dwords * 4);
1410         ctx->pm4_cdwords += cp_dwords;
1411
1412         if (optional) {
1413                 uint32_t newword;
1414
1415                 newword = ctx->pm4[start_dword];
1416                 newword &= PKT_COUNT_C;
1417                 newword |= PKT_COUNT_S(new_dwords);
1418                 ctx->pm4[start_dword] = newword;
1419         }
1420 out:
1421         block->status ^= R600_BLOCK_STATUS_DIRTY;
1422         block->nreg_dirty = 0;
1423         LIST_DELINIT(&block->list);
1424 }
1425
1426 void r600_context_block_resource_emit_dirty(struct r600_context *ctx, struct r600_block *block)
1427 {
1428         int id;
1429         int cp_dwords = block->pm4_ndwords;
1430         int nbo = block->nbo;
1431
1432         ctx->flags |= R600_CONTEXT_CHECK_EVENT_FLUSH;
1433
1434         if (block->status & R600_BLOCK_STATUS_RESOURCE_VERTEX) {
1435                 nbo = 1;
1436                 cp_dwords -= 2; /* don't copy the second NOP */
1437         }
1438
1439         for (int j = 0; j < nbo; j++) {
1440                 if (block->pm4_bo_index[j]) {
1441                         /* find relocation */
1442                         id = block->pm4_bo_index[j];
1443                         r600_context_bo_reloc(ctx,
1444                                               &block->pm4[block->reloc[id].bo_pm4_index],
1445                                               block->reloc[id].bo);
1446                         r600_context_bo_flush(ctx,
1447                                               block->reloc[id].flush_flags,
1448                                               block->reloc[id].flush_mask,
1449                                               block->reloc[id].bo);
1450                 }
1451         }
1452         ctx->flags &= ~R600_CONTEXT_CHECK_EVENT_FLUSH;
1453
1454         memcpy(&ctx->pm4[ctx->pm4_cdwords], block->pm4, cp_dwords * 4);
1455         ctx->pm4_cdwords += cp_dwords;
1456
1457         block->status ^= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1458         block->nreg_dirty = 0;
1459         LIST_DELINIT(&block->list);
1460 }
1461
1462 void r600_context_flush_dest_caches(struct r600_context *ctx)
1463 {
1464         struct r600_bo *cb[8];
1465         struct r600_bo *db;
1466         int i;
1467
1468         if (!(ctx->flags & R600_CONTEXT_DST_CACHES_DIRTY))
1469                 return;
1470
1471         db = r600_context_reg_bo(ctx, R_02800C_DB_DEPTH_BASE);
1472         cb[0] = r600_context_reg_bo(ctx, R_028040_CB_COLOR0_BASE);
1473         cb[1] = r600_context_reg_bo(ctx, R_028044_CB_COLOR1_BASE);
1474         cb[2] = r600_context_reg_bo(ctx, R_028048_CB_COLOR2_BASE);
1475         cb[3] = r600_context_reg_bo(ctx, R_02804C_CB_COLOR3_BASE);
1476         cb[4] = r600_context_reg_bo(ctx, R_028050_CB_COLOR4_BASE);
1477         cb[5] = r600_context_reg_bo(ctx, R_028054_CB_COLOR5_BASE);
1478         cb[6] = r600_context_reg_bo(ctx, R_028058_CB_COLOR6_BASE);
1479         cb[7] = r600_context_reg_bo(ctx, R_02805C_CB_COLOR7_BASE);
1480
1481         ctx->flags |= R600_CONTEXT_CHECK_EVENT_FLUSH;
1482         /* flush the color buffers */
1483         for (i = 0; i < 8; i++) {
1484                 if (!cb[i])
1485                         continue;
1486
1487                 r600_context_bo_flush(ctx,
1488                                         (S_0085F0_CB0_DEST_BASE_ENA(1) << i) |
1489                                         S_0085F0_CB_ACTION_ENA(1),
1490                                         0, cb[i]);
1491         }
1492         if (db) {
1493                 r600_context_bo_flush(ctx, S_0085F0_DB_ACTION_ENA(1) | S_0085F0_DB_DEST_BASE_ENA(1), 0, db);
1494         }
1495         ctx->flags &= ~R600_CONTEXT_CHECK_EVENT_FLUSH;
1496         ctx->flags &= ~R600_CONTEXT_DST_CACHES_DIRTY;
1497 }
1498
1499 void r600_context_draw(struct r600_context *ctx, const struct r600_draw *draw)
1500 {
1501         unsigned ndwords = 7;
1502         struct r600_block *dirty_block = NULL;
1503         struct r600_block *next_block;
1504         uint32_t *pm4;
1505
1506         if (draw->indices) {
1507                 ndwords = 11;
1508                 /* make sure there is enough relocation space before scheduling draw */
1509                 if (ctx->creloc >= (ctx->nreloc - 1)) {
1510                         r600_context_flush(ctx);
1511                 }
1512         }
1513
1514         /* queries need some special values */
1515         if (ctx->num_query_running) {
1516                 if (ctx->radeon->family >= CHIP_RV770) {
1517                         r600_context_reg(ctx,
1518                                         R_028D0C_DB_RENDER_CONTROL,
1519                                         S_028D0C_R700_PERFECT_ZPASS_COUNTS(1),
1520                                         S_028D0C_R700_PERFECT_ZPASS_COUNTS(1));
1521                 }
1522                 r600_context_reg(ctx,
1523                                 R_028D10_DB_RENDER_OVERRIDE,
1524                                 S_028D10_NOOP_CULL_DISABLE(1),
1525                                 S_028D10_NOOP_CULL_DISABLE(1));
1526         }
1527
1528         /* update the max dword count to make sure we have enough space
1529          * reserved for flushing the destination caches */
1530         ctx->pm4_ndwords = RADEON_CTX_MAX_PM4 - ctx->num_dest_buffers * 7 - 16;
1531
1532         if ((ctx->pm4_dirty_cdwords + ndwords + ctx->pm4_cdwords) > ctx->pm4_ndwords) {
1533                 /* need to flush */
1534                 r600_context_flush(ctx);
1535         }
1536         /* at that point everythings is flushed and ctx->pm4_cdwords = 0 */
1537         if ((ctx->pm4_dirty_cdwords + ndwords) > ctx->pm4_ndwords) {
1538                 R600_ERR("context is too big to be scheduled\n");
1539                 return;
1540         }
1541         /* enough room to copy packet */
1542         LIST_FOR_EACH_ENTRY_SAFE(dirty_block, next_block, &ctx->dirty, list) {
1543                 r600_context_block_emit_dirty(ctx, dirty_block);
1544         }
1545
1546         LIST_FOR_EACH_ENTRY_SAFE(dirty_block, next_block, &ctx->resource_dirty, list) {
1547                 r600_context_block_resource_emit_dirty(ctx, dirty_block);
1548         }
1549
1550         /* draw packet */
1551         pm4 = &ctx->pm4[ctx->pm4_cdwords];
1552
1553         pm4[0] = PKT3(PKT3_INDEX_TYPE, 0, ctx->predicate_drawing);
1554         pm4[1] = draw->vgt_index_type;
1555         pm4[2] = PKT3(PKT3_NUM_INSTANCES, 0, ctx->predicate_drawing);
1556         pm4[3] = draw->vgt_num_instances;
1557         if (draw->indices) {
1558                 pm4[4] = PKT3(PKT3_DRAW_INDEX, 3, ctx->predicate_drawing);
1559                 pm4[5] = draw->indices_bo_offset + r600_bo_offset(draw->indices);
1560                 pm4[6] = 0;
1561                 pm4[7] = draw->vgt_num_indices;
1562                 pm4[8] = draw->vgt_draw_initiator;
1563                 pm4[9] = PKT3(PKT3_NOP, 0, ctx->predicate_drawing);
1564                 pm4[10] = 0;
1565                 r600_context_bo_reloc(ctx, &pm4[10], draw->indices);
1566         } else {
1567                 pm4[4] = PKT3(PKT3_DRAW_INDEX_AUTO, 1, ctx->predicate_drawing);
1568                 pm4[5] = draw->vgt_num_indices;
1569                 pm4[6] = draw->vgt_draw_initiator;
1570         }
1571         ctx->pm4_cdwords += ndwords;
1572
1573         ctx->flags |= (R600_CONTEXT_DST_CACHES_DIRTY | R600_CONTEXT_DRAW_PENDING);
1574
1575         /* all dirty state have been scheduled in current cs */
1576         ctx->pm4_dirty_cdwords = 0;
1577 }
1578
1579 void r600_context_flush(struct r600_context *ctx)
1580 {
1581         struct drm_radeon_cs drmib = {};
1582         struct drm_radeon_cs_chunk chunks[2];
1583         uint64_t chunk_array[2];
1584         unsigned fence;
1585         int r;
1586         struct r600_block *enable_block = NULL;
1587
1588         if (ctx->pm4_cdwords == ctx->init_dwords)
1589                 return;
1590
1591         /* suspend queries */
1592         r600_context_queries_suspend(ctx);
1593
1594         if (ctx->radeon->family >= CHIP_CEDAR)
1595                 evergreen_context_flush_dest_caches(ctx);
1596         else
1597                 r600_context_flush_dest_caches(ctx);
1598
1599         /* partial flush is needed to avoid lockups on some chips with user fences */
1600         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
1601         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
1602         /* emit fence */
1603         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
1604         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
1605         ctx->pm4[ctx->pm4_cdwords++] = 0;
1606         ctx->pm4[ctx->pm4_cdwords++] = (1 << 29) | (0 << 24);
1607         ctx->pm4[ctx->pm4_cdwords++] = ctx->radeon->fence;
1608         ctx->pm4[ctx->pm4_cdwords++] = 0;
1609         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
1610         ctx->pm4[ctx->pm4_cdwords++] = 0;
1611         r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], ctx->radeon->fence_bo);
1612
1613 #if 1
1614         /* emit cs */
1615         drmib.num_chunks = 2;
1616         drmib.chunks = (uint64_t)(uintptr_t)chunk_array;
1617         chunks[0].chunk_id = RADEON_CHUNK_ID_IB;
1618         chunks[0].length_dw = ctx->pm4_cdwords;
1619         chunks[0].chunk_data = (uint64_t)(uintptr_t)ctx->pm4;
1620         chunks[1].chunk_id = RADEON_CHUNK_ID_RELOCS;
1621         chunks[1].length_dw = ctx->creloc * sizeof(struct r600_reloc) / 4;
1622         chunks[1].chunk_data = (uint64_t)(uintptr_t)ctx->reloc;
1623         chunk_array[0] = (uint64_t)(uintptr_t)&chunks[0];
1624         chunk_array[1] = (uint64_t)(uintptr_t)&chunks[1];
1625         r = drmCommandWriteRead(ctx->radeon->fd, DRM_RADEON_CS, &drmib,
1626                                 sizeof(struct drm_radeon_cs));
1627 #else
1628         *ctx->radeon->cfence = ctx->radeon->fence;
1629 #endif
1630
1631         r600_context_update_fenced_list(ctx);
1632
1633         fence = ctx->radeon->fence + 1;
1634         if (fence < ctx->radeon->fence) {
1635                 /* wrap around */
1636                 fence = 1;
1637                 r600_context_fence_wraparound(ctx, fence);
1638         }
1639         ctx->radeon->fence = fence;
1640
1641         /* restart */
1642         for (int i = 0; i < ctx->creloc; i++) {
1643                 ctx->bo[i]->reloc = NULL;
1644                 ctx->bo[i]->last_flush = 0;
1645                 radeon_bo_reference(ctx->radeon, &ctx->bo[i], NULL);
1646         }
1647         ctx->creloc = 0;
1648         ctx->pm4_dirty_cdwords = 0;
1649         ctx->pm4_cdwords = 0;
1650         ctx->flags = 0;
1651
1652         r600_init_cs(ctx);
1653
1654         /* resume queries */
1655         r600_context_queries_resume(ctx, TRUE);
1656
1657         /* set all valid group as dirty so they get reemited on
1658          * next draw command
1659          */
1660         LIST_FOR_EACH_ENTRY(enable_block, &ctx->enable_list, enable_list) {
1661                 if (!(enable_block->flags & BLOCK_FLAG_RESOURCE)) {
1662                         if(!(enable_block->status & R600_BLOCK_STATUS_DIRTY)) {
1663                                 LIST_ADDTAIL(&enable_block->list,&ctx->dirty);
1664                                 enable_block->status |= R600_BLOCK_STATUS_DIRTY;
1665                         }
1666                 } else {
1667                         if(!(enable_block->status & R600_BLOCK_STATUS_RESOURCE_DIRTY)) {
1668                                 LIST_ADDTAIL(&enable_block->list,&ctx->resource_dirty);
1669                                 enable_block->status |= R600_BLOCK_STATUS_RESOURCE_DIRTY;
1670                         }
1671                 }
1672                 ctx->pm4_dirty_cdwords += enable_block->pm4_ndwords +
1673                         enable_block->pm4_flush_ndwords;
1674                 enable_block->nreg_dirty = enable_block->nreg;
1675         }
1676 }
1677
1678 void r600_context_emit_fence(struct r600_context *ctx, struct r600_bo *fence_bo, unsigned offset, unsigned value)
1679 {
1680         unsigned ndwords = 10;
1681
1682         if (((ctx->pm4_dirty_cdwords + ndwords + ctx->pm4_cdwords) > ctx->pm4_ndwords) ||
1683             (ctx->creloc >= (ctx->nreloc - 1))) {
1684                 /* need to flush */
1685                 r600_context_flush(ctx);
1686         }
1687
1688         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
1689         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
1690         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
1691         ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
1692         ctx->pm4[ctx->pm4_cdwords++] = offset << 2;             /* ADDRESS_LO */
1693         ctx->pm4[ctx->pm4_cdwords++] = (1 << 29) | (0 << 24);   /* DATA_SEL | INT_EN | ADDRESS_HI */
1694         ctx->pm4[ctx->pm4_cdwords++] = value;                   /* DATA_LO */
1695         ctx->pm4[ctx->pm4_cdwords++] = 0;                       /* DATA_HI */
1696         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
1697         ctx->pm4[ctx->pm4_cdwords++] = 0;
1698         r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], fence_bo);
1699 }
1700
1701 void r600_context_dump_bof(struct r600_context *ctx, const char *file)
1702 {
1703         bof_t *bcs, *blob, *array, *bo, *size, *handle, *device_id, *root;
1704         unsigned i;
1705
1706         root = device_id = bcs = blob = array = bo = size = handle = NULL;
1707         root = bof_object();
1708         if (root == NULL)
1709                 goto out_err;
1710         device_id = bof_int32(ctx->radeon->device);
1711         if (device_id == NULL)
1712                 goto out_err;
1713         if (bof_object_set(root, "device_id", device_id))
1714                 goto out_err;
1715         bof_decref(device_id);
1716         device_id = NULL;
1717         /* dump relocs */
1718         blob = bof_blob(ctx->creloc * 16, ctx->reloc);
1719         if (blob == NULL)
1720                 goto out_err;
1721         if (bof_object_set(root, "reloc", blob))
1722                 goto out_err;
1723         bof_decref(blob);
1724         blob = NULL;
1725         /* dump cs */
1726         blob = bof_blob(ctx->pm4_cdwords * 4, ctx->pm4);
1727         if (blob == NULL)
1728                 goto out_err;
1729         if (bof_object_set(root, "pm4", blob))
1730                 goto out_err;
1731         bof_decref(blob);
1732         blob = NULL;
1733         /* dump bo */
1734         array = bof_array();
1735         if (array == NULL)
1736                 goto out_err;
1737         for (i = 0; i < ctx->creloc; i++) {
1738                 struct radeon_bo *rbo = ctx->bo[i];
1739                 bo = bof_object();
1740                 if (bo == NULL)
1741                         goto out_err;
1742                 size = bof_int32(rbo->size);
1743                 if (size == NULL)
1744                         goto out_err;
1745                 if (bof_object_set(bo, "size", size))
1746                         goto out_err;
1747                 bof_decref(size);
1748                 size = NULL;
1749                 handle = bof_int32(rbo->handle);
1750                 if (handle == NULL)
1751                         goto out_err;
1752                 if (bof_object_set(bo, "handle", handle))
1753                         goto out_err;
1754                 bof_decref(handle);
1755                 handle = NULL;
1756                 radeon_bo_map(ctx->radeon, rbo);
1757                 blob = bof_blob(rbo->size, rbo->data);
1758                 radeon_bo_unmap(ctx->radeon, rbo);
1759                 if (blob == NULL)
1760                         goto out_err;
1761                 if (bof_object_set(bo, "data", blob))
1762                         goto out_err;
1763                 bof_decref(blob);
1764                 blob = NULL;
1765                 if (bof_array_append(array, bo))
1766                         goto out_err;
1767                 bof_decref(bo);
1768                 bo = NULL;
1769         }
1770         if (bof_object_set(root, "bo", array))
1771                 goto out_err;
1772         bof_dump_file(root, file);
1773 out_err:
1774         bof_decref(blob);
1775         bof_decref(array);
1776         bof_decref(bo);
1777         bof_decref(size);
1778         bof_decref(handle);
1779         bof_decref(device_id);
1780         bof_decref(root);
1781 }
1782
1783 static boolean r600_query_result(struct r600_context *ctx, struct r600_query *query, boolean wait)
1784 {
1785         unsigned results_base = query->results_start;
1786         u64 start, end;
1787         u32 *results, *current_result;
1788
1789         if (wait)
1790                 results = r600_bo_map(ctx->radeon, query->buffer, PB_USAGE_CPU_READ, NULL);
1791         else
1792                 results = r600_bo_map(ctx->radeon, query->buffer, PB_USAGE_DONTBLOCK | PB_USAGE_CPU_READ, NULL);
1793         if (!results)
1794                 return FALSE;
1795
1796
1797         /* count all results across all data blocks */
1798         while (results_base != query->results_end) {
1799                 current_result = (u32*)((char*)results + results_base);
1800
1801                 start = (u64)current_result[0] | (u64)current_result[1] << 32;
1802                 end = (u64)current_result[2] | (u64)current_result[3] << 32;
1803                 if (((start & 0x8000000000000000UL) && (end & 0x8000000000000000UL))
1804                     || query->type == PIPE_QUERY_TIME_ELAPSED) {
1805                         query->result += end - start;
1806                 }
1807
1808                 results_base += 4 * 4;
1809                 if (results_base >= query->buffer_size)
1810                         results_base = 0;
1811         }
1812
1813         query->results_start = query->results_end;
1814         r600_bo_unmap(ctx->radeon, query->buffer);
1815         return TRUE;
1816 }
1817
1818 void r600_query_begin(struct r600_context *ctx, struct r600_query *query)
1819 {
1820         unsigned required_space, new_results_end;
1821
1822         /* query request needs 6/8 dwords for begin + 6/8 dwords for end */
1823         if (query->type == PIPE_QUERY_TIME_ELAPSED)
1824                 required_space = 16;
1825         else
1826                 required_space = 12;
1827
1828         if ((required_space + ctx->pm4_cdwords) > ctx->pm4_ndwords) {
1829                 /* need to flush */
1830                 r600_context_flush(ctx);
1831         }
1832
1833         if (query->type == PIPE_QUERY_OCCLUSION_COUNTER) {
1834                 /* Count queries emitted without flushes, and flush if more than
1835                  * half of buffer used, to avoid overwriting results which may be
1836                  * still in use. */
1837                 if (query->state & R600_QUERY_STATE_FLUSHED) {
1838                         query->queries_emitted = 1;
1839                 } else {
1840                         if (++query->queries_emitted > query->buffer_size / query->result_size / 2)
1841                                 r600_context_flush(ctx);
1842                 }
1843         }
1844
1845         new_results_end = query->results_end + query->result_size;
1846         if (new_results_end >= query->buffer_size)
1847                 new_results_end = 0;
1848
1849         /* collect current results if query buffer is full */
1850         if (new_results_end == query->results_start) {
1851                 if (!(query->state & R600_QUERY_STATE_FLUSHED))
1852                         r600_context_flush(ctx);
1853                 r600_query_result(ctx, query, TRUE);
1854         }
1855
1856         if (query->type == PIPE_QUERY_OCCLUSION_COUNTER) {
1857                 u32 *results;
1858                 int i;
1859
1860                 results = r600_bo_map(ctx->radeon, query->buffer, PB_USAGE_CPU_WRITE, NULL);
1861                 if (results) {
1862                         results = (u32*)((char*)results + query->results_end);
1863                         memset(results, 0, query->result_size);
1864
1865                         /* Set top bits for unused backends */
1866                         for (i = 0; i < ctx->max_db; i++) {
1867                                 if (!(ctx->backend_mask & (1<<i))) {
1868                                         results[(i * 4)+1] = 0x80000000;
1869                                         results[(i * 4)+3] = 0x80000000;
1870                                 }
1871                         }
1872                         r600_bo_unmap(ctx->radeon, query->buffer);
1873                 }
1874         }
1875
1876         /* emit begin query */
1877         if (query->type == PIPE_QUERY_TIME_ELAPSED) {
1878                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
1879                 ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
1880                 ctx->pm4[ctx->pm4_cdwords++] = query->results_end + r600_bo_offset(query->buffer);
1881                 ctx->pm4[ctx->pm4_cdwords++] = (3 << 29);
1882                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1883                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1884         } else {
1885                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
1886                 ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
1887                 ctx->pm4[ctx->pm4_cdwords++] = query->results_end + r600_bo_offset(query->buffer);
1888                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1889         }
1890         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
1891         ctx->pm4[ctx->pm4_cdwords++] = 0;
1892         r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], query->buffer);
1893
1894         query->state |= R600_QUERY_STATE_STARTED;
1895         query->state ^= R600_QUERY_STATE_ENDED;
1896         ctx->num_query_running++;
1897 }
1898
1899 void r600_query_end(struct r600_context *ctx, struct r600_query *query)
1900 {
1901         /* emit end query */
1902         if (query->type == PIPE_QUERY_TIME_ELAPSED) {
1903                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
1904                 ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
1905                 ctx->pm4[ctx->pm4_cdwords++] = query->results_end + 8 + r600_bo_offset(query->buffer);
1906                 ctx->pm4[ctx->pm4_cdwords++] = (3 << 29);
1907                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1908                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1909         } else {
1910                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
1911                 ctx->pm4[ctx->pm4_cdwords++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
1912                 ctx->pm4[ctx->pm4_cdwords++] = query->results_end + 8 + r600_bo_offset(query->buffer);
1913                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1914         }
1915         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
1916         ctx->pm4[ctx->pm4_cdwords++] = 0;
1917         r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], query->buffer);
1918
1919         query->results_end += query->result_size;
1920         if (query->results_end >= query->buffer_size)
1921                 query->results_end = 0;
1922
1923         query->state ^= R600_QUERY_STATE_STARTED;
1924         query->state |= R600_QUERY_STATE_ENDED;
1925         query->state &= ~R600_QUERY_STATE_FLUSHED;
1926
1927         ctx->num_query_running--;
1928 }
1929
1930 void r600_query_predication(struct r600_context *ctx, struct r600_query *query, int operation,
1931                             int flag_wait)
1932 {
1933         if (operation == PREDICATION_OP_CLEAR) {
1934                 if (ctx->pm4_cdwords + 3 > ctx->pm4_ndwords)
1935                         r600_context_flush(ctx);
1936
1937                 ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_SET_PREDICATION, 1, 0);
1938                 ctx->pm4[ctx->pm4_cdwords++] = 0;
1939                 ctx->pm4[ctx->pm4_cdwords++] = PRED_OP(PREDICATION_OP_CLEAR);
1940         } else {
1941                 unsigned results_base = query->results_start;
1942                 unsigned count;
1943                 u32 op;
1944
1945                 /* find count of the query data blocks */
1946                 count = query->buffer_size + query->results_end - query->results_start;
1947                 if (count >= query->buffer_size) count-=query->buffer_size;
1948                 count /= query->result_size;
1949
1950                 if (ctx->pm4_cdwords + 5 * count > ctx->pm4_ndwords)
1951                         r600_context_flush(ctx);
1952
1953                 op = PRED_OP(operation) | PREDICATION_DRAW_VISIBLE |
1954                                 (flag_wait ? PREDICATION_HINT_WAIT : PREDICATION_HINT_NOWAIT_DRAW);
1955
1956                 /* emit predicate packets for all data blocks */
1957                 while (results_base != query->results_end) {
1958                         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_SET_PREDICATION, 1, 0);
1959                         ctx->pm4[ctx->pm4_cdwords++] = results_base + r600_bo_offset(query->buffer);
1960                         ctx->pm4[ctx->pm4_cdwords++] = op;
1961                         ctx->pm4[ctx->pm4_cdwords++] = PKT3(PKT3_NOP, 0, 0);
1962                         ctx->pm4[ctx->pm4_cdwords++] = 0;
1963                         r600_context_bo_reloc(ctx, &ctx->pm4[ctx->pm4_cdwords - 1], query->buffer);
1964                         results_base += query->result_size;
1965                         if (results_base >= query->buffer_size)
1966                                 results_base = 0;
1967                         /* set CONTINUE bit for all packets except the first */
1968                         op |= PREDICATION_CONTINUE;
1969                 }
1970         }
1971 }
1972
1973 struct r600_query *r600_context_query_create(struct r600_context *ctx, unsigned query_type)
1974 {
1975         struct r600_query *query;
1976
1977         if (query_type != PIPE_QUERY_OCCLUSION_COUNTER && query_type != PIPE_QUERY_TIME_ELAPSED)
1978                 return NULL;
1979
1980         query = calloc(1, sizeof(struct r600_query));
1981         if (query == NULL)
1982                 return NULL;
1983
1984         query->type = query_type;
1985         query->buffer_size = 4096;
1986
1987         if (query_type == PIPE_QUERY_OCCLUSION_COUNTER)
1988                 query->result_size = 4 * 4 * ctx->max_db;
1989         else
1990                 query->result_size = 4 * 4;
1991
1992         /* adjust buffer size to simplify offsets wrapping math */
1993         query->buffer_size -= query->buffer_size % query->result_size;
1994
1995         /* As of GL4, query buffers are normally read by the CPU after
1996          * being written by the gpu, hence staging is probably a good
1997          * usage pattern.
1998          */
1999         query->buffer = r600_bo(ctx->radeon, query->buffer_size, 1, 0,
2000                                 PIPE_USAGE_STAGING);
2001         if (!query->buffer) {
2002                 free(query);
2003                 return NULL;
2004         }
2005
2006         LIST_ADDTAIL(&query->list, &ctx->query_list);
2007
2008         return query;
2009 }
2010
2011 void r600_context_query_destroy(struct r600_context *ctx, struct r600_query *query)
2012 {
2013         r600_bo_reference(ctx->radeon, &query->buffer, NULL);
2014         LIST_DELINIT(&query->list);
2015         free(query);
2016 }
2017
2018 boolean r600_context_query_result(struct r600_context *ctx,
2019                                 struct r600_query *query,
2020                                 boolean wait, void *vresult)
2021 {
2022         uint64_t *result = (uint64_t*)vresult;
2023
2024         if (!(query->state & R600_QUERY_STATE_FLUSHED)) {
2025                 r600_context_flush(ctx);
2026         }
2027         if (!r600_query_result(ctx, query, wait))
2028                 return FALSE;
2029         if (query->type == PIPE_QUERY_TIME_ELAPSED)
2030                 *result = (1000000*query->result)/r600_get_clock_crystal_freq(ctx->radeon);
2031         else
2032                 *result = query->result;
2033         query->result = 0;
2034         return TRUE;
2035 }
2036
2037 void r600_context_queries_suspend(struct r600_context *ctx)
2038 {
2039         struct r600_query *query;
2040
2041         LIST_FOR_EACH_ENTRY(query, &ctx->query_list, list) {
2042                 if (query->state & R600_QUERY_STATE_STARTED) {
2043                         r600_query_end(ctx, query);
2044                         query->state |= R600_QUERY_STATE_SUSPENDED;
2045                 }
2046         }
2047 }
2048
2049 void r600_context_queries_resume(struct r600_context *ctx, boolean flushed)
2050 {
2051         struct r600_query *query;
2052
2053         LIST_FOR_EACH_ENTRY(query, &ctx->query_list, list) {
2054                 if (flushed)
2055                         query->state |= R600_QUERY_STATE_FLUSHED;
2056
2057                 if (query->state & R600_QUERY_STATE_SUSPENDED) {
2058                         r600_query_begin(ctx, query);
2059                         query->state ^= R600_QUERY_STATE_SUSPENDED;
2060                 }
2061         }
2062 }