drivers/gpu/drm/i915/gt/selftest_lrc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/prime_numbers.h>
   7
   8 #include "gem/i915_gem_internal.h"
   9
  10 #include "i915_selftest.h"
  11 #include "intel_engine_heartbeat.h"
  12 #include "intel_engine_pm.h"
  13 #include "intel_reset.h"
  14 #include "intel_ring.h"
  15 #include "selftest_engine_heartbeat.h"
  16 #include "selftests/i915_random.h"
  17 #include "selftests/igt_flush_test.h"
  18 #include "selftests/igt_live_test.h"
  19 #include "selftests/igt_spinner.h"
  20 #include "selftests/lib_sw_fence.h"
  21 #include "shmem_utils.h"
  22
  23 #include "gem/selftests/igt_gem_utils.h"
  24 #include "gem/selftests/mock_context.h"
  25
  26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
  27 #define NUM_GPR 16
  28 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
  29
  30 static struct i915_vma *create_scratch(struct intel_gt *gt)
  31 {
  32         return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
  33 }
  34
  35 static bool is_active(struct i915_request *rq)
  36 {
  37         if (i915_request_is_active(rq))
  38                 return true;
  39
  40         if (i915_request_on_hold(rq))
  41                 return true;
  42
  43         if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
  44                 return true;
  45
  46         return false;
  47 }
  48
  49 static int wait_for_submit(struct intel_engine_cs *engine,
  50                            struct i915_request *rq,
  51                            unsigned long timeout)
  52 {
  53         /* Ignore our own attempts to suppress excess tasklets */
  54         tasklet_hi_schedule(&engine->sched_engine->tasklet);
  55
  56         timeout += jiffies;
  57         do {
  58                 bool done = time_after(jiffies, timeout);
  59
  60                 if (i915_request_completed(rq)) /* that was quick! */
  61                         return 0;
  62
  63                 /* Wait until the HW has acknowleged the submission (or err) */
  64                 intel_engine_flush_submission(engine);
  65                 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
  66                         return 0;
  67
  68                 if (done)
  69                         return -ETIME;
  70
  71                 cond_resched();
  72         } while (1);
  73 }
  74
  75 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
  76 {
  77         const u32 offset =
  78                 i915_ggtt_offset(ce->engine->status_page.vma) +
  79                 offset_in_page(slot);
  80         struct i915_request *rq;
  81         u32 *cs;
  82
  83         rq = intel_context_create_request(ce);
  84         if (IS_ERR(rq))
  85                 return PTR_ERR(rq);
  86
  87         cs = intel_ring_begin(rq, 4);
  88         if (IS_ERR(cs)) {
  89                 i915_request_add(rq);
  90                 return PTR_ERR(cs);
  91         }
  92
  93         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  94         *cs++ = offset;
  95         *cs++ = 0;
  96         *cs++ = 1;
  97
  98         intel_ring_advance(rq, cs);
  99
 100         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 101         i915_request_add(rq);
 102         return 0;
 103 }
 104
 105 static int context_flush(struct intel_context *ce, long timeout)
 106 {
 107         struct i915_request *rq;
 108         struct dma_fence *fence;
 109         int err = 0;
 110
 111         rq = intel_engine_create_kernel_request(ce->engine);
 112         if (IS_ERR(rq))
 113                 return PTR_ERR(rq);
 114
 115         fence = i915_active_fence_get(&ce->timeline->last_request);
 116         if (fence) {
 117                 i915_request_await_dma_fence(rq, fence);
 118                 dma_fence_put(fence);
 119         }
 120
 121         rq = i915_request_get(rq);
 122         i915_request_add(rq);
 123         if (i915_request_wait(rq, 0, timeout) < 0)
 124                 err = -ETIME;
 125         i915_request_put(rq);
 126
 127         rmb(); /* We know the request is written, make sure all state is too! */
 128         return err;
 129 }
 130
 131 static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)
 132 {
 133         if ((lri & MI_LRI_LRM_CS_MMIO) == 0)
 134                 return ~0u;
 135
 136         if (GRAPHICS_VER(engine->i915) < 12)
 137                 return 0xfff;
 138
 139         switch (engine->class) {
 140         default:
 141         case RENDER_CLASS:
 142         case COMPUTE_CLASS:
 143                 return 0x07ff;
 144         case COPY_ENGINE_CLASS:
 145                 return 0x0fff;
 146         case VIDEO_DECODE_CLASS:
 147         case VIDEO_ENHANCEMENT_CLASS:
 148                 return 0x3fff;
 149         }
 150 }
 151
 152 static int live_lrc_layout(void *arg)
 153 {
 154         struct intel_gt *gt = arg;
 155         struct intel_engine_cs *engine;
 156         enum intel_engine_id id;
 157         u32 *lrc;
 158         int err;
 159
 160         /*
 161          * Check the registers offsets we use to create the initial reg state
 162          * match the layout saved by HW.
 163          */
 164
 165         lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
 166         if (!lrc)
 167                 return -ENOMEM;
 168         GEM_BUG_ON(offset_in_page(lrc));
 169
 170         err = 0;
 171         for_each_engine(engine, gt, id) {
 172                 u32 *hw;
 173                 int dw;
 174
 175                 if (!engine->default_state)
 176                         continue;
 177
 178                 hw = shmem_pin_map(engine->default_state);
 179                 if (!hw) {
 180                         err = -ENOMEM;
 181                         break;
 182                 }
 183                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 184
 185                 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
 186                                 engine->kernel_context, engine, true);
 187
 188                 dw = 0;
 189                 do {
 190                         u32 lri = READ_ONCE(hw[dw]);
 191                         u32 lri_mask;
 192
 193                         if (lri == 0) {
 194                                 dw++;
 195                                 continue;
 196                         }
 197
 198                         if (lrc[dw] == 0) {
 199                                 pr_debug("%s: skipped instruction %x at dword %d\n",
 200                                          engine->name, lri, dw);
 201                                 dw++;
 202                                 continue;
 203                         }
 204
 205                         if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 206                                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 207                                        engine->name, dw, lri);
 208                                 err = -EINVAL;
 209                                 break;
 210                         }
 211
 212                         if (lrc[dw] != lri) {
 213                                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
 214                                        engine->name, dw, lri, lrc[dw]);
 215                                 err = -EINVAL;
 216                                 break;
 217                         }
 218
 219                         /*
 220                          * When bit 19 of MI_LOAD_REGISTER_IMM instruction
 221                          * opcode is set on Gen12+ devices, HW does not
 222                          * care about certain register address offsets, and
 223                          * instead check the following for valid address
 224                          * ranges on specific engines:
 225                          * RCS && CCS: BITS(0 - 10)
 226                          * BCS: BITS(0 - 11)
 227                          * VECS && VCS: BITS(0 - 13)
 228                          */
 229                         lri_mask = get_lri_mask(engine, lri);
 230
 231                         lri &= 0x7f;
 232                         lri++;
 233                         dw++;
 234
 235                         while (lri) {
 236                                 u32 offset = READ_ONCE(hw[dw]);
 237
 238                                 if ((offset ^ lrc[dw]) & lri_mask) {
 239                                         pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
 240                                                engine->name, dw, offset, lrc[dw]);
 241                                         err = -EINVAL;
 242                                         break;
 243                                 }
 244
 245                                 /*
 246                                  * Skip over the actual register value as we
 247                                  * expect that to differ.
 248                                  */
 249                                 dw += 2;
 250                                 lri -= 2;
 251                         }
 252                 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 253
 254                 if (err) {
 255                         pr_info("%s: HW register image:\n", engine->name);
 256                         igt_hexdump(hw, PAGE_SIZE);
 257
 258                         pr_info("%s: SW register image:\n", engine->name);
 259                         igt_hexdump(lrc, PAGE_SIZE);
 260                 }
 261
 262                 shmem_unpin_map(engine->default_state, hw);
 263                 if (err)
 264                         break;
 265         }
 266
 267         free_page((unsigned long)lrc);
 268         return err;
 269 }
 270
 271 static int find_offset(const u32 *lri, u32 offset)
 272 {
 273         int i;
 274
 275         for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
 276                 if (lri[i] == offset)
 277                         return i;
 278
 279         return -1;
 280 }
 281
 282 static int live_lrc_fixed(void *arg)
 283 {
 284         struct intel_gt *gt = arg;
 285         struct intel_engine_cs *engine;
 286         enum intel_engine_id id;
 287         int err = 0;
 288
 289         /*
 290          * Check the assumed register offsets match the actual locations in
 291          * the context image.
 292          */
 293
 294         for_each_engine(engine, gt, id) {
 295                 const struct {
 296                         u32 reg;
 297                         u32 offset;
 298                         const char *name;
 299                 } tbl[] = {
 300                         {
 301                                 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
 302                                 CTX_RING_START - 1,
 303                                 "RING_START"
 304                         },
 305                         {
 306                                 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
 307                                 CTX_RING_CTL - 1,
 308                                 "RING_CTL"
 309                         },
 310                         {
 311                                 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
 312                                 CTX_RING_HEAD - 1,
 313                                 "RING_HEAD"
 314                         },
 315                         {
 316                                 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
 317                                 CTX_RING_TAIL - 1,
 318                                 "RING_TAIL"
 319                         },
 320                         {
 321                                 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
 322                                 lrc_ring_mi_mode(engine),
 323                                 "RING_MI_MODE"
 324                         },
 325                         {
 326                                 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
 327                                 CTX_BB_STATE - 1,
 328                                 "BB_STATE"
 329                         },
 330                         {
 331                                 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
 332                                 lrc_ring_wa_bb_per_ctx(engine),
 333                                 "RING_BB_PER_CTX_PTR"
 334                         },
 335                         {
 336                                 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
 337                                 lrc_ring_indirect_ptr(engine),
 338                                 "RING_INDIRECT_CTX_PTR"
 339                         },
 340                         {
 341                                 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
 342                                 lrc_ring_indirect_offset(engine),
 343                                 "RING_INDIRECT_CTX_OFFSET"
 344                         },
 345                         {
 346                                 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
 347                                 CTX_TIMESTAMP - 1,
 348                                 "RING_CTX_TIMESTAMP"
 349                         },
 350                         {
 351                                 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
 352                                 lrc_ring_gpr0(engine),
 353                                 "RING_CS_GPR0"
 354                         },
 355                         {
 356                                 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
 357                                 lrc_ring_cmd_buf_cctl(engine),
 358                                 "RING_CMD_BUF_CCTL"
 359                         },
 360                         { },
 361                 }, *t;
 362                 u32 *hw;
 363
 364                 if (!engine->default_state)
 365                         continue;
 366
 367                 hw = shmem_pin_map(engine->default_state);
 368                 if (!hw) {
 369                         err = -ENOMEM;
 370                         break;
 371                 }
 372                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 373
 374                 for (t = tbl; t->name; t++) {
 375                         int dw = find_offset(hw, t->reg);
 376
 377                         if (dw != t->offset) {
 378                                 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
 379                                        engine->name,
 380                                        t->name,
 381                                        t->reg,
 382                                        dw,
 383                                        t->offset);
 384                                 err = -EINVAL;
 385                         }
 386                 }
 387
 388                 shmem_unpin_map(engine->default_state, hw);
 389         }
 390
 391         return err;
 392 }
 393
 394 static int __live_lrc_state(struct intel_engine_cs *engine,
 395                             struct i915_vma *scratch)
 396 {
 397         struct intel_context *ce;
 398         struct i915_request *rq;
 399         struct i915_gem_ww_ctx ww;
 400         enum {
 401                 RING_START_IDX = 0,
 402                 RING_TAIL_IDX,
 403                 MAX_IDX
 404         };
 405         u32 expected[MAX_IDX];
 406         u32 *cs;
 407         int err;
 408         int n;
 409
 410         ce = intel_context_create(engine);
 411         if (IS_ERR(ce))
 412                 return PTR_ERR(ce);
 413
 414         i915_gem_ww_ctx_init(&ww, false);
 415 retry:
 416         err = i915_gem_object_lock(scratch->obj, &ww);
 417         if (!err)
 418                 err = intel_context_pin_ww(ce, &ww);
 419         if (err)
 420                 goto err_put;
 421
 422         rq = i915_request_create(ce);
 423         if (IS_ERR(rq)) {
 424                 err = PTR_ERR(rq);
 425                 goto err_unpin;
 426         }
 427
 428         cs = intel_ring_begin(rq, 4 * MAX_IDX);
 429         if (IS_ERR(cs)) {
 430                 err = PTR_ERR(cs);
 431                 i915_request_add(rq);
 432                 goto err_unpin;
 433         }
 434
 435         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 436         *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
 437         *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
 438         *cs++ = 0;
 439
 440         expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
 441
 442         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 443         *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
 444         *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
 445         *cs++ = 0;
 446
 447         err = i915_request_await_object(rq, scratch->obj, true);
 448         if (!err)
 449                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 450
 451         i915_request_get(rq);
 452         i915_request_add(rq);
 453         if (err)
 454                 goto err_rq;
 455
 456         intel_engine_flush_submission(engine);
 457         expected[RING_TAIL_IDX] = ce->ring->tail;
 458
 459         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 460                 err = -ETIME;
 461                 goto err_rq;
 462         }
 463
 464         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 465         if (IS_ERR(cs)) {
 466                 err = PTR_ERR(cs);
 467                 goto err_rq;
 468         }
 469
 470         for (n = 0; n < MAX_IDX; n++) {
 471                 if (cs[n] != expected[n]) {
 472                         pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
 473                                engine->name, n, cs[n], expected[n]);
 474                         err = -EINVAL;
 475                         break;
 476                 }
 477         }
 478
 479         i915_gem_object_unpin_map(scratch->obj);
 480
 481 err_rq:
 482         i915_request_put(rq);
 483 err_unpin:
 484         intel_context_unpin(ce);
 485 err_put:
 486         if (err == -EDEADLK) {
 487                 err = i915_gem_ww_ctx_backoff(&ww);
 488                 if (!err)
 489                         goto retry;
 490         }
 491         i915_gem_ww_ctx_fini(&ww);
 492         intel_context_put(ce);
 493         return err;
 494 }
 495
 496 static int live_lrc_state(void *arg)
 497 {
 498         struct intel_gt *gt = arg;
 499         struct intel_engine_cs *engine;
 500         struct i915_vma *scratch;
 501         enum intel_engine_id id;
 502         int err = 0;
 503
 504         /*
 505          * Check the live register state matches what we expect for this
 506          * intel_context.
 507          */
 508
 509         scratch = create_scratch(gt);
 510         if (IS_ERR(scratch))
 511                 return PTR_ERR(scratch);
 512
 513         for_each_engine(engine, gt, id) {
 514                 err = __live_lrc_state(engine, scratch);
 515                 if (err)
 516                         break;
 517         }
 518
 519         if (igt_flush_test(gt->i915))
 520                 err = -EIO;
 521
 522         i915_vma_unpin_and_release(&scratch, 0);
 523         return err;
 524 }
 525
 526 static int gpr_make_dirty(struct intel_context *ce)
 527 {
 528         struct i915_request *rq;
 529         u32 *cs;
 530         int n;
 531
 532         rq = intel_context_create_request(ce);
 533         if (IS_ERR(rq))
 534                 return PTR_ERR(rq);
 535
 536         cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
 537         if (IS_ERR(cs)) {
 538                 i915_request_add(rq);
 539                 return PTR_ERR(cs);
 540         }
 541
 542         *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
 543         for (n = 0; n < NUM_GPR_DW; n++) {
 544                 *cs++ = CS_GPR(ce->engine, n);
 545                 *cs++ = STACK_MAGIC;
 546         }
 547         *cs++ = MI_NOOP;
 548
 549         intel_ring_advance(rq, cs);
 550
 551         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 552         i915_request_add(rq);
 553
 554         return 0;
 555 }
 556
 557 static struct i915_request *
 558 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
 559 {
 560         const u32 offset =
 561                 i915_ggtt_offset(ce->engine->status_page.vma) +
 562                 offset_in_page(slot);
 563         struct i915_request *rq;
 564         u32 *cs;
 565         int err;
 566         int n;
 567
 568         rq = intel_context_create_request(ce);
 569         if (IS_ERR(rq))
 570                 return rq;
 571
 572         cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
 573         if (IS_ERR(cs)) {
 574                 i915_request_add(rq);
 575                 return ERR_CAST(cs);
 576         }
 577
 578         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 579         *cs++ = MI_NOOP;
 580
 581         *cs++ = MI_SEMAPHORE_WAIT |
 582                 MI_SEMAPHORE_GLOBAL_GTT |
 583                 MI_SEMAPHORE_POLL |
 584                 MI_SEMAPHORE_SAD_NEQ_SDD;
 585         *cs++ = 0;
 586         *cs++ = offset;
 587         *cs++ = 0;
 588
 589         for (n = 0; n < NUM_GPR_DW; n++) {
 590                 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 591                 *cs++ = CS_GPR(ce->engine, n);
 592                 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
 593                 *cs++ = 0;
 594         }
 595
 596         i915_vma_lock(scratch);
 597         err = i915_request_await_object(rq, scratch->obj, true);
 598         if (!err)
 599                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 600         i915_vma_unlock(scratch);
 601
 602         i915_request_get(rq);
 603         i915_request_add(rq);
 604         if (err) {
 605                 i915_request_put(rq);
 606                 rq = ERR_PTR(err);
 607         }
 608
 609         return rq;
 610 }
 611
 612 static int __live_lrc_gpr(struct intel_engine_cs *engine,
 613                           struct i915_vma *scratch,
 614                           bool preempt)
 615 {
 616         u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
 617         struct intel_context *ce;
 618         struct i915_request *rq;
 619         u32 *cs;
 620         int err;
 621         int n;
 622
 623         if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
 624                 return 0; /* GPR only on rcs0 for gen8 */
 625
 626         err = gpr_make_dirty(engine->kernel_context);
 627         if (err)
 628                 return err;
 629
 630         ce = intel_context_create(engine);
 631         if (IS_ERR(ce))
 632                 return PTR_ERR(ce);
 633
 634         rq = __gpr_read(ce, scratch, slot);
 635         if (IS_ERR(rq)) {
 636                 err = PTR_ERR(rq);
 637                 goto err_put;
 638         }
 639
 640         err = wait_for_submit(engine, rq, HZ / 2);
 641         if (err)
 642                 goto err_rq;
 643
 644         if (preempt) {
 645                 err = gpr_make_dirty(engine->kernel_context);
 646                 if (err)
 647                         goto err_rq;
 648
 649                 err = emit_semaphore_signal(engine->kernel_context, slot);
 650                 if (err)
 651                         goto err_rq;
 652
 653                 err = wait_for_submit(engine, rq, HZ / 2);
 654                 if (err)
 655                         goto err_rq;
 656         } else {
 657                 slot[0] = 1;
 658                 wmb();
 659         }
 660
 661         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 662                 err = -ETIME;
 663                 goto err_rq;
 664         }
 665
 666         cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
 667         if (IS_ERR(cs)) {
 668                 err = PTR_ERR(cs);
 669                 goto err_rq;
 670         }
 671
 672         for (n = 0; n < NUM_GPR_DW; n++) {
 673                 if (cs[n]) {
 674                         pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
 675                                engine->name,
 676                                n / 2, n & 1 ? "udw" : "ldw",
 677                                cs[n]);
 678                         err = -EINVAL;
 679                         break;
 680                 }
 681         }
 682
 683         i915_gem_object_unpin_map(scratch->obj);
 684
 685 err_rq:
 686         memset32(&slot[0], -1, 4);
 687         wmb();
 688         i915_request_put(rq);
 689 err_put:
 690         intel_context_put(ce);
 691         return err;
 692 }
 693
 694 static int live_lrc_gpr(void *arg)
 695 {
 696         struct intel_gt *gt = arg;
 697         struct intel_engine_cs *engine;
 698         struct i915_vma *scratch;
 699         enum intel_engine_id id;
 700         int err = 0;
 701
 702         /*
 703          * Check that GPR registers are cleared in new contexts as we need
 704          * to avoid leaking any information from previous contexts.
 705          */
 706
 707         scratch = create_scratch(gt);
 708         if (IS_ERR(scratch))
 709                 return PTR_ERR(scratch);
 710
 711         for_each_engine(engine, gt, id) {
 712                 st_engine_heartbeat_disable(engine);
 713
 714                 err = __live_lrc_gpr(engine, scratch, false);
 715                 if (err)
 716                         goto err;
 717
 718                 err = __live_lrc_gpr(engine, scratch, true);
 719                 if (err)
 720                         goto err;
 721
 722 err:
 723                 st_engine_heartbeat_enable(engine);
 724                 if (igt_flush_test(gt->i915))
 725                         err = -EIO;
 726                 if (err)
 727                         break;
 728         }
 729
 730         i915_vma_unpin_and_release(&scratch, 0);
 731         return err;
 732 }
 733
 734 static struct i915_request *
 735 create_timestamp(struct intel_context *ce, void *slot, int idx)
 736 {
 737         const u32 offset =
 738                 i915_ggtt_offset(ce->engine->status_page.vma) +
 739                 offset_in_page(slot);
 740         struct i915_request *rq;
 741         u32 *cs;
 742         int err;
 743
 744         rq = intel_context_create_request(ce);
 745         if (IS_ERR(rq))
 746                 return rq;
 747
 748         cs = intel_ring_begin(rq, 10);
 749         if (IS_ERR(cs)) {
 750                 err = PTR_ERR(cs);
 751                 goto err;
 752         }
 753
 754         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 755         *cs++ = MI_NOOP;
 756
 757         *cs++ = MI_SEMAPHORE_WAIT |
 758                 MI_SEMAPHORE_GLOBAL_GTT |
 759                 MI_SEMAPHORE_POLL |
 760                 MI_SEMAPHORE_SAD_NEQ_SDD;
 761         *cs++ = 0;
 762         *cs++ = offset;
 763         *cs++ = 0;
 764
 765         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 766         *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
 767         *cs++ = offset + idx * sizeof(u32);
 768         *cs++ = 0;
 769
 770         intel_ring_advance(rq, cs);
 771
 772         err = 0;
 773 err:
 774         i915_request_get(rq);
 775         i915_request_add(rq);
 776         if (err) {
 777                 i915_request_put(rq);
 778                 return ERR_PTR(err);
 779         }
 780
 781         return rq;
 782 }
 783
 784 struct lrc_timestamp {
 785         struct intel_engine_cs *engine;
 786         struct intel_context *ce[2];
 787         u32 poison;
 788 };
 789
 790 static bool timestamp_advanced(u32 start, u32 end)
 791 {
 792         return (s32)(end - start) > 0;
 793 }
 794
 795 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
 796 {
 797         u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
 798         struct i915_request *rq;
 799         u32 timestamp;
 800         int err = 0;
 801
 802         arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
 803         rq = create_timestamp(arg->ce[0], slot, 1);
 804         if (IS_ERR(rq))
 805                 return PTR_ERR(rq);
 806
 807         err = wait_for_submit(rq->engine, rq, HZ / 2);
 808         if (err)
 809                 goto err;
 810
 811         if (preempt) {
 812                 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
 813                 err = emit_semaphore_signal(arg->ce[1], slot);
 814                 if (err)
 815                         goto err;
 816         } else {
 817                 slot[0] = 1;
 818                 wmb();
 819         }
 820
 821         /* And wait for switch to kernel (to save our context to memory) */
 822         err = context_flush(arg->ce[0], HZ / 2);
 823         if (err)
 824                 goto err;
 825
 826         if (!timestamp_advanced(arg->poison, slot[1])) {
 827                 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
 828                        arg->engine->name, preempt ? "preempt" : "simple",
 829                        arg->poison, slot[1]);
 830                 err = -EINVAL;
 831         }
 832
 833         timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
 834         if (!timestamp_advanced(slot[1], timestamp)) {
 835                 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
 836                        arg->engine->name, preempt ? "preempt" : "simple",
 837                        slot[1], timestamp);
 838                 err = -EINVAL;
 839         }
 840
 841 err:
 842         memset32(slot, -1, 4);
 843         i915_request_put(rq);
 844         return err;
 845 }
 846
 847 static int live_lrc_timestamp(void *arg)
 848 {
 849         struct lrc_timestamp data = {};
 850         struct intel_gt *gt = arg;
 851         enum intel_engine_id id;
 852         const u32 poison[] = {
 853                 0,
 854                 S32_MAX,
 855                 (u32)S32_MAX + 1,
 856                 U32_MAX,
 857         };
 858
 859         /*
 860          * We want to verify that the timestamp is saved and restore across
 861          * context switches and is monotonic.
 862          *
 863          * So we do this with a little bit of LRC poisoning to check various
 864          * boundary conditions, and see what happens if we preempt the context
 865          * with a second request (carrying more poison into the timestamp).
 866          */
 867
 868         for_each_engine(data.engine, gt, id) {
 869                 int i, err = 0;
 870
 871                 st_engine_heartbeat_disable(data.engine);
 872
 873                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 874                         struct intel_context *tmp;
 875
 876                         tmp = intel_context_create(data.engine);
 877                         if (IS_ERR(tmp)) {
 878                                 err = PTR_ERR(tmp);
 879                                 goto err;
 880                         }
 881
 882                         err = intel_context_pin(tmp);
 883                         if (err) {
 884                                 intel_context_put(tmp);
 885                                 goto err;
 886                         }
 887
 888                         data.ce[i] = tmp;
 889                 }
 890
 891                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
 892                         data.poison = poison[i];
 893
 894                         err = __lrc_timestamp(&data, false);
 895                         if (err)
 896                                 break;
 897
 898                         err = __lrc_timestamp(&data, true);
 899                         if (err)
 900                                 break;
 901                 }
 902
 903 err:
 904                 st_engine_heartbeat_enable(data.engine);
 905                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 906                         if (!data.ce[i])
 907                                 break;
 908
 909                         intel_context_unpin(data.ce[i]);
 910                         intel_context_put(data.ce[i]);
 911                 }
 912
 913                 if (igt_flush_test(gt->i915))
 914                         err = -EIO;
 915                 if (err)
 916                         return err;
 917         }
 918
 919         return 0;
 920 }
 921
 922 static struct i915_vma *
 923 create_user_vma(struct i915_address_space *vm, unsigned long size)
 924 {
 925         struct drm_i915_gem_object *obj;
 926         struct i915_vma *vma;
 927         int err;
 928
 929         obj = i915_gem_object_create_internal(vm->i915, size);
 930         if (IS_ERR(obj))
 931                 return ERR_CAST(obj);
 932
 933         vma = i915_vma_instance(obj, vm, NULL);
 934         if (IS_ERR(vma)) {
 935                 i915_gem_object_put(obj);
 936                 return vma;
 937         }
 938
 939         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 940         if (err) {
 941                 i915_gem_object_put(obj);
 942                 return ERR_PTR(err);
 943         }
 944
 945         return vma;
 946 }
 947
 948 static u32 safe_poison(u32 offset, u32 poison)
 949 {
 950         /*
 951          * Do not enable predication as it will nop all subsequent commands,
 952          * not only disabling the tests (by preventing all the other SRM) but
 953          * also preventing the arbitration events at the end of the request.
 954          */
 955         if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))
 956                 poison &= ~REG_BIT(0);
 957
 958         return poison;
 959 }
 960
 961 static struct i915_vma *
 962 store_context(struct intel_context *ce, struct i915_vma *scratch)
 963 {
 964         struct i915_vma *batch;
 965         u32 dw, x, *cs, *hw;
 966         u32 *defaults;
 967
 968         batch = create_user_vma(ce->vm, SZ_64K);
 969         if (IS_ERR(batch))
 970                 return batch;
 971
 972         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 973         if (IS_ERR(cs)) {
 974                 i915_vma_put(batch);
 975                 return ERR_CAST(cs);
 976         }
 977
 978         defaults = shmem_pin_map(ce->engine->default_state);
 979         if (!defaults) {
 980                 i915_gem_object_unpin_map(batch->obj);
 981                 i915_vma_put(batch);
 982                 return ERR_PTR(-ENOMEM);
 983         }
 984
 985         x = 0;
 986         dw = 0;
 987         hw = defaults;
 988         hw += LRC_STATE_OFFSET / sizeof(*hw);
 989         do {
 990                 u32 len = hw[dw] & 0x7f;
 991
 992                 if (hw[dw] == 0) {
 993                         dw++;
 994                         continue;
 995                 }
 996
 997                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 998                         dw += len + 2;
 999                         continue;
1000                 }
1001
1002                 dw++;
1003                 len = (len + 1) / 2;
1004                 while (len--) {
1005                         *cs++ = MI_STORE_REGISTER_MEM_GEN8;
1006                         *cs++ = hw[dw];
1007                         *cs++ = lower_32_bits(scratch->node.start + x);
1008                         *cs++ = upper_32_bits(scratch->node.start + x);
1009
1010                         dw += 2;
1011                         x += 4;
1012                 }
1013         } while (dw < PAGE_SIZE / sizeof(u32) &&
1014                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1015
1016         *cs++ = MI_BATCH_BUFFER_END;
1017
1018         shmem_unpin_map(ce->engine->default_state, defaults);
1019
1020         i915_gem_object_flush_map(batch->obj);
1021         i915_gem_object_unpin_map(batch->obj);
1022
1023         return batch;
1024 }
1025
1026 static int move_to_active(struct i915_request *rq,
1027                           struct i915_vma *vma,
1028                           unsigned int flags)
1029 {
1030         int err;
1031
1032         i915_vma_lock(vma);
1033         err = i915_request_await_object(rq, vma->obj, flags);
1034         if (!err)
1035                 err = i915_vma_move_to_active(vma, rq, flags);
1036         i915_vma_unlock(vma);
1037
1038         return err;
1039 }
1040
1041 static struct i915_request *
1042 record_registers(struct intel_context *ce,
1043                  struct i915_vma *before,
1044                  struct i915_vma *after,
1045                  u32 *sema)
1046 {
1047         struct i915_vma *b_before, *b_after;
1048         struct i915_request *rq;
1049         u32 *cs;
1050         int err;
1051
1052         b_before = store_context(ce, before);
1053         if (IS_ERR(b_before))
1054                 return ERR_CAST(b_before);
1055
1056         b_after = store_context(ce, after);
1057         if (IS_ERR(b_after)) {
1058                 rq = ERR_CAST(b_after);
1059                 goto err_before;
1060         }
1061
1062         rq = intel_context_create_request(ce);
1063         if (IS_ERR(rq))
1064                 goto err_after;
1065
1066         err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1067         if (err)
1068                 goto err_rq;
1069
1070         err = move_to_active(rq, b_before, 0);
1071         if (err)
1072                 goto err_rq;
1073
1074         err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1075         if (err)
1076                 goto err_rq;
1077
1078         err = move_to_active(rq, b_after, 0);
1079         if (err)
1080                 goto err_rq;
1081
1082         cs = intel_ring_begin(rq, 14);
1083         if (IS_ERR(cs)) {
1084                 err = PTR_ERR(cs);
1085                 goto err_rq;
1086         }
1087
1088         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1089         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1090         *cs++ = lower_32_bits(b_before->node.start);
1091         *cs++ = upper_32_bits(b_before->node.start);
1092
1093         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1094         *cs++ = MI_SEMAPHORE_WAIT |
1095                 MI_SEMAPHORE_GLOBAL_GTT |
1096                 MI_SEMAPHORE_POLL |
1097                 MI_SEMAPHORE_SAD_NEQ_SDD;
1098         *cs++ = 0;
1099         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1100                 offset_in_page(sema);
1101         *cs++ = 0;
1102         *cs++ = MI_NOOP;
1103
1104         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1105         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1106         *cs++ = lower_32_bits(b_after->node.start);
1107         *cs++ = upper_32_bits(b_after->node.start);
1108
1109         intel_ring_advance(rq, cs);
1110
1111         WRITE_ONCE(*sema, 0);
1112         i915_request_get(rq);
1113         i915_request_add(rq);
1114 err_after:
1115         i915_vma_put(b_after);
1116 err_before:
1117         i915_vma_put(b_before);
1118         return rq;
1119
1120 err_rq:
1121         i915_request_add(rq);
1122         rq = ERR_PTR(err);
1123         goto err_after;
1124 }
1125
1126 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1127 {
1128         struct i915_vma *batch;
1129         u32 dw, *cs, *hw;
1130         u32 *defaults;
1131
1132         batch = create_user_vma(ce->vm, SZ_64K);
1133         if (IS_ERR(batch))
1134                 return batch;
1135
1136         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1137         if (IS_ERR(cs)) {
1138                 i915_vma_put(batch);
1139                 return ERR_CAST(cs);
1140         }
1141
1142         defaults = shmem_pin_map(ce->engine->default_state);
1143         if (!defaults) {
1144                 i915_gem_object_unpin_map(batch->obj);
1145                 i915_vma_put(batch);
1146                 return ERR_PTR(-ENOMEM);
1147         }
1148
1149         dw = 0;
1150         hw = defaults;
1151         hw += LRC_STATE_OFFSET / sizeof(*hw);
1152         do {
1153                 u32 len = hw[dw] & 0x7f;
1154
1155                 if (hw[dw] == 0) {
1156                         dw++;
1157                         continue;
1158                 }
1159
1160                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1161                         dw += len + 2;
1162                         continue;
1163                 }
1164
1165                 dw++;
1166                 len = (len + 1) / 2;
1167                 *cs++ = MI_LOAD_REGISTER_IMM(len);
1168                 while (len--) {
1169                         *cs++ = hw[dw];
1170                         *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
1171                                                                   MI_LRI_LRM_CS_MMIO),
1172                                             poison);
1173                         dw += 2;
1174                 }
1175         } while (dw < PAGE_SIZE / sizeof(u32) &&
1176                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1177
1178         *cs++ = MI_BATCH_BUFFER_END;
1179
1180         shmem_unpin_map(ce->engine->default_state, defaults);
1181
1182         i915_gem_object_flush_map(batch->obj);
1183         i915_gem_object_unpin_map(batch->obj);
1184
1185         return batch;
1186 }
1187
1188 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1189 {
1190         struct i915_request *rq;
1191         struct i915_vma *batch;
1192         u32 *cs;
1193         int err;
1194
1195         batch = load_context(ce, poison);
1196         if (IS_ERR(batch))
1197                 return PTR_ERR(batch);
1198
1199         rq = intel_context_create_request(ce);
1200         if (IS_ERR(rq)) {
1201                 err = PTR_ERR(rq);
1202                 goto err_batch;
1203         }
1204
1205         err = move_to_active(rq, batch, 0);
1206         if (err)
1207                 goto err_rq;
1208
1209         cs = intel_ring_begin(rq, 8);
1210         if (IS_ERR(cs)) {
1211                 err = PTR_ERR(cs);
1212                 goto err_rq;
1213         }
1214
1215         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1216         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1217         *cs++ = lower_32_bits(batch->node.start);
1218         *cs++ = upper_32_bits(batch->node.start);
1219
1220         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1221         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1222                 offset_in_page(sema);
1223         *cs++ = 0;
1224         *cs++ = 1;
1225
1226         intel_ring_advance(rq, cs);
1227
1228         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1229 err_rq:
1230         i915_request_add(rq);
1231 err_batch:
1232         i915_vma_put(batch);
1233         return err;
1234 }
1235
1236 static bool is_moving(u32 a, u32 b)
1237 {
1238         return a != b;
1239 }
1240
1241 static int compare_isolation(struct intel_engine_cs *engine,
1242                              struct i915_vma *ref[2],
1243                              struct i915_vma *result[2],
1244                              struct intel_context *ce,
1245                              u32 poison)
1246 {
1247         u32 x, dw, *hw, *lrc;
1248         u32 *A[2], *B[2];
1249         u32 *defaults;
1250         int err = 0;
1251
1252         A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1253         if (IS_ERR(A[0]))
1254                 return PTR_ERR(A[0]);
1255
1256         A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1257         if (IS_ERR(A[1])) {
1258                 err = PTR_ERR(A[1]);
1259                 goto err_A0;
1260         }
1261
1262         B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1263         if (IS_ERR(B[0])) {
1264                 err = PTR_ERR(B[0]);
1265                 goto err_A1;
1266         }
1267
1268         B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1269         if (IS_ERR(B[1])) {
1270                 err = PTR_ERR(B[1]);
1271                 goto err_B0;
1272         }
1273
1274         lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1275                                                i915_coherent_map_type(engine->i915,
1276                                                                       ce->state->obj,
1277                                                                       false));
1278         if (IS_ERR(lrc)) {
1279                 err = PTR_ERR(lrc);
1280                 goto err_B1;
1281         }
1282         lrc += LRC_STATE_OFFSET / sizeof(*hw);
1283
1284         defaults = shmem_pin_map(ce->engine->default_state);
1285         if (!defaults) {
1286                 err = -ENOMEM;
1287                 goto err_lrc;
1288         }
1289
1290         x = 0;
1291         dw = 0;
1292         hw = defaults;
1293         hw += LRC_STATE_OFFSET / sizeof(*hw);
1294         do {
1295                 u32 len = hw[dw] & 0x7f;
1296
1297                 if (hw[dw] == 0) {
1298                         dw++;
1299                         continue;
1300                 }
1301
1302                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1303                         dw += len + 2;
1304                         continue;
1305                 }
1306
1307                 dw++;
1308                 len = (len + 1) / 2;
1309                 while (len--) {
1310                         if (!is_moving(A[0][x], A[1][x]) &&
1311                             (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1312                                 switch (hw[dw] & 4095) {
1313                                 case 0x30: /* RING_HEAD */
1314                                 case 0x34: /* RING_TAIL */
1315                                         break;
1316
1317                                 default:
1318                                         pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1319                                                engine->name, dw,
1320                                                hw[dw], hw[dw + 1],
1321                                                A[0][x], B[0][x], B[1][x],
1322                                                poison, lrc[dw + 1]);
1323                                         err = -EINVAL;
1324                                 }
1325                         }
1326                         dw += 2;
1327                         x++;
1328                 }
1329         } while (dw < PAGE_SIZE / sizeof(u32) &&
1330                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1331
1332         shmem_unpin_map(ce->engine->default_state, defaults);
1333 err_lrc:
1334         i915_gem_object_unpin_map(ce->state->obj);
1335 err_B1:
1336         i915_gem_object_unpin_map(result[1]->obj);
1337 err_B0:
1338         i915_gem_object_unpin_map(result[0]->obj);
1339 err_A1:
1340         i915_gem_object_unpin_map(ref[1]->obj);
1341 err_A0:
1342         i915_gem_object_unpin_map(ref[0]->obj);
1343         return err;
1344 }
1345
1346 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1347 {
1348         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1349         struct i915_vma *ref[2], *result[2];
1350         struct intel_context *A, *B;
1351         struct i915_request *rq;
1352         int err;
1353
1354         A = intel_context_create(engine);
1355         if (IS_ERR(A))
1356                 return PTR_ERR(A);
1357
1358         B = intel_context_create(engine);
1359         if (IS_ERR(B)) {
1360                 err = PTR_ERR(B);
1361                 goto err_A;
1362         }
1363
1364         ref[0] = create_user_vma(A->vm, SZ_64K);
1365         if (IS_ERR(ref[0])) {
1366                 err = PTR_ERR(ref[0]);
1367                 goto err_B;
1368         }
1369
1370         ref[1] = create_user_vma(A->vm, SZ_64K);
1371         if (IS_ERR(ref[1])) {
1372                 err = PTR_ERR(ref[1]);
1373                 goto err_ref0;
1374         }
1375
1376         rq = record_registers(A, ref[0], ref[1], sema);
1377         if (IS_ERR(rq)) {
1378                 err = PTR_ERR(rq);
1379                 goto err_ref1;
1380         }
1381
1382         WRITE_ONCE(*sema, 1);
1383         wmb();
1384
1385         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1386                 i915_request_put(rq);
1387                 err = -ETIME;
1388                 goto err_ref1;
1389         }
1390         i915_request_put(rq);
1391
1392         result[0] = create_user_vma(A->vm, SZ_64K);
1393         if (IS_ERR(result[0])) {
1394                 err = PTR_ERR(result[0]);
1395                 goto err_ref1;
1396         }
1397
1398         result[1] = create_user_vma(A->vm, SZ_64K);
1399         if (IS_ERR(result[1])) {
1400                 err = PTR_ERR(result[1]);
1401                 goto err_result0;
1402         }
1403
1404         rq = record_registers(A, result[0], result[1], sema);
1405         if (IS_ERR(rq)) {
1406                 err = PTR_ERR(rq);
1407                 goto err_result1;
1408         }
1409
1410         err = poison_registers(B, poison, sema);
1411         if (err) {
1412                 WRITE_ONCE(*sema, -1);
1413                 i915_request_put(rq);
1414                 goto err_result1;
1415         }
1416
1417         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1418                 i915_request_put(rq);
1419                 err = -ETIME;
1420                 goto err_result1;
1421         }
1422         i915_request_put(rq);
1423
1424         err = compare_isolation(engine, ref, result, A, poison);
1425
1426 err_result1:
1427         i915_vma_put(result[1]);
1428 err_result0:
1429         i915_vma_put(result[0]);
1430 err_ref1:
1431         i915_vma_put(ref[1]);
1432 err_ref0:
1433         i915_vma_put(ref[0]);
1434 err_B:
1435         intel_context_put(B);
1436 err_A:
1437         intel_context_put(A);
1438         return err;
1439 }
1440
1441 static bool skip_isolation(const struct intel_engine_cs *engine)
1442 {
1443         if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1444                 return true;
1445
1446         if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1447                 return true;
1448
1449         return false;
1450 }
1451
1452 static int live_lrc_isolation(void *arg)
1453 {
1454         struct intel_gt *gt = arg;
1455         struct intel_engine_cs *engine;
1456         enum intel_engine_id id;
1457         const u32 poison[] = {
1458                 STACK_MAGIC,
1459                 0x3a3a3a3a,
1460                 0x5c5c5c5c,
1461                 0xffffffff,
1462                 0xffff0000,
1463         };
1464         int err = 0;
1465
1466         /*
1467          * Our goal is try and verify that per-context state cannot be
1468          * tampered with by another non-privileged client.
1469          *
1470          * We take the list of context registers from the LRI in the default
1471          * context image and attempt to modify that list from a remote context.
1472          */
1473
1474         for_each_engine(engine, gt, id) {
1475                 int i;
1476
1477                 /* Just don't even ask */
1478                 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1479                     skip_isolation(engine))
1480                         continue;
1481
1482                 intel_engine_pm_get(engine);
1483                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1484                         int result;
1485
1486                         result = __lrc_isolation(engine, poison[i]);
1487                         if (result && !err)
1488                                 err = result;
1489
1490                         result = __lrc_isolation(engine, ~poison[i]);
1491                         if (result && !err)
1492                                 err = result;
1493                 }
1494                 intel_engine_pm_put(engine);
1495                 if (igt_flush_test(gt->i915)) {
1496                         err = -EIO;
1497                         break;
1498                 }
1499         }
1500
1501         return err;
1502 }
1503
1504 static int indirect_ctx_submit_req(struct intel_context *ce)
1505 {
1506         struct i915_request *rq;
1507         int err = 0;
1508
1509         rq = intel_context_create_request(ce);
1510         if (IS_ERR(rq))
1511                 return PTR_ERR(rq);
1512
1513         i915_request_get(rq);
1514         i915_request_add(rq);
1515
1516         if (i915_request_wait(rq, 0, HZ / 5) < 0)
1517                 err = -ETIME;
1518
1519         i915_request_put(rq);
1520
1521         return err;
1522 }
1523
1524 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1525 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1526
1527 static u32 *
1528 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1529 {
1530         *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1531                 MI_SRM_LRM_GLOBAL_GTT |
1532                 MI_LRI_LRM_CS_MMIO;
1533         *cs++ = i915_mmio_reg_offset(RING_START(0));
1534         *cs++ = i915_ggtt_offset(ce->state) +
1535                 context_wa_bb_offset(ce) +
1536                 CTX_BB_CANARY_OFFSET;
1537         *cs++ = 0;
1538
1539         return cs;
1540 }
1541
1542 static void
1543 indirect_ctx_bb_setup(struct intel_context *ce)
1544 {
1545         u32 *cs = context_indirect_bb(ce);
1546
1547         cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1548
1549         setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1550 }
1551
1552 static bool check_ring_start(struct intel_context *ce)
1553 {
1554         const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1555                 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1556
1557         if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1558                 return true;
1559
1560         pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1561                ctx_bb[CTX_BB_CANARY_INDEX],
1562                ce->lrc_reg_state[CTX_RING_START]);
1563
1564         return false;
1565 }
1566
1567 static int indirect_ctx_bb_check(struct intel_context *ce)
1568 {
1569         int err;
1570
1571         err = indirect_ctx_submit_req(ce);
1572         if (err)
1573                 return err;
1574
1575         if (!check_ring_start(ce))
1576                 return -EINVAL;
1577
1578         return 0;
1579 }
1580
1581 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1582 {
1583         struct intel_context *a, *b;
1584         int err;
1585
1586         a = intel_context_create(engine);
1587         if (IS_ERR(a))
1588                 return PTR_ERR(a);
1589         err = intel_context_pin(a);
1590         if (err)
1591                 goto put_a;
1592
1593         b = intel_context_create(engine);
1594         if (IS_ERR(b)) {
1595                 err = PTR_ERR(b);
1596                 goto unpin_a;
1597         }
1598         err = intel_context_pin(b);
1599         if (err)
1600                 goto put_b;
1601
1602         /* We use the already reserved extra page in context state */
1603         if (!a->wa_bb_page) {
1604                 GEM_BUG_ON(b->wa_bb_page);
1605                 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1606                 goto unpin_b;
1607         }
1608
1609         /*
1610          * In order to test that our per context bb is truly per context,
1611          * and executes at the intended spot on context restoring process,
1612          * make the batch store the ring start value to memory.
1613          * As ring start is restored apriori of starting the indirect ctx bb and
1614          * as it will be different for each context, it fits to this purpose.
1615          */
1616         indirect_ctx_bb_setup(a);
1617         indirect_ctx_bb_setup(b);
1618
1619         err = indirect_ctx_bb_check(a);
1620         if (err)
1621                 goto unpin_b;
1622
1623         err = indirect_ctx_bb_check(b);
1624
1625 unpin_b:
1626         intel_context_unpin(b);
1627 put_b:
1628         intel_context_put(b);
1629 unpin_a:
1630         intel_context_unpin(a);
1631 put_a:
1632         intel_context_put(a);
1633
1634         return err;
1635 }
1636
1637 static int live_lrc_indirect_ctx_bb(void *arg)
1638 {
1639         struct intel_gt *gt = arg;
1640         struct intel_engine_cs *engine;
1641         enum intel_engine_id id;
1642         int err = 0;
1643
1644         for_each_engine(engine, gt, id) {
1645                 intel_engine_pm_get(engine);
1646                 err = __live_lrc_indirect_ctx_bb(engine);
1647                 intel_engine_pm_put(engine);
1648
1649                 if (igt_flush_test(gt->i915))
1650                         err = -EIO;
1651
1652                 if (err)
1653                         break;
1654         }
1655
1656         return err;
1657 }
1658
1659 static void garbage_reset(struct intel_engine_cs *engine,
1660                           struct i915_request *rq)
1661 {
1662         const unsigned int bit = I915_RESET_ENGINE + engine->id;
1663         unsigned long *lock = &engine->gt->reset.flags;
1664
1665         local_bh_disable();
1666         if (!test_and_set_bit(bit, lock)) {
1667                 tasklet_disable(&engine->sched_engine->tasklet);
1668
1669                 if (!rq->fence.error)
1670                         __intel_engine_reset_bh(engine, NULL);
1671
1672                 tasklet_enable(&engine->sched_engine->tasklet);
1673                 clear_and_wake_up_bit(bit, lock);
1674         }
1675         local_bh_enable();
1676 }
1677
1678 static struct i915_request *garbage(struct intel_context *ce,
1679                                     struct rnd_state *prng)
1680 {
1681         struct i915_request *rq;
1682         int err;
1683
1684         err = intel_context_pin(ce);
1685         if (err)
1686                 return ERR_PTR(err);
1687
1688         prandom_bytes_state(prng,
1689                             ce->lrc_reg_state,
1690                             ce->engine->context_size -
1691                             LRC_STATE_OFFSET);
1692
1693         rq = intel_context_create_request(ce);
1694         if (IS_ERR(rq)) {
1695                 err = PTR_ERR(rq);
1696                 goto err_unpin;
1697         }
1698
1699         i915_request_get(rq);
1700         i915_request_add(rq);
1701         return rq;
1702
1703 err_unpin:
1704         intel_context_unpin(ce);
1705         return ERR_PTR(err);
1706 }
1707
1708 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1709 {
1710         struct intel_context *ce;
1711         struct i915_request *hang;
1712         int err = 0;
1713
1714         ce = intel_context_create(engine);
1715         if (IS_ERR(ce))
1716                 return PTR_ERR(ce);
1717
1718         hang = garbage(ce, prng);
1719         if (IS_ERR(hang)) {
1720                 err = PTR_ERR(hang);
1721                 goto err_ce;
1722         }
1723
1724         if (wait_for_submit(engine, hang, HZ / 2)) {
1725                 i915_request_put(hang);
1726                 err = -ETIME;
1727                 goto err_ce;
1728         }
1729
1730         intel_context_set_banned(ce);
1731         garbage_reset(engine, hang);
1732
1733         intel_engine_flush_submission(engine);
1734         if (!hang->fence.error) {
1735                 i915_request_put(hang);
1736                 pr_err("%s: corrupted context was not reset\n",
1737                        engine->name);
1738                 err = -EINVAL;
1739                 goto err_ce;
1740         }
1741
1742         if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1743                 pr_err("%s: corrupted context did not recover\n",
1744                        engine->name);
1745                 i915_request_put(hang);
1746                 err = -EIO;
1747                 goto err_ce;
1748         }
1749         i915_request_put(hang);
1750
1751 err_ce:
1752         intel_context_put(ce);
1753         return err;
1754 }
1755
1756 static int live_lrc_garbage(void *arg)
1757 {
1758         struct intel_gt *gt = arg;
1759         struct intel_engine_cs *engine;
1760         enum intel_engine_id id;
1761
1762         /*
1763          * Verify that we can recover if one context state is completely
1764          * corrupted.
1765          */
1766
1767         if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1768                 return 0;
1769
1770         for_each_engine(engine, gt, id) {
1771                 I915_RND_STATE(prng);
1772                 int err = 0, i;
1773
1774                 if (!intel_has_reset_engine(engine->gt))
1775                         continue;
1776
1777                 intel_engine_pm_get(engine);
1778                 for (i = 0; i < 3; i++) {
1779                         err = __lrc_garbage(engine, &prng);
1780                         if (err)
1781                                 break;
1782                 }
1783                 intel_engine_pm_put(engine);
1784
1785                 if (igt_flush_test(gt->i915))
1786                         err = -EIO;
1787                 if (err)
1788                         return err;
1789         }
1790
1791         return 0;
1792 }
1793
1794 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1795 {
1796         struct intel_context *ce;
1797         struct i915_request *rq;
1798         IGT_TIMEOUT(end_time);
1799         int err;
1800
1801         ce = intel_context_create(engine);
1802         if (IS_ERR(ce))
1803                 return PTR_ERR(ce);
1804
1805         ce->stats.runtime.num_underflow = 0;
1806         ce->stats.runtime.max_underflow = 0;
1807
1808         do {
1809                 unsigned int loop = 1024;
1810
1811                 while (loop) {
1812                         rq = intel_context_create_request(ce);
1813                         if (IS_ERR(rq)) {
1814                                 err = PTR_ERR(rq);
1815                                 goto err_rq;
1816                         }
1817
1818                         if (--loop == 0)
1819                                 i915_request_get(rq);
1820
1821                         i915_request_add(rq);
1822                 }
1823
1824                 if (__igt_timeout(end_time, NULL))
1825                         break;
1826
1827                 i915_request_put(rq);
1828         } while (1);
1829
1830         err = i915_request_wait(rq, 0, HZ / 5);
1831         if (err < 0) {
1832                 pr_err("%s: request not completed!\n", engine->name);
1833                 goto err_wait;
1834         }
1835
1836         igt_flush_test(engine->i915);
1837
1838         pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1839                 engine->name,
1840                 intel_context_get_total_runtime_ns(ce),
1841                 intel_context_get_avg_runtime_ns(ce));
1842
1843         err = 0;
1844         if (ce->stats.runtime.num_underflow) {
1845                 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1846                        engine->name,
1847                        ce->stats.runtime.num_underflow,
1848                        ce->stats.runtime.max_underflow);
1849                 GEM_TRACE_DUMP();
1850                 err = -EOVERFLOW;
1851         }
1852
1853 err_wait:
1854         i915_request_put(rq);
1855 err_rq:
1856         intel_context_put(ce);
1857         return err;
1858 }
1859
1860 static int live_pphwsp_runtime(void *arg)
1861 {
1862         struct intel_gt *gt = arg;
1863         struct intel_engine_cs *engine;
1864         enum intel_engine_id id;
1865         int err = 0;
1866
1867         /*
1868          * Check that cumulative context runtime as stored in the pphwsp[16]
1869          * is monotonic.
1870          */
1871
1872         for_each_engine(engine, gt, id) {
1873                 err = __live_pphwsp_runtime(engine);
1874                 if (err)
1875                         break;
1876         }
1877
1878         if (igt_flush_test(gt->i915))
1879                 err = -EIO;
1880
1881         return err;
1882 }
1883
1884 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1885 {
1886         static const struct i915_subtest tests[] = {
1887                 SUBTEST(live_lrc_layout),
1888                 SUBTEST(live_lrc_fixed),
1889                 SUBTEST(live_lrc_state),
1890                 SUBTEST(live_lrc_gpr),
1891                 SUBTEST(live_lrc_isolation),
1892                 SUBTEST(live_lrc_timestamp),
1893                 SUBTEST(live_lrc_garbage),
1894                 SUBTEST(live_pphwsp_runtime),
1895                 SUBTEST(live_lrc_indirect_ctx_bb),
1896         };
1897
1898         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1899                 return 0;
1900
1901         return intel_gt_live_subtests(tests, to_gt(i915));
1902 }