2 * SPDX-License-Identifier: MIT
4 * Copyright © 2018 Intel Corporation
7 #include <linux/prime_numbers.h>
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
13 #include "i915_selftest.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
20 #include "gem/selftests/igt_gem_utils.h"
21 #include "gem/selftests/mock_context.h"
23 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27 static struct i915_vma *create_scratch(struct intel_gt *gt)
29 struct drm_i915_gem_object *obj;
33 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
37 i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
39 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
41 i915_gem_object_put(obj);
45 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
47 i915_gem_object_put(obj);
54 static void engine_heartbeat_disable(struct intel_engine_cs *engine)
56 engine->props.heartbeat_interval_ms = 0;
58 intel_engine_pm_get(engine);
59 intel_engine_park_heartbeat(engine);
62 static void engine_heartbeat_enable(struct intel_engine_cs *engine)
64 intel_engine_pm_put(engine);
66 engine->props.heartbeat_interval_ms =
67 engine->defaults.heartbeat_interval_ms;
70 static bool is_active(struct i915_request *rq)
72 if (i915_request_is_active(rq))
75 if (i915_request_on_hold(rq))
78 if (i915_request_started(rq))
84 static int wait_for_submit(struct intel_engine_cs *engine,
85 struct i915_request *rq,
86 unsigned long timeout)
90 bool done = time_after(jiffies, timeout);
92 if (i915_request_completed(rq)) /* that was quick! */
95 /* Wait until the HW has acknowleged the submission (or err) */
96 intel_engine_flush_submission(engine);
97 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
107 static int wait_for_reset(struct intel_engine_cs *engine,
108 struct i915_request *rq,
109 unsigned long timeout)
115 intel_engine_flush_submission(engine);
117 if (READ_ONCE(engine->execlists.pending[0]))
120 if (i915_request_completed(rq))
123 if (READ_ONCE(rq->fence.error))
125 } while (time_before(jiffies, timeout));
127 flush_scheduled_work();
129 if (rq->fence.error != -EIO) {
130 pr_err("%s: hanging request %llx:%lld not reset\n",
137 /* Give the request a jiffie to complete after flushing the worker */
138 if (i915_request_wait(rq, 0,
139 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
140 pr_err("%s: hanging request %llx:%lld did not complete\n",
150 static int live_sanitycheck(void *arg)
152 struct intel_gt *gt = arg;
153 struct intel_engine_cs *engine;
154 enum intel_engine_id id;
155 struct igt_spinner spin;
158 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
161 if (igt_spinner_init(&spin, gt))
164 for_each_engine(engine, gt, id) {
165 struct intel_context *ce;
166 struct i915_request *rq;
168 ce = intel_context_create(engine);
174 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
180 i915_request_add(rq);
181 if (!igt_wait_for_spinner(&spin, rq)) {
182 GEM_TRACE("spinner failed to start\n");
184 intel_gt_set_wedged(gt);
189 igt_spinner_end(&spin);
190 if (igt_flush_test(gt->i915)) {
196 intel_context_put(ce);
201 igt_spinner_fini(&spin);
205 static int live_unlite_restore(struct intel_gt *gt, int prio)
207 struct intel_engine_cs *engine;
208 enum intel_engine_id id;
209 struct igt_spinner spin;
213 * Check that we can correctly context switch between 2 instances
214 * on the same engine from the same parent context.
217 if (igt_spinner_init(&spin, gt))
221 for_each_engine(engine, gt, id) {
222 struct intel_context *ce[2] = {};
223 struct i915_request *rq[2];
224 struct igt_live_test t;
227 if (prio && !intel_engine_has_preemption(engine))
230 if (!intel_engine_can_store_dword(engine))
233 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
237 engine_heartbeat_disable(engine);
239 for (n = 0; n < ARRAY_SIZE(ce); n++) {
240 struct intel_context *tmp;
242 tmp = intel_context_create(engine);
248 err = intel_context_pin(tmp);
250 intel_context_put(tmp);
255 * Setup the pair of contexts such that if we
256 * lite-restore using the RING_TAIL from ce[1] it
257 * will execute garbage from ce[0]->ring.
259 memset(tmp->ring->vaddr,
260 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
261 tmp->ring->vma->size);
265 GEM_BUG_ON(!ce[1]->ring->size);
266 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
267 __execlists_update_reg_state(ce[1], engine, ce[1]->ring->head);
269 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
271 err = PTR_ERR(rq[0]);
275 i915_request_get(rq[0]);
276 i915_request_add(rq[0]);
277 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
279 if (!igt_wait_for_spinner(&spin, rq[0])) {
280 i915_request_put(rq[0]);
284 rq[1] = i915_request_create(ce[1]);
286 err = PTR_ERR(rq[1]);
287 i915_request_put(rq[0]);
293 * Ensure we do the switch to ce[1] on completion.
295 * rq[0] is already submitted, so this should reduce
296 * to a no-op (a wait on a request on the same engine
297 * uses the submit fence, not the completion fence),
298 * but it will install a dependency on rq[1] for rq[0]
299 * that will prevent the pair being reordered by
302 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
305 i915_request_get(rq[1]);
306 i915_request_add(rq[1]);
307 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
308 i915_request_put(rq[0]);
311 struct i915_sched_attr attr = {
315 /* Alternatively preempt the spinner with ce[1] */
316 engine->schedule(rq[1], &attr);
319 /* And switch back to ce[0] for good measure */
320 rq[0] = i915_request_create(ce[0]);
322 err = PTR_ERR(rq[0]);
323 i915_request_put(rq[1]);
327 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
328 i915_request_get(rq[0]);
329 i915_request_add(rq[0]);
330 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
331 i915_request_put(rq[1]);
332 i915_request_put(rq[0]);
335 tasklet_kill(&engine->execlists.tasklet); /* flush submission */
336 igt_spinner_end(&spin);
337 for (n = 0; n < ARRAY_SIZE(ce); n++) {
338 if (IS_ERR_OR_NULL(ce[n]))
341 intel_context_unpin(ce[n]);
342 intel_context_put(ce[n]);
345 engine_heartbeat_enable(engine);
346 if (igt_live_test_end(&t))
352 igt_spinner_fini(&spin);
356 static int live_unlite_switch(void *arg)
358 return live_unlite_restore(arg, 0);
361 static int live_unlite_preempt(void *arg)
363 return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX));
366 static int live_pin_rewind(void *arg)
368 struct intel_gt *gt = arg;
369 struct intel_engine_cs *engine;
370 enum intel_engine_id id;
374 * We have to be careful not to trust intel_ring too much, for example
375 * ring->head is updated upon retire which is out of sync with pinning
376 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
377 * or else we risk writing an older, stale value.
379 * To simulate this, let's apply a bit of deliberate sabotague.
382 for_each_engine(engine, gt, id) {
383 struct intel_context *ce;
384 struct i915_request *rq;
385 struct intel_ring *ring;
386 struct igt_live_test t;
388 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
393 ce = intel_context_create(engine);
399 err = intel_context_pin(ce);
401 intel_context_put(ce);
405 /* Keep the context awake while we play games */
406 err = i915_active_acquire(&ce->active);
408 intel_context_unpin(ce);
409 intel_context_put(ce);
414 /* Poison the ring, and offset the next request from HEAD */
415 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
416 ring->emit = ring->size / 2;
417 ring->tail = ring->emit;
418 GEM_BUG_ON(ring->head);
420 intel_context_unpin(ce);
422 /* Submit a simple nop request */
423 GEM_BUG_ON(intel_context_is_pinned(ce));
424 rq = intel_context_create_request(ce);
425 i915_active_release(&ce->active); /* e.g. async retire */
426 intel_context_put(ce);
431 GEM_BUG_ON(!rq->head);
432 i915_request_add(rq);
434 /* Expect not to hang! */
435 if (igt_live_test_end(&t)) {
444 static int live_hold_reset(void *arg)
446 struct intel_gt *gt = arg;
447 struct intel_engine_cs *engine;
448 enum intel_engine_id id;
449 struct igt_spinner spin;
453 * In order to support offline error capture for fast preempt reset,
454 * we need to decouple the guilty request and ensure that it and its
455 * descendents are not executed while the capture is in progress.
458 if (!intel_has_reset_engine(gt))
461 if (igt_spinner_init(&spin, gt))
464 for_each_engine(engine, gt, id) {
465 struct intel_context *ce;
466 struct i915_request *rq;
468 ce = intel_context_create(engine);
474 engine_heartbeat_disable(engine);
476 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
481 i915_request_add(rq);
483 if (!igt_wait_for_spinner(&spin, rq)) {
484 intel_gt_set_wedged(gt);
489 /* We have our request executing, now remove it and reset */
491 if (test_and_set_bit(I915_RESET_ENGINE + id,
493 intel_gt_set_wedged(gt);
497 tasklet_disable(&engine->execlists.tasklet);
499 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
500 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
502 i915_request_get(rq);
503 execlists_hold(engine, rq);
504 GEM_BUG_ON(!i915_request_on_hold(rq));
506 intel_engine_reset(engine, NULL);
507 GEM_BUG_ON(rq->fence.error != -EIO);
509 tasklet_enable(&engine->execlists.tasklet);
510 clear_and_wake_up_bit(I915_RESET_ENGINE + id,
513 /* Check that we do not resubmit the held request */
514 if (!i915_request_wait(rq, 0, HZ / 5)) {
515 pr_err("%s: on hold request completed!\n",
517 i915_request_put(rq);
521 GEM_BUG_ON(!i915_request_on_hold(rq));
523 /* But is resubmitted on release */
524 execlists_unhold(engine, rq);
525 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
526 pr_err("%s: held request did not complete!\n",
528 intel_gt_set_wedged(gt);
531 i915_request_put(rq);
534 engine_heartbeat_enable(engine);
535 intel_context_put(ce);
540 igt_spinner_fini(&spin);
544 static const char *error_repr(int err)
546 return err ? "bad" : "good";
549 static int live_error_interrupt(void *arg)
551 static const struct error_phase {
552 enum { GOOD = 0, BAD = -EIO } error[2];
557 { { GOOD, GOOD } }, /* sentinel */
559 struct intel_gt *gt = arg;
560 struct intel_engine_cs *engine;
561 enum intel_engine_id id;
564 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
565 * of invalid commands in user batches that will cause a GPU hang.
566 * This is a faster mechanism than using hangcheck/heartbeats, but
567 * only detects problems the HW knows about -- it will not warn when
570 * To verify our detection and reset, we throw some invalid commands
571 * at the HW and wait for the interrupt.
574 if (!intel_has_reset_engine(gt))
577 for_each_engine(engine, gt, id) {
578 const struct error_phase *p;
581 engine_heartbeat_disable(engine);
583 for (p = phases; p->error[0] != GOOD; p++) {
584 struct i915_request *client[ARRAY_SIZE(phases->error)];
588 memset(client, 0, sizeof(*client));
589 for (i = 0; i < ARRAY_SIZE(client); i++) {
590 struct intel_context *ce;
591 struct i915_request *rq;
593 ce = intel_context_create(engine);
599 rq = intel_context_create_request(ce);
600 intel_context_put(ce);
606 if (rq->engine->emit_init_breadcrumb) {
607 err = rq->engine->emit_init_breadcrumb(rq);
609 i915_request_add(rq);
614 cs = intel_ring_begin(rq, 2);
616 i915_request_add(rq);
629 client[i] = i915_request_get(rq);
630 i915_request_add(rq);
633 err = wait_for_submit(engine, client[0], HZ / 2);
635 pr_err("%s: first request did not start within time!\n",
641 for (i = 0; i < ARRAY_SIZE(client); i++) {
642 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
643 pr_debug("%s: %s request incomplete!\n",
645 error_repr(p->error[i]));
647 if (!i915_request_started(client[i])) {
648 pr_err("%s: %s request not started!\n",
650 error_repr(p->error[i]));
655 /* Kick the tasklet to process the error */
656 intel_engine_flush_submission(engine);
657 if (client[i]->fence.error != p->error[i]) {
658 pr_err("%s: %s request (%s) with wrong error code: %d\n",
660 error_repr(p->error[i]),
661 i915_request_completed(client[i]) ? "completed" : "running",
662 client[i]->fence.error);
669 for (i = 0; i < ARRAY_SIZE(client); i++)
671 i915_request_put(client[i]);
673 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
674 engine->name, p - phases,
675 p->error[0], p->error[1]);
680 engine_heartbeat_enable(engine);
682 intel_gt_set_wedged(gt);
691 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
695 cs = intel_ring_begin(rq, 10);
699 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
701 *cs++ = MI_SEMAPHORE_WAIT |
702 MI_SEMAPHORE_GLOBAL_GTT |
704 MI_SEMAPHORE_SAD_NEQ_SDD;
706 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
710 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
711 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
721 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
723 intel_ring_advance(rq, cs);
727 static struct i915_request *
728 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
730 struct intel_context *ce;
731 struct i915_request *rq;
734 ce = intel_context_create(engine);
738 rq = intel_context_create_request(ce);
743 if (rq->engine->emit_init_breadcrumb)
744 err = rq->engine->emit_init_breadcrumb(rq);
746 err = emit_semaphore_chain(rq, vma, idx);
748 i915_request_get(rq);
749 i915_request_add(rq);
754 intel_context_put(ce);
759 release_queue(struct intel_engine_cs *engine,
760 struct i915_vma *vma,
763 struct i915_sched_attr attr = {
766 struct i915_request *rq;
769 rq = intel_engine_create_kernel_request(engine);
773 cs = intel_ring_begin(rq, 4);
775 i915_request_add(rq);
779 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
780 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
784 intel_ring_advance(rq, cs);
786 i915_request_get(rq);
787 i915_request_add(rq);
790 engine->schedule(rq, &attr);
791 local_bh_enable(); /* kick tasklet */
793 i915_request_put(rq);
799 slice_semaphore_queue(struct intel_engine_cs *outer,
800 struct i915_vma *vma,
803 struct intel_engine_cs *engine;
804 struct i915_request *head;
805 enum intel_engine_id id;
808 head = semaphore_queue(outer, vma, n++);
810 return PTR_ERR(head);
812 for_each_engine(engine, outer->gt, id) {
813 for (i = 0; i < count; i++) {
814 struct i915_request *rq;
816 rq = semaphore_queue(engine, vma, n++);
822 i915_request_put(rq);
826 err = release_queue(outer, vma, n, INT_MAX);
830 if (i915_request_wait(head, 0,
831 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
832 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
835 intel_gt_set_wedged(outer->gt);
840 i915_request_put(head);
844 static int live_timeslice_preempt(void *arg)
846 struct intel_gt *gt = arg;
847 struct drm_i915_gem_object *obj;
848 struct i915_vma *vma;
854 * If a request takes too long, we would like to give other users
855 * a fair go on the GPU. In particular, users may create batches
856 * that wait upon external input, where that input may even be
857 * supplied by another GPU job. To avoid blocking forever, we
858 * need to preempt the current task and replace it with another
861 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
864 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
868 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
874 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
876 err = PTR_ERR(vaddr);
880 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
884 err = i915_vma_sync(vma);
888 for_each_prime_number_from(count, 1, 16) {
889 struct intel_engine_cs *engine;
890 enum intel_engine_id id;
892 for_each_engine(engine, gt, id) {
893 if (!intel_engine_has_preemption(engine))
896 memset(vaddr, 0, PAGE_SIZE);
898 engine_heartbeat_disable(engine);
899 err = slice_semaphore_queue(engine, vma, count);
900 engine_heartbeat_enable(engine);
904 if (igt_flush_test(gt->i915)) {
914 i915_gem_object_unpin_map(obj);
916 i915_gem_object_put(obj);
920 static struct i915_request *
921 create_rewinder(struct intel_context *ce,
922 struct i915_request *wait,
926 i915_ggtt_offset(ce->engine->status_page.vma) +
927 offset_in_page(slot);
928 struct i915_request *rq;
932 rq = intel_context_create_request(ce);
937 err = i915_request_await_dma_fence(rq, &wait->fence);
942 cs = intel_ring_begin(rq, 14);
948 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
951 *cs++ = MI_SEMAPHORE_WAIT |
952 MI_SEMAPHORE_GLOBAL_GTT |
954 MI_SEMAPHORE_SAD_GTE_SDD;
959 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
960 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
961 *cs++ = offset + idx * sizeof(u32);
964 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
969 intel_ring_advance(rq, cs);
971 rq->sched.attr.priority = I915_PRIORITY_MASK;
974 i915_request_get(rq);
975 i915_request_add(rq);
977 i915_request_put(rq);
984 static int live_timeslice_rewind(void *arg)
986 struct intel_gt *gt = arg;
987 struct intel_engine_cs *engine;
988 enum intel_engine_id id;
991 * The usual presumption on timeslice expiration is that we replace
992 * the active context with another. However, given a chain of
993 * dependencies we may end up with replacing the context with itself,
994 * but only a few of those requests, forcing us to rewind the
995 * RING_TAIL of the original request.
997 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1000 for_each_engine(engine, gt, id) {
1001 enum { A1, A2, B1 };
1002 enum { X = 1, Z, Y };
1003 struct i915_request *rq[3] = {};
1004 struct intel_context *ce;
1005 unsigned long timeslice;
1009 if (!intel_engine_has_timeslices(engine))
1013 * A:rq1 -- semaphore wait, timestamp X
1014 * A:rq2 -- write timestamp Y
1016 * B:rq1 [await A:rq1] -- write timestamp Z
1018 * Force timeslice, release semaphore.
1020 * Expect execution/evaluation order XZY
1023 engine_heartbeat_disable(engine);
1024 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1026 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1028 ce = intel_context_create(engine);
1034 rq[0] = create_rewinder(ce, NULL, slot, X);
1035 if (IS_ERR(rq[0])) {
1036 intel_context_put(ce);
1040 rq[1] = create_rewinder(ce, NULL, slot, Y);
1041 intel_context_put(ce);
1045 err = wait_for_submit(engine, rq[1], HZ / 2);
1047 pr_err("%s: failed to submit first context\n",
1052 ce = intel_context_create(engine);
1058 rq[2] = create_rewinder(ce, rq[0], slot, Z);
1059 intel_context_put(ce);
1063 err = wait_for_submit(engine, rq[2], HZ / 2);
1065 pr_err("%s: failed to submit second context\n",
1070 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1071 if (i915_request_is_active(rq[A2])) { /* semaphore yielded! */
1072 /* Wait for the timeslice to kick in */
1073 del_timer(&engine->execlists.timer);
1074 tasklet_hi_schedule(&engine->execlists.tasklet);
1075 intel_engine_flush_submission(engine);
1077 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1078 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1079 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1080 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1082 /* Release the hounds! */
1084 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1086 for (i = 1; i <= 3; i++) {
1087 unsigned long timeout = jiffies + HZ / 2;
1089 while (!READ_ONCE(slot[i]) &&
1090 time_before(jiffies, timeout))
1093 if (!time_before(jiffies, timeout)) {
1094 pr_err("%s: rq[%d] timed out\n",
1095 engine->name, i - 1);
1100 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1104 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1105 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1113 memset32(&slot[0], -1, 4);
1116 engine->props.timeslice_duration_ms = timeslice;
1117 engine_heartbeat_enable(engine);
1118 for (i = 0; i < 3; i++)
1119 i915_request_put(rq[i]);
1120 if (igt_flush_test(gt->i915))
1129 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1131 struct i915_request *rq;
1133 rq = intel_engine_create_kernel_request(engine);
1137 i915_request_get(rq);
1138 i915_request_add(rq);
1143 static long timeslice_threshold(const struct intel_engine_cs *engine)
1145 return 2 * msecs_to_jiffies_timeout(timeslice(engine)) + 1;
1148 static int live_timeslice_queue(void *arg)
1150 struct intel_gt *gt = arg;
1151 struct drm_i915_gem_object *obj;
1152 struct intel_engine_cs *engine;
1153 enum intel_engine_id id;
1154 struct i915_vma *vma;
1159 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1160 * timeslicing between them disabled, we *do* enable timeslicing
1161 * if the queue demands it. (Normally, we do not submit if
1162 * ELSP[1] is already occupied, so must rely on timeslicing to
1163 * eject ELSP[0] in favour of the queue.)
1165 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1168 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1170 return PTR_ERR(obj);
1172 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1178 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1179 if (IS_ERR(vaddr)) {
1180 err = PTR_ERR(vaddr);
1184 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1188 err = i915_vma_sync(vma);
1192 for_each_engine(engine, gt, id) {
1193 struct i915_sched_attr attr = {
1194 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1196 struct i915_request *rq, *nop;
1198 if (!intel_engine_has_preemption(engine))
1201 engine_heartbeat_disable(engine);
1202 memset(vaddr, 0, PAGE_SIZE);
1204 /* ELSP[0]: semaphore wait */
1205 rq = semaphore_queue(engine, vma, 0);
1210 engine->schedule(rq, &attr);
1211 err = wait_for_submit(engine, rq, HZ / 2);
1213 pr_err("%s: Timed out trying to submit semaphores\n",
1218 /* ELSP[1]: nop request */
1219 nop = nop_request(engine);
1224 err = wait_for_submit(engine, nop, HZ / 2);
1225 i915_request_put(nop);
1227 pr_err("%s: Timed out trying to submit nop\n",
1232 GEM_BUG_ON(i915_request_completed(rq));
1233 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1235 /* Queue: semaphore signal, matching priority as semaphore */
1236 err = release_queue(engine, vma, 1, effective_prio(rq));
1240 /* Wait until we ack the release_queue and start timeslicing */
1243 intel_engine_flush_submission(engine);
1244 } while (READ_ONCE(engine->execlists.pending[0]));
1246 if (!READ_ONCE(engine->execlists.timer.expires) &&
1247 execlists_active(&engine->execlists) == rq &&
1248 !i915_request_completed(rq)) {
1249 struct drm_printer p =
1250 drm_info_printer(gt->i915->drm.dev);
1252 GEM_TRACE_ERR("%s: Failed to enable timeslicing!\n",
1254 intel_engine_dump(engine, &p,
1255 "%s\n", engine->name);
1258 memset(vaddr, 0xff, PAGE_SIZE);
1262 /* Timeslice every jiffy, so within 2 we should signal */
1263 if (i915_request_wait(rq, 0, timeslice_threshold(engine)) < 0) {
1264 struct drm_printer p =
1265 drm_info_printer(gt->i915->drm.dev);
1267 pr_err("%s: Failed to timeslice into queue\n",
1269 intel_engine_dump(engine, &p,
1270 "%s\n", engine->name);
1272 memset(vaddr, 0xff, PAGE_SIZE);
1276 i915_request_put(rq);
1278 engine_heartbeat_enable(engine);
1284 i915_vma_unpin(vma);
1286 i915_gem_object_unpin_map(obj);
1288 i915_gem_object_put(obj);
1292 static int live_busywait_preempt(void *arg)
1294 struct intel_gt *gt = arg;
1295 struct i915_gem_context *ctx_hi, *ctx_lo;
1296 struct intel_engine_cs *engine;
1297 struct drm_i915_gem_object *obj;
1298 struct i915_vma *vma;
1299 enum intel_engine_id id;
1304 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1305 * preempt the busywaits used to synchronise between rings.
1308 ctx_hi = kernel_context(gt->i915);
1311 ctx_hi->sched.priority =
1312 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1314 ctx_lo = kernel_context(gt->i915);
1317 ctx_lo->sched.priority =
1318 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1320 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1326 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
1332 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1338 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1342 err = i915_vma_sync(vma);
1346 for_each_engine(engine, gt, id) {
1347 struct i915_request *lo, *hi;
1348 struct igt_live_test t;
1351 if (!intel_engine_has_preemption(engine))
1354 if (!intel_engine_can_store_dword(engine))
1357 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1363 * We create two requests. The low priority request
1364 * busywaits on a semaphore (inside the ringbuffer where
1365 * is should be preemptible) and the high priority requests
1366 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1367 * allowing the first request to complete. If preemption
1368 * fails, we hang instead.
1371 lo = igt_request_alloc(ctx_lo, engine);
1377 cs = intel_ring_begin(lo, 8);
1380 i915_request_add(lo);
1384 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1385 *cs++ = i915_ggtt_offset(vma);
1389 /* XXX Do we need a flush + invalidate here? */
1391 *cs++ = MI_SEMAPHORE_WAIT |
1392 MI_SEMAPHORE_GLOBAL_GTT |
1394 MI_SEMAPHORE_SAD_EQ_SDD;
1396 *cs++ = i915_ggtt_offset(vma);
1399 intel_ring_advance(lo, cs);
1401 i915_request_get(lo);
1402 i915_request_add(lo);
1404 if (wait_for(READ_ONCE(*map), 10)) {
1405 i915_request_put(lo);
1410 /* Low priority request should be busywaiting now */
1411 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1412 i915_request_put(lo);
1413 pr_err("%s: Busywaiting request did not!\n",
1419 hi = igt_request_alloc(ctx_hi, engine);
1422 i915_request_put(lo);
1426 cs = intel_ring_begin(hi, 4);
1429 i915_request_add(hi);
1430 i915_request_put(lo);
1434 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1435 *cs++ = i915_ggtt_offset(vma);
1439 intel_ring_advance(hi, cs);
1440 i915_request_add(hi);
1442 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1443 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1445 pr_err("%s: Failed to preempt semaphore busywait!\n",
1448 intel_engine_dump(engine, &p, "%s\n", engine->name);
1451 i915_request_put(lo);
1452 intel_gt_set_wedged(gt);
1456 GEM_BUG_ON(READ_ONCE(*map));
1457 i915_request_put(lo);
1459 if (igt_live_test_end(&t)) {
1467 i915_vma_unpin(vma);
1469 i915_gem_object_unpin_map(obj);
1471 i915_gem_object_put(obj);
1473 kernel_context_close(ctx_lo);
1475 kernel_context_close(ctx_hi);
1479 static struct i915_request *
1480 spinner_create_request(struct igt_spinner *spin,
1481 struct i915_gem_context *ctx,
1482 struct intel_engine_cs *engine,
1485 struct intel_context *ce;
1486 struct i915_request *rq;
1488 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1490 return ERR_CAST(ce);
1492 rq = igt_spinner_create_request(spin, ce, arb);
1493 intel_context_put(ce);
1497 static int live_preempt(void *arg)
1499 struct intel_gt *gt = arg;
1500 struct i915_gem_context *ctx_hi, *ctx_lo;
1501 struct igt_spinner spin_hi, spin_lo;
1502 struct intel_engine_cs *engine;
1503 enum intel_engine_id id;
1506 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1509 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
1510 pr_err("Logical preemption supported, but not exposed\n");
1512 if (igt_spinner_init(&spin_hi, gt))
1515 if (igt_spinner_init(&spin_lo, gt))
1518 ctx_hi = kernel_context(gt->i915);
1521 ctx_hi->sched.priority =
1522 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1524 ctx_lo = kernel_context(gt->i915);
1527 ctx_lo->sched.priority =
1528 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1530 for_each_engine(engine, gt, id) {
1531 struct igt_live_test t;
1532 struct i915_request *rq;
1534 if (!intel_engine_has_preemption(engine))
1537 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1542 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1549 i915_request_add(rq);
1550 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1551 GEM_TRACE("lo spinner failed to start\n");
1553 intel_gt_set_wedged(gt);
1558 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1561 igt_spinner_end(&spin_lo);
1566 i915_request_add(rq);
1567 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1568 GEM_TRACE("hi spinner failed to start\n");
1570 intel_gt_set_wedged(gt);
1575 igt_spinner_end(&spin_hi);
1576 igt_spinner_end(&spin_lo);
1578 if (igt_live_test_end(&t)) {
1586 kernel_context_close(ctx_lo);
1588 kernel_context_close(ctx_hi);
1590 igt_spinner_fini(&spin_lo);
1592 igt_spinner_fini(&spin_hi);
1596 static int live_late_preempt(void *arg)
1598 struct intel_gt *gt = arg;
1599 struct i915_gem_context *ctx_hi, *ctx_lo;
1600 struct igt_spinner spin_hi, spin_lo;
1601 struct intel_engine_cs *engine;
1602 struct i915_sched_attr attr = {};
1603 enum intel_engine_id id;
1606 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1609 if (igt_spinner_init(&spin_hi, gt))
1612 if (igt_spinner_init(&spin_lo, gt))
1615 ctx_hi = kernel_context(gt->i915);
1619 ctx_lo = kernel_context(gt->i915);
1623 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1624 ctx_lo->sched.priority = I915_USER_PRIORITY(1);
1626 for_each_engine(engine, gt, id) {
1627 struct igt_live_test t;
1628 struct i915_request *rq;
1630 if (!intel_engine_has_preemption(engine))
1633 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1638 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1645 i915_request_add(rq);
1646 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1647 pr_err("First context failed to start\n");
1651 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1654 igt_spinner_end(&spin_lo);
1659 i915_request_add(rq);
1660 if (igt_wait_for_spinner(&spin_hi, rq)) {
1661 pr_err("Second context overtook first?\n");
1665 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1666 engine->schedule(rq, &attr);
1668 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1669 pr_err("High priority context failed to preempt the low priority context\n");
1674 igt_spinner_end(&spin_hi);
1675 igt_spinner_end(&spin_lo);
1677 if (igt_live_test_end(&t)) {
1685 kernel_context_close(ctx_lo);
1687 kernel_context_close(ctx_hi);
1689 igt_spinner_fini(&spin_lo);
1691 igt_spinner_fini(&spin_hi);
1695 igt_spinner_end(&spin_hi);
1696 igt_spinner_end(&spin_lo);
1697 intel_gt_set_wedged(gt);
1702 struct preempt_client {
1703 struct igt_spinner spin;
1704 struct i915_gem_context *ctx;
1707 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1709 c->ctx = kernel_context(gt->i915);
1713 if (igt_spinner_init(&c->spin, gt))
1719 kernel_context_close(c->ctx);
1723 static void preempt_client_fini(struct preempt_client *c)
1725 igt_spinner_fini(&c->spin);
1726 kernel_context_close(c->ctx);
1729 static int live_nopreempt(void *arg)
1731 struct intel_gt *gt = arg;
1732 struct intel_engine_cs *engine;
1733 struct preempt_client a, b;
1734 enum intel_engine_id id;
1738 * Verify that we can disable preemption for an individual request
1739 * that may be being observed and not want to be interrupted.
1742 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1745 if (preempt_client_init(gt, &a))
1747 if (preempt_client_init(gt, &b))
1749 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1751 for_each_engine(engine, gt, id) {
1752 struct i915_request *rq_a, *rq_b;
1754 if (!intel_engine_has_preemption(engine))
1757 engine->execlists.preempt_hang.count = 0;
1759 rq_a = spinner_create_request(&a.spin,
1763 err = PTR_ERR(rq_a);
1767 /* Low priority client, but unpreemptable! */
1768 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1770 i915_request_add(rq_a);
1771 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1772 pr_err("First client failed to start\n");
1776 rq_b = spinner_create_request(&b.spin,
1780 err = PTR_ERR(rq_b);
1784 i915_request_add(rq_b);
1786 /* B is much more important than A! (But A is unpreemptable.) */
1787 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
1789 /* Wait long enough for preemption and timeslicing */
1790 if (igt_wait_for_spinner(&b.spin, rq_b)) {
1791 pr_err("Second client started too early!\n");
1795 igt_spinner_end(&a.spin);
1797 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1798 pr_err("Second client failed to start\n");
1802 igt_spinner_end(&b.spin);
1804 if (engine->execlists.preempt_hang.count) {
1805 pr_err("Preemption recorded x%d; should have been suppressed!\n",
1806 engine->execlists.preempt_hang.count);
1811 if (igt_flush_test(gt->i915))
1817 preempt_client_fini(&b);
1819 preempt_client_fini(&a);
1823 igt_spinner_end(&b.spin);
1824 igt_spinner_end(&a.spin);
1825 intel_gt_set_wedged(gt);
1830 struct live_preempt_cancel {
1831 struct intel_engine_cs *engine;
1832 struct preempt_client a, b;
1835 static int __cancel_active0(struct live_preempt_cancel *arg)
1837 struct i915_request *rq;
1838 struct igt_live_test t;
1841 /* Preempt cancel of ELSP0 */
1842 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1843 if (igt_live_test_begin(&t, arg->engine->i915,
1844 __func__, arg->engine->name))
1847 rq = spinner_create_request(&arg->a.spin,
1848 arg->a.ctx, arg->engine,
1853 clear_bit(CONTEXT_BANNED, &rq->context->flags);
1854 i915_request_get(rq);
1855 i915_request_add(rq);
1856 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1861 intel_context_set_banned(rq->context);
1862 err = intel_engine_pulse(arg->engine);
1866 err = wait_for_reset(arg->engine, rq, HZ / 2);
1868 pr_err("Cancelled inflight0 request did not reset\n");
1873 i915_request_put(rq);
1874 if (igt_live_test_end(&t))
1879 static int __cancel_active1(struct live_preempt_cancel *arg)
1881 struct i915_request *rq[2] = {};
1882 struct igt_live_test t;
1885 /* Preempt cancel of ELSP1 */
1886 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1887 if (igt_live_test_begin(&t, arg->engine->i915,
1888 __func__, arg->engine->name))
1891 rq[0] = spinner_create_request(&arg->a.spin,
1892 arg->a.ctx, arg->engine,
1893 MI_NOOP); /* no preemption */
1895 return PTR_ERR(rq[0]);
1897 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
1898 i915_request_get(rq[0]);
1899 i915_request_add(rq[0]);
1900 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1905 rq[1] = spinner_create_request(&arg->b.spin,
1906 arg->b.ctx, arg->engine,
1908 if (IS_ERR(rq[1])) {
1909 err = PTR_ERR(rq[1]);
1913 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
1914 i915_request_get(rq[1]);
1915 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1916 i915_request_add(rq[1]);
1920 intel_context_set_banned(rq[1]->context);
1921 err = intel_engine_pulse(arg->engine);
1925 igt_spinner_end(&arg->a.spin);
1926 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
1930 if (rq[0]->fence.error != 0) {
1931 pr_err("Normal inflight0 request did not complete\n");
1936 if (rq[1]->fence.error != -EIO) {
1937 pr_err("Cancelled inflight1 request did not report -EIO\n");
1943 i915_request_put(rq[1]);
1944 i915_request_put(rq[0]);
1945 if (igt_live_test_end(&t))
1950 static int __cancel_queued(struct live_preempt_cancel *arg)
1952 struct i915_request *rq[3] = {};
1953 struct igt_live_test t;
1956 /* Full ELSP and one in the wings */
1957 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1958 if (igt_live_test_begin(&t, arg->engine->i915,
1959 __func__, arg->engine->name))
1962 rq[0] = spinner_create_request(&arg->a.spin,
1963 arg->a.ctx, arg->engine,
1966 return PTR_ERR(rq[0]);
1968 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
1969 i915_request_get(rq[0]);
1970 i915_request_add(rq[0]);
1971 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1976 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
1977 if (IS_ERR(rq[1])) {
1978 err = PTR_ERR(rq[1]);
1982 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
1983 i915_request_get(rq[1]);
1984 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1985 i915_request_add(rq[1]);
1989 rq[2] = spinner_create_request(&arg->b.spin,
1990 arg->a.ctx, arg->engine,
1992 if (IS_ERR(rq[2])) {
1993 err = PTR_ERR(rq[2]);
1997 i915_request_get(rq[2]);
1998 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
1999 i915_request_add(rq[2]);
2003 intel_context_set_banned(rq[2]->context);
2004 err = intel_engine_pulse(arg->engine);
2008 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2012 if (rq[0]->fence.error != -EIO) {
2013 pr_err("Cancelled inflight0 request did not report -EIO\n");
2018 if (rq[1]->fence.error != 0) {
2019 pr_err("Normal inflight1 request did not complete\n");
2024 if (rq[2]->fence.error != -EIO) {
2025 pr_err("Cancelled queued request did not report -EIO\n");
2031 i915_request_put(rq[2]);
2032 i915_request_put(rq[1]);
2033 i915_request_put(rq[0]);
2034 if (igt_live_test_end(&t))
2039 static int __cancel_hostile(struct live_preempt_cancel *arg)
2041 struct i915_request *rq;
2044 /* Preempt cancel non-preemptible spinner in ELSP0 */
2045 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2048 if (!intel_has_reset_engine(arg->engine->gt))
2051 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2052 rq = spinner_create_request(&arg->a.spin,
2053 arg->a.ctx, arg->engine,
2054 MI_NOOP); /* preemption disabled */
2058 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2059 i915_request_get(rq);
2060 i915_request_add(rq);
2061 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2066 intel_context_set_banned(rq->context);
2067 err = intel_engine_pulse(arg->engine); /* force reset */
2071 err = wait_for_reset(arg->engine, rq, HZ / 2);
2073 pr_err("Cancelled inflight0 request did not reset\n");
2078 i915_request_put(rq);
2079 if (igt_flush_test(arg->engine->i915))
2084 static int live_preempt_cancel(void *arg)
2086 struct intel_gt *gt = arg;
2087 struct live_preempt_cancel data;
2088 enum intel_engine_id id;
2092 * To cancel an inflight context, we need to first remove it from the
2093 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2096 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2099 if (preempt_client_init(gt, &data.a))
2101 if (preempt_client_init(gt, &data.b))
2104 for_each_engine(data.engine, gt, id) {
2105 if (!intel_engine_has_preemption(data.engine))
2108 err = __cancel_active0(&data);
2112 err = __cancel_active1(&data);
2116 err = __cancel_queued(&data);
2120 err = __cancel_hostile(&data);
2127 preempt_client_fini(&data.b);
2129 preempt_client_fini(&data.a);
2134 igt_spinner_end(&data.b.spin);
2135 igt_spinner_end(&data.a.spin);
2136 intel_gt_set_wedged(gt);
2140 static int live_suppress_self_preempt(void *arg)
2142 struct intel_gt *gt = arg;
2143 struct intel_engine_cs *engine;
2144 struct i915_sched_attr attr = {
2145 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
2147 struct preempt_client a, b;
2148 enum intel_engine_id id;
2152 * Verify that if a preemption request does not cause a change in
2153 * the current execution order, the preempt-to-idle injection is
2154 * skipped and that we do not accidentally apply it after the CS
2158 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2161 if (intel_uc_uses_guc_submission(>->uc))
2162 return 0; /* presume black blox */
2164 if (intel_vgpu_active(gt->i915))
2165 return 0; /* GVT forces single port & request submission */
2167 if (preempt_client_init(gt, &a))
2169 if (preempt_client_init(gt, &b))
2172 for_each_engine(engine, gt, id) {
2173 struct i915_request *rq_a, *rq_b;
2176 if (!intel_engine_has_preemption(engine))
2179 if (igt_flush_test(gt->i915))
2182 intel_engine_pm_get(engine);
2183 engine->execlists.preempt_hang.count = 0;
2185 rq_a = spinner_create_request(&a.spin,
2189 err = PTR_ERR(rq_a);
2190 intel_engine_pm_put(engine);
2194 i915_request_add(rq_a);
2195 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2196 pr_err("First client failed to start\n");
2197 intel_engine_pm_put(engine);
2201 /* Keep postponing the timer to avoid premature slicing */
2202 mod_timer(&engine->execlists.timer, jiffies + HZ);
2203 for (depth = 0; depth < 8; depth++) {
2204 rq_b = spinner_create_request(&b.spin,
2208 err = PTR_ERR(rq_b);
2209 intel_engine_pm_put(engine);
2212 i915_request_add(rq_b);
2214 GEM_BUG_ON(i915_request_completed(rq_a));
2215 engine->schedule(rq_a, &attr);
2216 igt_spinner_end(&a.spin);
2218 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2219 pr_err("Second client failed to start\n");
2220 intel_engine_pm_put(engine);
2227 igt_spinner_end(&a.spin);
2229 if (engine->execlists.preempt_hang.count) {
2230 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2232 engine->execlists.preempt_hang.count,
2234 intel_engine_pm_put(engine);
2239 intel_engine_pm_put(engine);
2240 if (igt_flush_test(gt->i915))
2246 preempt_client_fini(&b);
2248 preempt_client_fini(&a);
2252 igt_spinner_end(&b.spin);
2253 igt_spinner_end(&a.spin);
2254 intel_gt_set_wedged(gt);
2259 static int __i915_sw_fence_call
2260 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
2265 static struct i915_request *dummy_request(struct intel_engine_cs *engine)
2267 struct i915_request *rq;
2269 rq = kzalloc(sizeof(*rq), GFP_KERNEL);
2273 rq->engine = engine;
2275 spin_lock_init(&rq->lock);
2276 INIT_LIST_HEAD(&rq->fence.cb_list);
2277 rq->fence.lock = &rq->lock;
2278 rq->fence.ops = &i915_fence_ops;
2280 i915_sched_node_init(&rq->sched);
2282 /* mark this request as permanently incomplete */
2283 rq->fence.seqno = 1;
2284 BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
2285 rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
2286 GEM_BUG_ON(i915_request_completed(rq));
2288 i915_sw_fence_init(&rq->submit, dummy_notify);
2289 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
2291 spin_lock_init(&rq->lock);
2292 rq->fence.lock = &rq->lock;
2293 INIT_LIST_HEAD(&rq->fence.cb_list);
2298 static void dummy_request_free(struct i915_request *dummy)
2300 /* We have to fake the CS interrupt to kick the next request */
2301 i915_sw_fence_commit(&dummy->submit);
2303 i915_request_mark_complete(dummy);
2304 dma_fence_signal(&dummy->fence);
2306 i915_sched_node_fini(&dummy->sched);
2307 i915_sw_fence_fini(&dummy->submit);
2309 dma_fence_free(&dummy->fence);
2312 static int live_suppress_wait_preempt(void *arg)
2314 struct intel_gt *gt = arg;
2315 struct preempt_client client[4];
2316 struct i915_request *rq[ARRAY_SIZE(client)] = {};
2317 struct intel_engine_cs *engine;
2318 enum intel_engine_id id;
2323 * Waiters are given a little priority nudge, but not enough
2324 * to actually cause any preemption. Double check that we do
2325 * not needlessly generate preempt-to-idle cycles.
2328 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2331 if (preempt_client_init(gt, &client[0])) /* ELSP[0] */
2333 if (preempt_client_init(gt, &client[1])) /* ELSP[1] */
2335 if (preempt_client_init(gt, &client[2])) /* head of queue */
2337 if (preempt_client_init(gt, &client[3])) /* bystander */
2340 for_each_engine(engine, gt, id) {
2343 if (!intel_engine_has_preemption(engine))
2346 if (!engine->emit_init_breadcrumb)
2349 for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
2350 struct i915_request *dummy;
2352 engine->execlists.preempt_hang.count = 0;
2354 dummy = dummy_request(engine);
2358 for (i = 0; i < ARRAY_SIZE(client); i++) {
2359 struct i915_request *this;
2361 this = spinner_create_request(&client[i].spin,
2362 client[i].ctx, engine,
2365 err = PTR_ERR(this);
2369 /* Disable NEWCLIENT promotion */
2370 __i915_active_fence_set(&i915_request_timeline(this)->last_request,
2373 rq[i] = i915_request_get(this);
2374 i915_request_add(this);
2377 dummy_request_free(dummy);
2379 GEM_BUG_ON(i915_request_completed(rq[0]));
2380 if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
2381 pr_err("%s: First client failed to start\n",
2385 GEM_BUG_ON(!i915_request_started(rq[0]));
2387 if (i915_request_wait(rq[depth],
2390 pr_err("%s: Waiter depth:%d completed!\n",
2391 engine->name, depth);
2395 for (i = 0; i < ARRAY_SIZE(client); i++) {
2396 igt_spinner_end(&client[i].spin);
2397 i915_request_put(rq[i]);
2401 if (igt_flush_test(gt->i915))
2404 if (engine->execlists.preempt_hang.count) {
2405 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
2407 engine->execlists.preempt_hang.count,
2417 preempt_client_fini(&client[3]);
2419 preempt_client_fini(&client[2]);
2421 preempt_client_fini(&client[1]);
2423 preempt_client_fini(&client[0]);
2427 for (i = 0; i < ARRAY_SIZE(client); i++) {
2428 igt_spinner_end(&client[i].spin);
2429 i915_request_put(rq[i]);
2431 intel_gt_set_wedged(gt);
2436 static int live_chain_preempt(void *arg)
2438 struct intel_gt *gt = arg;
2439 struct intel_engine_cs *engine;
2440 struct preempt_client hi, lo;
2441 enum intel_engine_id id;
2445 * Build a chain AB...BA between two contexts (A, B) and request
2446 * preemption of the last request. It should then complete before
2447 * the previously submitted spinner in B.
2450 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2453 if (preempt_client_init(gt, &hi))
2456 if (preempt_client_init(gt, &lo))
2459 for_each_engine(engine, gt, id) {
2460 struct i915_sched_attr attr = {
2461 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
2463 struct igt_live_test t;
2464 struct i915_request *rq;
2465 int ring_size, count, i;
2467 if (!intel_engine_has_preemption(engine))
2470 rq = spinner_create_request(&lo.spin,
2476 i915_request_get(rq);
2477 i915_request_add(rq);
2479 ring_size = rq->wa_tail - rq->head;
2481 ring_size += rq->ring->size;
2482 ring_size = rq->ring->size / ring_size;
2483 pr_debug("%s(%s): Using maximum of %d requests\n",
2484 __func__, engine->name, ring_size);
2486 igt_spinner_end(&lo.spin);
2487 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2488 pr_err("Timed out waiting to flush %s\n", engine->name);
2489 i915_request_put(rq);
2492 i915_request_put(rq);
2494 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2499 for_each_prime_number_from(count, 1, ring_size) {
2500 rq = spinner_create_request(&hi.spin,
2505 i915_request_add(rq);
2506 if (!igt_wait_for_spinner(&hi.spin, rq))
2509 rq = spinner_create_request(&lo.spin,
2514 i915_request_add(rq);
2516 for (i = 0; i < count; i++) {
2517 rq = igt_request_alloc(lo.ctx, engine);
2520 i915_request_add(rq);
2523 rq = igt_request_alloc(hi.ctx, engine);
2527 i915_request_get(rq);
2528 i915_request_add(rq);
2529 engine->schedule(rq, &attr);
2531 igt_spinner_end(&hi.spin);
2532 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2533 struct drm_printer p =
2534 drm_info_printer(gt->i915->drm.dev);
2536 pr_err("Failed to preempt over chain of %d\n",
2538 intel_engine_dump(engine, &p,
2539 "%s\n", engine->name);
2540 i915_request_put(rq);
2543 igt_spinner_end(&lo.spin);
2544 i915_request_put(rq);
2546 rq = igt_request_alloc(lo.ctx, engine);
2550 i915_request_get(rq);
2551 i915_request_add(rq);
2553 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2554 struct drm_printer p =
2555 drm_info_printer(gt->i915->drm.dev);
2557 pr_err("Failed to flush low priority chain of %d requests\n",
2559 intel_engine_dump(engine, &p,
2560 "%s\n", engine->name);
2562 i915_request_put(rq);
2565 i915_request_put(rq);
2568 if (igt_live_test_end(&t)) {
2576 preempt_client_fini(&lo);
2578 preempt_client_fini(&hi);
2582 igt_spinner_end(&hi.spin);
2583 igt_spinner_end(&lo.spin);
2584 intel_gt_set_wedged(gt);
2589 static int create_gang(struct intel_engine_cs *engine,
2590 struct i915_request **prev)
2592 struct drm_i915_gem_object *obj;
2593 struct intel_context *ce;
2594 struct i915_request *rq;
2595 struct i915_vma *vma;
2599 ce = intel_context_create(engine);
2603 obj = i915_gem_object_create_internal(engine->i915, 4096);
2609 vma = i915_vma_instance(obj, ce->vm, NULL);
2615 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2619 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2623 /* Semaphore target: spin until zero */
2624 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2626 *cs++ = MI_SEMAPHORE_WAIT |
2628 MI_SEMAPHORE_SAD_EQ_SDD;
2630 *cs++ = lower_32_bits(vma->node.start);
2631 *cs++ = upper_32_bits(vma->node.start);
2634 u64 offset = (*prev)->batch->node.start;
2636 /* Terminate the spinner in the next lower priority batch. */
2637 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2638 *cs++ = lower_32_bits(offset);
2639 *cs++ = upper_32_bits(offset);
2643 *cs++ = MI_BATCH_BUFFER_END;
2644 i915_gem_object_flush_map(obj);
2645 i915_gem_object_unpin_map(obj);
2647 rq = intel_context_create_request(ce);
2651 rq->batch = i915_vma_get(vma);
2652 i915_request_get(rq);
2655 err = i915_request_await_object(rq, vma->obj, false);
2657 err = i915_vma_move_to_active(vma, rq, 0);
2659 err = rq->engine->emit_bb_start(rq,
2662 i915_vma_unlock(vma);
2663 i915_request_add(rq);
2667 i915_gem_object_put(obj);
2668 intel_context_put(ce);
2670 rq->client_link.next = &(*prev)->client_link;
2675 i915_vma_put(rq->batch);
2676 i915_request_put(rq);
2678 i915_gem_object_put(obj);
2680 intel_context_put(ce);
2684 static int live_preempt_gang(void *arg)
2686 struct intel_gt *gt = arg;
2687 struct intel_engine_cs *engine;
2688 enum intel_engine_id id;
2690 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2694 * Build as long a chain of preempters as we can, with each
2695 * request higher priority than the last. Once we are ready, we release
2696 * the last batch which then precolates down the chain, each releasing
2697 * the next oldest in turn. The intent is to simply push as hard as we
2698 * can with the number of preemptions, trying to exceed narrow HW
2699 * limits. At a minimum, we insist that we can sort all the user
2700 * high priority levels into execution order.
2703 for_each_engine(engine, gt, id) {
2704 struct i915_request *rq = NULL;
2705 struct igt_live_test t;
2706 IGT_TIMEOUT(end_time);
2711 if (!intel_engine_has_preemption(engine))
2714 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2718 struct i915_sched_attr attr = {
2719 .priority = I915_USER_PRIORITY(prio++),
2722 err = create_gang(engine, &rq);
2726 /* Submit each spinner at increasing priority */
2727 engine->schedule(rq, &attr);
2729 if (prio <= I915_PRIORITY_MAX)
2732 if (prio > (INT_MAX >> I915_USER_PRIORITY_SHIFT))
2735 if (__igt_timeout(end_time, NULL))
2738 pr_debug("%s: Preempt chain of %d requests\n",
2739 engine->name, prio);
2742 * Such that the last spinner is the highest priority and
2743 * should execute first. When that spinner completes,
2744 * it will terminate the next lowest spinner until there
2745 * are no more spinners and the gang is complete.
2747 cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
2750 i915_gem_object_unpin_map(rq->batch->obj);
2753 intel_gt_set_wedged(gt);
2756 while (rq) { /* wait for each rq from highest to lowest prio */
2757 struct i915_request *n =
2758 list_next_entry(rq, client_link);
2760 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
2761 struct drm_printer p =
2762 drm_info_printer(engine->i915->drm.dev);
2764 pr_err("Failed to flush chain of %d requests, at %d\n",
2765 prio, rq_prio(rq) >> I915_USER_PRIORITY_SHIFT);
2766 intel_engine_dump(engine, &p,
2767 "%s\n", engine->name);
2772 i915_vma_put(rq->batch);
2773 i915_request_put(rq);
2777 if (igt_live_test_end(&t))
2786 static struct i915_vma *
2787 create_gpr_user(struct intel_engine_cs *engine,
2788 struct i915_vma *result,
2789 unsigned int offset)
2791 struct drm_i915_gem_object *obj;
2792 struct i915_vma *vma;
2797 obj = i915_gem_object_create_internal(engine->i915, 4096);
2799 return ERR_CAST(obj);
2801 vma = i915_vma_instance(obj, result->vm, NULL);
2803 i915_gem_object_put(obj);
2807 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2810 return ERR_PTR(err);
2813 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2816 return ERR_CAST(cs);
2819 /* All GPR are clear for new contexts. We use GPR(0) as a constant */
2820 *cs++ = MI_LOAD_REGISTER_IMM(1);
2821 *cs++ = CS_GPR(engine, 0);
2824 for (i = 1; i < NUM_GPR; i++) {
2830 * As we read and write into the context saved GPR[i], if
2831 * we restart this batch buffer from an earlier point, we
2832 * will repeat the increment and store a value > 1.
2835 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
2836 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
2837 *cs++ = MI_MATH_ADD;
2838 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
2840 addr = result->node.start + offset + i * sizeof(*cs);
2841 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
2842 *cs++ = CS_GPR(engine, 2 * i);
2843 *cs++ = lower_32_bits(addr);
2844 *cs++ = upper_32_bits(addr);
2846 *cs++ = MI_SEMAPHORE_WAIT |
2848 MI_SEMAPHORE_SAD_GTE_SDD;
2850 *cs++ = lower_32_bits(result->node.start);
2851 *cs++ = upper_32_bits(result->node.start);
2854 *cs++ = MI_BATCH_BUFFER_END;
2855 i915_gem_object_flush_map(obj);
2856 i915_gem_object_unpin_map(obj);
2861 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
2863 struct drm_i915_gem_object *obj;
2864 struct i915_vma *vma;
2867 obj = i915_gem_object_create_internal(gt->i915, sz);
2869 return ERR_CAST(obj);
2871 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
2873 i915_gem_object_put(obj);
2877 err = i915_ggtt_pin(vma, 0, 0);
2880 return ERR_PTR(err);
2886 static struct i915_request *
2887 create_gpr_client(struct intel_engine_cs *engine,
2888 struct i915_vma *global,
2889 unsigned int offset)
2891 struct i915_vma *batch, *vma;
2892 struct intel_context *ce;
2893 struct i915_request *rq;
2896 ce = intel_context_create(engine);
2898 return ERR_CAST(ce);
2900 vma = i915_vma_instance(global->obj, ce->vm, NULL);
2906 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2910 batch = create_gpr_user(engine, vma, offset);
2911 if (IS_ERR(batch)) {
2912 err = PTR_ERR(batch);
2916 rq = intel_context_create_request(ce);
2923 err = i915_request_await_object(rq, vma->obj, false);
2925 err = i915_vma_move_to_active(vma, rq, 0);
2926 i915_vma_unlock(vma);
2928 i915_vma_lock(batch);
2930 err = i915_request_await_object(rq, batch->obj, false);
2932 err = i915_vma_move_to_active(batch, rq, 0);
2934 err = rq->engine->emit_bb_start(rq,
2937 i915_vma_unlock(batch);
2938 i915_vma_unpin(batch);
2941 i915_request_get(rq);
2942 i915_request_add(rq);
2945 i915_vma_put(batch);
2947 i915_vma_unpin(vma);
2949 intel_context_put(ce);
2950 return err ? ERR_PTR(err) : rq;
2953 static int preempt_user(struct intel_engine_cs *engine,
2954 struct i915_vma *global,
2957 struct i915_sched_attr attr = {
2958 .priority = I915_PRIORITY_MAX
2960 struct i915_request *rq;
2964 rq = intel_engine_create_kernel_request(engine);
2968 cs = intel_ring_begin(rq, 4);
2970 i915_request_add(rq);
2974 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
2975 *cs++ = i915_ggtt_offset(global);
2979 intel_ring_advance(rq, cs);
2981 i915_request_get(rq);
2982 i915_request_add(rq);
2984 engine->schedule(rq, &attr);
2986 if (i915_request_wait(rq, 0, HZ / 2) < 0)
2988 i915_request_put(rq);
2993 static int live_preempt_user(void *arg)
2995 struct intel_gt *gt = arg;
2996 struct intel_engine_cs *engine;
2997 struct i915_vma *global;
2998 enum intel_engine_id id;
3002 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
3006 * In our other tests, we look at preemption in carefully
3007 * controlled conditions in the ringbuffer. Since most of the
3008 * time is spent in user batches, most of our preemptions naturally
3009 * occur there. We want to verify that when we preempt inside a batch
3010 * we continue on from the current instruction and do not roll back
3011 * to the start, or another earlier arbitration point.
3013 * To verify this, we create a batch which is a mixture of
3014 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3015 * a few preempting contexts thrown into the mix, we look for any
3016 * repeated instructions (which show up as incorrect values).
3019 global = create_global(gt, 4096);
3021 return PTR_ERR(global);
3023 result = i915_gem_object_pin_map(global->obj, I915_MAP_WC);
3024 if (IS_ERR(result)) {
3025 i915_vma_unpin_and_release(&global, 0);
3026 return PTR_ERR(result);
3029 for_each_engine(engine, gt, id) {
3030 struct i915_request *client[3] = {};
3031 struct igt_live_test t;
3034 if (!intel_engine_has_preemption(engine))
3037 if (IS_GEN(gt->i915, 8) && engine->class != RENDER_CLASS)
3038 continue; /* we need per-context GPR */
3040 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3045 memset(result, 0, 4096);
3047 for (i = 0; i < ARRAY_SIZE(client); i++) {
3048 struct i915_request *rq;
3050 rq = create_gpr_client(engine, global,
3051 NUM_GPR * i * sizeof(u32));
3058 /* Continuously preempt the set of 3 running contexts */
3059 for (i = 1; i <= NUM_GPR; i++) {
3060 err = preempt_user(engine, global, i);
3065 if (READ_ONCE(result[0]) != NUM_GPR) {
3066 pr_err("%s: Failed to release semaphore\n",
3072 for (i = 0; i < ARRAY_SIZE(client); i++) {
3075 if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3080 for (gpr = 1; gpr < NUM_GPR; gpr++) {
3081 if (result[NUM_GPR * i + gpr] != 1) {
3082 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3084 i, gpr, result[NUM_GPR * i + gpr]);
3092 for (i = 0; i < ARRAY_SIZE(client); i++) {
3096 i915_request_put(client[i]);
3099 /* Flush the semaphores on error */
3100 smp_store_mb(result[0], -1);
3101 if (igt_live_test_end(&t))
3107 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3111 static int live_preempt_timeout(void *arg)
3113 struct intel_gt *gt = arg;
3114 struct i915_gem_context *ctx_hi, *ctx_lo;
3115 struct igt_spinner spin_lo;
3116 struct intel_engine_cs *engine;
3117 enum intel_engine_id id;
3121 * Check that we force preemption to occur by cancelling the previous
3122 * context if it refuses to yield the GPU.
3124 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
3127 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
3130 if (!intel_has_reset_engine(gt))
3133 if (igt_spinner_init(&spin_lo, gt))
3136 ctx_hi = kernel_context(gt->i915);
3139 ctx_hi->sched.priority =
3140 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
3142 ctx_lo = kernel_context(gt->i915);
3145 ctx_lo->sched.priority =
3146 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
3148 for_each_engine(engine, gt, id) {
3149 unsigned long saved_timeout;
3150 struct i915_request *rq;
3152 if (!intel_engine_has_preemption(engine))
3155 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3156 MI_NOOP); /* preemption disabled */
3162 i915_request_add(rq);
3163 if (!igt_wait_for_spinner(&spin_lo, rq)) {
3164 intel_gt_set_wedged(gt);
3169 rq = igt_request_alloc(ctx_hi, engine);
3171 igt_spinner_end(&spin_lo);
3176 /* Flush the previous CS ack before changing timeouts */
3177 while (READ_ONCE(engine->execlists.pending[0]))
3180 saved_timeout = engine->props.preempt_timeout_ms;
3181 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3183 i915_request_get(rq);
3184 i915_request_add(rq);
3186 intel_engine_flush_submission(engine);
3187 engine->props.preempt_timeout_ms = saved_timeout;
3189 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3190 intel_gt_set_wedged(gt);
3191 i915_request_put(rq);
3196 igt_spinner_end(&spin_lo);
3197 i915_request_put(rq);
3202 kernel_context_close(ctx_lo);
3204 kernel_context_close(ctx_hi);
3206 igt_spinner_fini(&spin_lo);
3210 static int random_range(struct rnd_state *rnd, int min, int max)
3212 return i915_prandom_u32_max_state(max - min, rnd) + min;
3215 static int random_priority(struct rnd_state *rnd)
3217 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3220 struct preempt_smoke {
3221 struct intel_gt *gt;
3222 struct i915_gem_context **contexts;
3223 struct intel_engine_cs *engine;
3224 struct drm_i915_gem_object *batch;
3225 unsigned int ncontext;
3226 struct rnd_state prng;
3227 unsigned long count;
3230 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3232 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3236 static int smoke_submit(struct preempt_smoke *smoke,
3237 struct i915_gem_context *ctx, int prio,
3238 struct drm_i915_gem_object *batch)
3240 struct i915_request *rq;
3241 struct i915_vma *vma = NULL;
3245 struct i915_address_space *vm;
3247 vm = i915_gem_context_get_vm_rcu(ctx);
3248 vma = i915_vma_instance(batch, vm, NULL);
3251 return PTR_ERR(vma);
3253 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3258 ctx->sched.priority = prio;
3260 rq = igt_request_alloc(ctx, smoke->engine);
3268 err = i915_request_await_object(rq, vma->obj, false);
3270 err = i915_vma_move_to_active(vma, rq, 0);
3272 err = rq->engine->emit_bb_start(rq,
3275 i915_vma_unlock(vma);
3278 i915_request_add(rq);
3282 i915_vma_unpin(vma);
3287 static int smoke_crescendo_thread(void *arg)
3289 struct preempt_smoke *smoke = arg;
3290 IGT_TIMEOUT(end_time);
3291 unsigned long count;
3295 struct i915_gem_context *ctx = smoke_context(smoke);
3298 err = smoke_submit(smoke,
3299 ctx, count % I915_PRIORITY_MAX,
3305 } while (!__igt_timeout(end_time, NULL));
3307 smoke->count = count;
3311 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3312 #define BATCH BIT(0)
3314 struct task_struct *tsk[I915_NUM_ENGINES] = {};
3315 struct preempt_smoke arg[I915_NUM_ENGINES];
3316 struct intel_engine_cs *engine;
3317 enum intel_engine_id id;
3318 unsigned long count;
3321 for_each_engine(engine, smoke->gt, id) {
3323 arg[id].engine = engine;
3324 if (!(flags & BATCH))
3325 arg[id].batch = NULL;
3328 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
3329 "igt/smoke:%d", id);
3330 if (IS_ERR(tsk[id])) {
3331 err = PTR_ERR(tsk[id]);
3334 get_task_struct(tsk[id]);
3337 yield(); /* start all threads before we kthread_stop() */
3340 for_each_engine(engine, smoke->gt, id) {
3343 if (IS_ERR_OR_NULL(tsk[id]))
3346 status = kthread_stop(tsk[id]);
3350 count += arg[id].count;
3352 put_task_struct(tsk[id]);
3355 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3357 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
3361 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3363 enum intel_engine_id id;
3364 IGT_TIMEOUT(end_time);
3365 unsigned long count;
3369 for_each_engine(smoke->engine, smoke->gt, id) {
3370 struct i915_gem_context *ctx = smoke_context(smoke);
3373 err = smoke_submit(smoke,
3374 ctx, random_priority(&smoke->prng),
3375 flags & BATCH ? smoke->batch : NULL);
3381 } while (!__igt_timeout(end_time, NULL));
3383 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3385 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
3389 static int live_preempt_smoke(void *arg)
3391 struct preempt_smoke smoke = {
3393 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3396 const unsigned int phase[] = { 0, BATCH };
3397 struct igt_live_test t;
3402 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.gt->i915))
3405 smoke.contexts = kmalloc_array(smoke.ncontext,
3406 sizeof(*smoke.contexts),
3408 if (!smoke.contexts)
3412 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3413 if (IS_ERR(smoke.batch)) {
3414 err = PTR_ERR(smoke.batch);
3418 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
3423 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3424 cs[n] = MI_ARB_CHECK;
3425 cs[n] = MI_BATCH_BUFFER_END;
3426 i915_gem_object_flush_map(smoke.batch);
3427 i915_gem_object_unpin_map(smoke.batch);
3429 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3434 for (n = 0; n < smoke.ncontext; n++) {
3435 smoke.contexts[n] = kernel_context(smoke.gt->i915);
3436 if (!smoke.contexts[n])
3440 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3441 err = smoke_crescendo(&smoke, phase[n]);
3445 err = smoke_random(&smoke, phase[n]);
3451 if (igt_live_test_end(&t))
3454 for (n = 0; n < smoke.ncontext; n++) {
3455 if (!smoke.contexts[n])
3457 kernel_context_close(smoke.contexts[n]);
3461 i915_gem_object_put(smoke.batch);
3463 kfree(smoke.contexts);
3468 static int nop_virtual_engine(struct intel_gt *gt,
3469 struct intel_engine_cs **siblings,
3470 unsigned int nsibling,
3473 #define CHAIN BIT(0)
3475 IGT_TIMEOUT(end_time);
3476 struct i915_request *request[16] = {};
3477 struct intel_context *ve[16];
3478 unsigned long n, prime, nc;
3479 struct igt_live_test t;
3480 ktime_t times[2] = {};
3483 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3485 for (n = 0; n < nctx; n++) {
3486 ve[n] = intel_execlists_create_virtual(siblings, nsibling);
3487 if (IS_ERR(ve[n])) {
3488 err = PTR_ERR(ve[n]);
3493 err = intel_context_pin(ve[n]);
3495 intel_context_put(ve[n]);
3501 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3505 for_each_prime_number_from(prime, 1, 8192) {
3506 times[1] = ktime_get_raw();
3508 if (flags & CHAIN) {
3509 for (nc = 0; nc < nctx; nc++) {
3510 for (n = 0; n < prime; n++) {
3511 struct i915_request *rq;
3513 rq = i915_request_create(ve[nc]);
3520 i915_request_put(request[nc]);
3521 request[nc] = i915_request_get(rq);
3522 i915_request_add(rq);
3526 for (n = 0; n < prime; n++) {
3527 for (nc = 0; nc < nctx; nc++) {
3528 struct i915_request *rq;
3530 rq = i915_request_create(ve[nc]);
3537 i915_request_put(request[nc]);
3538 request[nc] = i915_request_get(rq);
3539 i915_request_add(rq);
3544 for (nc = 0; nc < nctx; nc++) {
3545 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3546 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3547 __func__, ve[0]->engine->name,
3548 request[nc]->fence.context,
3549 request[nc]->fence.seqno);
3551 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3552 __func__, ve[0]->engine->name,
3553 request[nc]->fence.context,
3554 request[nc]->fence.seqno);
3556 intel_gt_set_wedged(gt);
3561 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3563 times[0] = times[1];
3565 for (nc = 0; nc < nctx; nc++) {
3566 i915_request_put(request[nc]);
3570 if (__igt_timeout(end_time, NULL))
3574 err = igt_live_test_end(&t);
3578 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3579 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3580 prime, div64_u64(ktime_to_ns(times[1]), prime));
3583 if (igt_flush_test(gt->i915))
3586 for (nc = 0; nc < nctx; nc++) {
3587 i915_request_put(request[nc]);
3588 intel_context_unpin(ve[nc]);
3589 intel_context_put(ve[nc]);
3594 static int live_virtual_engine(void *arg)
3596 struct intel_gt *gt = arg;
3597 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3598 struct intel_engine_cs *engine;
3599 enum intel_engine_id id;
3600 unsigned int class, inst;
3603 if (intel_uc_uses_guc_submission(>->uc))
3606 for_each_engine(engine, gt, id) {
3607 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3609 pr_err("Failed to wrap engine %s: err=%d\n",
3615 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3619 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3620 if (!gt->engine_class[class][inst])
3623 siblings[nsibling++] = gt->engine_class[class][inst];
3628 for (n = 1; n <= nsibling + 1; n++) {
3629 err = nop_virtual_engine(gt, siblings, nsibling,
3635 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3643 static int mask_virtual_engine(struct intel_gt *gt,
3644 struct intel_engine_cs **siblings,
3645 unsigned int nsibling)
3647 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3648 struct intel_context *ve;
3649 struct igt_live_test t;
3654 * Check that by setting the execution mask on a request, we can
3655 * restrict it to our desired engine within the virtual engine.
3658 ve = intel_execlists_create_virtual(siblings, nsibling);
3664 err = intel_context_pin(ve);
3668 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3672 for (n = 0; n < nsibling; n++) {
3673 request[n] = i915_request_create(ve);
3674 if (IS_ERR(request[n])) {
3675 err = PTR_ERR(request[n]);
3680 /* Reverse order as it's more likely to be unnatural */
3681 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3683 i915_request_get(request[n]);
3684 i915_request_add(request[n]);
3687 for (n = 0; n < nsibling; n++) {
3688 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3689 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3690 __func__, ve->engine->name,
3691 request[n]->fence.context,
3692 request[n]->fence.seqno);
3694 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3695 __func__, ve->engine->name,
3696 request[n]->fence.context,
3697 request[n]->fence.seqno);
3699 intel_gt_set_wedged(gt);
3704 if (request[n]->engine != siblings[nsibling - n - 1]) {
3705 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3706 request[n]->engine->name,
3707 siblings[nsibling - n - 1]->name);
3713 err = igt_live_test_end(&t);
3715 if (igt_flush_test(gt->i915))
3718 for (n = 0; n < nsibling; n++)
3719 i915_request_put(request[n]);
3722 intel_context_unpin(ve);
3724 intel_context_put(ve);
3729 static int live_virtual_mask(void *arg)
3731 struct intel_gt *gt = arg;
3732 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3733 unsigned int class, inst;
3736 if (intel_uc_uses_guc_submission(>->uc))
3739 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3740 unsigned int nsibling;
3743 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3744 if (!gt->engine_class[class][inst])
3747 siblings[nsibling++] = gt->engine_class[class][inst];
3752 err = mask_virtual_engine(gt, siblings, nsibling);
3760 static int preserved_virtual_engine(struct intel_gt *gt,
3761 struct intel_engine_cs **siblings,
3762 unsigned int nsibling)
3764 struct i915_request *last = NULL;
3765 struct intel_context *ve;
3766 struct i915_vma *scratch;
3767 struct igt_live_test t;
3772 scratch = create_scratch(siblings[0]->gt);
3773 if (IS_ERR(scratch))
3774 return PTR_ERR(scratch);
3776 err = i915_vma_sync(scratch);
3780 ve = intel_execlists_create_virtual(siblings, nsibling);
3786 err = intel_context_pin(ve);
3790 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3794 for (n = 0; n < NUM_GPR_DW; n++) {
3795 struct intel_engine_cs *engine = siblings[n % nsibling];
3796 struct i915_request *rq;
3798 rq = i915_request_create(ve);
3804 i915_request_put(last);
3805 last = i915_request_get(rq);
3807 cs = intel_ring_begin(rq, 8);
3809 i915_request_add(rq);
3814 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3815 *cs++ = CS_GPR(engine, n);
3816 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
3819 *cs++ = MI_LOAD_REGISTER_IMM(1);
3820 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
3824 intel_ring_advance(rq, cs);
3826 /* Restrict this request to run on a particular engine */
3827 rq->execution_mask = engine->mask;
3828 i915_request_add(rq);
3831 if (i915_request_wait(last, 0, HZ / 5) < 0) {
3836 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3842 for (n = 0; n < NUM_GPR_DW; n++) {
3844 pr_err("Incorrect value[%d] found for GPR[%d]\n",
3851 i915_gem_object_unpin_map(scratch->obj);
3854 if (igt_live_test_end(&t))
3856 i915_request_put(last);
3858 intel_context_unpin(ve);
3860 intel_context_put(ve);
3862 i915_vma_unpin_and_release(&scratch, 0);
3866 static int live_virtual_preserved(void *arg)
3868 struct intel_gt *gt = arg;
3869 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3870 unsigned int class, inst;
3873 * Check that the context image retains non-privileged (user) registers
3874 * from one engine to the next. For this we check that the CS_GPR
3878 if (intel_uc_uses_guc_submission(>->uc))
3881 /* As we use CS_GPR we cannot run before they existed on all engines. */
3882 if (INTEL_GEN(gt->i915) < 9)
3885 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3889 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3890 if (!gt->engine_class[class][inst])
3893 siblings[nsibling++] = gt->engine_class[class][inst];
3898 err = preserved_virtual_engine(gt, siblings, nsibling);
3906 static int bond_virtual_engine(struct intel_gt *gt,
3908 struct intel_engine_cs **siblings,
3909 unsigned int nsibling,
3911 #define BOND_SCHEDULE BIT(0)
3913 struct intel_engine_cs *master;
3914 struct i915_request *rq[16];
3915 enum intel_engine_id id;
3916 struct igt_spinner spin;
3921 * A set of bonded requests is intended to be run concurrently
3922 * across a number of engines. We use one request per-engine
3923 * and a magic fence to schedule each of the bonded requests
3924 * at the same time. A consequence of our current scheduler is that
3925 * we only move requests to the HW ready queue when the request
3926 * becomes ready, that is when all of its prerequisite fences have
3927 * been signaled. As one of those fences is the master submit fence,
3928 * there is a delay on all secondary fences as the HW may be
3929 * currently busy. Equally, as all the requests are independent,
3930 * they may have other fences that delay individual request
3931 * submission to HW. Ergo, we do not guarantee that all requests are
3932 * immediately submitted to HW at the same time, just that if the
3933 * rules are abided by, they are ready at the same time as the
3934 * first is submitted. Userspace can embed semaphores in its batch
3935 * to ensure parallel execution of its phases as it requires.
3936 * Though naturally it gets requested that perhaps the scheduler should
3937 * take care of parallel execution, even across preemption events on
3938 * different HW. (The proper answer is of course "lalalala".)
3940 * With the submit-fence, we have identified three possible phases
3941 * of synchronisation depending on the master fence: queued (not
3942 * ready), executing, and signaled. The first two are quite simple
3943 * and checked below. However, the signaled master fence handling is
3944 * contentious. Currently we do not distinguish between a signaled
3945 * fence and an expired fence, as once signaled it does not convey
3946 * any information about the previous execution. It may even be freed
3947 * and hence checking later it may not exist at all. Ergo we currently
3948 * do not apply the bonding constraint for an already signaled fence,
3949 * as our expectation is that it should not constrain the secondaries
3950 * and is outside of the scope of the bonded request API (i.e. all
3951 * userspace requests are meant to be running in parallel). As
3952 * it imposes no constraint, and is effectively a no-op, we do not
3953 * check below as normal execution flows are checked extensively above.
3955 * XXX Is the degenerate handling of signaled submit fences the
3956 * expected behaviour for userpace?
3959 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
3961 if (igt_spinner_init(&spin, gt))
3965 rq[0] = ERR_PTR(-ENOMEM);
3966 for_each_engine(master, gt, id) {
3967 struct i915_sw_fence fence = {};
3968 struct intel_context *ce;
3970 if (master->class == class)
3973 ce = intel_context_create(master);
3979 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
3981 rq[0] = igt_spinner_create_request(&spin, ce, MI_NOOP);
3982 intel_context_put(ce);
3983 if (IS_ERR(rq[0])) {
3984 err = PTR_ERR(rq[0]);
3987 i915_request_get(rq[0]);
3989 if (flags & BOND_SCHEDULE) {
3990 onstack_fence_init(&fence);
3991 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
3996 i915_request_add(rq[0]);
4000 if (!(flags & BOND_SCHEDULE) &&
4001 !igt_wait_for_spinner(&spin, rq[0])) {
4006 for (n = 0; n < nsibling; n++) {
4007 struct intel_context *ve;
4009 ve = intel_execlists_create_virtual(siblings, nsibling);
4012 onstack_fence_fini(&fence);
4016 err = intel_virtual_engine_attach_bond(ve->engine,
4020 intel_context_put(ve);
4021 onstack_fence_fini(&fence);
4025 err = intel_context_pin(ve);
4026 intel_context_put(ve);
4028 onstack_fence_fini(&fence);
4032 rq[n + 1] = i915_request_create(ve);
4033 intel_context_unpin(ve);
4034 if (IS_ERR(rq[n + 1])) {
4035 err = PTR_ERR(rq[n + 1]);
4036 onstack_fence_fini(&fence);
4039 i915_request_get(rq[n + 1]);
4041 err = i915_request_await_execution(rq[n + 1],
4043 ve->engine->bond_execute);
4044 i915_request_add(rq[n + 1]);
4046 onstack_fence_fini(&fence);
4050 onstack_fence_fini(&fence);
4051 intel_engine_flush_submission(master);
4052 igt_spinner_end(&spin);
4054 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
4055 pr_err("Master request did not execute (on %s)!\n",
4056 rq[0]->engine->name);
4061 for (n = 0; n < nsibling; n++) {
4062 if (i915_request_wait(rq[n + 1], 0,
4063 MAX_SCHEDULE_TIMEOUT) < 0) {
4068 if (rq[n + 1]->engine != siblings[n]) {
4069 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
4071 rq[n + 1]->engine->name,
4072 rq[0]->engine->name);
4078 for (n = 0; !IS_ERR(rq[n]); n++)
4079 i915_request_put(rq[n]);
4080 rq[0] = ERR_PTR(-ENOMEM);
4084 for (n = 0; !IS_ERR(rq[n]); n++)
4085 i915_request_put(rq[n]);
4086 if (igt_flush_test(gt->i915))
4089 igt_spinner_fini(&spin);
4093 static int live_virtual_bond(void *arg)
4095 static const struct phase {
4100 { "schedule", BOND_SCHEDULE },
4103 struct intel_gt *gt = arg;
4104 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4105 unsigned int class, inst;
4108 if (intel_uc_uses_guc_submission(>->uc))
4111 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4112 const struct phase *p;
4116 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
4117 if (!gt->engine_class[class][inst])
4120 GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings));
4121 siblings[nsibling++] = gt->engine_class[class][inst];
4126 for (p = phases; p->name; p++) {
4127 err = bond_virtual_engine(gt,
4128 class, siblings, nsibling,
4131 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
4132 __func__, p->name, class, nsibling, err);
4141 static int reset_virtual_engine(struct intel_gt *gt,
4142 struct intel_engine_cs **siblings,
4143 unsigned int nsibling)
4145 struct intel_engine_cs *engine;
4146 struct intel_context *ve;
4147 struct igt_spinner spin;
4148 struct i915_request *rq;
4153 * In order to support offline error capture for fast preempt reset,
4154 * we need to decouple the guilty request and ensure that it and its
4155 * descendents are not executed while the capture is in progress.
4158 if (igt_spinner_init(&spin, gt))
4161 ve = intel_execlists_create_virtual(siblings, nsibling);
4167 for (n = 0; n < nsibling; n++)
4168 engine_heartbeat_disable(siblings[n]);
4170 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4175 i915_request_add(rq);
4177 if (!igt_wait_for_spinner(&spin, rq)) {
4178 intel_gt_set_wedged(gt);
4183 engine = rq->engine;
4184 GEM_BUG_ON(engine == ve->engine);
4186 /* Take ownership of the reset and tasklet */
4187 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
4188 >->reset.flags)) {
4189 intel_gt_set_wedged(gt);
4193 tasklet_disable(&engine->execlists.tasklet);
4195 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
4196 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4198 /* Fake a preemption event; failed of course */
4199 spin_lock_irq(&engine->active.lock);
4200 __unwind_incomplete_requests(engine);
4201 spin_unlock_irq(&engine->active.lock);
4202 GEM_BUG_ON(rq->engine != ve->engine);
4204 /* Reset the engine while keeping our active request on hold */
4205 execlists_hold(engine, rq);
4206 GEM_BUG_ON(!i915_request_on_hold(rq));
4208 intel_engine_reset(engine, NULL);
4209 GEM_BUG_ON(rq->fence.error != -EIO);
4211 /* Release our grasp on the engine, letting CS flow again */
4212 tasklet_enable(&engine->execlists.tasklet);
4213 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, >->reset.flags);
4215 /* Check that we do not resubmit the held request */
4216 i915_request_get(rq);
4217 if (!i915_request_wait(rq, 0, HZ / 5)) {
4218 pr_err("%s: on hold request completed!\n",
4220 intel_gt_set_wedged(gt);
4224 GEM_BUG_ON(!i915_request_on_hold(rq));
4226 /* But is resubmitted on release */
4227 execlists_unhold(engine, rq);
4228 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4229 pr_err("%s: held request did not complete!\n",
4231 intel_gt_set_wedged(gt);
4236 i915_request_put(rq);
4238 for (n = 0; n < nsibling; n++)
4239 engine_heartbeat_enable(siblings[n]);
4241 intel_context_put(ve);
4243 igt_spinner_fini(&spin);
4247 static int live_virtual_reset(void *arg)
4249 struct intel_gt *gt = arg;
4250 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4251 unsigned int class, inst;
4254 * Check that we handle a reset event within a virtual engine.
4255 * Only the physical engine is reset, but we have to check the flow
4256 * of the virtual requests around the reset, and make sure it is not
4260 if (intel_uc_uses_guc_submission(>->uc))
4263 if (!intel_has_reset_engine(gt))
4266 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4270 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
4271 if (!gt->engine_class[class][inst])
4274 siblings[nsibling++] = gt->engine_class[class][inst];
4279 err = reset_virtual_engine(gt, siblings, nsibling);
4287 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4289 static const struct i915_subtest tests[] = {
4290 SUBTEST(live_sanitycheck),
4291 SUBTEST(live_unlite_switch),
4292 SUBTEST(live_unlite_preempt),
4293 SUBTEST(live_pin_rewind),
4294 SUBTEST(live_hold_reset),
4295 SUBTEST(live_error_interrupt),
4296 SUBTEST(live_timeslice_preempt),
4297 SUBTEST(live_timeslice_rewind),
4298 SUBTEST(live_timeslice_queue),
4299 SUBTEST(live_busywait_preempt),
4300 SUBTEST(live_preempt),
4301 SUBTEST(live_late_preempt),
4302 SUBTEST(live_nopreempt),
4303 SUBTEST(live_preempt_cancel),
4304 SUBTEST(live_suppress_self_preempt),
4305 SUBTEST(live_suppress_wait_preempt),
4306 SUBTEST(live_chain_preempt),
4307 SUBTEST(live_preempt_gang),
4308 SUBTEST(live_preempt_timeout),
4309 SUBTEST(live_preempt_user),
4310 SUBTEST(live_preempt_smoke),
4311 SUBTEST(live_virtual_engine),
4312 SUBTEST(live_virtual_mask),
4313 SUBTEST(live_virtual_preserved),
4314 SUBTEST(live_virtual_bond),
4315 SUBTEST(live_virtual_reset),
4318 if (!HAS_EXECLISTS(i915))
4321 if (intel_gt_is_wedged(&i915->gt))
4324 return intel_gt_live_subtests(tests, &i915->gt);
4327 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
4330 i915_ggtt_offset(ce->engine->status_page.vma) +
4331 offset_in_page(slot);
4332 struct i915_request *rq;
4335 rq = intel_context_create_request(ce);
4339 cs = intel_ring_begin(rq, 4);
4341 i915_request_add(rq);
4345 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
4350 intel_ring_advance(rq, cs);
4352 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
4353 i915_request_add(rq);
4357 static int context_flush(struct intel_context *ce, long timeout)
4359 struct i915_request *rq;
4360 struct dma_fence *fence;
4363 rq = intel_engine_create_kernel_request(ce->engine);
4367 fence = i915_active_fence_get(&ce->timeline->last_request);
4369 i915_request_await_dma_fence(rq, fence);
4370 dma_fence_put(fence);
4373 rq = i915_request_get(rq);
4374 i915_request_add(rq);
4375 if (i915_request_wait(rq, 0, timeout) < 0)
4377 i915_request_put(rq);
4379 rmb(); /* We know the request is written, make sure all state is too! */
4383 static int live_lrc_layout(void *arg)
4385 struct intel_gt *gt = arg;
4386 struct intel_engine_cs *engine;
4387 enum intel_engine_id id;
4392 * Check the registers offsets we use to create the initial reg state
4393 * match the layout saved by HW.
4396 lrc = kmalloc(PAGE_SIZE, GFP_KERNEL);
4401 for_each_engine(engine, gt, id) {
4405 if (!engine->default_state)
4408 hw = shmem_pin_map(engine->default_state);
4413 hw += LRC_STATE_OFFSET / sizeof(*hw);
4415 execlists_init_reg_state(memset(lrc, POISON_INUSE, PAGE_SIZE),
4416 engine->kernel_context,
4418 engine->kernel_context->ring,
4431 pr_debug("%s: skipped instruction %x at dword %d\n",
4432 engine->name, lri, dw);
4437 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
4438 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
4439 engine->name, dw, lri);
4444 if (lrc[dw] != lri) {
4445 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
4446 engine->name, dw, lri, lrc[dw]);
4456 if (hw[dw] != lrc[dw]) {
4457 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
4458 engine->name, dw, hw[dw], lrc[dw]);
4464 * Skip over the actual register value as we
4465 * expect that to differ.
4470 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
4473 pr_info("%s: HW register image:\n", engine->name);
4474 igt_hexdump(hw, PAGE_SIZE);
4476 pr_info("%s: SW register image:\n", engine->name);
4477 igt_hexdump(lrc, PAGE_SIZE);
4480 shmem_unpin_map(engine->default_state, hw);
4489 static int find_offset(const u32 *lri, u32 offset)
4493 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
4494 if (lri[i] == offset)
4500 static int live_lrc_fixed(void *arg)
4502 struct intel_gt *gt = arg;
4503 struct intel_engine_cs *engine;
4504 enum intel_engine_id id;
4508 * Check the assumed register offsets match the actual locations in
4509 * the context image.
4512 for_each_engine(engine, gt, id) {
4519 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
4524 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
4529 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
4534 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
4539 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
4540 lrc_ring_mi_mode(engine),
4544 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
4549 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
4550 lrc_ring_wa_bb_per_ctx(engine),
4551 "RING_BB_PER_CTX_PTR"
4554 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
4555 lrc_ring_indirect_ptr(engine),
4556 "RING_INDIRECT_CTX_PTR"
4559 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
4560 lrc_ring_indirect_offset(engine),
4561 "RING_INDIRECT_CTX_OFFSET"
4564 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
4566 "RING_CTX_TIMESTAMP"
4569 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
4570 lrc_ring_gpr0(engine),
4574 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
4575 lrc_ring_cmd_buf_cctl(engine),
4582 if (!engine->default_state)
4585 hw = shmem_pin_map(engine->default_state);
4590 hw += LRC_STATE_OFFSET / sizeof(*hw);
4592 for (t = tbl; t->name; t++) {
4593 int dw = find_offset(hw, t->reg);
4595 if (dw != t->offset) {
4596 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
4606 shmem_unpin_map(engine->default_state, hw);
4612 static int __live_lrc_state(struct intel_engine_cs *engine,
4613 struct i915_vma *scratch)
4615 struct intel_context *ce;
4616 struct i915_request *rq;
4622 u32 expected[MAX_IDX];
4627 ce = intel_context_create(engine);
4631 err = intel_context_pin(ce);
4635 rq = i915_request_create(ce);
4641 cs = intel_ring_begin(rq, 4 * MAX_IDX);
4644 i915_request_add(rq);
4648 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4649 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
4650 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
4653 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
4655 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4656 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
4657 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
4660 i915_vma_lock(scratch);
4661 err = i915_request_await_object(rq, scratch->obj, true);
4663 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
4664 i915_vma_unlock(scratch);
4666 i915_request_get(rq);
4667 i915_request_add(rq);
4671 intel_engine_flush_submission(engine);
4672 expected[RING_TAIL_IDX] = ce->ring->tail;
4674 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4679 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4685 for (n = 0; n < MAX_IDX; n++) {
4686 if (cs[n] != expected[n]) {
4687 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
4688 engine->name, n, cs[n], expected[n]);
4694 i915_gem_object_unpin_map(scratch->obj);
4697 i915_request_put(rq);
4699 intel_context_unpin(ce);
4701 intel_context_put(ce);
4705 static int live_lrc_state(void *arg)
4707 struct intel_gt *gt = arg;
4708 struct intel_engine_cs *engine;
4709 struct i915_vma *scratch;
4710 enum intel_engine_id id;
4714 * Check the live register state matches what we expect for this
4718 scratch = create_scratch(gt);
4719 if (IS_ERR(scratch))
4720 return PTR_ERR(scratch);
4722 for_each_engine(engine, gt, id) {
4723 err = __live_lrc_state(engine, scratch);
4728 if (igt_flush_test(gt->i915))
4731 i915_vma_unpin_and_release(&scratch, 0);
4735 static int gpr_make_dirty(struct intel_context *ce)
4737 struct i915_request *rq;
4741 rq = intel_context_create_request(ce);
4745 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
4747 i915_request_add(rq);
4751 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
4752 for (n = 0; n < NUM_GPR_DW; n++) {
4753 *cs++ = CS_GPR(ce->engine, n);
4754 *cs++ = STACK_MAGIC;
4758 intel_ring_advance(rq, cs);
4760 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
4761 i915_request_add(rq);
4766 static struct i915_request *
4767 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
4770 i915_ggtt_offset(ce->engine->status_page.vma) +
4771 offset_in_page(slot);
4772 struct i915_request *rq;
4777 rq = intel_context_create_request(ce);
4781 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
4783 i915_request_add(rq);
4784 return ERR_CAST(cs);
4787 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
4790 *cs++ = MI_SEMAPHORE_WAIT |
4791 MI_SEMAPHORE_GLOBAL_GTT |
4793 MI_SEMAPHORE_SAD_NEQ_SDD;
4798 for (n = 0; n < NUM_GPR_DW; n++) {
4799 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4800 *cs++ = CS_GPR(ce->engine, n);
4801 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4805 i915_vma_lock(scratch);
4806 err = i915_request_await_object(rq, scratch->obj, true);
4808 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
4809 i915_vma_unlock(scratch);
4811 i915_request_get(rq);
4812 i915_request_add(rq);
4814 i915_request_put(rq);
4821 static int __live_lrc_gpr(struct intel_engine_cs *engine,
4822 struct i915_vma *scratch,
4825 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
4826 struct intel_context *ce;
4827 struct i915_request *rq;
4832 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
4833 return 0; /* GPR only on rcs0 for gen8 */
4835 err = gpr_make_dirty(engine->kernel_context);
4839 ce = intel_context_create(engine);
4843 rq = __gpr_read(ce, scratch, slot);
4849 err = wait_for_submit(engine, rq, HZ / 2);
4854 err = gpr_make_dirty(engine->kernel_context);
4858 err = emit_semaphore_signal(engine->kernel_context, slot);
4866 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4871 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4877 for (n = 0; n < NUM_GPR_DW; n++) {
4879 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
4881 n / 2, n & 1 ? "udw" : "ldw",
4888 i915_gem_object_unpin_map(scratch->obj);
4891 memset32(&slot[0], -1, 4);
4893 i915_request_put(rq);
4895 intel_context_put(ce);
4899 static int live_lrc_gpr(void *arg)
4901 struct intel_gt *gt = arg;
4902 struct intel_engine_cs *engine;
4903 struct i915_vma *scratch;
4904 enum intel_engine_id id;
4908 * Check that GPR registers are cleared in new contexts as we need
4909 * to avoid leaking any information from previous contexts.
4912 scratch = create_scratch(gt);
4913 if (IS_ERR(scratch))
4914 return PTR_ERR(scratch);
4916 for_each_engine(engine, gt, id) {
4917 engine_heartbeat_disable(engine);
4919 err = __live_lrc_gpr(engine, scratch, false);
4923 err = __live_lrc_gpr(engine, scratch, true);
4928 engine_heartbeat_enable(engine);
4929 if (igt_flush_test(gt->i915))
4935 i915_vma_unpin_and_release(&scratch, 0);
4939 static struct i915_request *
4940 create_timestamp(struct intel_context *ce, void *slot, int idx)
4943 i915_ggtt_offset(ce->engine->status_page.vma) +
4944 offset_in_page(slot);
4945 struct i915_request *rq;
4949 rq = intel_context_create_request(ce);
4953 cs = intel_ring_begin(rq, 10);
4959 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
4962 *cs++ = MI_SEMAPHORE_WAIT |
4963 MI_SEMAPHORE_GLOBAL_GTT |
4965 MI_SEMAPHORE_SAD_NEQ_SDD;
4970 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4971 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
4972 *cs++ = offset + idx * sizeof(u32);
4975 intel_ring_advance(rq, cs);
4977 rq->sched.attr.priority = I915_PRIORITY_MASK;
4980 i915_request_get(rq);
4981 i915_request_add(rq);
4983 i915_request_put(rq);
4984 return ERR_PTR(err);
4990 struct lrc_timestamp {
4991 struct intel_engine_cs *engine;
4992 struct intel_context *ce[2];
4996 static bool timestamp_advanced(u32 start, u32 end)
4998 return (s32)(end - start) > 0;
5001 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
5003 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
5004 struct i915_request *rq;
5008 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
5009 rq = create_timestamp(arg->ce[0], slot, 1);
5013 err = wait_for_submit(rq->engine, rq, HZ / 2);
5018 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
5019 err = emit_semaphore_signal(arg->ce[1], slot);
5027 /* And wait for switch to kernel (to save our context to memory) */
5028 err = context_flush(arg->ce[0], HZ / 2);
5032 if (!timestamp_advanced(arg->poison, slot[1])) {
5033 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
5034 arg->engine->name, preempt ? "preempt" : "simple",
5035 arg->poison, slot[1]);
5039 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
5040 if (!timestamp_advanced(slot[1], timestamp)) {
5041 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
5042 arg->engine->name, preempt ? "preempt" : "simple",
5043 slot[1], timestamp);
5048 memset32(slot, -1, 4);
5049 i915_request_put(rq);
5053 static int live_lrc_timestamp(void *arg)
5055 struct lrc_timestamp data = {};
5056 struct intel_gt *gt = arg;
5057 enum intel_engine_id id;
5058 const u32 poison[] = {
5066 * We want to verify that the timestamp is saved and restore across
5067 * context switches and is monotonic.
5069 * So we do this with a little bit of LRC poisoning to check various
5070 * boundary conditions, and see what happens if we preempt the context
5071 * with a second request (carrying more poison into the timestamp).
5074 for_each_engine(data.engine, gt, id) {
5077 engine_heartbeat_disable(data.engine);
5079 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
5080 struct intel_context *tmp;
5082 tmp = intel_context_create(data.engine);
5088 err = intel_context_pin(tmp);
5090 intel_context_put(tmp);
5097 for (i = 0; i < ARRAY_SIZE(poison); i++) {
5098 data.poison = poison[i];
5100 err = __lrc_timestamp(&data, false);
5104 err = __lrc_timestamp(&data, true);
5110 engine_heartbeat_enable(data.engine);
5111 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
5115 intel_context_unpin(data.ce[i]);
5116 intel_context_put(data.ce[i]);
5119 if (igt_flush_test(gt->i915))
5128 static struct i915_vma *
5129 create_user_vma(struct i915_address_space *vm, unsigned long size)
5131 struct drm_i915_gem_object *obj;
5132 struct i915_vma *vma;
5135 obj = i915_gem_object_create_internal(vm->i915, size);
5137 return ERR_CAST(obj);
5139 vma = i915_vma_instance(obj, vm, NULL);
5141 i915_gem_object_put(obj);
5145 err = i915_vma_pin(vma, 0, 0, PIN_USER);
5147 i915_gem_object_put(obj);
5148 return ERR_PTR(err);
5154 static struct i915_vma *
5155 store_context(struct intel_context *ce, struct i915_vma *scratch)
5157 struct i915_vma *batch;
5158 u32 dw, x, *cs, *hw;
5161 batch = create_user_vma(ce->vm, SZ_64K);
5165 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
5167 i915_vma_put(batch);
5168 return ERR_CAST(cs);
5171 defaults = shmem_pin_map(ce->engine->default_state);
5173 i915_gem_object_unpin_map(batch->obj);
5174 i915_vma_put(batch);
5175 return ERR_PTR(-ENOMEM);
5181 hw += LRC_STATE_OFFSET / sizeof(*hw);
5183 u32 len = hw[dw] & 0x7f;
5190 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5196 len = (len + 1) / 2;
5198 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
5200 *cs++ = lower_32_bits(scratch->node.start + x);
5201 *cs++ = upper_32_bits(scratch->node.start + x);
5206 } while (dw < PAGE_SIZE / sizeof(u32) &&
5207 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5209 *cs++ = MI_BATCH_BUFFER_END;
5211 shmem_unpin_map(ce->engine->default_state, defaults);
5213 i915_gem_object_flush_map(batch->obj);
5214 i915_gem_object_unpin_map(batch->obj);
5219 static int move_to_active(struct i915_request *rq,
5220 struct i915_vma *vma,
5226 err = i915_request_await_object(rq, vma->obj, flags);
5228 err = i915_vma_move_to_active(vma, rq, flags);
5229 i915_vma_unlock(vma);
5234 static struct i915_request *
5235 record_registers(struct intel_context *ce,
5236 struct i915_vma *before,
5237 struct i915_vma *after,
5240 struct i915_vma *b_before, *b_after;
5241 struct i915_request *rq;
5245 b_before = store_context(ce, before);
5246 if (IS_ERR(b_before))
5247 return ERR_CAST(b_before);
5249 b_after = store_context(ce, after);
5250 if (IS_ERR(b_after)) {
5251 rq = ERR_CAST(b_after);
5255 rq = intel_context_create_request(ce);
5259 err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
5263 err = move_to_active(rq, b_before, 0);
5267 err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
5271 err = move_to_active(rq, b_after, 0);
5275 cs = intel_ring_begin(rq, 14);
5281 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5282 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5283 *cs++ = lower_32_bits(b_before->node.start);
5284 *cs++ = upper_32_bits(b_before->node.start);
5286 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5287 *cs++ = MI_SEMAPHORE_WAIT |
5288 MI_SEMAPHORE_GLOBAL_GTT |
5290 MI_SEMAPHORE_SAD_NEQ_SDD;
5292 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
5293 offset_in_page(sema);
5297 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5298 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5299 *cs++ = lower_32_bits(b_after->node.start);
5300 *cs++ = upper_32_bits(b_after->node.start);
5302 intel_ring_advance(rq, cs);
5304 WRITE_ONCE(*sema, 0);
5305 i915_request_get(rq);
5306 i915_request_add(rq);
5308 i915_vma_put(b_after);
5310 i915_vma_put(b_before);
5314 i915_request_add(rq);
5319 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
5321 struct i915_vma *batch;
5325 batch = create_user_vma(ce->vm, SZ_64K);
5329 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
5331 i915_vma_put(batch);
5332 return ERR_CAST(cs);
5335 defaults = shmem_pin_map(ce->engine->default_state);
5337 i915_gem_object_unpin_map(batch->obj);
5338 i915_vma_put(batch);
5339 return ERR_PTR(-ENOMEM);
5344 hw += LRC_STATE_OFFSET / sizeof(*hw);
5346 u32 len = hw[dw] & 0x7f;
5353 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5359 len = (len + 1) / 2;
5360 *cs++ = MI_LOAD_REGISTER_IMM(len);
5366 } while (dw < PAGE_SIZE / sizeof(u32) &&
5367 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5369 *cs++ = MI_BATCH_BUFFER_END;
5371 shmem_unpin_map(ce->engine->default_state, defaults);
5373 i915_gem_object_flush_map(batch->obj);
5374 i915_gem_object_unpin_map(batch->obj);
5379 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
5381 struct i915_request *rq;
5382 struct i915_vma *batch;
5386 batch = load_context(ce, poison);
5388 return PTR_ERR(batch);
5390 rq = intel_context_create_request(ce);
5396 err = move_to_active(rq, batch, 0);
5400 cs = intel_ring_begin(rq, 8);
5406 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5407 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5408 *cs++ = lower_32_bits(batch->node.start);
5409 *cs++ = upper_32_bits(batch->node.start);
5411 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
5412 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
5413 offset_in_page(sema);
5417 intel_ring_advance(rq, cs);
5419 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
5421 i915_request_add(rq);
5423 i915_vma_put(batch);
5427 static bool is_moving(u32 a, u32 b)
5432 static int compare_isolation(struct intel_engine_cs *engine,
5433 struct i915_vma *ref[2],
5434 struct i915_vma *result[2],
5435 struct intel_context *ce,
5438 u32 x, dw, *hw, *lrc;
5443 A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
5445 return PTR_ERR(A[0]);
5447 A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
5449 err = PTR_ERR(A[1]);
5453 B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
5455 err = PTR_ERR(B[0]);
5459 B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
5461 err = PTR_ERR(B[1]);
5465 lrc = i915_gem_object_pin_map(ce->state->obj,
5466 i915_coherent_map_type(engine->i915));
5471 lrc += LRC_STATE_OFFSET / sizeof(*hw);
5473 defaults = shmem_pin_map(ce->engine->default_state);
5482 hw += LRC_STATE_OFFSET / sizeof(*hw);
5484 u32 len = hw[dw] & 0x7f;
5491 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5497 len = (len + 1) / 2;
5499 if (!is_moving(A[0][x], A[1][x]) &&
5500 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
5501 switch (hw[dw] & 4095) {
5502 case 0x30: /* RING_HEAD */
5503 case 0x34: /* RING_TAIL */
5507 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
5510 A[0][x], B[0][x], B[1][x],
5511 poison, lrc[dw + 1]);
5518 } while (dw < PAGE_SIZE / sizeof(u32) &&
5519 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5521 shmem_unpin_map(ce->engine->default_state, defaults);
5523 i915_gem_object_unpin_map(ce->state->obj);
5525 i915_gem_object_unpin_map(result[1]->obj);
5527 i915_gem_object_unpin_map(result[0]->obj);
5529 i915_gem_object_unpin_map(ref[1]->obj);
5531 i915_gem_object_unpin_map(ref[0]->obj);
5535 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
5537 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
5538 struct i915_vma *ref[2], *result[2];
5539 struct intel_context *A, *B;
5540 struct i915_request *rq;
5543 A = intel_context_create(engine);
5547 B = intel_context_create(engine);
5553 ref[0] = create_user_vma(A->vm, SZ_64K);
5554 if (IS_ERR(ref[0])) {
5555 err = PTR_ERR(ref[0]);
5559 ref[1] = create_user_vma(A->vm, SZ_64K);
5560 if (IS_ERR(ref[1])) {
5561 err = PTR_ERR(ref[1]);
5565 rq = record_registers(A, ref[0], ref[1], sema);
5571 WRITE_ONCE(*sema, 1);
5574 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5575 i915_request_put(rq);
5579 i915_request_put(rq);
5581 result[0] = create_user_vma(A->vm, SZ_64K);
5582 if (IS_ERR(result[0])) {
5583 err = PTR_ERR(result[0]);
5587 result[1] = create_user_vma(A->vm, SZ_64K);
5588 if (IS_ERR(result[1])) {
5589 err = PTR_ERR(result[1]);
5593 rq = record_registers(A, result[0], result[1], sema);
5599 err = poison_registers(B, poison, sema);
5601 WRITE_ONCE(*sema, -1);
5602 i915_request_put(rq);
5606 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5607 i915_request_put(rq);
5611 i915_request_put(rq);
5613 err = compare_isolation(engine, ref, result, A, poison);
5616 i915_vma_put(result[1]);
5618 i915_vma_put(result[0]);
5620 i915_vma_put(ref[1]);
5622 i915_vma_put(ref[0]);
5624 intel_context_put(B);
5626 intel_context_put(A);
5630 static bool skip_isolation(const struct intel_engine_cs *engine)
5632 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
5635 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
5641 static int live_lrc_isolation(void *arg)
5643 struct intel_gt *gt = arg;
5644 struct intel_engine_cs *engine;
5645 enum intel_engine_id id;
5646 const u32 poison[] = {
5656 * Our goal is try and verify that per-context state cannot be
5657 * tampered with by another non-privileged client.
5659 * We take the list of context registers from the LRI in the default
5660 * context image and attempt to modify that list from a remote context.
5663 for_each_engine(engine, gt, id) {
5666 /* Just don't even ask */
5667 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
5668 skip_isolation(engine))
5671 intel_engine_pm_get(engine);
5672 for (i = 0; i < ARRAY_SIZE(poison); i++) {
5675 result = __lrc_isolation(engine, poison[i]);
5679 result = __lrc_isolation(engine, ~poison[i]);
5683 intel_engine_pm_put(engine);
5684 if (igt_flush_test(gt->i915)) {
5693 static int indirect_ctx_submit_req(struct intel_context *ce)
5695 struct i915_request *rq;
5698 rq = intel_context_create_request(ce);
5702 i915_request_get(rq);
5703 i915_request_add(rq);
5705 if (i915_request_wait(rq, 0, HZ / 5) < 0)
5708 i915_request_put(rq);
5713 #define CTX_BB_CANARY_OFFSET (3 * 1024)
5714 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))
5717 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
5719 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
5720 MI_SRM_LRM_GLOBAL_GTT |
5722 *cs++ = i915_mmio_reg_offset(RING_START(0));
5723 *cs++ = i915_ggtt_offset(ce->state) +
5724 context_wa_bb_offset(ce) +
5725 CTX_BB_CANARY_OFFSET;
5732 indirect_ctx_bb_setup(struct intel_context *ce)
5734 u32 *cs = context_indirect_bb(ce);
5736 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
5738 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
5741 static bool check_ring_start(struct intel_context *ce)
5743 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
5744 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
5746 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
5749 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
5750 ctx_bb[CTX_BB_CANARY_INDEX],
5751 ce->lrc_reg_state[CTX_RING_START]);
5756 static int indirect_ctx_bb_check(struct intel_context *ce)
5760 err = indirect_ctx_submit_req(ce);
5764 if (!check_ring_start(ce))
5770 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
5772 struct intel_context *a, *b;
5775 a = intel_context_create(engine);
5778 err = intel_context_pin(a);
5782 b = intel_context_create(engine);
5787 err = intel_context_pin(b);
5791 /* We use the already reserved extra page in context state */
5792 if (!a->wa_bb_page) {
5793 GEM_BUG_ON(b->wa_bb_page);
5794 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
5799 * In order to test that our per context bb is truly per context,
5800 * and executes at the intended spot on context restoring process,
5801 * make the batch store the ring start value to memory.
5802 * As ring start is restored apriori of starting the indirect ctx bb and
5803 * as it will be different for each context, it fits to this purpose.
5805 indirect_ctx_bb_setup(a);
5806 indirect_ctx_bb_setup(b);
5808 err = indirect_ctx_bb_check(a);
5812 err = indirect_ctx_bb_check(b);
5815 intel_context_unpin(b);
5817 intel_context_put(b);
5819 intel_context_unpin(a);
5821 intel_context_put(a);
5826 static int live_lrc_indirect_ctx_bb(void *arg)
5828 struct intel_gt *gt = arg;
5829 struct intel_engine_cs *engine;
5830 enum intel_engine_id id;
5833 for_each_engine(engine, gt, id) {
5834 intel_engine_pm_get(engine);
5835 err = __live_lrc_indirect_ctx_bb(engine);
5836 intel_engine_pm_put(engine);
5838 if (igt_flush_test(gt->i915))
5848 static void garbage_reset(struct intel_engine_cs *engine,
5849 struct i915_request *rq)
5851 const unsigned int bit = I915_RESET_ENGINE + engine->id;
5852 unsigned long *lock = &engine->gt->reset.flags;
5854 if (test_and_set_bit(bit, lock))
5857 tasklet_disable(&engine->execlists.tasklet);
5859 if (!rq->fence.error)
5860 intel_engine_reset(engine, NULL);
5862 tasklet_enable(&engine->execlists.tasklet);
5863 clear_and_wake_up_bit(bit, lock);
5866 static struct i915_request *garbage(struct intel_context *ce,
5867 struct rnd_state *prng)
5869 struct i915_request *rq;
5872 err = intel_context_pin(ce);
5874 return ERR_PTR(err);
5876 prandom_bytes_state(prng,
5878 ce->engine->context_size -
5881 rq = intel_context_create_request(ce);
5887 i915_request_get(rq);
5888 i915_request_add(rq);
5892 intel_context_unpin(ce);
5893 return ERR_PTR(err);
5896 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
5898 struct intel_context *ce;
5899 struct i915_request *hang;
5902 ce = intel_context_create(engine);
5906 hang = garbage(ce, prng);
5908 err = PTR_ERR(hang);
5912 if (wait_for_submit(engine, hang, HZ / 2)) {
5913 i915_request_put(hang);
5918 intel_context_set_banned(ce);
5919 garbage_reset(engine, hang);
5921 intel_engine_flush_submission(engine);
5922 if (!hang->fence.error) {
5923 i915_request_put(hang);
5924 pr_err("%s: corrupted context was not reset\n",
5930 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
5931 pr_err("%s: corrupted context did not recover\n",
5933 i915_request_put(hang);
5937 i915_request_put(hang);
5940 intel_context_put(ce);
5944 static int live_lrc_garbage(void *arg)
5946 struct intel_gt *gt = arg;
5947 struct intel_engine_cs *engine;
5948 enum intel_engine_id id;
5951 * Verify that we can recover if one context state is completely
5955 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
5958 for_each_engine(engine, gt, id) {
5959 I915_RND_STATE(prng);
5962 if (!intel_has_reset_engine(engine->gt))
5965 intel_engine_pm_get(engine);
5966 for (i = 0; i < 3; i++) {
5967 err = __lrc_garbage(engine, &prng);
5971 intel_engine_pm_put(engine);
5973 if (igt_flush_test(gt->i915))
5982 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
5984 struct intel_context *ce;
5985 struct i915_request *rq;
5986 IGT_TIMEOUT(end_time);
5989 ce = intel_context_create(engine);
5993 ce->runtime.num_underflow = 0;
5994 ce->runtime.max_underflow = 0;
5997 unsigned int loop = 1024;
6000 rq = intel_context_create_request(ce);
6007 i915_request_get(rq);
6009 i915_request_add(rq);
6012 if (__igt_timeout(end_time, NULL))
6015 i915_request_put(rq);
6018 err = i915_request_wait(rq, 0, HZ / 5);
6020 pr_err("%s: request not completed!\n", engine->name);
6024 igt_flush_test(engine->i915);
6026 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
6028 intel_context_get_total_runtime_ns(ce),
6029 intel_context_get_avg_runtime_ns(ce));
6032 if (ce->runtime.num_underflow) {
6033 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
6035 ce->runtime.num_underflow,
6036 ce->runtime.max_underflow);
6042 i915_request_put(rq);
6044 intel_context_put(ce);
6048 static int live_pphwsp_runtime(void *arg)
6050 struct intel_gt *gt = arg;
6051 struct intel_engine_cs *engine;
6052 enum intel_engine_id id;
6056 * Check that cumulative context runtime as stored in the pphwsp[16]
6060 for_each_engine(engine, gt, id) {
6061 err = __live_pphwsp_runtime(engine);
6066 if (igt_flush_test(gt->i915))
6072 int intel_lrc_live_selftests(struct drm_i915_private *i915)
6074 static const struct i915_subtest tests[] = {
6075 SUBTEST(live_lrc_layout),
6076 SUBTEST(live_lrc_fixed),
6077 SUBTEST(live_lrc_state),
6078 SUBTEST(live_lrc_gpr),
6079 SUBTEST(live_lrc_isolation),
6080 SUBTEST(live_lrc_timestamp),
6081 SUBTEST(live_lrc_garbage),
6082 SUBTEST(live_pphwsp_runtime),
6083 SUBTEST(live_lrc_indirect_ctx_bb),
6086 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
6089 return intel_gt_live_subtests(tests, &i915->gt);