1 // SPDX-License-Identifier: MIT
3 * Copyright © 2016-2019 Intel Corporation
6 #include <linux/types.h>
8 #include "gt/intel_gt.h"
9 #include "intel_guc_reg.h"
10 #include "intel_huc.h"
11 #include "intel_huc_print.h"
14 #include "pxp/intel_pxp_cmd_interface_43.h"
16 #include <linux/device/bus.h>
17 #include <linux/mei_aux.h>
22 * The HuC is a dedicated microcontroller for usage in media HEVC (High
23 * Efficiency Video Coding) operations. Userspace can directly use the firmware
24 * capabilities by adding HuC specific commands to batch buffers.
26 * The kernel driver is only responsible for loading the HuC firmware and
27 * triggering its security authentication. This is done differently depending
29 * - older platforms (from Gen9 to most Gen12s): the load is performed via DMA
30 * and the authentication via GuC
31 * - DG2: load and authentication are both performed via GSC.
32 * - MTL and newer platforms: the load is performed via DMA (same as with
33 * not-DG2 older platforms), while the authentication is done in 2-steps,
34 * a first auth for clear-media workloads via GuC and a second one for all
36 * On platforms where the GuC does the authentication, to correctly do so the
37 * HuC binary must be loaded before the GuC one.
38 * Loading the HuC is optional; however, not using the HuC might negatively
39 * impact power usage and/or performance of media workloads, depending on the
41 * HuC must be reloaded on events that cause the WOPCM to lose its contents
42 * (S3/S4, FLR); on older platforms the HuC must also be reloaded on GuC/GT
43 * reset, while on newer ones it will survive that.
45 * See https://github.com/intel/media-driver for the latest details on HuC
50 * DOC: HuC Memory Management
52 * Similarly to the GuC, the HuC can't do any memory allocations on its own,
53 * with the difference being that the allocations for HuC usage are handled by
54 * the userspace driver instead of the kernel one. The HuC accesses the memory
55 * via the PPGTT belonging to the context loaded on the VCS executing the
56 * HuC-specific commands.
60 * MEI-GSC load is an async process. The probing of the exposed aux device
61 * (see intel_gsc.c) usually happens a few seconds after i915 probe, depending
62 * on when the kernel schedules it. Unless something goes terribly wrong, we're
63 * guaranteed for this to happen during boot, so the big timeout is a safety net
64 * that we never expect to need.
65 * MEI-PXP + HuC load usually takes ~300ms, but if the GSC needs to be resumed
66 * and/or reset, this can take longer. Note that the kernel might schedule
67 * other work between the i915 init/resume and the MEI one, which can add to
70 #define GSC_INIT_TIMEOUT_MS 10000
71 #define PXP_INIT_TIMEOUT_MS 5000
73 static int sw_fence_dummy_notify(struct i915_sw_fence *sf,
74 enum i915_sw_fence_notify state)
79 static void __delayed_huc_load_complete(struct intel_huc *huc)
81 if (!i915_sw_fence_done(&huc->delayed_load.fence))
82 i915_sw_fence_complete(&huc->delayed_load.fence);
85 static void delayed_huc_load_complete(struct intel_huc *huc)
87 hrtimer_cancel(&huc->delayed_load.timer);
88 __delayed_huc_load_complete(huc);
91 static void __gsc_init_error(struct intel_huc *huc)
93 huc->delayed_load.status = INTEL_HUC_DELAYED_LOAD_ERROR;
94 __delayed_huc_load_complete(huc);
97 static void gsc_init_error(struct intel_huc *huc)
99 hrtimer_cancel(&huc->delayed_load.timer);
100 __gsc_init_error(huc);
103 static void gsc_init_done(struct intel_huc *huc)
105 hrtimer_cancel(&huc->delayed_load.timer);
107 /* MEI-GSC init is done, now we wait for MEI-PXP to bind */
108 huc->delayed_load.status = INTEL_HUC_WAITING_ON_PXP;
109 if (!i915_sw_fence_done(&huc->delayed_load.fence))
110 hrtimer_start(&huc->delayed_load.timer,
111 ms_to_ktime(PXP_INIT_TIMEOUT_MS),
115 static enum hrtimer_restart huc_delayed_load_timer_callback(struct hrtimer *hrtimer)
117 struct intel_huc *huc = container_of(hrtimer, struct intel_huc, delayed_load.timer);
119 if (!intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC)) {
120 if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_GSC)
121 huc_notice(huc, "timed out waiting for MEI GSC\n");
122 else if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_PXP)
123 huc_notice(huc, "timed out waiting for MEI PXP\n");
125 MISSING_CASE(huc->delayed_load.status);
127 __gsc_init_error(huc);
130 return HRTIMER_NORESTART;
133 static void huc_delayed_load_start(struct intel_huc *huc)
137 GEM_BUG_ON(intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC));
140 * On resume we don't have to wait for MEI-GSC to be re-probed, but we
141 * do need to wait for MEI-PXP to reset & re-bind
143 switch (huc->delayed_load.status) {
144 case INTEL_HUC_WAITING_ON_GSC:
145 delay = ms_to_ktime(GSC_INIT_TIMEOUT_MS);
147 case INTEL_HUC_WAITING_ON_PXP:
148 delay = ms_to_ktime(PXP_INIT_TIMEOUT_MS);
156 * This fence is always complete unless we're waiting for the
157 * GSC device to come up to load the HuC. We arm the fence here
158 * and complete it when we confirm that the HuC is loaded from
159 * the PXP bind callback.
161 GEM_BUG_ON(!i915_sw_fence_done(&huc->delayed_load.fence));
162 i915_sw_fence_fini(&huc->delayed_load.fence);
163 i915_sw_fence_reinit(&huc->delayed_load.fence);
164 i915_sw_fence_await(&huc->delayed_load.fence);
165 i915_sw_fence_commit(&huc->delayed_load.fence);
167 hrtimer_start(&huc->delayed_load.timer, delay, HRTIMER_MODE_REL);
170 static int gsc_notifier(struct notifier_block *nb, unsigned long action, void *data)
172 struct device *dev = data;
173 struct intel_huc *huc = container_of(nb, struct intel_huc, delayed_load.nb);
174 struct intel_gsc_intf *intf = &huc_to_gt(huc)->gsc.intf[0];
176 if (!intf->adev || &intf->adev->aux_dev.dev != dev)
180 case BUS_NOTIFY_BOUND_DRIVER: /* mei driver bound to aux device */
184 case BUS_NOTIFY_DRIVER_NOT_BOUND: /* mei driver fails to be bound */
185 case BUS_NOTIFY_UNBIND_DRIVER: /* mei driver about to be unbound */
186 huc_info(huc, "MEI driver not bound, disabling load\n");
194 void intel_huc_register_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus)
198 if (!intel_huc_is_loaded_by_gsc(huc))
201 huc->delayed_load.nb.notifier_call = gsc_notifier;
202 ret = bus_register_notifier(bus, &huc->delayed_load.nb);
204 huc_err(huc, "failed to register GSC notifier %pe\n", ERR_PTR(ret));
205 huc->delayed_load.nb.notifier_call = NULL;
210 void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus)
212 if (!huc->delayed_load.nb.notifier_call)
215 delayed_huc_load_complete(huc);
217 bus_unregister_notifier(bus, &huc->delayed_load.nb);
218 huc->delayed_load.nb.notifier_call = NULL;
221 static void delayed_huc_load_init(struct intel_huc *huc)
224 * Initialize fence to be complete as this is expected to be complete
225 * unless there is a delayed HuC load in progress.
227 i915_sw_fence_init(&huc->delayed_load.fence,
228 sw_fence_dummy_notify);
229 i915_sw_fence_commit(&huc->delayed_load.fence);
231 hrtimer_init(&huc->delayed_load.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
232 huc->delayed_load.timer.function = huc_delayed_load_timer_callback;
235 static void delayed_huc_load_fini(struct intel_huc *huc)
238 * the fence is initialized in init_early, so we need to clean it up
239 * even if HuC loading is off.
241 delayed_huc_load_complete(huc);
242 i915_sw_fence_fini(&huc->delayed_load.fence);
245 int intel_huc_sanitize(struct intel_huc *huc)
247 delayed_huc_load_complete(huc);
248 intel_uc_fw_sanitize(&huc->fw);
252 static bool vcs_supported(struct intel_gt *gt)
254 intel_engine_mask_t mask = gt->info.engine_mask;
257 * We reach here from i915_driver_early_probe for the primary GT before
258 * its engine mask is set, so we use the device info engine mask for it;
259 * this means we're not taking VCS fusing into account, but if the
260 * primary GT supports VCS engines we expect at least one of them to
261 * remain unfused so we're fine.
262 * For other GTs we expect the GT-specific mask to be set before we
263 * call this function.
265 GEM_BUG_ON(!gt_is_root(gt) && !gt->info.engine_mask);
268 mask = RUNTIME_INFO(gt->i915)->platform_engine_mask;
270 mask = gt->info.engine_mask;
272 return __ENGINE_INSTANCES_MASK(mask, VCS0, I915_MAX_VCS);
275 void intel_huc_init_early(struct intel_huc *huc)
277 struct drm_i915_private *i915 = huc_to_gt(huc)->i915;
278 struct intel_gt *gt = huc_to_gt(huc);
280 intel_uc_fw_init_early(&huc->fw, INTEL_UC_FW_TYPE_HUC, true);
283 * we always init the fence as already completed, even if HuC is not
284 * supported. This way we don't have to distinguish between HuC not
285 * supported/disabled or already loaded, and can focus on if the load
286 * is currently in progress (fence not complete) or not, which is what
287 * we care about for stalling userspace submissions.
289 delayed_huc_load_init(huc);
291 if (!vcs_supported(gt)) {
292 intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_NOT_SUPPORTED);
296 if (GRAPHICS_VER(i915) >= 11) {
297 huc->status[INTEL_HUC_AUTH_BY_GUC].reg = GEN11_HUC_KERNEL_LOAD_INFO;
298 huc->status[INTEL_HUC_AUTH_BY_GUC].mask = HUC_LOAD_SUCCESSFUL;
299 huc->status[INTEL_HUC_AUTH_BY_GUC].value = HUC_LOAD_SUCCESSFUL;
301 huc->status[INTEL_HUC_AUTH_BY_GUC].reg = HUC_STATUS2;
302 huc->status[INTEL_HUC_AUTH_BY_GUC].mask = HUC_FW_VERIFIED;
303 huc->status[INTEL_HUC_AUTH_BY_GUC].value = HUC_FW_VERIFIED;
307 huc->status[INTEL_HUC_AUTH_BY_GSC].reg = GEN11_HUC_KERNEL_LOAD_INFO;
308 huc->status[INTEL_HUC_AUTH_BY_GSC].mask = HUC_LOAD_SUCCESSFUL;
309 huc->status[INTEL_HUC_AUTH_BY_GSC].value = HUC_LOAD_SUCCESSFUL;
311 huc->status[INTEL_HUC_AUTH_BY_GSC].reg = HECI_FWSTS5(MTL_GSC_HECI1_BASE);
312 huc->status[INTEL_HUC_AUTH_BY_GSC].mask = HECI_FWSTS5_HUC_AUTH_DONE;
313 huc->status[INTEL_HUC_AUTH_BY_GSC].value = HECI_FWSTS5_HUC_AUTH_DONE;
317 #define HUC_LOAD_MODE_STRING(x) (x ? "GSC" : "legacy")
318 static int check_huc_loading_mode(struct intel_huc *huc)
320 struct intel_gt *gt = huc_to_gt(huc);
321 bool gsc_enabled = huc->fw.has_gsc_headers;
324 * The fuse for HuC load via GSC is only valid on platforms that have
327 if (HAS_GUC_DEPRIVILEGE(gt->i915))
328 huc->loaded_via_gsc = intel_uncore_read(gt->uncore, GUC_SHIM_CONTROL2) &
331 if (huc->loaded_via_gsc && !gsc_enabled) {
332 huc_err(huc, "HW requires a GSC-enabled blob, but we found a legacy one\n");
337 * On newer platforms we have GSC-enabled binaries but we load the HuC
338 * via DMA. To do so we need to find the location of the legacy-style
339 * binary inside the GSC-enabled one, which we do at fetch time. Make
340 * sure that we were able to do so if the fuse says we need to load via
341 * DMA and the binary is GSC-enabled.
343 if (!huc->loaded_via_gsc && gsc_enabled && !huc->fw.dma_start_offset) {
344 huc_err(huc, "HW in DMA mode, but we have an incompatible GSC-enabled blob\n");
349 * If the HuC is loaded via GSC, we need to be able to access the GSC.
350 * On DG2 this is done via the mei components, while on newer platforms
351 * it is done via the GSCCS,
353 if (huc->loaded_via_gsc) {
354 if (IS_DG2(gt->i915)) {
355 if (!IS_ENABLED(CONFIG_INTEL_MEI_PXP) ||
356 !IS_ENABLED(CONFIG_INTEL_MEI_GSC)) {
357 huc_info(huc, "can't load due to missing mei modules\n");
361 if (!HAS_ENGINE(gt, GSC0)) {
362 huc_info(huc, "can't load due to missing GSCCS\n");
368 huc_dbg(huc, "loaded by GSC = %s\n", str_yes_no(huc->loaded_via_gsc));
373 int intel_huc_init(struct intel_huc *huc)
375 struct intel_gt *gt = huc_to_gt(huc);
378 err = check_huc_loading_mode(huc);
382 if (HAS_ENGINE(gt, GSC0)) {
383 struct i915_vma *vma;
385 vma = intel_guc_allocate_vma(>->uc.guc, PXP43_HUC_AUTH_INOUT_SIZE * 2);
388 huc_info(huc, "Failed to allocate heci pkt\n");
395 err = intel_uc_fw_init(&huc->fw);
399 intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOADABLE);
405 i915_vma_unpin_and_release(&huc->heci_pkt, 0);
407 intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_INIT_FAIL);
408 huc_info(huc, "initialization failed %pe\n", ERR_PTR(err));
412 void intel_huc_fini(struct intel_huc *huc)
415 * the fence is initialized in init_early, so we need to clean it up
416 * even if HuC loading is off.
418 delayed_huc_load_fini(huc);
421 i915_vma_unpin_and_release(&huc->heci_pkt, 0);
423 if (intel_uc_fw_is_loadable(&huc->fw))
424 intel_uc_fw_fini(&huc->fw);
427 void intel_huc_suspend(struct intel_huc *huc)
429 if (!intel_uc_fw_is_loadable(&huc->fw))
433 * in the unlikely case that we're suspending before the GSC has
434 * completed its loading sequence, just stop waiting. We'll restart
437 delayed_huc_load_complete(huc);
440 static const char *auth_mode_string(struct intel_huc *huc,
441 enum intel_huc_authentication_type type)
443 bool partial = huc->fw.has_gsc_headers && type == INTEL_HUC_AUTH_BY_GUC;
445 return partial ? "clear media" : "all workloads";
448 int intel_huc_wait_for_auth_complete(struct intel_huc *huc,
449 enum intel_huc_authentication_type type)
451 struct intel_gt *gt = huc_to_gt(huc);
454 ret = __intel_wait_for_register(gt->uncore,
455 huc->status[type].reg,
456 huc->status[type].mask,
457 huc->status[type].value,
460 /* mark the load process as complete even if the wait failed */
461 delayed_huc_load_complete(huc);
464 huc_err(huc, "firmware not verified for %s: %pe\n",
465 auth_mode_string(huc, type), ERR_PTR(ret));
466 intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
470 intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING);
471 huc_info(huc, "authenticated for %s\n", auth_mode_string(huc, type));
476 * intel_huc_auth() - Authenticate HuC uCode
477 * @huc: intel_huc structure
478 * @type: authentication type (via GuC or via GSC)
480 * Called after HuC and GuC firmware loading during intel_uc_init_hw().
482 * This function invokes the GuC action to authenticate the HuC firmware,
483 * passing the offset of the RSA signature to intel_guc_auth_huc(). It then
484 * waits for up to 50ms for firmware verification ACK.
486 int intel_huc_auth(struct intel_huc *huc, enum intel_huc_authentication_type type)
488 struct intel_gt *gt = huc_to_gt(huc);
489 struct intel_guc *guc = >->uc.guc;
492 if (!intel_uc_fw_is_loaded(&huc->fw))
495 /* GSC will do the auth with the load */
496 if (intel_huc_is_loaded_by_gsc(huc))
499 if (intel_huc_is_authenticated(huc, type))
502 ret = i915_inject_probe_error(gt->i915, -ENXIO);
507 case INTEL_HUC_AUTH_BY_GUC:
508 ret = intel_guc_auth_huc(guc, intel_guc_ggtt_offset(guc, huc->fw.rsa_data));
510 case INTEL_HUC_AUTH_BY_GSC:
511 ret = intel_huc_fw_auth_via_gsccs(huc);
520 /* Check authentication status, it should be done by now */
521 ret = intel_huc_wait_for_auth_complete(huc, type);
528 huc_probe_error(huc, "%s authentication failed %pe\n",
529 auth_mode_string(huc, type), ERR_PTR(ret));
533 bool intel_huc_is_authenticated(struct intel_huc *huc,
534 enum intel_huc_authentication_type type)
536 struct intel_gt *gt = huc_to_gt(huc);
537 intel_wakeref_t wakeref;
540 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
541 status = intel_uncore_read(gt->uncore, huc->status[type].reg);
543 return (status & huc->status[type].mask) == huc->status[type].value;
546 static bool huc_is_fully_authenticated(struct intel_huc *huc)
548 struct intel_uc_fw *huc_fw = &huc->fw;
550 if (!huc_fw->has_gsc_headers)
551 return intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GUC);
552 else if (intel_huc_is_loaded_by_gsc(huc) || HAS_ENGINE(huc_to_gt(huc), GSC0))
553 return intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC);
559 * intel_huc_check_status() - check HuC status
560 * @huc: intel_huc structure
562 * This function reads status register to verify if HuC
563 * firmware was successfully loaded.
565 * The return values match what is expected for the I915_PARAM_HUC_STATUS
568 int intel_huc_check_status(struct intel_huc *huc)
570 struct intel_uc_fw *huc_fw = &huc->fw;
572 switch (__intel_uc_fw_status(huc_fw)) {
573 case INTEL_UC_FIRMWARE_NOT_SUPPORTED:
575 case INTEL_UC_FIRMWARE_DISABLED:
577 case INTEL_UC_FIRMWARE_MISSING:
579 case INTEL_UC_FIRMWARE_ERROR:
581 case INTEL_UC_FIRMWARE_INIT_FAIL:
583 case INTEL_UC_FIRMWARE_LOAD_FAIL:
590 * GSC-enabled binaries loaded via DMA are first partially
591 * authenticated by GuC and then fully authenticated by GSC
593 if (huc_is_fully_authenticated(huc))
594 return 1; /* full auth */
595 else if (huc_fw->has_gsc_headers && !intel_huc_is_loaded_by_gsc(huc) &&
596 intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GUC))
597 return 2; /* clear media only */
602 static bool huc_has_delayed_load(struct intel_huc *huc)
604 return intel_huc_is_loaded_by_gsc(huc) &&
605 (huc->delayed_load.status != INTEL_HUC_DELAYED_LOAD_ERROR);
608 void intel_huc_update_auth_status(struct intel_huc *huc)
610 if (!intel_uc_fw_is_loadable(&huc->fw))
613 if (!huc->fw.has_gsc_headers)
616 if (huc_is_fully_authenticated(huc))
617 intel_uc_fw_change_status(&huc->fw,
618 INTEL_UC_FIRMWARE_RUNNING);
619 else if (huc_has_delayed_load(huc))
620 huc_delayed_load_start(huc);
624 * intel_huc_load_status - dump information about HuC load status
626 * @p: the &drm_printer
628 * Pretty printer for HuC load status.
630 void intel_huc_load_status(struct intel_huc *huc, struct drm_printer *p)
632 struct intel_gt *gt = huc_to_gt(huc);
633 intel_wakeref_t wakeref;
635 if (!intel_huc_is_supported(huc)) {
636 drm_printf(p, "HuC not supported\n");
640 if (!intel_huc_is_wanted(huc)) {
641 drm_printf(p, "HuC disabled\n");
645 intel_uc_fw_dump(&huc->fw, p);
647 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
648 drm_printf(p, "HuC status: 0x%08x\n",
649 intel_uncore_read(gt->uncore, huc->status[INTEL_HUC_AUTH_BY_GUC].reg));