1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
15 #include <linux/vmalloc.h>
17 #include <trace/events/habanalabs.h>
19 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
21 static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = {
22 [COMMS_NOOP] = __stringify(COMMS_NOOP),
23 [COMMS_CLR_STS] = __stringify(COMMS_CLR_STS),
24 [COMMS_RST_STATE] = __stringify(COMMS_RST_STATE),
25 [COMMS_PREP_DESC] = __stringify(COMMS_PREP_DESC),
26 [COMMS_DATA_RDY] = __stringify(COMMS_DATA_RDY),
27 [COMMS_EXEC] = __stringify(COMMS_EXEC),
28 [COMMS_RST_DEV] = __stringify(COMMS_RST_DEV),
29 [COMMS_GOTO_WFE] = __stringify(COMMS_GOTO_WFE),
30 [COMMS_SKIP_BMC] = __stringify(COMMS_SKIP_BMC),
31 [COMMS_PREP_DESC_ELBI] = __stringify(COMMS_PREP_DESC_ELBI),
34 static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = {
35 [COMMS_STS_NOOP] = __stringify(COMMS_STS_NOOP),
36 [COMMS_STS_ACK] = __stringify(COMMS_STS_ACK),
37 [COMMS_STS_OK] = __stringify(COMMS_STS_OK),
38 [COMMS_STS_ERR] = __stringify(COMMS_STS_ERR),
39 [COMMS_STS_VALID_ERR] = __stringify(COMMS_STS_VALID_ERR),
40 [COMMS_STS_TIMEOUT_ERR] = __stringify(COMMS_STS_TIMEOUT_ERR),
43 static char *extract_fw_ver_from_str(const char *fw_str)
45 char *str, *fw_ver, *whitespace;
48 fw_ver = kmalloc(VERSION_MAX_LEN, GFP_KERNEL);
52 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
56 /* Skip the fw- part */
58 ver_offset = str - fw_str;
60 /* Copy until the next whitespace */
61 whitespace = strnstr(str, " ", VERSION_MAX_LEN - ver_offset);
65 strscpy(fw_ver, str, whitespace - str + 1);
75 * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32.
76 * @str: the given string
77 * @ver_num: the pointer to the extracted u32 to be returned to the caller.
78 * @given_char: the given char at the end of the u32 in the string
80 * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL
82 static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char)
84 char num_str[8] = {}, *ch;
86 ch = strchrnul(str, given_char);
87 if (*ch == '\0' || ch == str || ch - str >= sizeof(num_str))
90 memcpy(num_str, str, ch - str);
91 if (kstrtou32(num_str, 10, ver_num))
97 * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor
98 * from the version string
99 * @hdev: pointer to the hl_device
100 * @fw_str: the FW's version string
102 * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver.
104 * fw_str is expected to have one of two possible formats, examples:
105 * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3'
106 * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3'
107 * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0.
109 * Return: 0 for success or a negative error code for failure.
111 static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str)
115 end = strnstr(fw_str, "-rc-", VERSION_MAX_LEN);
120 end = strnstr(fw_str, "-fw-", VERSION_MAX_LEN);
128 for (start = end - 1; start != fw_str; start--) {
136 /* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */
138 start = extract_u32_until_given_char(start, &hdev->fw_sw_major_ver, '.');
143 start = extract_u32_until_given_char(start, &hdev->fw_sw_minor_ver, '.');
148 start = extract_u32_until_given_char(start, &hdev->fw_sw_sub_minor_ver, '-');
155 hdev->fw_sw_major_ver = 0;
156 hdev->fw_sw_minor_ver = 0;
157 hdev->fw_sw_sub_minor_ver = 0;
162 * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string.
163 * @hdev: pointer to the hl_device
164 * @preboot_ver: the FW's version string
166 * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3
167 * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver.
169 * Return: 0 on success, negative error code for failure.
171 static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)
173 preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_major_ver, '.');
175 dev_err(hdev->dev, "Error parsing preboot major version\n");
181 preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_minor_ver, '.');
183 dev_err(hdev->dev, "Error parsing preboot minor version\n");
189 hdev->fw_inner_major_ver = 0;
190 hdev->fw_inner_minor_ver = 0;
194 static int hl_request_fw(struct hl_device *hdev,
195 const struct firmware **firmware_p,
201 rc = request_firmware(firmware_p, fw_name, hdev->dev);
203 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
208 fw_size = (*firmware_p)->size;
209 if ((fw_size % 4) != 0) {
210 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
216 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
218 if (fw_size > FW_FILE_MAX_SIZE) {
220 "FW file size %zu exceeds maximum of %u bytes\n",
221 fw_size, FW_FILE_MAX_SIZE);
229 release_firmware(*firmware_p);
235 * hl_release_firmware() - release FW
239 * note: this inline function added to serve as a comprehensive mirror for the
240 * hl_request_fw function.
242 static inline void hl_release_firmware(const struct firmware *fw)
244 release_firmware(fw);
248 * hl_fw_copy_fw_to_device() - copy FW to device
250 * @hdev: pointer to hl_device structure.
252 * @dst: IO memory mapped address space to copy firmware to
253 * @src_offset: offset in src FW to copy from
254 * @size: amount of bytes to copy (0 to copy the whole binary)
256 * actual copy of FW binary data to device, shared by static and dynamic loaders
258 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
259 const struct firmware *fw, void __iomem *dst,
260 u32 src_offset, u32 size)
264 /* size 0 indicates to copy the whole file */
268 if (src_offset + size > fw->size) {
270 "size to copy(%u) and offset(%u) are invalid\n",
275 fw_data = (const void *) fw->data;
277 memcpy_toio(dst, fw_data + src_offset, size);
282 * hl_fw_copy_msg_to_device() - copy message to device
284 * @hdev: pointer to hl_device structure.
286 * @dst: IO memory mapped address space to copy firmware to
287 * @src_offset: offset in src message to copy from
288 * @size: amount of bytes to copy (0 to copy the whole binary)
290 * actual copy of message data to device.
292 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
293 struct lkd_msg_comms *msg, void __iomem *dst,
294 u32 src_offset, u32 size)
298 /* size 0 indicates to copy the whole file */
300 size = sizeof(struct lkd_msg_comms);
302 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
304 "size to copy(%u) and offset(%u) are invalid\n",
309 msg_data = (void *) msg;
311 memcpy_toio(dst, msg_data + src_offset, size);
317 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
319 * @hdev: pointer to hl_device structure.
320 * @fw_name: the firmware image name
321 * @dst: IO memory mapped address space to copy firmware to
322 * @src_offset: offset in src FW to copy from
323 * @size: amount of bytes to copy (0 to copy the whole binary)
325 * Copy fw code from firmware file to device memory.
327 * Return: 0 on success, non-zero for failure.
329 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
330 void __iomem *dst, u32 src_offset, u32 size)
332 const struct firmware *fw;
335 rc = hl_request_fw(hdev, &fw, fw_name);
339 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
341 hl_release_firmware(fw);
345 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value)
347 struct cpucp_packet pkt = {};
349 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
350 pkt.value = cpu_to_le64(value);
352 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
355 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
356 u16 len, u32 timeout, u64 *result)
358 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
359 struct asic_fixed_properties *prop = &hdev->asic_prop;
360 struct cpucp_packet *pkt;
361 dma_addr_t pkt_dma_addr;
362 struct hl_bd *sent_bd;
363 u32 tmp, expected_ack_val, pi, opcode;
366 pkt = hl_cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr);
369 "Failed to allocate DMA memory for packet to CPU\n");
373 memcpy(pkt, msg, len);
375 mutex_lock(&hdev->send_cpu_message_lock);
377 /* CPU-CP messages can be sent during soft-reset */
378 if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
383 if (hdev->device_cpu_disabled) {
388 /* set fence to a non valid value */
389 pkt->fence = cpu_to_le32(UINT_MAX);
393 * The CPU queue is a synchronous queue with an effective depth of
394 * a single entry (although it is allocated with room for multiple
395 * entries). We lock on it using 'send_cpu_message_lock' which
396 * serializes accesses to the CPU queue.
397 * Which means that we don't need to lock the access to the entire H/W
398 * queues module when submitting a JOB to the CPU queue.
400 hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
402 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
403 expected_ack_val = queue->pi;
405 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
407 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
408 (tmp == expected_ack_val), 1000,
411 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
413 if (rc == -ETIMEDOUT) {
414 /* If FW performed reset just before sending it a packet, we will get a timeout.
415 * This is expected behavior, hence no need for error message.
417 if (!hl_device_operational(hdev, NULL) && !hdev->reset_info.in_compute_reset)
418 dev_dbg(hdev->dev, "Device CPU packet timeout (0x%x) due to FW reset\n",
421 dev_err(hdev->dev, "Device CPU packet timeout (status = 0x%x)\n", tmp);
422 hdev->device_cpu_disabled = true;
426 tmp = le32_to_cpu(pkt->ctl);
428 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
430 opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT;
432 if (!prop->supports_advanced_cpucp_rc) {
433 dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode);
435 goto scrub_descriptor;
439 case cpucp_packet_invalid:
441 "CPU packet %d is not supported by F/W\n", opcode);
443 case cpucp_packet_fault:
445 "F/W failed processing CPU packet %d\n", opcode);
447 case cpucp_packet_invalid_pkt:
449 "CPU packet %d is not supported by F/W\n", opcode);
451 case cpucp_packet_invalid_params:
453 "F/W reports invalid parameters for CPU packet %d\n", opcode);
458 "Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode);
461 /* propagate the return code from the f/w to the callers who want to check it */
468 *result = le64_to_cpu(pkt->result);
472 /* Scrub previous buffer descriptor 'ctl' field which contains the
473 * previous PI value written during packet submission.
474 * We must do this or else F/W can read an old value upon queue wraparound.
476 sent_bd = queue->kernel_address;
477 sent_bd += hl_pi_2_offset(pi);
478 sent_bd->ctl = cpu_to_le32(UINT_MAX);
481 mutex_unlock(&hdev->send_cpu_message_lock);
483 hl_cpu_accessible_dma_pool_free(hdev, len, pkt);
488 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
490 struct cpucp_packet pkt;
494 memset(&pkt, 0, sizeof(pkt));
496 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
497 CPUCP_PKT_CTL_OPCODE_SHIFT);
498 pkt.value = cpu_to_le64(event_type);
500 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
504 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
509 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
512 struct cpucp_unmask_irq_arr_packet *pkt;
513 size_t total_pkt_size;
517 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
520 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
521 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
523 /* total_pkt_size is casted to u16 later on */
524 if (total_pkt_size > USHRT_MAX) {
525 dev_err(hdev->dev, "too many elements in IRQ array\n");
529 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
533 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
534 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
536 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
537 CPUCP_PKT_CTL_OPCODE_SHIFT);
539 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
540 total_pkt_size, 0, &result);
543 dev_err(hdev->dev, "failed to unmask IRQ array\n");
550 int hl_fw_test_cpu_queue(struct hl_device *hdev)
552 struct cpucp_packet test_pkt = {};
556 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
557 CPUCP_PKT_CTL_OPCODE_SHIFT);
558 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
560 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
561 sizeof(test_pkt), 0, &result);
564 if (result != CPUCP_PACKET_FENCE_VAL)
566 "CPU queue test failed (%#08llx)\n", result);
568 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
574 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
575 dma_addr_t *dma_handle)
579 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
581 *dma_handle = hdev->cpu_accessible_dma_address +
582 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
584 return (void *) (uintptr_t) kernel_addr;
587 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
590 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
594 int hl_fw_send_soft_reset(struct hl_device *hdev)
596 struct cpucp_packet pkt;
599 memset(&pkt, 0, sizeof(pkt));
600 pkt.ctl = cpu_to_le32(CPUCP_PACKET_SOFT_RESET << CPUCP_PKT_CTL_OPCODE_SHIFT);
601 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
603 dev_err(hdev->dev, "failed to send soft-reset msg (err = %d)\n", rc);
608 int hl_fw_send_device_activity(struct hl_device *hdev, bool open)
610 struct cpucp_packet pkt;
613 memset(&pkt, 0, sizeof(pkt));
614 pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
615 pkt.value = cpu_to_le64(open);
616 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
618 dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open);
623 int hl_fw_send_heartbeat(struct hl_device *hdev)
625 struct cpucp_packet hb_pkt;
629 memset(&hb_pkt, 0, sizeof(hb_pkt));
630 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
631 CPUCP_PKT_CTL_OPCODE_SHIFT);
632 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
634 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
635 sizeof(hb_pkt), 0, &result);
637 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
640 if (le32_to_cpu(hb_pkt.status_mask) &
641 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
642 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
649 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
652 bool err_exists = false;
654 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
657 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
659 "Device boot error - DRAM initialization failed\n");
663 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
664 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
668 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
670 "Device boot error - Thermal Sensor initialization failed\n");
674 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
675 if (hdev->bmc_enable) {
677 "Device boot error - Skipped waiting for BMC\n");
681 "Device boot message - Skipped waiting for BMC\n");
682 /* This is an info so we don't want it to disable the
685 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
689 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
691 "Device boot error - Serdes data from BMC not available\n");
695 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
697 "Device boot error - NIC F/W initialization failed\n");
701 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
703 "Device boot warning - security not ready\n");
707 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
708 dev_err(hdev->dev, "Device boot error - security failure\n");
712 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
713 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
717 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
718 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
722 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
723 dev_err(hdev->dev, "Device boot error - PLL failure\n");
727 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
728 /* Ignore this bit, don't prevent driver loading */
729 dev_dbg(hdev->dev, "device unusable status is set\n");
730 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
733 if (err_val & CPU_BOOT_ERR0_BINNING_FAIL) {
734 dev_err(hdev->dev, "Device boot error - binning failure\n");
738 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
739 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
741 if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {
742 dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");
746 /* All warnings should go here in order not to reach the unknown error validation */
747 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
749 "Device boot warning - Skipped DRAM initialization\n");
750 /* This is a warning so we don't want it to disable the
753 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
756 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
758 "Device boot warning - Failed to load preboot primary image\n");
759 /* This is a warning so we don't want it to disable the
760 * device as we have a secondary preboot image
762 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
765 if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
767 "Device boot warning - TPM failure\n");
768 /* This is a warning so we don't want it to disable the
771 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
774 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
776 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
780 /* return error only if it's in the predefined mask */
781 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
782 lower_32_bits(hdev->boot_error_status_mask)))
788 /* placeholder for ERR1 as no errors defined there yet */
789 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
793 * keep this variable to preserve the logic of the function.
794 * this way it would require less modifications when error will be
797 bool err_exists = false;
799 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
802 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
803 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
805 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
807 "Device boot error - unknown ERR1 error 0x%08x\n",
812 /* return error only if it's in the predefined mask */
813 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
814 upper_32_bits(hdev->boot_error_status_mask)))
820 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
821 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
822 u32 cpu_boot_dev_status1_reg)
824 u32 err_val, status_val;
825 bool err_exists = false;
827 /* Some of the firmware status codes are deprecated in newer f/w
828 * versions. In those versions, the errors are reported
829 * in different registers. Therefore, we need to check those
830 * registers and print the exact errors. Moreover, there
831 * may be multiple errors, so we need to report on each error
832 * separately. Some of the error codes might indicate a state
833 * that is not an error per-se, but it is an error in production
836 err_val = RREG32(boot_err0_reg);
837 status_val = RREG32(cpu_boot_dev_status0_reg);
838 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
840 err_val = RREG32(boot_err1_reg);
841 status_val = RREG32(cpu_boot_dev_status1_reg);
842 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
850 int hl_fw_cpucp_info_get(struct hl_device *hdev,
851 u32 sts_boot_dev_sts0_reg,
852 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
855 struct asic_fixed_properties *prop = &hdev->asic_prop;
856 struct cpucp_packet pkt = {};
857 dma_addr_t cpucp_info_dma_addr;
858 void *cpucp_info_cpu_addr;
863 cpucp_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, sizeof(struct cpucp_info),
864 &cpucp_info_dma_addr);
865 if (!cpucp_info_cpu_addr) {
867 "Failed to allocate DMA memory for CPU-CP info packet\n");
871 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
873 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
874 CPUCP_PKT_CTL_OPCODE_SHIFT);
875 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
876 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
878 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
879 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
882 "Failed to handle CPU-CP info pkt, error %d\n", rc);
886 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
887 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
889 dev_err(hdev->dev, "Errors in device boot\n");
893 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
894 sizeof(prop->cpucp_info));
896 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
899 "Failed to build hwmon channel info, error %d\n", rc);
904 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
906 dev_info(hdev->dev, "Linux version %s", kernel_ver);
910 /* assume EQ code doesn't need to check eqe index */
911 hdev->event_queue.check_eqe_index = false;
913 /* Read FW application security bits again */
914 if (prop->fw_cpu_boot_dev_sts0_valid) {
915 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
916 if (prop->fw_app_cpu_boot_dev_sts0 &
917 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
918 hdev->event_queue.check_eqe_index = true;
921 if (prop->fw_cpu_boot_dev_sts1_valid)
922 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
925 hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr);
930 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
932 struct cpucp_array_data_packet *pkt;
933 size_t total_pkt_size, data_size;
937 /* skip sending this info for unsupported ASICs */
938 if (!hdev->asic_funcs->get_msi_info)
941 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
942 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
944 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
945 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
947 /* total_pkt_size is casted to u16 later on */
948 if (total_pkt_size > USHRT_MAX) {
949 dev_err(hdev->dev, "CPUCP array data is too big\n");
953 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
957 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
959 memset((void *) &pkt->data, 0xFF, data_size);
960 hdev->asic_funcs->get_msi_info(pkt->data);
962 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
963 CPUCP_PKT_CTL_OPCODE_SHIFT);
965 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
966 total_pkt_size, 0, &result);
969 * in case packet result is invalid it means that FW does not support
970 * this feature and will use default/hard coded MSI values. no reason
973 if (rc && result == cpucp_packet_invalid)
977 dev_err(hdev->dev, "failed to send CPUCP array data\n");
984 int hl_fw_cpucp_handshake(struct hl_device *hdev,
985 u32 sts_boot_dev_sts0_reg,
986 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
991 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
992 sts_boot_dev_sts1_reg, boot_err0_reg,
997 return hl_fw_send_msi_info_msg(hdev);
1000 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
1002 struct cpucp_packet pkt = {};
1003 void *eeprom_info_cpu_addr;
1004 dma_addr_t eeprom_info_dma_addr;
1008 eeprom_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, max_size,
1009 &eeprom_info_dma_addr);
1010 if (!eeprom_info_cpu_addr) {
1012 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
1016 memset(eeprom_info_cpu_addr, 0, max_size);
1018 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
1019 CPUCP_PKT_CTL_OPCODE_SHIFT);
1020 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
1021 pkt.data_max_size = cpu_to_le32(max_size);
1023 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1024 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
1028 "Failed to handle CPU-CP EEPROM packet, error %d\n",
1033 /* result contains the actual size */
1034 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
1037 hl_cpu_accessible_dma_pool_free(hdev, max_size, eeprom_info_cpu_addr);
1042 int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data)
1044 struct cpucp_monitor_dump *mon_dump_cpu_addr;
1045 dma_addr_t mon_dump_dma_addr;
1046 struct cpucp_packet pkt = {};
1053 data_size = sizeof(struct cpucp_monitor_dump);
1054 mon_dump_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, data_size, &mon_dump_dma_addr);
1055 if (!mon_dump_cpu_addr) {
1057 "Failed to allocate DMA memory for CPU-CP monitor-dump packet\n");
1061 memset(mon_dump_cpu_addr, 0, data_size);
1063 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MONITOR_DUMP_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1064 pkt.addr = cpu_to_le64(mon_dump_dma_addr);
1065 pkt.data_max_size = cpu_to_le32(data_size);
1067 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1068 HL_CPUCP_MON_DUMP_TIMEOUT_USEC, &result);
1070 dev_err(hdev->dev, "Failed to handle CPU-CP monitor-dump packet, error %d\n", rc);
1074 /* result contains the actual size */
1075 src_ptr = (__le32 *) mon_dump_cpu_addr;
1077 for (i = 0; i < (data_size / sizeof(u32)); i++) {
1078 *dst_ptr = le32_to_cpu(*src_ptr);
1084 hl_cpu_accessible_dma_pool_free(hdev, data_size, mon_dump_cpu_addr);
1089 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
1090 struct hl_info_pci_counters *counters)
1092 struct cpucp_packet pkt = {};
1096 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1097 CPUCP_PKT_CTL_OPCODE_SHIFT);
1099 /* Fetch PCI rx counter */
1100 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
1101 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1102 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1105 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1108 counters->rx_throughput = result;
1110 memset(&pkt, 0, sizeof(pkt));
1111 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1112 CPUCP_PKT_CTL_OPCODE_SHIFT);
1114 /* Fetch PCI tx counter */
1115 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
1116 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1117 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1120 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1123 counters->tx_throughput = result;
1125 /* Fetch PCI replay counter */
1126 memset(&pkt, 0, sizeof(pkt));
1127 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
1128 CPUCP_PKT_CTL_OPCODE_SHIFT);
1130 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1131 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1134 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1137 counters->replay_cnt = (u32) result;
1142 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
1144 struct cpucp_packet pkt = {};
1148 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
1149 CPUCP_PKT_CTL_OPCODE_SHIFT);
1151 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1152 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1155 "Failed to handle CpuCP total energy pkt, error %d\n",
1160 *total_energy = result;
1165 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
1166 enum pll_index *pll_index)
1168 struct asic_fixed_properties *prop = &hdev->asic_prop;
1169 u8 pll_byte, pll_bit_off;
1173 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
1174 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
1178 * in case we are working with legacy FW (each asic has unique
1179 * PLL numbering) use the driver based index as they are
1180 * aligned with fw legacy numbering
1182 *pll_index = input_pll_index;
1186 /* retrieve a FW compatible PLL index based on
1187 * ASIC specific user request
1189 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
1190 if (fw_pll_idx < 0) {
1191 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
1192 input_pll_index, fw_pll_idx);
1196 /* PLL map is a u8 array */
1197 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
1198 pll_bit_off = fw_pll_idx & 0x7;
1200 if (!(pll_byte & BIT(pll_bit_off))) {
1201 dev_err(hdev->dev, "PLL index %d is not supported\n",
1206 *pll_index = fw_pll_idx;
1211 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
1214 struct cpucp_packet pkt;
1215 enum pll_index used_pll_idx;
1219 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
1223 memset(&pkt, 0, sizeof(pkt));
1225 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
1226 CPUCP_PKT_CTL_OPCODE_SHIFT);
1227 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
1229 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1230 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1232 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
1236 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
1237 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
1238 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
1239 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
1244 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
1246 struct cpucp_packet pkt;
1250 memset(&pkt, 0, sizeof(pkt));
1252 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
1253 CPUCP_PKT_CTL_OPCODE_SHIFT);
1254 pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
1256 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1257 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1259 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
1268 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
1269 struct cpucp_hbm_row_info *info)
1271 struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1272 dma_addr_t cpucp_repl_rows_info_dma_addr;
1273 struct cpucp_packet pkt = {};
1277 cpucp_repl_rows_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev,
1278 sizeof(struct cpucp_hbm_row_info),
1279 &cpucp_repl_rows_info_dma_addr);
1280 if (!cpucp_repl_rows_info_cpu_addr) {
1282 "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1286 memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1288 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1289 CPUCP_PKT_CTL_OPCODE_SHIFT);
1290 pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1291 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1293 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1294 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1297 "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1301 memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1304 hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_hbm_row_info),
1305 cpucp_repl_rows_info_cpu_addr);
1310 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1312 struct cpucp_packet pkt;
1316 memset(&pkt, 0, sizeof(pkt));
1318 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1320 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1323 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1327 *pend_rows_num = (u32) result;
1332 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1334 struct cpucp_packet pkt;
1337 memset(&pkt, 0, sizeof(pkt));
1339 pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1340 pkt.value = cpu_to_le64(asid);
1342 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1343 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1346 "Failed on ASID configuration request for engine core, error %d\n",
1352 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1354 struct static_fw_load_mgr *static_loader =
1355 &hdev->fw_loader.static_loader;
1358 if (hdev->asic_prop.dynamic_fw_load) {
1359 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1360 COMMS_RST_DEV, 0, false,
1361 hdev->fw_loader.cpu_timeout);
1363 dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1365 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1369 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1371 struct static_fw_load_mgr *static_loader =
1372 &hdev->fw_loader.static_loader;
1375 if (hdev->device_cpu_is_halted)
1378 /* Stop device CPU to make sure nothing bad happens */
1379 if (hdev->asic_prop.dynamic_fw_load) {
1380 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1381 COMMS_GOTO_WFE, 0, false,
1382 hdev->fw_loader.cpu_timeout);
1384 dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1386 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1387 msleep(static_loader->cpu_reset_wait_msec);
1389 /* Must clear this register in order to prevent preboot
1390 * from reading WFE after reboot
1392 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1395 hdev->device_cpu_is_halted = true;
1398 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1400 /* Some of the status codes below are deprecated in newer f/w
1401 * versions but we keep them here for backward compatibility
1404 case CPU_BOOT_STATUS_NA:
1406 "Device boot progress - BTL/ROM did NOT run\n");
1408 case CPU_BOOT_STATUS_IN_WFE:
1410 "Device boot progress - Stuck inside WFE loop\n");
1412 case CPU_BOOT_STATUS_IN_BTL:
1414 "Device boot progress - Stuck in BTL\n");
1416 case CPU_BOOT_STATUS_IN_PREBOOT:
1418 "Device boot progress - Stuck in Preboot\n");
1420 case CPU_BOOT_STATUS_IN_SPL:
1422 "Device boot progress - Stuck in SPL\n");
1424 case CPU_BOOT_STATUS_IN_UBOOT:
1426 "Device boot progress - Stuck in u-boot\n");
1428 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1430 "Device boot progress - DRAM initialization failed\n");
1432 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1434 "Device boot progress - Cannot boot\n");
1436 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1438 "Device boot progress - Thermal Sensor initialization failed\n");
1440 case CPU_BOOT_STATUS_SECURITY_READY:
1442 "Device boot progress - Stuck in preboot after security initialization\n");
1446 "Device boot progress - Invalid or unexpected status code %d\n", status);
1451 int hl_fw_wait_preboot_ready(struct hl_device *hdev)
1453 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
1457 /* Need to check two possible scenarios:
1459 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1460 * the preboot is waiting for the boot fit
1462 * All other status values - for older firmwares where the uboot was
1463 * loaded from the FLASH
1465 rc = hl_poll_timeout(
1467 pre_fw_load->cpu_boot_status_reg,
1469 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1470 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1471 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1472 hdev->fw_poll_interval_usec,
1473 pre_fw_load->wait_for_preboot_timeout);
1476 detect_cpu_boot_status(hdev, status);
1477 dev_err(hdev->dev, "CPU boot ready timeout (status = %d)\n", status);
1479 /* If we read all FF, then something is totally wrong, no point
1480 * of reading specific errors
1483 fw_read_errors(hdev, pre_fw_load->boot_err0_reg,
1484 pre_fw_load->boot_err1_reg,
1485 pre_fw_load->sts_boot_dev_sts0_reg,
1486 pre_fw_load->sts_boot_dev_sts1_reg);
1490 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_PREBOOT_CPU;
1495 static int hl_fw_read_preboot_caps(struct hl_device *hdev)
1497 struct pre_fw_load_props *pre_fw_load;
1498 struct asic_fixed_properties *prop;
1502 prop = &hdev->asic_prop;
1503 pre_fw_load = &hdev->fw_loader.pre_fw_load;
1505 rc = hl_fw_wait_preboot_ready(hdev);
1510 * the registers DEV_STS* contain FW capabilities/features.
1511 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1513 * In the first read of this register we store the value of this
1514 * register ONLY if the register is enabled (which will be propagated
1515 * to next stages) and also mark the register as valid.
1516 * In case it is not enabled the stored value will be left 0- all
1517 * caps/features are off
1519 reg_val = RREG32(pre_fw_load->sts_boot_dev_sts0_reg);
1520 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1521 prop->fw_cpu_boot_dev_sts0_valid = true;
1522 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1525 reg_val = RREG32(pre_fw_load->sts_boot_dev_sts1_reg);
1526 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1527 prop->fw_cpu_boot_dev_sts1_valid = true;
1528 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1531 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1532 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1534 /* initialize FW loader once we know what load protocol is used */
1535 hdev->asic_funcs->init_firmware_loader(hdev);
1537 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1538 prop->dynamic_fw_load ? "dynamic" : "legacy");
1542 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1543 enum hl_fw_component fwc)
1545 struct asic_fixed_properties *prop = &hdev->asic_prop;
1546 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1547 struct static_fw_load_mgr *static_loader;
1548 char *dest, *boot_ver, *preboot_ver;
1553 static_loader = &hdev->fw_loader.static_loader;
1556 case FW_COMP_BOOT_FIT:
1557 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1558 dest = prop->uboot_ver;
1560 limit = static_loader->boot_fit_version_max_off;
1562 case FW_COMP_PREBOOT:
1563 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1564 dest = prop->preboot_ver;
1566 limit = static_loader->preboot_version_max_off;
1569 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1573 ver_off &= static_loader->sram_offset_mask;
1575 if (ver_off < limit) {
1577 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1580 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1582 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1586 if (fwc == FW_COMP_BOOT_FIT) {
1587 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1589 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1592 } else if (fwc == FW_COMP_PREBOOT) {
1593 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1595 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1596 strscpy(btl_ver, prop->preboot_ver,
1597 min((int) (preboot_ver - prop->preboot_ver),
1599 dev_info(hdev->dev, "%s\n", btl_ver);
1602 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1604 dev_info(hdev->dev, "preboot version %s\n",
1614 * hl_fw_preboot_update_state - update internal data structures during
1615 * handshake with preboot
1618 * @hdev: pointer to the habanalabs device structure
1620 * @return 0 on success, otherwise non-zero error code
1622 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1624 struct asic_fixed_properties *prop = &hdev->asic_prop;
1625 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1627 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1628 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1630 /* We read boot_dev_sts registers multiple times during boot:
1631 * 1. preboot - a. Check whether the security status bits are valid
1632 * b. Check whether fw security is enabled
1633 * c. Check whether hard reset is done by preboot
1634 * 2. boot cpu - a. Fetch boot cpu security status
1635 * b. Check whether hard reset is done by boot cpu
1636 * 3. FW application - a. Fetch fw application security status
1637 * b. Check whether hard reset is done by fw app
1639 prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1641 prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN);
1643 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1646 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1649 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1650 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1652 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1653 prop->fw_security_enabled ? "enabled" : "disabled");
1655 dev_dbg(hdev->dev, "GIC controller is %s\n",
1656 prop->gic_interrupts_enable ? "enabled" : "disabled");
1659 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1663 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1670 int hl_fw_read_preboot_status(struct hl_device *hdev)
1674 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1677 /* get FW pre-load parameters */
1678 hdev->asic_funcs->init_firmware_preload_params(hdev);
1681 * In order to determine boot method (static VS dynamic) we need to
1682 * read the boot caps register
1684 rc = hl_fw_read_preboot_caps(hdev);
1688 hl_fw_preboot_update_state(hdev);
1690 /* no need to read preboot status in dynamic load */
1691 if (hdev->asic_prop.dynamic_fw_load)
1694 return hl_fw_static_read_preboot_status(hdev);
1697 /* associate string with COMM status */
1698 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1699 [COMMS_STS_NOOP] = "NOOP",
1700 [COMMS_STS_ACK] = "ACK",
1701 [COMMS_STS_OK] = "OK",
1702 [COMMS_STS_ERR] = "ERR",
1703 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1704 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1708 * hl_fw_dynamic_report_error_status - report error status
1710 * @hdev: pointer to the habanalabs device structure
1711 * @status: value of FW status register
1712 * @expected_status: the expected status
1714 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1716 enum comms_sts expected_status)
1718 enum comms_sts comm_status =
1719 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1721 if (comm_status < COMMS_STS_INVLD_LAST)
1722 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1723 hl_dynamic_fw_status_str[comm_status],
1724 hl_dynamic_fw_status_str[expected_status]);
1726 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1728 hl_dynamic_fw_status_str[expected_status]);
1732 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1734 * @hdev: pointer to the habanalabs device structure
1735 * @fw_loader: managing structure for loading device's FW
1736 * @cmd: LKD to FW cmd code
1737 * @size: size of next FW component to be loaded (0 if not necessary)
1739 * LDK to FW exact command layout is defined at struct comms_command.
1740 * note: the size argument is used only when the next FW component should be
1741 * loaded, otherwise it shall be 0. the size is used by the FW in later
1742 * protocol stages and when sending only indicating the amount of memory
1743 * to be allocated by the FW to receive the next boot component.
1745 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1746 struct fw_load_mgr *fw_loader,
1747 enum comms_cmd cmd, unsigned int size)
1749 struct cpu_dyn_regs *dyn_regs;
1752 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1754 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1755 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1757 trace_habanalabs_comms_send_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1758 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1762 * hl_fw_dynamic_extract_fw_response - update the FW response
1764 * @hdev: pointer to the habanalabs device structure
1765 * @fw_loader: managing structure for loading device's FW
1766 * @response: FW response
1767 * @status: the status read from CPU status register
1769 * @return 0 on success, otherwise non-zero error code
1771 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1772 struct fw_load_mgr *fw_loader,
1773 struct fw_response *response,
1776 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1777 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1778 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1779 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1781 if ((response->ram_type != COMMS_SRAM) &&
1782 (response->ram_type != COMMS_DRAM)) {
1783 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1784 response->ram_type);
1792 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1794 * @hdev: pointer to the habanalabs device structure
1795 * @fw_loader: managing structure for loading device's FW
1796 * @expected_status: expected status to wait for
1797 * @timeout: timeout for status wait
1799 * @return 0 on success, otherwise non-zero error code
1801 * waiting for status from FW include polling the FW status register until
1802 * expected status is received or timeout occurs (whatever occurs first).
1804 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1805 struct fw_load_mgr *fw_loader,
1806 enum comms_sts expected_status,
1809 struct cpu_dyn_regs *dyn_regs;
1813 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1815 trace_habanalabs_comms_wait_status(hdev->dev, comms_sts_str_arr[expected_status]);
1817 /* Wait for expected status */
1818 rc = hl_poll_timeout(
1820 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1822 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1823 hdev->fw_comms_poll_interval_usec,
1827 hl_fw_dynamic_report_error_status(hdev, status,
1832 trace_habanalabs_comms_wait_status_done(hdev->dev, comms_sts_str_arr[expected_status]);
1835 * skip storing FW response for NOOP to preserve the actual desired
1838 if (expected_status == COMMS_STS_NOOP)
1841 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1842 &fw_loader->dynamic_loader.response,
1848 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1850 * @hdev: pointer to the habanalabs device structure
1851 * @fw_loader: managing structure for loading device's FW
1853 * @return 0 on success, otherwise non-zero error code
1855 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1856 * from FW) we need to clear the CPU status register in order to avoid garbage
1857 * between command cycles.
1858 * This is done by sending clear command and polling the CPU to LKD status
1859 * register to hold the status NOOP
1861 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1862 struct fw_load_mgr *fw_loader)
1864 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1866 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1867 fw_loader->cpu_timeout);
1871 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1873 * @hdev: pointer to the habanalabs device structure
1874 * @fw_loader: managing structure for loading device's FW
1875 * @cmd: LKD to FW cmd code
1876 * @size: size of next FW component to be loaded (0 if not necessary)
1877 * @wait_ok: if true also wait for OK response from FW
1878 * @timeout: timeout for status wait
1880 * @return 0 on success, otherwise non-zero error code
1883 * when sending protocol command we have the following steps:
1884 * - send clear (clear command and verify clear status register)
1885 * - send the actual protocol command
1886 * - wait for ACK on the protocol command
1889 * if, in addition, the specific protocol command should wait for OK then:
1895 * send clear: this is necessary in order to clear the status register to avoid
1896 * leftovers between command
1897 * NOOP command: necessary to avoid loop on the clear command by the FW
1899 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1900 struct fw_load_mgr *fw_loader,
1901 enum comms_cmd cmd, unsigned int size,
1902 bool wait_ok, u32 timeout)
1906 trace_habanalabs_comms_protocol_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1908 /* first send clear command to clean former commands */
1909 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1913 /* send the actual command */
1914 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1916 /* wait for ACK for the command */
1917 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1922 /* clear command to prepare for NOOP command */
1923 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1927 /* send the actual NOOP command */
1928 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1933 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1938 /* clear command to prepare for NOOP command */
1939 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1943 /* send the actual NOOP command */
1944 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1950 * hl_fw_compat_crc32 - CRC compatible with FW
1952 * @data: pointer to the data
1953 * @size: size of the data
1955 * @return the CRC32 result
1957 * NOTE: kernel's CRC32 differs from standard CRC32 calculation.
1958 * in order to be aligned we need to flip the bits of both the input
1959 * initial CRC and kernel's CRC32 result.
1960 * in addition both sides use initial CRC of 0,
1962 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1964 return ~crc32_le(~((u32)0), data, size);
1968 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1969 * transfer (image or descriptor) between
1972 * @hdev: pointer to the habanalabs device structure
1973 * @addr: device address of memory transfer
1974 * @size: memory transfer size
1975 * @region: PCI memory region
1977 * @return 0 on success, otherwise non-zero error code
1979 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1980 u64 addr, size_t size,
1981 struct pci_mem_region *region)
1985 /* now make sure that the memory transfer is within region's bounds */
1986 end_addr = addr + size;
1987 if (end_addr >= region->region_base + region->region_size) {
1989 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1995 * now make sure memory transfer is within predefined BAR bounds.
1996 * this is to make sure we do not need to set the bar (e.g. for DRAM
1999 if (end_addr >= region->region_base - region->offset_in_bar +
2002 "FW image beyond PCI BAR bounds\n");
2010 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
2012 * @hdev: pointer to the habanalabs device structure
2013 * @fw_loader: managing structure for loading device's FW
2014 * @fw_desc: the descriptor from FW
2016 * @return 0 on success, otherwise non-zero error code
2018 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
2019 struct fw_load_mgr *fw_loader,
2020 struct lkd_fw_comms_desc *fw_desc)
2022 struct pci_mem_region *region;
2023 enum pci_region region_id;
2030 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC)
2031 dev_dbg(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
2032 fw_desc->header.magic);
2034 if (fw_desc->header.version != HL_COMMS_DESC_VER)
2035 dev_dbg(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
2036 fw_desc->header.version);
2039 * Calc CRC32 of data without header. use the size of the descriptor
2040 * reported by firmware, without calculating it ourself, to allow adding
2041 * more fields to the lkd_fw_comms_desc structure.
2042 * note that no alignment/stride address issues here as all structures
2043 * are 64 bit padded.
2045 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
2046 data_size = le16_to_cpu(fw_desc->header.size);
2048 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
2049 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
2050 dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
2051 data_crc32, fw_desc->header.crc32);
2055 /* find memory region to which to copy the image */
2056 addr = le64_to_cpu(fw_desc->img_addr);
2057 region_id = hl_get_pci_memory_region(hdev, addr);
2058 if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) {
2059 dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr);
2063 region = &hdev->pci_mem_region[region_id];
2065 /* store the region for the copy stage */
2066 fw_loader->dynamic_loader.image_region = region;
2069 * here we know that the start address is valid, now make sure that the
2070 * image is within region's bounds
2072 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
2073 fw_loader->dynamic_loader.fw_image_size,
2076 dev_err(hdev->dev, "invalid mem transfer request for FW image\n");
2080 /* here we can mark the descriptor as valid as the content has been validated */
2081 fw_loader->dynamic_loader.fw_desc_valid = true;
2086 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
2087 struct fw_response *response,
2088 struct pci_mem_region *region)
2093 device_addr = region->region_base + response->ram_offset;
2096 * validate that the descriptor is within region's bounds
2097 * Note that as the start address was supplied according to the RAM
2098 * type- testing only the end address is enough
2100 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
2101 sizeof(struct lkd_fw_comms_desc),
2107 * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw
2109 * @hdev: pointer to the habanalabs device structure
2110 * @fw_desc: the descriptor from FW
2112 static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev,
2113 struct lkd_fw_comms_desc *fw_desc)
2118 for (i = 0 ; i < LKD_FW_ASCII_MSG_MAX ; i++) {
2119 if (!fw_desc->ascii_msg[i].valid)
2122 /* force NULL termination */
2123 msg = fw_desc->ascii_msg[i].msg;
2124 msg[LKD_FW_ASCII_MSG_MAX_LEN - 1] = '\0';
2126 switch (fw_desc->ascii_msg[i].msg_lvl) {
2127 case LKD_FW_ASCII_MSG_ERR:
2128 dev_err(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2130 case LKD_FW_ASCII_MSG_WRN:
2131 dev_warn(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2133 case LKD_FW_ASCII_MSG_INF:
2134 dev_info(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2137 dev_dbg(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2144 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
2146 * @hdev: pointer to the habanalabs device structure
2147 * @fw_loader: managing structure for loading device's FW
2149 * @return 0 on success, otherwise non-zero error code
2151 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
2152 struct fw_load_mgr *fw_loader)
2154 struct lkd_fw_comms_desc *fw_desc;
2155 struct pci_mem_region *region;
2156 struct fw_response *response;
2160 enum pci_region region_id;
2163 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2164 response = &fw_loader->dynamic_loader.response;
2166 region_id = (response->ram_type == COMMS_SRAM) ?
2167 PCI_REGION_SRAM : PCI_REGION_DRAM;
2169 region = &hdev->pci_mem_region[region_id];
2171 rc = hl_fw_dynamic_validate_response(hdev, response, region);
2174 "invalid mem transfer request for FW descriptor\n");
2179 * extract address to copy the descriptor from
2180 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
2182 * it will be marked again as valid once validated
2184 fw_loader->dynamic_loader.fw_desc_valid = false;
2185 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2186 response->ram_offset;
2189 * We do the copy of the fw descriptor in 2 phases:
2190 * 1. copy the header + data info according to our lkd_fw_comms_desc definition.
2191 * then we're able to read the actual data size provided by fw.
2192 * this is needed for cases where data in descriptor was changed(add/remove)
2193 * in embedded specs header file before updating lkd copy of the header file
2194 * 2. copy descriptor to temporary buffer with aligned size and send it to validation
2196 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
2197 fw_data_size = le16_to_cpu(fw_desc->header.size);
2199 temp_fw_desc = vzalloc(sizeof(struct comms_desc_header) + fw_data_size);
2203 memcpy_fromio(temp_fw_desc, src, sizeof(struct comms_desc_header) + fw_data_size);
2205 rc = hl_fw_dynamic_validate_descriptor(hdev, fw_loader,
2206 (struct lkd_fw_comms_desc *) temp_fw_desc);
2209 hl_fw_dynamic_read_descriptor_msg(hdev, temp_fw_desc);
2211 vfree(temp_fw_desc);
2217 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
2219 * @hdev: pointer to the habanalabs device structure
2220 * @fw_loader: managing structure for loading device's FW
2221 * @next_image_size: size to allocate for next FW component
2223 * @return 0 on success, otherwise non-zero error code
2225 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
2226 struct fw_load_mgr *fw_loader,
2227 size_t next_image_size)
2231 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
2232 next_image_size, true,
2233 fw_loader->cpu_timeout);
2237 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
2241 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
2243 * @hdev: pointer to the habanalabs device structure
2244 * @fwc: the firmware component
2245 * @fw_version: fw component's version string
2247 static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
2248 enum hl_fw_component fwc,
2249 const char *fw_version)
2251 struct asic_fixed_properties *prop = &hdev->asic_prop;
2252 char *preboot_ver, *boot_ver;
2257 case FW_COMP_BOOT_FIT:
2258 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
2259 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
2261 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
2266 case FW_COMP_PREBOOT:
2267 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
2268 preboot_ver = strnstr(prop->preboot_ver, "Preboot", VERSION_MAX_LEN);
2269 dev_info(hdev->dev, "preboot full version: '%s'\n", preboot_ver);
2271 if (preboot_ver && preboot_ver != prop->preboot_ver) {
2272 strscpy(btl_ver, prop->preboot_ver,
2273 min((int) (preboot_ver - prop->preboot_ver), 31));
2274 dev_info(hdev->dev, "%s\n", btl_ver);
2277 rc = hl_get_sw_major_minor_subminor(hdev, preboot_ver);
2280 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
2282 rc = hl_get_preboot_major_minor(hdev, preboot_ver);
2290 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2298 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
2300 * @hdev: pointer to the habanalabs device structure
2301 * @fw: fw descriptor
2302 * @fw_loader: managing structure for loading device's FW
2304 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
2305 const struct firmware *fw,
2306 struct fw_load_mgr *fw_loader)
2308 struct lkd_fw_comms_desc *fw_desc;
2309 struct pci_mem_region *region;
2314 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2315 addr = le64_to_cpu(fw_desc->img_addr);
2317 /* find memory region to which to copy the image */
2318 region = fw_loader->dynamic_loader.image_region;
2320 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2321 (addr - region->region_base);
2323 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
2324 fw_loader->boot_fit_img.src_off,
2325 fw_loader->boot_fit_img.copy_size);
2331 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
2333 * @hdev: pointer to the habanalabs device structure
2335 * @fw_loader: managing structure for loading device's FW
2337 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
2338 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
2340 struct lkd_fw_comms_desc *fw_desc;
2341 struct pci_mem_region *region;
2346 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2347 addr = le64_to_cpu(fw_desc->img_addr);
2349 /* find memory region to which to copy the image */
2350 region = fw_loader->dynamic_loader.image_region;
2352 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2353 (addr - region->region_base);
2355 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2361 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2364 * @hdev: pointer to the habanalabs device structure
2365 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2366 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2368 * @return 0 on success, otherwise non-zero error code
2370 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2371 u32 cpu_boot_dev_sts0_reg,
2372 u32 cpu_boot_dev_sts1_reg)
2374 struct asic_fixed_properties *prop = &hdev->asic_prop;
2376 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2378 /* Read boot_cpu status bits */
2379 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2380 prop->fw_bootfit_cpu_boot_dev_sts0 =
2381 RREG32(cpu_boot_dev_sts0_reg);
2383 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2384 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2386 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2387 prop->fw_bootfit_cpu_boot_dev_sts0);
2390 if (prop->fw_cpu_boot_dev_sts1_valid) {
2391 prop->fw_bootfit_cpu_boot_dev_sts1 =
2392 RREG32(cpu_boot_dev_sts1_reg);
2394 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2395 prop->fw_bootfit_cpu_boot_dev_sts1);
2398 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2399 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2402 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2404 struct cpu_dyn_regs *dyn_regs =
2405 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2407 /* Check whether all 3 interrupt interfaces are set, if not use a
2410 if (!hdev->asic_prop.gic_interrupts_enable &&
2411 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2412 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2413 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2414 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2417 "Using a single interrupt interface towards cpucp");
2421 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2423 * @hdev: pointer to the habanalabs device structure
2424 * @fw_loader: managing structure for loading device's FW
2425 * @load_fwc: the FW component to be loaded
2426 * @img_ld_timeout: image load timeout
2428 * @return 0 on success, otherwise non-zero error code
2430 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2431 struct fw_load_mgr *fw_loader,
2432 enum hl_fw_component load_fwc,
2435 enum hl_fw_component cur_fwc;
2436 const struct firmware *fw;
2441 * when loading image we have one of 2 scenarios:
2442 * 1. current FW component is preboot and we want to load boot-fit
2443 * 2. current FW component is boot-fit and we want to load linux
2445 if (load_fwc == FW_COMP_BOOT_FIT) {
2446 cur_fwc = FW_COMP_PREBOOT;
2447 fw_name = fw_loader->boot_fit_img.image_name;
2449 cur_fwc = FW_COMP_BOOT_FIT;
2450 fw_name = fw_loader->linux_img.image_name;
2453 /* request FW in order to communicate to FW the size to be allocated */
2454 rc = hl_request_fw(hdev, &fw, fw_name);
2458 /* store the image size for future validation */
2459 fw_loader->dynamic_loader.fw_image_size = fw->size;
2461 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2465 /* read preboot version */
2466 rc = hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2467 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2471 /* update state according to boot stage */
2472 if (cur_fwc == FW_COMP_BOOT_FIT) {
2473 struct cpu_dyn_regs *dyn_regs;
2475 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2476 hl_fw_boot_fit_update_state(hdev,
2477 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2478 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2481 /* copy boot fit to space allocated by FW */
2482 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2486 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2488 fw_loader->cpu_timeout);
2492 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2497 hl_release_firmware(fw);
2501 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2502 struct fw_load_mgr *fw_loader)
2504 struct dynamic_fw_load_mgr *dyn_loader;
2508 dyn_loader = &fw_loader->dynamic_loader;
2511 * Make sure CPU boot-loader is running
2512 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2513 * yet there is a debug scenario in which we loading uboot (without Linux)
2514 * which at later stage is relocated to DRAM. In this case we expect
2515 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2518 rc = hl_poll_timeout(
2520 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2522 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2523 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2524 hdev->fw_poll_interval_usec,
2525 dyn_loader->wait_for_bl_timeout);
2527 dev_err(hdev->dev, "failed to wait for boot (status = %d)\n", status);
2531 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2535 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2536 struct fw_load_mgr *fw_loader)
2538 struct dynamic_fw_load_mgr *dyn_loader;
2542 dyn_loader = &fw_loader->dynamic_loader;
2544 /* Make sure CPU linux is running */
2546 rc = hl_poll_timeout(
2548 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2550 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2551 hdev->fw_poll_interval_usec,
2552 fw_loader->cpu_timeout);
2554 dev_err(hdev->dev, "failed to wait for Linux (status = %d)\n", status);
2558 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2563 * hl_fw_linux_update_state - update internal data structures after Linux
2565 * Note: Linux initialization is comprised mainly
2566 * of two stages - loading kernel (SRAM_AVAIL)
2568 * Therefore reading boot device status in any of
2569 * these stages might result in different values.
2571 * @hdev: pointer to the habanalabs device structure
2572 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2573 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2575 * @return 0 on success, otherwise non-zero error code
2577 static void hl_fw_linux_update_state(struct hl_device *hdev,
2578 u32 cpu_boot_dev_sts0_reg,
2579 u32 cpu_boot_dev_sts1_reg)
2581 struct asic_fixed_properties *prop = &hdev->asic_prop;
2583 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2585 /* Read FW application security bits */
2586 if (prop->fw_cpu_boot_dev_sts0_valid) {
2587 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2589 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2590 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2592 if (prop->fw_app_cpu_boot_dev_sts0 &
2593 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2594 prop->gic_interrupts_enable = false;
2597 "Firmware application CPU status0 %#x\n",
2598 prop->fw_app_cpu_boot_dev_sts0);
2600 dev_dbg(hdev->dev, "GIC controller is %s\n",
2601 prop->gic_interrupts_enable ?
2602 "enabled" : "disabled");
2605 if (prop->fw_cpu_boot_dev_sts1_valid) {
2606 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2609 "Firmware application CPU status1 %#x\n",
2610 prop->fw_app_cpu_boot_dev_sts1);
2613 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2614 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2616 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2620 * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2622 * @hdev: pointer to the habanalabs device structure
2623 * @fw_loader: managing structure for loading device's FW
2624 * @msg_type: message type
2625 * @data: data to be sent
2627 * @return 0 on success, otherwise non-zero error code
2629 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2630 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2632 struct lkd_msg_comms *msg;
2635 msg = kzalloc(sizeof(*msg), GFP_KERNEL);
2639 /* create message to be sent */
2640 msg->header.type = msg_type;
2641 msg->header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2642 msg->header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2645 case HL_COMMS_RESET_CAUSE_TYPE:
2646 msg->reset_cause = *(__u8 *) data;
2651 "Send COMMS message - invalid message type %u\n",
2657 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2658 sizeof(struct lkd_msg_comms));
2662 /* copy message to space allocated by FW */
2663 rc = hl_fw_dynamic_copy_msg(hdev, msg, fw_loader);
2667 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2669 fw_loader->cpu_timeout);
2673 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2675 fw_loader->cpu_timeout);
2683 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2685 * @hdev: pointer to the habanalabs device structure
2686 * @fw_loader: managing structure for loading device's FW
2688 * @return 0 on success, otherwise non-zero error code
2690 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2691 * the communication is done using registers:
2692 * - LKD command register
2693 * - FW status register
2694 * the protocol is race free. this goal is achieved by splitting the requests
2695 * and response to known synchronization points between the LKD and the FW.
2696 * each response to LKD request is known and bound to a predefined timeout.
2697 * in case of timeout expiration without the desired status from FW- the
2698 * protocol (and hence the boot) will fail.
2700 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2701 struct fw_load_mgr *fw_loader)
2703 struct cpu_dyn_regs *dyn_regs;
2704 int rc, fw_error_rc;
2707 "Loading %sfirmware to device, may take some time...\n",
2708 hdev->asic_prop.fw_security_enabled ? "secured " : "");
2710 /* initialize FW descriptor as invalid */
2711 fw_loader->dynamic_loader.fw_desc_valid = false;
2714 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2715 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2717 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2719 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2721 fw_loader->cpu_timeout);
2725 if (hdev->reset_info.curr_reset_cause) {
2726 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2727 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2731 /* Clear current reset cause */
2732 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2735 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2736 struct lkd_fw_binning_info *binning_info;
2738 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2742 /* read preboot version */
2743 rc = hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2744 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2749 /* read binning info from preboot */
2750 if (hdev->support_preboot_binning) {
2751 binning_info = &fw_loader->dynamic_loader.comm_desc.binning_info;
2752 hdev->tpc_binning = le64_to_cpu(binning_info->tpc_mask_l);
2753 hdev->dram_binning = le32_to_cpu(binning_info->dram_mask);
2754 hdev->edma_binning = le32_to_cpu(binning_info->edma_mask);
2755 hdev->decoder_binning = le32_to_cpu(binning_info->dec_mask);
2756 hdev->rotator_binning = le32_to_cpu(binning_info->rot_mask);
2758 rc = hdev->asic_funcs->set_dram_properties(hdev);
2762 rc = hdev->asic_funcs->set_binning_masks(hdev);
2767 "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x, rot:0x%x\n",
2768 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2769 hdev->decoder_binning, hdev->rotator_binning);
2775 /* load boot fit to FW */
2776 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2777 fw_loader->boot_fit_timeout);
2779 dev_err(hdev->dev, "failed to load boot fit\n");
2784 * when testing FW load (without Linux) on PLDM we don't want to
2785 * wait until boot fit is active as it may take several hours.
2786 * instead, we load the bootfit and let it do all initialization in
2789 if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2792 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2796 /* Enable DRAM scrambling before Linux boot and after successful
2799 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2801 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2802 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2806 if (fw_loader->skip_bmc) {
2807 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2810 fw_loader->cpu_timeout);
2812 dev_err(hdev->dev, "failed to load boot fit\n");
2817 /* load Linux image to FW */
2818 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2819 fw_loader->cpu_timeout);
2821 dev_err(hdev->dev, "failed to load Linux\n");
2825 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2829 hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2830 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2832 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2835 if (fw_loader->dynamic_loader.fw_desc_valid) {
2836 fw_error_rc = fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2837 le32_to_cpu(dyn_regs->cpu_boot_err1),
2838 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2839 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2849 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2851 * @hdev: pointer to the habanalabs device structure
2852 * @fw_loader: managing structure for loading device's FW
2854 * @return 0 on success, otherwise non-zero error code
2856 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2857 struct fw_load_mgr *fw_loader)
2859 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2860 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2861 struct static_fw_load_mgr *static_loader;
2862 u32 cpu_boot_status_reg;
2865 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2868 /* init common loader parameters */
2869 cpu_timeout = fw_loader->cpu_timeout;
2871 /* init static loader parameters */
2872 static_loader = &fw_loader->static_loader;
2873 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2874 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2875 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2876 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2877 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2879 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2880 cpu_timeout / USEC_PER_SEC);
2882 /* Wait for boot FIT request */
2883 rc = hl_poll_timeout(
2885 cpu_boot_status_reg,
2887 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2888 hdev->fw_poll_interval_usec,
2889 fw_loader->boot_fit_timeout);
2893 "No boot fit request received (status = %d), resuming boot\n", status);
2895 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2899 /* Clear device CPU message status */
2900 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2902 /* Signal device CPU that boot loader is ready */
2903 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2905 /* Poll for CPU device ack */
2906 rc = hl_poll_timeout(
2910 status == CPU_MSG_OK,
2911 hdev->fw_poll_interval_usec,
2912 fw_loader->boot_fit_timeout);
2916 "Timeout waiting for boot fit load ack (status = %d)\n", status);
2921 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2925 * Make sure CPU boot-loader is running
2926 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2927 * yet there is a debug scenario in which we loading uboot (without Linux)
2928 * which at later stage is relocated to DRAM. In this case we expect
2929 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2932 rc = hl_poll_timeout(
2934 cpu_boot_status_reg,
2936 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2937 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2938 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2939 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2940 hdev->fw_poll_interval_usec,
2943 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2945 /* Read U-Boot version now in case we will later fail */
2946 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2948 /* update state according to boot stage */
2949 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2950 cpu_boot_dev_status1_reg);
2953 detect_cpu_boot_status(hdev, status);
2958 /* Enable DRAM scrambling before Linux boot and after successful
2961 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2963 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2964 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2969 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2975 "Loading firmware to device, may take some time...\n");
2977 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2981 if (fw_loader->skip_bmc) {
2982 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2984 rc = hl_poll_timeout(
2986 cpu_boot_status_reg,
2988 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2989 hdev->fw_poll_interval_usec,
2994 "Failed to get ACK on skipping BMC (status = %d)\n",
2996 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3002 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
3004 rc = hl_poll_timeout(
3006 cpu_boot_status_reg,
3008 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
3009 hdev->fw_poll_interval_usec,
3013 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3016 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
3018 "Device reports FIT image is corrupted\n");
3021 "Failed to load firmware to device (status = %d)\n",
3028 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3029 fw_loader->static_loader.boot_err1_reg,
3030 cpu_boot_dev_status0_reg,
3031 cpu_boot_dev_status1_reg);
3035 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
3036 cpu_boot_dev_status1_reg);
3041 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3042 fw_loader->static_loader.boot_err1_reg,
3043 cpu_boot_dev_status0_reg,
3044 cpu_boot_dev_status1_reg);
3050 * hl_fw_init_cpu - initialize the device CPU
3052 * @hdev: pointer to the habanalabs device structure
3054 * @return 0 on success, otherwise non-zero error code
3056 * perform necessary initializations for device's CPU. takes into account if
3057 * init protocol is static or dynamic.
3059 int hl_fw_init_cpu(struct hl_device *hdev)
3061 struct asic_fixed_properties *prop = &hdev->asic_prop;
3062 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3064 return prop->dynamic_fw_load ?
3065 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
3066 hl_fw_static_init_cpu(hdev, fw_loader);
3069 void hl_fw_set_pll_profile(struct hl_device *hdev)
3071 hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
3072 hdev->asic_prop.max_freq_value);
3075 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
3079 if (!hl_device_operational(hdev, NULL))
3088 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
3091 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
3095 *max_clk = (value / 1000 / 1000);
3097 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
3100 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
3104 *cur_clk = (value / 1000 / 1000);
3109 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
3111 struct cpucp_packet pkt;
3116 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3120 memset(&pkt, 0, sizeof(pkt));
3123 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
3124 CPUCP_PKT_CTL_OPCODE_SHIFT);
3126 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3128 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3130 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3133 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
3138 return (long) result;
3141 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
3143 struct cpucp_packet pkt;
3147 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3151 memset(&pkt, 0, sizeof(pkt));
3153 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3154 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3155 pkt.value = cpu_to_le64(freq);
3157 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3160 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
3164 long hl_fw_get_max_power(struct hl_device *hdev)
3166 struct cpucp_packet pkt;
3170 memset(&pkt, 0, sizeof(pkt));
3172 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3174 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3177 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
3184 void hl_fw_set_max_power(struct hl_device *hdev)
3186 struct cpucp_packet pkt;
3189 /* TODO: remove this after simulator supports this packet */
3193 memset(&pkt, 0, sizeof(pkt));
3195 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3196 pkt.value = cpu_to_le64(hdev->max_power);
3198 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3201 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
3204 static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size,
3205 u32 nonce, u32 timeout)
3207 struct cpucp_packet pkt = {};
3208 dma_addr_t req_dma_addr;
3212 req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr);
3213 if (!req_cpu_addr) {
3215 "Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id);
3219 memset(data, 0, size);
3221 pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT);
3222 pkt.addr = cpu_to_le64(req_dma_addr);
3223 pkt.data_max_size = cpu_to_le32(size);
3224 pkt.nonce = cpu_to_le32(nonce);
3226 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
3230 "Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc);
3234 memcpy(data, req_cpu_addr, size);
3237 hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr);
3242 int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,
3245 return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info,
3246 sizeof(struct cpucp_sec_attest_info), nonce,
3247 HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);
3250 int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
3251 dma_addr_t buff, u32 *size)
3253 struct cpucp_packet pkt = {};
3257 pkt.ctl = cpu_to_le32(CPUCP_PACKET_GENERIC_PASSTHROUGH << CPUCP_PKT_CTL_OPCODE_SHIFT);
3258 pkt.addr = cpu_to_le64(buff);
3259 pkt.data_max_size = cpu_to_le32(*size);
3260 pkt.pkt_subidx = cpu_to_le32(sub_opcode);
3262 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)&pkt, sizeof(pkt),
3263 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
3265 dev_err(hdev->dev, "failed to send CPUCP data of generic fw pkt\n");
3267 dev_dbg(hdev->dev, "generic pkt was successful, result: 0x%llx\n", result);
3269 *size = (u32)result;