drivers/accel/habanalabs/common/firmware_if.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9 #include "../include/common/hl_boot_if.h"
  10
  11 #include <linux/firmware.h>
  12 #include <linux/crc32.h>
  13 #include <linux/slab.h>
  14 #include <linux/ctype.h>
  15 #include <linux/vmalloc.h>
  16
  17 #include <trace/events/habanalabs.h>
  18
  19 #define FW_FILE_MAX_SIZE                0x1400000 /* maximum size of 20MB */
  20
  21 static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = {
  22         [COMMS_NOOP] = __stringify(COMMS_NOOP),
  23         [COMMS_CLR_STS] = __stringify(COMMS_CLR_STS),
  24         [COMMS_RST_STATE] = __stringify(COMMS_RST_STATE),
  25         [COMMS_PREP_DESC] = __stringify(COMMS_PREP_DESC),
  26         [COMMS_DATA_RDY] = __stringify(COMMS_DATA_RDY),
  27         [COMMS_EXEC] = __stringify(COMMS_EXEC),
  28         [COMMS_RST_DEV] = __stringify(COMMS_RST_DEV),
  29         [COMMS_GOTO_WFE] = __stringify(COMMS_GOTO_WFE),
  30         [COMMS_SKIP_BMC] = __stringify(COMMS_SKIP_BMC),
  31         [COMMS_PREP_DESC_ELBI] = __stringify(COMMS_PREP_DESC_ELBI),
  32 };
  33
  34 static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = {
  35         [COMMS_STS_NOOP] = __stringify(COMMS_STS_NOOP),
  36         [COMMS_STS_ACK] = __stringify(COMMS_STS_ACK),
  37         [COMMS_STS_OK] = __stringify(COMMS_STS_OK),
  38         [COMMS_STS_ERR] = __stringify(COMMS_STS_ERR),
  39         [COMMS_STS_VALID_ERR] = __stringify(COMMS_STS_VALID_ERR),
  40         [COMMS_STS_TIMEOUT_ERR] = __stringify(COMMS_STS_TIMEOUT_ERR),
  41 };
  42
  43 static char *extract_fw_ver_from_str(const char *fw_str)
  44 {
  45         char *str, *fw_ver, *whitespace;
  46         u32 ver_offset;
  47
  48         fw_ver = kmalloc(VERSION_MAX_LEN, GFP_KERNEL);
  49         if (!fw_ver)
  50                 return NULL;
  51
  52         str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
  53         if (!str)
  54                 goto free_fw_ver;
  55
  56         /* Skip the fw- part */
  57         str += 3;
  58         ver_offset = str - fw_str;
  59
  60         /* Copy until the next whitespace */
  61         whitespace = strnstr(str, " ", VERSION_MAX_LEN - ver_offset);
  62         if (!whitespace)
  63                 goto free_fw_ver;
  64
  65         strscpy(fw_ver, str, whitespace - str + 1);
  66
  67         return fw_ver;
  68
  69 free_fw_ver:
  70         kfree(fw_ver);
  71         return NULL;
  72 }
  73
  74 /**
  75  * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32.
  76  * @str: the given string
  77  * @ver_num: the pointer to the extracted u32 to be returned to the caller.
  78  * @given_char: the given char at the end of the u32 in the string
  79  *
  80  * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL
  81  */
  82 static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char)
  83 {
  84         char num_str[8] = {}, *ch;
  85
  86         ch = strchrnul(str, given_char);
  87         if (*ch == '\0' || ch == str || ch - str >= sizeof(num_str))
  88                 return NULL;
  89
  90         memcpy(num_str, str, ch - str);
  91         if (kstrtou32(num_str, 10, ver_num))
  92                 return NULL;
  93         return ch;
  94 }
  95
  96 /**
  97  * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor
  98  *                                    from the version string
  99  * @hdev: pointer to the hl_device
 100  * @fw_str: the FW's version string
 101  *
 102  * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver.
 103  *
 104  * fw_str is expected to have one of two possible formats, examples:
 105  * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3'
 106  * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3'
 107  * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0.
 108  *
 109  * Return: 0 for success or a negative error code for failure.
 110  */
 111 static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str)
 112 {
 113         char *end, *start;
 114
 115         end = strnstr(fw_str, "-rc-", VERSION_MAX_LEN);
 116         if (end == fw_str)
 117                 return -EINVAL;
 118
 119         if (!end)
 120                 end = strnstr(fw_str, "-fw-", VERSION_MAX_LEN);
 121
 122         if (end == fw_str)
 123                 return -EINVAL;
 124
 125         if (!end)
 126                 return -EINVAL;
 127
 128         for (start = end - 1; start != fw_str; start--) {
 129                 if (*start == '-')
 130                         break;
 131         }
 132
 133         if (start == fw_str)
 134                 return -EINVAL;
 135
 136         /* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */
 137         start++;
 138         start = extract_u32_until_given_char(start, &hdev->fw_sw_major_ver, '.');
 139         if (!start)
 140                 goto err_zero_ver;
 141
 142         start++;
 143         start = extract_u32_until_given_char(start, &hdev->fw_sw_minor_ver, '.');
 144         if (!start)
 145                 goto err_zero_ver;
 146
 147         start++;
 148         start = extract_u32_until_given_char(start, &hdev->fw_sw_sub_minor_ver, '-');
 149         if (!start)
 150                 goto err_zero_ver;
 151
 152         return 0;
 153
 154 err_zero_ver:
 155         hdev->fw_sw_major_ver = 0;
 156         hdev->fw_sw_minor_ver = 0;
 157         hdev->fw_sw_sub_minor_ver = 0;
 158         return -EINVAL;
 159 }
 160
 161 /**
 162  * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string.
 163  * @hdev: pointer to the hl_device
 164  * @preboot_ver: the FW's version string
 165  *
 166  * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3
 167  * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver.
 168  *
 169  * Return: 0 on success, negative error code for failure.
 170  */
 171 static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)
 172 {
 173         preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_major_ver, '.');
 174         if (!preboot_ver) {
 175                 dev_err(hdev->dev, "Error parsing preboot major version\n");
 176                 goto err_zero_ver;
 177         }
 178
 179         preboot_ver++;
 180
 181         preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_minor_ver, '.');
 182         if (!preboot_ver) {
 183                 dev_err(hdev->dev, "Error parsing preboot minor version\n");
 184                 goto err_zero_ver;
 185         }
 186         return 0;
 187
 188 err_zero_ver:
 189         hdev->fw_inner_major_ver = 0;
 190         hdev->fw_inner_minor_ver = 0;
 191         return -EINVAL;
 192 }
 193
 194 static int hl_request_fw(struct hl_device *hdev,
 195                                 const struct firmware **firmware_p,
 196                                 const char *fw_name)
 197 {
 198         size_t fw_size;
 199         int rc;
 200
 201         rc = request_firmware(firmware_p, fw_name, hdev->dev);
 202         if (rc) {
 203                 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
 204                                 fw_name, rc);
 205                 goto out;
 206         }
 207
 208         fw_size = (*firmware_p)->size;
 209         if ((fw_size % 4) != 0) {
 210                 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
 211                                 fw_name, fw_size);
 212                 rc = -EINVAL;
 213                 goto release_fw;
 214         }
 215
 216         dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
 217
 218         if (fw_size > FW_FILE_MAX_SIZE) {
 219                 dev_err(hdev->dev,
 220                         "FW file size %zu exceeds maximum of %u bytes\n",
 221                         fw_size, FW_FILE_MAX_SIZE);
 222                 rc = -EINVAL;
 223                 goto release_fw;
 224         }
 225
 226         return 0;
 227
 228 release_fw:
 229         release_firmware(*firmware_p);
 230 out:
 231         return rc;
 232 }
 233
 234 /**
 235  * hl_release_firmware() - release FW
 236  *
 237  * @fw: fw descriptor
 238  *
 239  * note: this inline function added to serve as a comprehensive mirror for the
 240  *       hl_request_fw function.
 241  */
 242 static inline void hl_release_firmware(const struct firmware *fw)
 243 {
 244         release_firmware(fw);
 245 }
 246
 247 /**
 248  * hl_fw_copy_fw_to_device() - copy FW to device
 249  *
 250  * @hdev: pointer to hl_device structure.
 251  * @fw: fw descriptor
 252  * @dst: IO memory mapped address space to copy firmware to
 253  * @src_offset: offset in src FW to copy from
 254  * @size: amount of bytes to copy (0 to copy the whole binary)
 255  *
 256  * actual copy of FW binary data to device, shared by static and dynamic loaders
 257  */
 258 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
 259                                 const struct firmware *fw, void __iomem *dst,
 260                                 u32 src_offset, u32 size)
 261 {
 262         const void *fw_data;
 263
 264         /* size 0 indicates to copy the whole file */
 265         if (!size)
 266                 size = fw->size;
 267
 268         if (src_offset + size > fw->size) {
 269                 dev_err(hdev->dev,
 270                         "size to copy(%u) and offset(%u) are invalid\n",
 271                         size, src_offset);
 272                 return -EINVAL;
 273         }
 274
 275         fw_data = (const void *) fw->data;
 276
 277         memcpy_toio(dst, fw_data + src_offset, size);
 278         return 0;
 279 }
 280
 281 /**
 282  * hl_fw_copy_msg_to_device() - copy message to device
 283  *
 284  * @hdev: pointer to hl_device structure.
 285  * @msg: message
 286  * @dst: IO memory mapped address space to copy firmware to
 287  * @src_offset: offset in src message to copy from
 288  * @size: amount of bytes to copy (0 to copy the whole binary)
 289  *
 290  * actual copy of message data to device.
 291  */
 292 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
 293                 struct lkd_msg_comms *msg, void __iomem *dst,
 294                 u32 src_offset, u32 size)
 295 {
 296         void *msg_data;
 297
 298         /* size 0 indicates to copy the whole file */
 299         if (!size)
 300                 size = sizeof(struct lkd_msg_comms);
 301
 302         if (src_offset + size > sizeof(struct lkd_msg_comms)) {
 303                 dev_err(hdev->dev,
 304                         "size to copy(%u) and offset(%u) are invalid\n",
 305                         size, src_offset);
 306                 return -EINVAL;
 307         }
 308
 309         msg_data = (void *) msg;
 310
 311         memcpy_toio(dst, msg_data + src_offset, size);
 312
 313         return 0;
 314 }
 315
 316 /**
 317  * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
 318  *
 319  * @hdev: pointer to hl_device structure.
 320  * @fw_name: the firmware image name
 321  * @dst: IO memory mapped address space to copy firmware to
 322  * @src_offset: offset in src FW to copy from
 323  * @size: amount of bytes to copy (0 to copy the whole binary)
 324  *
 325  * Copy fw code from firmware file to device memory.
 326  *
 327  * Return: 0 on success, non-zero for failure.
 328  */
 329 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 330                                 void __iomem *dst, u32 src_offset, u32 size)
 331 {
 332         const struct firmware *fw;
 333         int rc;
 334
 335         rc = hl_request_fw(hdev, &fw, fw_name);
 336         if (rc)
 337                 return rc;
 338
 339         rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
 340
 341         hl_release_firmware(fw);
 342         return rc;
 343 }
 344
 345 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value)
 346 {
 347         struct cpucp_packet pkt = {};
 348
 349         pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
 350         pkt.value = cpu_to_le64(value);
 351
 352         return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
 353 }
 354
 355 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
 356                                 u16 len, u32 timeout, u64 *result)
 357 {
 358         struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
 359         struct asic_fixed_properties *prop = &hdev->asic_prop;
 360         struct cpucp_packet *pkt;
 361         dma_addr_t pkt_dma_addr;
 362         struct hl_bd *sent_bd;
 363         u32 tmp, expected_ack_val, pi, opcode;
 364         int rc;
 365
 366         pkt = hl_cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr);
 367         if (!pkt) {
 368                 dev_err(hdev->dev,
 369                         "Failed to allocate DMA memory for packet to CPU\n");
 370                 return -ENOMEM;
 371         }
 372
 373         memcpy(pkt, msg, len);
 374
 375         mutex_lock(&hdev->send_cpu_message_lock);
 376
 377         /* CPU-CP messages can be sent during soft-reset */
 378         if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
 379                 rc = 0;
 380                 goto out;
 381         }
 382
 383         if (hdev->device_cpu_disabled) {
 384                 rc = -EIO;
 385                 goto out;
 386         }
 387
 388         /* set fence to a non valid value */
 389         pkt->fence = cpu_to_le32(UINT_MAX);
 390         pi = queue->pi;
 391
 392         /*
 393          * The CPU queue is a synchronous queue with an effective depth of
 394          * a single entry (although it is allocated with room for multiple
 395          * entries). We lock on it using 'send_cpu_message_lock' which
 396          * serializes accesses to the CPU queue.
 397          * Which means that we don't need to lock the access to the entire H/W
 398          * queues module when submitting a JOB to the CPU queue.
 399          */
 400         hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
 401
 402         if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
 403                 expected_ack_val = queue->pi;
 404         else
 405                 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
 406
 407         rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
 408                                 (tmp == expected_ack_val), 1000,
 409                                 timeout, true);
 410
 411         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
 412
 413         if (rc == -ETIMEDOUT) {
 414                 /* If FW performed reset just before sending it a packet, we will get a timeout.
 415                  * This is expected behavior, hence no need for error message.
 416                  */
 417                 if (!hl_device_operational(hdev, NULL) && !hdev->reset_info.in_compute_reset)
 418                         dev_dbg(hdev->dev, "Device CPU packet timeout (0x%x) due to FW reset\n",
 419                                         tmp);
 420                 else
 421                         dev_err(hdev->dev, "Device CPU packet timeout (status = 0x%x)\n", tmp);
 422                 hdev->device_cpu_disabled = true;
 423                 goto out;
 424         }
 425
 426         tmp = le32_to_cpu(pkt->ctl);
 427
 428         rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
 429         if (rc) {
 430                 opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT;
 431
 432                 if (!prop->supports_advanced_cpucp_rc) {
 433                         dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode);
 434                         rc = -EIO;
 435                         goto scrub_descriptor;
 436                 }
 437
 438                 switch (rc) {
 439                 case cpucp_packet_invalid:
 440                         dev_err(hdev->dev,
 441                                 "CPU packet %d is not supported by F/W\n", opcode);
 442                         break;
 443                 case cpucp_packet_fault:
 444                         dev_err(hdev->dev,
 445                                 "F/W failed processing CPU packet %d\n", opcode);
 446                         break;
 447                 case cpucp_packet_invalid_pkt:
 448                         dev_dbg(hdev->dev,
 449                                 "CPU packet %d is not supported by F/W\n", opcode);
 450                         break;
 451                 case cpucp_packet_invalid_params:
 452                         dev_err(hdev->dev,
 453                                 "F/W reports invalid parameters for CPU packet %d\n", opcode);
 454                         break;
 455
 456                 default:
 457                         dev_err(hdev->dev,
 458                                 "Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode);
 459                 }
 460
 461                 /* propagate the return code from the f/w to the callers who want to check it */
 462                 if (result)
 463                         *result = rc;
 464
 465                 rc = -EIO;
 466
 467         } else if (result) {
 468                 *result = le64_to_cpu(pkt->result);
 469         }
 470
 471 scrub_descriptor:
 472         /* Scrub previous buffer descriptor 'ctl' field which contains the
 473          * previous PI value written during packet submission.
 474          * We must do this or else F/W can read an old value upon queue wraparound.
 475          */
 476         sent_bd = queue->kernel_address;
 477         sent_bd += hl_pi_2_offset(pi);
 478         sent_bd->ctl = cpu_to_le32(UINT_MAX);
 479
 480 out:
 481         mutex_unlock(&hdev->send_cpu_message_lock);
 482
 483         hl_cpu_accessible_dma_pool_free(hdev, len, pkt);
 484
 485         return rc;
 486 }
 487
 488 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
 489 {
 490         struct cpucp_packet pkt;
 491         u64 result;
 492         int rc;
 493
 494         memset(&pkt, 0, sizeof(pkt));
 495
 496         pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
 497                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 498         pkt.value = cpu_to_le64(event_type);
 499
 500         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 501                                                 0, &result);
 502
 503         if (rc)
 504                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
 505
 506         return rc;
 507 }
 508
 509 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
 510                 size_t irq_arr_size)
 511 {
 512         struct cpucp_unmask_irq_arr_packet *pkt;
 513         size_t total_pkt_size;
 514         u64 result;
 515         int rc;
 516
 517         total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
 518                         irq_arr_size;
 519
 520         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 521         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 522
 523         /* total_pkt_size is casted to u16 later on */
 524         if (total_pkt_size > USHRT_MAX) {
 525                 dev_err(hdev->dev, "too many elements in IRQ array\n");
 526                 return -EINVAL;
 527         }
 528
 529         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 530         if (!pkt)
 531                 return -ENOMEM;
 532
 533         pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
 534         memcpy(&pkt->irqs, irq_arr, irq_arr_size);
 535
 536         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
 537                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 538
 539         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
 540                                                 total_pkt_size, 0, &result);
 541
 542         if (rc)
 543                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
 544
 545         kfree(pkt);
 546
 547         return rc;
 548 }
 549
 550 int hl_fw_test_cpu_queue(struct hl_device *hdev)
 551 {
 552         struct cpucp_packet test_pkt = {};
 553         u64 result;
 554         int rc;
 555
 556         test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 557                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 558         test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 559
 560         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
 561                                                 sizeof(test_pkt), 0, &result);
 562
 563         if (!rc) {
 564                 if (result != CPUCP_PACKET_FENCE_VAL)
 565                         dev_err(hdev->dev,
 566                                 "CPU queue test failed (%#08llx)\n", result);
 567         } else {
 568                 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
 569         }
 570
 571         return rc;
 572 }
 573
 574 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
 575                                                 dma_addr_t *dma_handle)
 576 {
 577         u64 kernel_addr;
 578
 579         kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
 580
 581         *dma_handle = hdev->cpu_accessible_dma_address +
 582                 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
 583
 584         return (void *) (uintptr_t) kernel_addr;
 585 }
 586
 587 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 588                                         void *vaddr)
 589 {
 590         gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
 591                         size);
 592 }
 593
 594 int hl_fw_send_soft_reset(struct hl_device *hdev)
 595 {
 596         struct cpucp_packet pkt;
 597         int rc;
 598
 599         memset(&pkt, 0, sizeof(pkt));
 600         pkt.ctl = cpu_to_le32(CPUCP_PACKET_SOFT_RESET << CPUCP_PKT_CTL_OPCODE_SHIFT);
 601         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
 602         if (rc)
 603                 dev_err(hdev->dev, "failed to send soft-reset msg (err = %d)\n", rc);
 604
 605         return rc;
 606 }
 607
 608 int hl_fw_send_device_activity(struct hl_device *hdev, bool open)
 609 {
 610         struct cpucp_packet pkt;
 611         int rc;
 612
 613         memset(&pkt, 0, sizeof(pkt));
 614         pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
 615         pkt.value = cpu_to_le64(open);
 616         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
 617         if (rc)
 618                 dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open);
 619
 620         return rc;
 621 }
 622
 623 int hl_fw_send_heartbeat(struct hl_device *hdev)
 624 {
 625         struct cpucp_packet hb_pkt;
 626         u64 result;
 627         int rc;
 628
 629         memset(&hb_pkt, 0, sizeof(hb_pkt));
 630         hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
 631                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
 632         hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
 633
 634         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
 635                                                 sizeof(hb_pkt), 0, &result);
 636
 637         if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
 638                 return -EIO;
 639
 640         if (le32_to_cpu(hb_pkt.status_mask) &
 641                                         CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
 642                 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
 643                 rc = -EIO;
 644         }
 645
 646         return rc;
 647 }
 648
 649 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 650                                                                 u32 sts_val)
 651 {
 652         bool err_exists = false;
 653
 654         if (!(err_val & CPU_BOOT_ERR0_ENABLED))
 655                 return false;
 656
 657         if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
 658                 dev_err(hdev->dev,
 659                         "Device boot error - DRAM initialization failed\n");
 660                 err_exists = true;
 661         }
 662
 663         if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
 664                 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
 665                 err_exists = true;
 666         }
 667
 668         if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
 669                 dev_err(hdev->dev,
 670                         "Device boot error - Thermal Sensor initialization failed\n");
 671                 err_exists = true;
 672         }
 673
 674         if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
 675                 if (hdev->bmc_enable) {
 676                         dev_err(hdev->dev,
 677                                 "Device boot error - Skipped waiting for BMC\n");
 678                         err_exists = true;
 679                 } else {
 680                         dev_info(hdev->dev,
 681                                 "Device boot message - Skipped waiting for BMC\n");
 682                         /* This is an info so we don't want it to disable the
 683                          * device
 684                          */
 685                         err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
 686                 }
 687         }
 688
 689         if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
 690                 dev_err(hdev->dev,
 691                         "Device boot error - Serdes data from BMC not available\n");
 692                 err_exists = true;
 693         }
 694
 695         if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
 696                 dev_err(hdev->dev,
 697                         "Device boot error - NIC F/W initialization failed\n");
 698                 err_exists = true;
 699         }
 700
 701         if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
 702                 dev_err(hdev->dev,
 703                         "Device boot warning - security not ready\n");
 704                 err_exists = true;
 705         }
 706
 707         if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
 708                 dev_err(hdev->dev, "Device boot error - security failure\n");
 709                 err_exists = true;
 710         }
 711
 712         if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
 713                 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
 714                 err_exists = true;
 715         }
 716
 717         if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
 718                 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
 719                 err_exists = true;
 720         }
 721
 722         if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
 723                 dev_err(hdev->dev, "Device boot error - PLL failure\n");
 724                 err_exists = true;
 725         }
 726
 727         if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
 728                 /* Ignore this bit, don't prevent driver loading */
 729                 dev_dbg(hdev->dev, "device unusable status is set\n");
 730                 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
 731         }
 732
 733         if (err_val & CPU_BOOT_ERR0_BINNING_FAIL) {
 734                 dev_err(hdev->dev, "Device boot error - binning failure\n");
 735                 err_exists = true;
 736         }
 737
 738         if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
 739                 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
 740
 741         if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {
 742                 dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");
 743                 err_exists = true;
 744         }
 745
 746         /* All warnings should go here in order not to reach the unknown error validation */
 747         if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
 748                 dev_warn(hdev->dev,
 749                         "Device boot warning - Skipped DRAM initialization\n");
 750                 /* This is a warning so we don't want it to disable the
 751                  * device
 752                  */
 753                 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
 754         }
 755
 756         if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
 757                 dev_warn(hdev->dev,
 758                         "Device boot warning - Failed to load preboot primary image\n");
 759                 /* This is a warning so we don't want it to disable the
 760                  * device as we have a secondary preboot image
 761                  */
 762                 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
 763         }
 764
 765         if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
 766                 dev_warn(hdev->dev,
 767                         "Device boot warning - TPM failure\n");
 768                 /* This is a warning so we don't want it to disable the
 769                  * device
 770                  */
 771                 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
 772         }
 773
 774         if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
 775                 dev_err(hdev->dev,
 776                         "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
 777                 err_exists = true;
 778         }
 779
 780         /* return error only if it's in the predefined mask */
 781         if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
 782                                 lower_32_bits(hdev->boot_error_status_mask)))
 783                 return true;
 784
 785         return false;
 786 }
 787
 788 /* placeholder for ERR1 as no errors defined there yet */
 789 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
 790                                                                 u32 sts_val)
 791 {
 792         /*
 793          * keep this variable to preserve the logic of the function.
 794          * this way it would require less modifications when error will be
 795          * added to DEV_ERR1
 796          */
 797         bool err_exists = false;
 798
 799         if (!(err_val & CPU_BOOT_ERR1_ENABLED))
 800                 return false;
 801
 802         if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
 803                 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
 804
 805         if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
 806                 dev_err(hdev->dev,
 807                         "Device boot error - unknown ERR1 error 0x%08x\n",
 808                                                                 err_val);
 809                 err_exists = true;
 810         }
 811
 812         /* return error only if it's in the predefined mask */
 813         if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
 814                                 upper_32_bits(hdev->boot_error_status_mask)))
 815                 return true;
 816
 817         return false;
 818 }
 819
 820 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
 821                                 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
 822                                 u32 cpu_boot_dev_status1_reg)
 823 {
 824         u32 err_val, status_val;
 825         bool err_exists = false;
 826
 827         /* Some of the firmware status codes are deprecated in newer f/w
 828          * versions. In those versions, the errors are reported
 829          * in different registers. Therefore, we need to check those
 830          * registers and print the exact errors. Moreover, there
 831          * may be multiple errors, so we need to report on each error
 832          * separately. Some of the error codes might indicate a state
 833          * that is not an error per-se, but it is an error in production
 834          * environment
 835          */
 836         err_val = RREG32(boot_err0_reg);
 837         status_val = RREG32(cpu_boot_dev_status0_reg);
 838         err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
 839
 840         err_val = RREG32(boot_err1_reg);
 841         status_val = RREG32(cpu_boot_dev_status1_reg);
 842         err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
 843
 844         if (err_exists)
 845                 return -EIO;
 846
 847         return 0;
 848 }
 849
 850 int hl_fw_cpucp_info_get(struct hl_device *hdev,
 851                                 u32 sts_boot_dev_sts0_reg,
 852                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 853                                 u32 boot_err1_reg)
 854 {
 855         struct asic_fixed_properties *prop = &hdev->asic_prop;
 856         struct cpucp_packet pkt = {};
 857         dma_addr_t cpucp_info_dma_addr;
 858         void *cpucp_info_cpu_addr;
 859         char *kernel_ver;
 860         u64 result;
 861         int rc;
 862
 863         cpucp_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, sizeof(struct cpucp_info),
 864                                                                 &cpucp_info_dma_addr);
 865         if (!cpucp_info_cpu_addr) {
 866                 dev_err(hdev->dev,
 867                         "Failed to allocate DMA memory for CPU-CP info packet\n");
 868                 return -ENOMEM;
 869         }
 870
 871         memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
 872
 873         pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
 874                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 875         pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
 876         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
 877
 878         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 879                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
 880         if (rc) {
 881                 dev_err(hdev->dev,
 882                         "Failed to handle CPU-CP info pkt, error %d\n", rc);
 883                 goto out;
 884         }
 885
 886         rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
 887                                 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
 888         if (rc) {
 889                 dev_err(hdev->dev, "Errors in device boot\n");
 890                 goto out;
 891         }
 892
 893         memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
 894                         sizeof(prop->cpucp_info));
 895
 896         rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
 897         if (rc) {
 898                 dev_err(hdev->dev,
 899                         "Failed to build hwmon channel info, error %d\n", rc);
 900                 rc = -EFAULT;
 901                 goto out;
 902         }
 903
 904         kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
 905         if (kernel_ver) {
 906                 dev_info(hdev->dev, "Linux version %s", kernel_ver);
 907                 kfree(kernel_ver);
 908         }
 909
 910         /* assume EQ code doesn't need to check eqe index */
 911         hdev->event_queue.check_eqe_index = false;
 912
 913         /* Read FW application security bits again */
 914         if (prop->fw_cpu_boot_dev_sts0_valid) {
 915                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
 916                 if (prop->fw_app_cpu_boot_dev_sts0 &
 917                                 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
 918                         hdev->event_queue.check_eqe_index = true;
 919         }
 920
 921         if (prop->fw_cpu_boot_dev_sts1_valid)
 922                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
 923
 924 out:
 925         hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr);
 926
 927         return rc;
 928 }
 929
 930 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
 931 {
 932         struct cpucp_array_data_packet *pkt;
 933         size_t total_pkt_size, data_size;
 934         u64 result;
 935         int rc;
 936
 937         /* skip sending this info for unsupported ASICs */
 938         if (!hdev->asic_funcs->get_msi_info)
 939                 return 0;
 940
 941         data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
 942         total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
 943
 944         /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
 945         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
 946
 947         /* total_pkt_size is casted to u16 later on */
 948         if (total_pkt_size > USHRT_MAX) {
 949                 dev_err(hdev->dev, "CPUCP array data is too big\n");
 950                 return -EINVAL;
 951         }
 952
 953         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
 954         if (!pkt)
 955                 return -ENOMEM;
 956
 957         pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
 958
 959         memset((void *) &pkt->data, 0xFF, data_size);
 960         hdev->asic_funcs->get_msi_info(pkt->data);
 961
 962         pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
 963                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
 964
 965         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
 966                                                 total_pkt_size, 0, &result);
 967
 968         /*
 969          * in case packet result is invalid it means that FW does not support
 970          * this feature and will use default/hard coded MSI values. no reason
 971          * to stop the boot
 972          */
 973         if (rc && result == cpucp_packet_invalid)
 974                 rc = 0;
 975
 976         if (rc)
 977                 dev_err(hdev->dev, "failed to send CPUCP array data\n");
 978
 979         kfree(pkt);
 980
 981         return rc;
 982 }
 983
 984 int hl_fw_cpucp_handshake(struct hl_device *hdev,
 985                                 u32 sts_boot_dev_sts0_reg,
 986                                 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
 987                                 u32 boot_err1_reg)
 988 {
 989         int rc;
 990
 991         rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
 992                                         sts_boot_dev_sts1_reg, boot_err0_reg,
 993                                         boot_err1_reg);
 994         if (rc)
 995                 return rc;
 996
 997         return hl_fw_send_msi_info_msg(hdev);
 998 }
 999
1000 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
1001 {
1002         struct cpucp_packet pkt = {};
1003         void *eeprom_info_cpu_addr;
1004         dma_addr_t eeprom_info_dma_addr;
1005         u64 result;
1006         int rc;
1007
1008         eeprom_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, max_size,
1009                                                                         &eeprom_info_dma_addr);
1010         if (!eeprom_info_cpu_addr) {
1011                 dev_err(hdev->dev,
1012                         "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
1013                 return -ENOMEM;
1014         }
1015
1016         memset(eeprom_info_cpu_addr, 0, max_size);
1017
1018         pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
1019                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
1020         pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
1021         pkt.data_max_size = cpu_to_le32(max_size);
1022
1023         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1024                         HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
1025
1026         if (rc) {
1027                 dev_err(hdev->dev,
1028                         "Failed to handle CPU-CP EEPROM packet, error %d\n",
1029                         rc);
1030                 goto out;
1031         }
1032
1033         /* result contains the actual size */
1034         memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
1035
1036 out:
1037         hl_cpu_accessible_dma_pool_free(hdev, max_size, eeprom_info_cpu_addr);
1038
1039         return rc;
1040 }
1041
1042 int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data)
1043 {
1044         struct cpucp_monitor_dump *mon_dump_cpu_addr;
1045         dma_addr_t mon_dump_dma_addr;
1046         struct cpucp_packet pkt = {};
1047         size_t data_size;
1048         __le32 *src_ptr;
1049         u32 *dst_ptr;
1050         u64 result;
1051         int i, rc;
1052
1053         data_size = sizeof(struct cpucp_monitor_dump);
1054         mon_dump_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, data_size, &mon_dump_dma_addr);
1055         if (!mon_dump_cpu_addr) {
1056                 dev_err(hdev->dev,
1057                         "Failed to allocate DMA memory for CPU-CP monitor-dump packet\n");
1058                 return -ENOMEM;
1059         }
1060
1061         memset(mon_dump_cpu_addr, 0, data_size);
1062
1063         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MONITOR_DUMP_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1064         pkt.addr = cpu_to_le64(mon_dump_dma_addr);
1065         pkt.data_max_size = cpu_to_le32(data_size);
1066
1067         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1068                                                         HL_CPUCP_MON_DUMP_TIMEOUT_USEC, &result);
1069         if (rc) {
1070                 dev_err(hdev->dev, "Failed to handle CPU-CP monitor-dump packet, error %d\n", rc);
1071                 goto out;
1072         }
1073
1074         /* result contains the actual size */
1075         src_ptr = (__le32 *) mon_dump_cpu_addr;
1076         dst_ptr = data;
1077         for (i = 0; i < (data_size / sizeof(u32)); i++) {
1078                 *dst_ptr = le32_to_cpu(*src_ptr);
1079                 src_ptr++;
1080                 dst_ptr++;
1081         }
1082
1083 out:
1084         hl_cpu_accessible_dma_pool_free(hdev, data_size, mon_dump_cpu_addr);
1085
1086         return rc;
1087 }
1088
1089 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
1090                 struct hl_info_pci_counters *counters)
1091 {
1092         struct cpucp_packet pkt = {};
1093         u64 result;
1094         int rc;
1095
1096         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1097                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1098
1099         /* Fetch PCI rx counter */
1100         pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
1101         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1102                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1103         if (rc) {
1104                 dev_err(hdev->dev,
1105                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1106                 return rc;
1107         }
1108         counters->rx_throughput = result;
1109
1110         memset(&pkt, 0, sizeof(pkt));
1111         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1112                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1113
1114         /* Fetch PCI tx counter */
1115         pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
1116         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1117                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1118         if (rc) {
1119                 dev_err(hdev->dev,
1120                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1121                 return rc;
1122         }
1123         counters->tx_throughput = result;
1124
1125         /* Fetch PCI replay counter */
1126         memset(&pkt, 0, sizeof(pkt));
1127         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
1128                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1129
1130         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1131                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1132         if (rc) {
1133                 dev_err(hdev->dev,
1134                         "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1135                 return rc;
1136         }
1137         counters->replay_cnt = (u32) result;
1138
1139         return rc;
1140 }
1141
1142 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
1143 {
1144         struct cpucp_packet pkt = {};
1145         u64 result;
1146         int rc;
1147
1148         pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
1149                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
1150
1151         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1152                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1153         if (rc) {
1154                 dev_err(hdev->dev,
1155                         "Failed to handle CpuCP total energy pkt, error %d\n",
1156                                 rc);
1157                 return rc;
1158         }
1159
1160         *total_energy = result;
1161
1162         return rc;
1163 }
1164
1165 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
1166                                                 enum pll_index *pll_index)
1167 {
1168         struct asic_fixed_properties *prop = &hdev->asic_prop;
1169         u8 pll_byte, pll_bit_off;
1170         bool dynamic_pll;
1171         int fw_pll_idx;
1172
1173         dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
1174                                                 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
1175
1176         if (!dynamic_pll) {
1177                 /*
1178                  * in case we are working with legacy FW (each asic has unique
1179                  * PLL numbering) use the driver based index as they are
1180                  * aligned with fw legacy numbering
1181                  */
1182                 *pll_index = input_pll_index;
1183                 return 0;
1184         }
1185
1186         /* retrieve a FW compatible PLL index based on
1187          * ASIC specific user request
1188          */
1189         fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
1190         if (fw_pll_idx < 0) {
1191                 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
1192                         input_pll_index, fw_pll_idx);
1193                 return -EINVAL;
1194         }
1195
1196         /* PLL map is a u8 array */
1197         pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
1198         pll_bit_off = fw_pll_idx & 0x7;
1199
1200         if (!(pll_byte & BIT(pll_bit_off))) {
1201                 dev_err(hdev->dev, "PLL index %d is not supported\n",
1202                         fw_pll_idx);
1203                 return -EINVAL;
1204         }
1205
1206         *pll_index = fw_pll_idx;
1207
1208         return 0;
1209 }
1210
1211 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
1212                 u16 *pll_freq_arr)
1213 {
1214         struct cpucp_packet pkt;
1215         enum pll_index used_pll_idx;
1216         u64 result;
1217         int rc;
1218
1219         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
1220         if (rc)
1221                 return rc;
1222
1223         memset(&pkt, 0, sizeof(pkt));
1224
1225         pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
1226                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
1227         pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
1228
1229         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1230                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1231         if (rc) {
1232                 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
1233                 return rc;
1234         }
1235
1236         pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
1237         pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
1238         pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
1239         pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
1240
1241         return 0;
1242 }
1243
1244 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
1245 {
1246         struct cpucp_packet pkt;
1247         u64 result;
1248         int rc;
1249
1250         memset(&pkt, 0, sizeof(pkt));
1251
1252         pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
1253                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
1254         pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
1255
1256         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1257                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1258         if (rc) {
1259                 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
1260                 return rc;
1261         }
1262
1263         *power = result;
1264
1265         return rc;
1266 }
1267
1268 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
1269                                 struct cpucp_hbm_row_info *info)
1270 {
1271         struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1272         dma_addr_t cpucp_repl_rows_info_dma_addr;
1273         struct cpucp_packet pkt = {};
1274         u64 result;
1275         int rc;
1276
1277         cpucp_repl_rows_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev,
1278                                                         sizeof(struct cpucp_hbm_row_info),
1279                                                         &cpucp_repl_rows_info_dma_addr);
1280         if (!cpucp_repl_rows_info_cpu_addr) {
1281                 dev_err(hdev->dev,
1282                         "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1283                 return -ENOMEM;
1284         }
1285
1286         memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1287
1288         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1289                                         CPUCP_PKT_CTL_OPCODE_SHIFT);
1290         pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1291         pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1292
1293         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1294                                         HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1295         if (rc) {
1296                 dev_err(hdev->dev,
1297                         "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1298                 goto out;
1299         }
1300
1301         memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1302
1303 out:
1304         hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_hbm_row_info),
1305                                                 cpucp_repl_rows_info_cpu_addr);
1306
1307         return rc;
1308 }
1309
1310 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1311 {
1312         struct cpucp_packet pkt;
1313         u64 result;
1314         int rc;
1315
1316         memset(&pkt, 0, sizeof(pkt));
1317
1318         pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1319
1320         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1321         if (rc) {
1322                 dev_err(hdev->dev,
1323                                 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1324                 goto out;
1325         }
1326
1327         *pend_rows_num = (u32) result;
1328 out:
1329         return rc;
1330 }
1331
1332 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1333 {
1334         struct cpucp_packet pkt;
1335         int rc;
1336
1337         memset(&pkt, 0, sizeof(pkt));
1338
1339         pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1340         pkt.value = cpu_to_le64(asid);
1341
1342         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1343                                                 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1344         if (rc)
1345                 dev_err(hdev->dev,
1346                         "Failed on ASID configuration request for engine core, error %d\n",
1347                         rc);
1348
1349         return rc;
1350 }
1351
1352 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1353 {
1354         struct static_fw_load_mgr *static_loader =
1355                         &hdev->fw_loader.static_loader;
1356         int rc;
1357
1358         if (hdev->asic_prop.dynamic_fw_load) {
1359                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1360                                 COMMS_RST_DEV, 0, false,
1361                                 hdev->fw_loader.cpu_timeout);
1362                 if (rc)
1363                         dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1364         } else {
1365                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1366         }
1367 }
1368
1369 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1370 {
1371         struct static_fw_load_mgr *static_loader =
1372                         &hdev->fw_loader.static_loader;
1373         int rc;
1374
1375         if (hdev->device_cpu_is_halted)
1376                 return;
1377
1378         /* Stop device CPU to make sure nothing bad happens */
1379         if (hdev->asic_prop.dynamic_fw_load) {
1380                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1381                                 COMMS_GOTO_WFE, 0, false,
1382                                 hdev->fw_loader.cpu_timeout);
1383                 if (rc)
1384                         dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1385         } else {
1386                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1387                 msleep(static_loader->cpu_reset_wait_msec);
1388
1389                 /* Must clear this register in order to prevent preboot
1390                  * from reading WFE after reboot
1391                  */
1392                 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1393         }
1394
1395         hdev->device_cpu_is_halted = true;
1396 }
1397
1398 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1399 {
1400         /* Some of the status codes below are deprecated in newer f/w
1401          * versions but we keep them here for backward compatibility
1402          */
1403         switch (status) {
1404         case CPU_BOOT_STATUS_NA:
1405                 dev_err(hdev->dev,
1406                         "Device boot progress - BTL/ROM did NOT run\n");
1407                 break;
1408         case CPU_BOOT_STATUS_IN_WFE:
1409                 dev_err(hdev->dev,
1410                         "Device boot progress - Stuck inside WFE loop\n");
1411                 break;
1412         case CPU_BOOT_STATUS_IN_BTL:
1413                 dev_err(hdev->dev,
1414                         "Device boot progress - Stuck in BTL\n");
1415                 break;
1416         case CPU_BOOT_STATUS_IN_PREBOOT:
1417                 dev_err(hdev->dev,
1418                         "Device boot progress - Stuck in Preboot\n");
1419                 break;
1420         case CPU_BOOT_STATUS_IN_SPL:
1421                 dev_err(hdev->dev,
1422                         "Device boot progress - Stuck in SPL\n");
1423                 break;
1424         case CPU_BOOT_STATUS_IN_UBOOT:
1425                 dev_err(hdev->dev,
1426                         "Device boot progress - Stuck in u-boot\n");
1427                 break;
1428         case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1429                 dev_err(hdev->dev,
1430                         "Device boot progress - DRAM initialization failed\n");
1431                 break;
1432         case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1433                 dev_err(hdev->dev,
1434                         "Device boot progress - Cannot boot\n");
1435                 break;
1436         case CPU_BOOT_STATUS_TS_INIT_FAIL:
1437                 dev_err(hdev->dev,
1438                         "Device boot progress - Thermal Sensor initialization failed\n");
1439                 break;
1440         case CPU_BOOT_STATUS_SECURITY_READY:
1441                 dev_err(hdev->dev,
1442                         "Device boot progress - Stuck in preboot after security initialization\n");
1443                 break;
1444         default:
1445                 dev_err(hdev->dev,
1446                         "Device boot progress - Invalid or unexpected status code %d\n", status);
1447                 break;
1448         }
1449 }
1450
1451 int hl_fw_wait_preboot_ready(struct hl_device *hdev)
1452 {
1453         struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
1454         u32 status;
1455         int rc;
1456
1457         /* Need to check two possible scenarios:
1458          *
1459          * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1460          * the preboot is waiting for the boot fit
1461          *
1462          * All other status values - for older firmwares where the uboot was
1463          * loaded from the FLASH
1464          */
1465         rc = hl_poll_timeout(
1466                 hdev,
1467                 pre_fw_load->cpu_boot_status_reg,
1468                 status,
1469                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1470                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1471                 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1472                 hdev->fw_poll_interval_usec,
1473                 pre_fw_load->wait_for_preboot_timeout);
1474
1475         if (rc) {
1476                 detect_cpu_boot_status(hdev, status);
1477                 dev_err(hdev->dev, "CPU boot ready timeout (status = %d)\n", status);
1478
1479                 /* If we read all FF, then something is totally wrong, no point
1480                  * of reading specific errors
1481                  */
1482                 if (status != -1)
1483                         fw_read_errors(hdev, pre_fw_load->boot_err0_reg,
1484                                                 pre_fw_load->boot_err1_reg,
1485                                                 pre_fw_load->sts_boot_dev_sts0_reg,
1486                                                 pre_fw_load->sts_boot_dev_sts1_reg);
1487                 return -EIO;
1488         }
1489
1490         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_PREBOOT_CPU;
1491
1492         return 0;
1493 }
1494
1495 static int hl_fw_read_preboot_caps(struct hl_device *hdev)
1496 {
1497         struct pre_fw_load_props *pre_fw_load;
1498         struct asic_fixed_properties *prop;
1499         u32 reg_val;
1500         int rc;
1501
1502         prop = &hdev->asic_prop;
1503         pre_fw_load = &hdev->fw_loader.pre_fw_load;
1504
1505         rc = hl_fw_wait_preboot_ready(hdev);
1506         if (rc)
1507                 return rc;
1508
1509         /*
1510          * the registers DEV_STS* contain FW capabilities/features.
1511          * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1512          * is set.
1513          * In the first read of this register we store the value of this
1514          * register ONLY if the register is enabled (which will be propagated
1515          * to next stages) and also mark the register as valid.
1516          * In case it is not enabled the stored value will be left 0- all
1517          * caps/features are off
1518          */
1519         reg_val = RREG32(pre_fw_load->sts_boot_dev_sts0_reg);
1520         if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1521                 prop->fw_cpu_boot_dev_sts0_valid = true;
1522                 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1523         }
1524
1525         reg_val = RREG32(pre_fw_load->sts_boot_dev_sts1_reg);
1526         if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1527                 prop->fw_cpu_boot_dev_sts1_valid = true;
1528                 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1529         }
1530
1531         prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1532                                                 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1533
1534         /* initialize FW loader once we know what load protocol is used */
1535         hdev->asic_funcs->init_firmware_loader(hdev);
1536
1537         dev_dbg(hdev->dev, "Attempting %s FW load\n",
1538                         prop->dynamic_fw_load ? "dynamic" : "legacy");
1539         return 0;
1540 }
1541
1542 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1543                                         enum hl_fw_component fwc)
1544 {
1545         struct asic_fixed_properties *prop = &hdev->asic_prop;
1546         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1547         struct static_fw_load_mgr *static_loader;
1548         char *dest, *boot_ver, *preboot_ver;
1549         u32 ver_off, limit;
1550         const char *name;
1551         char btl_ver[32];
1552
1553         static_loader = &hdev->fw_loader.static_loader;
1554
1555         switch (fwc) {
1556         case FW_COMP_BOOT_FIT:
1557                 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1558                 dest = prop->uboot_ver;
1559                 name = "Boot-fit";
1560                 limit = static_loader->boot_fit_version_max_off;
1561                 break;
1562         case FW_COMP_PREBOOT:
1563                 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1564                 dest = prop->preboot_ver;
1565                 name = "Preboot";
1566                 limit = static_loader->preboot_version_max_off;
1567                 break;
1568         default:
1569                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1570                 return -EIO;
1571         }
1572
1573         ver_off &= static_loader->sram_offset_mask;
1574
1575         if (ver_off < limit) {
1576                 memcpy_fromio(dest,
1577                         hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1578                         VERSION_MAX_LEN);
1579         } else {
1580                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1581                                                                 name, ver_off);
1582                 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1583                 return -EIO;
1584         }
1585
1586         if (fwc == FW_COMP_BOOT_FIT) {
1587                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1588                 if (boot_ver) {
1589                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1590                         kfree(boot_ver);
1591                 }
1592         } else if (fwc == FW_COMP_PREBOOT) {
1593                 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1594                                                 VERSION_MAX_LEN);
1595                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1596                         strscpy(btl_ver, prop->preboot_ver,
1597                                 min((int) (preboot_ver - prop->preboot_ver),
1598                                                                         31));
1599                         dev_info(hdev->dev, "%s\n", btl_ver);
1600                 }
1601
1602                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1603                 if (preboot_ver) {
1604                         dev_info(hdev->dev, "preboot version %s\n",
1605                                                                 preboot_ver);
1606                         kfree(preboot_ver);
1607                 }
1608         }
1609
1610         return 0;
1611 }
1612
1613 /**
1614  * hl_fw_preboot_update_state - update internal data structures during
1615  *                              handshake with preboot
1616  *
1617  *
1618  * @hdev: pointer to the habanalabs device structure
1619  *
1620  * @return 0 on success, otherwise non-zero error code
1621  */
1622 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1623 {
1624         struct asic_fixed_properties *prop = &hdev->asic_prop;
1625         u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1626
1627         cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1628         cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1629
1630         /* We read boot_dev_sts registers multiple times during boot:
1631          * 1. preboot - a. Check whether the security status bits are valid
1632          *              b. Check whether fw security is enabled
1633          *              c. Check whether hard reset is done by preboot
1634          * 2. boot cpu - a. Fetch boot cpu security status
1635          *               b. Check whether hard reset is done by boot cpu
1636          * 3. FW application - a. Fetch fw application security status
1637          *                     b. Check whether hard reset is done by fw app
1638          */
1639         prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1640
1641         prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN);
1642
1643         dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1644                                                         cpu_boot_dev_sts0);
1645
1646         dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1647                                                         cpu_boot_dev_sts1);
1648
1649         dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1650                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1651
1652         dev_dbg(hdev->dev, "firmware-level security is %s\n",
1653                         prop->fw_security_enabled ? "enabled" : "disabled");
1654
1655         dev_dbg(hdev->dev, "GIC controller is %s\n",
1656                         prop->gic_interrupts_enable ? "enabled" : "disabled");
1657 }
1658
1659 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1660 {
1661         int rc;
1662
1663         rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1664         if (rc)
1665                 return rc;
1666
1667         return 0;
1668 }
1669
1670 int hl_fw_read_preboot_status(struct hl_device *hdev)
1671 {
1672         int rc;
1673
1674         if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1675                 return 0;
1676
1677         /* get FW pre-load parameters  */
1678         hdev->asic_funcs->init_firmware_preload_params(hdev);
1679
1680         /*
1681          * In order to determine boot method (static VS dynamic) we need to
1682          * read the boot caps register
1683          */
1684         rc = hl_fw_read_preboot_caps(hdev);
1685         if (rc)
1686                 return rc;
1687
1688         hl_fw_preboot_update_state(hdev);
1689
1690         /* no need to read preboot status in dynamic load */
1691         if (hdev->asic_prop.dynamic_fw_load)
1692                 return 0;
1693
1694         return hl_fw_static_read_preboot_status(hdev);
1695 }
1696
1697 /* associate string with COMM status */
1698 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1699         [COMMS_STS_NOOP] = "NOOP",
1700         [COMMS_STS_ACK] = "ACK",
1701         [COMMS_STS_OK] = "OK",
1702         [COMMS_STS_ERR] = "ERR",
1703         [COMMS_STS_VALID_ERR] = "VALID_ERR",
1704         [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1705 };
1706
1707 /**
1708  * hl_fw_dynamic_report_error_status - report error status
1709  *
1710  * @hdev: pointer to the habanalabs device structure
1711  * @status: value of FW status register
1712  * @expected_status: the expected status
1713  */
1714 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1715                                                 u32 status,
1716                                                 enum comms_sts expected_status)
1717 {
1718         enum comms_sts comm_status =
1719                                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1720
1721         if (comm_status < COMMS_STS_INVLD_LAST)
1722                 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1723                                 hl_dynamic_fw_status_str[comm_status],
1724                                 hl_dynamic_fw_status_str[expected_status]);
1725         else
1726                 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1727                                 comm_status,
1728                                 hl_dynamic_fw_status_str[expected_status]);
1729 }
1730
1731 /**
1732  * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1733  *
1734  * @hdev: pointer to the habanalabs device structure
1735  * @fw_loader: managing structure for loading device's FW
1736  * @cmd: LKD to FW cmd code
1737  * @size: size of next FW component to be loaded (0 if not necessary)
1738  *
1739  * LDK to FW exact command layout is defined at struct comms_command.
1740  * note: the size argument is used only when the next FW component should be
1741  *       loaded, otherwise it shall be 0. the size is used by the FW in later
1742  *       protocol stages and when sending only indicating the amount of memory
1743  *       to be allocated by the FW to receive the next boot component.
1744  */
1745 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1746                                 struct fw_load_mgr *fw_loader,
1747                                 enum comms_cmd cmd, unsigned int size)
1748 {
1749         struct cpu_dyn_regs *dyn_regs;
1750         u32 val;
1751
1752         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1753
1754         val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1755         val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1756
1757         trace_habanalabs_comms_send_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1758         WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1759 }
1760
1761 /**
1762  * hl_fw_dynamic_extract_fw_response - update the FW response
1763  *
1764  * @hdev: pointer to the habanalabs device structure
1765  * @fw_loader: managing structure for loading device's FW
1766  * @response: FW response
1767  * @status: the status read from CPU status register
1768  *
1769  * @return 0 on success, otherwise non-zero error code
1770  */
1771 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1772                                                 struct fw_load_mgr *fw_loader,
1773                                                 struct fw_response *response,
1774                                                 u32 status)
1775 {
1776         response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1777         response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1778                                                 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1779         response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1780
1781         if ((response->ram_type != COMMS_SRAM) &&
1782                                         (response->ram_type != COMMS_DRAM)) {
1783                 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1784                                                         response->ram_type);
1785                 return -EIO;
1786         }
1787
1788         return 0;
1789 }
1790
1791 /**
1792  * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1793  *
1794  * @hdev: pointer to the habanalabs device structure
1795  * @fw_loader: managing structure for loading device's FW
1796  * @expected_status: expected status to wait for
1797  * @timeout: timeout for status wait
1798  *
1799  * @return 0 on success, otherwise non-zero error code
1800  *
1801  * waiting for status from FW include polling the FW status register until
1802  * expected status is received or timeout occurs (whatever occurs first).
1803  */
1804 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1805                                                 struct fw_load_mgr *fw_loader,
1806                                                 enum comms_sts expected_status,
1807                                                 u32 timeout)
1808 {
1809         struct cpu_dyn_regs *dyn_regs;
1810         u32 status;
1811         int rc;
1812
1813         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1814
1815         trace_habanalabs_comms_wait_status(hdev->dev, comms_sts_str_arr[expected_status]);
1816
1817         /* Wait for expected status */
1818         rc = hl_poll_timeout(
1819                 hdev,
1820                 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1821                 status,
1822                 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1823                 hdev->fw_comms_poll_interval_usec,
1824                 timeout);
1825
1826         if (rc) {
1827                 hl_fw_dynamic_report_error_status(hdev, status,
1828                                                         expected_status);
1829                 return -EIO;
1830         }
1831
1832         trace_habanalabs_comms_wait_status_done(hdev->dev, comms_sts_str_arr[expected_status]);
1833
1834         /*
1835          * skip storing FW response for NOOP to preserve the actual desired
1836          * FW status
1837          */
1838         if (expected_status == COMMS_STS_NOOP)
1839                 return 0;
1840
1841         rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1842                                         &fw_loader->dynamic_loader.response,
1843                                         status);
1844         return rc;
1845 }
1846
1847 /**
1848  * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1849  *
1850  * @hdev: pointer to the habanalabs device structure
1851  * @fw_loader: managing structure for loading device's FW
1852  *
1853  * @return 0 on success, otherwise non-zero error code
1854  *
1855  * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1856  * from FW) we need to clear the CPU status register in order to avoid garbage
1857  * between command cycles.
1858  * This is done by sending clear command and polling the CPU to LKD status
1859  * register to hold the status NOOP
1860  */
1861 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1862                                                 struct fw_load_mgr *fw_loader)
1863 {
1864         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1865
1866         return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1867                                                         fw_loader->cpu_timeout);
1868 }
1869
1870 /**
1871  * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1872  *
1873  * @hdev: pointer to the habanalabs device structure
1874  * @fw_loader: managing structure for loading device's FW
1875  * @cmd: LKD to FW cmd code
1876  * @size: size of next FW component to be loaded (0 if not necessary)
1877  * @wait_ok: if true also wait for OK response from FW
1878  * @timeout: timeout for status wait
1879  *
1880  * @return 0 on success, otherwise non-zero error code
1881  *
1882  * brief:
1883  * when sending protocol command we have the following steps:
1884  * - send clear (clear command and verify clear status register)
1885  * - send the actual protocol command
1886  * - wait for ACK on the protocol command
1887  * - send clear
1888  * - send NOOP
1889  * if, in addition, the specific protocol command should wait for OK then:
1890  * - wait for OK
1891  * - send clear
1892  * - send NOOP
1893  *
1894  * NOTES:
1895  * send clear: this is necessary in order to clear the status register to avoid
1896  *             leftovers between command
1897  * NOOP command: necessary to avoid loop on the clear command by the FW
1898  */
1899 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1900                                 struct fw_load_mgr *fw_loader,
1901                                 enum comms_cmd cmd, unsigned int size,
1902                                 bool wait_ok, u32 timeout)
1903 {
1904         int rc;
1905
1906         trace_habanalabs_comms_protocol_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1907
1908         /* first send clear command to clean former commands */
1909         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1910         if (rc)
1911                 return rc;
1912
1913         /* send the actual command */
1914         hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1915
1916         /* wait for ACK for the command */
1917         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1918                                                                 timeout);
1919         if (rc)
1920                 return rc;
1921
1922         /* clear command to prepare for NOOP command */
1923         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1924         if (rc)
1925                 return rc;
1926
1927         /* send the actual NOOP command */
1928         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1929
1930         if (!wait_ok)
1931                 return 0;
1932
1933         rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1934                                                                 timeout);
1935         if (rc)
1936                 return rc;
1937
1938         /* clear command to prepare for NOOP command */
1939         rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1940         if (rc)
1941                 return rc;
1942
1943         /* send the actual NOOP command */
1944         hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1945
1946         return 0;
1947 }
1948
1949 /**
1950  * hl_fw_compat_crc32 - CRC compatible with FW
1951  *
1952  * @data: pointer to the data
1953  * @size: size of the data
1954  *
1955  * @return the CRC32 result
1956  *
1957  * NOTE: kernel's CRC32 differs from standard CRC32 calculation.
1958  *       in order to be aligned we need to flip the bits of both the input
1959  *       initial CRC and kernel's CRC32 result.
1960  *       in addition both sides use initial CRC of 0,
1961  */
1962 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1963 {
1964         return ~crc32_le(~((u32)0), data, size);
1965 }
1966
1967 /**
1968  * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1969  *                                        transfer (image or descriptor) between
1970  *                                        host and FW
1971  *
1972  * @hdev: pointer to the habanalabs device structure
1973  * @addr: device address of memory transfer
1974  * @size: memory transfer size
1975  * @region: PCI memory region
1976  *
1977  * @return 0 on success, otherwise non-zero error code
1978  */
1979 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1980                                                 u64 addr, size_t size,
1981                                                 struct pci_mem_region *region)
1982 {
1983         u64 end_addr;
1984
1985         /* now make sure that the memory transfer is within region's bounds */
1986         end_addr = addr + size;
1987         if (end_addr >= region->region_base + region->region_size) {
1988                 dev_err(hdev->dev,
1989                         "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1990                                                         end_addr);
1991                 return -EIO;
1992         }
1993
1994         /*
1995          * now make sure memory transfer is within predefined BAR bounds.
1996          * this is to make sure we do not need to set the bar (e.g. for DRAM
1997          * memory transfers)
1998          */
1999         if (end_addr >= region->region_base - region->offset_in_bar +
2000                                                         region->bar_size) {
2001                 dev_err(hdev->dev,
2002                         "FW image beyond PCI BAR bounds\n");
2003                 return -EIO;
2004         }
2005
2006         return 0;
2007 }
2008
2009 /**
2010  * hl_fw_dynamic_validate_descriptor - validate FW descriptor
2011  *
2012  * @hdev: pointer to the habanalabs device structure
2013  * @fw_loader: managing structure for loading device's FW
2014  * @fw_desc: the descriptor from FW
2015  *
2016  * @return 0 on success, otherwise non-zero error code
2017  */
2018 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
2019                                         struct fw_load_mgr *fw_loader,
2020                                         struct lkd_fw_comms_desc *fw_desc)
2021 {
2022         struct pci_mem_region *region;
2023         enum pci_region region_id;
2024         size_t data_size;
2025         u32 data_crc32;
2026         u8 *data_ptr;
2027         u64 addr;
2028         int rc;
2029
2030         if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC)
2031                 dev_dbg(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
2032                                 fw_desc->header.magic);
2033
2034         if (fw_desc->header.version != HL_COMMS_DESC_VER)
2035                 dev_dbg(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
2036                                 fw_desc->header.version);
2037
2038         /*
2039          * Calc CRC32 of data without header. use the size of the descriptor
2040          * reported by firmware, without calculating it ourself, to allow adding
2041          * more fields to the lkd_fw_comms_desc structure.
2042          * note that no alignment/stride address issues here as all structures
2043          * are 64 bit padded.
2044          */
2045         data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
2046         data_size = le16_to_cpu(fw_desc->header.size);
2047
2048         data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
2049         if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
2050                 dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
2051                         data_crc32, fw_desc->header.crc32);
2052                 return -EIO;
2053         }
2054
2055         /* find memory region to which to copy the image */
2056         addr = le64_to_cpu(fw_desc->img_addr);
2057         region_id = hl_get_pci_memory_region(hdev, addr);
2058         if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) {
2059                 dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr);
2060                 return -EIO;
2061         }
2062
2063         region = &hdev->pci_mem_region[region_id];
2064
2065         /* store the region for the copy stage */
2066         fw_loader->dynamic_loader.image_region = region;
2067
2068         /*
2069          * here we know that the start address is valid, now make sure that the
2070          * image is within region's bounds
2071          */
2072         rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
2073                                         fw_loader->dynamic_loader.fw_image_size,
2074                                         region);
2075         if (rc) {
2076                 dev_err(hdev->dev, "invalid mem transfer request for FW image\n");
2077                 return rc;
2078         }
2079
2080         /* here we can mark the descriptor as valid as the content has been validated */
2081         fw_loader->dynamic_loader.fw_desc_valid = true;
2082
2083         return 0;
2084 }
2085
2086 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
2087                                                 struct fw_response *response,
2088                                                 struct pci_mem_region *region)
2089 {
2090         u64 device_addr;
2091         int rc;
2092
2093         device_addr = region->region_base + response->ram_offset;
2094
2095         /*
2096          * validate that the descriptor is within region's bounds
2097          * Note that as the start address was supplied according to the RAM
2098          * type- testing only the end address is enough
2099          */
2100         rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
2101                                         sizeof(struct lkd_fw_comms_desc),
2102                                         region);
2103         return rc;
2104 }
2105
2106 /*
2107  * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw
2108  *
2109  * @hdev: pointer to the habanalabs device structure
2110  * @fw_desc: the descriptor from FW
2111  */
2112 static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev,
2113                                         struct lkd_fw_comms_desc *fw_desc)
2114 {
2115         int i;
2116         char *msg;
2117
2118         for (i = 0 ; i < LKD_FW_ASCII_MSG_MAX ; i++) {
2119                 if (!fw_desc->ascii_msg[i].valid)
2120                         return;
2121
2122                 /* force NULL termination */
2123                 msg = fw_desc->ascii_msg[i].msg;
2124                 msg[LKD_FW_ASCII_MSG_MAX_LEN - 1] = '\0';
2125
2126                 switch (fw_desc->ascii_msg[i].msg_lvl) {
2127                 case LKD_FW_ASCII_MSG_ERR:
2128                         dev_err(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2129                         break;
2130                 case LKD_FW_ASCII_MSG_WRN:
2131                         dev_warn(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2132                         break;
2133                 case LKD_FW_ASCII_MSG_INF:
2134                         dev_info(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2135                         break;
2136                 default:
2137                         dev_dbg(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2138                         break;
2139                 }
2140         }
2141 }
2142
2143 /**
2144  * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
2145  *
2146  * @hdev: pointer to the habanalabs device structure
2147  * @fw_loader: managing structure for loading device's FW
2148  *
2149  * @return 0 on success, otherwise non-zero error code
2150  */
2151 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
2152                                                 struct fw_load_mgr *fw_loader)
2153 {
2154         struct lkd_fw_comms_desc *fw_desc;
2155         struct pci_mem_region *region;
2156         struct fw_response *response;
2157         void *temp_fw_desc;
2158         void __iomem *src;
2159         u16 fw_data_size;
2160         enum pci_region region_id;
2161         int rc;
2162
2163         fw_desc = &fw_loader->dynamic_loader.comm_desc;
2164         response = &fw_loader->dynamic_loader.response;
2165
2166         region_id = (response->ram_type == COMMS_SRAM) ?
2167                                         PCI_REGION_SRAM : PCI_REGION_DRAM;
2168
2169         region = &hdev->pci_mem_region[region_id];
2170
2171         rc = hl_fw_dynamic_validate_response(hdev, response, region);
2172         if (rc) {
2173                 dev_err(hdev->dev,
2174                         "invalid mem transfer request for FW descriptor\n");
2175                 return rc;
2176         }
2177
2178         /*
2179          * extract address to copy the descriptor from
2180          * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
2181          * as invalid.
2182          * it will be marked again as valid once validated
2183          */
2184         fw_loader->dynamic_loader.fw_desc_valid = false;
2185         src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2186                                                         response->ram_offset;
2187
2188         /*
2189          * We do the copy of the fw descriptor in 2 phases:
2190          * 1. copy the header + data info according to our lkd_fw_comms_desc definition.
2191          *    then we're able to read the actual data size provided by fw.
2192          *    this is needed for cases where data in descriptor was changed(add/remove)
2193          *    in embedded specs header file before updating lkd copy of the header file
2194          * 2. copy descriptor to temporary buffer with aligned size and send it to validation
2195          */
2196         memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
2197         fw_data_size = le16_to_cpu(fw_desc->header.size);
2198
2199         temp_fw_desc = vzalloc(sizeof(struct comms_desc_header) + fw_data_size);
2200         if (!temp_fw_desc)
2201                 return -ENOMEM;
2202
2203         memcpy_fromio(temp_fw_desc, src, sizeof(struct comms_desc_header) + fw_data_size);
2204
2205         rc = hl_fw_dynamic_validate_descriptor(hdev, fw_loader,
2206                                         (struct lkd_fw_comms_desc *) temp_fw_desc);
2207
2208         if (!rc)
2209                 hl_fw_dynamic_read_descriptor_msg(hdev, temp_fw_desc);
2210
2211         vfree(temp_fw_desc);
2212
2213         return rc;
2214 }
2215
2216 /**
2217  * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
2218  *
2219  * @hdev: pointer to the habanalabs device structure
2220  * @fw_loader: managing structure for loading device's FW
2221  * @next_image_size: size to allocate for next FW component
2222  *
2223  * @return 0 on success, otherwise non-zero error code
2224  */
2225 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
2226                                                 struct fw_load_mgr *fw_loader,
2227                                                 size_t next_image_size)
2228 {
2229         int rc;
2230
2231         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
2232                                                 next_image_size, true,
2233                                                 fw_loader->cpu_timeout);
2234         if (rc)
2235                 return rc;
2236
2237         return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
2238 }
2239
2240 /**
2241  * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
2242  *
2243  * @hdev: pointer to the habanalabs device structure
2244  * @fwc: the firmware component
2245  * @fw_version: fw component's version string
2246  */
2247 static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
2248                                         enum hl_fw_component fwc,
2249                                         const char *fw_version)
2250 {
2251         struct asic_fixed_properties *prop = &hdev->asic_prop;
2252         char *preboot_ver, *boot_ver;
2253         char btl_ver[32];
2254         int rc;
2255
2256         switch (fwc) {
2257         case FW_COMP_BOOT_FIT:
2258                 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
2259                 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
2260                 if (boot_ver) {
2261                         dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
2262                         kfree(boot_ver);
2263                 }
2264
2265                 break;
2266         case FW_COMP_PREBOOT:
2267                 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
2268                 preboot_ver = strnstr(prop->preboot_ver, "Preboot", VERSION_MAX_LEN);
2269                 dev_info(hdev->dev, "preboot full version: '%s'\n", preboot_ver);
2270
2271                 if (preboot_ver && preboot_ver != prop->preboot_ver) {
2272                         strscpy(btl_ver, prop->preboot_ver,
2273                                 min((int) (preboot_ver - prop->preboot_ver), 31));
2274                         dev_info(hdev->dev, "%s\n", btl_ver);
2275                 }
2276
2277                 rc = hl_get_sw_major_minor_subminor(hdev, preboot_ver);
2278                 if (rc)
2279                         return rc;
2280                 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
2281                 if (preboot_ver) {
2282                         rc = hl_get_preboot_major_minor(hdev, preboot_ver);
2283                         kfree(preboot_ver);
2284                         if (rc)
2285                                 return rc;
2286                 }
2287
2288                 break;
2289         default:
2290                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2291                 return -EINVAL;
2292         }
2293
2294         return 0;
2295 }
2296
2297 /**
2298  * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
2299  *
2300  * @hdev: pointer to the habanalabs device structure
2301  * @fw: fw descriptor
2302  * @fw_loader: managing structure for loading device's FW
2303  */
2304 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
2305                                                 const struct firmware *fw,
2306                                                 struct fw_load_mgr *fw_loader)
2307 {
2308         struct lkd_fw_comms_desc *fw_desc;
2309         struct pci_mem_region *region;
2310         void __iomem *dest;
2311         u64 addr;
2312         int rc;
2313
2314         fw_desc = &fw_loader->dynamic_loader.comm_desc;
2315         addr = le64_to_cpu(fw_desc->img_addr);
2316
2317         /* find memory region to which to copy the image */
2318         region = fw_loader->dynamic_loader.image_region;
2319
2320         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2321                                         (addr - region->region_base);
2322
2323         rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
2324                                         fw_loader->boot_fit_img.src_off,
2325                                         fw_loader->boot_fit_img.copy_size);
2326
2327         return rc;
2328 }
2329
2330 /**
2331  * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
2332  *
2333  * @hdev: pointer to the habanalabs device structure
2334  * @msg: message
2335  * @fw_loader: managing structure for loading device's FW
2336  */
2337 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
2338                 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
2339 {
2340         struct lkd_fw_comms_desc *fw_desc;
2341         struct pci_mem_region *region;
2342         void __iomem *dest;
2343         u64 addr;
2344         int rc;
2345
2346         fw_desc = &fw_loader->dynamic_loader.comm_desc;
2347         addr = le64_to_cpu(fw_desc->img_addr);
2348
2349         /* find memory region to which to copy the image */
2350         region = fw_loader->dynamic_loader.image_region;
2351
2352         dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2353                                         (addr - region->region_base);
2354
2355         rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2356
2357         return rc;
2358 }
2359
2360 /**
2361  * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2362  *                               is loaded
2363  *
2364  * @hdev: pointer to the habanalabs device structure
2365  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2366  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2367  *
2368  * @return 0 on success, otherwise non-zero error code
2369  */
2370 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2371                                                 u32 cpu_boot_dev_sts0_reg,
2372                                                 u32 cpu_boot_dev_sts1_reg)
2373 {
2374         struct asic_fixed_properties *prop = &hdev->asic_prop;
2375
2376         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2377
2378         /* Read boot_cpu status bits */
2379         if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2380                 prop->fw_bootfit_cpu_boot_dev_sts0 =
2381                                 RREG32(cpu_boot_dev_sts0_reg);
2382
2383                 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2384                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2385
2386                 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2387                                         prop->fw_bootfit_cpu_boot_dev_sts0);
2388         }
2389
2390         if (prop->fw_cpu_boot_dev_sts1_valid) {
2391                 prop->fw_bootfit_cpu_boot_dev_sts1 =
2392                                 RREG32(cpu_boot_dev_sts1_reg);
2393
2394                 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2395                                         prop->fw_bootfit_cpu_boot_dev_sts1);
2396         }
2397
2398         dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2399                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2400 }
2401
2402 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2403 {
2404         struct cpu_dyn_regs *dyn_regs =
2405                         &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2406
2407         /* Check whether all 3 interrupt interfaces are set, if not use a
2408          * single interface
2409          */
2410         if (!hdev->asic_prop.gic_interrupts_enable &&
2411                         !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2412                                 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2413                 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2414                 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2415
2416                 dev_warn(hdev->dev,
2417                         "Using a single interrupt interface towards cpucp");
2418         }
2419 }
2420 /**
2421  * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2422  *
2423  * @hdev: pointer to the habanalabs device structure
2424  * @fw_loader: managing structure for loading device's FW
2425  * @load_fwc: the FW component to be loaded
2426  * @img_ld_timeout: image load timeout
2427  *
2428  * @return 0 on success, otherwise non-zero error code
2429  */
2430 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2431                                                 struct fw_load_mgr *fw_loader,
2432                                                 enum hl_fw_component load_fwc,
2433                                                 u32 img_ld_timeout)
2434 {
2435         enum hl_fw_component cur_fwc;
2436         const struct firmware *fw;
2437         char *fw_name;
2438         int rc = 0;
2439
2440         /*
2441          * when loading image we have one of 2 scenarios:
2442          * 1. current FW component is preboot and we want to load boot-fit
2443          * 2. current FW component is boot-fit and we want to load linux
2444          */
2445         if (load_fwc == FW_COMP_BOOT_FIT) {
2446                 cur_fwc = FW_COMP_PREBOOT;
2447                 fw_name = fw_loader->boot_fit_img.image_name;
2448         } else {
2449                 cur_fwc = FW_COMP_BOOT_FIT;
2450                 fw_name = fw_loader->linux_img.image_name;
2451         }
2452
2453         /* request FW in order to communicate to FW the size to be allocated */
2454         rc = hl_request_fw(hdev, &fw, fw_name);
2455         if (rc)
2456                 return rc;
2457
2458         /* store the image size for future validation */
2459         fw_loader->dynamic_loader.fw_image_size = fw->size;
2460
2461         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2462         if (rc)
2463                 goto release_fw;
2464
2465         /* read preboot version */
2466         rc = hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2467                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2468         if (rc)
2469                 goto release_fw;
2470
2471         /* update state according to boot stage */
2472         if (cur_fwc == FW_COMP_BOOT_FIT) {
2473                 struct cpu_dyn_regs *dyn_regs;
2474
2475                 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2476                 hl_fw_boot_fit_update_state(hdev,
2477                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2478                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2479         }
2480
2481         /* copy boot fit to space allocated by FW */
2482         rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2483         if (rc)
2484                 goto release_fw;
2485
2486         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2487                                                 0, true,
2488                                                 fw_loader->cpu_timeout);
2489         if (rc)
2490                 goto release_fw;
2491
2492         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2493                                                 0, false,
2494                                                 img_ld_timeout);
2495
2496 release_fw:
2497         hl_release_firmware(fw);
2498         return rc;
2499 }
2500
2501 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2502                                         struct fw_load_mgr *fw_loader)
2503 {
2504         struct dynamic_fw_load_mgr *dyn_loader;
2505         u32 status;
2506         int rc;
2507
2508         dyn_loader = &fw_loader->dynamic_loader;
2509
2510         /*
2511          * Make sure CPU boot-loader is running
2512          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2513          * yet there is a debug scenario in which we loading uboot (without Linux)
2514          * which at later stage is relocated to DRAM. In this case we expect
2515          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2516          * poll flags
2517          */
2518         rc = hl_poll_timeout(
2519                 hdev,
2520                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2521                 status,
2522                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2523                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2524                 hdev->fw_poll_interval_usec,
2525                 dyn_loader->wait_for_bl_timeout);
2526         if (rc) {
2527                 dev_err(hdev->dev, "failed to wait for boot (status = %d)\n", status);
2528                 return rc;
2529         }
2530
2531         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2532         return 0;
2533 }
2534
2535 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2536                                                 struct fw_load_mgr *fw_loader)
2537 {
2538         struct dynamic_fw_load_mgr *dyn_loader;
2539         u32 status;
2540         int rc;
2541
2542         dyn_loader = &fw_loader->dynamic_loader;
2543
2544         /* Make sure CPU linux is running */
2545
2546         rc = hl_poll_timeout(
2547                 hdev,
2548                 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2549                 status,
2550                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2551                 hdev->fw_poll_interval_usec,
2552                 fw_loader->cpu_timeout);
2553         if (rc) {
2554                 dev_err(hdev->dev, "failed to wait for Linux (status = %d)\n", status);
2555                 return rc;
2556         }
2557
2558         dev_dbg(hdev->dev, "Boot status = %d\n", status);
2559         return 0;
2560 }
2561
2562 /**
2563  * hl_fw_linux_update_state -   update internal data structures after Linux
2564  *                              is loaded.
2565  *                              Note: Linux initialization is comprised mainly
2566  *                              of two stages - loading kernel (SRAM_AVAIL)
2567  *                              & loading ARMCP.
2568  *                              Therefore reading boot device status in any of
2569  *                              these stages might result in different values.
2570  *
2571  * @hdev: pointer to the habanalabs device structure
2572  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2573  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2574  *
2575  * @return 0 on success, otherwise non-zero error code
2576  */
2577 static void hl_fw_linux_update_state(struct hl_device *hdev,
2578                                                 u32 cpu_boot_dev_sts0_reg,
2579                                                 u32 cpu_boot_dev_sts1_reg)
2580 {
2581         struct asic_fixed_properties *prop = &hdev->asic_prop;
2582
2583         hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2584
2585         /* Read FW application security bits */
2586         if (prop->fw_cpu_boot_dev_sts0_valid) {
2587                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2588
2589                 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2590                                                         CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2591
2592                 if (prop->fw_app_cpu_boot_dev_sts0 &
2593                                 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2594                         prop->gic_interrupts_enable = false;
2595
2596                 dev_dbg(hdev->dev,
2597                         "Firmware application CPU status0 %#x\n",
2598                         prop->fw_app_cpu_boot_dev_sts0);
2599
2600                 dev_dbg(hdev->dev, "GIC controller is %s\n",
2601                                 prop->gic_interrupts_enable ?
2602                                                 "enabled" : "disabled");
2603         }
2604
2605         if (prop->fw_cpu_boot_dev_sts1_valid) {
2606                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2607
2608                 dev_dbg(hdev->dev,
2609                         "Firmware application CPU status1 %#x\n",
2610                         prop->fw_app_cpu_boot_dev_sts1);
2611         }
2612
2613         dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2614                         prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2615
2616         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2617 }
2618
2619 /**
2620  * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2621  *
2622  * @hdev: pointer to the habanalabs device structure
2623  * @fw_loader: managing structure for loading device's FW
2624  * @msg_type: message type
2625  * @data: data to be sent
2626  *
2627  * @return 0 on success, otherwise non-zero error code
2628  */
2629 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2630                 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2631 {
2632         struct lkd_msg_comms *msg;
2633         int rc;
2634
2635         msg = kzalloc(sizeof(*msg), GFP_KERNEL);
2636         if (!msg)
2637                 return -ENOMEM;
2638
2639         /* create message to be sent */
2640         msg->header.type = msg_type;
2641         msg->header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2642         msg->header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2643
2644         switch (msg_type) {
2645         case HL_COMMS_RESET_CAUSE_TYPE:
2646                 msg->reset_cause = *(__u8 *) data;
2647                 break;
2648
2649         default:
2650                 dev_err(hdev->dev,
2651                         "Send COMMS message - invalid message type %u\n",
2652                         msg_type);
2653                 rc = -EINVAL;
2654                 goto out;
2655         }
2656
2657         rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2658                         sizeof(struct lkd_msg_comms));
2659         if (rc)
2660                 goto out;
2661
2662         /* copy message to space allocated by FW */
2663         rc = hl_fw_dynamic_copy_msg(hdev, msg, fw_loader);
2664         if (rc)
2665                 goto out;
2666
2667         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2668                                                 0, true,
2669                                                 fw_loader->cpu_timeout);
2670         if (rc)
2671                 goto out;
2672
2673         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2674                                                 0, true,
2675                                                 fw_loader->cpu_timeout);
2676
2677 out:
2678         kfree(msg);
2679         return rc;
2680 }
2681
2682 /**
2683  * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2684  *
2685  * @hdev: pointer to the habanalabs device structure
2686  * @fw_loader: managing structure for loading device's FW
2687  *
2688  * @return 0 on success, otherwise non-zero error code
2689  *
2690  * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2691  * the communication is done using registers:
2692  * - LKD command register
2693  * - FW status register
2694  * the protocol is race free. this goal is achieved by splitting the requests
2695  * and response to known synchronization points between the LKD and the FW.
2696  * each response to LKD request is known and bound to a predefined timeout.
2697  * in case of timeout expiration without the desired status from FW- the
2698  * protocol (and hence the boot) will fail.
2699  */
2700 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2701                                         struct fw_load_mgr *fw_loader)
2702 {
2703         struct cpu_dyn_regs *dyn_regs;
2704         int rc, fw_error_rc;
2705
2706         dev_info(hdev->dev,
2707                 "Loading %sfirmware to device, may take some time...\n",
2708                 hdev->asic_prop.fw_security_enabled ? "secured " : "");
2709
2710         /* initialize FW descriptor as invalid */
2711         fw_loader->dynamic_loader.fw_desc_valid = false;
2712
2713         /*
2714          * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2715          * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2716          */
2717         dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2718
2719         rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2720                                                 0, true,
2721                                                 fw_loader->cpu_timeout);
2722         if (rc)
2723                 goto protocol_err;
2724
2725         if (hdev->reset_info.curr_reset_cause) {
2726                 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2727                                 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2728                 if (rc)
2729                         goto protocol_err;
2730
2731                 /* Clear current reset cause */
2732                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2733         }
2734
2735         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2736                 struct lkd_fw_binning_info *binning_info;
2737
2738                 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2739                 if (rc)
2740                         goto protocol_err;
2741
2742                 /* read preboot version */
2743                 rc = hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2744                                 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2745
2746                 if (rc)
2747                         return rc;
2748
2749                 /* read binning info from preboot */
2750                 if (hdev->support_preboot_binning) {
2751                         binning_info = &fw_loader->dynamic_loader.comm_desc.binning_info;
2752                         hdev->tpc_binning = le64_to_cpu(binning_info->tpc_mask_l);
2753                         hdev->dram_binning = le32_to_cpu(binning_info->dram_mask);
2754                         hdev->edma_binning = le32_to_cpu(binning_info->edma_mask);
2755                         hdev->decoder_binning = le32_to_cpu(binning_info->dec_mask);
2756                         hdev->rotator_binning = le32_to_cpu(binning_info->rot_mask);
2757
2758                         rc = hdev->asic_funcs->set_dram_properties(hdev);
2759                         if (rc)
2760                                 return rc;
2761
2762                         rc = hdev->asic_funcs->set_binning_masks(hdev);
2763                         if (rc)
2764                                 return rc;
2765
2766                         dev_dbg(hdev->dev,
2767                                 "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x, rot:0x%x\n",
2768                                 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2769                                 hdev->decoder_binning, hdev->rotator_binning);
2770                 }
2771
2772                 return 0;
2773         }
2774
2775         /* load boot fit to FW */
2776         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2777                                                 fw_loader->boot_fit_timeout);
2778         if (rc) {
2779                 dev_err(hdev->dev, "failed to load boot fit\n");
2780                 goto protocol_err;
2781         }
2782
2783         /*
2784          * when testing FW load (without Linux) on PLDM we don't want to
2785          * wait until boot fit is active as it may take several hours.
2786          * instead, we load the bootfit and let it do all initialization in
2787          * the background.
2788          */
2789         if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2790                 return 0;
2791
2792         rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2793         if (rc)
2794                 goto protocol_err;
2795
2796         /* Enable DRAM scrambling before Linux boot and after successful
2797          *  UBoot
2798          */
2799         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2800
2801         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2802                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2803                 return 0;
2804         }
2805
2806         if (fw_loader->skip_bmc) {
2807                 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2808                                                         COMMS_SKIP_BMC, 0,
2809                                                         true,
2810                                                         fw_loader->cpu_timeout);
2811                 if (rc) {
2812                         dev_err(hdev->dev, "failed to load boot fit\n");
2813                         goto protocol_err;
2814                 }
2815         }
2816
2817         /* load Linux image to FW */
2818         rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2819                                                         fw_loader->cpu_timeout);
2820         if (rc) {
2821                 dev_err(hdev->dev, "failed to load Linux\n");
2822                 goto protocol_err;
2823         }
2824
2825         rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2826         if (rc)
2827                 goto protocol_err;
2828
2829         hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2830                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2831
2832         hl_fw_dynamic_update_linux_interrupt_if(hdev);
2833
2834 protocol_err:
2835         if (fw_loader->dynamic_loader.fw_desc_valid) {
2836                 fw_error_rc = fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2837                                 le32_to_cpu(dyn_regs->cpu_boot_err1),
2838                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2839                                 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2840
2841                 if (fw_error_rc)
2842                         return fw_error_rc;
2843         }
2844
2845         return rc;
2846 }
2847
2848 /**
2849  * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2850  *
2851  * @hdev: pointer to the habanalabs device structure
2852  * @fw_loader: managing structure for loading device's FW
2853  *
2854  * @return 0 on success, otherwise non-zero error code
2855  */
2856 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2857                                         struct fw_load_mgr *fw_loader)
2858 {
2859         u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2860         u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2861         struct static_fw_load_mgr *static_loader;
2862         u32 cpu_boot_status_reg;
2863         int rc;
2864
2865         if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2866                 return 0;
2867
2868         /* init common loader parameters */
2869         cpu_timeout = fw_loader->cpu_timeout;
2870
2871         /* init static loader parameters */
2872         static_loader = &fw_loader->static_loader;
2873         cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2874         msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2875         cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2876         cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2877         cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2878
2879         dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2880                 cpu_timeout / USEC_PER_SEC);
2881
2882         /* Wait for boot FIT request */
2883         rc = hl_poll_timeout(
2884                 hdev,
2885                 cpu_boot_status_reg,
2886                 status,
2887                 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2888                 hdev->fw_poll_interval_usec,
2889                 fw_loader->boot_fit_timeout);
2890
2891         if (rc) {
2892                 dev_dbg(hdev->dev,
2893                         "No boot fit request received (status = %d), resuming boot\n", status);
2894         } else {
2895                 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2896                 if (rc)
2897                         goto out;
2898
2899                 /* Clear device CPU message status */
2900                 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2901
2902                 /* Signal device CPU that boot loader is ready */
2903                 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2904
2905                 /* Poll for CPU device ack */
2906                 rc = hl_poll_timeout(
2907                         hdev,
2908                         cpu_msg_status_reg,
2909                         status,
2910                         status == CPU_MSG_OK,
2911                         hdev->fw_poll_interval_usec,
2912                         fw_loader->boot_fit_timeout);
2913
2914                 if (rc) {
2915                         dev_err(hdev->dev,
2916                                 "Timeout waiting for boot fit load ack (status = %d)\n", status);
2917                         goto out;
2918                 }
2919
2920                 /* Clear message */
2921                 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2922         }
2923
2924         /*
2925          * Make sure CPU boot-loader is running
2926          * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2927          * yet there is a debug scenario in which we loading uboot (without Linux)
2928          * which at later stage is relocated to DRAM. In this case we expect
2929          * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2930          * poll flags
2931          */
2932         rc = hl_poll_timeout(
2933                 hdev,
2934                 cpu_boot_status_reg,
2935                 status,
2936                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2937                 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2938                 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2939                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2940                 hdev->fw_poll_interval_usec,
2941                 cpu_timeout);
2942
2943         dev_dbg(hdev->dev, "uboot status = %d\n", status);
2944
2945         /* Read U-Boot version now in case we will later fail */
2946         hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2947
2948         /* update state according to boot stage */
2949         hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2950                                                 cpu_boot_dev_status1_reg);
2951
2952         if (rc) {
2953                 detect_cpu_boot_status(hdev, status);
2954                 rc = -EIO;
2955                 goto out;
2956         }
2957
2958         /* Enable DRAM scrambling before Linux boot and after successful
2959          *  UBoot
2960          */
2961         hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2962
2963         if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2964                 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2965                 rc = 0;
2966                 goto out;
2967         }
2968
2969         if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2970                 rc = 0;
2971                 goto out;
2972         }
2973
2974         dev_info(hdev->dev,
2975                 "Loading firmware to device, may take some time...\n");
2976
2977         rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2978         if (rc)
2979                 goto out;
2980
2981         if (fw_loader->skip_bmc) {
2982                 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2983
2984                 rc = hl_poll_timeout(
2985                         hdev,
2986                         cpu_boot_status_reg,
2987                         status,
2988                         (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2989                         hdev->fw_poll_interval_usec,
2990                         cpu_timeout);
2991
2992                 if (rc) {
2993                         dev_err(hdev->dev,
2994                                 "Failed to get ACK on skipping BMC (status = %d)\n",
2995                                 status);
2996                         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2997                         rc = -EIO;
2998                         goto out;
2999                 }
3000         }
3001
3002         WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
3003
3004         rc = hl_poll_timeout(
3005                 hdev,
3006                 cpu_boot_status_reg,
3007                 status,
3008                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
3009                 hdev->fw_poll_interval_usec,
3010                 cpu_timeout);
3011
3012         /* Clear message */
3013         WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3014
3015         if (rc) {
3016                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
3017                         dev_err(hdev->dev,
3018                                 "Device reports FIT image is corrupted\n");
3019                 else
3020                         dev_err(hdev->dev,
3021                                 "Failed to load firmware to device (status = %d)\n",
3022                                 status);
3023
3024                 rc = -EIO;
3025                 goto out;
3026         }
3027
3028         rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3029                                         fw_loader->static_loader.boot_err1_reg,
3030                                         cpu_boot_dev_status0_reg,
3031                                         cpu_boot_dev_status1_reg);
3032         if (rc)
3033                 return rc;
3034
3035         hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
3036                                                 cpu_boot_dev_status1_reg);
3037
3038         return 0;
3039
3040 out:
3041         fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3042                                         fw_loader->static_loader.boot_err1_reg,
3043                                         cpu_boot_dev_status0_reg,
3044                                         cpu_boot_dev_status1_reg);
3045
3046         return rc;
3047 }
3048
3049 /**
3050  * hl_fw_init_cpu - initialize the device CPU
3051  *
3052  * @hdev: pointer to the habanalabs device structure
3053  *
3054  * @return 0 on success, otherwise non-zero error code
3055  *
3056  * perform necessary initializations for device's CPU. takes into account if
3057  * init protocol is static or dynamic.
3058  */
3059 int hl_fw_init_cpu(struct hl_device *hdev)
3060 {
3061         struct asic_fixed_properties *prop = &hdev->asic_prop;
3062         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3063
3064         return  prop->dynamic_fw_load ?
3065                         hl_fw_dynamic_init_cpu(hdev, fw_loader) :
3066                         hl_fw_static_init_cpu(hdev, fw_loader);
3067 }
3068
3069 void hl_fw_set_pll_profile(struct hl_device *hdev)
3070 {
3071         hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
3072                                 hdev->asic_prop.max_freq_value);
3073 }
3074
3075 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
3076 {
3077         long value;
3078
3079         if (!hl_device_operational(hdev, NULL))
3080                 return -ENODEV;
3081
3082         if (!hdev->pdev) {
3083                 *cur_clk = 0;
3084                 *max_clk = 0;
3085                 return 0;
3086         }
3087
3088         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
3089
3090         if (value < 0) {
3091                 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
3092                 return value;
3093         }
3094
3095         *max_clk = (value / 1000 / 1000);
3096
3097         value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
3098
3099         if (value < 0) {
3100                 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
3101                 return value;
3102         }
3103
3104         *cur_clk = (value / 1000 / 1000);
3105
3106         return 0;
3107 }
3108
3109 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
3110 {
3111         struct cpucp_packet pkt;
3112         u32 used_pll_idx;
3113         u64 result;
3114         int rc;
3115
3116         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3117         if (rc)
3118                 return rc;
3119
3120         memset(&pkt, 0, sizeof(pkt));
3121
3122         if (curr)
3123                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
3124                                                 CPUCP_PKT_CTL_OPCODE_SHIFT);
3125         else
3126                 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3127
3128         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3129
3130         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3131
3132         if (rc) {
3133                 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
3134                         used_pll_idx, rc);
3135                 return rc;
3136         }
3137
3138         return (long) result;
3139 }
3140
3141 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
3142 {
3143         struct cpucp_packet pkt;
3144         u32 used_pll_idx;
3145         int rc;
3146
3147         rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3148         if (rc)
3149                 return;
3150
3151         memset(&pkt, 0, sizeof(pkt));
3152
3153         pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3154         pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3155         pkt.value = cpu_to_le64(freq);
3156
3157         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3158
3159         if (rc)
3160                 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
3161                         used_pll_idx, rc);
3162 }
3163
3164 long hl_fw_get_max_power(struct hl_device *hdev)
3165 {
3166         struct cpucp_packet pkt;
3167         u64 result;
3168         int rc;
3169
3170         memset(&pkt, 0, sizeof(pkt));
3171
3172         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3173
3174         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3175
3176         if (rc) {
3177                 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
3178                 return rc;
3179         }
3180
3181         return result;
3182 }
3183
3184 void hl_fw_set_max_power(struct hl_device *hdev)
3185 {
3186         struct cpucp_packet pkt;
3187         int rc;
3188
3189         /* TODO: remove this after simulator supports this packet */
3190         if (!hdev->pdev)
3191                 return;
3192
3193         memset(&pkt, 0, sizeof(pkt));
3194
3195         pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3196         pkt.value = cpu_to_le64(hdev->max_power);
3197
3198         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3199
3200         if (rc)
3201                 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
3202 }
3203
3204 static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size,
3205                                         u32 nonce, u32 timeout)
3206 {
3207         struct cpucp_packet pkt = {};
3208         dma_addr_t req_dma_addr;
3209         void *req_cpu_addr;
3210         int rc;
3211
3212         req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr);
3213         if (!req_cpu_addr) {
3214                 dev_err(hdev->dev,
3215                         "Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id);
3216                 return -ENOMEM;
3217         }
3218
3219         memset(data, 0, size);
3220
3221         pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT);
3222         pkt.addr = cpu_to_le64(req_dma_addr);
3223         pkt.data_max_size = cpu_to_le32(size);
3224         pkt.nonce = cpu_to_le32(nonce);
3225
3226         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
3227                                         timeout, NULL);
3228         if (rc) {
3229                 dev_err(hdev->dev,
3230                         "Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc);
3231                 goto out;
3232         }
3233
3234         memcpy(data, req_cpu_addr, size);
3235
3236 out:
3237         hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr);
3238
3239         return rc;
3240 }
3241
3242 int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,
3243                                 u32 nonce)
3244 {
3245         return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info,
3246                                         sizeof(struct cpucp_sec_attest_info), nonce,
3247                                         HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);
3248 }
3249
3250 int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
3251                                                 dma_addr_t buff, u32 *size)
3252 {
3253         struct cpucp_packet pkt = {};
3254         u64 result;
3255         int rc = 0;
3256
3257         pkt.ctl = cpu_to_le32(CPUCP_PACKET_GENERIC_PASSTHROUGH << CPUCP_PKT_CTL_OPCODE_SHIFT);
3258         pkt.addr = cpu_to_le64(buff);
3259         pkt.data_max_size = cpu_to_le32(*size);
3260         pkt.pkt_subidx = cpu_to_le32(sub_opcode);
3261
3262         rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)&pkt, sizeof(pkt),
3263                                                 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
3264         if (rc)
3265                 dev_err(hdev->dev, "failed to send CPUCP data of generic fw pkt\n");
3266         else
3267                 dev_dbg(hdev->dev, "generic pkt was successful, result: 0x%llx\n", result);
3268
3269         *size = (u32)result;
3270
3271         return rc;
3272 }