drivers/accel/habanalabs/gaudi2/gaudi2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2020-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "gaudi2P.h"
   9 #include "gaudi2_masks.h"
  10 #include "../include/gaudi2/gaudi2_special_blocks.h"
  11 #include "../include/hw_ip/mmu/mmu_general.h"
  12 #include "../include/hw_ip/mmu/mmu_v2_0.h"
  13 #include "../include/gaudi2/gaudi2_packets.h"
  14 #include "../include/gaudi2/gaudi2_reg_map.h"
  15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h"
  16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h"
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/hwmon.h>
  21 #include <linux/iommu.h>
  22
  23 #define GAUDI2_DMA_POOL_BLK_SIZE                SZ_256          /* 256 bytes */
  24
  25 #define GAUDI2_RESET_TIMEOUT_MSEC               2000            /* 2000ms */
  26
  27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC          500000          /* 500ms */
  28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC         25000           /* 25s */
  29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC         25000           /* 25s */
  30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC     3000000         /* 3s */
  31 #define GAUDI2_RESET_POLL_CNT                   3
  32 #define GAUDI2_RESET_WAIT_MSEC                  1               /* 1ms */
  33 #define GAUDI2_CPU_RESET_WAIT_MSEC              100             /* 100ms */
  34 #define GAUDI2_PLDM_RESET_WAIT_MSEC             1000            /* 1s */
  35 #define GAUDI2_CB_POOL_CB_CNT                   512
  36 #define GAUDI2_CB_POOL_CB_SIZE                  SZ_128K         /* 128KB */
  37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC          4000000         /* 4s */
  38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC         25000000        /* 25s */
  39 #define GAUDI2_TEST_QUEUE_WAIT_USEC             100000          /* 100ms */
  40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC        1000000         /* 1s */
  41
  42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT          3
  43
  44 /*
  45  * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
  46  * and the code relies on that value (for array size etc..) we define another value
  47  * for MAX faulty TPCs which reflects the cluster binning requirements
  48  */
  49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS         1
  50 #define MAX_FAULTY_XBARS                        1
  51 #define MAX_FAULTY_EDMAS                        1
  52 #define MAX_FAULTY_DECODERS                     1
  53
  54 #define GAUDI2_TPC_FULL_MASK                    0x1FFFFFF
  55 #define GAUDI2_HIF_HMMU_FULL_MASK               0xFFFF
  56 #define GAUDI2_DECODER_FULL_MASK                0x3FF
  57
  58 #define GAUDI2_NA_EVENT_CAUSE                   0xFF
  59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE              18
  60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE        25
  61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE          3
  62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE         14
  63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE         3
  64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE          2
  65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE             22
  66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE            31
  67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE             25
  68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE             16
  69 #define GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE        5
  70 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE         7
  71 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE       8
  72 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE         19
  73 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE             9
  74 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE          3
  75 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE   3
  76 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE      2
  77 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE       2
  78 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE       2
  79 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE          5
  80
  81 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC       (MMU_CONFIG_TIMEOUT_USEC * 10)
  82 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC            (MMU_CONFIG_TIMEOUT_USEC * 200)
  83 #define GAUDI2_ARB_WDT_TIMEOUT                  (0x1000000)
  84
  85 #define GAUDI2_VDEC_TIMEOUT_USEC                10000           /* 10ms */
  86 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC           (GAUDI2_VDEC_TIMEOUT_USEC * 100)
  87
  88 #define KDMA_TIMEOUT_USEC                       USEC_PER_SEC
  89
  90 #define IS_DMA_IDLE(dma_core_sts0)      \
  91         (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK)))
  92
  93 #define IS_DMA_HALTED(dma_core_sts1)    \
  94         ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK))
  95
  96 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
  97
  98 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK))
  99
 100 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \
 101         ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \
 102         (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \
 103         (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK)))
 104
 105 #define PCIE_DEC_EN_MASK                        0x300
 106 #define DEC_WORK_STATE_IDLE                     0
 107 #define DEC_WORK_STATE_PEND                     3
 108 #define IS_DEC_IDLE(dec_swreg15) \
 109         (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \
 110         ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) ==  DEC_WORK_STATE_PEND)
 111
 112 /* HBM MMU address scrambling parameters */
 113 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE            SZ_8M
 114 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT           26
 115 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT           0
 116 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK        DRAM_VA_HINT_MASK
 117 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR  16
 118 #define MMU_RANGE_INV_VA_LSB_SHIFT              12
 119 #define MMU_RANGE_INV_VA_MSB_SHIFT              44
 120 #define MMU_RANGE_INV_EN_SHIFT                  0
 121 #define MMU_RANGE_INV_ASID_EN_SHIFT             1
 122 #define MMU_RANGE_INV_ASID_SHIFT                2
 123
 124 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has
 125  * a 2 entries FIFO, and hence it is not enabled for it.
 126  */
 127 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK         GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
 128 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK         GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
 129
 130 #define GAUDI2_MAX_STRING_LEN                   64
 131
 132 #define GAUDI2_VDEC_MSIX_ENTRIES                (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
 133                                                         GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
 134
 135 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
 136
 137 /* RAZWI initiator coordinates */
 138 #define RAZWI_GET_AXUSER_XY(x) \
 139         ((x & 0xF8001FF0) >> 4)
 140
 141 #define RAZWI_GET_AXUSER_LOW_XY(x) \
 142         ((x & 0x00001FF0) >> 4)
 143
 144 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT         0
 145 #define RAZWI_INITIATOR_AXUER_L_X_MASK          0x1F
 146 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT         5
 147 #define RAZWI_INITIATOR_AXUER_L_Y_MASK          0xF
 148
 149 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT         23
 150 #define RAZWI_INITIATOR_AXUER_H_X_MASK          0x1F
 151
 152 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \
 153         ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \
 154                 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT))
 155
 156 #define RAZWI_INITIATOR_ID_X_HIGH(x) \
 157                 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT)
 158
 159 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \
 160         (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh))
 161
 162 #define PSOC_RAZWI_ENG_STR_SIZE 128
 163 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5
 164
 165 /* HW scrambles only bits 0-25 */
 166 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26)
 167
 168 struct gaudi2_razwi_info {
 169         u32 axuser_xy;
 170         u32 rtr_ctrl;
 171         u16 eng_id;
 172         char *eng_name;
 173 };
 174
 175 static struct gaudi2_razwi_info common_razwi_info[] = {
 176                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE,
 177                                 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"},
 178                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
 179                                 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"},
 180                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE,
 181                                 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"},
 182                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
 183                                 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"},
 184                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE,
 185                                 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"},
 186                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
 187                                 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"},
 188                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE,
 189                                 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"},
 190                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
 191                                 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"},
 192                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE,
 193                                 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"},
 194                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE,
 195                                 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"},
 196                 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE,
 197                                 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"},
 198                 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE,
 199                                 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"},
 200                 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE,
 201                                 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"},
 202                 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE,
 203                                 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"},
 204                 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE,
 205                                 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"},
 206                 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE,
 207                                 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"},
 208                 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE,
 209                                 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"},
 210                 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE,
 211                                 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"},
 212                 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE,
 213                                 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"},
 214                 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE,
 215                                 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"},
 216                 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE,
 217                                 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"},
 218                 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE,
 219                                 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"},
 220                 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE,
 221                                 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"},
 222                 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE,
 223                                 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"},
 224                 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE,
 225                                 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"},
 226                 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE,
 227                                 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"},
 228                 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE,
 229                                 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"},
 230                 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE,
 231                                 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"},
 232                 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE,
 233                                 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"},
 234                 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE,
 235                                 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"},
 236                 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE,
 237                                 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"},
 238                 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE,
 239                                 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"},
 240                 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE,
 241                                 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"},
 242                 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE,
 243                                 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"},
 244                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
 245                                 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"},
 246                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE,
 247                                 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"},
 248                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE,
 249                                 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"},
 250                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE,
 251                                 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"},
 252                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
 253                                 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"},
 254                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE,
 255                                 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"},
 256                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
 257                                 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"},
 258                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
 259                                 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"},
 260                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE,
 261                                 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"},
 262                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE,
 263                                 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"},
 264                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE,
 265                                 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"},
 266                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
 267                                 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"},
 268                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
 269                                 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"},
 270                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
 271                                 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"},
 272                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE,
 273                                 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"},
 274                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
 275                                 GAUDI2_ENGINE_ID_SIZE, "PMMU"},
 276                 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE,
 277                                 GAUDI2_ENGINE_ID_SIZE, "PCIE"},
 278                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE,
 279                                 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"},
 280                 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE,
 281                                 GAUDI2_ENGINE_ID_KDMA, "KDMA"},
 282                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE,
 283                                 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"},
 284                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE,
 285                                 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"},
 286                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE,
 287                                 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"},
 288                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE,
 289                                 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"},
 290                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
 291                                 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"},
 292                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
 293                                 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"},
 294                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
 295                                 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"},
 296                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
 297                                 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"},
 298                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
 299                                 GAUDI2_ENGINE_ID_SIZE, "HMMU0"},
 300                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
 301                                 GAUDI2_ENGINE_ID_SIZE, "HMMU1"},
 302                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
 303                                 GAUDI2_ENGINE_ID_SIZE, "HMMU2"},
 304                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
 305                                 GAUDI2_ENGINE_ID_SIZE, "HMMU3"},
 306                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
 307                                 GAUDI2_ENGINE_ID_SIZE, "HMMU4"},
 308                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
 309                                 GAUDI2_ENGINE_ID_SIZE, "HMMU5"},
 310                 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
 311                                 GAUDI2_ENGINE_ID_SIZE, "HMMU6"},
 312                 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
 313                                 GAUDI2_ENGINE_ID_SIZE, "HMMU7"},
 314                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
 315                                 GAUDI2_ENGINE_ID_SIZE, "HMMU8"},
 316                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
 317                                 GAUDI2_ENGINE_ID_SIZE, "HMMU9"},
 318                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
 319                                 GAUDI2_ENGINE_ID_SIZE, "HMMU10"},
 320                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
 321                                 GAUDI2_ENGINE_ID_SIZE, "HMMU11"},
 322                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
 323                                 GAUDI2_ENGINE_ID_SIZE, "HMMU12"},
 324                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
 325                                 GAUDI2_ENGINE_ID_SIZE, "HMMU13"},
 326                 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
 327                                 GAUDI2_ENGINE_ID_SIZE, "HMMU14"},
 328                 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
 329                                 GAUDI2_ENGINE_ID_SIZE, "HMMU15"},
 330                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
 331                                 GAUDI2_ENGINE_ID_ROT_0, "ROT0"},
 332                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
 333                                 GAUDI2_ENGINE_ID_ROT_1, "ROT1"},
 334                 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
 335                                 GAUDI2_ENGINE_ID_PSOC, "CPU"},
 336                 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE,
 337                                 GAUDI2_ENGINE_ID_PSOC, "PSOC"}
 338 };
 339
 340 static struct gaudi2_razwi_info mme_razwi_info[] = {
 341                 /* MME X high coordinate is N/A, hence using only low coordinates */
 342                 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
 343                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"},
 344                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
 345                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"},
 346                 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
 347                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"},
 348                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
 349                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"},
 350                 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
 351                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"},
 352                 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
 353                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"},
 354                 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
 355                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"},
 356                 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
 357                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"},
 358                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
 359                                 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"},
 360                 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
 361                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"},
 362                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
 363                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"},
 364                 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
 365                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"},
 366                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
 367                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"},
 368                 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
 369                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"},
 370                 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
 371                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"},
 372                 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
 373                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"},
 374                 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
 375                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"},
 376                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
 377                                 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"},
 378                 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
 379                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"},
 380                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
 381                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"},
 382                 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
 383                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"},
 384                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
 385                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"},
 386                 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
 387                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"},
 388                 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
 389                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"},
 390                 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
 391                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"},
 392                 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
 393                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"},
 394                 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
 395                                 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"},
 396                 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
 397                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"},
 398                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
 399                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"},
 400                 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
 401                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"},
 402                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
 403                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"},
 404                 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
 405                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"},
 406                 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
 407                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"},
 408                 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
 409                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"},
 410                 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
 411                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"},
 412                 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
 413                                 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"}
 414 };
 415
 416 enum hl_pmmu_fatal_cause {
 417         LATENCY_RD_OUT_FIFO_OVERRUN,
 418         LATENCY_WR_OUT_FIFO_OVERRUN,
 419 };
 420
 421 enum hl_pcie_drain_ind_cause {
 422         LBW_AXI_DRAIN_IND,
 423         HBW_AXI_DRAIN_IND
 424 };
 425
 426 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = {
 427         [HBM_ID0] = 0xFFFC,
 428         [HBM_ID1] = 0xFFCF,
 429         [HBM_ID2] = 0xF7F7,
 430         [HBM_ID3] = 0x7F7F,
 431         [HBM_ID4] = 0xFCFF,
 432         [HBM_ID5] = 0xCFFF,
 433 };
 434
 435 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = {
 436         [0] = HBM_ID0,
 437         [1] = HBM_ID1,
 438         [2] = HBM_ID4,
 439         [3] = HBM_ID5,
 440 };
 441
 442 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = {
 443         [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0,
 444         [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2,
 445         [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1,
 446         [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3,
 447         [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2,
 448         [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4,
 449         [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3,
 450         [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5,
 451 };
 452
 453 static const int gaudi2_qman_async_event_id[] = {
 454         [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM,
 455         [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM,
 456         [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM,
 457         [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM,
 458         [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM,
 459         [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM,
 460         [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM,
 461         [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM,
 462         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM,
 463         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM,
 464         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM,
 465         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM,
 466         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM,
 467         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM,
 468         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM,
 469         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM,
 470         [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM,
 471         [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM,
 472         [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM,
 473         [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM,
 474         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM,
 475         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM,
 476         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM,
 477         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM,
 478         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM,
 479         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM,
 480         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM,
 481         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM,
 482         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM,
 483         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM,
 484         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM,
 485         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM,
 486         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM,
 487         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM,
 488         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM,
 489         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM,
 490         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM,
 491         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM,
 492         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM,
 493         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM,
 494         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM,
 495         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM,
 496         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM,
 497         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM,
 498         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM,
 499         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM,
 500         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM,
 501         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM,
 502         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM,
 503         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM,
 504         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM,
 505         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM,
 506         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM,
 507         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM,
 508         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM,
 509         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM,
 510         [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM,
 511         [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM,
 512         [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM,
 513         [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM,
 514         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM,
 515         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM,
 516         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM,
 517         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM,
 518         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM,
 519         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM,
 520         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM,
 521         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM,
 522         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM,
 523         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM,
 524         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM,
 525         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM,
 526         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM,
 527         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM,
 528         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM,
 529         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM,
 530         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM,
 531         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM,
 532         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM,
 533         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM,
 534         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM,
 535         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM,
 536         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM,
 537         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM,
 538         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM,
 539         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM,
 540         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM,
 541         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM,
 542         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM,
 543         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM,
 544         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM,
 545         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM,
 546         [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM,
 547         [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM,
 548         [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM,
 549         [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM,
 550         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM,
 551         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM,
 552         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM,
 553         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM,
 554         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM,
 555         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM,
 556         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM,
 557         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM,
 558         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM,
 559         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM,
 560         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM,
 561         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM,
 562         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM,
 563         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM,
 564         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM,
 565         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM,
 566         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM,
 567         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM,
 568         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM,
 569         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM,
 570         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM,
 571         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM,
 572         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM,
 573         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM,
 574         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM,
 575         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM,
 576         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM,
 577         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM,
 578         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM,
 579         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM,
 580         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM,
 581         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM,
 582         [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM,
 583         [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM,
 584         [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM,
 585         [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM,
 586         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM,
 587         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM,
 588         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM,
 589         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM,
 590         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM,
 591         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM,
 592         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM,
 593         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM,
 594         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM,
 595         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM,
 596         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM,
 597         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM,
 598         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM,
 599         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM,
 600         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM,
 601         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM,
 602         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM,
 603         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM,
 604         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM,
 605         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM,
 606         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM,
 607         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM,
 608         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM,
 609         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM,
 610         [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0,
 611         [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0,
 612         [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0,
 613         [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0,
 614         [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1,
 615         [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1,
 616         [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1,
 617         [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1,
 618         [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0,
 619         [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0,
 620         [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0,
 621         [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0,
 622         [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1,
 623         [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1,
 624         [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1,
 625         [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1,
 626         [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0,
 627         [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0,
 628         [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0,
 629         [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0,
 630         [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1,
 631         [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1,
 632         [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1,
 633         [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1,
 634         [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0,
 635         [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0,
 636         [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0,
 637         [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0,
 638         [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1,
 639         [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1,
 640         [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1,
 641         [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1,
 642         [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0,
 643         [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0,
 644         [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0,
 645         [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0,
 646         [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1,
 647         [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1,
 648         [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1,
 649         [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1,
 650         [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0,
 651         [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0,
 652         [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0,
 653         [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0,
 654         [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1,
 655         [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1,
 656         [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1,
 657         [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1,
 658         [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0,
 659         [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0,
 660         [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0,
 661         [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0,
 662         [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1,
 663         [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1,
 664         [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1,
 665         [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1,
 666         [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0,
 667         [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0,
 668         [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0,
 669         [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0,
 670         [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1,
 671         [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1,
 672         [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1,
 673         [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1,
 674         [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0,
 675         [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0,
 676         [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0,
 677         [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0,
 678         [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1,
 679         [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1,
 680         [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1,
 681         [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1,
 682         [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0,
 683         [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0,
 684         [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0,
 685         [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0,
 686         [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1,
 687         [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1,
 688         [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1,
 689         [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1,
 690         [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0,
 691         [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0,
 692         [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0,
 693         [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0,
 694         [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1,
 695         [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1,
 696         [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1,
 697         [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1,
 698         [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0,
 699         [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0,
 700         [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0,
 701         [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0,
 702         [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1,
 703         [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1,
 704         [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1,
 705         [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1,
 706         [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
 707         [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
 708         [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
 709         [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
 710         [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
 711         [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
 712         [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
 713         [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM
 714 };
 715
 716 static const int gaudi2_dma_core_async_event_id[] = {
 717         [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE,
 718         [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE,
 719         [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE,
 720         [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE,
 721         [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE,
 722         [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE,
 723         [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE,
 724         [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE,
 725         [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE,
 726         [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE,
 727         [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE,
 728 };
 729
 730 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = {
 731         "qman sei intr",
 732         "arc sei intr"
 733 };
 734
 735 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = {
 736         "AXI_TERMINATOR WR",
 737         "AXI_TERMINATOR RD",
 738         "AXI SPLIT SEI Status"
 739 };
 740
 741 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = {
 742         "cbu_bresp_sei_intr_cause",
 743         "cbu_rresp_sei_intr_cause",
 744         "lbu_bresp_sei_intr_cause",
 745         "lbu_rresp_sei_intr_cause",
 746         "cbu_axi_split_intr_cause",
 747         "lbu_axi_split_intr_cause",
 748         "arc_ip_excptn_sei_intr_cause",
 749         "dmi_bresp_sei_intr_cause",
 750         "aux2apb_err_sei_intr_cause",
 751         "cfg_lbw_wr_terminated_intr_cause",
 752         "cfg_lbw_rd_terminated_intr_cause",
 753         "cfg_dccm_wr_terminated_intr_cause",
 754         "cfg_dccm_rd_terminated_intr_cause",
 755         "cfg_hbw_rd_terminated_intr_cause"
 756 };
 757
 758 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = {
 759         "msix_vcd_hbw_sei",
 760         "msix_l2c_hbw_sei",
 761         "msix_nrm_hbw_sei",
 762         "msix_abnrm_hbw_sei",
 763         "msix_vcd_lbw_sei",
 764         "msix_l2c_lbw_sei",
 765         "msix_nrm_lbw_sei",
 766         "msix_abnrm_lbw_sei",
 767         "apb_vcd_lbw_sei",
 768         "apb_l2c_lbw_sei",
 769         "apb_nrm_lbw_sei",
 770         "apb_abnrm_lbw_sei",
 771         "dec_sei",
 772         "dec_apb_sei",
 773         "trc_apb_sei",
 774         "lbw_mstr_if_sei",
 775         "axi_split_bresp_err_sei",
 776         "hbw_axi_wr_viol_sei",
 777         "hbw_axi_rd_viol_sei",
 778         "lbw_axi_wr_viol_sei",
 779         "lbw_axi_rd_viol_sei",
 780         "vcd_spi",
 781         "l2c_spi",
 782         "nrm_spi",
 783         "abnrm_spi",
 784 };
 785
 786 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = {
 787         "PQ AXI HBW error",
 788         "CQ AXI HBW error",
 789         "CP AXI HBW error",
 790         "CP error due to undefined OPCODE",
 791         "CP encountered STOP OPCODE",
 792         "CP AXI LBW error",
 793         "CP WRREG32 or WRBULK returned error",
 794         "N/A",
 795         "FENCE 0 inc over max value and clipped",
 796         "FENCE 1 inc over max value and clipped",
 797         "FENCE 2 inc over max value and clipped",
 798         "FENCE 3 inc over max value and clipped",
 799         "FENCE 0 dec under min value and clipped",
 800         "FENCE 1 dec under min value and clipped",
 801         "FENCE 2 dec under min value and clipped",
 802         "FENCE 3 dec under min value and clipped",
 803         "CPDMA Up overflow",
 804         "PQC L2H error"
 805 };
 806
 807 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = {
 808         "RSVD0",
 809         "CQ AXI HBW error",
 810         "CP AXI HBW error",
 811         "CP error due to undefined OPCODE",
 812         "CP encountered STOP OPCODE",
 813         "CP AXI LBW error",
 814         "CP WRREG32 or WRBULK returned error",
 815         "N/A",
 816         "FENCE 0 inc over max value and clipped",
 817         "FENCE 1 inc over max value and clipped",
 818         "FENCE 2 inc over max value and clipped",
 819         "FENCE 3 inc over max value and clipped",
 820         "FENCE 0 dec under min value and clipped",
 821         "FENCE 1 dec under min value and clipped",
 822         "FENCE 2 dec under min value and clipped",
 823         "FENCE 3 dec under min value and clipped",
 824         "CPDMA Up overflow",
 825         "RSVD17",
 826         "CQ_WR_IFIFO_CI_ERR",
 827         "CQ_WR_CTL_CI_ERR",
 828         "ARC_CQF_RD_ERR",
 829         "ARC_CQ_WR_IFIFO_CI_ERR",
 830         "ARC_CQ_WR_CTL_CI_ERR",
 831         "ARC_AXI_ERR",
 832         "CP_SWITCH_WDT_ERR"
 833 };
 834
 835 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = {
 836         "Choice push while full error",
 837         "Choice Q watchdog error",
 838         "MSG AXI LBW returned with error"
 839 };
 840
 841 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = {
 842         "qm_axi_err",
 843         "qm_trace_fence_events",
 844         "qm_sw_err",
 845         "qm_cp_sw_stop",
 846         "lbw_mstr_rresp_err",
 847         "lbw_mstr_bresp_err",
 848         "lbw_msg_slverr",
 849         "hbw_msg_slverr",
 850         "wbc_slverr",
 851         "hbw_mstr_rresp_err",
 852         "hbw_mstr_bresp_err",
 853         "sb_resp_intr",
 854         "mrsb_resp_intr",
 855         "core_dw_status_0",
 856         "core_dw_status_1",
 857         "core_dw_status_2",
 858         "core_dw_status_3",
 859         "core_dw_status_4",
 860         "core_dw_status_5",
 861         "core_dw_status_6",
 862         "core_dw_status_7",
 863         "async_arc2cpu_sei_intr",
 864 };
 865
 866 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = {
 867         "tpc_address_exceed_slm",
 868         "tpc_div_by_0",
 869         "tpc_spu_mac_overflow",
 870         "tpc_spu_addsub_overflow",
 871         "tpc_spu_abs_overflow",
 872         "tpc_spu_fma_fp_dst_nan",
 873         "tpc_spu_fma_fp_dst_inf",
 874         "tpc_spu_convert_fp_dst_nan",
 875         "tpc_spu_convert_fp_dst_inf",
 876         "tpc_spu_fp_dst_denorm",
 877         "tpc_vpu_mac_overflow",
 878         "tpc_vpu_addsub_overflow",
 879         "tpc_vpu_abs_overflow",
 880         "tpc_vpu_convert_fp_dst_nan",
 881         "tpc_vpu_convert_fp_dst_inf",
 882         "tpc_vpu_fma_fp_dst_nan",
 883         "tpc_vpu_fma_fp_dst_inf",
 884         "tpc_vpu_fp_dst_denorm",
 885         "tpc_assertions",
 886         "tpc_illegal_instruction",
 887         "tpc_pc_wrap_around",
 888         "tpc_qm_sw_err",
 889         "tpc_hbw_rresp_err",
 890         "tpc_hbw_bresp_err",
 891         "tpc_lbw_rresp_err",
 892         "tpc_lbw_bresp_err",
 893         "st_unlock_already_locked",
 894         "invalid_lock_access",
 895         "LD_L protection violation",
 896         "ST_L protection violation",
 897         "D$ L0CS mismatch",
 898 };
 899
 900 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = {
 901         "agu_resp_intr",
 902         "qman_axi_err",
 903         "wap sei (wbc axi err)",
 904         "arc sei",
 905         "cfg access error",
 906         "qm_sw_err",
 907         "sbte_dbg_intr_0",
 908         "sbte_dbg_intr_1",
 909         "sbte_dbg_intr_2",
 910         "sbte_dbg_intr_3",
 911         "sbte_dbg_intr_4",
 912         "sbte_prtn_intr_0",
 913         "sbte_prtn_intr_1",
 914         "sbte_prtn_intr_2",
 915         "sbte_prtn_intr_3",
 916         "sbte_prtn_intr_4",
 917 };
 918
 919 static const char * const guadi2_mme_sbte_error_cause[GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE] = {
 920         "i0",
 921         "i1",
 922         "i2",
 923         "i3",
 924         "i4",
 925 };
 926
 927 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = {
 928         "WBC ERR RESP_0",
 929         "WBC ERR RESP_1",
 930         "AP SOURCE POS INF",
 931         "AP SOURCE NEG INF",
 932         "AP SOURCE NAN",
 933         "AP RESULT POS INF",
 934         "AP RESULT NEG INF",
 935 };
 936
 937 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
 938         "HBW Read returned with error RRESP",
 939         "HBW write returned with error BRESP",
 940         "LBW write returned with error BRESP",
 941         "descriptor_fifo_overflow",
 942         "KDMA SB LBW Read returned with error",
 943         "KDMA WBC LBW Write returned with error",
 944         "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
 945         "WRONG CFG FOR COMMIT IN LIN DMA"
 946 };
 947
 948 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
 949         "HBW/LBW Read returned with error RRESP",
 950         "HBW/LBW write returned with error BRESP",
 951         "LBW write returned with error BRESP",
 952         "descriptor_fifo_overflow",
 953         "KDMA SB LBW Read returned with error",
 954         "KDMA WBC LBW Write returned with error",
 955         "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
 956         "WRONG CFG FOR COMMIT IN LIN DMA"
 957 };
 958
 959 struct gaudi2_sm_sei_cause_data {
 960         const char *cause_name;
 961         const char *log_name;
 962 };
 963
 964 static const struct gaudi2_sm_sei_cause_data
 965 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = {
 966         {"calculated SO value overflow/underflow", "SOB ID"},
 967         {"payload address of monitor is not aligned to 4B", "monitor addr"},
 968         {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"},
 969 };
 970
 971 static const char * const
 972 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = {
 973         "LATENCY_RD_OUT_FIFO_OVERRUN",
 974         "LATENCY_WR_OUT_FIFO_OVERRUN",
 975 };
 976
 977 static const char * const
 978 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = {
 979         "LATENCY_RD_OUT_FIFO_OVERRUN",
 980         "LATENCY_WR_OUT_FIFO_OVERRUN",
 981 };
 982
 983 static const char * const
 984 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = {
 985         "AXI drain HBW",
 986         "AXI drain LBW",
 987 };
 988
 989 static const char * const
 990 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = {
 991         "HBW error response",
 992         "LBW error response",
 993         "TLP is blocked by RR"
 994 };
 995
 996 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = {
 997         [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE,
 998         [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE,
 999         [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE,
1000         [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE,
1001         [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE,
1002         [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE,
1003         [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE,
1004         [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE,
1005         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE,
1006         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE,
1007         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE,
1008         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE,
1009         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE,
1010         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE,
1011         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE,
1012         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE,
1013         [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE,
1014         [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE,
1015         [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE,
1016         [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE,
1017         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE,
1018         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE,
1019         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE,
1020         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE,
1021         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE,
1022         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE,
1023         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE,
1024         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE,
1025         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE,
1026         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE,
1027         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE,
1028         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE,
1029         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE,
1030         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE,
1031         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE,
1032         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE,
1033         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE,
1034         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE,
1035         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE,
1036         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE,
1037         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE,
1038         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE,
1039         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE,
1040         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE,
1041         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE,
1042         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE,
1043         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE,
1044         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE,
1045         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE,
1046         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE,
1047         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE,
1048         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE,
1049         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE,
1050         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE,
1051         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE,
1052         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE,
1053         [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE,
1054         [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE,
1055         [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE,
1056         [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE,
1057         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE,
1058         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE,
1059         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE,
1060         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE,
1061         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE,
1062         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE,
1063         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE,
1064         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE,
1065         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE,
1066         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE,
1067         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE,
1068         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE,
1069         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE,
1070         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE,
1071         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE,
1072         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE,
1073         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE,
1074         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE,
1075         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE,
1076         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE,
1077         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE,
1078         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE,
1079         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE,
1080         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE,
1081         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE,
1082         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE,
1083         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE,
1084         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE,
1085         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE,
1086         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE,
1087         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE,
1088         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE,
1089         [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE,
1090         [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE,
1091         [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE,
1092         [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE,
1093         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE,
1094         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE,
1095         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE,
1096         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE,
1097         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE,
1098         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE,
1099         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE,
1100         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE,
1101         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE,
1102         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE,
1103         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE,
1104         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE,
1105         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE,
1106         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE,
1107         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE,
1108         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE,
1109         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE,
1110         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE,
1111         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE,
1112         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE,
1113         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE,
1114         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE,
1115         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE,
1116         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE,
1117         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE,
1118         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE,
1119         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE,
1120         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE,
1121         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE,
1122         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE,
1123         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE,
1124         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE,
1125         [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE,
1126         [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE,
1127         [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE,
1128         [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE,
1129         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE,
1130         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE,
1131         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE,
1132         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE,
1133         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE,
1134         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE,
1135         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE,
1136         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE,
1137         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE,
1138         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE,
1139         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE,
1140         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE,
1141         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE,
1142         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE,
1143         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE,
1144         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE,
1145         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE,
1146         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE,
1147         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE,
1148         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE,
1149         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE,
1150         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE,
1151         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE,
1152         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE,
1153         [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE,
1154         [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE,
1155         [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE,
1156         [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE,
1157         [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE,
1158         [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE,
1159         [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE,
1160         [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE,
1161         [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE,
1162         [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE,
1163         [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE,
1164         [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE,
1165         [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE,
1166         [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE,
1167         [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE,
1168         [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE,
1169         [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE,
1170         [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE,
1171         [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE,
1172         [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE,
1173         [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE,
1174         [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE,
1175         [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE,
1176         [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE,
1177         [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE,
1178         [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE,
1179         [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE,
1180         [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE,
1181         [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE,
1182         [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE,
1183         [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE,
1184         [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE,
1185         [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE,
1186         [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE,
1187         [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE,
1188         [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE,
1189         [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE,
1190         [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE,
1191         [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE,
1192         [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE,
1193         [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE,
1194         [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE,
1195         [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE,
1196         [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE,
1197         [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE,
1198         [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE,
1199         [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE,
1200         [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE,
1201         [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE,
1202         [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE,
1203         [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE,
1204         [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE,
1205         [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE,
1206         [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE,
1207         [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE,
1208         [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE,
1209         [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE,
1210         [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE,
1211         [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE,
1212         [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE,
1213         [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE,
1214         [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE,
1215         [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE,
1216         [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE,
1217         [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE,
1218         [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE,
1219         [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE,
1220         [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE,
1221         [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE,
1222         [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE,
1223         [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE,
1224         [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE,
1225         [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE,
1226         [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE,
1227         [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE,
1228         [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE,
1229         [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE,
1230         [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE,
1231         [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE,
1232         [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE,
1233         [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE,
1234         [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE,
1235         [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE,
1236         [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE,
1237         [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE,
1238         [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE,
1239         [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE,
1240         [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE,
1241         [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE,
1242         [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE,
1243         [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE,
1244         [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE,
1245         [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE,
1246         [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE,
1247         [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE,
1248         [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE,
1249         [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE,
1250         [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE,
1251         [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE,
1252         [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE,
1253         [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE,
1254         [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE,
1255         [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE,
1256         [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE
1257 };
1258
1259 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = {
1260         [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE,
1261         [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE,
1262         [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE,
1263         [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE,
1264         [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE,
1265         [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE,
1266         [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE,
1267         [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE,
1268         [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE,
1269         [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE,
1270         [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE,
1271         [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE,
1272         [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE,
1273         [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE,
1274         [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE,
1275         [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE,
1276         [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE,
1277         [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE,
1278         [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE,
1279         [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE,
1280         [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE,
1281         [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE,
1282         [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE,
1283         [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE,
1284         [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE,
1285         [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE,
1286         [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE,
1287         [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE,
1288         [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE,
1289         [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE,
1290         [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE,
1291         [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE,
1292         [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE,
1293         [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE,
1294         [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE,
1295         [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE,
1296         [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE,
1297         [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE,
1298         [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE,
1299         [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE,
1300         [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE,
1301         [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE,
1302         [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE,
1303         [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE,
1304         [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE,
1305         [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE,
1306         [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE,
1307         [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE,
1308         [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE,
1309         [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE,
1310         [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE,
1311         [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE,
1312         [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE,
1313         [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE,
1314         [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE,
1315         [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE,
1316         [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE,
1317         [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE,
1318         [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE,
1319         [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE,
1320         [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE,
1321         [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE,
1322         [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE,
1323         [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE,
1324         [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE,
1325         [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE,
1326         [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE,
1327         [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE,
1328         [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE,
1329 };
1330
1331 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = {
1332         [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE,
1333         [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE,
1334         [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE,
1335         [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE,
1336         [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE,
1337         [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE,
1338         [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE,
1339         [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE,
1340         [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE,
1341         [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE,
1342         [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE,
1343         [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE,
1344         [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE,
1345         [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE,
1346         [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE,
1347         [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE,
1348         [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE,
1349         [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE,
1350         [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE,
1351         [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE,
1352         [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE,
1353         [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE,
1354         [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE,
1355         [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE,
1356         [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE,
1357         [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE,
1358         [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE,
1359         [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE,
1360         [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE,
1361         [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE,
1362         [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE,
1363         [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE,
1364         [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE,
1365         [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE,
1366         [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE,
1367         [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE,
1368         [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE,
1369         [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE,
1370         [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE,
1371         [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE,
1372         [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE,
1373         [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE,
1374         [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE,
1375         [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE,
1376         [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE,
1377         [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE,
1378         [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE,
1379         [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE,
1380         [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE,
1381         [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE,
1382         [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE,
1383         [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE,
1384         [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE,
1385         [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE,
1386         [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE,
1387         [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE,
1388         [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE,
1389         [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE,
1390         [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE,
1391         [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE,
1392         [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE,
1393         [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE,
1394         [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE,
1395         [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE,
1396         [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE,
1397         [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE,
1398         [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE,
1399         [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE,
1400         [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE,
1401 };
1402
1403 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = {
1404         [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE,
1405         [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE,
1406         [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE,
1407         [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE,
1408 };
1409
1410 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = {
1411         [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0,
1412         [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0,
1413         [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0,
1414         [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0,
1415         [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1,
1416         [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1,
1417         [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1,
1418         [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1,
1419         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0,
1420         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0,
1421         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0,
1422         [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0,
1423         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1,
1424         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1,
1425         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1,
1426         [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1,
1427         [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0,
1428         [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0,
1429         [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0,
1430         [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0,
1431         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0,
1432         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0,
1433         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0,
1434         [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0,
1435         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1,
1436         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1,
1437         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1,
1438         [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1,
1439         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2,
1440         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2,
1441         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2,
1442         [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2,
1443         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3,
1444         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3,
1445         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3,
1446         [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3,
1447         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4,
1448         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4,
1449         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4,
1450         [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4,
1451         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5,
1452         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5,
1453         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5,
1454         [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5,
1455         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24,
1456         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24,
1457         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24,
1458         [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24,
1459         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2,
1460         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2,
1461         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2,
1462         [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2,
1463         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3,
1464         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3,
1465         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3,
1466         [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3,
1467         [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4,
1468         [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4,
1469         [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4,
1470         [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4,
1471         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6,
1472         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6,
1473         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6,
1474         [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6,
1475         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7,
1476         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7,
1477         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7,
1478         [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7,
1479         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8,
1480         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8,
1481         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8,
1482         [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8,
1483         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9,
1484         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9,
1485         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9,
1486         [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9,
1487         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10,
1488         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10,
1489         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10,
1490         [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10,
1491         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11,
1492         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11,
1493         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11,
1494         [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11,
1495         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4,
1496         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4,
1497         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4,
1498         [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4,
1499         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5,
1500         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5,
1501         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5,
1502         [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5,
1503         [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1,
1504         [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1,
1505         [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1,
1506         [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1,
1507         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12,
1508         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12,
1509         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12,
1510         [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12,
1511         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13,
1512         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13,
1513         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13,
1514         [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13,
1515         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14,
1516         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14,
1517         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14,
1518         [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14,
1519         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15,
1520         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15,
1521         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15,
1522         [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15,
1523         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16,
1524         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16,
1525         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16,
1526         [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16,
1527         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17,
1528         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17,
1529         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17,
1530         [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17,
1531         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6,
1532         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6,
1533         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6,
1534         [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6,
1535         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7,
1536         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7,
1537         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7,
1538         [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7,
1539         [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5,
1540         [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5,
1541         [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5,
1542         [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5,
1543         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18,
1544         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18,
1545         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18,
1546         [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18,
1547         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19,
1548         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19,
1549         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19,
1550         [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19,
1551         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20,
1552         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20,
1553         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20,
1554         [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20,
1555         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21,
1556         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21,
1557         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21,
1558         [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21,
1559         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22,
1560         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22,
1561         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22,
1562         [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22,
1563         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23,
1564         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23,
1565         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23,
1566         [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23,
1567         [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0,
1568         [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0,
1569         [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0,
1570         [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0,
1571         [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1,
1572         [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1,
1573         [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1,
1574         [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1,
1575         [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2,
1576         [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2,
1577         [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2,
1578         [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2,
1579         [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3,
1580         [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3,
1581         [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3,
1582         [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3,
1583         [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4,
1584         [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4,
1585         [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4,
1586         [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4,
1587         [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5,
1588         [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5,
1589         [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5,
1590         [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5,
1591         [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6,
1592         [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6,
1593         [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6,
1594         [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6,
1595         [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7,
1596         [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7,
1597         [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7,
1598         [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7,
1599         [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8,
1600         [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8,
1601         [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8,
1602         [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8,
1603         [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9,
1604         [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9,
1605         [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9,
1606         [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9,
1607         [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10,
1608         [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10,
1609         [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10,
1610         [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10,
1611         [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11,
1612         [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11,
1613         [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11,
1614         [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11,
1615         [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12,
1616         [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12,
1617         [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12,
1618         [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12,
1619         [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13,
1620         [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13,
1621         [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13,
1622         [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13,
1623         [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14,
1624         [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14,
1625         [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14,
1626         [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14,
1627         [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15,
1628         [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15,
1629         [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15,
1630         [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15,
1631         [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16,
1632         [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16,
1633         [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16,
1634         [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16,
1635         [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17,
1636         [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17,
1637         [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17,
1638         [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17,
1639         [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18,
1640         [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18,
1641         [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18,
1642         [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18,
1643         [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19,
1644         [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19,
1645         [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19,
1646         [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19,
1647         [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20,
1648         [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20,
1649         [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20,
1650         [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20,
1651         [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21,
1652         [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21,
1653         [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21,
1654         [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21,
1655         [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22,
1656         [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22,
1657         [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22,
1658         [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22,
1659         [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23,
1660         [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23,
1661         [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23,
1662         [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23,
1663         [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0,
1664         [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0,
1665         [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0,
1666         [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0,
1667         [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1,
1668         [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1,
1669         [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1,
1670         [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1
1671 };
1672
1673 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = {
1674         [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE,
1675         [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE,
1676         [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE,
1677         [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE,
1678         [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE,
1679         [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE,
1680         [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE,
1681         [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE,
1682         [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE,
1683         [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE,
1684         [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE
1685 };
1686
1687 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = {
1688         [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE,
1689         [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE,
1690         [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE,
1691         [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE
1692 };
1693
1694 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = {
1695         [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE,
1696         [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE,
1697         [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE,
1698         [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE,
1699         [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE,
1700         [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE,
1701         [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE,
1702         [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE,
1703         [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE,
1704         [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE,
1705         [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE,
1706         [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE,
1707         [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE,
1708         [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE,
1709         [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE,
1710         [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE,
1711         [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE,
1712         [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE,
1713         [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE,
1714         [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE,
1715         [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE,
1716         [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE,
1717         [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE,
1718         [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE,
1719         [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE,
1720 };
1721
1722 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = {
1723         [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE,
1724         [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE,
1725         [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE,
1726         [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE,
1727         [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE,
1728         [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE,
1729         [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE,
1730         [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE,
1731         [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE,
1732         [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE,
1733         [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE,
1734         [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE,
1735         [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE,
1736         [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE,
1737         [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE,
1738         [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE,
1739         [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE,
1740         [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE,
1741         [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE,
1742         [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE,
1743         [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE,
1744         [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE,
1745         [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE,
1746         [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE,
1747         [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE,
1748 };
1749
1750 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = {
1751         [ROTATOR_ID_0] = mmROT0_BASE,
1752         [ROTATOR_ID_1] = mmROT1_BASE
1753 };
1754
1755 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = {
1756         [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0,
1757         [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0,
1758         [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0,
1759         [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0,
1760         [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0,
1761         [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0,
1762         [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0,
1763         [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0,
1764         [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0,
1765         [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0,
1766         [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0,
1767         [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0,
1768         [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0,
1769         [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0,
1770         [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0,
1771         [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0,
1772         [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0,
1773         [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0,
1774         [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0,
1775         [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0,
1776         [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0,
1777         [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0,
1778         [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0,
1779         [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0,
1780         [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0,
1781 };
1782
1783 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = {
1784         [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0,
1785         [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0,
1786 };
1787
1788 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = {
1789         [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0,
1790         [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1,
1791         [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2,
1792         [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3,
1793         [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4,
1794         [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5,
1795         [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0,
1796         [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1,
1797         [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2,
1798         [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3,
1799         [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4,
1800         [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5,
1801         [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0,
1802         [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1,
1803         [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2,
1804         [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3,
1805         [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4,
1806         [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5,
1807         [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0,
1808         [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1,
1809         [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2,
1810         [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3,
1811         [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4,
1812         [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5,
1813         /* the PCI TPC is placed last (mapped liked HW) */
1814         [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6,
1815 };
1816
1817 static const u32 gaudi2_mme_engine_id_to_mme_id[] = {
1818         [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0,
1819         [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1,
1820         [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2,
1821         [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3,
1822 };
1823
1824 static const u32 gaudi2_edma_engine_id_to_edma_id[] = {
1825         [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0,
1826         [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1,
1827         [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0,
1828         [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1,
1829         [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2,
1830         [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3,
1831         [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4,
1832         [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5,
1833         [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6,
1834         [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7,
1835         [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA,
1836 };
1837
1838 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1839         GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
1840         GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0,
1841         GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
1842         GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0,
1843         GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
1844         GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0,
1845         GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0,
1846         GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0,
1847 };
1848
1849 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = {
1850         "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal",
1851         "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal",
1852         "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal",
1853         "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal",
1854         "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal",
1855         "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal",
1856         "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal",
1857         "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal",
1858         "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal",
1859         "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal"
1860 };
1861
1862 enum rtr_id {
1863         DCORE0_RTR0,
1864         DCORE0_RTR1,
1865         DCORE0_RTR2,
1866         DCORE0_RTR3,
1867         DCORE0_RTR4,
1868         DCORE0_RTR5,
1869         DCORE0_RTR6,
1870         DCORE0_RTR7,
1871         DCORE1_RTR0,
1872         DCORE1_RTR1,
1873         DCORE1_RTR2,
1874         DCORE1_RTR3,
1875         DCORE1_RTR4,
1876         DCORE1_RTR5,
1877         DCORE1_RTR6,
1878         DCORE1_RTR7,
1879         DCORE2_RTR0,
1880         DCORE2_RTR1,
1881         DCORE2_RTR2,
1882         DCORE2_RTR3,
1883         DCORE2_RTR4,
1884         DCORE2_RTR5,
1885         DCORE2_RTR6,
1886         DCORE2_RTR7,
1887         DCORE3_RTR0,
1888         DCORE3_RTR1,
1889         DCORE3_RTR2,
1890         DCORE3_RTR3,
1891         DCORE3_RTR4,
1892         DCORE3_RTR5,
1893         DCORE3_RTR6,
1894         DCORE3_RTR7,
1895 };
1896
1897 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
1898         DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3,
1899         DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4,
1900         DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1,
1901         DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6,
1902         DCORE0_RTR0
1903 };
1904
1905 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
1906         DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2,
1907         DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5,
1908         DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0,
1909         DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7,
1910         DCORE0_RTR0
1911 };
1912
1913 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = {
1914         DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0,
1915         DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0
1916 };
1917
1918 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = {
1919         DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1,
1920         DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0
1921 };
1922
1923 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = {
1924         DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0,
1925         DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7
1926 };
1927
1928 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = {
1929         DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0,
1930         DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7
1931 };
1932
1933 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1934         mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1935         mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1936         mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1937         mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1938         mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1939         mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1940         mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1941         mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE
1942 };
1943
1944 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = {
1945         DCORE0_RTR0, DCORE0_RTR0
1946 };
1947
1948 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = {
1949         DCORE0_RTR2, DCORE0_RTR2
1950 };
1951
1952 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = {
1953         DCORE2_RTR0, DCORE3_RTR7
1954 };
1955
1956 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = {
1957         DCORE2_RTR2, DCORE3_RTR5
1958 };
1959
1960 struct mme_initiators_rtr_id {
1961         u32 wap0;
1962         u32 wap1;
1963         u32 write;
1964         u32 read;
1965         u32 sbte0;
1966         u32 sbte1;
1967         u32 sbte2;
1968         u32 sbte3;
1969         u32 sbte4;
1970 };
1971
1972 enum mme_initiators {
1973         MME_WAP0 = 0,
1974         MME_WAP1,
1975         MME_WRITE,
1976         MME_READ,
1977         MME_SBTE0,
1978         MME_SBTE1,
1979         MME_SBTE2,
1980         MME_SBTE3,
1981         MME_SBTE4,
1982         MME_INITIATORS_MAX
1983 };
1984
1985 static const struct mme_initiators_rtr_id
1986 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = {
1987         { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7,
1988         .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6},
1989         { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8,
1990         .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8},
1991         { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23,
1992         .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23},
1993         { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30,
1994         .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28},
1995 };
1996
1997 enum razwi_event_sources {
1998         RAZWI_TPC,
1999         RAZWI_MME,
2000         RAZWI_EDMA,
2001         RAZWI_PDMA,
2002         RAZWI_NIC,
2003         RAZWI_DEC,
2004         RAZWI_ROT
2005 };
2006
2007 struct hbm_mc_error_causes {
2008         u32 mask;
2009         char cause[50];
2010 };
2011
2012 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS;
2013
2014 /* Special blocks iterator is currently used to configure security protection bits,
2015  * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2016  * must be skipped. Following configurations are commonly used for both PB config
2017  * and global error reading, since currently they both share the same settings.
2018  * Once it changes, we must remember to use separate configurations for either one.
2019  */
2020 static int gaudi2_iterator_skip_block_types[] = {
2021                 GAUDI2_BLOCK_TYPE_PLL,
2022                 GAUDI2_BLOCK_TYPE_EU_BIST,
2023                 GAUDI2_BLOCK_TYPE_HBM,
2024                 GAUDI2_BLOCK_TYPE_XFT
2025 };
2026
2027 static struct range gaudi2_iterator_skip_block_ranges[] = {
2028                 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */
2029                 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE},
2030                 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE},
2031                 /* Skip all CPU blocks except for CPU_IF */
2032                 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE},
2033                 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE}
2034 };
2035
2036 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = {
2037         {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"},
2038         {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2039         {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2040         {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"},
2041         {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"},
2042 };
2043
2044 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = {
2045         [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even",
2046         [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd",
2047         [HBM_SEI_READ_ERR] = "SEI read data error",
2048         [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error",
2049         [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted",
2050         [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail",
2051         [HBM_SEI_DFI] = "SEI DFI error",
2052         [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read",
2053         [HBM_SEI_BIST_FAIL] = "SEI BIST fail"
2054 };
2055
2056 struct mmu_spi_sei_cause {
2057         char cause[50];
2058         int clear_bit;
2059 };
2060
2061 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = {
2062         {"page fault", 1},              /* INTERRUPT_CLR[1] */
2063         {"page access", 1},             /* INTERRUPT_CLR[1] */
2064         {"bypass ddr", 2},              /* INTERRUPT_CLR[2] */
2065         {"multi hit", 2},               /* INTERRUPT_CLR[2] */
2066         {"mmu rei0", -1},               /* no clear register bit */
2067         {"mmu rei1", -1},               /* no clear register bit */
2068         {"stlb rei0", -1},              /* no clear register bit */
2069         {"stlb rei1", -1},              /* no clear register bit */
2070         {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */
2071         {"rr privileged read hit", 2},  /* INTERRUPT_CLR[2] */
2072         {"rr secure write hit", 2},     /* INTERRUPT_CLR[2] */
2073         {"rr secure read hit", 2},      /* INTERRUPT_CLR[2] */
2074         {"bist_fail no use", 2},        /* INTERRUPT_CLR[2] */
2075         {"bist_fail no use", 2},        /* INTERRUPT_CLR[2] */
2076         {"bist_fail no use", 2},        /* INTERRUPT_CLR[2] */
2077         {"bist_fail no use", 2},        /* INTERRUPT_CLR[2] */
2078         {"slave error", 16},            /* INTERRUPT_CLR[16] */
2079         {"dec error", 17},              /* INTERRUPT_CLR[17] */
2080         {"burst fifo full", 2}          /* INTERRUPT_CLR[2] */
2081 };
2082
2083 struct gaudi2_cache_invld_params {
2084         u64 start_va;
2085         u64 end_va;
2086         u32 inv_start_val;
2087         u32 flags;
2088         bool range_invalidation;
2089 };
2090
2091 struct gaudi2_tpc_idle_data {
2092         struct engines_data *e;
2093         unsigned long *mask;
2094         bool *is_idle;
2095         const char *tpc_fmt;
2096 };
2097
2098 struct gaudi2_tpc_mmu_data {
2099         u32 rw_asid;
2100 };
2101
2102 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0};
2103
2104 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val);
2105 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id);
2106 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id);
2107 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2108 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2109 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
2110 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
2111                                                                                 bool is_memset);
2112 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2113                 struct engines_data *e);
2114 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2115                 struct engines_data *e);
2116 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2117                 struct engines_data *e);
2118 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
2119 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr);
2120
2121 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev)
2122 {
2123
2124 }
2125
2126 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev)
2127 {
2128         return sizeof(struct packet_msg_short);
2129 }
2130
2131 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev)
2132 {
2133         return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence);
2134 }
2135
2136 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx)
2137 {
2138         struct asic_fixed_properties *prop = &hdev->asic_prop;
2139         int dcore, inst, tpc_seq;
2140         u32 offset;
2141
2142         /* init the return code */
2143         ctx->rc = 0;
2144
2145         for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) {
2146                 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) {
2147                         tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
2148
2149                         if (!(prop->tpc_enabled_mask & BIT(tpc_seq)))
2150                                 continue;
2151
2152                         offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst);
2153
2154                         ctx->fn(hdev, dcore, inst, offset, ctx);
2155                         if (ctx->rc) {
2156                                 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n",
2157                                                         dcore, inst);
2158                                 return;
2159                         }
2160                 }
2161         }
2162
2163         if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6)))
2164                 return;
2165
2166         /* special check for PCI TPC (DCORE0_TPC6) */
2167         offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1);
2168         ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx);
2169         if (ctx->rc)
2170                 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n");
2171 }
2172
2173 static bool gaudi2_host_phys_addr_valid(u64 addr)
2174 {
2175         if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1))
2176                 return true;
2177
2178         return false;
2179 }
2180
2181 static int set_number_of_functional_hbms(struct hl_device *hdev)
2182 {
2183         struct asic_fixed_properties *prop = &hdev->asic_prop;
2184         u8 faulty_hbms = hweight64(hdev->dram_binning);
2185
2186         /* check if all HBMs should be used */
2187         if (!faulty_hbms) {
2188                 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n");
2189                 prop->num_functional_hbms = GAUDI2_HBM_NUM;
2190                 return 0;
2191         }
2192
2193         /*
2194          * check for error condition in which number of binning
2195          * candidates is higher than the maximum supported by the
2196          * driver (in which case binning mask shall be ignored and driver will
2197          * set the default)
2198          */
2199         if (faulty_hbms > MAX_FAULTY_HBMS) {
2200                 dev_err(hdev->dev,
2201                         "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n",
2202                         MAX_FAULTY_HBMS, hdev->dram_binning);
2203                 return -EINVAL;
2204         }
2205
2206         /*
2207          * by default, number of functional HBMs in Gaudi2 is always
2208          * GAUDI2_HBM_NUM - 1.
2209          */
2210         prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms;
2211         return 0;
2212 }
2213
2214 static int gaudi2_set_dram_properties(struct hl_device *hdev)
2215 {
2216         struct asic_fixed_properties *prop = &hdev->asic_prop;
2217         u32 basic_hbm_page_size;
2218         int rc;
2219
2220         rc = set_number_of_functional_hbms(hdev);
2221         if (rc)
2222                 return -EINVAL;
2223
2224         /*
2225          * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround
2226          * in which we are using x16 bigger page size to be able to populate the entire
2227          * HBM mappings in the TLB
2228          */
2229         basic_hbm_page_size = prop->num_functional_hbms * SZ_8M;
2230         prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size;
2231         prop->device_mem_alloc_default_page_size = prop->dram_page_size;
2232         prop->dram_size = prop->num_functional_hbms * SZ_16G;
2233         prop->dram_base_address = DRAM_PHYS_BASE;
2234         prop->dram_end_address = prop->dram_base_address + prop->dram_size;
2235         prop->dram_supports_virtual_memory = true;
2236
2237         prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size;
2238         prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK;
2239         prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START;
2240         prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END;
2241
2242         /* since DRAM page size differs from DMMU page size we need to allocate
2243          * DRAM memory in units of dram_page size and mapping this memory in
2244          * units of DMMU page size. we overcome this size mismatch using a
2245          * scrambling routine which takes a DRAM page and converts it to a DMMU
2246          * page.
2247          * We therefore:
2248          * 1. partition the virtual address space to DRAM-page (whole) pages.
2249          *    (suppose we get n such pages)
2250          * 2. limit the amount of virtual address space we got from 1 above to
2251          *    a multiple of 64M as we don't want the scrambled address to cross
2252          *    the DRAM virtual address space.
2253          *    ( m = (n * DRAM_page_size) / DMMU_page_size).
2254          * 3. determine the and address accordingly
2255          *    end_addr = start_addr + m * 48M
2256          *
2257          *    the DRAM address MSBs (63:48) are not part of the roundup calculation
2258          */
2259         prop->dmmu.start_addr = prop->dram_base_address +
2260                         (prop->dram_page_size *
2261                                 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size));
2262
2263         prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size *
2264                         div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size);
2265
2266         return 0;
2267 }
2268
2269 static int gaudi2_set_fixed_properties(struct hl_device *hdev)
2270 {
2271         struct asic_fixed_properties *prop = &hdev->asic_prop;
2272         struct hw_queue_properties *q_props;
2273         u32 num_sync_stream_queues = 0;
2274         int i;
2275
2276         prop->max_queues = GAUDI2_QUEUE_ID_SIZE;
2277         prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties),
2278                                         GFP_KERNEL);
2279
2280         if (!prop->hw_queues_props)
2281                 return -ENOMEM;
2282
2283         q_props = prop->hw_queues_props;
2284
2285         for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) {
2286                 q_props[i].type = QUEUE_TYPE_HW;
2287                 q_props[i].driver_only = 0;
2288
2289                 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) {
2290                         q_props[i].supports_sync_stream = 0;
2291                 } else {
2292                         q_props[i].supports_sync_stream = 1;
2293                         num_sync_stream_queues++;
2294                 }
2295
2296                 q_props[i].cb_alloc_flags = CB_ALLOC_USER;
2297         }
2298
2299         q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU;
2300         q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1;
2301         q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL;
2302
2303         prop->cache_line_size = DEVICE_CACHE_LINE_SIZE;
2304         prop->cfg_base_address = CFG_BASE;
2305         prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0;
2306         prop->host_base_address = HOST_PHYS_BASE_0;
2307         prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0;
2308         prop->max_pending_cs = GAUDI2_MAX_PENDING_CS;
2309         prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER;
2310         prop->user_dec_intr_count = NUMBER_OF_DEC;
2311         prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1;
2312         prop->completion_mode = HL_COMPLETION_MODE_CS;
2313         prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER;
2314         prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER;
2315
2316         prop->sram_base_address = SRAM_BASE_ADDR;
2317         prop->sram_size = SRAM_SIZE;
2318         prop->sram_end_address = prop->sram_base_address + prop->sram_size;
2319         prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET;
2320
2321         prop->hints_range_reservation = true;
2322
2323         prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1;
2324
2325         if (hdev->pldm)
2326                 prop->mmu_pgt_size = 0x800000; /* 8MB */
2327         else
2328                 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE;
2329
2330         prop->mmu_pte_size = HL_PTE_SIZE;
2331         prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE;
2332         prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE;
2333
2334         prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT;
2335         prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT;
2336         prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT;
2337         prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT;
2338         prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT;
2339         prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK;
2340         prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK;
2341         prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK;
2342         prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK;
2343         prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK;
2344         prop->dmmu.page_size = PAGE_SIZE_1GB;
2345         prop->dmmu.num_hops = MMU_ARCH_6_HOPS;
2346         prop->dmmu.last_mask = LAST_MASK;
2347         prop->dmmu.host_resident = 1;
2348         prop->dmmu.hop_table_size = prop->mmu_hop_table_size;
2349         prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
2350
2351         /*
2352          * this is done in order to be able to validate FW descriptor (i.e. validating that
2353          * the addresses and allocated space for FW image does not cross memory bounds).
2354          * for this reason we set the DRAM size to the minimum possible and later it will
2355          * be modified according to what reported in the cpucp info packet
2356          */
2357         prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G;
2358
2359         hdev->pmmu_huge_range = true;
2360         prop->pmmu.host_resident = 1;
2361         prop->pmmu.num_hops = MMU_ARCH_6_HOPS;
2362         prop->pmmu.last_mask = LAST_MASK;
2363         prop->pmmu.hop_table_size = prop->mmu_hop_table_size;
2364         prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
2365
2366         prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START;
2367         prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END;
2368         prop->hints_host_hpage_reserved_va_range.start_addr =
2369                         RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START;
2370         prop->hints_host_hpage_reserved_va_range.end_addr =
2371                         RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END;
2372
2373         if (PAGE_SIZE == SZ_64K) {
2374                 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K;
2375                 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K;
2376                 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K;
2377                 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K;
2378                 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K;
2379                 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K;
2380                 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K;
2381                 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K;
2382                 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K;
2383                 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K;
2384                 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K;
2385                 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K;
2386                 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
2387                 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
2388                 prop->pmmu.page_size = PAGE_SIZE_64KB;
2389
2390                 /* shifts and masks are the same in PMMU and HPMMU */
2391                 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
2392                 prop->pmmu_huge.page_size = PAGE_SIZE_16MB;
2393                 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
2394                 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
2395         } else {
2396                 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K;
2397                 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K;
2398                 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K;
2399                 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K;
2400                 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K;
2401                 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K;
2402                 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K;
2403                 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K;
2404                 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K;
2405                 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K;
2406                 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K;
2407                 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K;
2408                 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
2409                 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
2410                 prop->pmmu.page_size = PAGE_SIZE_4KB;
2411
2412                 /* shifts and masks are the same in PMMU and HPMMU */
2413                 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
2414                 prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
2415                 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
2416                 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
2417         }
2418
2419         prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE;
2420         prop->num_engine_cores = CPU_ID_MAX;
2421         prop->cfg_size = CFG_SIZE;
2422         prop->max_asid = MAX_ASID;
2423         prop->num_of_events = GAUDI2_EVENT_SIZE;
2424
2425         prop->supports_engine_modes = true;
2426
2427         prop->dc_power_default = DC_POWER_DEFAULT;
2428
2429         prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT;
2430         prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE;
2431         prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE;
2432         prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
2433
2434         strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
2435
2436         prop->mme_master_slave_mode = 1;
2437
2438         prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER +
2439                                         (num_sync_stream_queues * HL_RSVD_SOBS);
2440
2441         prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER +
2442                                         (num_sync_stream_queues * HL_RSVD_MONS);
2443
2444         prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST;
2445         prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT;
2446         prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE;
2447
2448         prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER;
2449
2450         prop->fw_cpu_boot_dev_sts0_valid = false;
2451         prop->fw_cpu_boot_dev_sts1_valid = false;
2452         prop->hard_reset_done_by_fw = false;
2453         prop->gic_interrupts_enable = true;
2454
2455         prop->server_type = HL_SERVER_TYPE_UNKNOWN;
2456
2457         prop->max_dec = NUMBER_OF_DEC;
2458
2459         prop->clk_pll_index = HL_GAUDI2_MME_PLL;
2460
2461         prop->dma_mask = 64;
2462
2463         prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0;
2464
2465         return 0;
2466 }
2467
2468 static int gaudi2_pci_bars_map(struct hl_device *hdev)
2469 {
2470         static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"};
2471         bool is_wc[3] = {false, false, true};
2472         int rc;
2473
2474         rc = hl_pci_bars_map(hdev, name, is_wc);
2475         if (rc)
2476                 return rc;
2477
2478         hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR);
2479
2480         return 0;
2481 }
2482
2483 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
2484 {
2485         struct gaudi2_device *gaudi2 = hdev->asic_specific;
2486         struct hl_inbound_pci_region pci_region;
2487         u64 old_addr = addr;
2488         int rc;
2489
2490         if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr))
2491                 return old_addr;
2492
2493         if (hdev->asic_prop.iatu_done_by_fw)
2494                 return U64_MAX;
2495
2496         /* Inbound Region 2 - Bar 4 - Point to DRAM */
2497         pci_region.mode = PCI_BAR_MATCH_MODE;
2498         pci_region.bar = DRAM_BAR_ID;
2499         pci_region.addr = addr;
2500         rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
2501         if (rc)
2502                 return U64_MAX;
2503
2504         if (gaudi2) {
2505                 old_addr = gaudi2->dram_bar_cur_addr;
2506                 gaudi2->dram_bar_cur_addr = addr;
2507         }
2508
2509         return old_addr;
2510 }
2511
2512 static int gaudi2_init_iatu(struct hl_device *hdev)
2513 {
2514         struct hl_inbound_pci_region inbound_region;
2515         struct hl_outbound_pci_region outbound_region;
2516         u32 bar_addr_low, bar_addr_high;
2517         int rc;
2518
2519         if (hdev->asic_prop.iatu_done_by_fw)
2520                 return 0;
2521
2522         /* Temporary inbound Region 0 - Bar 0 - Point to CFG
2523          * We must map this region in BAR match mode in order to
2524          * fetch BAR physical base address
2525          */
2526         inbound_region.mode = PCI_BAR_MATCH_MODE;
2527         inbound_region.bar = SRAM_CFG_BAR_ID;
2528         /* Base address must be aligned to Bar size which is 256 MB */
2529         inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF;
2530         rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2531         if (rc)
2532                 return rc;
2533
2534         /* Fetch physical BAR address */
2535         bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF);
2536         bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF;
2537
2538         hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low;
2539
2540         /* Inbound Region 0 - Bar 0 - Point to CFG */
2541         inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2542         inbound_region.bar = SRAM_CFG_BAR_ID;
2543         inbound_region.offset_in_bar = 0;
2544         inbound_region.addr = STM_FLASH_BASE_ADDR;
2545         inbound_region.size = CFG_REGION_SIZE;
2546         rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2547         if (rc)
2548                 return rc;
2549
2550         /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */
2551         inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2552         inbound_region.bar = SRAM_CFG_BAR_ID;
2553         inbound_region.offset_in_bar = CFG_REGION_SIZE;
2554         inbound_region.addr = BAR0_RSRVD_BASE_ADDR;
2555         inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE;
2556         rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
2557         if (rc)
2558                 return rc;
2559
2560         /* Inbound Region 2 - Bar 4 - Point to DRAM */
2561         inbound_region.mode = PCI_BAR_MATCH_MODE;
2562         inbound_region.bar = DRAM_BAR_ID;
2563         inbound_region.addr = DRAM_PHYS_BASE;
2564         rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
2565         if (rc)
2566                 return rc;
2567
2568         /* Outbound Region 0 - Point to Host */
2569         outbound_region.addr = HOST_PHYS_BASE_0;
2570         outbound_region.size = HOST_PHYS_SIZE_0;
2571         rc = hl_pci_set_outbound_region(hdev, &outbound_region);
2572
2573         return rc;
2574 }
2575
2576 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev)
2577 {
2578         return RREG32(mmHW_STATE);
2579 }
2580
2581 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev)
2582 {
2583         struct asic_fixed_properties *prop = &hdev->asic_prop;
2584
2585         /*
2586          * check for error condition in which number of binning candidates
2587          * is higher than the maximum supported by the driver
2588          */
2589         if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) {
2590                 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n",
2591                                         MAX_CLUSTER_BINNING_FAULTY_TPCS,
2592                                         hdev->tpc_binning);
2593                 return -EINVAL;
2594         }
2595
2596         prop->tpc_binning_mask = hdev->tpc_binning;
2597         prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK;
2598
2599         return 0;
2600 }
2601
2602 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev)
2603 {
2604         struct asic_fixed_properties *prop = &hdev->asic_prop;
2605         struct hw_queue_properties *q_props = prop->hw_queues_props;
2606         u64 tpc_binning_mask;
2607         u8 subst_idx = 0;
2608         int i, rc;
2609
2610         rc = gaudi2_tpc_binning_init_prop(hdev);
2611         if (rc)
2612                 return rc;
2613
2614         tpc_binning_mask = prop->tpc_binning_mask;
2615
2616         for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) {
2617                 u8 subst_seq, binned, qid_base;
2618
2619                 if (tpc_binning_mask == 0)
2620                         break;
2621
2622                 if (subst_idx == 0) {
2623                         subst_seq = TPC_ID_DCORE0_TPC6;
2624                         qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
2625                 } else {
2626                         subst_seq = TPC_ID_DCORE3_TPC5;
2627                         qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0;
2628                 }
2629
2630
2631                 /* clear bit from mask */
2632                 binned = __ffs(tpc_binning_mask);
2633                 /*
2634                  * Coverity complains about possible out-of-bound access in
2635                  * clear_bit
2636                  */
2637                 if (binned >= TPC_ID_SIZE) {
2638                         dev_err(hdev->dev,
2639                                 "Invalid binned TPC (binning mask: %llx)\n",
2640                                 tpc_binning_mask);
2641                         return -EINVAL;
2642                 }
2643                 clear_bit(binned, (unsigned long *)&tpc_binning_mask);
2644
2645                 /* also clear replacing TPC bit from enabled mask */
2646                 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask);
2647
2648                 /* bin substite TPC's Qs */
2649                 q_props[qid_base].binned = 1;
2650                 q_props[qid_base + 1].binned = 1;
2651                 q_props[qid_base + 2].binned = 1;
2652                 q_props[qid_base + 3].binned = 1;
2653
2654                 subst_idx++;
2655         }
2656
2657         return 0;
2658 }
2659
2660 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev)
2661 {
2662         struct asic_fixed_properties *prop = &hdev->asic_prop;
2663         u8 num_faulty;
2664
2665         num_faulty = hweight32(hdev->decoder_binning);
2666
2667         /*
2668          * check for error condition in which number of binning candidates
2669          * is higher than the maximum supported by the driver
2670          */
2671         if (num_faulty > MAX_FAULTY_DECODERS) {
2672                 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n",
2673                                                 hdev->decoder_binning);
2674                 return -EINVAL;
2675         }
2676
2677         prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK);
2678
2679         if (prop->decoder_binning_mask)
2680                 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1));
2681         else
2682                 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK;
2683
2684         return 0;
2685 }
2686
2687 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev)
2688 {
2689         struct asic_fixed_properties *prop = &hdev->asic_prop;
2690
2691         /* check if we should override default binning */
2692         if (!hdev->dram_binning) {
2693                 prop->dram_binning_mask = 0;
2694                 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK;
2695                 return;
2696         }
2697
2698         /* set DRAM binning constraints */
2699         prop->faulty_dram_cluster_map |= hdev->dram_binning;
2700         prop->dram_binning_mask = hdev->dram_binning;
2701         prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5);
2702 }
2703
2704 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev)
2705 {
2706         struct asic_fixed_properties *prop = &hdev->asic_prop;
2707         struct hw_queue_properties *q_props;
2708         u8 seq, num_faulty;
2709
2710         num_faulty = hweight32(hdev->edma_binning);
2711
2712         /*
2713          * check for error condition in which number of binning candidates
2714          * is higher than the maximum supported by the driver
2715          */
2716         if (num_faulty > MAX_FAULTY_EDMAS) {
2717                 dev_err(hdev->dev,
2718                         "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n",
2719                         hdev->edma_binning);
2720                 return -EINVAL;
2721         }
2722
2723         if (!hdev->edma_binning) {
2724                 prop->edma_binning_mask = 0;
2725                 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK;
2726                 return 0;
2727         }
2728
2729         seq = __ffs((unsigned long)hdev->edma_binning);
2730
2731         /* set binning constraints */
2732         prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]);
2733         prop->edma_binning_mask = hdev->edma_binning;
2734         prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1);
2735
2736         /* bin substitute EDMA's queue */
2737         q_props = prop->hw_queues_props;
2738         q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1;
2739         q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1;
2740         q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1;
2741         q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1;
2742
2743         return 0;
2744 }
2745
2746 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask)
2747 {
2748         struct asic_fixed_properties *prop = &hdev->asic_prop;
2749         u8 num_faulty, seq;
2750
2751         /* check if we should override default binning */
2752         if (!xbar_edge_iso_mask) {
2753                 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK;
2754                 return 0;
2755         }
2756
2757         /*
2758          * note that it can be set to value other than 0 only after cpucp packet (i.e.
2759          * only the FW can set a redundancy value). for user it'll always be 0.
2760          */
2761         num_faulty = hweight32(xbar_edge_iso_mask);
2762
2763         /*
2764          * check for error condition in which number of binning candidates
2765          * is higher than the maximum supported by the driver
2766          */
2767         if (num_faulty > MAX_FAULTY_XBARS) {
2768                 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n",
2769                                                                         MAX_FAULTY_XBARS);
2770                 return -EINVAL;
2771         }
2772
2773         seq = __ffs((unsigned long)xbar_edge_iso_mask);
2774
2775         /* set binning constraints */
2776         prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]);
2777         prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK;
2778
2779         return 0;
2780 }
2781
2782 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask)
2783 {
2784         int rc;
2785
2786         /*
2787          * mark all clusters as good, each component will "fail" cluster
2788          * based on eFuse/user values.
2789          * If more than single cluster is faulty- the chip is unusable
2790          */
2791         hdev->asic_prop.faulty_dram_cluster_map = 0;
2792
2793         gaudi2_set_dram_binning_masks(hdev);
2794
2795         rc = gaudi2_set_edma_binning_masks(hdev);
2796         if (rc)
2797                 return rc;
2798
2799         rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask);
2800         if (rc)
2801                 return rc;
2802
2803
2804         /* always initially set to full mask */
2805         hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK;
2806
2807         return 0;
2808 }
2809
2810 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev)
2811 {
2812         struct asic_fixed_properties *prop = &hdev->asic_prop;
2813         int rc;
2814
2815         rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask);
2816         if (rc)
2817                 return rc;
2818
2819         /* if we have DRAM binning reported by FW we should perform cluster config  */
2820         if (prop->faulty_dram_cluster_map) {
2821                 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map);
2822
2823                 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq];
2824         }
2825
2826         return 0;
2827 }
2828
2829 static int gaudi2_set_binning_masks(struct hl_device *hdev)
2830 {
2831         int rc;
2832
2833         rc = gaudi2_set_cluster_binning_masks(hdev);
2834         if (rc)
2835                 return rc;
2836
2837         rc = gaudi2_set_tpc_binning_masks(hdev);
2838         if (rc)
2839                 return rc;
2840
2841         rc = gaudi2_set_dec_binning_masks(hdev);
2842         if (rc)
2843                 return rc;
2844
2845         return 0;
2846 }
2847
2848 static int gaudi2_cpucp_info_get(struct hl_device *hdev)
2849 {
2850         struct gaudi2_device *gaudi2 = hdev->asic_specific;
2851         struct asic_fixed_properties *prop = &hdev->asic_prop;
2852         long max_power;
2853         u64 dram_size;
2854         int rc;
2855
2856         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2857                 return 0;
2858
2859         /* No point of asking this information again when not doing hard reset, as the device
2860          * CPU hasn't been reset
2861          */
2862         if (hdev->reset_info.in_compute_reset)
2863                 return 0;
2864
2865         rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
2866                                                                                 mmCPU_BOOT_ERR1);
2867         if (rc)
2868                 return rc;
2869
2870         dram_size = le64_to_cpu(prop->cpucp_info.dram_size);
2871         if (dram_size) {
2872                 /* we can have wither 5 or 6 HBMs. other values are invalid */
2873
2874                 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) &&
2875                                         (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) {
2876                         dev_err(hdev->dev,
2877                                 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n",
2878                                 dram_size, prop->dram_size);
2879                         dram_size = prop->dram_size;
2880                 }
2881
2882                 prop->dram_size = dram_size;
2883                 prop->dram_end_address = prop->dram_base_address + dram_size;
2884         }
2885
2886         if (!strlen(prop->cpucp_info.card_name))
2887                 strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
2888
2889         /* Overwrite binning masks with the actual binning values from F/W */
2890         hdev->dram_binning = prop->cpucp_info.dram_binning_mask;
2891         hdev->edma_binning = prop->cpucp_info.edma_binning_mask;
2892         hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);
2893         hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));
2894
2895         dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n",
2896                         hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2897                         hdev->decoder_binning);
2898
2899         /*
2900          * at this point the DRAM parameters need to be updated according to data obtained
2901          * from the FW
2902          */
2903         rc = hdev->asic_funcs->set_dram_properties(hdev);
2904         if (rc)
2905                 return rc;
2906
2907         rc = hdev->asic_funcs->set_binning_masks(hdev);
2908         if (rc)
2909                 return rc;
2910
2911         max_power = hl_fw_get_max_power(hdev);
2912         if (max_power < 0)
2913                 return max_power;
2914
2915         prop->max_power_default = (u64) max_power;
2916
2917         return 0;
2918 }
2919
2920 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev)
2921 {
2922         struct gaudi2_device *gaudi2 = hdev->asic_specific;
2923         u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS];
2924         int rc;
2925
2926         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2927                 return 0;
2928
2929         rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr);
2930         if (rc)
2931                 return rc;
2932
2933         hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3];
2934
2935         return 0;
2936 }
2937
2938 static int gaudi2_early_init(struct hl_device *hdev)
2939 {
2940         struct asic_fixed_properties *prop = &hdev->asic_prop;
2941         struct pci_dev *pdev = hdev->pdev;
2942         resource_size_t pci_bar_size;
2943         int rc;
2944
2945         rc = gaudi2_set_fixed_properties(hdev);
2946         if (rc)
2947                 return rc;
2948
2949         /* Check BAR sizes */
2950         pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID);
2951
2952         if (pci_bar_size != CFG_BAR_SIZE) {
2953                 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2954                         SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE);
2955                 rc = -ENODEV;
2956                 goto free_queue_props;
2957         }
2958
2959         pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID);
2960         if (pci_bar_size != MSIX_BAR_SIZE) {
2961                 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2962                         MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE);
2963                 rc = -ENODEV;
2964                 goto free_queue_props;
2965         }
2966
2967         prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID);
2968         hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID);
2969
2970         /*
2971          * Only in pldm driver config iATU
2972          */
2973         if (hdev->pldm)
2974                 hdev->asic_prop.iatu_done_by_fw = false;
2975         else
2976                 hdev->asic_prop.iatu_done_by_fw = true;
2977
2978         rc = hl_pci_init(hdev);
2979         if (rc)
2980                 goto free_queue_props;
2981
2982         /* Before continuing in the initialization, we need to read the preboot
2983          * version to determine whether we run with a security-enabled firmware
2984          */
2985         rc = hl_fw_read_preboot_status(hdev);
2986         if (rc) {
2987                 if (hdev->reset_on_preboot_fail)
2988                         /* we are already on failure flow, so don't check if hw_fini fails. */
2989                         hdev->asic_funcs->hw_fini(hdev, true, false);
2990                 goto pci_fini;
2991         }
2992
2993         if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
2994                 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
2995                 rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2996                 if (rc) {
2997                         dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
2998                         goto pci_fini;
2999                 }
3000         }
3001
3002         return 0;
3003
3004 pci_fini:
3005         hl_pci_fini(hdev);
3006 free_queue_props:
3007         kfree(hdev->asic_prop.hw_queues_props);
3008         return rc;
3009 }
3010
3011 static int gaudi2_early_fini(struct hl_device *hdev)
3012 {
3013         kfree(hdev->asic_prop.hw_queues_props);
3014         hl_pci_fini(hdev);
3015
3016         return 0;
3017 }
3018
3019 static bool gaudi2_is_arc_nic_owned(u64 arc_id)
3020 {
3021         switch (arc_id) {
3022         case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
3023                 return true;
3024         default:
3025                 return false;
3026         }
3027 }
3028
3029 static bool gaudi2_is_arc_tpc_owned(u64 arc_id)
3030 {
3031         switch (arc_id) {
3032         case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
3033                 return true;
3034         default:
3035                 return false;
3036         }
3037 }
3038
3039 static void gaudi2_init_arcs(struct hl_device *hdev)
3040 {
3041         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3042         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3043         u64 arc_id;
3044         u32 i;
3045
3046         for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) {
3047                 if (gaudi2_is_arc_enabled(hdev, i))
3048                         continue;
3049
3050                 gaudi2_set_arc_id_cap(hdev, i);
3051         }
3052
3053         for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
3054                 if (!gaudi2_is_queue_enabled(hdev, i))
3055                         continue;
3056
3057                 arc_id = gaudi2_queue_id_to_arc_id[i];
3058                 if (gaudi2_is_arc_enabled(hdev, arc_id))
3059                         continue;
3060
3061                 if (gaudi2_is_arc_nic_owned(arc_id) &&
3062                                 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)))
3063                         continue;
3064
3065                 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized &
3066                                                         BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)))
3067                         continue;
3068
3069                 gaudi2_set_arc_id_cap(hdev, arc_id);
3070         }
3071
3072         /* Fetch ARC scratchpad address */
3073         hdev->asic_prop.engine_core_interrupt_reg_addr =
3074                 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl);
3075 }
3076
3077 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
3078 {
3079         u32 reg_base, reg_val;
3080         int rc;
3081
3082         switch (cpu_id) {
3083         case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3:
3084                 /* Each ARC scheduler has 2 consecutive DCCM blocks */
3085                 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3086                                                 ARC_DCCM_BLOCK_SIZE * 2, true);
3087                 if (rc)
3088                         return rc;
3089                 break;
3090         case CPU_ID_SCHED_ARC4:
3091         case CPU_ID_SCHED_ARC5:
3092         case CPU_ID_MME_QMAN_ARC0:
3093         case CPU_ID_MME_QMAN_ARC1:
3094                 reg_base = gaudi2_arc_blocks_bases[cpu_id];
3095
3096                 /* Scrub lower DCCM block */
3097                 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3098                                                 ARC_DCCM_BLOCK_SIZE, true);
3099                 if (rc)
3100                         return rc;
3101
3102                 /* Switch to upper DCCM block */
3103                 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1);
3104                 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
3105
3106                 /* Scrub upper DCCM block */
3107                 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3108                                                 ARC_DCCM_BLOCK_SIZE, true);
3109                 if (rc)
3110                         return rc;
3111
3112                 /* Switch to lower DCCM block */
3113                 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0);
3114                 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
3115                 break;
3116         default:
3117                 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3118                                                 ARC_DCCM_BLOCK_SIZE, true);
3119                 if (rc)
3120                         return rc;
3121         }
3122
3123         return 0;
3124 }
3125
3126 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
3127 {
3128         u16 arc_id;
3129         int rc;
3130
3131         for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) {
3132                 if (!gaudi2_is_arc_enabled(hdev, arc_id))
3133                         continue;
3134
3135                 rc = gaudi2_scrub_arc_dccm(hdev, arc_id);
3136                 if (rc)
3137                         return rc;
3138         }
3139
3140         return 0;
3141 }
3142
3143 static int gaudi2_late_init(struct hl_device *hdev)
3144 {
3145         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3146         int rc;
3147
3148         hdev->asic_prop.supports_advanced_cpucp_rc = true;
3149
3150         rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS,
3151                                         gaudi2->virt_msix_db_dma_addr);
3152         if (rc) {
3153                 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
3154                 return rc;
3155         }
3156
3157         rc = gaudi2_fetch_psoc_frequency(hdev);
3158         if (rc) {
3159                 dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
3160                 goto disable_pci_access;
3161         }
3162
3163         gaudi2_init_arcs(hdev);
3164
3165         rc = gaudi2_scrub_arcs_dccm(hdev);
3166         if (rc) {
3167                 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
3168                 goto disable_pci_access;
3169         }
3170
3171         gaudi2_init_security(hdev);
3172
3173         return 0;
3174
3175 disable_pci_access:
3176         hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
3177
3178         return rc;
3179 }
3180
3181 static void gaudi2_late_fini(struct hl_device *hdev)
3182 {
3183         hl_hwmon_release_resources(hdev);
3184 }
3185
3186 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx)
3187 {
3188         struct user_mapped_block *blocks = gaudi2->mapped_blocks;
3189
3190         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3191         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3192         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3193         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3194         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3195         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3196         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3197         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3198         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3199         HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3200 }
3201
3202 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev)
3203 {
3204         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3205         struct user_mapped_block *blocks = gaudi2->mapped_blocks;
3206         u32 block_size, umr_start_idx, num_umr_blocks;
3207         int i;
3208
3209         for (i = 0 ; i < NUM_ARC_CPUS ; i++) {
3210                 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3)
3211                         block_size = ARC_DCCM_BLOCK_SIZE * 2;
3212                 else
3213                         block_size = ARC_DCCM_BLOCK_SIZE;
3214
3215                 blocks[i].address = gaudi2_arc_dccm_bases[i];
3216                 blocks[i].size = block_size;
3217         }
3218
3219         blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE;
3220         blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE;
3221
3222         blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE;
3223         blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE;
3224
3225         blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE;
3226         blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE;
3227
3228         blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE;
3229         blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE;
3230
3231         blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE;
3232         blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE;
3233
3234         blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE;
3235         blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE;
3236
3237         blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE;
3238         blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE;
3239
3240         blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE;
3241         blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE;
3242
3243         umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS;
3244         num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS;
3245         for (i = 0 ; i < num_umr_blocks ; i++) {
3246                 u8 nic_id, umr_block_id;
3247
3248                 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS;
3249                 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS;
3250
3251                 blocks[umr_start_idx + i].address =
3252                         mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE +
3253                         (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET +
3254                         (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET +
3255                         umr_block_id * NIC_UMR_OFFSET;
3256                 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE;
3257         }
3258
3259         /* Expose decoder HW configuration block to user */
3260         gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX);
3261
3262         for (i = 1; i < NUM_OF_DCORES; ++i) {
3263                 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE;
3264                 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE;
3265
3266                 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address =
3267                                                 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET;
3268
3269                 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address =
3270                                                 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET;
3271         }
3272 }
3273
3274 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
3275 {
3276         dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
3277         void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {};
3278         int i, j, rc = 0;
3279
3280         /* The device ARC works with 32-bits addresses, and because there is a single HW register
3281          * that holds the extension bits (49..28), these bits must be identical in all the allocated
3282          * range.
3283          */
3284
3285         for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
3286                 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,
3287                                                         &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO);
3288                 if (!virt_addr_arr[i]) {
3289                         rc = -ENOMEM;
3290                         goto free_dma_mem_arr;
3291                 }
3292
3293                 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
3294                 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr))
3295                         break;
3296         }
3297
3298         if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) {
3299                 dev_err(hdev->dev,
3300                         "MSB of ARC accessible DMA memory are not identical in all range\n");
3301                 rc = -EFAULT;
3302                 goto free_dma_mem_arr;
3303         }
3304
3305         hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
3306         hdev->cpu_accessible_dma_address = dma_addr_arr[i];
3307
3308 free_dma_mem_arr:
3309         for (j = 0 ; j < i ; j++)
3310                 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],
3311                                                 dma_addr_arr[j]);
3312
3313         return rc;
3314 }
3315
3316 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev)
3317 {
3318         struct asic_fixed_properties *prop = &hdev->asic_prop;
3319         struct pci_mem_region *region;
3320
3321         /* CFG */
3322         region = &hdev->pci_mem_region[PCI_REGION_CFG];
3323         region->region_base = CFG_BASE;
3324         region->region_size = CFG_SIZE;
3325         region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR;
3326         region->bar_size = CFG_BAR_SIZE;
3327         region->bar_id = SRAM_CFG_BAR_ID;
3328         region->used = 1;
3329
3330         /* SRAM */
3331         region = &hdev->pci_mem_region[PCI_REGION_SRAM];
3332         region->region_base = SRAM_BASE_ADDR;
3333         region->region_size = SRAM_SIZE;
3334         region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE;
3335         region->bar_size = CFG_BAR_SIZE;
3336         region->bar_id = SRAM_CFG_BAR_ID;
3337         region->used = 1;
3338
3339         /* DRAM */
3340         region = &hdev->pci_mem_region[PCI_REGION_DRAM];
3341         region->region_base = DRAM_PHYS_BASE;
3342         region->region_size = hdev->asic_prop.dram_size;
3343         region->offset_in_bar = 0;
3344         region->bar_size = prop->dram_pci_bar_size;
3345         region->bar_id = DRAM_BAR_ID;
3346         region->used = 1;
3347 }
3348
3349 static void gaudi2_user_interrupt_setup(struct hl_device *hdev)
3350 {
3351         struct asic_fixed_properties *prop = &hdev->asic_prop;
3352         int i, j, k;
3353
3354         /* Initialize TPC interrupt */
3355         HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC);
3356
3357         /* Initialize unexpected error interrupt */
3358         HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0,
3359                                                 HL_USR_INTERRUPT_UNEXPECTED);
3360
3361         /* Initialize common user CQ interrupt */
3362         HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,
3363                                 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ);
3364
3365         /* Initialize common decoder interrupt */
3366         HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev,
3367                                 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER);
3368
3369         /* User interrupts structure holds both decoder and user interrupts from various engines.
3370          * We first initialize the decoder interrupts and then we add the user interrupts.
3371          * The only limitation is that the last decoder interrupt id must be smaller
3372          * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time.
3373          */
3374
3375         /* Initialize decoder interrupts, expose only normal interrupts,
3376          * error interrupts to be handled by driver
3377          */
3378         for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM;
3379                                                                                 i += 2, j++)
3380                 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i,
3381                                                 HL_USR_INTERRUPT_DECODER);
3382
3383         for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++)
3384                 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ);
3385 }
3386
3387 static inline int gaudi2_get_non_zero_random_int(void)
3388 {
3389         int rand = get_random_u32();
3390
3391         return rand ? rand : 1;
3392 }
3393
3394 static void gaudi2_special_blocks_free(struct hl_device *hdev)
3395 {
3396         struct asic_fixed_properties *prop = &hdev->asic_prop;
3397         struct hl_skip_blocks_cfg *skip_special_blocks_cfg =
3398                         &prop->skip_special_blocks_cfg;
3399
3400         kfree(prop->special_blocks);
3401         kfree(skip_special_blocks_cfg->block_types);
3402         kfree(skip_special_blocks_cfg->block_ranges);
3403 }
3404
3405 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev)
3406 {
3407         gaudi2_special_blocks_free(hdev);
3408 }
3409
3410 static bool gaudi2_special_block_skip(struct hl_device *hdev,
3411                 struct hl_special_blocks_cfg *special_blocks_cfg,
3412                 u32 blk_idx, u32 major, u32 minor, u32 sub_minor)
3413 {
3414         return false;
3415 }
3416
3417 static int gaudi2_special_blocks_config(struct hl_device *hdev)
3418 {
3419         struct asic_fixed_properties *prop = &hdev->asic_prop;
3420         int i, rc;
3421
3422         /* Configure Special blocks */
3423         prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE;
3424         prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks);
3425         prop->special_blocks = kmalloc_array(prop->num_of_special_blocks,
3426                         sizeof(*prop->special_blocks), GFP_KERNEL);
3427         if (!prop->special_blocks)
3428                 return -ENOMEM;
3429
3430         for (i = 0 ; i < prop->num_of_special_blocks ; i++)
3431                 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i],
3432                                 sizeof(*prop->special_blocks));
3433
3434         /* Configure when to skip Special blocks */
3435         memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg));
3436         prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip;
3437
3438         if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) {
3439                 prop->skip_special_blocks_cfg.block_types =
3440                                 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types),
3441                                         sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL);
3442                 if (!prop->skip_special_blocks_cfg.block_types) {
3443                         rc = -ENOMEM;
3444                         goto free_special_blocks;
3445                 }
3446
3447                 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types,
3448                                 sizeof(gaudi2_iterator_skip_block_types));
3449
3450                 prop->skip_special_blocks_cfg.block_types_len =
3451                                         ARRAY_SIZE(gaudi2_iterator_skip_block_types);
3452         }
3453
3454         if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) {
3455                 prop->skip_special_blocks_cfg.block_ranges =
3456                                 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges),
3457                                         sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL);
3458                 if (!prop->skip_special_blocks_cfg.block_ranges) {
3459                         rc = -ENOMEM;
3460                         goto free_skip_special_blocks_types;
3461                 }
3462
3463                 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++)
3464                         memcpy(&prop->skip_special_blocks_cfg.block_ranges[i],
3465                                         &gaudi2_iterator_skip_block_ranges[i],
3466                                         sizeof(struct range));
3467
3468                 prop->skip_special_blocks_cfg.block_ranges_len =
3469                                         ARRAY_SIZE(gaudi2_iterator_skip_block_ranges);
3470         }
3471
3472         return 0;
3473
3474 free_skip_special_blocks_types:
3475         kfree(prop->skip_special_blocks_cfg.block_types);
3476 free_special_blocks:
3477         kfree(prop->special_blocks);
3478
3479         return rc;
3480 }
3481
3482 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev)
3483 {
3484         return gaudi2_special_blocks_config(hdev);
3485 }
3486
3487 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev)
3488 {
3489         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3490         struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
3491         int i;
3492
3493         for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
3494                 /* bail-out if this is an allocation failure point */
3495                 if (!msg_info[i].kern_addr)
3496                         break;
3497
3498                 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr);
3499                 msg_info[i].kern_addr = NULL;
3500         }
3501 }
3502
3503 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev)
3504 {
3505         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3506         struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
3507         int i, rc;
3508
3509         /* allocate a message-short buf for each Q we intend to test */
3510         for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
3511                 msg_info[i].kern_addr =
3512                         (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short),
3513                                                         GFP_KERNEL, &msg_info[i].dma_addr);
3514                 if (!msg_info[i].kern_addr) {
3515                         dev_err(hdev->dev,
3516                                 "Failed to allocate dma memory for H/W queue %d testing\n", i);
3517                         rc = -ENOMEM;
3518                         goto err_exit;
3519                 }
3520         }
3521
3522         return 0;
3523
3524 err_exit:
3525         gaudi2_test_queues_msgs_free(hdev);
3526         return rc;
3527 }
3528
3529 static int gaudi2_sw_init(struct hl_device *hdev)
3530 {
3531         struct asic_fixed_properties *prop = &hdev->asic_prop;
3532         struct gaudi2_device *gaudi2;
3533         int i, rc;
3534
3535         /* Allocate device structure */
3536         gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL);
3537         if (!gaudi2)
3538                 return -ENOMEM;
3539
3540         for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) {
3541                 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid)
3542                         continue;
3543
3544                 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) {
3545                         dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n",
3546                                 GAUDI2_EVENT_SIZE);
3547                         rc = -EINVAL;
3548                         goto free_gaudi2_device;
3549                 }
3550
3551                 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id;
3552         }
3553
3554         for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++)
3555                 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int();
3556
3557         gaudi2->cpucp_info_get = gaudi2_cpucp_info_get;
3558
3559         hdev->asic_specific = gaudi2;
3560
3561         /* Create DMA pool for small allocations.
3562          * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped
3563          * PI/CI registers allocated from this pool have this restriction
3564          */
3565         hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev,
3566                                         GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0);
3567         if (!hdev->dma_pool) {
3568                 dev_err(hdev->dev, "failed to create DMA pool\n");
3569                 rc = -ENOMEM;
3570                 goto free_gaudi2_device;
3571         }
3572
3573         rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev);
3574         if (rc)
3575                 goto free_dma_pool;
3576
3577         hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
3578         if (!hdev->cpu_accessible_dma_pool) {
3579                 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n");
3580                 rc = -ENOMEM;
3581                 goto free_cpu_dma_mem;
3582         }
3583
3584         rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem,
3585                                 HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
3586         if (rc) {
3587                 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n");
3588                 rc = -EFAULT;
3589                 goto free_cpu_accessible_dma_pool;
3590         }
3591
3592         gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size,
3593                                                                 &gaudi2->virt_msix_db_dma_addr);
3594         if (!gaudi2->virt_msix_db_cpu_addr) {
3595                 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n");
3596                 rc = -ENOMEM;
3597                 goto free_cpu_accessible_dma_pool;
3598         }
3599
3600         spin_lock_init(&gaudi2->hw_queues_lock);
3601
3602         gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE,
3603                                                         &gaudi2->scratchpad_bus_address,
3604                                                         GFP_KERNEL | __GFP_ZERO);
3605         if (!gaudi2->scratchpad_kernel_address) {
3606                 rc = -ENOMEM;
3607                 goto free_virt_msix_db_mem;
3608         }
3609
3610         gaudi2_user_mapped_blocks_init(hdev);
3611
3612         /* Initialize user interrupts */
3613         gaudi2_user_interrupt_setup(hdev);
3614
3615         hdev->supports_coresight = true;
3616         hdev->supports_sync_stream = true;
3617         hdev->supports_cb_mapping = true;
3618         hdev->supports_wait_for_multi_cs = false;
3619
3620         prop->supports_compute_reset = true;
3621
3622         /* Event queue sanity check added in FW version 1.11 */
3623         if (hl_is_fw_sw_ver_below(hdev, 1, 11))
3624                 hdev->event_queue.check_eqe_index = false;
3625         else
3626                 hdev->event_queue.check_eqe_index = true;
3627
3628         hdev->asic_funcs->set_pci_memory_regions(hdev);
3629
3630         rc = gaudi2_special_blocks_iterator_config(hdev);
3631         if (rc)
3632                 goto free_scratchpad_mem;
3633
3634         rc = gaudi2_test_queues_msgs_alloc(hdev);
3635         if (rc)
3636                 goto special_blocks_free;
3637
3638         return 0;
3639
3640 special_blocks_free:
3641         gaudi2_special_blocks_iterator_free(hdev);
3642 free_scratchpad_mem:
3643         hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3644                                   gaudi2->scratchpad_bus_address);
3645 free_virt_msix_db_mem:
3646         hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3647 free_cpu_accessible_dma_pool:
3648         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3649 free_cpu_dma_mem:
3650         hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3651                                         hdev->cpu_accessible_dma_address);
3652 free_dma_pool:
3653         dma_pool_destroy(hdev->dma_pool);
3654 free_gaudi2_device:
3655         kfree(gaudi2);
3656         return rc;
3657 }
3658
3659 static int gaudi2_sw_fini(struct hl_device *hdev)
3660 {
3661         struct asic_fixed_properties *prop = &hdev->asic_prop;
3662         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3663
3664         gaudi2_test_queues_msgs_free(hdev);
3665
3666         gaudi2_special_blocks_iterator_free(hdev);
3667
3668         hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3669
3670         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3671
3672         hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3673                                                 hdev->cpu_accessible_dma_address);
3674
3675         hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3676                                         gaudi2->scratchpad_bus_address);
3677
3678         dma_pool_destroy(hdev->dma_pool);
3679
3680         kfree(gaudi2);
3681
3682         return 0;
3683 }
3684
3685 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base)
3686 {
3687         WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP |
3688                                                 QM_GLBL_CFG1_CQF_STOP |
3689                                                 QM_GLBL_CFG1_CP_STOP);
3690
3691         /* stop also the ARC */
3692         WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP);
3693 }
3694
3695 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base)
3696 {
3697         WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH |
3698                                                 QM_GLBL_CFG1_CQF_FLUSH |
3699                                                 QM_GLBL_CFG1_CP_FLUSH);
3700 }
3701
3702 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base)
3703 {
3704         WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH);
3705 }
3706
3707 /**
3708  * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3709  *
3710  * @hdev: pointer to the habanalabs device structure
3711  * @queue_id: queue to clear fence counters to
3712  * @skip_fence: if true set maximum fence value to all fence counters to avoid
3713  *              getting stuck on any fence value. otherwise set all fence
3714  *              counters to 0 (standard clear of fence counters)
3715  */
3716 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id,
3717                                                 bool skip_fence)
3718 {
3719         u32 size, reg_base;
3720         u32 addr, val;
3721
3722         reg_base = gaudi2_qm_blocks_bases[queue_id];
3723
3724         addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET;
3725         size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0;
3726
3727         /*
3728          * in case we want to make sure that QM that is stuck on a fence will
3729          * be released we should set the fence counter to a higher value that
3730          * the value the QM waiting for. to comply with any fence counter of
3731          * any value we set maximum fence value to all counters
3732          */
3733         val = skip_fence ? U32_MAX : 0;
3734         gaudi2_memset_device_lbw(hdev, addr, size, val);
3735 }
3736
3737 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id)
3738 {
3739         u32 reg_base = gaudi2_qm_blocks_bases[queue_id];
3740
3741         gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true);
3742         gaudi2_flush_qman_common(hdev, reg_base);
3743         gaudi2_flush_qman_arc_common(hdev, reg_base);
3744 }
3745
3746 static void gaudi2_stop_dma_qmans(struct hl_device *hdev)
3747 {
3748         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3749         int dcore, inst;
3750
3751         if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3752                 goto stop_edma_qmans;
3753
3754         /* Stop CPs of PDMA QMANs */
3755         gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE);
3756         gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE);
3757
3758 stop_edma_qmans:
3759         if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3760                 return;
3761
3762         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3763                 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3764                         u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3765                         u32 qm_base;
3766
3767                         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3768                                 continue;
3769
3770                         qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3771                                         inst * DCORE_EDMA_OFFSET;
3772
3773                         /* Stop CPs of EDMA QMANs */
3774                         gaudi2_stop_qman_common(hdev, qm_base);
3775                 }
3776         }
3777 }
3778
3779 static void gaudi2_stop_mme_qmans(struct hl_device *hdev)
3780 {
3781         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3782         u32 offset, i;
3783
3784         offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3785
3786         for (i = 0 ; i < NUM_OF_DCORES ; i++) {
3787                 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)))
3788                         continue;
3789
3790                 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3791         }
3792 }
3793
3794 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev)
3795 {
3796         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3797         u32 reg_base;
3798         int i;
3799
3800         if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3801                 return;
3802
3803         for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3804                 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3805                         continue;
3806
3807                 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
3808                 gaudi2_stop_qman_common(hdev, reg_base);
3809         }
3810 }
3811
3812 static void gaudi2_stop_rot_qmans(struct hl_device *hdev)
3813 {
3814         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3815         u32 reg_base;
3816         int i;
3817
3818         if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3819                 return;
3820
3821         for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3822                 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3823                         continue;
3824
3825                 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
3826                 gaudi2_stop_qman_common(hdev, reg_base);
3827         }
3828 }
3829
3830 static void gaudi2_stop_nic_qmans(struct hl_device *hdev)
3831 {
3832         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3833         u32 reg_base, queue_id;
3834         int i;
3835
3836         if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
3837                 return;
3838
3839         queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
3840
3841         for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
3842                 if (!(hdev->nic_ports_mask & BIT(i)))
3843                         continue;
3844
3845                 reg_base = gaudi2_qm_blocks_bases[queue_id];
3846                 gaudi2_stop_qman_common(hdev, reg_base);
3847         }
3848 }
3849
3850 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base)
3851 {
3852         u32 reg_val;
3853
3854         reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1);
3855         WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val);
3856 }
3857
3858 static void gaudi2_dma_stall(struct hl_device *hdev)
3859 {
3860         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3861         int dcore, inst;
3862
3863         if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3864                 goto stall_edma;
3865
3866         gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE);
3867         gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE);
3868
3869 stall_edma:
3870         if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3871                 return;
3872
3873         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3874                 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3875                         u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3876                         u32 core_base;
3877
3878                         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3879                                 continue;
3880
3881                         core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET +
3882                                         inst * DCORE_EDMA_OFFSET;
3883
3884                         /* Stall CPs of EDMA QMANs */
3885                         gaudi2_stall_dma_common(hdev, core_base);
3886                 }
3887         }
3888 }
3889
3890 static void gaudi2_mme_stall(struct hl_device *hdev)
3891 {
3892         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3893         u32 offset, i;
3894
3895         offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL;
3896
3897         for (i = 0 ; i < NUM_OF_DCORES ; i++)
3898                 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3899                         WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1);
3900 }
3901
3902 static void gaudi2_tpc_stall(struct hl_device *hdev)
3903 {
3904         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3905         u32 reg_base;
3906         int i;
3907
3908         if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3909                 return;
3910
3911         for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3912                 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3913                         continue;
3914
3915                 reg_base = gaudi2_tpc_cfg_blocks_bases[i];
3916                 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1);
3917         }
3918 }
3919
3920 static void gaudi2_rotator_stall(struct hl_device *hdev)
3921 {
3922         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3923         u32 reg_val;
3924         int i;
3925
3926         if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3927                 return;
3928
3929         reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) |
3930                         FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) |
3931                         FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1);
3932
3933         for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3934                 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3935                         continue;
3936
3937                 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val);
3938         }
3939 }
3940
3941 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base)
3942 {
3943         WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0);
3944 }
3945
3946 static void gaudi2_disable_dma_qmans(struct hl_device *hdev)
3947 {
3948         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3949         int dcore, inst;
3950
3951         if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3952                 goto stop_edma_qmans;
3953
3954         gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE);
3955         gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE);
3956
3957 stop_edma_qmans:
3958         if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3959                 return;
3960
3961         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3962                 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3963                         u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3964                         u32 qm_base;
3965
3966                         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3967                                 continue;
3968
3969                         qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3970                                         inst * DCORE_EDMA_OFFSET;
3971
3972                         /* Disable CPs of EDMA QMANs */
3973                         gaudi2_disable_qman_common(hdev, qm_base);
3974                 }
3975         }
3976 }
3977
3978 static void gaudi2_disable_mme_qmans(struct hl_device *hdev)
3979 {
3980         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3981         u32 offset, i;
3982
3983         offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3984
3985         for (i = 0 ; i < NUM_OF_DCORES ; i++)
3986                 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3987                         gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3988 }
3989
3990 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev)
3991 {
3992         struct gaudi2_device *gaudi2 = hdev->asic_specific;
3993         u32 reg_base;
3994         int i;
3995
3996         if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3997                 return;
3998
3999         for (i = 0 ; i < TPC_ID_SIZE ; i++) {
4000                 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
4001                         continue;
4002
4003                 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
4004                 gaudi2_disable_qman_common(hdev, reg_base);
4005         }
4006 }
4007
4008 static void gaudi2_disable_rot_qmans(struct hl_device *hdev)
4009 {
4010         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4011         u32 reg_base;
4012         int i;
4013
4014         if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
4015                 return;
4016
4017         for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
4018                 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
4019                         continue;
4020
4021                 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
4022                 gaudi2_disable_qman_common(hdev, reg_base);
4023         }
4024 }
4025
4026 static void gaudi2_disable_nic_qmans(struct hl_device *hdev)
4027 {
4028         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4029         u32 reg_base, queue_id;
4030         int i;
4031
4032         if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
4033                 return;
4034
4035         queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
4036
4037         for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
4038                 if (!(hdev->nic_ports_mask & BIT(i)))
4039                         continue;
4040
4041                 reg_base = gaudi2_qm_blocks_bases[queue_id];
4042                 gaudi2_disable_qman_common(hdev, reg_base);
4043         }
4044 }
4045
4046 static void gaudi2_enable_timestamp(struct hl_device *hdev)
4047 {
4048         /* Disable the timestamp counter */
4049         WREG32(mmPSOC_TIMESTAMP_BASE, 0);
4050
4051         /* Zero the lower/upper parts of the 64-bit counter */
4052         WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0);
4053         WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0);
4054
4055         /* Enable the counter */
4056         WREG32(mmPSOC_TIMESTAMP_BASE, 1);
4057 }
4058
4059 static void gaudi2_disable_timestamp(struct hl_device *hdev)
4060 {
4061         /* Disable the timestamp counter */
4062         WREG32(mmPSOC_TIMESTAMP_BASE, 0);
4063 }
4064
4065 static const char *gaudi2_irq_name(u16 irq_number)
4066 {
4067         switch (irq_number) {
4068         case GAUDI2_IRQ_NUM_EVENT_QUEUE:
4069                 return "gaudi2 cpu eq";
4070         case GAUDI2_IRQ_NUM_COMPLETION:
4071                 return "gaudi2 completion";
4072         case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM:
4073                 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM];
4074         case GAUDI2_IRQ_NUM_TPC_ASSERT:
4075                 return "gaudi2 tpc assert";
4076         case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR:
4077                 return "gaudi2 unexpected error";
4078         case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST:
4079                 return "gaudi2 user completion";
4080         default:
4081                 return "invalid";
4082         }
4083 }
4084
4085 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num)
4086 {
4087         int i, irq, relative_idx;
4088         struct hl_dec *dec;
4089
4090         for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) {
4091                 irq = pci_irq_vector(hdev->pdev, i);
4092                 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
4093
4094                 dec = hdev->dec + relative_idx / 2;
4095
4096                 /* We pass different structures depending on the irq handler. For the abnormal
4097                  * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
4098                  * user_interrupt entry
4099                  */
4100                 free_irq(irq, ((relative_idx % 2) ?
4101                                 (void *) dec :
4102                                 (void *) &hdev->user_interrupt[dec->core_id]));
4103         }
4104 }
4105
4106 static int gaudi2_dec_enable_msix(struct hl_device *hdev)
4107 {
4108         int rc, i, irq_init_cnt, irq, relative_idx;
4109         struct hl_dec *dec;
4110
4111         for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0;
4112                         i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM;
4113                         i++, irq_init_cnt++) {
4114
4115                 irq = pci_irq_vector(hdev->pdev, i);
4116                 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
4117
4118                 /* We pass different structures depending on the irq handler. For the abnormal
4119                  * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
4120                  * user_interrupt entry
4121                  *
4122                  * TODO: change the dec abnrm to threaded irq
4123                  */
4124
4125                 dec = hdev->dec + relative_idx / 2;
4126                 if (relative_idx % 2) {
4127                         rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0,
4128                                                 gaudi2_irq_name(i), (void *) dec);
4129                 } else {
4130                         rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4131                                         hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4132                                         gaudi2_irq_name(i),
4133                                         (void *) &hdev->user_interrupt[dec->core_id]);
4134                 }
4135
4136                 if (rc) {
4137                         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4138                         goto free_dec_irqs;
4139                 }
4140         }
4141
4142         return 0;
4143
4144 free_dec_irqs:
4145         gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt));
4146         return rc;
4147 }
4148
4149 static int gaudi2_enable_msix(struct hl_device *hdev)
4150 {
4151         struct asic_fixed_properties *prop = &hdev->asic_prop;
4152         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4153         int rc, irq, i, j, user_irq_init_cnt;
4154         struct hl_cq *cq;
4155
4156         if (gaudi2->hw_cap_initialized & HW_CAP_MSIX)
4157                 return 0;
4158
4159         rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES,
4160                                         PCI_IRQ_MSIX);
4161         if (rc < 0) {
4162                 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n",
4163                         GAUDI2_MSIX_ENTRIES, rc);
4164                 return rc;
4165         }
4166
4167         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4168         cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4169         rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq);
4170         if (rc) {
4171                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4172                 goto free_irq_vectors;
4173         }
4174
4175         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4176         rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE),
4177                         &hdev->event_queue);
4178         if (rc) {
4179                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4180                 goto free_completion_irq;
4181         }
4182
4183         rc = gaudi2_dec_enable_msix(hdev);
4184         if (rc) {
4185                 dev_err(hdev->dev, "Failed to enable decoder IRQ");
4186                 goto free_event_irq;
4187         }
4188
4189         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4190         rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4191                         hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4192                         gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt);
4193         if (rc) {
4194                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4195                 goto free_dec_irq;
4196         }
4197
4198         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4199         rc = request_irq(irq, hl_irq_handler_user_interrupt, 0,
4200                         gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR),
4201                                         &hdev->unexpected_error_interrupt);
4202         if (rc) {
4203                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4204                 goto free_tpc_irq;
4205         }
4206
4207         for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0;
4208                         user_irq_init_cnt < prop->user_interrupt_count;
4209                         i++, j++, user_irq_init_cnt++) {
4210
4211                 irq = pci_irq_vector(hdev->pdev, i);
4212                 rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4213                                                 hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4214                                                 gaudi2_irq_name(i), &hdev->user_interrupt[j]);
4215
4216                 if (rc) {
4217                         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4218                         goto free_user_irq;
4219                 }
4220         }
4221
4222         gaudi2->hw_cap_initialized |= HW_CAP_MSIX;
4223
4224         return 0;
4225
4226 free_user_irq:
4227         for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count;
4228                         i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) {
4229
4230                 irq = pci_irq_vector(hdev->pdev, i);
4231                 free_irq(irq, &hdev->user_interrupt[j]);
4232         }
4233         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4234         free_irq(irq, &hdev->unexpected_error_interrupt);
4235 free_tpc_irq:
4236         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4237         free_irq(irq, &hdev->tpc_interrupt);
4238 free_dec_irq:
4239         gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1);
4240 free_event_irq:
4241         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4242         free_irq(irq, cq);
4243
4244 free_completion_irq:
4245         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4246         free_irq(irq, cq);
4247
4248 free_irq_vectors:
4249         pci_free_irq_vectors(hdev->pdev);
4250
4251         return rc;
4252 }
4253
4254 static void gaudi2_sync_irqs(struct hl_device *hdev)
4255 {
4256         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4257         int i, j;
4258         int irq;
4259
4260         if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
4261                 return;
4262
4263         /* Wait for all pending IRQs to be finished */
4264         synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION));
4265
4266         for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) {
4267                 irq = pci_irq_vector(hdev->pdev, i);
4268                 synchronize_irq(irq);
4269         }
4270
4271         synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT));
4272         synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR));
4273
4274         for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;
4275                                                                                 i++, j++) {
4276                 irq = pci_irq_vector(hdev->pdev, i);
4277                 synchronize_irq(irq);
4278         }
4279
4280         synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));
4281 }
4282
4283 static void gaudi2_disable_msix(struct hl_device *hdev)
4284 {
4285         struct asic_fixed_properties *prop = &hdev->asic_prop;
4286         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4287         struct hl_cq *cq;
4288         int irq, i, j, k;
4289
4290         if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
4291                 return;
4292
4293         gaudi2_sync_irqs(hdev);
4294
4295         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4296         free_irq(irq, &hdev->event_queue);
4297
4298         gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
4299
4300         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4301         free_irq(irq, &hdev->tpc_interrupt);
4302
4303         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4304         free_irq(irq, &hdev->unexpected_error_interrupt);
4305
4306         for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0;
4307                         k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {
4308
4309                 irq = pci_irq_vector(hdev->pdev, i);
4310                 free_irq(irq, &hdev->user_interrupt[j]);
4311         }
4312
4313         irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4314         cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4315         free_irq(irq, cq);
4316
4317         pci_free_irq_vectors(hdev->pdev);
4318
4319         gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX;
4320 }
4321
4322 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id)
4323 {
4324         u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
4325         u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
4326         u32 timeout_usec, dec_id, dec_bit, offset, graceful;
4327         int rc;
4328
4329         if (hdev->pldm)
4330                 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
4331         else
4332                 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
4333
4334         for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
4335                 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
4336                 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4337                         continue;
4338
4339                 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET;
4340
4341                 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0);
4342
4343                 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
4344
4345                 /* Wait till all traffic from decoder stops
4346                  * before apply core reset.
4347                  */
4348                 rc = hl_poll_timeout(
4349                                 hdev,
4350                                 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset,
4351                                 graceful,
4352                                 (graceful & graceful_pend_mask),
4353                                 100,
4354                                 timeout_usec);
4355                 if (rc)
4356                         dev_err(hdev->dev,
4357                                 "Failed to stop traffic from DCORE%d Decoder %d\n",
4358                                 dcore_id, dec_id);
4359         }
4360 }
4361
4362 static void gaudi2_stop_pcie_dec(struct hl_device *hdev)
4363 {
4364         u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
4365         u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
4366         u32 timeout_usec, dec_id, dec_bit, offset, graceful;
4367         int rc;
4368
4369         if (hdev->pldm)
4370                 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
4371         else
4372                 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
4373
4374         for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
4375                 dec_bit = PCIE_DEC_SHIFT + dec_id;
4376                 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4377                         continue;
4378
4379                 offset = dec_id * PCIE_VDEC_OFFSET;
4380
4381                 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0);
4382
4383                 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
4384
4385                 /* Wait till all traffic from decoder stops
4386                  * before apply core reset.
4387                  */
4388                 rc = hl_poll_timeout(
4389                                 hdev,
4390                                 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset,
4391                                 graceful,
4392                                 (graceful & graceful_pend_mask),
4393                                 100,
4394                                 timeout_usec);
4395                 if (rc)
4396                         dev_err(hdev->dev,
4397                                 "Failed to stop traffic from PCIe Decoder %d\n",
4398                                 dec_id);
4399         }
4400 }
4401
4402 static void gaudi2_stop_dec(struct hl_device *hdev)
4403 {
4404         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4405         int dcore_id;
4406
4407         if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0)
4408                 return;
4409
4410         for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
4411                 gaudi2_stop_dcore_dec(hdev, dcore_id);
4412
4413         gaudi2_stop_pcie_dec(hdev);
4414 }
4415
4416 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4417 {
4418         u32 reg_base, reg_val;
4419
4420         reg_base = gaudi2_arc_blocks_bases[cpu_id];
4421         if (run_mode == HL_ENGINE_CORE_RUN)
4422                 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1);
4423         else
4424                 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1);
4425
4426         WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val);
4427 }
4428
4429 static void gaudi2_halt_arcs(struct hl_device *hdev)
4430 {
4431         u16 arc_id;
4432
4433         for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) {
4434                 if (gaudi2_is_arc_enabled(hdev, arc_id))
4435                         gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT);
4436         }
4437 }
4438
4439 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4440 {
4441         int rc;
4442         u32 reg_base, val, ack_mask, timeout_usec = 100000;
4443
4444         if (hdev->pldm)
4445                 timeout_usec *= 100;
4446
4447         reg_base = gaudi2_arc_blocks_bases[cpu_id];
4448         if (run_mode == HL_ENGINE_CORE_RUN)
4449                 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK;
4450         else
4451                 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK;
4452
4453         rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET,
4454                                 val, ((val & ack_mask) == ack_mask),
4455                                 1000, timeout_usec);
4456
4457         if (!rc) {
4458                 /* Clear */
4459                 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0);
4460                 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val);
4461         }
4462
4463         return rc;
4464 }
4465
4466 static void gaudi2_reset_arcs(struct hl_device *hdev)
4467 {
4468         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4469         u16 arc_id;
4470
4471         if (!gaudi2)
4472                 return;
4473
4474         for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++)
4475                 if (gaudi2_is_arc_enabled(hdev, arc_id))
4476                         gaudi2_clr_arc_id_cap(hdev, arc_id);
4477 }
4478
4479 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev)
4480 {
4481         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4482         u32 queue_id;
4483         int i;
4484
4485         if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
4486                 return;
4487
4488         queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
4489
4490         for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
4491                 if (!(hdev->nic_ports_mask & BIT(i)))
4492                         continue;
4493
4494                 gaudi2_qman_manual_flush_common(hdev, queue_id);
4495         }
4496 }
4497
4498 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids,
4499                                         u32 num_cores, u32 core_command)
4500 {
4501         int i, rc;
4502
4503         for (i = 0 ; i < num_cores ; i++) {
4504                 if (gaudi2_is_arc_enabled(hdev, core_ids[i]))
4505                         gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command);
4506         }
4507
4508         for (i = 0 ; i < num_cores ; i++) {
4509                 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) {
4510                         rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command);
4511
4512                         if (rc) {
4513                                 dev_err(hdev->dev, "failed to %s arc: %d\n",
4514                                         (core_command == HL_ENGINE_CORE_HALT) ?
4515                                         "HALT" : "RUN", core_ids[i]);
4516                                 return -1;
4517                         }
4518                 }
4519         }
4520
4521         return 0;
4522 }
4523
4524 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4525 {
4526         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4527         u32 reg_base, reg_addr, reg_val, tpc_id;
4528
4529         if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
4530                 return 0;
4531
4532         tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id];
4533         if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id)))
4534                 return 0;
4535
4536         reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id];
4537         reg_addr = reg_base + TPC_CFG_STALL_OFFSET;
4538         reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK,
4539                         (engine_command == HL_ENGINE_STALL) ? 1 : 0);
4540         WREG32(reg_addr, reg_val);
4541
4542         if (engine_command == HL_ENGINE_RESUME) {
4543                 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id];
4544                 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET;
4545                 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK);
4546         }
4547
4548         return 0;
4549 }
4550
4551 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4552 {
4553         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4554         u32 reg_base, reg_addr, reg_val, mme_id;
4555
4556         mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id];
4557         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id)))
4558                 return 0;
4559
4560         reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id];
4561         reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET;
4562         reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK,
4563                         (engine_command == HL_ENGINE_STALL) ? 1 : 0);
4564         WREG32(reg_addr, reg_val);
4565
4566         return 0;
4567 }
4568
4569 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4570 {
4571         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4572         u32 reg_base, reg_addr, reg_val, edma_id;
4573
4574         if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
4575                 return 0;
4576
4577         edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id];
4578         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id)))
4579                 return 0;
4580
4581         reg_base = gaudi2_dma_core_blocks_bases[edma_id];
4582         reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET;
4583         reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK,
4584                         (engine_command == HL_ENGINE_STALL) ? 1 : 0);
4585         WREG32(reg_addr, reg_val);
4586
4587         if (engine_command == HL_ENGINE_STALL) {
4588                 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) |
4589                                 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1);
4590                 WREG32(reg_addr, reg_val);
4591         }
4592
4593         return 0;
4594 }
4595
4596 static int gaudi2_set_engine_modes(struct hl_device *hdev,
4597                 u32 *engine_ids, u32 num_engines, u32 engine_command)
4598 {
4599         int i, rc;
4600
4601         for (i = 0 ; i < num_engines ; ++i) {
4602                 switch (engine_ids[i]) {
4603                 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5:
4604                 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5:
4605                 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5:
4606                 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5:
4607                         rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command);
4608                         if (rc)
4609                                 return rc;
4610
4611                         break;
4612                 case GAUDI2_DCORE0_ENGINE_ID_MME:
4613                 case GAUDI2_DCORE1_ENGINE_ID_MME:
4614                 case GAUDI2_DCORE2_ENGINE_ID_MME:
4615                 case GAUDI2_DCORE3_ENGINE_ID_MME:
4616                         rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command);
4617                         if (rc)
4618                                 return rc;
4619
4620                         break;
4621                 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1:
4622                 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1:
4623                 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1:
4624                 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1:
4625                         rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command);
4626                         if (rc)
4627                                 return rc;
4628
4629                         break;
4630                 default:
4631                         dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]);
4632                         return -EINVAL;
4633                 }
4634         }
4635
4636         return 0;
4637 }
4638
4639 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids,
4640                                         u32 num_engines, u32 engine_command)
4641 {
4642         switch (engine_command) {
4643         case HL_ENGINE_CORE_HALT:
4644         case HL_ENGINE_CORE_RUN:
4645                 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command);
4646
4647         case HL_ENGINE_STALL:
4648         case HL_ENGINE_RESUME:
4649                 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command);
4650
4651         default:
4652                 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command);
4653                 return -EINVAL;
4654         }
4655 }
4656
4657 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4658 {
4659         u32 wait_timeout_ms;
4660
4661         if (hdev->pldm)
4662                 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC;
4663         else
4664                 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC;
4665
4666         if (fw_reset)
4667                 goto skip_engines;
4668
4669         gaudi2_stop_dma_qmans(hdev);
4670         gaudi2_stop_mme_qmans(hdev);
4671         gaudi2_stop_tpc_qmans(hdev);
4672         gaudi2_stop_rot_qmans(hdev);
4673         gaudi2_stop_nic_qmans(hdev);
4674         msleep(wait_timeout_ms);
4675
4676         gaudi2_halt_arcs(hdev);
4677         gaudi2_dma_stall(hdev);
4678         gaudi2_mme_stall(hdev);
4679         gaudi2_tpc_stall(hdev);
4680         gaudi2_rotator_stall(hdev);
4681
4682         msleep(wait_timeout_ms);
4683
4684         gaudi2_stop_dec(hdev);
4685
4686         /*
4687          * in case of soft reset do a manual flush for QMANs (currently called
4688          * only for NIC QMANs
4689          */
4690         if (!hard_reset)
4691                 gaudi2_nic_qmans_manual_flush(hdev);
4692
4693         gaudi2_disable_dma_qmans(hdev);
4694         gaudi2_disable_mme_qmans(hdev);
4695         gaudi2_disable_tpc_qmans(hdev);
4696         gaudi2_disable_rot_qmans(hdev);
4697         gaudi2_disable_nic_qmans(hdev);
4698         gaudi2_disable_timestamp(hdev);
4699
4700 skip_engines:
4701         if (hard_reset) {
4702                 gaudi2_disable_msix(hdev);
4703                 return;
4704         }
4705
4706         gaudi2_sync_irqs(hdev);
4707 }
4708
4709 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev)
4710 {
4711         struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
4712
4713         pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
4714         pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0;
4715         pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1;
4716         pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;
4717         pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;
4718         pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC;
4719 }
4720
4721 static void gaudi2_init_firmware_loader(struct hl_device *hdev)
4722 {
4723         struct fw_load_mgr *fw_loader = &hdev->fw_loader;
4724         struct dynamic_fw_load_mgr *dynamic_loader;
4725         struct cpu_dyn_regs *dyn_regs;
4726
4727         /* fill common fields */
4728         fw_loader->fw_comp_loaded = FW_TYPE_NONE;
4729         fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE;
4730         fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE;
4731         fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC;
4732         fw_loader->skip_bmc = false;
4733         fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
4734         fw_loader->dram_bar_id = DRAM_BAR_ID;
4735         fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;
4736
4737         /* here we update initial values for few specific dynamic regs (as
4738          * before reading the first descriptor from FW those value has to be
4739          * hard-coded). in later stages of the protocol those values will be
4740          * updated automatically by reading the FW descriptor so data there
4741          * will always be up-to-date
4742          */
4743         dynamic_loader = &hdev->fw_loader.dynamic_loader;
4744         dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
4745         dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
4746         dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
4747         dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC;
4748 }
4749
4750 static int gaudi2_init_cpu(struct hl_device *hdev)
4751 {
4752         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4753         int rc;
4754
4755         if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
4756                 return 0;
4757
4758         if (gaudi2->hw_cap_initialized & HW_CAP_CPU)
4759                 return 0;
4760
4761         rc = hl_fw_init_cpu(hdev);
4762         if (rc)
4763                 return rc;
4764
4765         gaudi2->hw_cap_initialized |= HW_CAP_CPU;
4766
4767         return 0;
4768 }
4769
4770 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
4771 {
4772         struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
4773         struct asic_fixed_properties *prop = &hdev->asic_prop;
4774         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4775         struct cpu_dyn_regs *dyn_regs;
4776         struct hl_eq *eq;
4777         u32 status;
4778         int err;
4779
4780         if (!hdev->cpu_queues_enable)
4781                 return 0;
4782
4783         if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
4784                 return 0;
4785
4786         eq = &hdev->event_queue;
4787
4788         dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4789
4790         WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
4791         WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
4792
4793         WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
4794         WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
4795
4796         WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address));
4797         WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address));
4798
4799         WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
4800         WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
4801         WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
4802
4803         /* Used for EQ CI */
4804         WREG32(mmCPU_IF_EQ_RD_OFFS, 0);
4805
4806         WREG32(mmCPU_IF_PF_PQ_PI, 0);
4807
4808         WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
4809
4810         /* Let the ARC know we are ready as it is now handling those queues  */
4811
4812         WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
4813                 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
4814
4815         err = hl_poll_timeout(
4816                 hdev,
4817                 mmCPU_IF_QUEUE_INIT,
4818                 status,
4819                 (status == PQ_INIT_STATUS_READY_FOR_HOST),
4820                 1000,
4821                 cpu_timeout);
4822
4823         if (err) {
4824                 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n");
4825                 return -EIO;
4826         }
4827
4828         /* update FW application security bits */
4829         if (prop->fw_cpu_boot_dev_sts0_valid)
4830                 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
4831
4832         if (prop->fw_cpu_boot_dev_sts1_valid)
4833                 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
4834
4835         gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q;
4836         return 0;
4837 }
4838
4839 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base,
4840                                 u32 queue_id_base)
4841 {
4842         struct hl_hw_queue *q;
4843         u32 pq_id, pq_offset;
4844
4845         for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
4846                 q = &hdev->kernel_queues[queue_id_base + pq_id];
4847                 pq_offset = pq_id * 4;
4848
4849                 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset,
4850                                 lower_32_bits(q->bus_address));
4851                 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset,
4852                                 upper_32_bits(q->bus_address));
4853                 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH));
4854                 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0);
4855                 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0);
4856         }
4857 }
4858
4859 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base)
4860 {
4861         u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi;
4862
4863         mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4864         mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4865         so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4866         so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4867
4868         for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) {
4869                 cp_offset = cp_id * 4;
4870
4871                 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo);
4872                 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi);
4873                 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo);
4874                 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi);
4875         }
4876
4877         /* allow QMANs to accept work from ARC CQF */
4878         WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1));
4879 }
4880
4881 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base,
4882                                 u32 queue_id_base)
4883 {
4884         struct gaudi2_device *gaudi2 = hdev->asic_specific;
4885         u32 pq_id, pq_offset, so_base_lo, so_base_hi;
4886
4887         so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4888         so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4889
4890         for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
4891                 pq_offset = pq_id * 4;
4892
4893                 /* Configure QMAN HBW to scratchpad as it is not needed */
4894                 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset,
4895                                 lower_32_bits(gaudi2->scratchpad_bus_address));
4896                 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset,
4897                                 upper_32_bits(gaudi2->scratchpad_bus_address));
4898                 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset,
4899                                 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry)));
4900
4901                 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0);
4902                 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA);
4903                 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo);
4904                 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi);
4905         }
4906
4907         /* Enable QMAN H/W completion */
4908         WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
4909 }
4910
4911 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base)
4912 {
4913         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4914         u32 sp_reg_addr;
4915
4916         switch (queue_id_base) {
4917         case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3:
4918                 fallthrough;
4919         case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
4920                 fallthrough;
4921         case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
4922                 fallthrough;
4923         case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
4924                 fallthrough;
4925         case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
4926                 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);
4927                 break;
4928         case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
4929                 fallthrough;
4930         case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
4931                 fallthrough;
4932         case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
4933                 fallthrough;
4934         case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
4935                 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);
4936                 break;
4937         case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
4938                 fallthrough;
4939         case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
4940                 fallthrough;
4941         case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
4942                 fallthrough;
4943         case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
4944                 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);
4945                 break;
4946         case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3:
4947                 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl);
4948                 break;
4949         case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3:
4950                 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);
4951                 break;
4952         default:
4953                 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base);
4954                 return 0;
4955         }
4956
4957         return sp_reg_addr;
4958 }
4959
4960 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base,
4961                                         u32 queue_id_base)
4962 {
4963         u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset;
4964         int map_table_entry;
4965
4966         WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot);
4967
4968         irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base);
4969         WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset));
4970         WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset));
4971
4972         map_table_entry = gaudi2_qman_async_event_id[queue_id_base];
4973         WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET,
4974                 gaudi2_irq_map_table[map_table_entry].cpu_id);
4975
4976         WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK);
4977
4978         WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT);
4979         WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0);
4980         WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0);
4981
4982         /* Enable the QMAN channel.
4983          * PDMA QMAN configuration is different, as we do not allow user to
4984          * access some of the CPs.
4985          * PDMA0: CP2/3 are reserved for the ARC usage.
4986          * PDMA1: CP1/2/3 are reserved for the ARC usage.
4987          */
4988         if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0])
4989                 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE);
4990         else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0])
4991                 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE);
4992         else
4993                 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE);
4994 }
4995
4996 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base,
4997                 u32 queue_id_base)
4998 {
4999         u32 pq_id;
5000
5001         for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++)
5002                 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION;
5003
5004         gaudi2_init_qman_pq(hdev, reg_base, queue_id_base);
5005         gaudi2_init_qman_cp(hdev, reg_base);
5006         gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base);
5007         gaudi2_init_qman_common(hdev, reg_base, queue_id_base);
5008 }
5009
5010 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base,
5011                                 u32 dma_core_id, bool is_secure)
5012 {
5013         u32 prot, irq_handler_offset;
5014         struct cpu_dyn_regs *dyn_regs;
5015         int map_table_entry;
5016
5017         prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT;
5018         if (is_secure)
5019                 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT;
5020
5021         WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot);
5022
5023         dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5024         irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);
5025
5026         WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET,
5027                         lower_32_bits(CFG_BASE + irq_handler_offset));
5028
5029         WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET,
5030                         upper_32_bits(CFG_BASE + irq_handler_offset));
5031
5032         map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id];
5033         WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET,
5034                 gaudi2_irq_map_table[map_table_entry].cpu_id);
5035
5036         /* Enable the DMA channel */
5037         WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT);
5038 }
5039
5040 static void gaudi2_init_kdma(struct hl_device *hdev)
5041 {
5042         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5043         u32 reg_base;
5044
5045         if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA)
5046                 return;
5047
5048         reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA];
5049
5050         gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true);
5051
5052         gaudi2->hw_cap_initialized |= HW_CAP_KDMA;
5053 }
5054
5055 static void gaudi2_init_pdma(struct hl_device *hdev)
5056 {
5057         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5058         u32 reg_base;
5059
5060         if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK)
5061                 return;
5062
5063         reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0];
5064         gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false);
5065
5066         reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0];
5067         gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0);
5068
5069         reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1];
5070         gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false);
5071
5072         reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0];
5073         gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0);
5074
5075         gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK;
5076 }
5077
5078 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq)
5079 {
5080         u32 reg_base, base_edma_core_id, base_edma_qman_id;
5081
5082         base_edma_core_id = DMA_CORE_ID_EDMA0 + seq;
5083         base_edma_qman_id = edma_stream_base[seq];
5084
5085         reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id];
5086         gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false);
5087
5088         reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id];
5089         gaudi2_init_qman(hdev, reg_base, base_edma_qman_id);
5090 }
5091
5092 static void gaudi2_init_edma(struct hl_device *hdev)
5093 {
5094         struct asic_fixed_properties *prop = &hdev->asic_prop;
5095         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5096         int dcore, inst;
5097
5098         if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK)
5099                 return;
5100
5101         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
5102                 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
5103                         u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
5104
5105                         if (!(prop->edma_enabled_mask & BIT(seq)))
5106                                 continue;
5107
5108                         gaudi2_init_edma_instance(hdev, seq);
5109
5110                         gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq);
5111                 }
5112         }
5113 }
5114
5115 /*
5116  * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5117  * @hdev: pointer to habanalabs device structure.
5118  * @sob_id: sync object ID.
5119  * @first_mon_id: ID of first monitor out of 3 consecutive monitors.
5120  * @interrupt_id: interrupt ID.
5121  *
5122  * Some initiators cannot have HBW address in their completion address registers, and thus cannot
5123  * write directly to the HBW host memory of the virtual MSI-X doorbell.
5124  * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write.
5125  *
5126  * The mechanism in the sync manager block is composed of a master monitor with 3 messages.
5127  * In addition to the HBW write, the other 2 messages are for preparing the monitor to next
5128  * completion, by decrementing the sync object value and re-arming the monitor.
5129  */
5130 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id,
5131                                                         u32 first_mon_id, u32 interrupt_id)
5132 {
5133         u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config;
5134         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5135         u64 addr;
5136         u8 mask;
5137
5138         /* Reset the SOB value */
5139         sob_offset = sob_id * sizeof(u32);
5140         WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
5141
5142         /* Configure 3 monitors:
5143          * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor)
5144          * 2. Decrement SOB value by 1.
5145          * 3. Re-arm the master monitor.
5146          */
5147
5148         first_mon_offset = first_mon_id * sizeof(u32);
5149
5150         /* 2nd monitor: Decrement SOB value by 1 */
5151         mon_offset = first_mon_offset + sizeof(u32);
5152
5153         addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
5154         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5155         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5156
5157         payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */
5158                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) |
5159                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1);
5160         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5161
5162         /* 3rd monitor: Re-arm the master monitor */
5163         mon_offset = first_mon_offset + 2 * sizeof(u32);
5164
5165         addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset;
5166         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5167         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5168
5169         sob_group = sob_id / 8;
5170         mask = ~BIT(sob_id & 0x7);
5171         mode = 0; /* comparison mode is "greater than or equal to" */
5172         arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) |
5173                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) |
5174                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) |
5175                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1);
5176
5177         payload = arm;
5178         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5179
5180         /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */
5181         mon_offset = first_mon_offset;
5182
5183         config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */
5184         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config);
5185
5186         addr = gaudi2->virt_msix_db_dma_addr;
5187         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5188         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5189
5190         payload = interrupt_id;
5191         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5192
5193         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm);
5194 }
5195
5196 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev)
5197 {
5198         u32 decoder_id, sob_id, first_mon_id, interrupt_id;
5199         struct asic_fixed_properties *prop = &hdev->asic_prop;
5200
5201         /* Decoder normal/abnormal interrupts */
5202         for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) {
5203                 if (!(prop->decoder_enabled_mask & BIT(decoder_id)))
5204                         continue;
5205
5206                 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
5207                 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id;
5208                 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id;
5209                 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5210
5211                 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
5212                 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id;
5213                 interrupt_id += 1;
5214                 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5215         }
5216 }
5217
5218 static void gaudi2_init_sm(struct hl_device *hdev)
5219 {
5220         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5221         u64 cq_address;
5222         u32 reg_val;
5223         int i;
5224
5225         /* Enable HBW/LBW CQ for completion monitors */
5226         reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
5227         reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1);
5228
5229         for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++)
5230                 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
5231
5232         /* Enable only HBW CQ for KDMA completion monitor */
5233         reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
5234         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
5235
5236         /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */
5237         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr));
5238         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr));
5239         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION);
5240
5241         for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) {
5242                 cq_address =
5243                         hdev->completion_queue[i].bus_address;
5244
5245                 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i),
5246                                                         lower_32_bits(cq_address));
5247                 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i),
5248                                                         upper_32_bits(cq_address));
5249                 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i),
5250                                                         ilog2(HL_CQ_SIZE_IN_BYTES));
5251         }
5252
5253         /* Configure kernel ASID and MMU BP*/
5254         WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000);
5255         WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0);
5256
5257         /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */
5258         gaudi2_prepare_sm_for_virt_msix_db(hdev);
5259 }
5260
5261 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base)
5262 {
5263         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5264         u32 reg_val;
5265         int i;
5266
5267         reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0);
5268         reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1);
5269         reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1);
5270         reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1);
5271         reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1);
5272         reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1);
5273
5274         WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val);
5275         WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF);
5276
5277         for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) {
5278                 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i);
5279                 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]);
5280         }
5281 }
5282
5283 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
5284                                                         bool config_qman_only)
5285 {
5286         u32 queue_id_base, reg_base;
5287
5288         switch (dcore_id) {
5289         case 0:
5290                 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
5291                 break;
5292         case 1:
5293                 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
5294                 break;
5295         case 2:
5296                 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
5297                 break;
5298         case 3:
5299                 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
5300                 break;
5301         default:
5302                 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);
5303                 return;
5304         }
5305
5306         if (!config_qman_only) {
5307                 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id];
5308                 gaudi2_init_mme_acc(hdev, reg_base);
5309         }
5310
5311         reg_base = gaudi2_qm_blocks_bases[queue_id_base];
5312         gaudi2_init_qman(hdev, reg_base, queue_id_base);
5313 }
5314
5315 static void gaudi2_init_mme(struct hl_device *hdev)
5316 {
5317         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5318         int i;
5319
5320         if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK)
5321                 return;
5322
5323         for (i = 0 ; i < NUM_OF_DCORES ; i++) {
5324                 gaudi2_init_dcore_mme(hdev, i, false);
5325
5326                 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i);
5327         }
5328 }
5329
5330 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base)
5331 {
5332         /* Mask arithmetic and QM interrupts in TPC */
5333         WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE);
5334
5335         /* Set 16 cache lines */
5336         WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET,
5337                         2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT);
5338 }
5339
5340 struct gaudi2_tpc_init_cfg_data {
5341         enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES];
5342 };
5343
5344 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst,
5345                                         u32 offset, struct iterate_module_ctx *ctx)
5346 {
5347         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5348         struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data;
5349         u32 queue_id_base;
5350         u8 seq;
5351
5352         queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN);
5353
5354         if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1))
5355                 /* gets last sequence number */
5356                 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE;
5357         else
5358                 seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
5359
5360         gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset);
5361         gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base);
5362
5363         gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq);
5364 }
5365
5366 static void gaudi2_init_tpc(struct hl_device *hdev)
5367 {
5368         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5369         struct gaudi2_tpc_init_cfg_data init_cfg_data;
5370         struct iterate_module_ctx tpc_iter;
5371
5372         if (!hdev->asic_prop.tpc_enabled_mask)
5373                 return;
5374
5375         if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK)
5376                 return;
5377
5378         init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0;
5379         init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0;
5380         init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0;
5381         init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0;
5382         tpc_iter.fn = &gaudi2_init_tpc_config;
5383         tpc_iter.data = &init_cfg_data;
5384         gaudi2_iterate_tpcs(hdev, &tpc_iter);
5385 }
5386
5387 static void gaudi2_init_rotator(struct hl_device *hdev)
5388 {
5389         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5390         u32 i, reg_base, queue_id;
5391
5392         queue_id = GAUDI2_QUEUE_ID_ROT_0_0;
5393
5394         for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
5395                 reg_base = gaudi2_qm_blocks_bases[queue_id];
5396                 gaudi2_init_qman(hdev, reg_base, queue_id);
5397
5398                 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i);
5399         }
5400 }
5401
5402 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id)
5403 {
5404         u32 sob_id;
5405
5406         /* VCMD normal interrupt */
5407         sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
5408         WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR,
5409                         mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
5410         WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
5411
5412         /* VCMD abnormal interrupt */
5413         sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
5414         WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR,
5415                         mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
5416         WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
5417 }
5418
5419 static void gaudi2_init_dec(struct hl_device *hdev)
5420 {
5421         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5422         u32 dcore_id, dec_id, dec_bit;
5423         u64 base_addr;
5424
5425         if (!hdev->asic_prop.decoder_enabled_mask)
5426                 return;
5427
5428         if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK)
5429                 return;
5430
5431         for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
5432                 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
5433                         dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
5434
5435                         if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5436                                 continue;
5437
5438                         base_addr =  mmDCORE0_DEC0_CMD_BASE +
5439                                         BRDG_CTRL_BLOCK_OFFSET +
5440                                         dcore_id * DCORE_OFFSET +
5441                                         dec_id * DCORE_VDEC_OFFSET;
5442
5443                         gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5444
5445                         gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
5446                 }
5447
5448         for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) {
5449                 dec_bit = PCIE_DEC_SHIFT + dec_id;
5450                 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5451                         continue;
5452
5453                 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET +
5454                                 dec_id * DCORE_VDEC_OFFSET;
5455
5456                 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5457
5458                 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
5459         }
5460 }
5461
5462 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev,
5463                                         u32 stlb_base, u32 asid, u64 phys_addr)
5464 {
5465         u32 status, timeout_usec;
5466         int rc;
5467
5468         if (hdev->pldm || !hdev->pdev)
5469                 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
5470         else
5471                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
5472
5473         WREG32(stlb_base + STLB_ASID_OFFSET, asid);
5474         WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
5475         WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT);
5476         WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000);
5477
5478         rc = hl_poll_timeout(
5479                 hdev,
5480                 stlb_base + STLB_BUSY_OFFSET,
5481                 status,
5482                 !(status & 0x80000000),
5483                 1000,
5484                 timeout_usec);
5485
5486         if (rc) {
5487                 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid);
5488                 return rc;
5489         }
5490
5491         return 0;
5492 }
5493
5494 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base,
5495                                         u32 start_offset, u32 inv_start_val,
5496                                         u32 flags)
5497 {
5498         /* clear PMMU mem line cache (only needed in mmu range invalidation) */
5499         if (flags & MMU_OP_CLEAR_MEMCACHE)
5500                 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1);
5501
5502         if (flags & MMU_OP_SKIP_LOW_CACHE_INV)
5503                 return;
5504
5505         WREG32(stlb_base + start_offset, inv_start_val);
5506 }
5507
5508 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base,
5509                                                 struct gaudi2_cache_invld_params *inv_params)
5510 {
5511         u32 status, timeout_usec, start_offset;
5512         int rc;
5513
5514         timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC :
5515                                         GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
5516
5517         /* poll PMMU mem line cache (only needed in mmu range invalidation) */
5518         if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) {
5519                 rc = hl_poll_timeout(
5520                         hdev,
5521                         mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS,
5522                         status,
5523                         status & 0x1,
5524                         1000,
5525                         timeout_usec);
5526
5527                 if (rc)
5528                         return rc;
5529
5530                 /* Need to manually reset the status to 0 */
5531                 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0);
5532         }
5533
5534         /* Lower cache does not work with cache lines, hence we can skip its
5535          * invalidation upon map and invalidate only upon unmap
5536          */
5537         if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV)
5538                 return 0;
5539
5540         start_offset = inv_params->range_invalidation ?
5541                         STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET;
5542
5543         rc = hl_poll_timeout(
5544                 hdev,
5545                 stlb_base + start_offset,
5546                 status,
5547                 !(status & 0x1),
5548                 1000,
5549                 timeout_usec);
5550
5551         return rc;
5552 }
5553
5554 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id)
5555 {
5556         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5557         u32 hw_cap;
5558
5559         hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id);
5560
5561         if (gaudi2->hw_cap_initialized & hw_cap)
5562                 return true;
5563
5564         return false;
5565 }
5566
5567 /* this function shall be called only for HMMUs for which capability bit is set */
5568 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id)
5569 {
5570         u32 offset;
5571
5572         offset =  (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
5573         return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset);
5574 }
5575
5576 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base,
5577                                                 struct gaudi2_cache_invld_params *inv_params)
5578 {
5579         u32 start_offset;
5580
5581         if (inv_params->range_invalidation) {
5582                 /* Set the addresses range
5583                  * Note: that the start address we set in register, is not included in
5584                  * the range of the invalidation, by design.
5585                  * that's why we need to set lower address than the one we actually
5586                  * want to be included in the range invalidation.
5587                  */
5588                 u64 start = inv_params->start_va - 1;
5589
5590                 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET;
5591
5592                 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET,
5593                                 start >> MMU_RANGE_INV_VA_LSB_SHIFT);
5594
5595                 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET,
5596                                 start >> MMU_RANGE_INV_VA_MSB_SHIFT);
5597
5598                 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET,
5599                                 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT);
5600
5601                 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET,
5602                                 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT);
5603         } else {
5604                 start_offset = STLB_INV_ALL_START_OFFSET;
5605         }
5606
5607         gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset,
5608                                                 inv_params->inv_start_val, inv_params->flags);
5609 }
5610
5611 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev,
5612                                                 int dcore_id, int hmmu_id,
5613                                                 struct gaudi2_cache_invld_params *inv_params)
5614 {
5615         u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
5616
5617         gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params);
5618 }
5619
5620 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev,
5621                                                 int dcore_id, int hmmu_id,
5622                                                 struct gaudi2_cache_invld_params *inv_params)
5623 {
5624         u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
5625
5626         return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params);
5627 }
5628
5629 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev,
5630                                                 struct gaudi2_cache_invld_params *inv_params)
5631 {
5632         int dcore_id, hmmu_id;
5633
5634         /* first send all invalidation commands */
5635         for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
5636                 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
5637                         if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5638                                 continue;
5639
5640                         gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params);
5641                 }
5642         }
5643
5644         /* next, poll all invalidations status */
5645         for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
5646                 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
5647                         int rc;
5648
5649                         if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5650                                 continue;
5651
5652                         rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id,
5653                                                                                 inv_params);
5654                         if (rc)
5655                                 return rc;
5656                 }
5657         }
5658
5659         return 0;
5660 }
5661
5662 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
5663 {
5664         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5665         struct gaudi2_cache_invld_params invld_params;
5666         int rc = 0;
5667
5668         if (hdev->reset_info.hard_reset_pending)
5669                 return rc;
5670
5671         invld_params.range_invalidation = false;
5672         invld_params.inv_start_val = 1;
5673
5674         if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
5675                 invld_params.flags = flags;
5676                 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5677                 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5678                                                                                 &invld_params);
5679         } else if (flags & MMU_OP_PHYS_PACK) {
5680                 invld_params.flags = 0;
5681                 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5682         }
5683
5684         return rc;
5685 }
5686
5687 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
5688                                 u32 flags, u32 asid, u64 va, u64 size)
5689 {
5690         struct gaudi2_cache_invld_params invld_params = {0};
5691         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5692         u64 start_va, end_va;
5693         u32 inv_start_val;
5694         int rc = 0;
5695
5696         if (hdev->reset_info.hard_reset_pending)
5697                 return 0;
5698
5699         inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT |
5700                         1 << MMU_RANGE_INV_ASID_EN_SHIFT |
5701                         asid << MMU_RANGE_INV_ASID_SHIFT);
5702         start_va = va;
5703         end_va = start_va + size;
5704
5705         if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
5706                 /* As range invalidation does not support zero address we will
5707                  * do full invalidation in this case
5708                  */
5709                 if (start_va) {
5710                         invld_params.range_invalidation = true;
5711                         invld_params.start_va = start_va;
5712                         invld_params.end_va = end_va;
5713                         invld_params.inv_start_val = inv_start_val;
5714                         invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE;
5715                 } else {
5716                         invld_params.range_invalidation = false;
5717                         invld_params.inv_start_val = 1;
5718                         invld_params.flags = flags;
5719                 }
5720
5721
5722                 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5723                 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5724                                                                                 &invld_params);
5725                 if (rc)
5726                         return rc;
5727
5728         } else if (flags & MMU_OP_PHYS_PACK) {
5729                 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va);
5730                 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va);
5731                 invld_params.inv_start_val = inv_start_val;
5732                 invld_params.flags = flags;
5733                 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5734         }
5735
5736         return rc;
5737 }
5738
5739 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base)
5740 {
5741         struct asic_fixed_properties *prop = &hdev->asic_prop;
5742         u64 hop0_addr;
5743         u32 asid, max_asid = prop->max_asid;
5744         int rc;
5745
5746         /* it takes too much time to init all of the ASIDs on palladium */
5747         if (hdev->pldm)
5748                 max_asid = min((u32) 8, max_asid);
5749
5750         for (asid = 0 ; asid < max_asid ; asid++) {
5751                 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr;
5752                 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr);
5753                 if (rc) {
5754                         dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid);
5755                         return rc;
5756                 }
5757         }
5758
5759         return 0;
5760 }
5761
5762 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base)
5763 {
5764         u32 status, timeout_usec;
5765         int rc;
5766
5767         if (hdev->pldm || !hdev->pdev)
5768                 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
5769         else
5770                 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
5771
5772         WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1);
5773
5774         rc = hl_poll_timeout(
5775                 hdev,
5776                 stlb_base + STLB_SRAM_INIT_OFFSET,
5777                 status,
5778                 !status,
5779                 1000,
5780                 timeout_usec);
5781
5782         if (rc)
5783                 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n");
5784
5785         rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base);
5786         if (rc)
5787                 return rc;
5788
5789         WREG32(mmu_base + MMU_BYPASS_OFFSET, 0);
5790
5791         rc = hl_poll_timeout(
5792                 hdev,
5793                 stlb_base + STLB_INV_ALL_START_OFFSET,
5794                 status,
5795                 !status,
5796                 1000,
5797                 timeout_usec);
5798
5799         if (rc)
5800                 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n");
5801
5802         WREG32(mmu_base + MMU_ENABLE_OFFSET, 1);
5803
5804         return rc;
5805 }
5806
5807 static int gaudi2_pci_mmu_init(struct hl_device *hdev)
5808 {
5809         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5810         u32 mmu_base, stlb_base;
5811         int rc;
5812
5813         if (gaudi2->hw_cap_initialized & HW_CAP_PMMU)
5814                 return 0;
5815
5816         mmu_base = mmPMMU_HBW_MMU_BASE;
5817         stlb_base = mmPMMU_HBW_STLB_BASE;
5818
5819         RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5820                 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) |
5821                 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) |
5822                 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) |
5823                 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) |
5824                 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT),
5825                 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5826                 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5827                 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5828                 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5829                 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5830
5831         WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0);
5832
5833         if (PAGE_SIZE == SZ_64K) {
5834                 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */
5835                 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET,
5836                         FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) |
5837                         FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) |
5838                         FIELD_PREP(
5839                                 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK,
5840                                 1),
5841                         DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK |
5842                         DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK |
5843                         DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK);
5844         }
5845
5846         WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK);
5847
5848         rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5849         if (rc)
5850                 return rc;
5851
5852         gaudi2->hw_cap_initialized |= HW_CAP_PMMU;
5853
5854         return 0;
5855 }
5856
5857 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id,
5858                                 int hmmu_id)
5859 {
5860         struct asic_fixed_properties *prop = &hdev->asic_prop;
5861         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5862         u32 offset, mmu_base, stlb_base, hw_cap;
5863         u8 dmmu_seq;
5864         int rc;
5865
5866         dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id;
5867         hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq;
5868
5869         /*
5870          * return if DMMU is already initialized or if it's not out of
5871          * isolation (due to cluster binning)
5872          */
5873         if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq)))
5874                 return 0;
5875
5876         offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
5877         mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset;
5878         stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset;
5879
5880         RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */,
5881                         MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK);
5882
5883         RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5884                 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) |
5885                 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) |
5886                 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) |
5887                 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) |
5888                 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3),
5889                         DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5890                         DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5891                         DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5892                         DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5893                         DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5894
5895         RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1,
5896                         STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK);
5897
5898         WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK);
5899
5900         rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5901         if (rc)
5902                 return rc;
5903
5904         gaudi2->hw_cap_initialized |= hw_cap;
5905
5906         return 0;
5907 }
5908
5909 static int gaudi2_hbm_mmu_init(struct hl_device *hdev)
5910 {
5911         int rc, dcore_id, hmmu_id;
5912
5913         for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
5914                 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) {
5915                         rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id);
5916                         if (rc)
5917                                 return rc;
5918                 }
5919
5920         return 0;
5921 }
5922
5923 static int gaudi2_mmu_init(struct hl_device *hdev)
5924 {
5925         int rc;
5926
5927         rc = gaudi2_pci_mmu_init(hdev);
5928         if (rc)
5929                 return rc;
5930
5931         rc = gaudi2_hbm_mmu_init(hdev);
5932         if (rc)
5933                 return rc;
5934
5935         return 0;
5936 }
5937
5938 static int gaudi2_hw_init(struct hl_device *hdev)
5939 {
5940         struct gaudi2_device *gaudi2 = hdev->asic_specific;
5941         int rc;
5942
5943         /* Let's mark in the H/W that we have reached this point. We check
5944          * this value in the reset_before_init function to understand whether
5945          * we need to reset the chip before doing H/W init. This register is
5946          * cleared by the H/W upon H/W reset
5947          */
5948         WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
5949
5950         /* Perform read from the device to make sure device is up */
5951         RREG32(mmHW_STATE);
5952
5953         /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE.
5954          * So we set it here and if anyone tries to move it later to
5955          * a different address, there will be an error
5956          */
5957         if (hdev->asic_prop.iatu_done_by_fw)
5958                 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE;
5959
5960         /*
5961          * Before pushing u-boot/linux to device, need to set the hbm bar to
5962          * base address of dram
5963          */
5964         if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
5965                 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n");
5966                 return -EIO;
5967         }
5968
5969         rc = gaudi2_init_cpu(hdev);
5970         if (rc) {
5971                 dev_err(hdev->dev, "failed to initialize CPU\n");
5972                 return rc;
5973         }
5974
5975         gaudi2_init_scrambler_hbm(hdev);
5976         gaudi2_init_kdma(hdev);
5977
5978         rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC);
5979         if (rc) {
5980                 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc);
5981                 return rc;
5982         }
5983
5984         rc = gaudi2->cpucp_info_get(hdev);
5985         if (rc) {
5986                 dev_err(hdev->dev, "Failed to get cpucp info\n");
5987                 return rc;
5988         }
5989
5990         rc = gaudi2_mmu_init(hdev);
5991         if (rc)
5992                 return rc;
5993
5994         gaudi2_init_pdma(hdev);
5995         gaudi2_init_edma(hdev);
5996         gaudi2_init_sm(hdev);
5997         gaudi2_init_tpc(hdev);
5998         gaudi2_init_mme(hdev);
5999         gaudi2_init_rotator(hdev);
6000         gaudi2_init_dec(hdev);
6001         gaudi2_enable_timestamp(hdev);
6002
6003         rc = gaudi2_coresight_init(hdev);
6004         if (rc)
6005                 goto disable_queues;
6006
6007         rc = gaudi2_enable_msix(hdev);
6008         if (rc)
6009                 goto disable_queues;
6010
6011         /* Perform read from the device to flush all configuration */
6012         RREG32(mmHW_STATE);
6013
6014         return 0;
6015
6016 disable_queues:
6017         gaudi2_disable_dma_qmans(hdev);
6018         gaudi2_disable_mme_qmans(hdev);
6019         gaudi2_disable_tpc_qmans(hdev);
6020         gaudi2_disable_rot_qmans(hdev);
6021         gaudi2_disable_nic_qmans(hdev);
6022
6023         gaudi2_disable_timestamp(hdev);
6024
6025         return rc;
6026 }
6027
6028 /**
6029  * gaudi2_send_hard_reset_cmd - common function to handle reset
6030  *
6031  * @hdev: pointer to the habanalabs device structure
6032  *
6033  * This function handles the various possible scenarios for reset.
6034  * It considers if reset is handled by driver\FW and what FW components are loaded
6035  */
6036 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev)
6037 {
6038         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6039         bool heartbeat_reset, preboot_only, cpu_initialized = false;
6040         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6041         u32 cpu_boot_status;
6042
6043         preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU);
6044         heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT);
6045
6046         /*
6047          * Handle corner case where failure was at cpu management app load,
6048          * and driver didn't detect any failure while loading the FW,
6049          * then at such scenario driver will send only HALT_MACHINE
6050          * and no one will respond to this request since FW already back to preboot
6051          * and it cannot handle such cmd.
6052          * In this case next time the management app loads it'll check on events register
6053          * which will still have the halt indication, and will reboot the device.
6054          * The solution is to let preboot clear all relevant registers before next boot
6055          * once driver send COMMS_RST_DEV.
6056          */
6057         cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS);
6058
6059         if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) &&
6060                         (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL))
6061                 cpu_initialized = true;
6062
6063         /*
6064          * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways:
6065          * 1. FW reset: FW initiate the reset sequence
6066          * 2. driver reset: FW will start HALT sequence (the preparations for the
6067          *                  reset but not the reset itself as it is not implemented
6068          *                  on their part) and LKD will wait to let FW complete the
6069          *                  sequence before issuing the reset
6070          */
6071         if (!preboot_only && cpu_initialized) {
6072                 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq),
6073                         gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id);
6074
6075                 msleep(GAUDI2_CPU_RESET_WAIT_MSEC);
6076         }
6077
6078         /*
6079          * When working with preboot (without Linux/Boot fit) we can
6080          * communicate only using the COMMS commands to issue halt/reset.
6081          *
6082          * For the case in which we are working with Linux/Bootfit this is a hail-mary
6083          * attempt to revive the card in the small chance that the f/w has
6084          * experienced a watchdog event, which caused it to return back to preboot.
6085          * In that case, triggering reset through GIC won't help. We need to
6086          * trigger the reset as if Linux wasn't loaded.
6087          *
6088          * We do it only if the reset cause was HB, because that would be the
6089          * indication of such an event.
6090          *
6091          * In case watchdog hasn't expired but we still got HB, then this won't
6092          * do any damage.
6093          */
6094
6095         if (heartbeat_reset || preboot_only || !cpu_initialized) {
6096                 if (hdev->asic_prop.hard_reset_done_by_fw)
6097                         hl_fw_ask_hard_reset_without_linux(hdev);
6098                 else
6099                         hl_fw_ask_halt_machine_without_linux(hdev);
6100         }
6101 }
6102
6103 /**
6104  * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6105  *
6106  * @hdev: pointer to the habanalabs device structure
6107  *
6108  * This function executes hard reset based on if driver/FW should do the reset
6109  */
6110 static void gaudi2_execute_hard_reset(struct hl_device *hdev)
6111 {
6112         if (hdev->asic_prop.hard_reset_done_by_fw) {
6113                 gaudi2_send_hard_reset_cmd(hdev);
6114                 return;
6115         }
6116
6117         /* Set device to handle FLR by H/W as we will put the device
6118          * CPU to halt mode
6119          */
6120         WREG32(mmPCIE_AUX_FLR_CTRL,
6121                         (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK));
6122
6123         gaudi2_send_hard_reset_cmd(hdev);
6124
6125         WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1);
6126 }
6127
6128 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
6129 {
6130         int i, rc = 0;
6131         u32 reg_val;
6132
6133         for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
6134                 rc = hl_poll_timeout(
6135                         hdev,
6136                         mmCPU_RST_STATUS_TO_HOST,
6137                         reg_val,
6138                         reg_val == CPU_RST_STATUS_SOFT_RST_DONE,
6139                         1000,
6140                         poll_timeout_us);
6141
6142         if (rc)
6143                 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
6144                                 reg_val);
6145         return rc;
6146 }
6147
6148 /**
6149  * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6150  *
6151  * @hdev: pointer to the habanalabs device structure
6152  * @driver_performs_reset: true if driver should perform reset instead of f/w.
6153  * @poll_timeout_us: time to wait for response from f/w.
6154  *
6155  * This function executes soft reset based on if driver/FW should do the reset
6156  */
6157 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset,
6158                                                 u32 poll_timeout_us)
6159 {
6160         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6161         int rc = 0;
6162
6163         if (!driver_performs_reset) {
6164                 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) {
6165                         /* set SP to indicate reset request sent to FW */
6166                         if (dyn_regs->cpu_rst_status)
6167                                 WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA);
6168                         else
6169                                 WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA);
6170                         WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq),
6171                                 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id);
6172
6173                         /* wait for f/w response */
6174                         rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
6175                 } else {
6176                         rc = hl_fw_send_soft_reset(hdev);
6177                 }
6178                 return rc;
6179         }
6180
6181         /* Block access to engines, QMANs and SM during reset, these
6182          * RRs will be reconfigured after soft reset.
6183          * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset.
6184          */
6185         gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1,
6186                                         mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE);
6187
6188         gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2,
6189                                 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE,
6190                                 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE);
6191
6192         WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1);
6193         return 0;
6194 }
6195
6196 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us)
6197 {
6198         int i, rc = 0;
6199         u32 reg_val;
6200
6201         /* We poll the BTM done indication multiple times after reset due to
6202          * a HW errata 'GAUDI2_0300'
6203          */
6204         for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
6205                 rc = hl_poll_timeout(
6206                         hdev,
6207                         mmPSOC_GLOBAL_CONF_BTM_FSM,
6208                         reg_val,
6209                         reg_val == 0,
6210                         1000,
6211                         poll_timeout_us);
6212
6213         if (rc)
6214                 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);
6215 }
6216
6217 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
6218 {
6219         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6220         u32 poll_timeout_us, reset_sleep_ms;
6221         bool driver_performs_reset = false;
6222         int rc;
6223
6224         if (hdev->pldm) {
6225                 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC :
6226                                                 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC;
6227                 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC;
6228         } else {
6229                 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC;
6230                 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC;
6231         }
6232
6233         if (fw_reset)
6234                 goto skip_reset;
6235
6236         gaudi2_reset_arcs(hdev);
6237
6238         if (hard_reset) {
6239                 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;
6240                 gaudi2_execute_hard_reset(hdev);
6241         } else {
6242                 /*
6243                  * As we have to support also work with preboot only (which does not supports
6244                  * soft reset) we have to make sure that security is disabled before letting driver
6245                  * do the reset. user shall control the BFE flags to avoid asking soft reset in
6246                  * secured device with preboot only.
6247                  */
6248                 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&
6249                                                         !hdev->asic_prop.fw_security_enabled);
6250                 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us);
6251                 if (rc)
6252                         return rc;
6253         }
6254
6255 skip_reset:
6256         if (driver_performs_reset || hard_reset) {
6257                 /*
6258                  * Instead of waiting for BTM indication we should wait for preboot ready:
6259                  * Consider the below scenario:
6260                  * 1. FW update is being triggered
6261                  *        - setting the dirty bit
6262                  * 2. hard reset will be triggered due to the dirty bit
6263                  * 3. FW initiates the reset:
6264                  *        - dirty bit cleared
6265                  *        - BTM indication cleared
6266                  *        - preboot ready indication cleared
6267                  * 4. during hard reset:
6268                  *        - BTM indication will be set
6269                  *        - BIST test performed and another reset triggered
6270                  * 5. only after this reset the preboot will set the preboot ready
6271                  *
6272                  * when polling on BTM indication alone we can lose sync with FW while trying to
6273                  * communicate with FW that is during reset.
6274                  * to overcome this we will always wait to preboot ready indication
6275                  */
6276
6277                 /* without this sleep reset will not work */
6278                 msleep(reset_sleep_ms);
6279
6280                 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU)
6281                         hl_fw_wait_preboot_ready(hdev);
6282                 else
6283                         gaudi2_poll_btm_indication(hdev, poll_timeout_us);
6284         }
6285
6286         if (!gaudi2)
6287                 return 0;
6288
6289         gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK);
6290         gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK);
6291
6292         /*
6293          * Clear NIC capability mask in order for driver to re-configure
6294          * NIC QMANs. NIC ports will not be re-configured during soft
6295          * reset as we call gaudi2_nic_init only during hard reset
6296          */
6297         gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK);
6298
6299         if (hard_reset) {
6300                 gaudi2->hw_cap_initialized &=
6301                         ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK |
6302                         HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q |
6303                         HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK |
6304                         HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA |
6305                         HW_CAP_MME_MASK | HW_CAP_ROT_MASK);
6306
6307                 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat));
6308         } else {
6309                 gaudi2->hw_cap_initialized &=
6310                         ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET |
6311                         HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK |
6312                         HW_CAP_ROT_MASK);
6313         }
6314         return 0;
6315 }
6316
6317 static int gaudi2_suspend(struct hl_device *hdev)
6318 {
6319         int rc;
6320
6321         rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
6322         if (rc)
6323                 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
6324
6325         return rc;
6326 }
6327
6328 static int gaudi2_resume(struct hl_device *hdev)
6329 {
6330         return gaudi2_init_iatu(hdev);
6331 }
6332
6333 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
6334                 void *cpu_addr, dma_addr_t dma_addr, size_t size)
6335 {
6336         int rc;
6337
6338         vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
6339                         VM_DONTCOPY | VM_NORESERVE);
6340
6341 #ifdef _HAS_DMA_MMAP_COHERENT
6342
6343         rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size);
6344         if (rc)
6345                 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
6346
6347 #else
6348
6349         rc = remap_pfn_range(vma, vma->vm_start,
6350                                 virt_to_phys(cpu_addr) >> PAGE_SHIFT,
6351                                 size, vma->vm_page_prot);
6352         if (rc)
6353                 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
6354
6355 #endif
6356
6357         return rc;
6358 }
6359
6360 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id)
6361 {
6362         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6363         u64 hw_cap_mask = 0;
6364         u64 hw_tpc_cap_bit = 0;
6365         u64 hw_nic_cap_bit = 0;
6366         u64 hw_test_cap_bit = 0;
6367
6368         switch (hw_queue_id) {
6369         case GAUDI2_QUEUE_ID_PDMA_0_0:
6370         case GAUDI2_QUEUE_ID_PDMA_0_1:
6371         case GAUDI2_QUEUE_ID_PDMA_1_0:
6372                 hw_cap_mask = HW_CAP_PDMA_MASK;
6373                 break;
6374         case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
6375                 hw_test_cap_bit = HW_CAP_EDMA_SHIFT +
6376                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2);
6377                 break;
6378         case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
6379                 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE +
6380                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2);
6381                 break;
6382         case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
6383                 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE +
6384                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2);
6385                 break;
6386         case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
6387                 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE +
6388                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2);
6389                 break;
6390
6391         case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
6392                 hw_test_cap_bit = HW_CAP_MME_SHIFT;
6393                 break;
6394
6395         case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
6396                 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1;
6397                 break;
6398
6399         case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
6400                 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2;
6401                 break;
6402
6403         case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
6404                 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3;
6405                 break;
6406
6407         case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3:
6408                 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT +
6409                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2);
6410
6411                 /* special case where cap bit refers to the first queue id */
6412                 if (!hw_tpc_cap_bit)
6413                         return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0));
6414                 break;
6415
6416         case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
6417                 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE +
6418                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2);
6419                 break;
6420
6421         case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
6422                 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) +
6423                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2);
6424                 break;
6425
6426         case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
6427                 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) +
6428                         ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2);
6429                 break;
6430
6431         case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
6432                 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE);
6433                 break;
6434
6435         case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3:
6436                 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2);
6437                 break;
6438
6439         case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3:
6440                 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2);
6441
6442                 /* special case where cap bit refers to the first queue id */
6443                 if (!hw_nic_cap_bit)
6444                         return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0));
6445                 break;
6446
6447         case GAUDI2_QUEUE_ID_CPU_PQ:
6448                 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q);
6449
6450         default:
6451                 return false;
6452         }
6453
6454         if (hw_tpc_cap_bit)
6455                 return  !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit));
6456
6457         if (hw_nic_cap_bit)
6458                 return  !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit));
6459
6460         if (hw_test_cap_bit)
6461                 hw_cap_mask = BIT_ULL(hw_test_cap_bit);
6462
6463         return !!(gaudi2->hw_cap_initialized & hw_cap_mask);
6464 }
6465
6466 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id)
6467 {
6468         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6469
6470         switch (arc_id) {
6471         case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6472         case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6473                 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id));
6474
6475         case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6476                 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
6477
6478         case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6479                 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
6480
6481         default:
6482                 return false;
6483         }
6484 }
6485
6486 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6487 {
6488         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6489
6490         switch (arc_id) {
6491         case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6492         case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6493                 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id));
6494                 break;
6495
6496         case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6497                 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
6498                 break;
6499
6500         case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6501                 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
6502                 break;
6503
6504         default:
6505                 return;
6506         }
6507 }
6508
6509 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6510 {
6511         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6512
6513         switch (arc_id) {
6514         case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6515         case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6516                 gaudi2->active_hw_arc |= BIT_ULL(arc_id);
6517                 break;
6518
6519         case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6520                 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0);
6521                 break;
6522
6523         case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6524                 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0);
6525                 break;
6526
6527         default:
6528                 return;
6529         }
6530 }
6531
6532 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
6533 {
6534         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6535         u32 pq_offset, reg_base, db_reg_offset, db_value;
6536
6537         if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) {
6538                 /*
6539                  * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs.
6540                  * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ
6541                  * number.
6542                  */
6543                 pq_offset = (hw_queue_id & 0x3) * 4;
6544                 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
6545                 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset;
6546         } else {
6547                 db_reg_offset = mmCPU_IF_PF_PQ_PI;
6548         }
6549
6550         db_value = pi;
6551
6552         /* ring the doorbell */
6553         WREG32(db_reg_offset, db_value);
6554
6555         if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) {
6556                 /* make sure device CPU will read latest data from host */
6557                 mb();
6558                 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
6559                         gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
6560         }
6561 }
6562
6563 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
6564 {
6565         __le64 *pbd = (__le64 *) bd;
6566
6567         /* The QMANs are on the host memory so a simple copy suffice */
6568         pqe[0] = pbd[0];
6569         pqe[1] = pbd[1];
6570 }
6571
6572 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size,
6573                                 dma_addr_t *dma_handle, gfp_t flags)
6574 {
6575         return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);
6576 }
6577
6578 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size,
6579                                 void *cpu_addr, dma_addr_t dma_handle)
6580 {
6581         dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
6582 }
6583
6584 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
6585                                 u32 timeout, u64 *result)
6586 {
6587         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6588
6589         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) {
6590                 if (result)
6591                         *result = 0;
6592                 return 0;
6593         }
6594
6595         if (!timeout)
6596                 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC;
6597
6598         return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result);
6599 }
6600
6601 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size,
6602                                 gfp_t mem_flags, dma_addr_t *dma_handle)
6603 {
6604         if (size > GAUDI2_DMA_POOL_BLK_SIZE)
6605                 return NULL;
6606
6607         return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
6608 }
6609
6610 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr)
6611 {
6612         dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
6613 }
6614
6615 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
6616                                                 dma_addr_t *dma_handle)
6617 {
6618         return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
6619 }
6620
6621 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
6622 {
6623         hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
6624 }
6625
6626 static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len,
6627                                         enum dma_data_direction dir)
6628 {
6629         dma_addr_t dma_addr;
6630
6631         dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir);
6632         if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr)))
6633                 return 0;
6634
6635         return dma_addr;
6636 }
6637
6638 static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len,
6639                                         enum dma_data_direction dir)
6640 {
6641         dma_unmap_single(&hdev->pdev->dev, addr, len, dir);
6642 }
6643
6644 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser)
6645 {
6646         struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
6647         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6648
6649         if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) {
6650                 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);
6651                 return -EINVAL;
6652         }
6653
6654         /* Just check if CB address is valid */
6655
6656         if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6657                                         parser->user_cb_size,
6658                                         asic_prop->sram_user_base_address,
6659                                         asic_prop->sram_end_address))
6660                 return 0;
6661
6662         if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6663                                         parser->user_cb_size,
6664                                         asic_prop->dram_user_base_address,
6665                                         asic_prop->dram_end_address))
6666                 return 0;
6667
6668         if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) &&
6669                 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6670                                                 parser->user_cb_size,
6671                                                 asic_prop->dmmu.start_addr,
6672                                                 asic_prop->dmmu.end_addr))
6673                 return 0;
6674
6675         if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) {
6676                 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6677                                         parser->user_cb_size,
6678                                         asic_prop->pmmu.start_addr,
6679                                         asic_prop->pmmu.end_addr) ||
6680                         hl_mem_area_inside_range(
6681                                         (u64) (uintptr_t) parser->user_cb,
6682                                         parser->user_cb_size,
6683                                         asic_prop->pmmu_huge.start_addr,
6684                                         asic_prop->pmmu_huge.end_addr))
6685                         return 0;
6686
6687         } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) {
6688                 if (!hdev->pdev)
6689                         return 0;
6690
6691                 if (!device_iommu_mapped(&hdev->pdev->dev))
6692                         return 0;
6693         }
6694
6695         dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n",
6696                 parser->user_cb, parser->user_cb_size);
6697
6698         return -EFAULT;
6699 }
6700
6701 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
6702 {
6703         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6704
6705         if (!parser->is_kernel_allocated_cb)
6706                 return gaudi2_validate_cb_address(hdev, parser);
6707
6708         if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
6709                 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n");
6710                 return -EINVAL;
6711         }
6712
6713         return 0;
6714 }
6715
6716 static int gaudi2_send_heartbeat(struct hl_device *hdev)
6717 {
6718         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6719
6720         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6721                 return 0;
6722
6723         return hl_fw_send_heartbeat(hdev);
6724 }
6725
6726 /* This is an internal helper function, used to update the KDMA mmu props.
6727  * Should be called with a proper kdma lock.
6728  */
6729 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev,
6730                                            bool mmu_bypass, u32 asid)
6731 {
6732         u32 rw_asid, rw_mmu_bp;
6733
6734         rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
6735                       (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
6736
6737         rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) |
6738                         (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT);
6739
6740         WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid);
6741         WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp);
6742 }
6743
6744 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id,
6745                                                 u32 mon_payload, u32 sync_value)
6746 {
6747         u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm;
6748         u8 mask;
6749
6750         sob_offset = sob_id * 4;
6751         mon_offset = mon_id * 4;
6752
6753         /* Reset the SOB value */
6754         WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
6755
6756         /* Configure this address with CQ_ID 0 because CQ_EN is set */
6757         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id);
6758
6759         /* Configure this address with CS index because CQ_EN is set */
6760         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload);
6761
6762         sync_group_id = sob_id / 8;
6763         mask = ~(1 << (sob_id & 0x7));
6764         mode = 1; /* comparison mode is "equal to" */
6765
6766         mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value);
6767         mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode);
6768         mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask);
6769         mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id);
6770         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm);
6771 }
6772
6773 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */
6774 static int gaudi2_send_job_to_kdma(struct hl_device *hdev,
6775                                         u64 src_addr, u64 dst_addr,
6776                                         u32 size, bool is_memset)
6777 {
6778         u32 comp_val, commit_mask, *polling_addr, timeout, status = 0;
6779         struct hl_cq_entry *cq_base;
6780         struct hl_cq *cq;
6781         u64 comp_addr;
6782         int rc;
6783
6784         gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION,
6785                                 GAUDI2_RESERVED_MON_KDMA_COMPLETION,
6786                                 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1);
6787
6788         comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 +
6789                         (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32));
6790
6791         comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
6792                         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
6793
6794         WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr));
6795         WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr));
6796         WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr));
6797         WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr));
6798         WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr));
6799         WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr));
6800         WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val);
6801         WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size);
6802
6803         commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |
6804                                 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);
6805
6806         if (is_memset)
6807                 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1);
6808
6809         WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask);
6810
6811         /* Wait for completion */
6812         cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION];
6813         cq_base = cq->kernel_address;
6814         polling_addr = (u32 *)&cq_base[cq->ci];
6815
6816         if (hdev->pldm)
6817                 /* for each 1MB 20 second of timeout */
6818                 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20;
6819         else
6820                 timeout = KDMA_TIMEOUT_USEC;
6821
6822         /* Polling */
6823         rc = hl_poll_timeout_memory(
6824                         hdev,
6825                         polling_addr,
6826                         status,
6827                         (status == 1),
6828                         1000,
6829                         timeout,
6830                         true);
6831
6832         *polling_addr = 0;
6833
6834         if (rc) {
6835                 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n");
6836                 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT);
6837                 return rc;
6838         }
6839
6840         cq->ci = hl_cq_inc_ptr(cq->ci);
6841
6842         return 0;
6843 }
6844
6845 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val)
6846 {
6847         u32 i;
6848
6849         for (i = 0 ; i < size ; i += sizeof(u32))
6850                 WREG32(addr + i, val);
6851 }
6852
6853 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable)
6854 {
6855         u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
6856
6857         if (enable) {
6858                 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE);
6859                 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0);
6860         } else {
6861                 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED);
6862                 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
6863         }
6864 }
6865
6866 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id)
6867 {
6868         return hdev->asic_prop.first_available_user_sob[0] +
6869                                 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0;
6870 }
6871
6872 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id)
6873 {
6874         u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6875         u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
6876
6877         /* Reset the SOB value */
6878         WREG32(sob_addr, 0);
6879 }
6880
6881 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val,
6882                                             struct gaudi2_queues_test_info *msg_info)
6883 {
6884         u32 sob_offset =  gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6885         u32 tmp, sob_base = 1;
6886         struct packet_msg_short *msg_short_pkt = msg_info->kern_addr;
6887         size_t pkt_size = sizeof(struct packet_msg_short);
6888         int rc;
6889
6890         tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) |
6891                 (1 << GAUDI2_PKT_CTL_EB_SHIFT) |
6892                 (1 << GAUDI2_PKT_CTL_MB_SHIFT) |
6893                 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) |
6894                 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT);
6895
6896         msg_short_pkt->value = cpu_to_le32(sob_val);
6897         msg_short_pkt->ctl = cpu_to_le32(tmp);
6898
6899         rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr);
6900         if (rc)
6901                 dev_err(hdev->dev,
6902                         "Failed to send msg_short packet to H/W queue %d\n", hw_queue_id);
6903
6904         return rc;
6905 }
6906
6907 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val)
6908 {
6909         u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6910         u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
6911         u32 timeout_usec, tmp;
6912         int rc;
6913
6914         if (hdev->pldm)
6915                 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC;
6916         else
6917                 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC;
6918
6919         rc = hl_poll_timeout(
6920                         hdev,
6921                         sob_addr,
6922                         tmp,
6923                         (tmp == sob_val),
6924                         1000,
6925                         timeout_usec);
6926
6927         if (rc == -ETIMEDOUT) {
6928                 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n",
6929                         hw_queue_id, tmp);
6930                 rc = -EIO;
6931         }
6932
6933         return rc;
6934 }
6935
6936 static int gaudi2_test_cpu_queue(struct hl_device *hdev)
6937 {
6938         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6939
6940         /*
6941          * check capability here as send_cpu_message() won't update the result
6942          * value if no capability
6943          */
6944         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6945                 return 0;
6946
6947         return hl_fw_test_cpu_queue(hdev);
6948 }
6949
6950 static int gaudi2_test_queues(struct hl_device *hdev)
6951 {
6952         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6953         struct gaudi2_queues_test_info *msg_info;
6954         u32 sob_val = 0x5a5a;
6955         int i, rc;
6956
6957         /* send test message on all enabled Qs */
6958         for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
6959                 if (!gaudi2_is_queue_enabled(hdev, i))
6960                         continue;
6961
6962                 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0];
6963                 gaudi2_qman_set_test_mode(hdev, i, true);
6964                 gaudi2_test_queue_clear(hdev, i);
6965                 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info);
6966                 if (rc)
6967                         goto done;
6968         }
6969
6970         rc = gaudi2_test_cpu_queue(hdev);
6971         if (rc)
6972                 goto done;
6973
6974         /* verify that all messages were processed */
6975         for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
6976                 if (!gaudi2_is_queue_enabled(hdev, i))
6977                         continue;
6978
6979                 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val);
6980                 if (rc)
6981                         /* chip is not usable, no need for cleanups, just bail-out with error */
6982                         goto done;
6983
6984                 gaudi2_test_queue_clear(hdev, i);
6985                 gaudi2_qman_set_test_mode(hdev, i, false);
6986         }
6987
6988 done:
6989         return rc;
6990 }
6991
6992 static int gaudi2_compute_reset_late_init(struct hl_device *hdev)
6993 {
6994         struct gaudi2_device *gaudi2 = hdev->asic_specific;
6995         size_t irq_arr_size;
6996         int rc;
6997
6998         gaudi2_init_arcs(hdev);
6999
7000         rc = gaudi2_scrub_arcs_dccm(hdev);
7001         if (rc) {
7002                 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
7003                 return rc;
7004         }
7005
7006         gaudi2_init_security(hdev);
7007
7008         /* Unmask all IRQs since some could have been received during the soft reset */
7009         irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]);
7010         return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);
7011 }
7012
7013 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7014                 struct engines_data *e)
7015 {
7016         u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
7017         struct asic_fixed_properties *prop = &hdev->asic_prop;
7018         unsigned long *mask = (unsigned long *) mask_arr;
7019         const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n";
7020         bool is_idle = true, is_eng_idle;
7021         int engine_idx, i, j;
7022         u64 offset;
7023
7024         if (e)
7025                 hl_engine_data_sprintf(e,
7026                         "\nCORE  EDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
7027                         "----  ----  -------  ------------  -------------  -------------\n");
7028
7029         for (i = 0; i < NUM_OF_DCORES; i++) {
7030                 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) {
7031                         int seq = i * NUM_OF_EDMA_PER_DCORE + j;
7032
7033                         if (!(prop->edma_enabled_mask & BIT(seq)))
7034                                 continue;
7035
7036                         engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 +
7037                                         i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
7038                         offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET;
7039
7040                         dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset);
7041                         dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset);
7042
7043                         qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset);
7044                         qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset);
7045                         qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset);
7046
7047                         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7048                                         IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
7049                         is_idle &= is_eng_idle;
7050
7051                         if (mask && !is_eng_idle)
7052                                 set_bit(engine_idx, mask);
7053
7054                         if (e)
7055                                 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N",
7056                                                         qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
7057                 }
7058         }
7059
7060         return is_idle;
7061 }
7062
7063 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7064                 struct engines_data *e)
7065 {
7066         u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
7067         unsigned long *mask = (unsigned long *) mask_arr;
7068         const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n";
7069         bool is_idle = true, is_eng_idle;
7070         int engine_idx, i;
7071         u64 offset;
7072
7073         if (e)
7074                 hl_engine_data_sprintf(e,
7075                                         "\nPDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
7076                                         "----  -------  ------------  -------------  -------------\n");
7077
7078         for (i = 0 ; i < NUM_OF_PDMA ; i++) {
7079                 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i;
7080                 offset = i * PDMA_OFFSET;
7081                 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset);
7082                 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset);
7083
7084                 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset);
7085                 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset);
7086                 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset);
7087
7088                 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7089                                 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
7090                 is_idle &= is_eng_idle;
7091
7092                 if (mask && !is_eng_idle)
7093                         set_bit(engine_idx, mask);
7094
7095                 if (e)
7096                         hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N",
7097                                                 qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
7098         }
7099
7100         return is_idle;
7101 }
7102
7103 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7104                 struct engines_data *e)
7105 {
7106         unsigned long *mask = (unsigned long *) mask_arr;
7107         const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";
7108         u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7109         bool is_idle = true, is_eng_idle;
7110         int engine_idx, i;
7111         u64 offset = 0;
7112
7113         /* NIC, twelve macros in Full chip */
7114         if (e && hdev->nic_ports_mask)
7115                 hl_engine_data_sprintf(e,
7116                                         "\nNIC  is_idle  QM_GLBL_STS0  QM_CGM_STS\n"
7117                                         "---  -------  ------------  ----------\n");
7118
7119         for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {
7120                 if (!(i & 1))
7121                         offset = i / 2 * NIC_OFFSET;
7122                 else
7123                         offset += NIC_QM_OFFSET;
7124
7125                 if (!(hdev->nic_ports_mask & BIT(i)))
7126                         continue;
7127
7128                 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i;
7129
7130
7131                 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset);
7132                 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset);
7133                 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset);
7134
7135                 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7136                 is_idle &= is_eng_idle;
7137
7138                 if (mask && !is_eng_idle)
7139                         set_bit(engine_idx, mask);
7140
7141                 if (e)
7142                         hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N",
7143                                                 qm_glbl_sts0, qm_cgm_sts);
7144         }
7145
7146         return is_idle;
7147 }
7148
7149 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7150                 struct engines_data *e)
7151 {
7152         u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts;
7153         unsigned long *mask = (unsigned long *) mask_arr;
7154         const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";
7155         bool is_idle = true, is_eng_idle;
7156         int engine_idx, i;
7157         u64 offset;
7158
7159         if (e)
7160                 hl_engine_data_sprintf(e,
7161                                         "\nMME  Stub  is_idle  QM_GLBL_STS0  MME_ARCH_STATUS\n"
7162                                         "---  ----  -------  ------------  ---------------\n");
7163         /* MME, one per Dcore */
7164         for (i = 0 ; i < NUM_OF_DCORES ; i++) {
7165                 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET;
7166                 offset = i * DCORE_OFFSET;
7167
7168                 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset);
7169                 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset);
7170                 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset);
7171
7172                 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7173                 is_idle &= is_eng_idle;
7174
7175                 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset);
7176                 is_eng_idle &= IS_MME_IDLE(mme_arch_sts);
7177                 is_idle &= is_eng_idle;
7178
7179                 if (e)
7180                         hl_engine_data_sprintf(e, mme_fmt, i, "N",
7181                                 is_eng_idle ? "Y" : "N",
7182                                 qm_glbl_sts0,
7183                                 mme_arch_sts);
7184
7185                 if (mask && !is_eng_idle)
7186                         set_bit(engine_idx, mask);
7187         }
7188
7189         return is_idle;
7190 }
7191
7192 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
7193                                         struct iterate_module_ctx *ctx)
7194 {
7195         struct gaudi2_tpc_idle_data *idle_data = ctx->data;
7196         u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7197         bool is_eng_idle;
7198         int engine_idx;
7199
7200         if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))
7201                 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
7202         else
7203                 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 +
7204                                 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst;
7205
7206         tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset);
7207         qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset);
7208         qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset);
7209         qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset);
7210
7211         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7212                                                 IS_TPC_IDLE(tpc_cfg_sts);
7213         *(idle_data->is_idle) &= is_eng_idle;
7214
7215         if (idle_data->mask && !is_eng_idle)
7216                 set_bit(engine_idx, idle_data->mask);
7217
7218         if (idle_data->e)
7219                 hl_engine_data_sprintf(idle_data->e,
7220                                         idle_data->tpc_fmt, dcore, inst,
7221                                         is_eng_idle ? "Y" : "N",
7222                                         qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
7223 }
7224
7225 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7226                 struct engines_data *e)
7227 {
7228         struct asic_fixed_properties *prop = &hdev->asic_prop;
7229         unsigned long *mask = (unsigned long *) mask_arr;
7230         bool is_idle = true;
7231
7232         struct gaudi2_tpc_idle_data tpc_idle_data = {
7233                 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",
7234                 .e = e,
7235                 .mask = mask,
7236                 .is_idle = &is_idle,
7237         };
7238         struct iterate_module_ctx tpc_iter = {
7239                 .fn = &gaudi2_is_tpc_engine_idle,
7240                 .data = &tpc_idle_data,
7241         };
7242
7243         if (e && prop->tpc_enabled_mask)
7244                 hl_engine_data_sprintf(e,
7245                         "\nCORE  TPC  is_idle  QM_GLBL_STS0  QM_CGM_STS  STATUS\n"
7246                         "----  ---  -------  ------------  ----------  ------\n");
7247
7248         gaudi2_iterate_tpcs(hdev, &tpc_iter);
7249
7250         return *tpc_idle_data.is_idle;
7251 }
7252
7253 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7254                 struct engines_data *e)
7255 {
7256         struct asic_fixed_properties *prop = &hdev->asic_prop;
7257         unsigned long *mask = (unsigned long *) mask_arr;
7258         const char *pcie_dec_fmt = "%-10d%-9s%#x\n";
7259         const char *dec_fmt = "%-6d%-5d%-9s%#x\n";
7260         bool is_idle = true, is_eng_idle;
7261         u32 dec_swreg15, dec_enabled_bit;
7262         int engine_idx, i, j;
7263         u64 offset;
7264
7265         /* Decoders, two each Dcore and two shared PCIe decoders */
7266         if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK)))
7267                 hl_engine_data_sprintf(e,
7268                         "\nCORE  DEC  is_idle  VSI_CMD_SWREG15\n"
7269                         "----  ---  -------  ---------------\n");
7270
7271         for (i = 0 ; i < NUM_OF_DCORES ; i++) {
7272                 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) {
7273                         dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j);
7274                         if (!(prop->decoder_enabled_mask & dec_enabled_bit))
7275                                 continue;
7276
7277                         engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 +
7278                                         i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
7279                         offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET;
7280
7281                         dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset);
7282                         is_eng_idle = IS_DEC_IDLE(dec_swreg15);
7283                         is_idle &= is_eng_idle;
7284
7285                         if (mask && !is_eng_idle)
7286                                 set_bit(engine_idx, mask);
7287
7288                         if (e)
7289                                 hl_engine_data_sprintf(e, dec_fmt, i, j,
7290                                                         is_eng_idle ? "Y" : "N", dec_swreg15);
7291                 }
7292         }
7293
7294         if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK))
7295                 hl_engine_data_sprintf(e,
7296                         "\nPCIe DEC  is_idle  VSI_CMD_SWREG15\n"
7297                         "--------  -------  ---------------\n");
7298
7299         /* Check shared(PCIe) decoders */
7300         for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) {
7301                 dec_enabled_bit = PCIE_DEC_SHIFT + i;
7302                 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit)))
7303                         continue;
7304
7305                 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i;
7306                 offset = i * DCORE_DEC_OFFSET;
7307                 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset);
7308                 is_eng_idle = IS_DEC_IDLE(dec_swreg15);
7309                 is_idle &= is_eng_idle;
7310
7311                 if (mask && !is_eng_idle)
7312                         set_bit(engine_idx, mask);
7313
7314                 if (e)
7315                         hl_engine_data_sprintf(e, pcie_dec_fmt, i,
7316                                                 is_eng_idle ? "Y" : "N", dec_swreg15);
7317         }
7318
7319         return is_idle;
7320 }
7321
7322 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7323                 struct engines_data *e)
7324 {
7325         const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n";
7326         unsigned long *mask = (unsigned long *) mask_arr;
7327         u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7328         bool is_idle = true, is_eng_idle;
7329         int engine_idx, i;
7330         u64 offset;
7331
7332         if (e)
7333                 hl_engine_data_sprintf(e,
7334                         "\nCORE  ROT  is_idle  QM_GLBL_STS0  QM_GLBL_STS1  QM_CGM_STS\n"
7335                         "----  ---  -------  ------------  ------------  ----------\n");
7336
7337         for (i = 0 ; i < NUM_OF_ROT ; i++) {
7338                 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i;
7339
7340                 offset = i * ROT_OFFSET;
7341
7342                 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset);
7343                 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset);
7344                 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset);
7345
7346                 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7347                 is_idle &= is_eng_idle;
7348
7349                 if (mask && !is_eng_idle)
7350                         set_bit(engine_idx, mask);
7351
7352                 if (e)
7353                         hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N",
7354                                                 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7355         }
7356
7357         return is_idle;
7358 }
7359
7360 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7361                                         struct engines_data *e)
7362 {
7363         bool is_idle = true;
7364
7365         is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e);
7366         is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e);
7367         is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e);
7368         is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e);
7369         is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e);
7370         is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e);
7371         is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e);
7372
7373         return is_idle;
7374 }
7375
7376 static void gaudi2_hw_queues_lock(struct hl_device *hdev)
7377         __acquires(&gaudi2->hw_queues_lock)
7378 {
7379         struct gaudi2_device *gaudi2 = hdev->asic_specific;
7380
7381         spin_lock(&gaudi2->hw_queues_lock);
7382 }
7383
7384 static void gaudi2_hw_queues_unlock(struct hl_device *hdev)
7385         __releases(&gaudi2->hw_queues_lock)
7386 {
7387         struct gaudi2_device *gaudi2 = hdev->asic_specific;
7388
7389         spin_unlock(&gaudi2->hw_queues_lock);
7390 }
7391
7392 static u32 gaudi2_get_pci_id(struct hl_device *hdev)
7393 {
7394         return hdev->pdev->device;
7395 }
7396
7397 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
7398 {
7399         struct gaudi2_device *gaudi2 = hdev->asic_specific;
7400
7401         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
7402                 return 0;
7403
7404         return hl_fw_get_eeprom_data(hdev, data, max_size);
7405 }
7406
7407 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val)
7408 {
7409         WREG32(mmCPU_IF_EQ_RD_OFFS, val);
7410 }
7411
7412 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
7413 {
7414         struct gaudi2_device *gaudi2 = hdev->asic_specific;
7415
7416         if (aggregate) {
7417                 *size = (u32) sizeof(gaudi2->events_stat_aggregate);
7418                 return gaudi2->events_stat_aggregate;
7419         }
7420
7421         *size = (u32) sizeof(gaudi2->events_stat);
7422         return gaudi2->events_stat;
7423 }
7424
7425 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id,
7426                                 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
7427 {
7428         u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) *
7429                         dcore_vdec_id + DCORE_OFFSET * dcore_id;
7430
7431         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
7432         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
7433
7434         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
7435         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
7436
7437         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
7438         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
7439
7440         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
7441         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
7442
7443         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
7444         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
7445 }
7446
7447 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid)
7448 {
7449         u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
7450                         (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
7451         struct asic_fixed_properties *prop = &hdev->asic_prop;
7452         u32 dcore_offset = dcore_id * DCORE_OFFSET;
7453         u32 vdec_id, i, ports_offset, reg_val;
7454         u8 edma_seq_base;
7455
7456         /* EDMA */
7457         edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE;
7458         if (prop->edma_enabled_mask & BIT(edma_seq_base)) {
7459                 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7460                 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7461                 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
7462                 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
7463         }
7464
7465         if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) {
7466                 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7467                 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7468                 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
7469                 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
7470         }
7471
7472         /* Sync Mngr */
7473         WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid);
7474         /*
7475          * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID
7476          * for any access type
7477          */
7478         if (dcore_id > 0) {
7479                 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) |
7480                           (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT);
7481                 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val);
7482                 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0);
7483         }
7484
7485         WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0);
7486         WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid);
7487
7488         for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) {
7489                 ports_offset = i * DCORE_MME_SBTE_OFFSET;
7490                 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP +
7491                                 dcore_offset + ports_offset, 0);
7492                 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID +
7493                                 dcore_offset + ports_offset, rw_asid);
7494         }
7495
7496         for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) {
7497                 ports_offset = i * DCORE_MME_WB_OFFSET;
7498                 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP +
7499                                 dcore_offset + ports_offset, 0);
7500                 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID +
7501                                 dcore_offset + ports_offset, rw_asid);
7502         }
7503
7504         WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7505         WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7506
7507         /*
7508          * Decoders
7509          */
7510         for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) {
7511                 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id))
7512                         gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0);
7513         }
7514 }
7515
7516 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev,
7517                                 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
7518 {
7519         u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id;
7520
7521         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
7522         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
7523
7524         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
7525         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
7526
7527         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
7528         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
7529
7530         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
7531         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
7532
7533         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
7534         WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
7535 }
7536
7537 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id,
7538                                                         u32 rw_asid, u32 rw_mmu_bp)
7539 {
7540         u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id;
7541
7542         WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp);
7543         WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid);
7544 }
7545
7546 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid)
7547 {
7548         u32 reg_base, reg_offset, reg_val = 0;
7549
7550         reg_base = gaudi2_arc_blocks_bases[cpu_id];
7551
7552         /* Enable MMU and configure asid for all relevant ARC regions */
7553         reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0);
7554         reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid);
7555
7556         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL);
7557         WREG32(reg_base + reg_offset, reg_val);
7558
7559         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW);
7560         WREG32(reg_base + reg_offset, reg_val);
7561
7562         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA);
7563         WREG32(reg_base + reg_offset, reg_val);
7564
7565         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA);
7566         WREG32(reg_base + reg_offset, reg_val);
7567
7568         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA);
7569         WREG32(reg_base + reg_offset, reg_val);
7570
7571         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE);
7572         WREG32(reg_base + reg_offset, reg_val);
7573
7574         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL);
7575         WREG32(reg_base + reg_offset, reg_val);
7576
7577         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL);
7578         WREG32(reg_base + reg_offset, reg_val);
7579
7580         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL);
7581         WREG32(reg_base + reg_offset, reg_val);
7582
7583         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL);
7584         WREG32(reg_base + reg_offset, reg_val);
7585
7586         reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL);
7587         WREG32(reg_base + reg_offset, reg_val);
7588 }
7589
7590 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid)
7591 {
7592         int i;
7593
7594         if (hdev->fw_components & FW_TYPE_BOOT_CPU)
7595                 return hl_fw_cpucp_engine_core_asid_set(hdev, asid);
7596
7597         for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
7598                 gaudi2_arc_mmu_prepare(hdev, i, asid);
7599
7600         for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
7601                 if (!gaudi2_is_queue_enabled(hdev, i))
7602                         continue;
7603
7604                 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid);
7605         }
7606
7607         return 0;
7608 }
7609
7610 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid)
7611 {
7612         struct asic_fixed_properties *prop = &hdev->asic_prop;
7613         u32 rw_asid, offset;
7614         int rc, i;
7615
7616         rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) |
7617                         FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid);
7618
7619         WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
7620         WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
7621         WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid);
7622         WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0);
7623
7624         WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
7625         WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
7626         WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid);
7627         WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0);
7628
7629         /* ROT */
7630         for (i = 0 ; i < NUM_OF_ROT ; i++) {
7631                 offset = i * ROT_OFFSET;
7632                 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid);
7633                 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
7634                 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK);
7635                 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK);
7636                 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK);
7637         }
7638
7639         /* Shared Decoders are the last bits in the decoders mask */
7640         if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0))
7641                 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0);
7642
7643         if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1))
7644                 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0);
7645
7646         /* arc farm arc dup eng */
7647         for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
7648                 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0);
7649
7650         rc = gaudi2_arc_mmu_prepare_all(hdev, asid);
7651         if (rc)
7652                 return rc;
7653
7654         return 0;
7655 }
7656
7657 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset,
7658                                         struct iterate_module_ctx *ctx)
7659 {
7660         struct gaudi2_tpc_mmu_data *mmu_data = ctx->data;
7661
7662         WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0);
7663         WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid);
7664         WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
7665         WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid);
7666 }
7667
7668 /* zero the MMUBP and set the ASID */
7669 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid)
7670 {
7671         struct gaudi2_device *gaudi2 = hdev->asic_specific;
7672         struct gaudi2_tpc_mmu_data tpc_mmu_data;
7673         struct iterate_module_ctx tpc_iter = {
7674                 .fn = &gaudi2_tpc_mmu_prepare,
7675                 .data = &tpc_mmu_data,
7676         };
7677         int rc, i;
7678
7679         if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) {
7680                 dev_crit(hdev->dev, "asid %u is too big\n", asid);
7681                 return -EINVAL;
7682         }
7683
7684         if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK))
7685                 return 0;
7686
7687         rc = gaudi2_mmu_shared_prepare(hdev, asid);
7688         if (rc)
7689                 return rc;
7690
7691         /* configure DCORE MMUs */
7692         tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
7693                                 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
7694         gaudi2_iterate_tpcs(hdev, &tpc_iter);
7695         for (i = 0 ; i < NUM_OF_DCORES ; i++)
7696                 gaudi2_mmu_dcore_prepare(hdev, i, asid);
7697
7698         return 0;
7699 }
7700
7701 static inline bool is_info_event(u32 event)
7702 {
7703         switch (event) {
7704         case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
7705         case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
7706
7707         /* return in case of NIC status event - these events are received periodically and not as
7708          * an indication to an error.
7709          */
7710         case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1:
7711                 return true;
7712         default:
7713                 return false;
7714         }
7715 }
7716
7717 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type,
7718                         bool ratelimited, const char *fmt, ...)
7719 {
7720         struct va_format vaf;
7721         va_list args;
7722
7723         va_start(args, fmt);
7724         vaf.fmt = fmt;
7725         vaf.va = &args;
7726
7727         if (ratelimited)
7728                 dev_err_ratelimited(hdev->dev, "%s: %pV\n",
7729                         gaudi2_irq_map_table[event_type].valid ?
7730                         gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf);
7731         else
7732                 dev_err(hdev->dev, "%s: %pV\n",
7733                         gaudi2_irq_map_table[event_type].valid ?
7734                         gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf);
7735
7736         va_end(args);
7737 }
7738
7739 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
7740                 struct hl_eq_ecc_data *ecc_data)
7741 {
7742         u64 ecc_address = 0, ecc_syndrom = 0;
7743         u8 memory_wrapper_idx = 0;
7744
7745         ecc_address = le64_to_cpu(ecc_data->ecc_address);
7746         ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
7747         memory_wrapper_idx = ecc_data->memory_wrapper_idx;
7748
7749         gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
7750                 "ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.",
7751                 ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical);
7752
7753         return !!ecc_data->is_critical;
7754 }
7755
7756 static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base)
7757 {
7758         u32 lo, hi, cq_ptr_size, arc_cq_ptr_size;
7759         u64 cq_ptr, arc_cq_ptr, cp_current_inst;
7760
7761         lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET);
7762         hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET);
7763         cq_ptr = ((u64) hi) << 32 | lo;
7764         cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET);
7765
7766         lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET);
7767         hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET);
7768         arc_cq_ptr = ((u64) hi) << 32 | lo;
7769         arc_cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET);
7770
7771         lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET);
7772         hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET);
7773         cp_current_inst = ((u64) hi) << 32 | lo;
7774
7775         dev_info(hdev->dev,
7776                 "LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n",
7777                 cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst);
7778 }
7779
7780 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type,
7781                                                         u64 qman_base, u32 qid_base)
7782 {
7783         u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0;
7784         u64 glbl_sts_addr, arb_err_addr;
7785         char reg_desc[32];
7786
7787         glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE);
7788         arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE);
7789
7790         /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */
7791         for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
7792                 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);
7793
7794                 if (!glbl_sts_val)
7795                         continue;
7796
7797                 if (i == QMAN_STREAMS) {
7798                         snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM");
7799                         num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE;
7800                 } else {
7801                         snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);
7802                         num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE;
7803                 }
7804
7805                 for (j = 0 ; j < num_error_causes ; j++)
7806                         if (glbl_sts_val & BIT(j)) {
7807                                 gaudi2_print_event(hdev, event_type, true,
7808                                         "%s. err cause: %s", reg_desc,
7809                                         i == QMAN_STREAMS ?
7810                                         gaudi2_lower_qman_error_cause[j] :
7811                                         gaudi2_qman_error_cause[j]);
7812                                 error_count++;
7813                         }
7814
7815                 if (i == QMAN_STREAMS)
7816                         print_lower_qman_data_on_err(hdev, qman_base);
7817         }
7818
7819         arb_err_val = RREG32(arb_err_addr);
7820
7821         if (!arb_err_val)
7822                 goto out;
7823
7824         for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {
7825                 if (arb_err_val & BIT(j)) {
7826                         gaudi2_print_event(hdev, event_type, true,
7827                                 "ARB_ERR. err cause: %s",
7828                                 gaudi2_qman_arb_error_cause[j]);
7829                         error_count++;
7830                 }
7831         }
7832
7833 out:
7834         return error_count;
7835 }
7836
7837 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
7838                         u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7839                         enum gaudi2_engine_id id, u64 *event_mask)
7840 {
7841         u32 razwi_hi, razwi_lo, razwi_xy;
7842         u16 eng_id = id;
7843         u8 rd_wr_flag;
7844
7845         if (is_write) {
7846                 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI);
7847                 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO);
7848                 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY);
7849                 rd_wr_flag = HL_RAZWI_WRITE;
7850         } else {
7851                 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI);
7852                 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO);
7853                 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY);
7854                 rd_wr_flag = HL_RAZWI_READ;
7855         }
7856
7857         hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1,
7858                                 rd_wr_flag | HL_RAZWI_HBW, event_mask);
7859
7860         dev_err_ratelimited(hdev->dev,
7861                 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n",
7862                 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy);
7863 }
7864
7865 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
7866                         u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7867                         enum gaudi2_engine_id id, u64 *event_mask)
7868 {
7869         u64 razwi_addr = CFG_BASE;
7870         u32 razwi_xy;
7871         u16 eng_id = id;
7872         u8 rd_wr_flag;
7873
7874         if (is_write) {
7875                 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI);
7876                 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY);
7877                 rd_wr_flag = HL_RAZWI_WRITE;
7878         } else {
7879                 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI);
7880                 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY);
7881                 rd_wr_flag = HL_RAZWI_READ;
7882         }
7883
7884         hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask);
7885         dev_err_ratelimited(hdev->dev,
7886                                 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n",
7887                                 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr,
7888                                                 razwi_xy);
7889 }
7890
7891 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev,
7892                                                 enum razwi_event_sources module, u8 module_idx)
7893 {
7894         switch (module) {
7895         case RAZWI_TPC:
7896                 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES))
7897                         return GAUDI2_DCORE0_ENGINE_ID_TPC_6;
7898                 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7899                                 (module_idx % NUM_OF_TPC_PER_DCORE) +
7900                                 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
7901
7902         case RAZWI_MME:
7903                 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) +
7904                         (module_idx * ENGINE_ID_DCORE_OFFSET));
7905
7906         case RAZWI_EDMA:
7907                 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7908                         (module_idx % NUM_OF_EDMA_PER_DCORE));
7909
7910         case RAZWI_PDMA:
7911                 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx);
7912
7913         case RAZWI_NIC:
7914                 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx));
7915
7916         case RAZWI_DEC:
7917                 if (module_idx == 8)
7918                         return GAUDI2_PCIE_ENGINE_ID_DEC_0;
7919
7920                 if (module_idx == 9)
7921                         return GAUDI2_PCIE_ENGINE_ID_DEC_1;
7922                                         ;
7923                 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7924                                 (module_idx % NUM_OF_DEC_PER_DCORE) +
7925                                 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
7926
7927         case RAZWI_ROT:
7928                 return GAUDI2_ENGINE_ID_ROT_0 + module_idx;
7929
7930         default:
7931                 return GAUDI2_ENGINE_ID_SIZE;
7932         }
7933 }
7934
7935 /*
7936  * This function handles RR(Range register) hit events.
7937  * raised be initiators not PSOC RAZWI.
7938  */
7939 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
7940                                 enum razwi_event_sources module, u8 module_idx,
7941                                 u8 module_sub_idx, u64 *event_mask)
7942 {
7943         bool via_sft = false;
7944         u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx;
7945         u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr;
7946         u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0;
7947         u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0;
7948         char initiator_name[64];
7949
7950         switch (module) {
7951         case RAZWI_TPC:
7952                 sprintf(initiator_name, "TPC_%u", module_idx);
7953                 if (hdev->tpc_binning) {
7954                         binned_idx = __ffs(hdev->tpc_binning);
7955                         if (binned_idx == module_idx)
7956                                 module_idx = TPC_ID_DCORE0_TPC6;
7957                 }
7958
7959                 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx];
7960
7961                 if (hl_is_fw_sw_ver_below(hdev, 1, 9) &&
7962                                 !hdev->asic_prop.fw_security_enabled &&
7963                                 ((module_idx == 0) || (module_idx == 1)))
7964                         lbw_rtr_id = DCORE0_RTR0;
7965                 else
7966                         lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx];
7967                 break;
7968         case RAZWI_MME:
7969                 sprintf(initiator_name, "MME_%u", module_idx);
7970                 switch (module_sub_idx) {
7971                 case MME_WAP0:
7972                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0;
7973                         break;
7974                 case MME_WAP1:
7975                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1;
7976                         break;
7977                 case MME_WRITE:
7978                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write;
7979                         break;
7980                 case MME_READ:
7981                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read;
7982                         break;
7983                 case MME_SBTE0:
7984                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0;
7985                         break;
7986                 case MME_SBTE1:
7987                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1;
7988                         break;
7989                 case MME_SBTE2:
7990                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2;
7991                         break;
7992                 case MME_SBTE3:
7993                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3;
7994                         break;
7995                 case MME_SBTE4:
7996                         hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4;
7997                         break;
7998                 default:
7999                         return;
8000                 }
8001                 lbw_rtr_id = hbw_rtr_id;
8002                 break;
8003         case RAZWI_EDMA:
8004                 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx];
8005                 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE;
8006                 /* SFT has separate MSTR_IF for LBW, only there we can
8007                  * read the LBW razwi related registers
8008                  */
8009                 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE +
8010                                                                 dcore_id * SFT_DCORE_OFFSET;
8011                 via_sft = true;
8012                 sprintf(initiator_name, "EDMA_%u", module_idx);
8013                 break;
8014         case RAZWI_PDMA:
8015                 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx];
8016                 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx];
8017                 sprintf(initiator_name, "PDMA_%u", module_idx);
8018                 break;
8019         case RAZWI_NIC:
8020                 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx];
8021                 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx];
8022                 sprintf(initiator_name, "NIC_%u", module_idx);
8023                 break;
8024         case RAZWI_DEC:
8025                 sprintf(initiator_name, "DEC_%u", module_idx);
8026                 if (hdev->decoder_binning) {
8027                         binned_idx = __ffs(hdev->decoder_binning);
8028                         if (binned_idx == module_idx)
8029                                 module_idx = DEC_ID_PCIE_VDEC1;
8030                 }
8031                 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx];
8032                 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx];
8033                 break;
8034         case RAZWI_ROT:
8035                 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx];
8036                 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx];
8037                 sprintf(initiator_name, "ROT_%u", module_idx);
8038                 break;
8039         default:
8040                 return;
8041         }
8042
8043         /* Find router mstr_if register base */
8044         if (!via_sft) {
8045                 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE;
8046                 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE;
8047                 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE +
8048                                 dcore_id * DCORE_OFFSET +
8049                                 dcore_rtr_id * DCORE_RTR_OFFSET +
8050                                 RTR_MSTR_IF_OFFSET;
8051                 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr +
8052                                 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET);
8053         }
8054
8055         /* Find out event cause by reading "RAZWI_HAPPENED" registers */
8056         hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED);
8057         hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED);
8058         lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED);
8059         lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED);
8060
8061         eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx);
8062         if (hbw_shrd_aw) {
8063                 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true,
8064                                                 initiator_name, eng_id, event_mask);
8065
8066                 /* Clear event indication */
8067                 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw);
8068         }
8069
8070         if (hbw_shrd_ar) {
8071                 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false,
8072                                                 initiator_name, eng_id, event_mask);
8073
8074                 /* Clear event indication */
8075                 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar);
8076         }
8077
8078         if (lbw_shrd_aw) {
8079                 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true,
8080                                                 initiator_name, eng_id, event_mask);
8081
8082                 /* Clear event indication */
8083                 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw);
8084         }
8085
8086         if (lbw_shrd_ar) {
8087                 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false,
8088                                                 initiator_name, eng_id, event_mask);
8089
8090                 /* Clear event indication */
8091                 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar);
8092         }
8093 }
8094
8095 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
8096 {
8097         struct asic_fixed_properties *prop = &hdev->asic_prop;
8098         u8 mod_idx, sub_mod;
8099
8100         /* check all TPCs */
8101         for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) {
8102                 if (prop->tpc_enabled_mask & BIT(mod_idx))
8103                         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL);
8104         }
8105
8106         /* check all MMEs */
8107         for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
8108                 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++)
8109                         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx,
8110                                                                         sub_mod, NULL);
8111
8112         /* check all EDMAs */
8113         for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
8114                 if (prop->edma_enabled_mask & BIT(mod_idx))
8115                         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL);
8116
8117         /* check all PDMAs */
8118         for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++)
8119                 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL);
8120
8121         /* check all NICs */
8122         for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++)
8123                 if (hdev->nic_ports_mask & BIT(mod_idx))
8124                         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,
8125                                                                 NULL);
8126
8127         /* check all DECs */
8128         for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++)
8129                 if (prop->decoder_enabled_mask & BIT(mod_idx))
8130                         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL);
8131
8132         /* check all ROTs */
8133         for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++)
8134                 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
8135 }
8136
8137 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size,
8138                                                 u32 axuser_xy, u32 *base, u16 *eng_id,
8139                                                 char *eng_name)
8140 {
8141
8142         int i, num_of_eng = 0;
8143         u16 str_size = 0;
8144
8145         for (i = 0 ; i < array_size ; i++) {
8146                 if (axuser_xy != razwi_info[i].axuser_xy)
8147                         continue;
8148
8149                 eng_id[num_of_eng] = razwi_info[i].eng_id;
8150                 base[num_of_eng] = razwi_info[i].rtr_ctrl;
8151                 if (!num_of_eng)
8152                         str_size += snprintf(eng_name + str_size,
8153                                                 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s",
8154                                                 razwi_info[i].eng_name);
8155                 else
8156                         str_size += snprintf(eng_name + str_size,
8157                                                 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s",
8158                                                 razwi_info[i].eng_name);
8159                 num_of_eng++;
8160         }
8161
8162         return num_of_eng;
8163 }
8164
8165 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg,
8166                                                 u64 *event_mask)
8167 {
8168         u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0;
8169         u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR];
8170         u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR];
8171         char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE];
8172         bool razwi_happened = false;
8173         u64 addr;
8174         int i;
8175
8176         num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info),
8177                                                         axuser_xy, base, eng_id, eng_name_str);
8178
8179         /* If no match for XY coordinates, try to find it in MME razwi table */
8180         if (!num_of_eng) {
8181                 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg);
8182                 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info,
8183                                                                 ARRAY_SIZE(mme_razwi_info),
8184                                                                 axuser_xy, base, eng_id,
8185                                                                 eng_name_str);
8186         }
8187
8188         for  (i = 0 ; i < num_of_eng ; i++) {
8189                 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) {
8190                         addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI);
8191                         addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO);
8192                         addr = ((u64)addr_hi << 32) + addr_lo;
8193                         if (addr) {
8194                                 dev_err(hdev->dev,
8195                                         "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
8196                                         eng_name_str, addr);
8197                                 hl_handle_razwi(hdev, addr, &eng_id[0],
8198                                         num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask);
8199                                 razwi_happened = true;
8200                         }
8201                 }
8202
8203                 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) {
8204                         addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI);
8205                         addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO);
8206                         addr = ((u64)addr_hi << 32) + addr_lo;
8207                         if (addr) {
8208                                 dev_err(hdev->dev,
8209                                         "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
8210                                         eng_name_str, addr);
8211                                 hl_handle_razwi(hdev, addr, &eng_id[0],
8212                                         num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask);
8213                                 razwi_happened = true;
8214                         }
8215                 }
8216
8217                 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) {
8218                         addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR);
8219                         if (addr_lo) {
8220                                 dev_err(hdev->dev,
8221                                         "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
8222                                         eng_name_str, addr_lo);
8223                                 hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8224                                         num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask);
8225                                 razwi_happened = true;
8226                         }
8227                 }
8228
8229                 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) {
8230                         addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR);
8231                         if (addr_lo) {
8232                                 dev_err(hdev->dev,
8233                                                 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
8234                                                 eng_name_str, addr_lo);
8235                                 hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8236                                         num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask);
8237                                 razwi_happened = true;
8238                         }
8239                 }
8240                 /* In common case the loop will break, when there is only one engine id, or
8241                  * several engines with the same router. The exceptional case is with psoc razwi
8242                  * from EDMA, where it's possible to get axuser id which fits 2 routers (2
8243                  * interfaces of sft router). In this case, maybe the first router won't hold info
8244                  * and we will need to iterate on the other router.
8245                  */
8246                 if (razwi_happened)
8247                         break;
8248         }
8249
8250         return razwi_happened;
8251 }
8252
8253 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */
8254 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask)
8255 {
8256         u32 razwi_mask_info, razwi_intr = 0, error_count = 0;
8257
8258         if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {
8259                 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT);
8260                 if (!razwi_intr)
8261                         return 0;
8262         }
8263
8264         razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO);
8265
8266         dev_err_ratelimited(hdev->dev,
8267                 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n",
8268                 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info),
8269                 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info),
8270                 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info),
8271                 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info),
8272                 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info));
8273
8274         if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask))
8275                 error_count++;
8276         else
8277                 dev_err_ratelimited(hdev->dev,
8278                                 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n",
8279                                 razwi_mask_info);
8280
8281         /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */
8282         if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))
8283                 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr);
8284
8285         return error_count;
8286 }
8287
8288 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type)
8289 {
8290         u32 i, sts_val, sts_clr_val = 0, error_count = 0;
8291
8292         sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET);
8293
8294         for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) {
8295                 if (sts_val & BIT(i)) {
8296                         gaudi2_print_event(hdev, event_type, true,
8297                                 "err cause: %s", gaudi2_qm_sei_error_cause[i]);
8298                         sts_clr_val |= BIT(i);
8299                         error_count++;
8300                 }
8301         }
8302
8303         WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val);
8304
8305         return error_count;
8306 }
8307
8308 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
8309                                         bool extended_err_check, u64 *event_mask)
8310 {
8311         enum razwi_event_sources module;
8312         u32 error_count = 0;
8313         u64 qman_base;
8314         u8 index;
8315
8316         switch (event_type) {
8317         case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP:
8318                 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
8319                 qman_base = mmDCORE0_TPC0_QM_BASE +
8320                                 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET +
8321                                 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET;
8322                 module = RAZWI_TPC;
8323                 break;
8324         case GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
8325                 qman_base = mmDCORE0_TPC6_QM_BASE;
8326                 module = RAZWI_TPC;
8327                 break;
8328         case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
8329         case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
8330         case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
8331         case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
8332                 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
8333                                 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
8334                                                 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
8335                 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET;
8336                 module = RAZWI_MME;
8337                 break;
8338         case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
8339         case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
8340                 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP;
8341                 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET;
8342                 module = RAZWI_PDMA;
8343                 break;
8344         case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
8345         case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
8346                 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
8347                 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET;
8348                 module = RAZWI_ROT;
8349                 break;
8350         default:
8351                 return 0;
8352         }
8353
8354         error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8355
8356         /* There is a single event per NIC macro, so should check its both QMAN blocks */
8357         if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE &&
8358                         event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE)
8359                 error_count += _gaudi2_handle_qm_sei_err(hdev,
8360                                         qman_base + NIC_QM_OFFSET, event_type);
8361
8362         if (extended_err_check) {
8363                 /* check if RAZWI happened */
8364                 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask);
8365                 hl_check_for_glbl_errors(hdev);
8366         }
8367
8368         return error_count;
8369 }
8370
8371 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
8372 {
8373         u32 qid_base, error_count = 0;
8374         u64 qman_base;
8375         u8 index = 0;
8376
8377         switch (event_type) {
8378         case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM:
8379                 index = event_type - GAUDI2_EVENT_TPC0_QM;
8380                 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS;
8381                 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8382                 break;
8383         case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM:
8384                 index = event_type - GAUDI2_EVENT_TPC6_QM;
8385                 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS;
8386                 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8387                 break;
8388         case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM:
8389                 index = event_type - GAUDI2_EVENT_TPC12_QM;
8390                 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS;
8391                 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8392                 break;
8393         case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM:
8394                 index = event_type - GAUDI2_EVENT_TPC18_QM;
8395                 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS;
8396                 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8397                 break;
8398         case GAUDI2_EVENT_TPC24_QM:
8399                 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
8400                 qman_base = mmDCORE0_TPC6_QM_BASE;
8401                 break;
8402         case GAUDI2_EVENT_MME0_QM:
8403                 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
8404                 qman_base = mmDCORE0_MME_QM_BASE;
8405                 break;
8406         case GAUDI2_EVENT_MME1_QM:
8407                 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
8408                 qman_base = mmDCORE1_MME_QM_BASE;
8409                 break;
8410         case GAUDI2_EVENT_MME2_QM:
8411                 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
8412                 qman_base = mmDCORE2_MME_QM_BASE;
8413                 break;
8414         case GAUDI2_EVENT_MME3_QM:
8415                 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
8416                 qman_base = mmDCORE3_MME_QM_BASE;
8417                 break;
8418         case GAUDI2_EVENT_HDMA0_QM:
8419                 index = 0;
8420                 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0;
8421                 qman_base = mmDCORE0_EDMA0_QM_BASE;
8422                 break;
8423         case GAUDI2_EVENT_HDMA1_QM:
8424                 index = 1;
8425                 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0;
8426                 qman_base = mmDCORE0_EDMA1_QM_BASE;
8427                 break;
8428         case GAUDI2_EVENT_HDMA2_QM:
8429                 index = 2;
8430                 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0;
8431                 qman_base = mmDCORE1_EDMA0_QM_BASE;
8432                 break;
8433         case GAUDI2_EVENT_HDMA3_QM:
8434                 index = 3;
8435                 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0;
8436                 qman_base = mmDCORE1_EDMA1_QM_BASE;
8437                 break;
8438         case GAUDI2_EVENT_HDMA4_QM:
8439                 index = 4;
8440                 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0;
8441                 qman_base = mmDCORE2_EDMA0_QM_BASE;
8442                 break;
8443         case GAUDI2_EVENT_HDMA5_QM:
8444                 index = 5;
8445                 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0;
8446                 qman_base = mmDCORE2_EDMA1_QM_BASE;
8447                 break;
8448         case GAUDI2_EVENT_HDMA6_QM:
8449                 index = 6;
8450                 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0;
8451                 qman_base = mmDCORE3_EDMA0_QM_BASE;
8452                 break;
8453         case GAUDI2_EVENT_HDMA7_QM:
8454                 index = 7;
8455                 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0;
8456                 qman_base = mmDCORE3_EDMA1_QM_BASE;
8457                 break;
8458         case GAUDI2_EVENT_PDMA0_QM:
8459                 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0;
8460                 qman_base = mmPDMA0_QM_BASE;
8461                 break;
8462         case GAUDI2_EVENT_PDMA1_QM:
8463                 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0;
8464                 qman_base = mmPDMA1_QM_BASE;
8465                 break;
8466         case GAUDI2_EVENT_ROTATOR0_ROT0_QM:
8467                 qid_base = GAUDI2_QUEUE_ID_ROT_0_0;
8468                 qman_base = mmROT0_QM_BASE;
8469                 break;
8470         case GAUDI2_EVENT_ROTATOR1_ROT1_QM:
8471                 qid_base = GAUDI2_QUEUE_ID_ROT_1_0;
8472                 qman_base = mmROT1_QM_BASE;
8473                 break;
8474         default:
8475                 return 0;
8476         }
8477
8478         error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, qid_base);
8479
8480         /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */
8481         if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) {
8482                 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8483                 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask);
8484         }
8485
8486         hl_check_for_glbl_errors(hdev);
8487
8488         return error_count;
8489 }
8490
8491 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type)
8492 {
8493         u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm;
8494
8495         for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) {
8496                 sts_clr_val = 0;
8497                 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS +
8498                                 (arc_farm * ARC_FARM_OFFSET));
8499
8500                 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) {
8501                         if (sts_val & BIT(i)) {
8502                                 gaudi2_print_event(hdev, event_type, true,
8503                                                 "ARC FARM ARC %u err cause: %s",
8504                                                 arc_farm, gaudi2_arc_sei_error_cause[i]);
8505                                 sts_clr_val |= BIT(i);
8506                                 error_count++;
8507                         }
8508                 }
8509                 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET),
8510                                 sts_clr_val);
8511         }
8512
8513         hl_check_for_glbl_errors(hdev);
8514
8515         return error_count;
8516 }
8517
8518 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type)
8519 {
8520         u32 i, sts_val, sts_clr_val = 0, error_count = 0;
8521
8522         sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS);
8523
8524         for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) {
8525                 if (sts_val & BIT(i)) {
8526                         gaudi2_print_event(hdev, event_type, true,
8527                                 "err cause: %s", gaudi2_cpu_sei_error_cause[i]);
8528                         sts_clr_val |= BIT(i);
8529                         error_count++;
8530                 }
8531         }
8532
8533         hl_check_for_glbl_errors(hdev);
8534
8535         WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val);
8536
8537         return error_count;
8538 }
8539
8540 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type,
8541                                         struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
8542                                         u64 *event_mask)
8543 {
8544         u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
8545         u32 error_count = 0;
8546         int i;
8547
8548         for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++)
8549                 if (intr_cause_data & BIT(i)) {
8550                         gaudi2_print_event(hdev, event_type, true,
8551                                 "err cause: %s", guadi2_rot_error_cause[i]);
8552                         error_count++;
8553                 }
8554
8555         /* check if RAZWI happened */
8556         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask);
8557         hl_check_for_glbl_errors(hdev);
8558
8559         return error_count;
8560 }
8561
8562 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev,  u8 tpc_index, u16 event_type,
8563                                         struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
8564                                         u64 *event_mask)
8565 {
8566         u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
8567         u32 error_count = 0;
8568         int i;
8569
8570         for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++)
8571                 if (intr_cause_data & BIT(i)) {
8572                         gaudi2_print_event(hdev, event_type, true,
8573                                 "interrupt cause: %s",  gaudi2_tpc_interrupts_cause[i]);
8574                         error_count++;
8575                 }
8576
8577         /* check if RAZWI happened */
8578         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask);
8579         hl_check_for_glbl_errors(hdev);
8580
8581         return error_count;
8582 }
8583
8584 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type,
8585                                         u64 *event_mask)
8586 {
8587         u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0;
8588         int i;
8589
8590         if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES)
8591                 /* DCORE DEC */
8592                 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR +
8593                                 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) +
8594                                 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE);
8595         else
8596                 /* PCIE DEC */
8597                 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET *
8598                                 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES);
8599
8600         sts_val = RREG32(sts_addr);
8601
8602         for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) {
8603                 if (sts_val & BIT(i)) {
8604                         gaudi2_print_event(hdev, event_type, true,
8605                                 "err cause: %s", gaudi2_dec_error_cause[i]);
8606                         sts_clr_val |= BIT(i);
8607                         error_count++;
8608                 }
8609         }
8610
8611         /* check if RAZWI happened */
8612         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask);
8613         hl_check_for_glbl_errors(hdev);
8614
8615         /* Write 1 clear errors */
8616         WREG32(sts_addr, sts_clr_val);
8617
8618         return error_count;
8619 }
8620
8621 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8622                                         u64 *event_mask)
8623 {
8624         u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0;
8625         int i;
8626
8627         sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index;
8628         sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index;
8629
8630         sts_val = RREG32(sts_addr);
8631
8632         for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) {
8633                 if (sts_val & BIT(i)) {
8634                         gaudi2_print_event(hdev, event_type, true,
8635                                 "err cause: %s", guadi2_mme_error_cause[i]);
8636                         sts_clr_val |= BIT(i);
8637                         error_count++;
8638                 }
8639         }
8640
8641         /* check if RAZWI happened */
8642         for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++)
8643                 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask);
8644
8645         hl_check_for_glbl_errors(hdev);
8646
8647         WREG32(sts_clr_addr, sts_clr_val);
8648
8649         return error_count;
8650 }
8651
8652 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type,
8653                                         u64 intr_cause_data)
8654 {
8655         int i, error_count = 0;
8656
8657         for (i = 0 ; i < GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE ; i++)
8658                 if (intr_cause_data & BIT(i)) {
8659                         gaudi2_print_event(hdev, event_type, true,
8660                                 "err cause: %s", guadi2_mme_sbte_error_cause[i]);
8661                         error_count++;
8662                 }
8663
8664         hl_check_for_glbl_errors(hdev);
8665
8666         return error_count;
8667 }
8668
8669 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8670                                         u64 *event_mask)
8671 {
8672         u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0;
8673         int i;
8674
8675         sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index;
8676         sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index;
8677
8678         sts_val = RREG32(sts_addr);
8679
8680         for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) {
8681                 if (sts_val & BIT(i)) {
8682                         gaudi2_print_event(hdev, event_type, true,
8683                                 "err cause: %s", guadi2_mme_wap_error_cause[i]);
8684                         sts_clr_val |= BIT(i);
8685                         error_count++;
8686                 }
8687         }
8688
8689         /* check if RAZWI happened on WAP0/1 */
8690         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask);
8691         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask);
8692         hl_check_for_glbl_errors(hdev);
8693
8694         WREG32(sts_clr_addr, sts_clr_val);
8695
8696         return error_count;
8697 }
8698
8699 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type,
8700                                         u64 intr_cause_data)
8701 {
8702         u32 error_count = 0;
8703         int i;
8704
8705         /* If an AXI read or write error is received, an error is reported and
8706          * interrupt message is sent. Due to an HW errata, when reading the cause
8707          * register of the KDMA engine, the reported error is always HBW even if
8708          * the actual error caused by a LBW KDMA transaction.
8709          */
8710         for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
8711                 if (intr_cause_data & BIT(i)) {
8712                         gaudi2_print_event(hdev, event_type, true,
8713                                 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]);
8714                         error_count++;
8715                 }
8716
8717         hl_check_for_glbl_errors(hdev);
8718
8719         return error_count;
8720 }
8721
8722 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause)
8723 {
8724         u32 error_count = 0;
8725         int i;
8726
8727         for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
8728                 if (intr_cause & BIT(i)) {
8729                         gaudi2_print_event(hdev, event_type, true,
8730                                 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]);
8731                         error_count++;
8732                 }
8733
8734         hl_check_for_glbl_errors(hdev);
8735
8736         return error_count;
8737 }
8738
8739 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask)
8740 {
8741         u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr;
8742
8743         razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED;
8744         if (RREG32(razwi_happened_addr)) {
8745                 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8746                                                         GAUDI2_ENGINE_ID_PCIE, event_mask);
8747                 WREG32(razwi_happened_addr, 0x1);
8748         }
8749
8750         razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED;
8751         if (RREG32(razwi_happened_addr)) {
8752                 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8753                                                         GAUDI2_ENGINE_ID_PCIE, event_mask);
8754                 WREG32(razwi_happened_addr, 0x1);
8755         }
8756
8757         razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED;
8758         if (RREG32(razwi_happened_addr)) {
8759                 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8760                                                         GAUDI2_ENGINE_ID_PCIE, event_mask);
8761                 WREG32(razwi_happened_addr, 0x1);
8762         }
8763
8764         razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED;
8765         if (RREG32(razwi_happened_addr)) {
8766                 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8767                                                         GAUDI2_ENGINE_ID_PCIE, event_mask);
8768                 WREG32(razwi_happened_addr, 0x1);
8769         }
8770 }
8771
8772 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type,
8773                                         u64 intr_cause_data, u64 *event_mask)
8774 {
8775         u32 error_count = 0;
8776         int i;
8777
8778         gaudi2_print_event(hdev, event_type, true,
8779                 "intr_cause_data: %#llx", intr_cause_data);
8780
8781         for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) {
8782                 if (!(intr_cause_data & BIT_ULL(i)))
8783                         continue;
8784
8785                 gaudi2_print_event(hdev, event_type, true,
8786                         "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]);
8787                 error_count++;
8788
8789                 /*
8790                  * Always check for LBW and HBW additional info as the indication itself is
8791                  * sometimes missing
8792                  */
8793         }
8794
8795         hl_check_for_glbl_errors(hdev);
8796         gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask);
8797
8798         return error_count;
8799 }
8800
8801 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type,
8802                                 u64 intr_cause_data)
8803
8804 {
8805         u32 error_count = 0;
8806         int i;
8807
8808         for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) {
8809                 if (intr_cause_data & BIT_ULL(i)) {
8810                         gaudi2_print_event(hdev, event_type, true,
8811                                 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]);
8812                         error_count++;
8813                 }
8814         }
8815
8816         return error_count;
8817 }
8818
8819 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data)
8820 {
8821         u32 error_count = 0;
8822         int i;
8823
8824         for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) {
8825                 if (intr_cause_data & BIT_ULL(i)) {
8826                         gaudi2_print_event(hdev, event_type, true,
8827                                 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]);
8828                         error_count++;
8829                 }
8830         }
8831
8832         return error_count;
8833 }
8834
8835 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu,
8836                                         u64 *event_mask)
8837 {
8838         u32 valid, val;
8839         u64 addr;
8840
8841         valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
8842
8843         if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK))
8844                 return;
8845
8846         val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE));
8847         addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK;
8848         addr <<= 32;
8849         addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA));
8850
8851         if (is_pmmu) {
8852                 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr);
8853         } else {
8854
8855                 addr = gaudi2_mmu_descramble_addr(hdev, addr);
8856                 addr &= HW_UNSCRAMBLED_BITS_MASK;
8857                 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n",
8858                                 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK);
8859         }
8860
8861         hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask);
8862
8863         WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
8864 }
8865
8866 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
8867 {
8868         u32 valid, val;
8869         u64 addr;
8870
8871         valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
8872
8873         if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK))
8874                 return;
8875
8876         val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE));
8877         addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK;
8878         addr <<= 32;
8879         addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA));
8880
8881         if (!is_pmmu)
8882                 addr = gaudi2_mmu_descramble_addr(hdev, addr);
8883
8884         dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",
8885                                 is_pmmu ? "PMMU" : "HMMU", addr);
8886         WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
8887 }
8888
8889 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type,
8890                                                 u64 mmu_base, bool is_pmmu, u64 *event_mask)
8891 {
8892         u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0;
8893         int i;
8894
8895         spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET);
8896
8897         for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) {
8898                 if (spi_sei_cause & BIT(i)) {
8899                         gaudi2_print_event(hdev, event_type, true,
8900                                 "err cause: %s", gaudi2_mmu_spi_sei[i].cause);
8901
8902                         if (i == 0)
8903                                 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask);
8904                         else if (i == 1)
8905                                 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
8906
8907                         if (gaudi2_mmu_spi_sei[i].clear_bit >= 0)
8908                                 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit);
8909
8910                         error_count++;
8911                 }
8912         }
8913
8914         /* Clear cause */
8915         WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause);
8916
8917         /* Clear interrupt */
8918         WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr);
8919
8920         return error_count;
8921 }
8922
8923 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index)
8924 {
8925         u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log,
8926                 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0;
8927         int i;
8928
8929         sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index;
8930         cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index;
8931
8932         sei_cause_val = RREG32(sei_cause_addr);
8933         sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val);
8934         cq_intr_val = RREG32(cq_intr_addr);
8935
8936         /* SEI interrupt */
8937         if (sei_cause_cause) {
8938                 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */
8939                 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK,
8940                                         sei_cause_val);
8941
8942                 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) {
8943                         if (!(sei_cause_cause & BIT(i)))
8944                                 continue;
8945
8946                         gaudi2_print_event(hdev, event_type, true,
8947                                 "err cause: %s. %s: 0x%X",
8948                                 gaudi2_sm_sei_cause[i].cause_name,
8949                                 gaudi2_sm_sei_cause[i].log_name,
8950                                 sei_cause_log);
8951                         error_count++;
8952                         break;
8953                 }
8954
8955                 /* Clear SM_SEI_CAUSE */
8956                 WREG32(sei_cause_addr, 0);
8957         }
8958
8959         /* CQ interrupt */
8960         if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) {
8961                 cq_intr_queue_index =
8962                                 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK,
8963                                         cq_intr_val);
8964
8965                 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n",
8966                                 sm_index, cq_intr_queue_index);
8967                 error_count++;
8968
8969                 /* Clear CQ_INTR */
8970                 WREG32(cq_intr_addr, 0);
8971         }
8972
8973         hl_check_for_glbl_errors(hdev);
8974
8975         return error_count;
8976 }
8977
8978 static u64 get_hmmu_base(u16 event_type)
8979 {
8980         u8 dcore, index_in_dcore;
8981
8982         switch (event_type) {
8983         case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP:
8984         case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR:
8985                 dcore = 0;
8986                 index_in_dcore = 0;
8987         break;
8988         case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP:
8989         case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR:
8990                 dcore = 1;
8991                 index_in_dcore = 0;
8992         break;
8993         case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP:
8994         case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR:
8995                 dcore = 0;
8996                 index_in_dcore = 1;
8997         break;
8998         case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP:
8999         case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR:
9000                 dcore = 1;
9001                 index_in_dcore = 1;
9002         break;
9003         case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP:
9004         case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR:
9005                 dcore = 3;
9006                 index_in_dcore = 2;
9007         break;
9008         case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP:
9009         case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR:
9010                 dcore = 2;
9011                 index_in_dcore = 2;
9012         break;
9013         case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP:
9014         case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR:
9015                 dcore = 3;
9016                 index_in_dcore = 3;
9017         break;
9018         case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP:
9019         case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR:
9020                 dcore = 2;
9021                 index_in_dcore = 3;
9022         break;
9023         case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP:
9024         case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR:
9025                 dcore = 0;
9026                 index_in_dcore = 2;
9027         break;
9028         case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP:
9029         case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR:
9030                 dcore = 1;
9031                 index_in_dcore = 2;
9032         break;
9033         case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP:
9034         case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR:
9035                 dcore = 0;
9036                 index_in_dcore = 3;
9037         break;
9038         case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP:
9039         case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR:
9040                 dcore = 1;
9041                 index_in_dcore = 3;
9042         break;
9043         case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9044         case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9045                 dcore = 3;
9046                 index_in_dcore = 0;
9047         break;
9048         case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP:
9049         case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR:
9050                 dcore = 2;
9051                 index_in_dcore = 0;
9052         break;
9053         case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP:
9054         case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR:
9055                 dcore = 3;
9056                 index_in_dcore = 1;
9057         break;
9058         case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP:
9059         case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR:
9060                 dcore = 2;
9061                 index_in_dcore = 1;
9062         break;
9063         default:
9064                 return ULONG_MAX;
9065         }
9066
9067         return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET;
9068 }
9069
9070 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9071 {
9072         bool is_pmmu = false;
9073         u32 error_count = 0;
9074         u64 mmu_base;
9075
9076         switch (event_type) {
9077         case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9078         case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9079                 mmu_base = get_hmmu_base(event_type);
9080                 break;
9081
9082         case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
9083         case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
9084                 is_pmmu = true;
9085                 mmu_base = mmPMMU_HBW_MMU_BASE;
9086                 break;
9087         default:
9088                 return 0;
9089         }
9090
9091         if (mmu_base == ULONG_MAX)
9092                 return 0;
9093
9094         error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base,
9095                                                         is_pmmu, event_mask);
9096         hl_check_for_glbl_errors(hdev);
9097
9098         return error_count;
9099 }
9100
9101
9102 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */
9103 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev,
9104                         struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt)
9105 {
9106         u32 addr, beat, beat_shift;
9107         bool rc = false;
9108
9109         dev_err_ratelimited(hdev->dev,
9110                         "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n",
9111                         FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt),
9112                         FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt),
9113                         FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt));
9114
9115         addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val);
9116         dev_err_ratelimited(hdev->dev,
9117                         "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n",
9118                         FIELD_GET(HBM_RD_ADDR_SID_MASK, addr),
9119                         FIELD_GET(HBM_RD_ADDR_BG_MASK, addr),
9120                         FIELD_GET(HBM_RD_ADDR_BA_MASK, addr),
9121                         FIELD_GET(HBM_RD_ADDR_COL_MASK, addr),
9122                         FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr));
9123
9124         /* For each beat (RDQS edge), look for possible errors and print relevant info */
9125         for (beat = 0 ; beat < 4 ; beat++) {
9126                 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9127                         (HBM_RD_ERR_SERR_BEAT0_MASK << beat))
9128                         dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n",
9129                                                 beat,
9130                                                 le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9131                                                 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
9132
9133                 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9134                         (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) {
9135                         dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n",
9136                                                 beat,
9137                                                 le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9138                                                 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
9139                         rc |= true;
9140                 }
9141
9142                 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT;
9143                 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9144                         (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) {
9145                         dev_err_ratelimited(hdev->dev,
9146                                         "Beat%d read PARITY: DM: %#x, PAR data: %#x\n",
9147                                         beat,
9148                                         le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9149                                         (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9150                                                 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >>
9151                                                 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift));
9152                         rc |= true;
9153                 }
9154
9155                 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat);
9156                 dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9157                                         le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2]));
9158                 dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9159                                         le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1]));
9160         }
9161
9162         return rc;
9163 }
9164
9165 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev,
9166                         struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt)
9167 {
9168         struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds;
9169         u32 i, curr_addr, derr = wr_par_err_data->dbg_derr;
9170
9171         dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt);
9172
9173         dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n",
9174                                 derr & 0x3, derr & 0xc);
9175
9176         /* JIRA H6-3286 - the following prints may not be valid */
9177         dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n");
9178         for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) {
9179                 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr);
9180                 dev_err_ratelimited(hdev->dev,
9181                                 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n",
9182                                 i,
9183                                 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr),
9184                                 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr),
9185                                 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr),
9186                                 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr));
9187         }
9188 }
9189
9190 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev,
9191                 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt)
9192 {
9193         __le32 *col_cmd = ca_par_err_data->dbg_col;
9194         __le16 *row_cmd = ca_par_err_data->dbg_row;
9195         u32 i;
9196
9197         dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt);
9198
9199         dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n");
9200         for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++)
9201                 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i,
9202                         le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0),
9203                         le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0));
9204 }
9205
9206 /* Returns true if hard reset is needed or false otherwise */
9207 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type,
9208                                         struct hl_eq_hbm_sei_data *sei_data)
9209 {
9210         bool require_hard_reset = false;
9211         u32 hbm_id, mc_id, cause_idx;
9212
9213         hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4;
9214         mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2;
9215
9216         cause_idx = sei_data->hdr.sei_cause;
9217         if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) {
9218                 gaudi2_print_event(hdev, event_type, true,
9219                         "err cause: %s",
9220                         "Invalid HBM SEI event cause (%d) provided by FW", cause_idx);
9221                 return true;
9222         }
9223
9224         gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical,
9225                 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s",
9226                 sei_data->hdr.is_critical ? "Critical" : "Non-critical",
9227                 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel,
9228                 hbm_mc_sei_cause[cause_idx]);
9229
9230         /* Print error-specific info */
9231         switch (cause_idx) {
9232         case HBM_SEI_CATTRIP:
9233                 require_hard_reset = true;
9234                 break;
9235
9236         case  HBM_SEI_CMD_PARITY_EVEN:
9237                 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info,
9238                                                 le32_to_cpu(sei_data->hdr.cnt));
9239                 require_hard_reset = true;
9240                 break;
9241
9242         case  HBM_SEI_CMD_PARITY_ODD:
9243                 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info,
9244                                                 le32_to_cpu(sei_data->hdr.cnt));
9245                 require_hard_reset = true;
9246                 break;
9247
9248         case HBM_SEI_WRITE_DATA_PARITY_ERR:
9249                 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info,
9250                                                 le32_to_cpu(sei_data->hdr.cnt));
9251                 require_hard_reset = true;
9252                 break;
9253
9254         case HBM_SEI_READ_ERR:
9255                 /* Unlike other SEI events, read error requires further processing of the
9256                  * raw data in order to determine the root cause.
9257                  */
9258                 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev,
9259                                                                 &sei_data->read_err_info,
9260                                                                 le32_to_cpu(sei_data->hdr.cnt));
9261                 break;
9262
9263         default:
9264                 break;
9265         }
9266
9267         require_hard_reset |= !!sei_data->hdr.is_critical;
9268
9269         return require_hard_reset;
9270 }
9271
9272 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type,
9273                                 u64 intr_cause_data)
9274 {
9275         if (intr_cause_data) {
9276                 gaudi2_print_event(hdev, event_type, true,
9277                         "temperature error cause: %#llx", intr_cause_data);
9278                 return 1;
9279         }
9280
9281         return 0;
9282 }
9283
9284 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data)
9285 {
9286         u32 i, error_count = 0;
9287
9288         for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++)
9289                 if (intr_cause_data & hbm_mc_spi[i].mask) {
9290                         dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n",
9291                                 hbm_mc_spi[i].cause);
9292                         error_count++;
9293                 }
9294
9295         return error_count;
9296 }
9297
9298 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9299 {
9300         ktime_t zero_time = ktime_set(0, 0);
9301
9302         mutex_lock(&hdev->clk_throttling.lock);
9303
9304         switch (event_type) {
9305         case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
9306                 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
9307                 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
9308                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
9309                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
9310                 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n");
9311                 break;
9312
9313         case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
9314                 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
9315                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
9316                 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n");
9317                 break;
9318
9319         case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
9320                 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
9321                 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
9322                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
9323                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
9324                 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9325                 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n");
9326                 break;
9327
9328         case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
9329                 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
9330                 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
9331                 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9332                 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n");
9333                 break;
9334
9335         default:
9336                 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type);
9337                 break;
9338         }
9339
9340         mutex_unlock(&hdev->clk_throttling.lock);
9341 }
9342
9343 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type,
9344                                         struct cpucp_pkt_sync_err *sync_err)
9345 {
9346         struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9347
9348         gaudi2_print_event(hdev, event_type, false,
9349                 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
9350                 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci),
9351                 q->pi, atomic_read(&q->ci));
9352 }
9353
9354 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type)
9355 {
9356         u32 p2p_intr, msix_gw_intr, error_count = 0;
9357
9358         p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR);
9359         msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR);
9360
9361         if (p2p_intr) {
9362                 gaudi2_print_event(hdev, event_type, true,
9363                         "pcie p2p transaction terminated due to security, req_id(0x%x)",
9364                         RREG32(mmPCIE_WRAP_P2P_REQ_ID));
9365
9366                 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1);
9367                 error_count++;
9368         }
9369
9370         if (msix_gw_intr) {
9371                 gaudi2_print_event(hdev, event_type, true,
9372                         "pcie msi-x gen denied due to vector num check failure, vec(0x%X)",
9373                         RREG32(mmPCIE_WRAP_MSIX_GW_VEC));
9374
9375                 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1);
9376                 error_count++;
9377         }
9378
9379         return error_count;
9380 }
9381
9382 static int gaudi2_handle_pcie_drain(struct hl_device *hdev,
9383                         struct hl_eq_pcie_drain_ind_data *drain_data)
9384 {
9385         u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0;
9386
9387         cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data);
9388         lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw);
9389         lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw);
9390         hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw);
9391         hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw);
9392
9393         if (cause & BIT_ULL(0)) {
9394                 dev_err_ratelimited(hdev->dev,
9395                         "PCIE AXI drain LBW completed, read_err %u, write_err %u\n",
9396                         !!lbw_rd, !!lbw_wr);
9397                 error_count++;
9398         }
9399
9400         if (cause & BIT_ULL(1)) {
9401                 dev_err_ratelimited(hdev->dev,
9402                         "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n",
9403                         hbw_rd, hbw_wr);
9404                 error_count++;
9405         }
9406
9407         return error_count;
9408 }
9409
9410 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data)
9411 {
9412         u32 error_count = 0;
9413         int i;
9414
9415         for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) {
9416                 if (intr_cause_data & BIT_ULL(i)) {
9417                         dev_err_ratelimited(hdev->dev, "PSOC %s completed\n",
9418                                 gaudi2_psoc_axi_drain_interrupts_cause[i]);
9419                         error_count++;
9420                 }
9421         }
9422
9423         hl_check_for_glbl_errors(hdev);
9424
9425         return error_count;
9426 }
9427
9428 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type,
9429                                         struct cpucp_pkt_sync_err *sync_err)
9430 {
9431         struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9432
9433         gaudi2_print_event(hdev, event_type, false,
9434                 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
9435                 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));
9436 }
9437
9438 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type,
9439                                         struct hl_eq_engine_arc_intr_data *data)
9440 {
9441         struct hl_engine_arc_dccm_queue_full_irq *q;
9442         u32 intr_type, engine_id;
9443         u64 payload;
9444
9445         intr_type = le32_to_cpu(data->intr_type);
9446         engine_id = le32_to_cpu(data->engine_id);
9447         payload = le64_to_cpu(data->payload);
9448
9449         switch (intr_type) {
9450         case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ:
9451                 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload;
9452
9453                 gaudi2_print_event(hdev, event_type, true,
9454                                 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u",
9455                                 engine_id, intr_type, q->queue_index);
9456                 return 1;
9457         default:
9458                 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type");
9459                 return 0;
9460         }
9461 }
9462
9463 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
9464 {
9465         struct gaudi2_device *gaudi2 = hdev->asic_specific;
9466         bool reset_required = false, is_critical = false;
9467         u32 index, ctl, reset_flags = 0, error_count = 0;
9468         u64 event_mask = 0;
9469         u16 event_type;
9470
9471         ctl = le32_to_cpu(eq_entry->hdr.ctl);
9472         event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
9473
9474         if (event_type >= GAUDI2_EVENT_SIZE) {
9475                 dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
9476                                 event_type, GAUDI2_EVENT_SIZE - 1);
9477                 return;
9478         }
9479
9480         gaudi2->events_stat[event_type]++;
9481         gaudi2->events_stat_aggregate[event_type]++;
9482
9483         switch (event_type) {
9484         case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR:
9485                 fallthrough;
9486         case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR:
9487                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9488                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9489                 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
9490                 is_critical = eq_entry->ecc_data.is_critical;
9491                 error_count++;
9492                 break;
9493
9494         case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM:
9495                 fallthrough;
9496         case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM:
9497                 fallthrough;
9498         case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1:
9499                 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask);
9500                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9501                 break;
9502
9503         case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0:
9504                 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type);
9505                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9506                 break;
9507
9508         case GAUDI2_EVENT_CPU_AXI_ERR_RSP:
9509                 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type);
9510                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9511                 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
9512                 break;
9513
9514         case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
9515         case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
9516                 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask);
9517                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9518                 break;
9519
9520         case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
9521         case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
9522                 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
9523                 error_count = gaudi2_handle_rot_err(hdev, index, event_type,
9524                                         &eq_entry->razwi_with_intr_cause, &event_mask);
9525                 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9526                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9527                 break;
9528
9529         case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
9530                 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
9531                 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9532                                                 &eq_entry->razwi_with_intr_cause, &event_mask);
9533                 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9534                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9535                 break;
9536
9537         case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE:
9538                 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;
9539                 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9540                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9541                 break;
9542
9543         case GAUDI2_EVENT_TPC0_KERNEL_ERR:
9544         case GAUDI2_EVENT_TPC1_KERNEL_ERR:
9545         case GAUDI2_EVENT_TPC2_KERNEL_ERR:
9546         case GAUDI2_EVENT_TPC3_KERNEL_ERR:
9547         case GAUDI2_EVENT_TPC4_KERNEL_ERR:
9548         case GAUDI2_EVENT_TPC5_KERNEL_ERR:
9549         case GAUDI2_EVENT_TPC6_KERNEL_ERR:
9550         case GAUDI2_EVENT_TPC7_KERNEL_ERR:
9551         case GAUDI2_EVENT_TPC8_KERNEL_ERR:
9552         case GAUDI2_EVENT_TPC9_KERNEL_ERR:
9553         case GAUDI2_EVENT_TPC10_KERNEL_ERR:
9554         case GAUDI2_EVENT_TPC11_KERNEL_ERR:
9555         case GAUDI2_EVENT_TPC12_KERNEL_ERR:
9556         case GAUDI2_EVENT_TPC13_KERNEL_ERR:
9557         case GAUDI2_EVENT_TPC14_KERNEL_ERR:
9558         case GAUDI2_EVENT_TPC15_KERNEL_ERR:
9559         case GAUDI2_EVENT_TPC16_KERNEL_ERR:
9560         case GAUDI2_EVENT_TPC17_KERNEL_ERR:
9561         case GAUDI2_EVENT_TPC18_KERNEL_ERR:
9562         case GAUDI2_EVENT_TPC19_KERNEL_ERR:
9563         case GAUDI2_EVENT_TPC20_KERNEL_ERR:
9564         case GAUDI2_EVENT_TPC21_KERNEL_ERR:
9565         case GAUDI2_EVENT_TPC22_KERNEL_ERR:
9566         case GAUDI2_EVENT_TPC23_KERNEL_ERR:
9567         case GAUDI2_EVENT_TPC24_KERNEL_ERR:
9568                 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) /
9569                         (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR);
9570                 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9571                                         &eq_entry->razwi_with_intr_cause, &event_mask);
9572                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9573                 break;
9574
9575         case GAUDI2_EVENT_DEC0_SPI:
9576         case GAUDI2_EVENT_DEC1_SPI:
9577         case GAUDI2_EVENT_DEC2_SPI:
9578         case GAUDI2_EVENT_DEC3_SPI:
9579         case GAUDI2_EVENT_DEC4_SPI:
9580         case GAUDI2_EVENT_DEC5_SPI:
9581         case GAUDI2_EVENT_DEC6_SPI:
9582         case GAUDI2_EVENT_DEC7_SPI:
9583         case GAUDI2_EVENT_DEC8_SPI:
9584         case GAUDI2_EVENT_DEC9_SPI:
9585                 index = (event_type - GAUDI2_EVENT_DEC0_SPI) /
9586                                 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI);
9587                 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9588                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9589                 break;
9590
9591         case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
9592         case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
9593         case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
9594         case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
9595                 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
9596                                 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
9597                                                 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
9598                 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9599                 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9600                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9601                 break;
9602
9603         case GAUDI2_EVENT_MME0_QMAN_SW_ERROR:
9604         case GAUDI2_EVENT_MME1_QMAN_SW_ERROR:
9605         case GAUDI2_EVENT_MME2_QMAN_SW_ERROR:
9606         case GAUDI2_EVENT_MME3_QMAN_SW_ERROR:
9607                 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) /
9608                                 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR -
9609                                         GAUDI2_EVENT_MME0_QMAN_SW_ERROR);
9610                 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9611                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9612                 break;
9613
9614         case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID:
9615         case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID:
9616         case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID:
9617         case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID:
9618                 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) /
9619                                 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID -
9620                                         GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID);
9621                 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask);
9622                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9623                 break;
9624
9625         case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP:
9626         case GAUDI2_EVENT_KDMA0_CORE:
9627                 error_count = gaudi2_handle_kdma_core_event(hdev, event_type,
9628                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9629                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9630                 break;
9631
9632         case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE:
9633                 error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9634                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9635                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9636                 break;
9637
9638         case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE:
9639                 error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9640                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9641                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9642                 break;
9643
9644         case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR:
9645                 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type,
9646                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask);
9647                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9648                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9649                 break;
9650
9651         case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9652         case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9653         case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
9654         case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
9655                 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask);
9656                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9657                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9658                 break;
9659
9660         case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL:
9661                 error_count = gaudi2_handle_hif_fatal(hdev, event_type,
9662                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9663                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9664                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9665                 break;
9666
9667         case GAUDI2_EVENT_PMMU_FATAL_0:
9668                 error_count = gaudi2_handle_pif_fatal(hdev, event_type,
9669                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9670                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9671                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9672                 break;
9673
9674         case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT:
9675                 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask);
9676                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9677                 break;
9678
9679         case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE:
9680                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9681                 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) {
9682                         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9683                         reset_required = true;
9684                 }
9685                 error_count++;
9686                 break;
9687
9688         case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5:
9689                 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type,
9690                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9691                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9692                 break;
9693
9694         case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI:
9695                 error_count = gaudi2_handle_hbm_mc_spi(hdev,
9696                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9697                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9698                 break;
9699
9700         case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE:
9701                 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
9702                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9703                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9704                 break;
9705
9706         case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN:
9707                 error_count = gaudi2_handle_psoc_drain(hdev,
9708                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9709                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9710                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9711                 break;
9712
9713         case GAUDI2_EVENT_CPU_AXI_ECC:
9714                 error_count = GAUDI2_NA_EVENT_CAUSE;
9715                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9716                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9717                 break;
9718         case GAUDI2_EVENT_CPU_L2_RAM_ECC:
9719                 error_count = GAUDI2_NA_EVENT_CAUSE;
9720                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9721                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9722                 break;
9723         case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP:
9724         case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP:
9725         case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP:
9726         case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP:
9727                 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type,
9728                                                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9729                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9730                 break;
9731         case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B:
9732                 error_count = GAUDI2_NA_EVENT_CAUSE;
9733                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9734                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9735                 break;
9736         case GAUDI2_EVENT_PSOC_AXI_ERR_RSP:
9737                 error_count = GAUDI2_NA_EVENT_CAUSE;
9738                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9739                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9740                 break;
9741         case GAUDI2_EVENT_PSOC_PRSTN_FALL:
9742                 error_count = GAUDI2_NA_EVENT_CAUSE;
9743                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9744                 break;
9745         case GAUDI2_EVENT_PCIE_APB_TIMEOUT:
9746                 error_count = GAUDI2_NA_EVENT_CAUSE;
9747                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9748                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9749                 break;
9750         case GAUDI2_EVENT_PCIE_FATAL_ERR:
9751                 error_count = GAUDI2_NA_EVENT_CAUSE;
9752                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9753                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9754                 break;
9755         case GAUDI2_EVENT_TPC0_BMON_SPMU:
9756         case GAUDI2_EVENT_TPC1_BMON_SPMU:
9757         case GAUDI2_EVENT_TPC2_BMON_SPMU:
9758         case GAUDI2_EVENT_TPC3_BMON_SPMU:
9759         case GAUDI2_EVENT_TPC4_BMON_SPMU:
9760         case GAUDI2_EVENT_TPC5_BMON_SPMU:
9761         case GAUDI2_EVENT_TPC6_BMON_SPMU:
9762         case GAUDI2_EVENT_TPC7_BMON_SPMU:
9763         case GAUDI2_EVENT_TPC8_BMON_SPMU:
9764         case GAUDI2_EVENT_TPC9_BMON_SPMU:
9765         case GAUDI2_EVENT_TPC10_BMON_SPMU:
9766         case GAUDI2_EVENT_TPC11_BMON_SPMU:
9767         case GAUDI2_EVENT_TPC12_BMON_SPMU:
9768         case GAUDI2_EVENT_TPC13_BMON_SPMU:
9769         case GAUDI2_EVENT_TPC14_BMON_SPMU:
9770         case GAUDI2_EVENT_TPC15_BMON_SPMU:
9771         case GAUDI2_EVENT_TPC16_BMON_SPMU:
9772         case GAUDI2_EVENT_TPC17_BMON_SPMU:
9773         case GAUDI2_EVENT_TPC18_BMON_SPMU:
9774         case GAUDI2_EVENT_TPC19_BMON_SPMU:
9775         case GAUDI2_EVENT_TPC20_BMON_SPMU:
9776         case GAUDI2_EVENT_TPC21_BMON_SPMU:
9777         case GAUDI2_EVENT_TPC22_BMON_SPMU:
9778         case GAUDI2_EVENT_TPC23_BMON_SPMU:
9779         case GAUDI2_EVENT_TPC24_BMON_SPMU:
9780         case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU:
9781         case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU:
9782         case GAUDI2_EVENT_MME0_WAP_BMON_SPMU:
9783         case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU:
9784         case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU:
9785         case GAUDI2_EVENT_MME1_WAP_BMON_SPMU:
9786         case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU:
9787         case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU:
9788         case GAUDI2_EVENT_MME2_WAP_BMON_SPMU:
9789         case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU:
9790         case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU:
9791         case GAUDI2_EVENT_MME3_WAP_BMON_SPMU:
9792         case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU:
9793                 fallthrough;
9794         case GAUDI2_EVENT_DEC0_BMON_SPMU:
9795         case GAUDI2_EVENT_DEC1_BMON_SPMU:
9796         case GAUDI2_EVENT_DEC2_BMON_SPMU:
9797         case GAUDI2_EVENT_DEC3_BMON_SPMU:
9798         case GAUDI2_EVENT_DEC4_BMON_SPMU:
9799         case GAUDI2_EVENT_DEC5_BMON_SPMU:
9800         case GAUDI2_EVENT_DEC6_BMON_SPMU:
9801         case GAUDI2_EVENT_DEC7_BMON_SPMU:
9802         case GAUDI2_EVENT_DEC8_BMON_SPMU:
9803         case GAUDI2_EVENT_DEC9_BMON_SPMU:
9804         case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU:
9805                 error_count = GAUDI2_NA_EVENT_CAUSE;
9806                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9807                 break;
9808
9809         case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
9810         case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
9811         case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
9812         case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
9813                 gaudi2_print_clk_change_info(hdev, event_type, &event_mask);
9814                 error_count = GAUDI2_NA_EVENT_CAUSE;
9815                 break;
9816
9817         case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC:
9818                 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err);
9819                 error_count = GAUDI2_NA_EVENT_CAUSE;
9820                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9821                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9822                 break;
9823
9824         case GAUDI2_EVENT_PCIE_FLR_REQUESTED:
9825                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9826                 error_count = GAUDI2_NA_EVENT_CAUSE;
9827                 /* Do nothing- FW will handle it */
9828                 break;
9829
9830         case GAUDI2_EVENT_PCIE_P2P_MSIX:
9831                 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type);
9832                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9833                 break;
9834
9835         case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE:
9836                 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE;
9837                 error_count = gaudi2_handle_sm_err(hdev, event_type, index);
9838                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9839                 break;
9840
9841         case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR:
9842                 error_count = GAUDI2_NA_EVENT_CAUSE;
9843                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9844                 break;
9845
9846         case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
9847                 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n",
9848                                                 le64_to_cpu(eq_entry->data[0]));
9849                 error_count = GAUDI2_NA_EVENT_CAUSE;
9850                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9851                 break;
9852         case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT:
9853                 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n",
9854                                                 le64_to_cpu(eq_entry->data[0]));
9855                 error_count = GAUDI2_NA_EVENT_CAUSE;
9856                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9857                 break;
9858
9859         case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED:
9860                 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err);
9861                 error_count = GAUDI2_NA_EVENT_CAUSE;
9862                 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9863                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9864                 break;
9865
9866         case GAUDI2_EVENT_ARC_DCCM_FULL:
9867                 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data);
9868                 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9869                 break;
9870
9871         case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED:
9872         case GAUDI2_EVENT_CPU_DEV_RESET_REQ:
9873                 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9874                 error_count = GAUDI2_NA_EVENT_CAUSE;
9875                 is_critical = true;
9876                 break;
9877
9878         default:
9879                 if (gaudi2_irq_map_table[event_type].valid) {
9880                         dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
9881                                                 event_type);
9882                         error_count = GAUDI2_NA_EVENT_CAUSE;
9883                 }
9884         }
9885
9886         /* Make sure to dump an error in case no error cause was printed so far.
9887          * Note that although we have counted the errors, we use this number as
9888          * a boolean.
9889          */
9890         if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type))
9891                 gaudi2_print_event(hdev, event_type, true, "%d", event_type);
9892         else if (error_count == 0)
9893                 gaudi2_print_event(hdev, event_type, true,
9894                                 "No error cause for H/W event %u", event_type);
9895
9896         if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) ||
9897                                 reset_required) {
9898                 if (reset_required ||
9899                                 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD))
9900                         reset_flags |= HL_DRV_RESET_HARD;
9901
9902                 if (hdev->hard_reset_on_fw_events ||
9903                                 (hdev->asic_prop.fw_security_enabled && is_critical))
9904                         goto reset_device;
9905         }
9906
9907         /* Send unmask irq only for interrupts not classified as MSG */
9908         if (!gaudi2_irq_map_table[event_type].msg)
9909                 hl_fw_unmask_irq(hdev, event_type);
9910
9911         if (event_mask)
9912                 hl_notifier_event_send_all(hdev, event_mask);
9913
9914         return;
9915
9916 reset_device:
9917         if (hdev->asic_prop.fw_security_enabled && is_critical) {
9918                 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW;
9919                 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
9920         } else {
9921                 reset_flags |= HL_DRV_RESET_DELAY;
9922         }
9923         /* escalate general hw errors to critical/fatal error */
9924         if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR)
9925                 hl_handle_critical_hw_err(hdev, event_type, &event_mask);
9926
9927         event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
9928         hl_device_cond_reset(hdev, reset_flags, event_mask);
9929 }
9930
9931 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev,
9932                         struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr,
9933                         u32 hw_queue_id, u32 size, u64 addr, u32 val)
9934 {
9935         u32 ctl, pkt_size;
9936         int rc = 0;
9937
9938         ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);
9939         ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1);
9940         ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1);
9941         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1);
9942
9943         lin_dma_pkt->ctl = cpu_to_le32(ctl);
9944         lin_dma_pkt->src_addr = cpu_to_le64(val);
9945         lin_dma_pkt->dst_addr = cpu_to_le64(addr);
9946         lin_dma_pkt->tsize = cpu_to_le32(size);
9947
9948         pkt_size = sizeof(struct packet_lin_dma);
9949
9950         rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
9951         if (rc)
9952                 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n",
9953                                 hw_queue_id);
9954
9955         return rc;
9956 }
9957
9958 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val)
9959 {
9960         u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
9961                                         GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
9962                                         GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
9963                                         GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0};
9964         u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val,
9965                 old_mmubp, mmubp, num_of_pkts, busy, pkt_size;
9966         u64 comp_addr, cur_addr = addr, end_addr = addr + size;
9967         struct asic_fixed_properties *prop = &hdev->asic_prop;
9968         void *lin_dma_pkts_arr;
9969         dma_addr_t pkt_dma_addr;
9970         int rc = 0, dma_num = 0;
9971
9972         if (prop->edma_enabled_mask == 0) {
9973                 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
9974                 return -EIO;
9975         }
9976
9977         sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
9978         sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
9979         comp_addr = CFG_BASE + sob_addr;
9980         comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
9981                 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
9982         mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) |
9983                 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1);
9984
9985         /* Calculate how many lin dma pkts we'll need */
9986         num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G);
9987         pkt_size = sizeof(struct packet_lin_dma);
9988
9989         lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts,
9990                                         &pkt_dma_addr, GFP_KERNEL);
9991         if (!lin_dma_pkts_arr)
9992                 return -ENOMEM;
9993
9994         /*
9995          * set mmu bypass for the scrubbing - all ddmas are configured the same so save
9996          * only the first one to restore later
9997          * also set the sob addr for all edma cores for completion.
9998          * set QM as trusted to allow it to access physical address with MMU bp.
9999          */
10000         old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP);
10001         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10002                 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10003                         u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
10004                         u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10005
10006                         if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10007                                 continue;
10008
10009                         WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP +
10010                                         edma_offset, mmubp);
10011                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset,
10012                                         lower_32_bits(comp_addr));
10013                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset,
10014                                         upper_32_bits(comp_addr));
10015                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset,
10016                                         comp_val);
10017                         gaudi2_qman_set_test_mode(hdev,
10018                                         edma_queues_id[dcore] + 4 * edma_idx, true);
10019                 }
10020         }
10021
10022         WREG32(sob_addr, 0);
10023
10024         while (cur_addr < end_addr) {
10025                 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10026                         for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10027                                 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10028
10029                                 if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10030                                         continue;
10031
10032                                 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr);
10033
10034                                 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev,
10035                                         (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num,
10036                                         pkt_dma_addr + dma_num * pkt_size,
10037                                         edma_queues_id[dcore] + edma_idx * 4,
10038                                         chunk_size, cur_addr, val);
10039                                 if (rc)
10040                                         goto end;
10041
10042                                 dma_num++;
10043                                 cur_addr += chunk_size;
10044                                 if (cur_addr == end_addr)
10045                                         break;
10046                         }
10047                 }
10048         }
10049
10050         rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
10051         if (rc) {
10052                 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
10053                 goto end;
10054         }
10055 end:
10056         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10057                 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10058                         u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
10059                         u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10060
10061                         if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10062                                 continue;
10063
10064                         WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp);
10065                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0);
10066                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0);
10067                         WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0);
10068                         gaudi2_qman_set_test_mode(hdev,
10069                                         edma_queues_id[dcore] + 4 * edma_idx, false);
10070                 }
10071         }
10072
10073         WREG32(sob_addr, 0);
10074         hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr);
10075
10076         return rc;
10077 }
10078
10079 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val)
10080 {
10081         int rc;
10082         struct asic_fixed_properties *prop = &hdev->asic_prop;
10083         u64 size = prop->dram_end_address - prop->dram_user_base_address;
10084
10085         rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val);
10086
10087         if (rc)
10088                 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n",
10089                                 prop->dram_user_base_address, size);
10090         return rc;
10091 }
10092
10093 static int gaudi2_scrub_device_mem(struct hl_device *hdev)
10094 {
10095         int rc;
10096         struct asic_fixed_properties *prop = &hdev->asic_prop;
10097         u64 val = hdev->memory_scrub_val;
10098         u64 addr, size;
10099
10100         if (!hdev->memory_scrub)
10101                 return 0;
10102
10103         /* scrub SRAM */
10104         addr = prop->sram_user_base_address;
10105         size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET);
10106         dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n",
10107                         addr, addr + size, val);
10108         rc = gaudi2_memset_device_memory(hdev, addr, size, val);
10109         if (rc) {
10110                 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc);
10111                 return rc;
10112         }
10113
10114         /* scrub DRAM */
10115         rc = gaudi2_scrub_device_dram(hdev, val);
10116         if (rc) {
10117                 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc);
10118                 return rc;
10119         }
10120         return 0;
10121 }
10122
10123 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev)
10124 {
10125         u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr,
10126                 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr;
10127         u32 val, size, offset;
10128         int dcore_id;
10129
10130         offset = hdev->asic_prop.first_available_cq[0] * 4;
10131         cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset;
10132         cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset;
10133         cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset;
10134         cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset;
10135         cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset;
10136         cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset;
10137         size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 -
10138                         (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset);
10139
10140         /* memset dcore0 CQ registers */
10141         gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10142         gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10143         gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10144         gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10145         gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10146         gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10147
10148         cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET;
10149         cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET;
10150         cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET;
10151         cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET;
10152         cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET;
10153         cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET;
10154         size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0;
10155
10156         for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10157                 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10158                 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10159                 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10160                 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10161                 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10162                 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10163
10164                 cq_lbw_l_addr += DCORE_OFFSET;
10165                 cq_lbw_h_addr += DCORE_OFFSET;
10166                 cq_lbw_data_addr += DCORE_OFFSET;
10167                 cq_base_l_addr += DCORE_OFFSET;
10168                 cq_base_h_addr += DCORE_OFFSET;
10169                 cq_size_addr += DCORE_OFFSET;
10170         }
10171
10172         offset = hdev->asic_prop.first_available_user_mon[0] * 4;
10173         addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset;
10174         val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT;
10175         size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset);
10176
10177         /* memset dcore0 monitors */
10178         gaudi2_memset_device_lbw(hdev, addr, size, val);
10179
10180         addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset;
10181         gaudi2_memset_device_lbw(hdev, addr, size, 0);
10182
10183         mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET;
10184         mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET;
10185         size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0;
10186
10187         for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10188                 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val);
10189                 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0);
10190                 mon_sts_addr += DCORE_OFFSET;
10191                 mon_cfg_addr += DCORE_OFFSET;
10192         }
10193
10194         offset = hdev->asic_prop.first_available_user_sob[0] * 4;
10195         addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset;
10196         val = 0;
10197         size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 -
10198                         (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
10199
10200         /* memset dcore0 sobs */
10201         gaudi2_memset_device_lbw(hdev, addr, size, val);
10202
10203         addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET;
10204         size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0;
10205
10206         for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10207                 gaudi2_memset_device_lbw(hdev, addr, size, val);
10208                 addr += DCORE_OFFSET;
10209         }
10210
10211         /* Flush all WREG to prevent race */
10212         val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
10213 }
10214
10215 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev)
10216 {
10217         u32 reg_base, hw_queue_id;
10218
10219         for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0;
10220                                                         hw_queue_id += NUM_OF_PQ_PER_QMAN) {
10221                 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10222                         continue;
10223
10224                 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10225
10226                 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
10227                 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
10228         }
10229
10230         /* Flush all WREG to prevent race */
10231         RREG32(mmPDMA0_QM_ARB_CFG_0);
10232 }
10233
10234 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev)
10235 {
10236         u32 reg_base, hw_queue_id;
10237
10238         for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3;
10239                                                         hw_queue_id += NUM_OF_PQ_PER_QMAN) {
10240                 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10241                         continue;
10242
10243                 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10244
10245                 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
10246                 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
10247         }
10248
10249         /* Flush all WREG to prevent race */
10250         RREG32(mmPDMA0_QM_ARB_CFG_0);
10251 }
10252
10253 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid)
10254 {
10255         return 0;
10256 }
10257
10258 static void gaudi2_restore_phase_topology(struct hl_device *hdev)
10259 {
10260 }
10261
10262 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx,
10263                                                 struct dup_block_ctx *cfg_ctx)
10264 {
10265         u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off;
10266         u8 seq;
10267         int i;
10268
10269         for (i = 0 ; i < cfg_ctx->instances ; i++) {
10270                 seq = block_idx * cfg_ctx->instances + i;
10271
10272                 /* skip disabled instance */
10273                 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq)))
10274                         continue;
10275
10276                 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off,
10277                                         cfg_ctx->data);
10278         }
10279 }
10280
10281 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx,
10282                                                 u64 mask)
10283 {
10284         int i;
10285
10286         cfg_ctx->enabled_mask = mask;
10287
10288         for (i = 0 ; i < cfg_ctx->blocks ; i++)
10289                 gaudi2_init_block_instances(hdev, i, cfg_ctx);
10290 }
10291
10292 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx)
10293 {
10294         gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX);
10295 }
10296
10297 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr)
10298 {
10299         void *host_mem_virtual_addr;
10300         dma_addr_t host_mem_dma_addr;
10301         u64 reserved_va_base;
10302         u32 pos, size_left, size_to_dma;
10303         struct hl_ctx *ctx;
10304         int rc = 0;
10305
10306         /* Fetch the ctx */
10307         ctx = hl_get_compute_ctx(hdev);
10308         if (!ctx) {
10309                 dev_err(hdev->dev, "No ctx available\n");
10310                 return -EINVAL;
10311         }
10312
10313         /* Allocate buffers for read and for poll */
10314         host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr,
10315                                                                 GFP_KERNEL | __GFP_ZERO);
10316         if (host_mem_virtual_addr == NULL) {
10317                 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n");
10318                 rc = -ENOMEM;
10319                 goto put_ctx;
10320         }
10321
10322         /* Reserve VM region on asic side */
10323         reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M,
10324                                                 HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
10325         if (!reserved_va_base) {
10326                 dev_err(hdev->dev, "Failed to reserve vmem on asic\n");
10327                 rc = -ENOMEM;
10328                 goto free_data_buffer;
10329         }
10330
10331         /* Create mapping on asic side */
10332         mutex_lock(&hdev->mmu_lock);
10333
10334         rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M);
10335         if (rc) {
10336                 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
10337                 goto unreserve_va;
10338         }
10339
10340         rc = hl_mmu_invalidate_cache_range(hdev, false,
10341                                       MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV,
10342                                       ctx->asid, reserved_va_base, SZ_2M);
10343         if (rc) {
10344                 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
10345                 goto unreserve_va;
10346         }
10347
10348         mutex_unlock(&hdev->mmu_lock);
10349
10350         /* Enable MMU on KDMA */
10351         gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);
10352
10353         pos = 0;
10354         size_left = size;
10355         size_to_dma = SZ_2M;
10356
10357         while (size_left > 0) {
10358                 if (size_left < SZ_2M)
10359                         size_to_dma = size_left;
10360
10361                 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false);
10362                 if (rc)
10363                         break;
10364
10365                 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma);
10366
10367                 if (size_left <= SZ_2M)
10368                         break;
10369
10370                 pos += SZ_2M;
10371                 addr += SZ_2M;
10372                 size_left -= SZ_2M;
10373         }
10374
10375         gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
10376
10377         mutex_lock(&hdev->mmu_lock);
10378
10379         rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
10380         if (rc)
10381                 goto unreserve_va;
10382
10383         rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
10384                                       ctx->asid, reserved_va_base, SZ_2M);
10385
10386 unreserve_va:
10387         mutex_unlock(&hdev->mmu_lock);
10388         hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
10389 free_data_buffer:
10390         hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr);
10391 put_ctx:
10392         hl_ctx_put(ctx);
10393
10394         return rc;
10395 }
10396
10397 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx)
10398 {
10399         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10400         int min_alloc_order, rc;
10401
10402         if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
10403                 return 0;
10404
10405         hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,
10406                                                                 HOST_SPACE_INTERNAL_CB_SZ,
10407                                                                 &hdev->internal_cb_pool_dma_addr,
10408                                                                 GFP_KERNEL | __GFP_ZERO);
10409
10410         if (!hdev->internal_cb_pool_virt_addr)
10411                 return -ENOMEM;
10412
10413         min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev),
10414                                         gaudi2_get_wait_cb_size(hdev)));
10415
10416         hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);
10417         if (!hdev->internal_cb_pool) {
10418                 dev_err(hdev->dev, "Failed to create internal CB pool\n");
10419                 rc = -ENOMEM;
10420                 goto free_internal_cb_pool;
10421         }
10422
10423         rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr,
10424                                 HOST_SPACE_INTERNAL_CB_SZ, -1);
10425         if (rc) {
10426                 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n");
10427                 rc = -EFAULT;
10428                 goto destroy_internal_cb_pool;
10429         }
10430
10431         hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,
10432                                         HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
10433
10434         if (!hdev->internal_cb_va_base) {
10435                 rc = -ENOMEM;
10436                 goto destroy_internal_cb_pool;
10437         }
10438
10439         mutex_lock(&hdev->mmu_lock);
10440
10441         rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
10442                                         HOST_SPACE_INTERNAL_CB_SZ);
10443         if (rc)
10444                 goto unreserve_internal_cb_pool;
10445
10446         rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
10447         if (rc)
10448                 goto unmap_internal_cb_pool;
10449
10450         mutex_unlock(&hdev->mmu_lock);
10451
10452         return 0;
10453
10454 unmap_internal_cb_pool:
10455         hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10456 unreserve_internal_cb_pool:
10457         mutex_unlock(&hdev->mmu_lock);
10458         hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10459 destroy_internal_cb_pool:
10460         gen_pool_destroy(hdev->internal_cb_pool);
10461 free_internal_cb_pool:
10462         hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10463                                         hdev->internal_cb_pool_dma_addr);
10464
10465         return rc;
10466 }
10467
10468 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx)
10469 {
10470         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10471
10472         if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
10473                 return;
10474
10475         mutex_lock(&hdev->mmu_lock);
10476         hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10477         hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10478         hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
10479         mutex_unlock(&hdev->mmu_lock);
10480
10481         gen_pool_destroy(hdev->internal_cb_pool);
10482
10483         hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10484                                         hdev->internal_cb_pool_dma_addr);
10485 }
10486
10487 static void gaudi2_restore_user_registers(struct hl_device *hdev)
10488 {
10489         gaudi2_restore_user_sm_registers(hdev);
10490         gaudi2_restore_user_qm_registers(hdev);
10491 }
10492
10493 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
10494 {
10495         struct hl_device *hdev = ctx->hdev;
10496         struct asic_fixed_properties *prop = &hdev->asic_prop;
10497         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10498         int rc;
10499
10500         rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
10501                                 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true);
10502         if (rc)
10503                 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n",
10504                         RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
10505
10506         return rc;
10507 }
10508
10509 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
10510 {
10511         struct hl_device *hdev = ctx->hdev;
10512         struct asic_fixed_properties *prop = &hdev->asic_prop;
10513         int rc;
10514
10515         rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
10516                                 prop->pmmu.page_size, true);
10517         if (rc)
10518                 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n",
10519                         RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
10520 }
10521
10522 static int gaudi2_ctx_init(struct hl_ctx *ctx)
10523 {
10524         int rc;
10525
10526         rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);
10527         if (rc)
10528                 return rc;
10529
10530         /* No need to clear user registers if the device has just
10531          * performed reset, we restore only nic qm registers
10532          */
10533         if (ctx->hdev->reset_upon_device_release)
10534                 gaudi2_restore_nic_qm_registers(ctx->hdev);
10535         else
10536                 gaudi2_restore_user_registers(ctx->hdev);
10537
10538         rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx);
10539         if (rc)
10540                 return rc;
10541
10542         rc = gaudi2_map_virtual_msix_doorbell_memory(ctx);
10543         if (rc)
10544                 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10545
10546         return rc;
10547 }
10548
10549 static void gaudi2_ctx_fini(struct hl_ctx *ctx)
10550 {
10551         if (ctx->asid == HL_KERNEL_ASID_ID)
10552                 return;
10553
10554         gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10555
10556         gaudi2_unmap_virtual_msix_doorbell_memory(ctx);
10557 }
10558
10559 static int gaudi2_pre_schedule_cs(struct hl_cs *cs)
10560 {
10561         struct hl_device *hdev = cs->ctx->hdev;
10562         int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
10563         u32 mon_payload, sob_id, mon_id;
10564
10565         if (!cs_needs_completion(cs))
10566                 return 0;
10567
10568         /*
10569          * First 64 SOB/MON are reserved for driver for QMAN auto completion
10570          * mechanism. Each SOB/MON pair are used for a pending CS with the same
10571          * cyclic index. The SOB value is increased when each of the CS jobs is
10572          * completed. When the SOB reaches the number of CS jobs, the monitor
10573          * generates MSI-X interrupt.
10574          */
10575
10576         sob_id = mon_id = index;
10577         mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |
10578                                 (1 << CQ_ENTRY_READY_SHIFT) | index;
10579
10580         gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload,
10581                                 cs->jobs_cnt);
10582
10583         return 0;
10584 }
10585
10586 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
10587 {
10588         return HL_INVALID_QUEUE;
10589 }
10590
10591 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb)
10592 {
10593         struct hl_cb *cb = data;
10594         struct packet_msg_short *pkt;
10595         u32 value, ctl, pkt_size = sizeof(*pkt);
10596
10597         pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size);
10598         memset(pkt, 0, pkt_size);
10599
10600         /* Inc by 1, Mode ADD */
10601         value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1);
10602         value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1);
10603
10604         ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4);
10605         ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */
10606         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10607         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb);
10608         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10609
10610         pkt->value = cpu_to_le32(value);
10611         pkt->ctl = cpu_to_le32(ctl);
10612
10613         return size + pkt_size;
10614 }
10615
10616 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr)
10617 {
10618         u32 ctl, pkt_size = sizeof(*pkt);
10619
10620         memset(pkt, 0, pkt_size);
10621
10622         ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
10623         ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0);  /* MON base */
10624         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10625         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10626         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0);
10627
10628         pkt->value = cpu_to_le32(value);
10629         pkt->ctl = cpu_to_le32(ctl);
10630
10631         return pkt_size;
10632 }
10633
10634 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt,
10635                                         u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr)
10636 {
10637         u32 ctl, value, pkt_size = sizeof(*pkt);
10638         u8 mask;
10639
10640         if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) {
10641                 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask);
10642                 return 0;
10643         }
10644
10645         memset(pkt, 0, pkt_size);
10646
10647         value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8);
10648         value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val);
10649         value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/
10650         value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask);
10651
10652         ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
10653         ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */
10654         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10655         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10656         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10657
10658         pkt->value = cpu_to_le32(value);
10659         pkt->ctl = cpu_to_le32(ctl);
10660
10661         return pkt_size;
10662 }
10663
10664 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt)
10665 {
10666         u32 ctl, cfg, pkt_size = sizeof(*pkt);
10667
10668         memset(pkt, 0, pkt_size);
10669
10670         cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1);
10671         cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1);
10672         cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2);
10673
10674         ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE);
10675         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10676         ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10677
10678         pkt->cfg = cpu_to_le32(cfg);
10679         pkt->ctl = cpu_to_le32(ctl);
10680
10681         return pkt_size;
10682 }
10683
10684 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop)
10685 {
10686         struct hl_cb *cb = prop->data;
10687         void *buf = (void *) (uintptr_t) (cb->kernel_address);
10688
10689         u64 monitor_base, fence_addr = 0;
10690         u32 stream_index, size = prop->size;
10691         u16 msg_addr_offset;
10692
10693         stream_index = prop->q_idx % 4;
10694         fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] +
10695                         QM_FENCE2_OFFSET + stream_index * 4;
10696
10697         /*
10698          * monitor_base should be the content of the base0 address registers,
10699          * so it will be added to the msg short offsets
10700          */
10701         monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;
10702
10703         /* First monitor config packet: low address of the sync */
10704         msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) -
10705                                 monitor_base;
10706
10707         size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset);
10708
10709         /* Second monitor config packet: high address of the sync */
10710         msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) -
10711                                 monitor_base;
10712
10713         size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset);
10714
10715         /*
10716          * Third monitor config packet: the payload, i.e. what to write when the
10717          * sync triggers
10718          */
10719         msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) -
10720                                 monitor_base;
10721
10722         size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset);
10723
10724         /* Fourth monitor config packet: bind the monitor to a sync object */
10725         msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base;
10726
10727         size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask,
10728                                                 prop->sob_val, msg_addr_offset);
10729
10730         /* Fence packet */
10731         size += gaudi2_add_fence_pkt(buf + size);
10732
10733         return size;
10734 }
10735
10736 static void gaudi2_reset_sob(struct hl_device *hdev, void *data)
10737 {
10738         struct hl_hw_sob *hw_sob = data;
10739
10740         dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id);
10741
10742         WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0);
10743
10744         kref_init(&hw_sob->kref);
10745 }
10746
10747 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group)
10748 {
10749 }
10750
10751 static u64 gaudi2_get_device_time(struct hl_device *hdev)
10752 {
10753         u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
10754
10755         return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
10756 }
10757
10758 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs)
10759 {
10760         return 0;
10761 }
10762
10763 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx,
10764                                         struct hl_cs *cs, u32 wait_queue_id,
10765                                         u32 collective_engine_id, u32 encaps_signal_offset)
10766 {
10767         return -EINVAL;
10768 }
10769
10770 /*
10771  * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
10772  *                   to DMMU page-size address (64MB) before mapping it in
10773  *                   the MMU.
10774  * The operation is performed on both the virtual and physical addresses.
10775  * for device with 6 HBMs the scramble is:
10776  * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48]
10777  *
10778  * Example:
10779  * =============================================================================
10780  * Allocated DRAM  Reserved VA      scrambled VA for MMU mapping    Scrambled PA
10781  * Phys address                                                     in MMU last
10782  *                                                                    HOP
10783  * =============================================================================
10784  * PA1 0x3000000  VA1 0x9C000000  SVA1= (VA1/48M)*64M 0xD0000000  <- PA1/48M 0x1
10785  * PA2 0x9000000  VA2 0x9F000000  SVA2= (VA2/48M)*64M 0xD4000000  <- PA2/48M 0x3
10786  * =============================================================================
10787  */
10788 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr)
10789 {
10790         struct asic_fixed_properties *prop = &hdev->asic_prop;
10791         u32 divisor, mod_va;
10792         u64 div_va;
10793
10794         /* accept any address in the DRAM address space */
10795         if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE,
10796                                                                         VA_HBM_SPACE_END)) {
10797
10798                 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
10799                 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va);
10800                 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) |
10801                         (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) |
10802                         (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT);
10803         }
10804
10805         return raw_addr;
10806 }
10807
10808 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr)
10809 {
10810         struct asic_fixed_properties *prop = &hdev->asic_prop;
10811         u32 divisor, mod_va;
10812         u64 div_va;
10813
10814         /* accept any address in the DRAM address space */
10815         if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE,
10816                                                                         VA_HBM_SPACE_END)) {
10817
10818                 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
10819                 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK,
10820                                         PAGE_SIZE_64MB, &mod_va);
10821
10822                 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) +
10823                                         (div_va * divisor + mod_va));
10824         }
10825
10826         return scrambled_addr;
10827 }
10828
10829 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id)
10830 {
10831         u32 base = 0, dcore_id, dec_id;
10832
10833         if (core_id >= NUMBER_OF_DEC) {
10834                 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id);
10835                 goto out;
10836         }
10837
10838         if (core_id < 8) {
10839                 dcore_id = core_id / NUM_OF_DEC_PER_DCORE;
10840                 dec_id = core_id % NUM_OF_DEC_PER_DCORE;
10841
10842                 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET +
10843                                 dec_id * DCORE_VDEC_OFFSET;
10844         } else {
10845                 /* PCIe Shared Decoder */
10846                 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET);
10847         }
10848 out:
10849         return base;
10850 }
10851
10852 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr,
10853                                 u32 *block_size, u32 *block_id)
10854 {
10855         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10856         int i;
10857
10858         for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) {
10859                 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) {
10860                         *block_id = i;
10861                         if (block_size)
10862                                 *block_size = gaudi2->mapped_blocks[i].size;
10863                         return 0;
10864                 }
10865         }
10866
10867         dev_err(hdev->dev, "Invalid block address %#llx", block_addr);
10868
10869         return -EINVAL;
10870 }
10871
10872 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
10873                         u32 block_id, u32 block_size)
10874 {
10875         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10876         u64 offset_in_bar;
10877         u64 address;
10878         int rc;
10879
10880         if (block_id >= NUM_USER_MAPPED_BLOCKS) {
10881                 dev_err(hdev->dev, "Invalid block id %u", block_id);
10882                 return -EINVAL;
10883         }
10884
10885         /* we allow mapping only an entire block */
10886         if (block_size != gaudi2->mapped_blocks[block_id].size) {
10887                 dev_err(hdev->dev, "Invalid block size %u", block_size);
10888                 return -EINVAL;
10889         }
10890
10891         offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR;
10892
10893         address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar;
10894
10895         vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
10896                         VM_DONTCOPY | VM_NORESERVE);
10897
10898         rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT,
10899                         block_size, vma->vm_page_prot);
10900         if (rc)
10901                 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
10902
10903         return rc;
10904 }
10905
10906 static void gaudi2_enable_events_from_fw(struct hl_device *hdev)
10907 {
10908         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10909
10910         struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
10911         u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq);
10912
10913         if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
10914                 WREG32(irq_handler_offset,
10915                         gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id);
10916 }
10917
10918 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base)
10919 {
10920         switch (mmu_id) {
10921         case HW_CAP_DCORE0_DMMU0:
10922                 *mmu_base = mmDCORE0_HMMU0_MMU_BASE;
10923                 break;
10924         case HW_CAP_DCORE0_DMMU1:
10925                 *mmu_base = mmDCORE0_HMMU1_MMU_BASE;
10926                 break;
10927         case HW_CAP_DCORE0_DMMU2:
10928                 *mmu_base = mmDCORE0_HMMU2_MMU_BASE;
10929                 break;
10930         case HW_CAP_DCORE0_DMMU3:
10931                 *mmu_base = mmDCORE0_HMMU3_MMU_BASE;
10932                 break;
10933         case HW_CAP_DCORE1_DMMU0:
10934                 *mmu_base = mmDCORE1_HMMU0_MMU_BASE;
10935                 break;
10936         case HW_CAP_DCORE1_DMMU1:
10937                 *mmu_base = mmDCORE1_HMMU1_MMU_BASE;
10938                 break;
10939         case HW_CAP_DCORE1_DMMU2:
10940                 *mmu_base = mmDCORE1_HMMU2_MMU_BASE;
10941                 break;
10942         case HW_CAP_DCORE1_DMMU3:
10943                 *mmu_base = mmDCORE1_HMMU3_MMU_BASE;
10944                 break;
10945         case HW_CAP_DCORE2_DMMU0:
10946                 *mmu_base = mmDCORE2_HMMU0_MMU_BASE;
10947                 break;
10948         case HW_CAP_DCORE2_DMMU1:
10949                 *mmu_base = mmDCORE2_HMMU1_MMU_BASE;
10950                 break;
10951         case HW_CAP_DCORE2_DMMU2:
10952                 *mmu_base = mmDCORE2_HMMU2_MMU_BASE;
10953                 break;
10954         case HW_CAP_DCORE2_DMMU3:
10955                 *mmu_base = mmDCORE2_HMMU3_MMU_BASE;
10956                 break;
10957         case HW_CAP_DCORE3_DMMU0:
10958                 *mmu_base = mmDCORE3_HMMU0_MMU_BASE;
10959                 break;
10960         case HW_CAP_DCORE3_DMMU1:
10961                 *mmu_base = mmDCORE3_HMMU1_MMU_BASE;
10962                 break;
10963         case HW_CAP_DCORE3_DMMU2:
10964                 *mmu_base = mmDCORE3_HMMU2_MMU_BASE;
10965                 break;
10966         case HW_CAP_DCORE3_DMMU3:
10967                 *mmu_base = mmDCORE3_HMMU3_MMU_BASE;
10968                 break;
10969         case HW_CAP_PMMU:
10970                 *mmu_base = mmPMMU_HBW_MMU_BASE;
10971                 break;
10972         default:
10973                 return -EINVAL;
10974         }
10975
10976         return 0;
10977 }
10978
10979 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id)
10980 {
10981         bool is_pmmu = (mmu_id == HW_CAP_PMMU);
10982         struct gaudi2_device *gaudi2 = hdev->asic_specific;
10983         u32 mmu_base;
10984
10985         if (!(gaudi2->hw_cap_initialized & mmu_id))
10986                 return;
10987
10988         if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base))
10989                 return;
10990
10991         gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL);
10992         gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
10993 }
10994
10995 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)
10996 {
10997         u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES;
10998
10999         /* check all HMMUs */
11000         for (i = 0 ; i < num_of_hmmus ; i++) {
11001                 mmu_id = HW_CAP_DCORE0_DMMU0 << i;
11002
11003                 if (mmu_cap_mask & mmu_id)
11004                         gaudi2_ack_mmu_error(hdev, mmu_id);
11005         }
11006
11007         /* check PMMU */
11008         if (mmu_cap_mask & HW_CAP_PMMU)
11009                 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU);
11010
11011         return 0;
11012 }
11013
11014 static void gaudi2_get_msi_info(__le32 *table)
11015 {
11016         table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX);
11017 }
11018
11019 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx)
11020 {
11021         switch (pll_idx) {
11022         case HL_GAUDI2_CPU_PLL: return CPU_PLL;
11023         case HL_GAUDI2_PCI_PLL: return PCI_PLL;
11024         case HL_GAUDI2_NIC_PLL: return NIC_PLL;
11025         case HL_GAUDI2_DMA_PLL: return DMA_PLL;
11026         case HL_GAUDI2_MESH_PLL: return MESH_PLL;
11027         case HL_GAUDI2_MME_PLL: return MME_PLL;
11028         case HL_GAUDI2_TPC_PLL: return TPC_PLL;
11029         case HL_GAUDI2_IF_PLL: return IF_PLL;
11030         case HL_GAUDI2_SRAM_PLL: return SRAM_PLL;
11031         case HL_GAUDI2_HBM_PLL: return HBM_PLL;
11032         case HL_GAUDI2_VID_PLL: return VID_PLL;
11033         case HL_GAUDI2_MSS_PLL: return MSS_PLL;
11034         default: return -EINVAL;
11035         }
11036 }
11037
11038 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map)
11039 {
11040         /* Not implemented */
11041         return 0;
11042 }
11043
11044 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon)
11045 {
11046         /* Not implemented */
11047         return 0;
11048 }
11049
11050 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset,
11051                                 struct hl_device *hdev, struct hl_mon_state_dump *mon)
11052 {
11053         /* Not implemented */
11054         return 0;
11055 }
11056
11057
11058 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset,
11059                                 u64 status_base_offset, enum hl_sync_engine_type engine_type,
11060                                 u32 engine_id, char **buf, size_t *size, size_t *offset)
11061 {
11062         /* Not implemented */
11063         return 0;
11064 }
11065
11066
11067 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = {
11068         .monitor_valid = gaudi2_monitor_valid,
11069         .print_single_monitor = gaudi2_print_single_monitor,
11070         .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map,
11071         .print_fences_single_engine = gaudi2_print_fences_single_engine,
11072 };
11073
11074 static void gaudi2_state_dump_init(struct hl_device *hdev)
11075 {
11076         /* Not implemented */
11077         hdev->state_dump_specs.props = gaudi2_state_dump_specs_props;
11078         hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs;
11079 }
11080
11081 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id)
11082 {
11083         return 0;
11084 }
11085
11086 static u32 *gaudi2_get_stream_master_qid_arr(void)
11087 {
11088         return NULL;
11089 }
11090
11091 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
11092                                 struct attribute_group *dev_vrm_attr_grp)
11093 {
11094         hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);
11095         hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp);
11096 }
11097
11098 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
11099                                         u32 page_size, u32 *real_page_size, bool is_dram_addr)
11100 {
11101         struct asic_fixed_properties *prop = &hdev->asic_prop;
11102
11103         /* for host pages the page size must be  */
11104         if (!is_dram_addr) {
11105                 if (page_size % mmu_prop->page_size)
11106                         goto page_size_err;
11107
11108                 *real_page_size = mmu_prop->page_size;
11109                 return 0;
11110         }
11111
11112         if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size))
11113                 goto page_size_err;
11114
11115         /*
11116          * MMU page size is different from DRAM page size (more precisely, DMMU page is greater
11117          * than DRAM page size).
11118          * for this reason work with the DRAM page size and let the MMU scrambling routine handle
11119          * this mismatch when calculating the address to place in the MMU page table.
11120          * (in that case also make sure that the dram_page_size is not greater than the
11121          * mmu page size)
11122          */
11123         *real_page_size = prop->dram_page_size;
11124
11125         return 0;
11126
11127 page_size_err:
11128         dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
11129                                                         page_size, mmu_prop->page_size >> 10);
11130         return -EFAULT;
11131 }
11132
11133 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data)
11134 {
11135         return -EOPNOTSUPP;
11136 }
11137
11138 int gaudi2_send_device_activity(struct hl_device *hdev, bool open)
11139 {
11140         struct gaudi2_device *gaudi2 = hdev->asic_specific;
11141
11142         if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
11143                 return 0;
11144
11145         return hl_fw_send_device_activity(hdev, open);
11146 }
11147
11148 static const struct hl_asic_funcs gaudi2_funcs = {
11149         .early_init = gaudi2_early_init,
11150         .early_fini = gaudi2_early_fini,
11151         .late_init = gaudi2_late_init,
11152         .late_fini = gaudi2_late_fini,
11153         .sw_init = gaudi2_sw_init,
11154         .sw_fini = gaudi2_sw_fini,
11155         .hw_init = gaudi2_hw_init,
11156         .hw_fini = gaudi2_hw_fini,
11157         .halt_engines = gaudi2_halt_engines,
11158         .suspend = gaudi2_suspend,
11159         .resume = gaudi2_resume,
11160         .mmap = gaudi2_mmap,
11161         .ring_doorbell = gaudi2_ring_doorbell,
11162         .pqe_write = gaudi2_pqe_write,
11163         .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent,
11164         .asic_dma_free_coherent = gaudi2_dma_free_coherent,
11165         .scrub_device_mem = gaudi2_scrub_device_mem,
11166         .scrub_device_dram = gaudi2_scrub_device_dram,
11167         .get_int_queue_base = NULL,
11168         .test_queues = gaudi2_test_queues,
11169         .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc,
11170         .asic_dma_pool_free = gaudi2_dma_pool_free,
11171         .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc,
11172         .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free,
11173         .asic_dma_unmap_single = gaudi2_dma_unmap_single,
11174         .asic_dma_map_single = gaudi2_dma_map_single,
11175         .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
11176         .cs_parser = gaudi2_cs_parser,
11177         .asic_dma_map_sgtable = hl_dma_map_sgtable,
11178         .add_end_of_cb_packets = NULL,
11179         .update_eq_ci = gaudi2_update_eq_ci,
11180         .context_switch = gaudi2_context_switch,
11181         .restore_phase_topology = gaudi2_restore_phase_topology,
11182         .debugfs_read_dma = gaudi2_debugfs_read_dma,
11183         .add_device_attr = gaudi2_add_device_attr,
11184         .handle_eqe = gaudi2_handle_eqe,
11185         .get_events_stat = gaudi2_get_events_stat,
11186         .read_pte = NULL,
11187         .write_pte = NULL,
11188         .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache,
11189         .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range,
11190         .mmu_prefetch_cache_range = NULL,
11191         .send_heartbeat = gaudi2_send_heartbeat,
11192         .debug_coresight = gaudi2_debug_coresight,
11193         .is_device_idle = gaudi2_is_device_idle,
11194         .compute_reset_late_init = gaudi2_compute_reset_late_init,
11195         .hw_queues_lock = gaudi2_hw_queues_lock,
11196         .hw_queues_unlock = gaudi2_hw_queues_unlock,
11197         .get_pci_id = gaudi2_get_pci_id,
11198         .get_eeprom_data = gaudi2_get_eeprom_data,
11199         .get_monitor_dump = gaudi2_get_monitor_dump,
11200         .send_cpu_message = gaudi2_send_cpu_message,
11201         .pci_bars_map = gaudi2_pci_bars_map,
11202         .init_iatu = gaudi2_init_iatu,
11203         .rreg = hl_rreg,
11204         .wreg = hl_wreg,
11205         .halt_coresight = gaudi2_halt_coresight,
11206         .ctx_init = gaudi2_ctx_init,
11207         .ctx_fini = gaudi2_ctx_fini,
11208         .pre_schedule_cs = gaudi2_pre_schedule_cs,
11209         .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq,
11210         .load_firmware_to_device = NULL,
11211         .load_boot_fit_to_device = NULL,
11212         .get_signal_cb_size = gaudi2_get_signal_cb_size,
11213         .get_wait_cb_size = gaudi2_get_wait_cb_size,
11214         .gen_signal_cb = gaudi2_gen_signal_cb,
11215         .gen_wait_cb = gaudi2_gen_wait_cb,
11216         .reset_sob = gaudi2_reset_sob,
11217         .reset_sob_group = gaudi2_reset_sob_group,
11218         .get_device_time = gaudi2_get_device_time,
11219         .pb_print_security_errors = gaudi2_pb_print_security_errors,
11220         .collective_wait_init_cs = gaudi2_collective_wait_init_cs,
11221         .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs,
11222         .get_dec_base_addr = gaudi2_get_dec_base_addr,
11223         .scramble_addr = gaudi2_mmu_scramble_addr,
11224         .descramble_addr = gaudi2_mmu_descramble_addr,
11225         .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors,
11226         .get_hw_block_id = gaudi2_get_hw_block_id,
11227         .hw_block_mmap = gaudi2_block_mmap,
11228         .enable_events_from_fw = gaudi2_enable_events_from_fw,
11229         .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error,
11230         .get_msi_info = gaudi2_get_msi_info,
11231         .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx,
11232         .init_firmware_preload_params = gaudi2_init_firmware_preload_params,
11233         .init_firmware_loader = gaudi2_init_firmware_loader,
11234         .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm,
11235         .state_dump_init = gaudi2_state_dump_init,
11236         .get_sob_addr = &gaudi2_get_sob_addr,
11237         .set_pci_memory_regions = gaudi2_set_pci_memory_regions,
11238         .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr,
11239         .check_if_razwi_happened = gaudi2_check_if_razwi_happened,
11240         .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size,
11241         .access_dev_mem = hl_access_dev_mem,
11242         .set_dram_bar_base = gaudi2_set_hbm_bar_base,
11243         .set_engine_cores = gaudi2_set_engine_cores,
11244         .set_engines = gaudi2_set_engines,
11245         .send_device_activity = gaudi2_send_device_activity,
11246         .set_dram_properties = gaudi2_set_dram_properties,
11247         .set_binning_masks = gaudi2_set_binning_masks,
11248 };
11249
11250 void gaudi2_set_asic_funcs(struct hl_device *hdev)
11251 {
11252         hdev->asic_funcs = &gaudi2_funcs;
11253 }