drivers/accel/ivpu/ivpu_hw_37xx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include "ivpu_drv.h"
   7 #include "ivpu_fw.h"
   8 #include "ivpu_hw_37xx_reg.h"
   9 #include "ivpu_hw_reg_io.h"
  10 #include "ivpu_hw.h"
  11 #include "ivpu_ipc.h"
  12 #include "ivpu_mmu.h"
  13 #include "ivpu_pm.h"
  14
  15 #define TILE_FUSE_ENABLE_BOTH        0x0
  16 #define TILE_SKU_BOTH_MTL            0x3630
  17
  18 /* Work point configuration values */
  19 #define CONFIG_1_TILE                0x01
  20 #define CONFIG_2_TILE                0x02
  21 #define PLL_RATIO_5_3                0x01
  22 #define PLL_RATIO_4_3                0x02
  23 #define WP_CONFIG(tile, ratio)       (((tile) << 8) | (ratio))
  24 #define WP_CONFIG_1_TILE_5_3_RATIO   WP_CONFIG(CONFIG_1_TILE, PLL_RATIO_5_3)
  25 #define WP_CONFIG_1_TILE_4_3_RATIO   WP_CONFIG(CONFIG_1_TILE, PLL_RATIO_4_3)
  26 #define WP_CONFIG_2_TILE_5_3_RATIO   WP_CONFIG(CONFIG_2_TILE, PLL_RATIO_5_3)
  27 #define WP_CONFIG_2_TILE_4_3_RATIO   WP_CONFIG(CONFIG_2_TILE, PLL_RATIO_4_3)
  28 #define WP_CONFIG_0_TILE_PLL_OFF     WP_CONFIG(0, 0)
  29
  30 #define PLL_REF_CLK_FREQ             (50 * 1000000)
  31 #define PLL_SIMULATION_FREQ          (10 * 1000000)
  32 #define PLL_DEFAULT_EPP_VALUE        0x80
  33
  34 #define TIM_SAFE_ENABLE              0xf1d0dead
  35 #define TIM_WATCHDOG_RESET_VALUE     0xffffffff
  36
  37 #define TIMEOUT_US                   (150 * USEC_PER_MSEC)
  38 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
  39 #define PLL_TIMEOUT_US               (1500 * USEC_PER_MSEC)
  40 #define IDLE_TIMEOUT_US              (500 * USEC_PER_MSEC)
  41
  42 #define ICB_0_IRQ_MASK ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  43                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  44                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  45                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  46                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  47                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  48                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  49
  50 #define ICB_1_IRQ_MASK ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  51                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  52                         (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  53
  54 #define ICB_0_1_IRQ_MASK ((((u64)ICB_1_IRQ_MASK) << 32) | ICB_0_IRQ_MASK)
  55
  56 #define BUTTRESS_IRQ_MASK ((REG_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE)) | \
  57                            (REG_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR)) | \
  58                            (REG_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, UFI_ERR)))
  59
  60 #define BUTTRESS_IRQ_ENABLE_MASK ((u32)~BUTTRESS_IRQ_MASK)
  61 #define BUTTRESS_IRQ_DISABLE_MASK ((u32)-1)
  62
  63 #define ITF_FIREWALL_VIOLATION_MASK ((REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  64                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  65                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  66                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  67                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  68                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  69                                      (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  70
  71 static char *ivpu_platform_to_str(u32 platform)
  72 {
  73         switch (platform) {
  74         case IVPU_PLATFORM_SILICON:
  75                 return "IVPU_PLATFORM_SILICON";
  76         case IVPU_PLATFORM_SIMICS:
  77                 return "IVPU_PLATFORM_SIMICS";
  78         case IVPU_PLATFORM_FPGA:
  79                 return "IVPU_PLATFORM_FPGA";
  80         default:
  81                 return "Invalid platform";
  82         }
  83 }
  84
  85 static void ivpu_hw_read_platform(struct ivpu_device *vdev)
  86 {
  87         u32 gen_ctrl = REGV_RD32(VPU_37XX_HOST_SS_GEN_CTRL);
  88         u32 platform = REG_GET_FLD(VPU_37XX_HOST_SS_GEN_CTRL, PS, gen_ctrl);
  89
  90         if  (platform == IVPU_PLATFORM_SIMICS || platform == IVPU_PLATFORM_FPGA)
  91                 vdev->platform = platform;
  92         else
  93                 vdev->platform = IVPU_PLATFORM_SILICON;
  94
  95         ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n",
  96                  ivpu_platform_to_str(vdev->platform), vdev->platform);
  97 }
  98
  99 static void ivpu_hw_wa_init(struct ivpu_device *vdev)
 100 {
 101         vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
 102         vdev->wa.clear_runtime_mem = false;
 103         vdev->wa.d3hot_after_power_off = true;
 104
 105         if (ivpu_device_id(vdev) == PCI_DEVICE_ID_MTL && ivpu_revision(vdev) < 4)
 106                 vdev->wa.interrupt_clear_with_0 = true;
 107
 108         IVPU_PRINT_WA(punit_disabled);
 109         IVPU_PRINT_WA(clear_runtime_mem);
 110         IVPU_PRINT_WA(d3hot_after_power_off);
 111         IVPU_PRINT_WA(interrupt_clear_with_0);
 112 }
 113
 114 static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
 115 {
 116         if (ivpu_is_simics(vdev) || ivpu_is_fpga(vdev)) {
 117                 vdev->timeout.boot = 100000;
 118                 vdev->timeout.jsm = 50000;
 119                 vdev->timeout.tdr = 2000000;
 120                 vdev->timeout.reschedule_suspend = 1000;
 121         } else {
 122                 vdev->timeout.boot = 1000;
 123                 vdev->timeout.jsm = 500;
 124                 vdev->timeout.tdr = 2000;
 125                 vdev->timeout.reschedule_suspend = 10;
 126         }
 127 }
 128
 129 static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev)
 130 {
 131         return REGB_POLL_FLD(VPU_37XX_BUTTRESS_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US);
 132 }
 133
 134 /* Send KMD initiated workpoint change */
 135 static int ivpu_pll_cmd_send(struct ivpu_device *vdev, u16 min_ratio, u16 max_ratio,
 136                              u16 target_ratio, u16 config)
 137 {
 138         int ret;
 139         u32 val;
 140
 141         ret = ivpu_pll_wait_for_cmd_send(vdev);
 142         if (ret) {
 143                 ivpu_err(vdev, "Failed to sync before WP request: %d\n", ret);
 144                 return ret;
 145         }
 146
 147         val = REGB_RD32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD0);
 148         val = REG_SET_FLD_NUM(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD0, MIN_RATIO, min_ratio, val);
 149         val = REG_SET_FLD_NUM(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD0, MAX_RATIO, max_ratio, val);
 150         REGB_WR32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD0, val);
 151
 152         val = REGB_RD32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD1);
 153         val = REG_SET_FLD_NUM(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD1, TARGET_RATIO, target_ratio, val);
 154         val = REG_SET_FLD_NUM(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD1, EPP, PLL_DEFAULT_EPP_VALUE, val);
 155         REGB_WR32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD1, val);
 156
 157         val = REGB_RD32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD2);
 158         val = REG_SET_FLD_NUM(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD2, CONFIG, config, val);
 159         REGB_WR32(VPU_37XX_BUTTRESS_WP_REQ_PAYLOAD2, val);
 160
 161         val = REGB_RD32(VPU_37XX_BUTTRESS_WP_REQ_CMD);
 162         val = REG_SET_FLD(VPU_37XX_BUTTRESS_WP_REQ_CMD, SEND, val);
 163         REGB_WR32(VPU_37XX_BUTTRESS_WP_REQ_CMD, val);
 164
 165         ret = ivpu_pll_wait_for_cmd_send(vdev);
 166         if (ret)
 167                 ivpu_err(vdev, "Failed to sync after WP request: %d\n", ret);
 168
 169         return ret;
 170 }
 171
 172 static int ivpu_pll_wait_for_lock(struct ivpu_device *vdev, bool enable)
 173 {
 174         u32 exp_val = enable ? 0x1 : 0x0;
 175
 176         if (IVPU_WA(punit_disabled))
 177                 return 0;
 178
 179         return REGB_POLL_FLD(VPU_37XX_BUTTRESS_PLL_STATUS, LOCK, exp_val, PLL_TIMEOUT_US);
 180 }
 181
 182 static int ivpu_pll_wait_for_status_ready(struct ivpu_device *vdev)
 183 {
 184         if (IVPU_WA(punit_disabled))
 185                 return 0;
 186
 187         return REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, READY, 1, PLL_TIMEOUT_US);
 188 }
 189
 190 static void ivpu_pll_init_frequency_ratios(struct ivpu_device *vdev)
 191 {
 192         struct ivpu_hw_info *hw = vdev->hw;
 193         u8 fuse_min_ratio, fuse_max_ratio, fuse_pn_ratio;
 194         u32 fmin_fuse, fmax_fuse;
 195
 196         fmin_fuse = REGB_RD32(VPU_37XX_BUTTRESS_FMIN_FUSE);
 197         fuse_min_ratio = REG_GET_FLD(VPU_37XX_BUTTRESS_FMIN_FUSE, MIN_RATIO, fmin_fuse);
 198         fuse_pn_ratio = REG_GET_FLD(VPU_37XX_BUTTRESS_FMIN_FUSE, PN_RATIO, fmin_fuse);
 199
 200         fmax_fuse = REGB_RD32(VPU_37XX_BUTTRESS_FMAX_FUSE);
 201         fuse_max_ratio = REG_GET_FLD(VPU_37XX_BUTTRESS_FMAX_FUSE, MAX_RATIO, fmax_fuse);
 202
 203         hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, fuse_min_ratio, fuse_max_ratio);
 204         hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, fuse_max_ratio);
 205         hw->pll.pn_ratio = clamp_t(u8, fuse_pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
 206 }
 207
 208 static int ivpu_hw_37xx_wait_for_vpuip_bar(struct ivpu_device *vdev)
 209 {
 210         return REGV_POLL_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, AON, 0, 100);
 211 }
 212
 213 static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable)
 214 {
 215         struct ivpu_hw_info *hw = vdev->hw;
 216         u16 target_ratio;
 217         u16 config;
 218         int ret;
 219
 220         if (IVPU_WA(punit_disabled)) {
 221                 ivpu_dbg(vdev, PM, "Skipping PLL request on %s\n",
 222                          ivpu_platform_to_str(vdev->platform));
 223                 return 0;
 224         }
 225
 226         if (enable) {
 227                 target_ratio = hw->pll.pn_ratio;
 228                 config = hw->config;
 229         } else {
 230                 target_ratio = 0;
 231                 config = 0;
 232         }
 233
 234         ivpu_dbg(vdev, PM, "PLL workpoint request: config 0x%04x pll ratio 0x%x\n",
 235                  config, target_ratio);
 236
 237         ret = ivpu_pll_cmd_send(vdev, hw->pll.min_ratio, hw->pll.max_ratio, target_ratio, config);
 238         if (ret) {
 239                 ivpu_err(vdev, "Failed to send PLL workpoint request: %d\n", ret);
 240                 return ret;
 241         }
 242
 243         ret = ivpu_pll_wait_for_lock(vdev, enable);
 244         if (ret) {
 245                 ivpu_err(vdev, "Timed out waiting for PLL lock\n");
 246                 return ret;
 247         }
 248
 249         if (enable) {
 250                 ret = ivpu_pll_wait_for_status_ready(vdev);
 251                 if (ret) {
 252                         ivpu_err(vdev, "Timed out waiting for PLL ready status\n");
 253                         return ret;
 254                 }
 255
 256                 ret = ivpu_hw_37xx_wait_for_vpuip_bar(vdev);
 257                 if (ret) {
 258                         ivpu_err(vdev, "Timed out waiting for VPUIP bar\n");
 259                         return ret;
 260                 }
 261         }
 262
 263         return 0;
 264 }
 265
 266 static int ivpu_pll_enable(struct ivpu_device *vdev)
 267 {
 268         return ivpu_pll_drive(vdev, true);
 269 }
 270
 271 static int ivpu_pll_disable(struct ivpu_device *vdev)
 272 {
 273         return ivpu_pll_drive(vdev, false);
 274 }
 275
 276 static void ivpu_boot_host_ss_rst_clr_assert(struct ivpu_device *vdev)
 277 {
 278         u32 val = 0;
 279
 280         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, TOP_NOC, val);
 281         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, DSS_MAS, val);
 282         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, MSS_MAS, val);
 283
 284         REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_CLR, val);
 285 }
 286
 287 static void ivpu_boot_host_ss_rst_drive(struct ivpu_device *vdev, bool enable)
 288 {
 289         u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_RST_SET);
 290
 291         if (enable) {
 292                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 293                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 294                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 295         } else {
 296                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 297                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 298                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 299         }
 300
 301         REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_SET, val);
 302 }
 303
 304 static void ivpu_boot_host_ss_clk_drive(struct ivpu_device *vdev, bool enable)
 305 {
 306         u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_CLK_SET);
 307
 308         if (enable) {
 309                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 310                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 311                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 312         } else {
 313                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 314                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 315                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 316         }
 317
 318         REGV_WR32(VPU_37XX_HOST_SS_CPR_CLK_SET, val);
 319 }
 320
 321 static int ivpu_boot_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 322 {
 323         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
 324
 325         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
 326                 return -EIO;
 327
 328         return 0;
 329 }
 330
 331 static int ivpu_boot_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 332 {
 333         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QACCEPTN);
 334
 335         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 336                 return -EIO;
 337
 338         return 0;
 339 }
 340
 341 static int ivpu_boot_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 342 {
 343         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QDENY);
 344
 345         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 346                 return -EIO;
 347
 348         return 0;
 349 }
 350
 351 static int ivpu_boot_top_noc_qrenqn_check(struct ivpu_device *vdev, u32 exp_val)
 352 {
 353         u32 val = REGV_RD32(MTL_VPU_TOP_NOC_QREQN);
 354
 355         if (!REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 356             !REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 357                 return -EIO;
 358
 359         return 0;
 360 }
 361
 362 static int ivpu_boot_top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 363 {
 364         u32 val = REGV_RD32(MTL_VPU_TOP_NOC_QACCEPTN);
 365
 366         if (!REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 367             !REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 368                 return -EIO;
 369
 370         return 0;
 371 }
 372
 373 static int ivpu_boot_top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 374 {
 375         u32 val = REGV_RD32(MTL_VPU_TOP_NOC_QDENY);
 376
 377         if (!REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 378             !REG_TEST_FLD_NUM(MTL_VPU_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 379                 return -EIO;
 380
 381         return 0;
 382 }
 383
 384 static int ivpu_boot_host_ss_configure(struct ivpu_device *vdev)
 385 {
 386         ivpu_boot_host_ss_rst_clr_assert(vdev);
 387
 388         return ivpu_boot_noc_qreqn_check(vdev, 0x0);
 389 }
 390
 391 static void ivpu_boot_vpu_idle_gen_disable(struct ivpu_device *vdev)
 392 {
 393         REGV_WR32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, 0x0);
 394 }
 395
 396 static int ivpu_boot_host_ss_axi_drive(struct ivpu_device *vdev, bool enable)
 397 {
 398         int ret;
 399         u32 val;
 400
 401         val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
 402         if (enable)
 403                 val = REG_SET_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 404         else
 405                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 406         REGV_WR32(VPU_37XX_HOST_SS_NOC_QREQN, val);
 407
 408         ret = ivpu_boot_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 409         if (ret) {
 410                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 411                 return ret;
 412         }
 413
 414         ret = ivpu_boot_noc_qdeny_check(vdev, 0x0);
 415         if (ret)
 416                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 417
 418         return ret;
 419 }
 420
 421 static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev)
 422 {
 423         return ivpu_boot_host_ss_axi_drive(vdev, true);
 424 }
 425
 426 static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable)
 427 {
 428         int ret;
 429         u32 val;
 430
 431         val = REGV_RD32(MTL_VPU_TOP_NOC_QREQN);
 432         if (enable) {
 433                 val = REG_SET_FLD(MTL_VPU_TOP_NOC_QREQN, CPU_CTRL, val);
 434                 val = REG_SET_FLD(MTL_VPU_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 435         } else {
 436                 val = REG_CLR_FLD(MTL_VPU_TOP_NOC_QREQN, CPU_CTRL, val);
 437                 val = REG_CLR_FLD(MTL_VPU_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 438         }
 439         REGV_WR32(MTL_VPU_TOP_NOC_QREQN, val);
 440
 441         ret = ivpu_boot_top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 442         if (ret) {
 443                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 444                 return ret;
 445         }
 446
 447         ret = ivpu_boot_top_noc_qdeny_check(vdev, 0x0);
 448         if (ret)
 449                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 450
 451         return ret;
 452 }
 453
 454 static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev)
 455 {
 456         return ivpu_boot_host_ss_top_noc_drive(vdev, true);
 457 }
 458
 459 static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable)
 460 {
 461         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 462
 463         if (enable)
 464                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 465         else
 466                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 467
 468         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 469 }
 470
 471 static void ivpu_boot_pwr_island_drive(struct ivpu_device *vdev, bool enable)
 472 {
 473         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0);
 474
 475         if (enable)
 476                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 477         else
 478                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 479
 480         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 481 }
 482
 483 static int ivpu_boot_wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
 484 {
 485         /* FPGA model (UPF) is not power aware, skipped Power Island polling */
 486         if (ivpu_is_fpga(vdev))
 487                 return 0;
 488
 489         return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU,
 490                              exp_val, PWR_ISLAND_STATUS_TIMEOUT_US);
 491 }
 492
 493 static void ivpu_boot_pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable)
 494 {
 495         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0);
 496
 497         if (enable)
 498                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 499         else
 500                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 501
 502         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, val);
 503 }
 504
 505 static void ivpu_boot_dpu_active_drive(struct ivpu_device *vdev, bool enable)
 506 {
 507         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE);
 508
 509         if (enable)
 510                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 511         else
 512                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 513
 514         REGV_WR32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, val);
 515 }
 516
 517 static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev)
 518 {
 519         int ret;
 520
 521         ivpu_boot_pwr_island_trickle_drive(vdev, true);
 522         ivpu_boot_pwr_island_drive(vdev, true);
 523
 524         ret = ivpu_boot_wait_for_pwr_island_status(vdev, 0x1);
 525         if (ret) {
 526                 ivpu_err(vdev, "Timed out waiting for power island status\n");
 527                 return ret;
 528         }
 529
 530         ret = ivpu_boot_top_noc_qrenqn_check(vdev, 0x0);
 531         if (ret) {
 532                 ivpu_err(vdev, "Failed qrenqn check %d\n", ret);
 533                 return ret;
 534         }
 535
 536         ivpu_boot_host_ss_clk_drive(vdev, true);
 537         ivpu_boot_pwr_island_isolation_drive(vdev, false);
 538         ivpu_boot_host_ss_rst_drive(vdev, true);
 539         ivpu_boot_dpu_active_drive(vdev, true);
 540
 541         return ret;
 542 }
 543
 544 static int ivpu_boot_pwr_domain_disable(struct ivpu_device *vdev)
 545 {
 546         ivpu_boot_dpu_active_drive(vdev, false);
 547         ivpu_boot_pwr_island_isolation_drive(vdev, true);
 548         ivpu_boot_pwr_island_trickle_drive(vdev, false);
 549         ivpu_boot_pwr_island_drive(vdev, false);
 550
 551         return ivpu_boot_wait_for_pwr_island_status(vdev, 0x0);
 552 }
 553
 554 static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev)
 555 {
 556         u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES);
 557
 558         val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, NOSNOOP_OVERRIDE_EN, val);
 559         val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AW_NOSNOOP_OVERRIDE, val);
 560         val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val);
 561
 562         REGV_WR32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 563 }
 564
 565 static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev)
 566 {
 567         u32 val = REGV_RD32(VPU_37XX_HOST_IF_TBU_MMUSSIDV);
 568
 569         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 570         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 571         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 572         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 573
 574         REGV_WR32(VPU_37XX_HOST_IF_TBU_MMUSSIDV, val);
 575 }
 576
 577 static void ivpu_boot_soc_cpu_boot(struct ivpu_device *vdev)
 578 {
 579         u32 val;
 580
 581         val = REGV_RD32(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC);
 582         val = REG_SET_FLD(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTRUN0, val);
 583
 584         val = REG_CLR_FLD(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTVEC, val);
 585         REGV_WR32(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 586
 587         val = REG_SET_FLD(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 588         REGV_WR32(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 589
 590         val = REG_CLR_FLD(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 591         REGV_WR32(MTL_VPU_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 592
 593         val = vdev->fw->entry_point >> 9;
 594         REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 595
 596         val = REG_SET_FLD(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, DONE, val);
 597         REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 598
 599         ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 600                  vdev->fw->entry_point == vdev->fw->cold_boot_entry_point ? "cold boot" : "resume");
 601 }
 602
 603 static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable)
 604 {
 605         int ret;
 606         u32 val;
 607
 608         ret = REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
 609         if (ret) {
 610                 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret);
 611                 return ret;
 612         }
 613
 614         val = REGB_RD32(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL);
 615         if (enable)
 616                 val = REG_SET_FLD(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL, I3, val);
 617         else
 618                 val = REG_CLR_FLD(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL, I3, val);
 619         REGB_WR32(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL, val);
 620
 621         ret = REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
 622         if (ret)
 623                 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret);
 624
 625         return ret;
 626 }
 627
 628 static int ivpu_hw_37xx_info_init(struct ivpu_device *vdev)
 629 {
 630         struct ivpu_hw_info *hw = vdev->hw;
 631
 632         hw->tile_fuse = TILE_FUSE_ENABLE_BOTH;
 633         hw->sku = TILE_SKU_BOTH_MTL;
 634         hw->config = WP_CONFIG_2_TILE_4_3_RATIO;
 635
 636         ivpu_pll_init_frequency_ratios(vdev);
 637
 638         ivpu_hw_init_range(&hw->ranges.global, 0x80000000, SZ_512M);
 639         ivpu_hw_init_range(&hw->ranges.user,   0xc0000000, 255 * SZ_1M);
 640         ivpu_hw_init_range(&hw->ranges.shave, 0x180000000, SZ_2G);
 641         ivpu_hw_init_range(&hw->ranges.dma,   0x200000000, SZ_8G);
 642
 643         ivpu_hw_read_platform(vdev);
 644         ivpu_hw_wa_init(vdev);
 645         ivpu_hw_timeouts_init(vdev);
 646
 647         return 0;
 648 }
 649
 650 static int ivpu_hw_37xx_reset(struct ivpu_device *vdev)
 651 {
 652         int ret = 0;
 653
 654         if (ivpu_boot_pwr_domain_disable(vdev)) {
 655                 ivpu_err(vdev, "Failed to disable power domain\n");
 656                 ret = -EIO;
 657         }
 658
 659         if (ivpu_pll_disable(vdev)) {
 660                 ivpu_err(vdev, "Failed to disable PLL\n");
 661                 ret = -EIO;
 662         }
 663
 664         return ret;
 665 }
 666
 667 static int ivpu_hw_37xx_d0i3_enable(struct ivpu_device *vdev)
 668 {
 669         int ret;
 670
 671         ret = ivpu_boot_d0i3_drive(vdev, true);
 672         if (ret)
 673                 ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret);
 674
 675         udelay(5); /* VPU requires 5 us to complete the transition */
 676
 677         return ret;
 678 }
 679
 680 static int ivpu_hw_37xx_d0i3_disable(struct ivpu_device *vdev)
 681 {
 682         int ret;
 683
 684         ret = ivpu_boot_d0i3_drive(vdev, false);
 685         if (ret)
 686                 ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret);
 687
 688         return ret;
 689 }
 690
 691 static int ivpu_hw_37xx_power_up(struct ivpu_device *vdev)
 692 {
 693         int ret;
 694
 695         ret = ivpu_hw_37xx_d0i3_disable(vdev);
 696         if (ret)
 697                 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
 698
 699         ret = ivpu_pll_enable(vdev);
 700         if (ret) {
 701                 ivpu_err(vdev, "Failed to enable PLL: %d\n", ret);
 702                 return ret;
 703         }
 704
 705         ret = ivpu_boot_host_ss_configure(vdev);
 706         if (ret) {
 707                 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
 708                 return ret;
 709         }
 710
 711         /*
 712          * The control circuitry for vpu_idle indication logic powers up active.
 713          * To ensure unnecessary low power mode signal from LRT during bring up,
 714          * KMD disables the circuitry prior to bringing up the Main Power island.
 715          */
 716         ivpu_boot_vpu_idle_gen_disable(vdev);
 717
 718         ret = ivpu_boot_pwr_domain_enable(vdev);
 719         if (ret) {
 720                 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
 721                 return ret;
 722         }
 723
 724         ret = ivpu_boot_host_ss_axi_enable(vdev);
 725         if (ret) {
 726                 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
 727                 return ret;
 728         }
 729
 730         ret = ivpu_boot_host_ss_top_noc_enable(vdev);
 731         if (ret)
 732                 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
 733
 734         return ret;
 735 }
 736
 737 static int ivpu_hw_37xx_boot_fw(struct ivpu_device *vdev)
 738 {
 739         ivpu_boot_no_snoop_enable(vdev);
 740         ivpu_boot_tbu_mmu_enable(vdev);
 741         ivpu_boot_soc_cpu_boot(vdev);
 742
 743         return 0;
 744 }
 745
 746 static bool ivpu_hw_37xx_is_idle(struct ivpu_device *vdev)
 747 {
 748         u32 val;
 749
 750         if (IVPU_WA(punit_disabled))
 751                 return true;
 752
 753         val = REGB_RD32(VPU_37XX_BUTTRESS_VPU_STATUS);
 754         return REG_TEST_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, READY, val) &&
 755                REG_TEST_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, IDLE, val);
 756 }
 757
 758 static int ivpu_hw_37xx_power_down(struct ivpu_device *vdev)
 759 {
 760         int ret = 0;
 761
 762         if (!ivpu_hw_37xx_is_idle(vdev))
 763                 ivpu_warn(vdev, "VPU not idle during power down\n");
 764
 765         if (ivpu_hw_37xx_reset(vdev)) {
 766                 ivpu_err(vdev, "Failed to reset VPU\n");
 767                 ret = -EIO;
 768         }
 769
 770         if (ivpu_hw_37xx_d0i3_enable(vdev)) {
 771                 ivpu_err(vdev, "Failed to enter D0I3\n");
 772                 ret = -EIO;
 773         }
 774
 775         return ret;
 776 }
 777
 778 static void ivpu_hw_37xx_wdt_disable(struct ivpu_device *vdev)
 779 {
 780         u32 val;
 781
 782         /* Enable writing and set non-zero WDT value */
 783         REGV_WR32(MTL_VPU_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 784         REGV_WR32(MTL_VPU_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 785
 786         /* Enable writing and disable watchdog timer */
 787         REGV_WR32(MTL_VPU_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 788         REGV_WR32(MTL_VPU_CPU_SS_TIM_WDOG_EN, 0);
 789
 790         /* Now clear the timeout interrupt */
 791         val = REGV_RD32(MTL_VPU_CPU_SS_TIM_GEN_CONFIG);
 792         val = REG_CLR_FLD(MTL_VPU_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 793         REGV_WR32(MTL_VPU_CPU_SS_TIM_GEN_CONFIG, val);
 794 }
 795
 796 static u32 ivpu_hw_37xx_pll_to_freq(u32 ratio, u32 config)
 797 {
 798         u32 pll_clock = PLL_REF_CLK_FREQ * ratio;
 799         u32 cpu_clock;
 800
 801         if ((config & 0xff) == PLL_RATIO_4_3)
 802                 cpu_clock = pll_clock * 2 / 4;
 803         else
 804                 cpu_clock = pll_clock * 2 / 5;
 805
 806         return cpu_clock;
 807 }
 808
 809 /* Register indirect accesses */
 810 static u32 ivpu_hw_37xx_reg_pll_freq_get(struct ivpu_device *vdev)
 811 {
 812         u32 pll_curr_ratio;
 813
 814         pll_curr_ratio = REGB_RD32(VPU_37XX_BUTTRESS_CURRENT_PLL);
 815         pll_curr_ratio &= VPU_37XX_BUTTRESS_CURRENT_PLL_RATIO_MASK;
 816
 817         if (!ivpu_is_silicon(vdev))
 818                 return PLL_SIMULATION_FREQ;
 819
 820         return ivpu_hw_37xx_pll_to_freq(pll_curr_ratio, vdev->hw->config);
 821 }
 822
 823 static u32 ivpu_hw_37xx_reg_telemetry_offset_get(struct ivpu_device *vdev)
 824 {
 825         return REGB_RD32(VPU_37XX_BUTTRESS_VPU_TELEMETRY_OFFSET);
 826 }
 827
 828 static u32 ivpu_hw_37xx_reg_telemetry_size_get(struct ivpu_device *vdev)
 829 {
 830         return REGB_RD32(VPU_37XX_BUTTRESS_VPU_TELEMETRY_SIZE);
 831 }
 832
 833 static u32 ivpu_hw_37xx_reg_telemetry_enable_get(struct ivpu_device *vdev)
 834 {
 835         return REGB_RD32(VPU_37XX_BUTTRESS_VPU_TELEMETRY_ENABLE);
 836 }
 837
 838 static void ivpu_hw_37xx_reg_db_set(struct ivpu_device *vdev, u32 db_id)
 839 {
 840         u32 reg_stride = MTL_VPU_CPU_SS_DOORBELL_1 - MTL_VPU_CPU_SS_DOORBELL_0;
 841         u32 val = REG_FLD(MTL_VPU_CPU_SS_DOORBELL_0, SET);
 842
 843         REGV_WR32I(MTL_VPU_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
 844 }
 845
 846 static u32 ivpu_hw_37xx_reg_ipc_rx_addr_get(struct ivpu_device *vdev)
 847 {
 848         return REGV_RD32(VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM);
 849 }
 850
 851 static u32 ivpu_hw_37xx_reg_ipc_rx_count_get(struct ivpu_device *vdev)
 852 {
 853         u32 count = REGV_RD32_SILENT(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT);
 854
 855         return REG_GET_FLD(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 856 }
 857
 858 static void ivpu_hw_37xx_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
 859 {
 860         REGV_WR32(MTL_VPU_CPU_SS_TIM_IPC_FIFO, vpu_addr);
 861 }
 862
 863 static void ivpu_hw_37xx_irq_clear(struct ivpu_device *vdev)
 864 {
 865         REGV_WR64(VPU_37XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK);
 866 }
 867
 868 static void ivpu_hw_37xx_irq_enable(struct ivpu_device *vdev)
 869 {
 870         REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK);
 871         REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK);
 872         REGB_WR32(VPU_37XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_ENABLE_MASK);
 873         REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
 874 }
 875
 876 static void ivpu_hw_37xx_irq_disable(struct ivpu_device *vdev)
 877 {
 878         REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
 879         REGB_WR32(VPU_37XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_DISABLE_MASK);
 880         REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
 881         REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, 0x0);
 882 }
 883
 884 static void ivpu_hw_37xx_irq_wdt_nce_handler(struct ivpu_device *vdev)
 885 {
 886         ivpu_err_ratelimited(vdev, "WDT NCE irq\n");
 887
 888         ivpu_pm_schedule_recovery(vdev);
 889 }
 890
 891 static void ivpu_hw_37xx_irq_wdt_mss_handler(struct ivpu_device *vdev)
 892 {
 893         ivpu_err_ratelimited(vdev, "WDT MSS irq\n");
 894
 895         ivpu_hw_wdt_disable(vdev);
 896         ivpu_pm_schedule_recovery(vdev);
 897 }
 898
 899 static void ivpu_hw_37xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
 900 {
 901         ivpu_err_ratelimited(vdev, "NOC Firewall irq\n");
 902
 903         ivpu_pm_schedule_recovery(vdev);
 904 }
 905
 906 /* Handler for IRQs from VPU core (irqV) */
 907 static u32 ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq)
 908 {
 909         u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
 910
 911         REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status);
 912
 913         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
 914                 ivpu_mmu_irq_evtq_handler(vdev);
 915
 916         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
 917                 ivpu_ipc_irq_handler(vdev);
 918
 919         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
 920                 ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
 921
 922         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
 923                 ivpu_mmu_irq_gerr_handler(vdev);
 924
 925         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
 926                 ivpu_hw_37xx_irq_wdt_mss_handler(vdev);
 927
 928         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
 929                 ivpu_hw_37xx_irq_wdt_nce_handler(vdev);
 930
 931         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
 932                 ivpu_hw_37xx_irq_noc_firewall_handler(vdev);
 933
 934         return status;
 935 }
 936
 937 /* Handler for IRQs from Buttress core (irqB) */
 938 static u32 ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
 939 {
 940         u32 status = REGB_RD32(VPU_37XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
 941         bool schedule_recovery = false;
 942
 943         if (status == 0)
 944                 return 0;
 945
 946         if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
 947                 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x",
 948                          REGB_RD32(VPU_37XX_BUTTRESS_CURRENT_PLL));
 949
 950         if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) {
 951                 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_37XX_BUTTRESS_ATS_ERR_LOG_0));
 952                 REGB_WR32(VPU_37XX_BUTTRESS_ATS_ERR_CLEAR, 0x1);
 953                 schedule_recovery = true;
 954         }
 955
 956         if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, UFI_ERR, status)) {
 957                 u32 ufi_log = REGB_RD32(VPU_37XX_BUTTRESS_UFI_ERR_LOG);
 958
 959                 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx",
 960                          ufi_log, REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, OPCODE, ufi_log),
 961                          REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, AXI_ID, ufi_log),
 962                          REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, CQ_ID, ufi_log));
 963                 REGB_WR32(VPU_37XX_BUTTRESS_UFI_ERR_CLEAR, 0x1);
 964                 schedule_recovery = true;
 965         }
 966
 967         /* This must be done after interrupts are cleared at the source. */
 968         if (IVPU_WA(interrupt_clear_with_0))
 969                 /*
 970                  * Writing 1 triggers an interrupt, so we can't perform read update write.
 971                  * Clear local interrupt status by writing 0 to all bits.
 972                  */
 973                 REGB_WR32(VPU_37XX_BUTTRESS_INTERRUPT_STAT, 0x0);
 974         else
 975                 REGB_WR32(VPU_37XX_BUTTRESS_INTERRUPT_STAT, status);
 976
 977         if (schedule_recovery)
 978                 ivpu_pm_schedule_recovery(vdev);
 979
 980         return status;
 981 }
 982
 983 static irqreturn_t ivpu_hw_37xx_irq_handler(int irq, void *ptr)
 984 {
 985         struct ivpu_device *vdev = ptr;
 986         u32 ret_irqv, ret_irqb;
 987
 988         REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
 989
 990         ret_irqv = ivpu_hw_37xx_irqv_handler(vdev, irq);
 991         ret_irqb = ivpu_hw_37xx_irqb_handler(vdev, irq);
 992
 993         /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
 994         REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
 995
 996         return IRQ_RETVAL(ret_irqb | ret_irqv);
 997 }
 998
 999 static void ivpu_hw_37xx_diagnose_failure(struct ivpu_device *vdev)
1000 {
1001         u32 irqv = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
1002         u32 irqb = REGB_RD32(VPU_37XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
1003
1004         if (ivpu_hw_37xx_reg_ipc_rx_count_get(vdev))
1005                 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1006
1007         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, irqv))
1008                 ivpu_err(vdev, "WDT MSS timeout detected\n");
1009
1010         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, irqv))
1011                 ivpu_err(vdev, "WDT NCE timeout detected\n");
1012
1013         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, irqv))
1014                 ivpu_err(vdev, "NOC Firewall irq detected\n");
1015
1016         if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, irqb))
1017                 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_37XX_BUTTRESS_ATS_ERR_LOG_0));
1018
1019         if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, UFI_ERR, irqb)) {
1020                 u32 ufi_log = REGB_RD32(VPU_37XX_BUTTRESS_UFI_ERR_LOG);
1021
1022                 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx",
1023                          ufi_log, REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, OPCODE, ufi_log),
1024                          REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, AXI_ID, ufi_log),
1025                          REG_GET_FLD(VPU_37XX_BUTTRESS_UFI_ERR_LOG, CQ_ID, ufi_log));
1026         }
1027 }
1028
1029 const struct ivpu_hw_ops ivpu_hw_37xx_ops = {
1030         .info_init = ivpu_hw_37xx_info_init,
1031         .power_up = ivpu_hw_37xx_power_up,
1032         .is_idle = ivpu_hw_37xx_is_idle,
1033         .power_down = ivpu_hw_37xx_power_down,
1034         .reset = ivpu_hw_37xx_reset,
1035         .boot_fw = ivpu_hw_37xx_boot_fw,
1036         .wdt_disable = ivpu_hw_37xx_wdt_disable,
1037         .diagnose_failure = ivpu_hw_37xx_diagnose_failure,
1038         .reg_pll_freq_get = ivpu_hw_37xx_reg_pll_freq_get,
1039         .reg_telemetry_offset_get = ivpu_hw_37xx_reg_telemetry_offset_get,
1040         .reg_telemetry_size_get = ivpu_hw_37xx_reg_telemetry_size_get,
1041         .reg_telemetry_enable_get = ivpu_hw_37xx_reg_telemetry_enable_get,
1042         .reg_db_set = ivpu_hw_37xx_reg_db_set,
1043         .reg_ipc_rx_addr_get = ivpu_hw_37xx_reg_ipc_rx_addr_get,
1044         .reg_ipc_rx_count_get = ivpu_hw_37xx_reg_ipc_rx_count_get,
1045         .reg_ipc_tx_set = ivpu_hw_37xx_reg_ipc_tx_set,
1046         .irq_clear = ivpu_hw_37xx_irq_clear,
1047         .irq_enable = ivpu_hw_37xx_irq_enable,
1048         .irq_disable = ivpu_hw_37xx_irq_disable,
1049         .irq_handler = ivpu_hw_37xx_irq_handler,
1050 };