drivers/accel/ivpu/ivpu_hw_40xx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include "ivpu_drv.h"
   7 #include "ivpu_fw.h"
   8 #include "ivpu_hw.h"
   9 #include "ivpu_hw_40xx_reg.h"
  10 #include "ivpu_hw_reg_io.h"
  11 #include "ivpu_ipc.h"
  12 #include "ivpu_mmu.h"
  13 #include "ivpu_pm.h"
  14
  15 #include <linux/dmi.h>
  16
  17 #define TILE_MAX_NUM                 6
  18 #define TILE_MAX_MASK                0x3f
  19
  20 #define LNL_HW_ID                    0x4040
  21
  22 #define SKU_TILE_SHIFT               0u
  23 #define SKU_TILE_MASK                0x0000ffffu
  24 #define SKU_HW_ID_SHIFT              16u
  25 #define SKU_HW_ID_MASK               0xffff0000u
  26
  27 #define PLL_CONFIG_DEFAULT           0x1
  28 #define PLL_CDYN_DEFAULT             0x80
  29 #define PLL_EPP_DEFAULT              0x80
  30 #define PLL_REF_CLK_FREQ             (50 * 1000000)
  31 #define PLL_RATIO_TO_FREQ(x)         ((x) * PLL_REF_CLK_FREQ)
  32
  33 #define PLL_PROFILING_FREQ_DEFAULT   38400000
  34 #define PLL_PROFILING_FREQ_HIGH      400000000
  35
  36 #define TIM_SAFE_ENABLE              0xf1d0dead
  37 #define TIM_WATCHDOG_RESET_VALUE     0xffffffff
  38
  39 #define TIMEOUT_US                   (150 * USEC_PER_MSEC)
  40 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
  41 #define PLL_TIMEOUT_US               (1500 * USEC_PER_MSEC)
  42
  43 #define WEIGHTS_DEFAULT              0xf711f711u
  44 #define WEIGHTS_ATS_DEFAULT          0x0000f711u
  45
  46 #define ICB_0_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  47                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  48                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  49                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  50                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  51                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  52                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  53
  54 #define ICB_1_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  55                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  56                         (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  57
  58 #define ICB_0_1_IRQ_MASK ((((u64)ICB_1_IRQ_MASK) << 32) | ICB_0_IRQ_MASK)
  59
  60 #define BUTTRESS_IRQ_MASK ((REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR)) | \
  61                            (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR)) | \
  62                            (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR)) | \
  63                            (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR)) | \
  64                            (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR)) | \
  65                            (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR)))
  66
  67 #define BUTTRESS_IRQ_ENABLE_MASK ((u32)~BUTTRESS_IRQ_MASK)
  68 #define BUTTRESS_IRQ_DISABLE_MASK ((u32)-1)
  69
  70 #define ITF_FIREWALL_VIOLATION_MASK ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  71                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  72                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  73                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  74                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  75                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  76                                      (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  77
  78 static char *ivpu_platform_to_str(u32 platform)
  79 {
  80         switch (platform) {
  81         case IVPU_PLATFORM_SILICON:
  82                 return "IVPU_PLATFORM_SILICON";
  83         case IVPU_PLATFORM_SIMICS:
  84                 return "IVPU_PLATFORM_SIMICS";
  85         case IVPU_PLATFORM_FPGA:
  86                 return "IVPU_PLATFORM_FPGA";
  87         default:
  88                 return "Invalid platform";
  89         }
  90 }
  91
  92 static const struct dmi_system_id ivpu_dmi_platform_simulation[] = {
  93         {
  94                 .ident = "Intel Simics",
  95                 .matches = {
  96                         DMI_MATCH(DMI_BOARD_NAME, "lnlrvp"),
  97                         DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
  98                         DMI_MATCH(DMI_BOARD_SERIAL, "123456789"),
  99                 },
 100         },
 101         {
 102                 .ident = "Intel Simics",
 103                 .matches = {
 104                         DMI_MATCH(DMI_BOARD_NAME, "Simics"),
 105                 },
 106         },
 107         { }
 108 };
 109
 110 static void ivpu_hw_read_platform(struct ivpu_device *vdev)
 111 {
 112         if (dmi_check_system(ivpu_dmi_platform_simulation))
 113                 vdev->platform = IVPU_PLATFORM_SIMICS;
 114         else
 115                 vdev->platform = IVPU_PLATFORM_SILICON;
 116
 117         ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n",
 118                  ivpu_platform_to_str(vdev->platform), vdev->platform);
 119 }
 120
 121 static void ivpu_hw_wa_init(struct ivpu_device *vdev)
 122 {
 123         vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
 124         vdev->wa.clear_runtime_mem = false;
 125
 126         if (ivpu_hw_gen(vdev) == IVPU_HW_40XX)
 127                 vdev->wa.disable_clock_relinquish = true;
 128
 129         IVPU_PRINT_WA(punit_disabled);
 130         IVPU_PRINT_WA(clear_runtime_mem);
 131         IVPU_PRINT_WA(disable_clock_relinquish);
 132 }
 133
 134 static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
 135 {
 136         if (ivpu_is_fpga(vdev)) {
 137                 vdev->timeout.boot = 100000;
 138                 vdev->timeout.jsm = 50000;
 139                 vdev->timeout.tdr = 2000000;
 140                 vdev->timeout.reschedule_suspend = 1000;
 141         } else if (ivpu_is_simics(vdev)) {
 142                 vdev->timeout.boot = 50;
 143                 vdev->timeout.jsm = 500;
 144                 vdev->timeout.tdr = 10000;
 145                 vdev->timeout.reschedule_suspend = 10;
 146         } else {
 147                 vdev->timeout.boot = 1000;
 148                 vdev->timeout.jsm = 500;
 149                 vdev->timeout.tdr = 2000;
 150                 vdev->timeout.reschedule_suspend = 10;
 151         }
 152 }
 153
 154 static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev)
 155 {
 156         return REGB_POLL_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US);
 157 }
 158
 159 static int ivpu_pll_cmd_send(struct ivpu_device *vdev, u16 min_ratio, u16 max_ratio,
 160                              u16 target_ratio, u16 epp, u16 config, u16 cdyn)
 161 {
 162         int ret;
 163         u32 val;
 164
 165         ret = ivpu_pll_wait_for_cmd_send(vdev);
 166         if (ret) {
 167                 ivpu_err(vdev, "Failed to sync before WP request: %d\n", ret);
 168                 return ret;
 169         }
 170
 171         val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0);
 172         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MIN_RATIO, min_ratio, val);
 173         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MAX_RATIO, max_ratio, val);
 174         REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, val);
 175
 176         val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1);
 177         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, TARGET_RATIO, target_ratio, val);
 178         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, EPP, epp, val);
 179         REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, val);
 180
 181         val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2);
 182         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CONFIG, config, val);
 183         val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CDYN, cdyn, val);
 184         REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, val);
 185
 186         val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_CMD);
 187         val = REG_SET_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, val);
 188         REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_CMD, val);
 189
 190         ret = ivpu_pll_wait_for_cmd_send(vdev);
 191         if (ret)
 192                 ivpu_err(vdev, "Failed to sync after WP request: %d\n", ret);
 193
 194         return ret;
 195 }
 196
 197 static int ivpu_pll_wait_for_status_ready(struct ivpu_device *vdev)
 198 {
 199         return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, 1, PLL_TIMEOUT_US);
 200 }
 201
 202 static int ivpu_wait_for_clock_own_resource_ack(struct ivpu_device *vdev)
 203 {
 204         if (ivpu_is_simics(vdev))
 205                 return 0;
 206
 207         return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US);
 208 }
 209
 210 static void ivpu_pll_init_frequency_ratios(struct ivpu_device *vdev)
 211 {
 212         struct ivpu_hw_info *hw = vdev->hw;
 213         u8 fuse_min_ratio, fuse_pn_ratio, fuse_max_ratio;
 214         u32 fmin_fuse, fmax_fuse;
 215
 216         fmin_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMIN_FUSE);
 217         fuse_min_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, MIN_RATIO, fmin_fuse);
 218         fuse_pn_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, PN_RATIO, fmin_fuse);
 219
 220         fmax_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMAX_FUSE);
 221         fuse_max_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMAX_FUSE, MAX_RATIO, fmax_fuse);
 222
 223         hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, fuse_min_ratio, fuse_max_ratio);
 224         hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, fuse_max_ratio);
 225         hw->pll.pn_ratio = clamp_t(u8, fuse_pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
 226 }
 227
 228 static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable)
 229 {
 230         u16 config = enable ? PLL_CONFIG_DEFAULT : 0;
 231         u16 cdyn = enable ? PLL_CDYN_DEFAULT : 0;
 232         u16 epp = enable ? PLL_EPP_DEFAULT : 0;
 233         struct ivpu_hw_info *hw = vdev->hw;
 234         u16 target_ratio = hw->pll.pn_ratio;
 235         int ret;
 236
 237         ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, epp: 0x%x, config: 0x%x, cdyn: 0x%x\n",
 238                  PLL_RATIO_TO_FREQ(target_ratio), epp, config, cdyn);
 239
 240         ret = ivpu_pll_cmd_send(vdev, hw->pll.min_ratio, hw->pll.max_ratio,
 241                                 target_ratio, epp, config, cdyn);
 242         if (ret) {
 243                 ivpu_err(vdev, "Failed to send PLL workpoint request: %d\n", ret);
 244                 return ret;
 245         }
 246
 247         if (enable) {
 248                 ret = ivpu_pll_wait_for_status_ready(vdev);
 249                 if (ret) {
 250                         ivpu_err(vdev, "Timed out waiting for PLL ready status\n");
 251                         return ret;
 252                 }
 253         }
 254
 255         return 0;
 256 }
 257
 258 static int ivpu_pll_enable(struct ivpu_device *vdev)
 259 {
 260         return ivpu_pll_drive(vdev, true);
 261 }
 262
 263 static int ivpu_pll_disable(struct ivpu_device *vdev)
 264 {
 265         return ivpu_pll_drive(vdev, false);
 266 }
 267
 268 static void ivpu_boot_host_ss_rst_drive(struct ivpu_device *vdev, bool enable)
 269 {
 270         u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN);
 271
 272         if (enable) {
 273                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 274                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 275                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 276         } else {
 277                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 278                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 279                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 280         }
 281
 282         REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val);
 283 }
 284
 285 static void ivpu_boot_host_ss_clk_drive(struct ivpu_device *vdev, bool enable)
 286 {
 287         u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN);
 288
 289         if (enable) {
 290                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 291                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 292                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 293         } else {
 294                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 295                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 296                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 297         }
 298
 299         REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val);
 300 }
 301
 302 static int ivpu_boot_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 303 {
 304         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
 305
 306         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
 307                 return -EIO;
 308
 309         return 0;
 310 }
 311
 312 static int ivpu_boot_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 313 {
 314         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN);
 315
 316         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 317                 return -EIO;
 318
 319         return 0;
 320 }
 321
 322 static int ivpu_boot_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 323 {
 324         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY);
 325
 326         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 327                 return -EIO;
 328
 329         return 0;
 330 }
 331
 332 static int ivpu_boot_top_noc_qrenqn_check(struct ivpu_device *vdev, u32 exp_val)
 333 {
 334         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 335
 336         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 337             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 338                 return -EIO;
 339
 340         return 0;
 341 }
 342
 343 static int ivpu_boot_top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 344 {
 345         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN);
 346
 347         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 348             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 349                 return -EIO;
 350
 351         return 0;
 352 }
 353
 354 static int ivpu_boot_top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 355 {
 356         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY);
 357
 358         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 359             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 360                 return -EIO;
 361
 362         return 0;
 363 }
 364
 365 static void ivpu_boot_idle_gen_drive(struct ivpu_device *vdev, bool enable)
 366 {
 367         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN);
 368
 369         if (enable)
 370                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 371         else
 372                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 373
 374         REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val);
 375 }
 376
 377 static int ivpu_boot_host_ss_check(struct ivpu_device *vdev)
 378 {
 379         int ret;
 380
 381         ret = ivpu_boot_noc_qreqn_check(vdev, 0x0);
 382         if (ret) {
 383                 ivpu_err(vdev, "Failed qreqn check: %d\n", ret);
 384                 return ret;
 385         }
 386
 387         ret = ivpu_boot_noc_qacceptn_check(vdev, 0x0);
 388         if (ret) {
 389                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 390                 return ret;
 391         }
 392
 393         ret = ivpu_boot_noc_qdeny_check(vdev, 0x0);
 394         if (ret)
 395                 ivpu_err(vdev, "Failed qdeny check %d\n", ret);
 396
 397         return ret;
 398 }
 399
 400 static int ivpu_boot_host_ss_axi_drive(struct ivpu_device *vdev, bool enable)
 401 {
 402         int ret;
 403         u32 val;
 404
 405         val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
 406         if (enable)
 407                 val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 408         else
 409                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 410         REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val);
 411
 412         ret = ivpu_boot_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 413         if (ret) {
 414                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 415                 return ret;
 416         }
 417
 418         ret = ivpu_boot_noc_qdeny_check(vdev, 0x0);
 419         if (ret) {
 420                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 421                 return ret;
 422         }
 423
 424         if (enable) {
 425                 REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT);
 426                 REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT);
 427         }
 428
 429         return ret;
 430 }
 431
 432 static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev)
 433 {
 434         return ivpu_boot_host_ss_axi_drive(vdev, true);
 435 }
 436
 437 static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable)
 438 {
 439         int ret;
 440         u32 val;
 441
 442         val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 443         if (enable) {
 444                 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 445                 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 446         } else {
 447                 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 448                 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 449         }
 450         REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val);
 451
 452         ret = ivpu_boot_top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 453         if (ret) {
 454                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 455                 return ret;
 456         }
 457
 458         ret = ivpu_boot_top_noc_qdeny_check(vdev, 0x0);
 459         if (ret)
 460                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 461
 462         return ret;
 463 }
 464
 465 static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev)
 466 {
 467         return ivpu_boot_host_ss_top_noc_drive(vdev, true);
 468 }
 469
 470 static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable)
 471 {
 472         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 473
 474         if (enable)
 475                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 476         else
 477                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 478
 479         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 480
 481         if (enable)
 482                 ndelay(500);
 483 }
 484
 485 static void ivpu_boot_pwr_island_drive(struct ivpu_device *vdev, bool enable)
 486 {
 487         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
 488
 489         if (enable)
 490                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 491         else
 492                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 493
 494         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 495
 496         if (!enable)
 497                 ndelay(500);
 498 }
 499
 500 static int ivpu_boot_wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
 501 {
 502         if (ivpu_is_fpga(vdev))
 503                 return 0;
 504
 505         return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU,
 506                              exp_val, PWR_ISLAND_STATUS_TIMEOUT_US);
 507 }
 508
 509 static void ivpu_boot_pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable)
 510 {
 511         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0);
 512
 513         if (enable)
 514                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 515         else
 516                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 517
 518         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val);
 519 }
 520
 521 static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev)
 522 {
 523         u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES);
 524
 525         val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val);
 526         val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val);
 527         val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
 528
 529         REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 530 }
 531
 532 static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev)
 533 {
 534         u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV);
 535
 536         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 537         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 538         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val);
 539         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val);
 540         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 541         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 542
 543         REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val);
 544 }
 545
 546 static int ivpu_boot_cpu_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 547 {
 548         u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN);
 549
 550         if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val))
 551                 return -EIO;
 552
 553         return 0;
 554 }
 555
 556 static int ivpu_boot_cpu_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 557 {
 558         u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY);
 559
 560         if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val))
 561                 return -EIO;
 562
 563         return 0;
 564 }
 565
 566 static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev)
 567 {
 568         int ret;
 569
 570         ret = ivpu_wait_for_clock_own_resource_ack(vdev);
 571         if (ret) {
 572                 ivpu_err(vdev, "Timed out waiting for clock own resource ACK\n");
 573                 return ret;
 574         }
 575
 576         ivpu_boot_pwr_island_trickle_drive(vdev, true);
 577         ivpu_boot_pwr_island_drive(vdev, true);
 578
 579         ret = ivpu_boot_wait_for_pwr_island_status(vdev, 0x1);
 580         if (ret) {
 581                 ivpu_err(vdev, "Timed out waiting for power island status\n");
 582                 return ret;
 583         }
 584
 585         ret = ivpu_boot_top_noc_qrenqn_check(vdev, 0x0);
 586         if (ret) {
 587                 ivpu_err(vdev, "Failed qrenqn check %d\n", ret);
 588                 return ret;
 589         }
 590
 591         ivpu_boot_host_ss_clk_drive(vdev, true);
 592         ivpu_boot_host_ss_rst_drive(vdev, true);
 593         ivpu_boot_pwr_island_isolation_drive(vdev, false);
 594
 595         return ret;
 596 }
 597
 598 static int ivpu_boot_soc_cpu_drive(struct ivpu_device *vdev, bool enable)
 599 {
 600         int ret;
 601         u32 val;
 602
 603         val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN);
 604         if (enable)
 605                 val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 606         else
 607                 val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 608         REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val);
 609
 610         ret = ivpu_boot_cpu_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 611         if (ret) {
 612                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 613                 return ret;
 614         }
 615
 616         ret = ivpu_boot_cpu_noc_qdeny_check(vdev, 0x0);
 617         if (ret)
 618                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 619
 620         return ret;
 621 }
 622
 623 static int ivpu_boot_soc_cpu_enable(struct ivpu_device *vdev)
 624 {
 625         return ivpu_boot_soc_cpu_drive(vdev, true);
 626 }
 627
 628 static int ivpu_boot_soc_cpu_boot(struct ivpu_device *vdev)
 629 {
 630         int ret;
 631         u32 val;
 632         u64 val64;
 633
 634         ret = ivpu_boot_soc_cpu_enable(vdev);
 635         if (ret) {
 636                 ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret);
 637                 return ret;
 638         }
 639
 640         val64 = vdev->fw->entry_point;
 641         val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1;
 642         REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64);
 643
 644         val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO);
 645         val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val);
 646         REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val);
 647
 648         ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 649                  ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume");
 650
 651         return 0;
 652 }
 653
 654 static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable)
 655 {
 656         int ret;
 657         u32 val;
 658
 659         ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
 660         if (ret) {
 661                 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret);
 662                 return ret;
 663         }
 664
 665         val = REGB_RD32(VPU_40XX_BUTTRESS_D0I3_CONTROL);
 666         if (enable)
 667                 val = REG_SET_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val);
 668         else
 669                 val = REG_CLR_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val);
 670         REGB_WR32(VPU_40XX_BUTTRESS_D0I3_CONTROL, val);
 671
 672         ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
 673         if (ret) {
 674                 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret);
 675                 return ret;
 676         }
 677
 678         return 0;
 679 }
 680
 681 static bool ivpu_tile_disable_check(u32 config)
 682 {
 683         /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */
 684         if (config == 0)
 685                 return true;
 686
 687         if (config > BIT(TILE_MAX_NUM - 1))
 688                 return false;
 689
 690         if ((config & (config - 1)) == 0)
 691                 return true;
 692
 693         return false;
 694 }
 695
 696 static int ivpu_hw_40xx_info_init(struct ivpu_device *vdev)
 697 {
 698         struct ivpu_hw_info *hw = vdev->hw;
 699         u32 tile_disable;
 700         u32 tile_enable;
 701         u32 fuse;
 702
 703         fuse = REGB_RD32(VPU_40XX_BUTTRESS_TILE_FUSE);
 704         if (!REG_TEST_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, VALID, fuse)) {
 705                 ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse);
 706                 return -EIO;
 707         }
 708
 709         tile_disable = REG_GET_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, CONFIG, fuse);
 710         if (!ivpu_tile_disable_check(tile_disable)) {
 711                 ivpu_err(vdev, "Fuse: Invalid tile disable config (0x%x)\n", tile_disable);
 712                 return -EIO;
 713         }
 714
 715         if (tile_disable)
 716                 ivpu_dbg(vdev, MISC, "Fuse: %d tiles enabled. Tile number %d disabled\n",
 717                          TILE_MAX_NUM - 1, ffs(tile_disable) - 1);
 718         else
 719                 ivpu_dbg(vdev, MISC, "Fuse: All %d tiles enabled\n", TILE_MAX_NUM);
 720
 721         tile_enable = (~tile_disable) & TILE_MAX_MASK;
 722
 723         hw->sku = REG_SET_FLD_NUM(SKU, HW_ID, LNL_HW_ID, hw->sku);
 724         hw->sku = REG_SET_FLD_NUM(SKU, TILE, tile_enable, hw->sku);
 725         hw->tile_fuse = tile_disable;
 726         hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
 727
 728         ivpu_pll_init_frequency_ratios(vdev);
 729
 730         ivpu_hw_init_range(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
 731         ivpu_hw_init_range(&vdev->hw->ranges.user,   0x80000000, SZ_256M);
 732         ivpu_hw_init_range(&vdev->hw->ranges.shave,  0x80000000 + SZ_256M, SZ_2G - SZ_256M);
 733         ivpu_hw_init_range(&vdev->hw->ranges.dma,   0x200000000, SZ_8G);
 734
 735         ivpu_hw_read_platform(vdev);
 736         ivpu_hw_wa_init(vdev);
 737         ivpu_hw_timeouts_init(vdev);
 738
 739         return 0;
 740 }
 741
 742 static int ivpu_hw_40xx_reset(struct ivpu_device *vdev)
 743 {
 744         int ret;
 745         u32 val;
 746
 747         ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US);
 748         if (ret) {
 749                 ivpu_err(vdev, "Wait for *_TRIGGER timed out\n");
 750                 return ret;
 751         }
 752
 753         val = REGB_RD32(VPU_40XX_BUTTRESS_IP_RESET);
 754         val = REG_SET_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, val);
 755         REGB_WR32(VPU_40XX_BUTTRESS_IP_RESET, val);
 756
 757         ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US);
 758         if (ret)
 759                 ivpu_err(vdev, "Timed out waiting for RESET completion\n");
 760
 761         return ret;
 762 }
 763
 764 static int ivpu_hw_40xx_d0i3_enable(struct ivpu_device *vdev)
 765 {
 766         int ret;
 767
 768         if (IVPU_WA(punit_disabled))
 769                 return 0;
 770
 771         ret = ivpu_boot_d0i3_drive(vdev, true);
 772         if (ret)
 773                 ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret);
 774
 775         udelay(5); /* VPU requires 5 us to complete the transition */
 776
 777         return ret;
 778 }
 779
 780 static int ivpu_hw_40xx_d0i3_disable(struct ivpu_device *vdev)
 781 {
 782         int ret;
 783
 784         if (IVPU_WA(punit_disabled))
 785                 return 0;
 786
 787         ret = ivpu_boot_d0i3_drive(vdev, false);
 788         if (ret)
 789                 ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret);
 790
 791         return ret;
 792 }
 793
 794 static void ivpu_hw_40xx_profiling_freq_reg_set(struct ivpu_device *vdev)
 795 {
 796         u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
 797
 798         if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT)
 799                 val = REG_CLR_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val);
 800         else
 801                 val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val);
 802
 803         REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val);
 804 }
 805
 806 static void ivpu_hw_40xx_ats_print(struct ivpu_device *vdev)
 807 {
 808         ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n",
 809                  REGB_RD32(VPU_40XX_BUTTRESS_HM_ATS) ? "Enable" : "Disable");
 810 }
 811
 812 static void ivpu_hw_40xx_clock_relinquish_disable(struct ivpu_device *vdev)
 813 {
 814         u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
 815
 816         val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, DISABLE_CLK_RELINQUISH, val);
 817         REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val);
 818 }
 819
 820 static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev)
 821 {
 822         int ret;
 823
 824         ret = ivpu_hw_40xx_reset(vdev);
 825         if (ret) {
 826                 ivpu_err(vdev, "Failed to reset HW: %d\n", ret);
 827                 return ret;
 828         }
 829
 830         ret = ivpu_hw_40xx_d0i3_disable(vdev);
 831         if (ret)
 832                 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
 833
 834         ret = ivpu_pll_enable(vdev);
 835         if (ret) {
 836                 ivpu_err(vdev, "Failed to enable PLL: %d\n", ret);
 837                 return ret;
 838         }
 839
 840         if (IVPU_WA(disable_clock_relinquish))
 841                 ivpu_hw_40xx_clock_relinquish_disable(vdev);
 842         ivpu_hw_40xx_profiling_freq_reg_set(vdev);
 843         ivpu_hw_40xx_ats_print(vdev);
 844
 845         ret = ivpu_boot_host_ss_check(vdev);
 846         if (ret) {
 847                 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
 848                 return ret;
 849         }
 850
 851         ivpu_boot_idle_gen_drive(vdev, false);
 852
 853         ret = ivpu_boot_pwr_domain_enable(vdev);
 854         if (ret) {
 855                 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
 856                 return ret;
 857         }
 858
 859         ret = ivpu_boot_host_ss_axi_enable(vdev);
 860         if (ret) {
 861                 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
 862                 return ret;
 863         }
 864
 865         ret = ivpu_boot_host_ss_top_noc_enable(vdev);
 866         if (ret)
 867                 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
 868
 869         return ret;
 870 }
 871
 872 static int ivpu_hw_40xx_boot_fw(struct ivpu_device *vdev)
 873 {
 874         int ret;
 875
 876         ivpu_boot_no_snoop_enable(vdev);
 877         ivpu_boot_tbu_mmu_enable(vdev);
 878
 879         ret = ivpu_boot_soc_cpu_boot(vdev);
 880         if (ret)
 881                 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
 882
 883         return ret;
 884 }
 885
 886 static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev)
 887 {
 888         u32 val;
 889
 890         if (IVPU_WA(punit_disabled))
 891                 return true;
 892
 893         val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
 894         return REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, val) &&
 895                REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val);
 896 }
 897
 898 static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev)
 899 {
 900         int ret = 0;
 901
 902         if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_reset(vdev))
 903                 ivpu_warn(vdev, "Failed to reset the VPU\n");
 904
 905         if (ivpu_pll_disable(vdev)) {
 906                 ivpu_err(vdev, "Failed to disable PLL\n");
 907                 ret = -EIO;
 908         }
 909
 910         if (ivpu_hw_40xx_d0i3_enable(vdev)) {
 911                 ivpu_err(vdev, "Failed to enter D0I3\n");
 912                 ret = -EIO;
 913         }
 914
 915         return ret;
 916 }
 917
 918 static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev)
 919 {
 920         u32 val;
 921
 922         REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 923         REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 924
 925         REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 926         REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0);
 927
 928         val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG);
 929         val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 930         REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
 931 }
 932
 933 /* Register indirect accesses */
 934 static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev)
 935 {
 936         u32 pll_curr_ratio;
 937
 938         pll_curr_ratio = REGB_RD32(VPU_40XX_BUTTRESS_PLL_FREQ);
 939         pll_curr_ratio &= VPU_40XX_BUTTRESS_PLL_FREQ_RATIO_MASK;
 940
 941         return PLL_RATIO_TO_FREQ(pll_curr_ratio);
 942 }
 943
 944 static u32 ivpu_hw_40xx_reg_telemetry_offset_get(struct ivpu_device *vdev)
 945 {
 946         return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_OFFSET);
 947 }
 948
 949 static u32 ivpu_hw_40xx_reg_telemetry_size_get(struct ivpu_device *vdev)
 950 {
 951         return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_SIZE);
 952 }
 953
 954 static u32 ivpu_hw_40xx_reg_telemetry_enable_get(struct ivpu_device *vdev)
 955 {
 956         return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_ENABLE);
 957 }
 958
 959 static void ivpu_hw_40xx_reg_db_set(struct ivpu_device *vdev, u32 db_id)
 960 {
 961         u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0;
 962         u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET);
 963
 964         REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
 965 }
 966
 967 static u32 ivpu_hw_40xx_reg_ipc_rx_addr_get(struct ivpu_device *vdev)
 968 {
 969         return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM);
 970 }
 971
 972 static u32 ivpu_hw_40xx_reg_ipc_rx_count_get(struct ivpu_device *vdev)
 973 {
 974         u32 count = REGV_RD32_SILENT(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT);
 975
 976         return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 977 }
 978
 979 static void ivpu_hw_40xx_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
 980 {
 981         REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
 982 }
 983
 984 static void ivpu_hw_40xx_irq_clear(struct ivpu_device *vdev)
 985 {
 986         REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK);
 987 }
 988
 989 static void ivpu_hw_40xx_irq_enable(struct ivpu_device *vdev)
 990 {
 991         REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK);
 992         REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK);
 993         REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_ENABLE_MASK);
 994         REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
 995 }
 996
 997 static void ivpu_hw_40xx_irq_disable(struct ivpu_device *vdev)
 998 {
 999         REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
1000         REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_DISABLE_MASK);
1001         REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
1002         REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul);
1003 }
1004
1005 static void ivpu_hw_40xx_irq_wdt_nce_handler(struct ivpu_device *vdev)
1006 {
1007         /* TODO: For LNN hang consider engine reset instead of full recovery */
1008         ivpu_pm_schedule_recovery(vdev);
1009 }
1010
1011 static void ivpu_hw_40xx_irq_wdt_mss_handler(struct ivpu_device *vdev)
1012 {
1013         ivpu_hw_wdt_disable(vdev);
1014         ivpu_pm_schedule_recovery(vdev);
1015 }
1016
1017 static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
1018 {
1019         ivpu_pm_schedule_recovery(vdev);
1020 }
1021
1022 /* Handler for IRQs from VPU core (irqV) */
1023 static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
1024 {
1025         u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
1026         irqreturn_t ret = IRQ_NONE;
1027
1028         if (!status)
1029                 return IRQ_NONE;
1030
1031         REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
1032
1033         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1034                 ivpu_mmu_irq_evtq_handler(vdev);
1035
1036         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1037                 ret |= ivpu_ipc_irq_handler(vdev);
1038
1039         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1040                 ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1041
1042         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1043                 ivpu_mmu_irq_gerr_handler(vdev);
1044
1045         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1046                 ivpu_hw_40xx_irq_wdt_mss_handler(vdev);
1047
1048         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1049                 ivpu_hw_40xx_irq_wdt_nce_handler(vdev);
1050
1051         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1052                 ivpu_hw_40xx_irq_noc_firewall_handler(vdev);
1053
1054         return ret;
1055 }
1056
1057 /* Handler for IRQs from Buttress core (irqB) */
1058 static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
1059 {
1060         bool schedule_recovery = false;
1061         u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
1062
1063         if (status == 0)
1064                 return IRQ_NONE;
1065
1066         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
1067                 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE");
1068
1069         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) {
1070                 ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
1071                          REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1),
1072                          REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2));
1073                 REGB_WR32(VPU_40XX_BUTTRESS_ATS_ERR_CLEAR, 0x1);
1074                 schedule_recovery = true;
1075         }
1076
1077         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, status)) {
1078                 ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG));
1079                 REGB_WR32(VPU_40XX_BUTTRESS_CFI0_ERR_CLEAR, 0x1);
1080                 schedule_recovery = true;
1081         }
1082
1083         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, status)) {
1084                 ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG));
1085                 REGB_WR32(VPU_40XX_BUTTRESS_CFI1_ERR_CLEAR, 0x1);
1086                 schedule_recovery = true;
1087         }
1088
1089         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, status)) {
1090                 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x",
1091                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW),
1092                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH));
1093                 REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_CLEAR, 0x1);
1094                 schedule_recovery = true;
1095         }
1096
1097         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, status)) {
1098                 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x",
1099                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW),
1100                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH));
1101                 REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_CLEAR, 0x1);
1102                 schedule_recovery = true;
1103         }
1104
1105         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, status)) {
1106                 ivpu_err(vdev, "Survivability error detected\n");
1107                 schedule_recovery = true;
1108         }
1109
1110         /* This must be done after interrupts are cleared at the source. */
1111         REGB_WR32(VPU_40XX_BUTTRESS_INTERRUPT_STAT, status);
1112
1113         if (schedule_recovery)
1114                 ivpu_pm_schedule_recovery(vdev);
1115
1116         return IRQ_HANDLED;
1117 }
1118
1119 static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr)
1120 {
1121         struct ivpu_device *vdev = ptr;
1122         irqreturn_t ret = IRQ_NONE;
1123
1124         REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
1125
1126         ret |= ivpu_hw_40xx_irqv_handler(vdev, irq);
1127         ret |= ivpu_hw_40xx_irqb_handler(vdev, irq);
1128
1129         /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
1130         REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
1131
1132         if (ret & IRQ_WAKE_THREAD)
1133                 return IRQ_WAKE_THREAD;
1134
1135         return ret;
1136 }
1137
1138 static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev)
1139 {
1140         u32 irqv = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
1141         u32 irqb = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
1142
1143         if (ivpu_hw_40xx_reg_ipc_rx_count_get(vdev))
1144                 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1145
1146         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, irqv))
1147                 ivpu_err(vdev, "WDT MSS timeout detected\n");
1148
1149         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, irqv))
1150                 ivpu_err(vdev, "WDT NCE timeout detected\n");
1151
1152         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, irqv))
1153                 ivpu_err(vdev, "NOC Firewall irq detected\n");
1154
1155         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, irqb)) {
1156                 ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
1157                          REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1),
1158                          REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2));
1159         }
1160
1161         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, irqb))
1162                 ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG));
1163
1164         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, irqb))
1165                 ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG));
1166
1167         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, irqb))
1168                 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n",
1169                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW),
1170                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH));
1171
1172         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, irqb))
1173                 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n",
1174                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW),
1175                          REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH));
1176
1177         if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, irqb))
1178                 ivpu_err(vdev, "Survivability error detected\n");
1179 }
1180
1181 const struct ivpu_hw_ops ivpu_hw_40xx_ops = {
1182         .info_init = ivpu_hw_40xx_info_init,
1183         .power_up = ivpu_hw_40xx_power_up,
1184         .is_idle = ivpu_hw_40xx_is_idle,
1185         .power_down = ivpu_hw_40xx_power_down,
1186         .reset = ivpu_hw_40xx_reset,
1187         .boot_fw = ivpu_hw_40xx_boot_fw,
1188         .wdt_disable = ivpu_hw_40xx_wdt_disable,
1189         .diagnose_failure = ivpu_hw_40xx_diagnose_failure,
1190         .reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get,
1191         .reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get,
1192         .reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get,
1193         .reg_telemetry_enable_get = ivpu_hw_40xx_reg_telemetry_enable_get,
1194         .reg_db_set = ivpu_hw_40xx_reg_db_set,
1195         .reg_ipc_rx_addr_get = ivpu_hw_40xx_reg_ipc_rx_addr_get,
1196         .reg_ipc_rx_count_get = ivpu_hw_40xx_reg_ipc_rx_count_get,
1197         .reg_ipc_tx_set = ivpu_hw_40xx_reg_ipc_tx_set,
1198         .irq_clear = ivpu_hw_40xx_irq_clear,
1199         .irq_enable = ivpu_hw_40xx_irq_enable,
1200         .irq_disable = ivpu_hw_40xx_irq_disable,
1201         .irq_handler = ivpu_hw_40xx_irq_handler,
1202 };