drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <linux/dma-mapping.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "soc15_common.h"
  33 #include "psp_v3_1.h"
  34 #include "psp_v10_0.h"
  35 #include "psp_v11_0.h"
  36 #include "psp_v12_0.h"
  37
  38 #include "amdgpu_ras.h"
  39
  40 static int psp_sysfs_init(struct amdgpu_device *adev);
  41 static void psp_sysfs_fini(struct amdgpu_device *adev);
  42
  43 static int psp_load_smu_fw(struct psp_context *psp);
  44
  45 /*
  46  * Due to DF Cstate management centralized to PMFW, the firmware
  47  * loading sequence will be updated as below:
  48  *   - Load KDB
  49  *   - Load SYS_DRV
  50  *   - Load tOS
  51  *   - Load PMFW
  52  *   - Setup TMR
  53  *   - Load other non-psp fw
  54  *   - Load ASD
  55  *   - Load XGMI/RAS/HDCP/DTM TA if any
  56  *
  57  * This new sequence is required for
  58  *   - Arcturus
  59  *   - Navi12 and onwards
  60  */
  61 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  62 {
  63         struct amdgpu_device *adev = psp->adev;
  64
  65         psp->pmfw_centralized_cstate_management = false;
  66
  67         if (amdgpu_sriov_vf(adev))
  68                 return;
  69
  70         if (adev->flags & AMD_IS_APU)
  71                 return;
  72
  73         if ((adev->asic_type == CHIP_ARCTURUS) ||
  74             (adev->asic_type >= CHIP_NAVI12))
  75                 psp->pmfw_centralized_cstate_management = true;
  76 }
  77
  78 static int psp_early_init(void *handle)
  79 {
  80         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  81         struct psp_context *psp = &adev->psp;
  82
  83         switch (adev->asic_type) {
  84         case CHIP_VEGA10:
  85         case CHIP_VEGA12:
  86                 psp_v3_1_set_psp_funcs(psp);
  87                 psp->autoload_supported = false;
  88                 break;
  89         case CHIP_RAVEN:
  90                 psp_v10_0_set_psp_funcs(psp);
  91                 psp->autoload_supported = false;
  92                 break;
  93         case CHIP_VEGA20:
  94         case CHIP_ARCTURUS:
  95                 psp_v11_0_set_psp_funcs(psp);
  96                 psp->autoload_supported = false;
  97                 break;
  98         case CHIP_NAVI10:
  99         case CHIP_NAVI14:
 100         case CHIP_NAVI12:
 101         case CHIP_SIENNA_CICHLID:
 102         case CHIP_NAVY_FLOUNDER:
 103                 psp_v11_0_set_psp_funcs(psp);
 104                 psp->autoload_supported = true;
 105                 break;
 106         case CHIP_RENOIR:
 107                 psp_v12_0_set_psp_funcs(psp);
 108                 break;
 109         default:
 110                 return -EINVAL;
 111         }
 112
 113         psp->adev = adev;
 114
 115         psp_check_pmfw_centralized_cstate_management(psp);
 116
 117         return 0;
 118 }
 119
 120 static void psp_memory_training_fini(struct psp_context *psp)
 121 {
 122         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 123
 124         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 125         kfree(ctx->sys_cache);
 126         ctx->sys_cache = NULL;
 127 }
 128
 129 static int psp_memory_training_init(struct psp_context *psp)
 130 {
 131         int ret;
 132         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 133
 134         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 135                 DRM_DEBUG("memory training is not supported!\n");
 136                 return 0;
 137         }
 138
 139         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 140         if (ctx->sys_cache == NULL) {
 141                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 142                 ret = -ENOMEM;
 143                 goto Err_out;
 144         }
 145
 146         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 147                   ctx->train_data_size,
 148                   ctx->p2c_train_data_offset,
 149                   ctx->c2p_train_data_offset);
 150         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 151         return 0;
 152
 153 Err_out:
 154         psp_memory_training_fini(psp);
 155         return ret;
 156 }
 157
 158 static int psp_sw_init(void *handle)
 159 {
 160         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 161         struct psp_context *psp = &adev->psp;
 162         int ret;
 163
 164         ret = psp_init_microcode(psp);
 165         if (ret) {
 166                 DRM_ERROR("Failed to load psp firmware!\n");
 167                 return ret;
 168         }
 169
 170         ret = psp_memory_training_init(psp);
 171         if (ret) {
 172                 DRM_ERROR("Failed to initialize memory training!\n");
 173                 return ret;
 174         }
 175         ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 176         if (ret) {
 177                 DRM_ERROR("Failed to process memory training!\n");
 178                 return ret;
 179         }
 180
 181         if (adev->asic_type == CHIP_NAVI10) {
 182                 ret= psp_sysfs_init(adev);
 183                 if (ret) {
 184                         return ret;
 185                 }
 186         }
 187
 188         return 0;
 189 }
 190
 191 static int psp_sw_fini(void *handle)
 192 {
 193         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 194
 195         psp_memory_training_fini(&adev->psp);
 196         release_firmware(adev->psp.sos_fw);
 197         adev->psp.sos_fw = NULL;
 198         release_firmware(adev->psp.asd_fw);
 199         adev->psp.asd_fw = NULL;
 200         release_firmware(adev->psp.ta_fw);
 201         adev->psp.ta_fw = NULL;
 202
 203         if (adev->asic_type == CHIP_NAVI10)
 204                 psp_sysfs_fini(adev);
 205
 206         return 0;
 207 }
 208
 209 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 210                  uint32_t reg_val, uint32_t mask, bool check_changed)
 211 {
 212         uint32_t val;
 213         int i;
 214         struct amdgpu_device *adev = psp->adev;
 215
 216         for (i = 0; i < adev->usec_timeout; i++) {
 217                 val = RREG32(reg_index);
 218                 if (check_changed) {
 219                         if (val != reg_val)
 220                                 return 0;
 221                 } else {
 222                         if ((val & mask) == reg_val)
 223                                 return 0;
 224                 }
 225                 udelay(1);
 226         }
 227
 228         return -ETIME;
 229 }
 230
 231 static int
 232 psp_cmd_submit_buf(struct psp_context *psp,
 233                    struct amdgpu_firmware_info *ucode,
 234                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 235 {
 236         int ret;
 237         int index;
 238         int timeout = 2000;
 239         bool ras_intr = false;
 240         bool skip_unsupport = false;
 241
 242         mutex_lock(&psp->mutex);
 243
 244         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 245
 246         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 247
 248         index = atomic_inc_return(&psp->fence_value);
 249         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 250         if (ret) {
 251                 atomic_dec(&psp->fence_value);
 252                 mutex_unlock(&psp->mutex);
 253                 return ret;
 254         }
 255
 256         amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 257         while (*((unsigned int *)psp->fence_buf) != index) {
 258                 if (--timeout == 0)
 259                         break;
 260                 /*
 261                  * Shouldn't wait for timeout when err_event_athub occurs,
 262                  * because gpu reset thread triggered and lock resource should
 263                  * be released for psp resume sequence.
 264                  */
 265                 ras_intr = amdgpu_ras_intr_triggered();
 266                 if (ras_intr)
 267                         break;
 268                 msleep(1);
 269                 amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 270         }
 271
 272         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 273         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 274                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 275
 276         /* In some cases, psp response status is not 0 even there is no
 277          * problem while the command is submitted. Some version of PSP FW
 278          * doesn't write 0 to that field.
 279          * So here we would like to only print a warning instead of an error
 280          * during psp initialization to avoid breaking hw_init and it doesn't
 281          * return -EINVAL.
 282          */
 283         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 284                 if (ucode)
 285                         DRM_WARN("failed to load ucode id (%d) ",
 286                                   ucode->ucode_id);
 287                 DRM_WARN("psp command (0x%X) failed and response status is (0x%X)\n",
 288                          psp->cmd_buf_mem->cmd_id,
 289                          psp->cmd_buf_mem->resp.status);
 290                 if (!timeout) {
 291                         mutex_unlock(&psp->mutex);
 292                         return -EINVAL;
 293                 }
 294         }
 295
 296         /* get xGMI session id from response buffer */
 297         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 298
 299         if (ucode) {
 300                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 301                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 302         }
 303         mutex_unlock(&psp->mutex);
 304
 305         return ret;
 306 }
 307
 308 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 309                                  struct psp_gfx_cmd_resp *cmd,
 310                                  uint64_t tmr_mc, uint32_t size)
 311 {
 312         if (amdgpu_sriov_vf(psp->adev))
 313                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 314         else
 315                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 316         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 317         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 318         cmd->cmd.cmd_setup_tmr.buf_size = size;
 319 }
 320
 321 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 322                                       uint64_t pri_buf_mc, uint32_t size)
 323 {
 324         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 325         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 326         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 327         cmd->cmd.cmd_load_toc.toc_size = size;
 328 }
 329
 330 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 331 static int psp_load_toc(struct psp_context *psp,
 332                         uint32_t *tmr_size)
 333 {
 334         int ret;
 335         struct psp_gfx_cmd_resp *cmd;
 336
 337         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 338         if (!cmd)
 339                 return -ENOMEM;
 340         /* Copy toc to psp firmware private buffer */
 341         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 342         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 343
 344         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 345
 346         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 347                                  psp->fence_buf_mc_addr);
 348         if (!ret)
 349                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 350         kfree(cmd);
 351         return ret;
 352 }
 353
 354 /* Set up Trusted Memory Region */
 355 static int psp_tmr_init(struct psp_context *psp)
 356 {
 357         int ret;
 358         int tmr_size;
 359         void *tmr_buf;
 360         void **pptr;
 361
 362         /*
 363          * According to HW engineer, they prefer the TMR address be "naturally
 364          * aligned" , e.g. the start address be an integer divide of TMR size.
 365          *
 366          * Note: this memory need be reserved till the driver
 367          * uninitializes.
 368          */
 369         tmr_size = PSP_TMR_SIZE;
 370
 371         /* For ASICs support RLC autoload, psp will parse the toc
 372          * and calculate the total size of TMR needed */
 373         if (!amdgpu_sriov_vf(psp->adev) &&
 374             psp->toc_start_addr &&
 375             psp->toc_bin_size &&
 376             psp->fw_pri_buf) {
 377                 ret = psp_load_toc(psp, &tmr_size);
 378                 if (ret) {
 379                         DRM_ERROR("Failed to load toc\n");
 380                         return ret;
 381                 }
 382         }
 383
 384         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 385         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 386                                       AMDGPU_GEM_DOMAIN_VRAM,
 387                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 388
 389         return ret;
 390 }
 391
 392 static int psp_clear_vf_fw(struct psp_context *psp)
 393 {
 394         int ret;
 395         struct psp_gfx_cmd_resp *cmd;
 396
 397         if (!amdgpu_sriov_vf(psp->adev) || psp->adev->asic_type != CHIP_NAVI12)
 398                 return 0;
 399
 400         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 401         if (!cmd)
 402                 return -ENOMEM;
 403
 404         cmd->cmd_id = GFX_CMD_ID_CLEAR_VF_FW;
 405
 406         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 407         kfree(cmd);
 408
 409         return ret;
 410 }
 411
 412 static int psp_tmr_load(struct psp_context *psp)
 413 {
 414         int ret;
 415         struct psp_gfx_cmd_resp *cmd;
 416
 417         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 418         if (!cmd)
 419                 return -ENOMEM;
 420
 421         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 422                              amdgpu_bo_size(psp->tmr_bo));
 423         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 424                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 425
 426         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 427                                  psp->fence_buf_mc_addr);
 428
 429         kfree(cmd);
 430
 431         return ret;
 432 }
 433
 434 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 435                                         struct psp_gfx_cmd_resp *cmd)
 436 {
 437         if (amdgpu_sriov_vf(psp->adev))
 438                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 439         else
 440                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 441 }
 442
 443 static int psp_tmr_unload(struct psp_context *psp)
 444 {
 445         int ret;
 446         struct psp_gfx_cmd_resp *cmd;
 447
 448         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 449         if (!cmd)
 450                 return -ENOMEM;
 451
 452         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 453         DRM_INFO("free PSP TMR buffer\n");
 454
 455         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 456                                  psp->fence_buf_mc_addr);
 457
 458         kfree(cmd);
 459
 460         return ret;
 461 }
 462
 463 static int psp_tmr_terminate(struct psp_context *psp)
 464 {
 465         int ret;
 466         void *tmr_buf;
 467         void **pptr;
 468
 469         ret = psp_tmr_unload(psp);
 470         if (ret)
 471                 return ret;
 472
 473         /* free TMR memory buffer */
 474         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 475         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 476
 477         return 0;
 478 }
 479
 480 static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 481                                 uint64_t asd_mc, uint32_t size)
 482 {
 483         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 484         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 485         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 486         cmd->cmd.cmd_load_ta.app_len = size;
 487
 488         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = 0;
 489         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = 0;
 490         cmd->cmd.cmd_load_ta.cmd_buf_len = 0;
 491 }
 492
 493 static int psp_asd_load(struct psp_context *psp)
 494 {
 495         int ret;
 496         struct psp_gfx_cmd_resp *cmd;
 497
 498         /* If PSP version doesn't match ASD version, asd loading will be failed.
 499          * add workaround to bypass it for sriov now.
 500          * TODO: add version check to make it common
 501          */
 502         if (amdgpu_sriov_vf(psp->adev) ||
 503             (psp->adev->asic_type == CHIP_NAVY_FLOUNDER))
 504                 return 0;
 505
 506         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 507         if (!cmd)
 508                 return -ENOMEM;
 509
 510         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 511         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 512
 513         psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 514                                   psp->asd_ucode_size);
 515
 516         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 517                                  psp->fence_buf_mc_addr);
 518         if (!ret) {
 519                 psp->asd_context.asd_initialized = true;
 520                 psp->asd_context.session_id = cmd->resp.session_id;
 521         }
 522
 523         kfree(cmd);
 524
 525         return ret;
 526 }
 527
 528 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 529                                        uint32_t session_id)
 530 {
 531         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 532         cmd->cmd.cmd_unload_ta.session_id = session_id;
 533 }
 534
 535 static int psp_asd_unload(struct psp_context *psp)
 536 {
 537         int ret;
 538         struct psp_gfx_cmd_resp *cmd;
 539
 540         if (amdgpu_sriov_vf(psp->adev))
 541                 return 0;
 542
 543         if (!psp->asd_context.asd_initialized)
 544                 return 0;
 545
 546         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 547         if (!cmd)
 548                 return -ENOMEM;
 549
 550         psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id);
 551
 552         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 553                                  psp->fence_buf_mc_addr);
 554         if (!ret)
 555                 psp->asd_context.asd_initialized = false;
 556
 557         kfree(cmd);
 558
 559         return ret;
 560 }
 561
 562 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 563                 uint32_t id, uint32_t value)
 564 {
 565         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 566         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 567         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 568 }
 569
 570 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 571                 uint32_t value)
 572 {
 573         struct psp_gfx_cmd_resp *cmd = NULL;
 574         int ret = 0;
 575
 576         if (reg >= PSP_REG_LAST)
 577                 return -EINVAL;
 578
 579         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 580         if (!cmd)
 581                 return -ENOMEM;
 582
 583         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 584         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 585
 586         kfree(cmd);
 587         return ret;
 588 }
 589
 590 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 591                                      uint64_t ta_bin_mc,
 592                                      uint32_t ta_bin_size,
 593                                      uint64_t ta_shared_mc,
 594                                      uint32_t ta_shared_size)
 595 {
 596         cmd->cmd_id                             = GFX_CMD_ID_LOAD_TA;
 597         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 598         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 599         cmd->cmd.cmd_load_ta.app_len            = ta_bin_size;
 600
 601         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ta_shared_mc);
 602         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ta_shared_mc);
 603         cmd->cmd.cmd_load_ta.cmd_buf_len         = ta_shared_size;
 604 }
 605
 606 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 607 {
 608         int ret;
 609
 610         /*
 611          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 612          * physical) for xgmi ta <-> Driver
 613          */
 614         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 615                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 616                                       &psp->xgmi_context.xgmi_shared_bo,
 617                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 618                                       &psp->xgmi_context.xgmi_shared_buf);
 619
 620         return ret;
 621 }
 622
 623 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 624                                        uint32_t ta_cmd_id,
 625                                        uint32_t session_id)
 626 {
 627         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 628         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 629         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 630 }
 631
 632 static int psp_ta_invoke(struct psp_context *psp,
 633                   uint32_t ta_cmd_id,
 634                   uint32_t session_id)
 635 {
 636         int ret;
 637         struct psp_gfx_cmd_resp *cmd;
 638
 639         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 640         if (!cmd)
 641                 return -ENOMEM;
 642
 643         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id);
 644
 645         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 646                                  psp->fence_buf_mc_addr);
 647
 648         kfree(cmd);
 649
 650         return ret;
 651 }
 652
 653 static int psp_xgmi_load(struct psp_context *psp)
 654 {
 655         int ret;
 656         struct psp_gfx_cmd_resp *cmd;
 657
 658         /*
 659          * TODO: bypass the loading in sriov for now
 660          */
 661
 662         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 663         if (!cmd)
 664                 return -ENOMEM;
 665
 666         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 667         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 668
 669         psp_prep_ta_load_cmd_buf(cmd,
 670                                  psp->fw_pri_mc_addr,
 671                                  psp->ta_xgmi_ucode_size,
 672                                  psp->xgmi_context.xgmi_shared_mc_addr,
 673                                  PSP_XGMI_SHARED_MEM_SIZE);
 674
 675         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 676                                  psp->fence_buf_mc_addr);
 677
 678         if (!ret) {
 679                 psp->xgmi_context.initialized = 1;
 680                 psp->xgmi_context.session_id = cmd->resp.session_id;
 681         }
 682
 683         kfree(cmd);
 684
 685         return ret;
 686 }
 687
 688 static int psp_xgmi_unload(struct psp_context *psp)
 689 {
 690         int ret;
 691         struct psp_gfx_cmd_resp *cmd;
 692         struct amdgpu_device *adev = psp->adev;
 693
 694         /* XGMI TA unload currently is not supported on Arcturus */
 695         if (adev->asic_type == CHIP_ARCTURUS)
 696                 return 0;
 697
 698         /*
 699          * TODO: bypass the unloading in sriov for now
 700          */
 701
 702         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 703         if (!cmd)
 704                 return -ENOMEM;
 705
 706         psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 707
 708         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 709                                  psp->fence_buf_mc_addr);
 710
 711         kfree(cmd);
 712
 713         return ret;
 714 }
 715
 716 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 717 {
 718         return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id);
 719 }
 720
 721 int psp_xgmi_terminate(struct psp_context *psp)
 722 {
 723         int ret;
 724
 725         if (!psp->xgmi_context.initialized)
 726                 return 0;
 727
 728         ret = psp_xgmi_unload(psp);
 729         if (ret)
 730                 return ret;
 731
 732         psp->xgmi_context.initialized = 0;
 733
 734         /* free xgmi shared memory */
 735         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 736                         &psp->xgmi_context.xgmi_shared_mc_addr,
 737                         &psp->xgmi_context.xgmi_shared_buf);
 738
 739         return 0;
 740 }
 741
 742 int psp_xgmi_initialize(struct psp_context *psp)
 743 {
 744         struct ta_xgmi_shared_memory *xgmi_cmd;
 745         int ret;
 746
 747         if (!psp->adev->psp.ta_fw ||
 748             !psp->adev->psp.ta_xgmi_ucode_size ||
 749             !psp->adev->psp.ta_xgmi_start_addr)
 750                 return -ENOENT;
 751
 752         if (!psp->xgmi_context.initialized) {
 753                 ret = psp_xgmi_init_shared_buf(psp);
 754                 if (ret)
 755                         return ret;
 756         }
 757
 758         /* Load XGMI TA */
 759         ret = psp_xgmi_load(psp);
 760         if (ret)
 761                 return ret;
 762
 763         /* Initialize XGMI session */
 764         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 765         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 766         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 767
 768         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 769
 770         return ret;
 771 }
 772
 773 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
 774 {
 775         struct ta_xgmi_shared_memory *xgmi_cmd;
 776         int ret;
 777
 778         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 779         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 780
 781         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
 782
 783         /* Invoke xgmi ta to get hive id */
 784         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 785         if (ret)
 786                 return ret;
 787
 788         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
 789
 790         return 0;
 791 }
 792
 793 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
 794 {
 795         struct ta_xgmi_shared_memory *xgmi_cmd;
 796         int ret;
 797
 798         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 799         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 800
 801         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
 802
 803         /* Invoke xgmi ta to get the node id */
 804         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 805         if (ret)
 806                 return ret;
 807
 808         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
 809
 810         return 0;
 811 }
 812
 813 int psp_xgmi_get_topology_info(struct psp_context *psp,
 814                                int number_devices,
 815                                struct psp_xgmi_topology_info *topology)
 816 {
 817         struct ta_xgmi_shared_memory *xgmi_cmd;
 818         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 819         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
 820         int i;
 821         int ret;
 822
 823         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 824                 return -EINVAL;
 825
 826         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 827         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 828
 829         /* Fill in the shared memory with topology information as input */
 830         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 831         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
 832         topology_info_input->num_nodes = number_devices;
 833
 834         for (i = 0; i < topology_info_input->num_nodes; i++) {
 835                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 836                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 837                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
 838                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 839         }
 840
 841         /* Invoke xgmi ta to get the topology information */
 842         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
 843         if (ret)
 844                 return ret;
 845
 846         /* Read the output topology information from the shared memory */
 847         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
 848         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
 849         for (i = 0; i < topology->num_nodes; i++) {
 850                 topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
 851                 topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
 852                 topology->nodes[i].is_sharing_enabled = topology_info_output->nodes[i].is_sharing_enabled;
 853                 topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine;
 854         }
 855
 856         return 0;
 857 }
 858
 859 int psp_xgmi_set_topology_info(struct psp_context *psp,
 860                                int number_devices,
 861                                struct psp_xgmi_topology_info *topology)
 862 {
 863         struct ta_xgmi_shared_memory *xgmi_cmd;
 864         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 865         int i;
 866
 867         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 868                 return -EINVAL;
 869
 870         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 871         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 872
 873         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 874         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
 875         topology_info_input->num_nodes = number_devices;
 876
 877         for (i = 0; i < topology_info_input->num_nodes; i++) {
 878                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 879                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 880                 topology_info_input->nodes[i].is_sharing_enabled = 1;
 881                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 882         }
 883
 884         /* Invoke xgmi ta to set topology information */
 885         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
 886 }
 887
 888 // ras begin
 889 static int psp_ras_init_shared_buf(struct psp_context *psp)
 890 {
 891         int ret;
 892
 893         /*
 894          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 895          * physical) for ras ta <-> Driver
 896          */
 897         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 898                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 899                         &psp->ras.ras_shared_bo,
 900                         &psp->ras.ras_shared_mc_addr,
 901                         &psp->ras.ras_shared_buf);
 902
 903         return ret;
 904 }
 905
 906 static int psp_ras_load(struct psp_context *psp)
 907 {
 908         int ret;
 909         struct psp_gfx_cmd_resp *cmd;
 910
 911         /*
 912          * TODO: bypass the loading in sriov for now
 913          */
 914         if (amdgpu_sriov_vf(psp->adev))
 915                 return 0;
 916
 917         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 918         if (!cmd)
 919                 return -ENOMEM;
 920
 921         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 922         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 923
 924         psp_prep_ta_load_cmd_buf(cmd,
 925                                  psp->fw_pri_mc_addr,
 926                                  psp->ta_ras_ucode_size,
 927                                  psp->ras.ras_shared_mc_addr,
 928                                  PSP_RAS_SHARED_MEM_SIZE);
 929
 930         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 931                         psp->fence_buf_mc_addr);
 932
 933         if (!ret) {
 934                 psp->ras.ras_initialized = true;
 935                 psp->ras.session_id = cmd->resp.session_id;
 936         }
 937
 938         kfree(cmd);
 939
 940         return ret;
 941 }
 942
 943 static int psp_ras_unload(struct psp_context *psp)
 944 {
 945         int ret;
 946         struct psp_gfx_cmd_resp *cmd;
 947
 948         /*
 949          * TODO: bypass the unloading in sriov for now
 950          */
 951         if (amdgpu_sriov_vf(psp->adev))
 952                 return 0;
 953
 954         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 955         if (!cmd)
 956                 return -ENOMEM;
 957
 958         psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 959
 960         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 961                         psp->fence_buf_mc_addr);
 962
 963         kfree(cmd);
 964
 965         return ret;
 966 }
 967
 968 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 969 {
 970         struct ta_ras_shared_memory *ras_cmd;
 971         int ret;
 972
 973         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 974
 975         /*
 976          * TODO: bypass the loading in sriov for now
 977          */
 978         if (amdgpu_sriov_vf(psp->adev))
 979                 return 0;
 980
 981         ret = psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id);
 982
 983         if (amdgpu_ras_intr_triggered())
 984                 return ret;
 985
 986         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
 987         {
 988                 DRM_WARN("RAS: Unsupported Interface");
 989                 return -EINVAL;
 990         }
 991
 992         if (!ret) {
 993                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
 994                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
 995
 996                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
 997                 }
 998                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
 999                         dev_warn(psp->adev->dev,
1000                                  "RAS internal register access blocked\n");
1001         }
1002
1003         return ret;
1004 }
1005
1006 int psp_ras_enable_features(struct psp_context *psp,
1007                 union ta_ras_cmd_input *info, bool enable)
1008 {
1009         struct ta_ras_shared_memory *ras_cmd;
1010         int ret;
1011
1012         if (!psp->ras.ras_initialized)
1013                 return -EINVAL;
1014
1015         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1016         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1017
1018         if (enable)
1019                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1020         else
1021                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1022
1023         ras_cmd->ras_in_message = *info;
1024
1025         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1026         if (ret)
1027                 return -EINVAL;
1028
1029         return ras_cmd->ras_status;
1030 }
1031
1032 static int psp_ras_terminate(struct psp_context *psp)
1033 {
1034         int ret;
1035
1036         /*
1037          * TODO: bypass the terminate in sriov for now
1038          */
1039         if (amdgpu_sriov_vf(psp->adev))
1040                 return 0;
1041
1042         if (!psp->ras.ras_initialized)
1043                 return 0;
1044
1045         ret = psp_ras_unload(psp);
1046         if (ret)
1047                 return ret;
1048
1049         psp->ras.ras_initialized = false;
1050
1051         /* free ras shared memory */
1052         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
1053                         &psp->ras.ras_shared_mc_addr,
1054                         &psp->ras.ras_shared_buf);
1055
1056         return 0;
1057 }
1058
1059 static int psp_ras_initialize(struct psp_context *psp)
1060 {
1061         int ret;
1062
1063         /*
1064          * TODO: bypass the initialize in sriov for now
1065          */
1066         if (amdgpu_sriov_vf(psp->adev))
1067                 return 0;
1068
1069         if (!psp->adev->psp.ta_ras_ucode_size ||
1070             !psp->adev->psp.ta_ras_start_addr) {
1071                 dev_info(psp->adev->dev, "RAS: optional ras ta ucode is not available\n");
1072                 return 0;
1073         }
1074
1075         if (!psp->ras.ras_initialized) {
1076                 ret = psp_ras_init_shared_buf(psp);
1077                 if (ret)
1078                         return ret;
1079         }
1080
1081         ret = psp_ras_load(psp);
1082         if (ret)
1083                 return ret;
1084
1085         return 0;
1086 }
1087
1088 int psp_ras_trigger_error(struct psp_context *psp,
1089                           struct ta_ras_trigger_error_input *info)
1090 {
1091         struct ta_ras_shared_memory *ras_cmd;
1092         int ret;
1093
1094         if (!psp->ras.ras_initialized)
1095                 return -EINVAL;
1096
1097         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1098         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1099
1100         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1101         ras_cmd->ras_in_message.trigger_error = *info;
1102
1103         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1104         if (ret)
1105                 return -EINVAL;
1106
1107         /* If err_event_athub occurs error inject was successful, however
1108            return status from TA is no long reliable */
1109         if (amdgpu_ras_intr_triggered())
1110                 return 0;
1111
1112         return ras_cmd->ras_status;
1113 }
1114 // ras end
1115
1116 // HDCP start
1117 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
1118 {
1119         int ret;
1120
1121         /*
1122          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1123          * physical) for hdcp ta <-> Driver
1124          */
1125         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
1126                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1127                                       &psp->hdcp_context.hdcp_shared_bo,
1128                                       &psp->hdcp_context.hdcp_shared_mc_addr,
1129                                       &psp->hdcp_context.hdcp_shared_buf);
1130
1131         return ret;
1132 }
1133
1134 static int psp_hdcp_load(struct psp_context *psp)
1135 {
1136         int ret;
1137         struct psp_gfx_cmd_resp *cmd;
1138
1139         /*
1140          * TODO: bypass the loading in sriov for now
1141          */
1142         if (amdgpu_sriov_vf(psp->adev))
1143                 return 0;
1144
1145         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1146         if (!cmd)
1147                 return -ENOMEM;
1148
1149         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1150         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
1151                psp->ta_hdcp_ucode_size);
1152
1153         psp_prep_ta_load_cmd_buf(cmd,
1154                                  psp->fw_pri_mc_addr,
1155                                  psp->ta_hdcp_ucode_size,
1156                                  psp->hdcp_context.hdcp_shared_mc_addr,
1157                                  PSP_HDCP_SHARED_MEM_SIZE);
1158
1159         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1160
1161         if (!ret) {
1162                 psp->hdcp_context.hdcp_initialized = true;
1163                 psp->hdcp_context.session_id = cmd->resp.session_id;
1164                 mutex_init(&psp->hdcp_context.mutex);
1165         }
1166
1167         kfree(cmd);
1168
1169         return ret;
1170 }
1171 static int psp_hdcp_initialize(struct psp_context *psp)
1172 {
1173         int ret;
1174
1175         /*
1176          * TODO: bypass the initialize in sriov for now
1177          */
1178         if (amdgpu_sriov_vf(psp->adev))
1179                 return 0;
1180
1181         if (!psp->adev->psp.ta_hdcp_ucode_size ||
1182             !psp->adev->psp.ta_hdcp_start_addr) {
1183                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1184                 return 0;
1185         }
1186
1187         if (!psp->hdcp_context.hdcp_initialized) {
1188                 ret = psp_hdcp_init_shared_buf(psp);
1189                 if (ret)
1190                         return ret;
1191         }
1192
1193         ret = psp_hdcp_load(psp);
1194         if (ret)
1195                 return ret;
1196
1197         return 0;
1198 }
1199
1200 static int psp_hdcp_unload(struct psp_context *psp)
1201 {
1202         int ret;
1203         struct psp_gfx_cmd_resp *cmd;
1204
1205         /*
1206          * TODO: bypass the unloading in sriov for now
1207          */
1208         if (amdgpu_sriov_vf(psp->adev))
1209                 return 0;
1210
1211         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1212         if (!cmd)
1213                 return -ENOMEM;
1214
1215         psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
1216
1217         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1218
1219         kfree(cmd);
1220
1221         return ret;
1222 }
1223
1224 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1225 {
1226         /*
1227          * TODO: bypass the loading in sriov for now
1228          */
1229         if (amdgpu_sriov_vf(psp->adev))
1230                 return 0;
1231
1232         return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id);
1233 }
1234
1235 static int psp_hdcp_terminate(struct psp_context *psp)
1236 {
1237         int ret;
1238
1239         /*
1240          * TODO: bypass the terminate in sriov for now
1241          */
1242         if (amdgpu_sriov_vf(psp->adev))
1243                 return 0;
1244
1245         if (!psp->hdcp_context.hdcp_initialized)
1246                 return 0;
1247
1248         ret = psp_hdcp_unload(psp);
1249         if (ret)
1250                 return ret;
1251
1252         psp->hdcp_context.hdcp_initialized = false;
1253
1254         /* free hdcp shared memory */
1255         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
1256                               &psp->hdcp_context.hdcp_shared_mc_addr,
1257                               &psp->hdcp_context.hdcp_shared_buf);
1258
1259         return 0;
1260 }
1261 // HDCP end
1262
1263 // DTM start
1264 static int psp_dtm_init_shared_buf(struct psp_context *psp)
1265 {
1266         int ret;
1267
1268         /*
1269          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1270          * physical) for dtm ta <-> Driver
1271          */
1272         ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE,
1273                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1274                                       &psp->dtm_context.dtm_shared_bo,
1275                                       &psp->dtm_context.dtm_shared_mc_addr,
1276                                       &psp->dtm_context.dtm_shared_buf);
1277
1278         return ret;
1279 }
1280
1281 static int psp_dtm_load(struct psp_context *psp)
1282 {
1283         int ret;
1284         struct psp_gfx_cmd_resp *cmd;
1285
1286         /*
1287          * TODO: bypass the loading in sriov for now
1288          */
1289         if (amdgpu_sriov_vf(psp->adev))
1290                 return 0;
1291
1292         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1293         if (!cmd)
1294                 return -ENOMEM;
1295
1296         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1297         memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size);
1298
1299         psp_prep_ta_load_cmd_buf(cmd,
1300                                  psp->fw_pri_mc_addr,
1301                                  psp->ta_dtm_ucode_size,
1302                                  psp->dtm_context.dtm_shared_mc_addr,
1303                                  PSP_DTM_SHARED_MEM_SIZE);
1304
1305         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1306
1307         if (!ret) {
1308                 psp->dtm_context.dtm_initialized = true;
1309                 psp->dtm_context.session_id = cmd->resp.session_id;
1310                 mutex_init(&psp->dtm_context.mutex);
1311         }
1312
1313         kfree(cmd);
1314
1315         return ret;
1316 }
1317
1318 static int psp_dtm_initialize(struct psp_context *psp)
1319 {
1320         int ret;
1321
1322         /*
1323          * TODO: bypass the initialize in sriov for now
1324          */
1325         if (amdgpu_sriov_vf(psp->adev))
1326                 return 0;
1327
1328         if (!psp->adev->psp.ta_dtm_ucode_size ||
1329             !psp->adev->psp.ta_dtm_start_addr) {
1330                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1331                 return 0;
1332         }
1333
1334         if (!psp->dtm_context.dtm_initialized) {
1335                 ret = psp_dtm_init_shared_buf(psp);
1336                 if (ret)
1337                         return ret;
1338         }
1339
1340         ret = psp_dtm_load(psp);
1341         if (ret)
1342                 return ret;
1343
1344         return 0;
1345 }
1346
1347 static int psp_dtm_unload(struct psp_context *psp)
1348 {
1349         int ret;
1350         struct psp_gfx_cmd_resp *cmd;
1351
1352         /*
1353          * TODO: bypass the unloading in sriov for now
1354          */
1355         if (amdgpu_sriov_vf(psp->adev))
1356                 return 0;
1357
1358         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1359         if (!cmd)
1360                 return -ENOMEM;
1361
1362         psp_prep_ta_unload_cmd_buf(cmd, psp->dtm_context.session_id);
1363
1364         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1365
1366         kfree(cmd);
1367
1368         return ret;
1369 }
1370
1371 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1372 {
1373         /*
1374          * TODO: bypass the loading in sriov for now
1375          */
1376         if (amdgpu_sriov_vf(psp->adev))
1377                 return 0;
1378
1379         return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id);
1380 }
1381
1382 static int psp_dtm_terminate(struct psp_context *psp)
1383 {
1384         int ret;
1385
1386         /*
1387          * TODO: bypass the terminate in sriov for now
1388          */
1389         if (amdgpu_sriov_vf(psp->adev))
1390                 return 0;
1391
1392         if (!psp->dtm_context.dtm_initialized)
1393                 return 0;
1394
1395         ret = psp_dtm_unload(psp);
1396         if (ret)
1397                 return ret;
1398
1399         psp->dtm_context.dtm_initialized = false;
1400
1401         /* free hdcp shared memory */
1402         amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo,
1403                               &psp->dtm_context.dtm_shared_mc_addr,
1404                               &psp->dtm_context.dtm_shared_buf);
1405
1406         return 0;
1407 }
1408 // DTM end
1409
1410 static int psp_hw_start(struct psp_context *psp)
1411 {
1412         struct amdgpu_device *adev = psp->adev;
1413         int ret;
1414
1415         if (!amdgpu_sriov_vf(adev)) {
1416                 if (psp->kdb_bin_size &&
1417                     (psp->funcs->bootloader_load_kdb != NULL)) {
1418                         ret = psp_bootloader_load_kdb(psp);
1419                         if (ret) {
1420                                 DRM_ERROR("PSP load kdb failed!\n");
1421                                 return ret;
1422                         }
1423                 }
1424
1425                 if (psp->spl_bin_size) {
1426                         ret = psp_bootloader_load_spl(psp);
1427                         if (ret) {
1428                                 DRM_ERROR("PSP load spl failed!\n");
1429                                 return ret;
1430                         }
1431                 }
1432
1433                 ret = psp_bootloader_load_sysdrv(psp);
1434                 if (ret) {
1435                         DRM_ERROR("PSP load sysdrv failed!\n");
1436                         return ret;
1437                 }
1438
1439                 ret = psp_bootloader_load_sos(psp);
1440                 if (ret) {
1441                         DRM_ERROR("PSP load sos failed!\n");
1442                         return ret;
1443                 }
1444         }
1445
1446         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
1447         if (ret) {
1448                 DRM_ERROR("PSP create ring failed!\n");
1449                 return ret;
1450         }
1451
1452         ret = psp_clear_vf_fw(psp);
1453         if (ret) {
1454                 DRM_ERROR("PSP clear vf fw!\n");
1455                 return ret;
1456         }
1457
1458         ret = psp_tmr_init(psp);
1459         if (ret) {
1460                 DRM_ERROR("PSP tmr init failed!\n");
1461                 return ret;
1462         }
1463
1464         /*
1465          * For ASICs with DF Cstate management centralized
1466          * to PMFW, TMR setup should be performed after PMFW
1467          * loaded and before other non-psp firmware loaded.
1468          */
1469         if (psp->pmfw_centralized_cstate_management) {
1470                 ret = psp_load_smu_fw(psp);
1471                 if (ret)
1472                         return ret;
1473         }
1474
1475         ret = psp_tmr_load(psp);
1476         if (ret) {
1477                 DRM_ERROR("PSP load tmr failed!\n");
1478                 return ret;
1479         }
1480
1481         return 0;
1482 }
1483
1484 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1485                            enum psp_gfx_fw_type *type)
1486 {
1487         switch (ucode->ucode_id) {
1488         case AMDGPU_UCODE_ID_SDMA0:
1489                 *type = GFX_FW_TYPE_SDMA0;
1490                 break;
1491         case AMDGPU_UCODE_ID_SDMA1:
1492                 *type = GFX_FW_TYPE_SDMA1;
1493                 break;
1494         case AMDGPU_UCODE_ID_SDMA2:
1495                 *type = GFX_FW_TYPE_SDMA2;
1496                 break;
1497         case AMDGPU_UCODE_ID_SDMA3:
1498                 *type = GFX_FW_TYPE_SDMA3;
1499                 break;
1500         case AMDGPU_UCODE_ID_SDMA4:
1501                 *type = GFX_FW_TYPE_SDMA4;
1502                 break;
1503         case AMDGPU_UCODE_ID_SDMA5:
1504                 *type = GFX_FW_TYPE_SDMA5;
1505                 break;
1506         case AMDGPU_UCODE_ID_SDMA6:
1507                 *type = GFX_FW_TYPE_SDMA6;
1508                 break;
1509         case AMDGPU_UCODE_ID_SDMA7:
1510                 *type = GFX_FW_TYPE_SDMA7;
1511                 break;
1512         case AMDGPU_UCODE_ID_CP_MES:
1513                 *type = GFX_FW_TYPE_CP_MES;
1514                 break;
1515         case AMDGPU_UCODE_ID_CP_MES_DATA:
1516                 *type = GFX_FW_TYPE_MES_STACK;
1517                 break;
1518         case AMDGPU_UCODE_ID_CP_CE:
1519                 *type = GFX_FW_TYPE_CP_CE;
1520                 break;
1521         case AMDGPU_UCODE_ID_CP_PFP:
1522                 *type = GFX_FW_TYPE_CP_PFP;
1523                 break;
1524         case AMDGPU_UCODE_ID_CP_ME:
1525                 *type = GFX_FW_TYPE_CP_ME;
1526                 break;
1527         case AMDGPU_UCODE_ID_CP_MEC1:
1528                 *type = GFX_FW_TYPE_CP_MEC;
1529                 break;
1530         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1531                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1532                 break;
1533         case AMDGPU_UCODE_ID_CP_MEC2:
1534                 *type = GFX_FW_TYPE_CP_MEC;
1535                 break;
1536         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1537                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1538                 break;
1539         case AMDGPU_UCODE_ID_RLC_G:
1540                 *type = GFX_FW_TYPE_RLC_G;
1541                 break;
1542         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1543                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1544                 break;
1545         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1546                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1547                 break;
1548         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1549                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1550                 break;
1551         case AMDGPU_UCODE_ID_SMC:
1552                 *type = GFX_FW_TYPE_SMU;
1553                 break;
1554         case AMDGPU_UCODE_ID_UVD:
1555                 *type = GFX_FW_TYPE_UVD;
1556                 break;
1557         case AMDGPU_UCODE_ID_UVD1:
1558                 *type = GFX_FW_TYPE_UVD1;
1559                 break;
1560         case AMDGPU_UCODE_ID_VCE:
1561                 *type = GFX_FW_TYPE_VCE;
1562                 break;
1563         case AMDGPU_UCODE_ID_VCN:
1564                 *type = GFX_FW_TYPE_VCN;
1565                 break;
1566         case AMDGPU_UCODE_ID_VCN1:
1567                 *type = GFX_FW_TYPE_VCN1;
1568                 break;
1569         case AMDGPU_UCODE_ID_DMCU_ERAM:
1570                 *type = GFX_FW_TYPE_DMCU_ERAM;
1571                 break;
1572         case AMDGPU_UCODE_ID_DMCU_INTV:
1573                 *type = GFX_FW_TYPE_DMCU_ISR;
1574                 break;
1575         case AMDGPU_UCODE_ID_VCN0_RAM:
1576                 *type = GFX_FW_TYPE_VCN0_RAM;
1577                 break;
1578         case AMDGPU_UCODE_ID_VCN1_RAM:
1579                 *type = GFX_FW_TYPE_VCN1_RAM;
1580                 break;
1581         case AMDGPU_UCODE_ID_DMCUB:
1582                 *type = GFX_FW_TYPE_DMUB;
1583                 break;
1584         case AMDGPU_UCODE_ID_MAXIMUM:
1585         default:
1586                 return -EINVAL;
1587         }
1588
1589         return 0;
1590 }
1591
1592 static void psp_print_fw_hdr(struct psp_context *psp,
1593                              struct amdgpu_firmware_info *ucode)
1594 {
1595         struct amdgpu_device *adev = psp->adev;
1596         struct common_firmware_header *hdr;
1597
1598         switch (ucode->ucode_id) {
1599         case AMDGPU_UCODE_ID_SDMA0:
1600         case AMDGPU_UCODE_ID_SDMA1:
1601         case AMDGPU_UCODE_ID_SDMA2:
1602         case AMDGPU_UCODE_ID_SDMA3:
1603         case AMDGPU_UCODE_ID_SDMA4:
1604         case AMDGPU_UCODE_ID_SDMA5:
1605         case AMDGPU_UCODE_ID_SDMA6:
1606         case AMDGPU_UCODE_ID_SDMA7:
1607                 hdr = (struct common_firmware_header *)
1608                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1609                 amdgpu_ucode_print_sdma_hdr(hdr);
1610                 break;
1611         case AMDGPU_UCODE_ID_CP_CE:
1612                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1613                 amdgpu_ucode_print_gfx_hdr(hdr);
1614                 break;
1615         case AMDGPU_UCODE_ID_CP_PFP:
1616                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1617                 amdgpu_ucode_print_gfx_hdr(hdr);
1618                 break;
1619         case AMDGPU_UCODE_ID_CP_ME:
1620                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1621                 amdgpu_ucode_print_gfx_hdr(hdr);
1622                 break;
1623         case AMDGPU_UCODE_ID_CP_MEC1:
1624                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1625                 amdgpu_ucode_print_gfx_hdr(hdr);
1626                 break;
1627         case AMDGPU_UCODE_ID_RLC_G:
1628                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1629                 amdgpu_ucode_print_rlc_hdr(hdr);
1630                 break;
1631         case AMDGPU_UCODE_ID_SMC:
1632                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1633                 amdgpu_ucode_print_smc_hdr(hdr);
1634                 break;
1635         default:
1636                 break;
1637         }
1638 }
1639
1640 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1641                                        struct psp_gfx_cmd_resp *cmd)
1642 {
1643         int ret;
1644         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1645
1646         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1647
1648         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1649         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1650         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1651         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1652
1653         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1654         if (ret)
1655                 DRM_ERROR("Unknown firmware type\n");
1656
1657         return ret;
1658 }
1659
1660 static int psp_execute_np_fw_load(struct psp_context *psp,
1661                                   struct amdgpu_firmware_info *ucode)
1662 {
1663         int ret = 0;
1664
1665         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1666         if (ret)
1667                 return ret;
1668
1669         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1670                                  psp->fence_buf_mc_addr);
1671
1672         return ret;
1673 }
1674
1675 static int psp_load_smu_fw(struct psp_context *psp)
1676 {
1677         int ret;
1678         struct amdgpu_device* adev = psp->adev;
1679         struct amdgpu_firmware_info *ucode =
1680                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1681         struct amdgpu_ras *ras = psp->ras.ras;
1682
1683         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
1684                 return 0;
1685
1686
1687         if (adev->in_gpu_reset && ras && ras->supported) {
1688                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
1689                 if (ret) {
1690                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
1691                 }
1692         }
1693
1694         ret = psp_execute_np_fw_load(psp, ucode);
1695
1696         if (ret)
1697                 DRM_ERROR("PSP load smu failed!\n");
1698
1699         return ret;
1700 }
1701
1702 static bool fw_load_skip_check(struct psp_context *psp,
1703                                struct amdgpu_firmware_info *ucode)
1704 {
1705         if (!ucode->fw)
1706                 return true;
1707
1708         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1709             (psp_smu_reload_quirk(psp) ||
1710              psp->autoload_supported ||
1711              psp->pmfw_centralized_cstate_management))
1712                 return true;
1713
1714         if (amdgpu_sriov_vf(psp->adev) &&
1715            (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1716             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1717             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1718             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1719             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1720             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1721             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1722             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1723             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
1724             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
1725             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
1726             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
1727             || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
1728                 /*skip ucode loading in SRIOV VF */
1729                 return true;
1730
1731         if (psp->autoload_supported &&
1732             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1733              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1734                 /* skip mec JT when autoload is enabled */
1735                 return true;
1736
1737         return false;
1738 }
1739
1740 static int psp_np_fw_load(struct psp_context *psp)
1741 {
1742         int i, ret;
1743         struct amdgpu_firmware_info *ucode;
1744         struct amdgpu_device* adev = psp->adev;
1745
1746         if (psp->autoload_supported &&
1747             !psp->pmfw_centralized_cstate_management) {
1748                 ret = psp_load_smu_fw(psp);
1749                 if (ret)
1750                         return ret;
1751         }
1752
1753         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1754                 ucode = &adev->firmware.ucode[i];
1755
1756                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1757                     !fw_load_skip_check(psp, ucode)) {
1758                         ret = psp_load_smu_fw(psp);
1759                         if (ret)
1760                                 return ret;
1761                         continue;
1762                 }
1763
1764                 if (fw_load_skip_check(psp, ucode))
1765                         continue;
1766
1767                 if (psp->autoload_supported &&
1768                     (adev->asic_type == CHIP_SIENNA_CICHLID ||
1769                      adev->asic_type == CHIP_NAVY_FLOUNDER) &&
1770                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
1771                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
1772                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
1773                         /* PSP only receive one SDMA fw for sienna_cichlid,
1774                          * as all four sdma fw are same */
1775                         continue;
1776
1777                 psp_print_fw_hdr(psp, ucode);
1778
1779                 ret = psp_execute_np_fw_load(psp, ucode);
1780                 if (ret)
1781                         return ret;
1782
1783                 /* Start rlc autoload after psp recieved all the gfx firmware */
1784                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
1785                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
1786                         ret = psp_rlc_autoload_start(psp);
1787                         if (ret) {
1788                                 DRM_ERROR("Failed to start rlc autoload\n");
1789                                 return ret;
1790                         }
1791                 }
1792         }
1793
1794         return 0;
1795 }
1796
1797 static int psp_load_fw(struct amdgpu_device *adev)
1798 {
1799         int ret;
1800         struct psp_context *psp = &adev->psp;
1801
1802         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1803                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1804                 goto skip_memalloc;
1805         }
1806
1807         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1808         if (!psp->cmd)
1809                 return -ENOMEM;
1810
1811         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1812                                         AMDGPU_GEM_DOMAIN_GTT,
1813                                         &psp->fw_pri_bo,
1814                                         &psp->fw_pri_mc_addr,
1815                                         &psp->fw_pri_buf);
1816         if (ret)
1817                 goto failed;
1818
1819         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1820                                         AMDGPU_GEM_DOMAIN_VRAM,
1821                                         &psp->fence_buf_bo,
1822                                         &psp->fence_buf_mc_addr,
1823                                         &psp->fence_buf);
1824         if (ret)
1825                 goto failed;
1826
1827         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1828                                       AMDGPU_GEM_DOMAIN_VRAM,
1829                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1830                                       (void **)&psp->cmd_buf_mem);
1831         if (ret)
1832                 goto failed;
1833
1834         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1835
1836         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1837         if (ret) {
1838                 DRM_ERROR("PSP ring init failed!\n");
1839                 goto failed;
1840         }
1841
1842 skip_memalloc:
1843         ret = psp_hw_start(psp);
1844         if (ret)
1845                 goto failed;
1846
1847         ret = psp_np_fw_load(psp);
1848         if (ret)
1849                 goto failed;
1850
1851         ret = psp_asd_load(psp);
1852         if (ret) {
1853                 DRM_ERROR("PSP load asd failed!\n");
1854                 return ret;
1855         }
1856
1857         if (psp->adev->psp.ta_fw) {
1858                 ret = psp_ras_initialize(psp);
1859                 if (ret)
1860                         dev_err(psp->adev->dev,
1861                                         "RAS: Failed to initialize RAS\n");
1862
1863                 ret = psp_hdcp_initialize(psp);
1864                 if (ret)
1865                         dev_err(psp->adev->dev,
1866                                 "HDCP: Failed to initialize HDCP\n");
1867
1868                 ret = psp_dtm_initialize(psp);
1869                 if (ret)
1870                         dev_err(psp->adev->dev,
1871                                 "DTM: Failed to initialize DTM\n");
1872         }
1873
1874         return 0;
1875
1876 failed:
1877         /*
1878          * all cleanup jobs (xgmi terminate, ras terminate,
1879          * ring destroy, cmd/fence/fw buffers destory,
1880          * psp->cmd destory) are delayed to psp_hw_fini
1881          */
1882         return ret;
1883 }
1884
1885 static int psp_hw_init(void *handle)
1886 {
1887         int ret;
1888         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1889
1890         mutex_lock(&adev->firmware.mutex);
1891         /*
1892          * This sequence is just used on hw_init only once, no need on
1893          * resume.
1894          */
1895         ret = amdgpu_ucode_init_bo(adev);
1896         if (ret)
1897                 goto failed;
1898
1899         ret = psp_load_fw(adev);
1900         if (ret) {
1901                 DRM_ERROR("PSP firmware loading failed\n");
1902                 goto failed;
1903         }
1904
1905         mutex_unlock(&adev->firmware.mutex);
1906         return 0;
1907
1908 failed:
1909         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1910         mutex_unlock(&adev->firmware.mutex);
1911         return -EINVAL;
1912 }
1913
1914 static int psp_hw_fini(void *handle)
1915 {
1916         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1917         struct psp_context *psp = &adev->psp;
1918         int ret;
1919
1920         if (psp->adev->psp.ta_fw) {
1921                 psp_ras_terminate(psp);
1922                 psp_dtm_terminate(psp);
1923                 psp_hdcp_terminate(psp);
1924         }
1925
1926         psp_asd_unload(psp);
1927         ret = psp_clear_vf_fw(psp);
1928         if (ret) {
1929                 DRM_ERROR("PSP clear vf fw!\n");
1930                 return ret;
1931         }
1932
1933         psp_tmr_terminate(psp);
1934         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1935
1936         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1937                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1938         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1939                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1940         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1941                               (void **)&psp->cmd_buf_mem);
1942
1943         kfree(psp->cmd);
1944         psp->cmd = NULL;
1945
1946         return 0;
1947 }
1948
1949 static int psp_suspend(void *handle)
1950 {
1951         int ret;
1952         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1953         struct psp_context *psp = &adev->psp;
1954
1955         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1956             psp->xgmi_context.initialized == 1) {
1957                 ret = psp_xgmi_terminate(psp);
1958                 if (ret) {
1959                         DRM_ERROR("Failed to terminate xgmi ta\n");
1960                         return ret;
1961                 }
1962         }
1963
1964         if (psp->adev->psp.ta_fw) {
1965                 ret = psp_ras_terminate(psp);
1966                 if (ret) {
1967                         DRM_ERROR("Failed to terminate ras ta\n");
1968                         return ret;
1969                 }
1970                 ret = psp_hdcp_terminate(psp);
1971                 if (ret) {
1972                         DRM_ERROR("Failed to terminate hdcp ta\n");
1973                         return ret;
1974                 }
1975                 ret = psp_dtm_terminate(psp);
1976                 if (ret) {
1977                         DRM_ERROR("Failed to terminate dtm ta\n");
1978                         return ret;
1979                 }
1980         }
1981
1982         ret = psp_asd_unload(psp);
1983         if (ret) {
1984                 DRM_ERROR("Failed to unload asd\n");
1985                 return ret;
1986         }
1987
1988         ret = psp_tmr_terminate(psp);
1989         if (ret) {
1990                 DRM_ERROR("Falied to terminate tmr\n");
1991                 return ret;
1992         }
1993
1994         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1995         if (ret) {
1996                 DRM_ERROR("PSP ring stop failed\n");
1997                 return ret;
1998         }
1999
2000         return 0;
2001 }
2002
2003 static int psp_resume(void *handle)
2004 {
2005         int ret;
2006         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2007         struct psp_context *psp = &adev->psp;
2008
2009         DRM_INFO("PSP is resuming...\n");
2010
2011         ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2012         if (ret) {
2013                 DRM_ERROR("Failed to process memory training!\n");
2014                 return ret;
2015         }
2016
2017         mutex_lock(&adev->firmware.mutex);
2018
2019         ret = psp_hw_start(psp);
2020         if (ret)
2021                 goto failed;
2022
2023         ret = psp_np_fw_load(psp);
2024         if (ret)
2025                 goto failed;
2026
2027         ret = psp_asd_load(psp);
2028         if (ret) {
2029                 DRM_ERROR("PSP load asd failed!\n");
2030                 goto failed;
2031         }
2032
2033         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2034                 ret = psp_xgmi_initialize(psp);
2035                 /* Warning the XGMI seesion initialize failure
2036                  * Instead of stop driver initialization
2037                  */
2038                 if (ret)
2039                         dev_err(psp->adev->dev,
2040                                 "XGMI: Failed to initialize XGMI session\n");
2041         }
2042
2043         if (psp->adev->psp.ta_fw) {
2044                 ret = psp_ras_initialize(psp);
2045                 if (ret)
2046                         dev_err(psp->adev->dev,
2047                                         "RAS: Failed to initialize RAS\n");
2048
2049                 ret = psp_hdcp_initialize(psp);
2050                 if (ret)
2051                         dev_err(psp->adev->dev,
2052                                 "HDCP: Failed to initialize HDCP\n");
2053
2054                 ret = psp_dtm_initialize(psp);
2055                 if (ret)
2056                         dev_err(psp->adev->dev,
2057                                 "DTM: Failed to initialize DTM\n");
2058         }
2059
2060         mutex_unlock(&adev->firmware.mutex);
2061
2062         return 0;
2063
2064 failed:
2065         DRM_ERROR("PSP resume failed\n");
2066         mutex_unlock(&adev->firmware.mutex);
2067         return ret;
2068 }
2069
2070 int psp_gpu_reset(struct amdgpu_device *adev)
2071 {
2072         int ret;
2073
2074         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2075                 return 0;
2076
2077         mutex_lock(&adev->psp.mutex);
2078         ret = psp_mode1_reset(&adev->psp);
2079         mutex_unlock(&adev->psp.mutex);
2080
2081         return ret;
2082 }
2083
2084 int psp_rlc_autoload_start(struct psp_context *psp)
2085 {
2086         int ret;
2087         struct psp_gfx_cmd_resp *cmd;
2088
2089         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
2090         if (!cmd)
2091                 return -ENOMEM;
2092
2093         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2094
2095         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2096                                  psp->fence_buf_mc_addr);
2097         kfree(cmd);
2098         return ret;
2099 }
2100
2101 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2102                         uint64_t cmd_gpu_addr, int cmd_size)
2103 {
2104         struct amdgpu_firmware_info ucode = {0};
2105
2106         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2107                 AMDGPU_UCODE_ID_VCN0_RAM;
2108         ucode.mc_addr = cmd_gpu_addr;
2109         ucode.ucode_size = cmd_size;
2110
2111         return psp_execute_np_fw_load(&adev->psp, &ucode);
2112 }
2113
2114 int psp_ring_cmd_submit(struct psp_context *psp,
2115                         uint64_t cmd_buf_mc_addr,
2116                         uint64_t fence_mc_addr,
2117                         int index)
2118 {
2119         unsigned int psp_write_ptr_reg = 0;
2120         struct psp_gfx_rb_frame *write_frame;
2121         struct psp_ring *ring = &psp->km_ring;
2122         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2123         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2124                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2125         struct amdgpu_device *adev = psp->adev;
2126         uint32_t ring_size_dw = ring->ring_size / 4;
2127         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2128
2129         /* KM (GPCOM) prepare write pointer */
2130         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2131
2132         /* Update KM RB frame pointer to new frame */
2133         /* write_frame ptr increments by size of rb_frame in bytes */
2134         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2135         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2136                 write_frame = ring_buffer_start;
2137         else
2138                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2139         /* Check invalid write_frame ptr address */
2140         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2141                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2142                           ring_buffer_start, ring_buffer_end, write_frame);
2143                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2144                 return -EINVAL;
2145         }
2146
2147         /* Initialize KM RB frame */
2148         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2149
2150         /* Update KM RB frame */
2151         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2152         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2153         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2154         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2155         write_frame->fence_value = index;
2156         amdgpu_asic_flush_hdp(adev, NULL);
2157
2158         /* Update the write Pointer in DWORDs */
2159         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2160         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2161         return 0;
2162 }
2163
2164 int psp_init_asd_microcode(struct psp_context *psp,
2165                            const char *chip_name)
2166 {
2167         struct amdgpu_device *adev = psp->adev;
2168         char fw_name[30];
2169         const struct psp_firmware_header_v1_0 *asd_hdr;
2170         int err = 0;
2171
2172         if (!chip_name) {
2173                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2174                 return -EINVAL;
2175         }
2176
2177         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2178         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2179         if (err)
2180                 goto out;
2181
2182         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2183         if (err)
2184                 goto out;
2185
2186         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2187         adev->psp.asd_fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2188         adev->psp.asd_feature_version = le32_to_cpu(asd_hdr->ucode_feature_version);
2189         adev->psp.asd_ucode_size = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2190         adev->psp.asd_start_addr = (uint8_t *)asd_hdr +
2191                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2192         return 0;
2193 out:
2194         dev_err(adev->dev, "fail to initialize asd microcode\n");
2195         release_firmware(adev->psp.asd_fw);
2196         adev->psp.asd_fw = NULL;
2197         return err;
2198 }
2199
2200 int psp_init_sos_microcode(struct psp_context *psp,
2201                            const char *chip_name)
2202 {
2203         struct amdgpu_device *adev = psp->adev;
2204         char fw_name[30];
2205         const struct psp_firmware_header_v1_0 *sos_hdr;
2206         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
2207         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
2208         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2209         int err = 0;
2210
2211         if (!chip_name) {
2212                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
2213                 return -EINVAL;
2214         }
2215
2216         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
2217         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
2218         if (err)
2219                 goto out;
2220
2221         err = amdgpu_ucode_validate(adev->psp.sos_fw);
2222         if (err)
2223                 goto out;
2224
2225         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
2226         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
2227
2228         switch (sos_hdr->header.header_version_major) {
2229         case 1:
2230                 adev->psp.sos_fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
2231                 adev->psp.sos_feature_version = le32_to_cpu(sos_hdr->ucode_feature_version);
2232                 adev->psp.sos_bin_size = le32_to_cpu(sos_hdr->sos_size_bytes);
2233                 adev->psp.sys_bin_size = le32_to_cpu(sos_hdr->sos_offset_bytes);
2234                 adev->psp.sys_start_addr = (uint8_t *)sos_hdr +
2235                                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2236                 adev->psp.sos_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2237                                 le32_to_cpu(sos_hdr->sos_offset_bytes);
2238                 if (sos_hdr->header.header_version_minor == 1) {
2239                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
2240                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_1->toc_size_bytes);
2241                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2242                                         le32_to_cpu(sos_hdr_v1_1->toc_offset_bytes);
2243                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_1->kdb_size_bytes);
2244                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2245                                         le32_to_cpu(sos_hdr_v1_1->kdb_offset_bytes);
2246                 }
2247                 if (sos_hdr->header.header_version_minor == 2) {
2248                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
2249                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_2->kdb_size_bytes);
2250                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2251                                                     le32_to_cpu(sos_hdr_v1_2->kdb_offset_bytes);
2252                 }
2253                 if (sos_hdr->header.header_version_minor == 3) {
2254                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
2255                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.toc_size_bytes);
2256                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2257                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc_offset_bytes);
2258                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_size_bytes);
2259                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2260                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_offset_bytes);
2261                         adev->psp.spl_bin_size = le32_to_cpu(sos_hdr_v1_3->spl_size_bytes);
2262                         adev->psp.spl_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2263                                 le32_to_cpu(sos_hdr_v1_3->spl_offset_bytes);
2264                 }
2265                 break;
2266         default:
2267                 dev_err(adev->dev,
2268                         "unsupported psp sos firmware\n");
2269                 err = -EINVAL;
2270                 goto out;
2271         }
2272
2273         return 0;
2274 out:
2275         dev_err(adev->dev,
2276                 "failed to init sos firmware\n");
2277         release_firmware(adev->psp.sos_fw);
2278         adev->psp.sos_fw = NULL;
2279
2280         return err;
2281 }
2282
2283 int parse_ta_bin_descriptor(struct psp_context *psp,
2284                             const struct ta_fw_bin_desc *desc,
2285                             const struct ta_firmware_header_v2_0 *ta_hdr)
2286 {
2287         uint8_t *ucode_start_addr  = NULL;
2288
2289         if (!psp || !desc || !ta_hdr)
2290                 return -EINVAL;
2291
2292         ucode_start_addr  = (uint8_t *)ta_hdr +
2293                             le32_to_cpu(desc->offset_bytes) +
2294                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
2295
2296         switch (desc->fw_type) {
2297         case TA_FW_TYPE_PSP_ASD:
2298                 psp->asd_fw_version        = le32_to_cpu(desc->fw_version);
2299                 psp->asd_feature_version   = le32_to_cpu(desc->fw_version);
2300                 psp->asd_ucode_size        = le32_to_cpu(desc->size_bytes);
2301                 psp->asd_start_addr        = ucode_start_addr;
2302                 break;
2303         case TA_FW_TYPE_PSP_XGMI:
2304                 psp->ta_xgmi_ucode_version = le32_to_cpu(desc->fw_version);
2305                 psp->ta_xgmi_ucode_size    = le32_to_cpu(desc->size_bytes);
2306                 psp->ta_xgmi_start_addr    = ucode_start_addr;
2307                 break;
2308         case TA_FW_TYPE_PSP_RAS:
2309                 psp->ta_ras_ucode_version  = le32_to_cpu(desc->fw_version);
2310                 psp->ta_ras_ucode_size     = le32_to_cpu(desc->size_bytes);
2311                 psp->ta_ras_start_addr     = ucode_start_addr;
2312                 break;
2313         case TA_FW_TYPE_PSP_HDCP:
2314                 psp->ta_hdcp_ucode_version = le32_to_cpu(desc->fw_version);
2315                 psp->ta_hdcp_ucode_size    = le32_to_cpu(desc->size_bytes);
2316                 psp->ta_hdcp_start_addr    = ucode_start_addr;
2317                 break;
2318         case TA_FW_TYPE_PSP_DTM:
2319                 psp->ta_dtm_ucode_version  = le32_to_cpu(desc->fw_version);
2320                 psp->ta_dtm_ucode_size     = le32_to_cpu(desc->size_bytes);
2321                 psp->ta_dtm_start_addr     = ucode_start_addr;
2322                 break;
2323         default:
2324                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
2325                 break;
2326         }
2327
2328         return 0;
2329 }
2330
2331 int psp_init_ta_microcode(struct psp_context *psp,
2332                           const char *chip_name)
2333 {
2334         struct amdgpu_device *adev = psp->adev;
2335         char fw_name[30];
2336         const struct ta_firmware_header_v2_0 *ta_hdr;
2337         int err = 0;
2338         int ta_index = 0;
2339
2340         if (!chip_name) {
2341                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
2342                 return -EINVAL;
2343         }
2344
2345         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
2346         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
2347         if (err)
2348                 goto out;
2349
2350         err = amdgpu_ucode_validate(adev->psp.ta_fw);
2351         if (err)
2352                 goto out;
2353
2354         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
2355
2356         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
2357                 dev_err(adev->dev, "unsupported TA header version\n");
2358                 err = -EINVAL;
2359                 goto out;
2360         }
2361
2362         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_TA_PACKAGING) {
2363                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
2364                 err = -EINVAL;
2365                 goto out;
2366         }
2367
2368         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
2369                 err = parse_ta_bin_descriptor(psp,
2370                                               &ta_hdr->ta_fw_bin[ta_index],
2371                                               ta_hdr);
2372                 if (err)
2373                         goto out;
2374         }
2375
2376         return 0;
2377 out:
2378         dev_err(adev->dev, "fail to initialize ta microcode\n");
2379         release_firmware(adev->psp.ta_fw);
2380         adev->psp.ta_fw = NULL;
2381         return err;
2382 }
2383
2384 static int psp_set_clockgating_state(void *handle,
2385                                      enum amd_clockgating_state state)
2386 {
2387         return 0;
2388 }
2389
2390 static int psp_set_powergating_state(void *handle,
2391                                      enum amd_powergating_state state)
2392 {
2393         return 0;
2394 }
2395
2396 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
2397                                          struct device_attribute *attr,
2398                                          char *buf)
2399 {
2400         struct drm_device *ddev = dev_get_drvdata(dev);
2401         struct amdgpu_device *adev = ddev->dev_private;
2402         uint32_t fw_ver;
2403         int ret;
2404
2405         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2406                 DRM_INFO("PSP block is not ready yet.");
2407                 return -EBUSY;
2408         }
2409
2410         mutex_lock(&adev->psp.mutex);
2411         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
2412         mutex_unlock(&adev->psp.mutex);
2413
2414         if (ret) {
2415                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
2416                 return ret;
2417         }
2418
2419         return snprintf(buf, PAGE_SIZE, "%x\n", fw_ver);
2420 }
2421
2422 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
2423                                                        struct device_attribute *attr,
2424                                                        const char *buf,
2425                                                        size_t count)
2426 {
2427         struct drm_device *ddev = dev_get_drvdata(dev);
2428         struct amdgpu_device *adev = ddev->dev_private;
2429         void *cpu_addr;
2430         dma_addr_t dma_addr;
2431         int ret;
2432         char fw_name[100];
2433         const struct firmware *usbc_pd_fw;
2434
2435         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2436                 DRM_INFO("PSP block is not ready yet.");
2437                 return -EBUSY;
2438         }
2439
2440         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
2441         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
2442         if (ret)
2443                 goto fail;
2444
2445         /* We need contiguous physical mem to place the FW  for psp to access */
2446         cpu_addr = dma_alloc_coherent(adev->dev, usbc_pd_fw->size, &dma_addr, GFP_KERNEL);
2447
2448         ret = dma_mapping_error(adev->dev, dma_addr);
2449         if (ret)
2450                 goto rel_buf;
2451
2452         memcpy_toio(cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
2453
2454         /*
2455          * x86 specific workaround.
2456          * Without it the buffer is invisible in PSP.
2457          *
2458          * TODO Remove once PSP starts snooping CPU cache
2459          */
2460 #ifdef CONFIG_X86
2461         clflush_cache_range(cpu_addr, (usbc_pd_fw->size & ~(L1_CACHE_BYTES - 1)));
2462 #endif
2463
2464         mutex_lock(&adev->psp.mutex);
2465         ret = psp_load_usbc_pd_fw(&adev->psp, dma_addr);
2466         mutex_unlock(&adev->psp.mutex);
2467
2468 rel_buf:
2469         dma_free_coherent(adev->dev, usbc_pd_fw->size, cpu_addr, dma_addr);
2470         release_firmware(usbc_pd_fw);
2471
2472 fail:
2473         if (ret) {
2474                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
2475                 return ret;
2476         }
2477
2478         return count;
2479 }
2480
2481 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
2482                    psp_usbc_pd_fw_sysfs_read,
2483                    psp_usbc_pd_fw_sysfs_write);
2484
2485
2486
2487 const struct amd_ip_funcs psp_ip_funcs = {
2488         .name = "psp",
2489         .early_init = psp_early_init,
2490         .late_init = NULL,
2491         .sw_init = psp_sw_init,
2492         .sw_fini = psp_sw_fini,
2493         .hw_init = psp_hw_init,
2494         .hw_fini = psp_hw_fini,
2495         .suspend = psp_suspend,
2496         .resume = psp_resume,
2497         .is_idle = NULL,
2498         .check_soft_reset = NULL,
2499         .wait_for_idle = NULL,
2500         .soft_reset = NULL,
2501         .set_clockgating_state = psp_set_clockgating_state,
2502         .set_powergating_state = psp_set_powergating_state,
2503 };
2504
2505 static int psp_sysfs_init(struct amdgpu_device *adev)
2506 {
2507         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
2508
2509         if (ret)
2510                 DRM_ERROR("Failed to create USBC PD FW control file!");
2511
2512         return ret;
2513 }
2514
2515 static void psp_sysfs_fini(struct amdgpu_device *adev)
2516 {
2517         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
2518 }
2519
2520 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
2521 {
2522         .type = AMD_IP_BLOCK_TYPE_PSP,
2523         .major = 3,
2524         .minor = 1,
2525         .rev = 0,
2526         .funcs = &psp_ip_funcs,
2527 };
2528
2529 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
2530 {
2531         .type = AMD_IP_BLOCK_TYPE_PSP,
2532         .major = 10,
2533         .minor = 0,
2534         .rev = 0,
2535         .funcs = &psp_ip_funcs,
2536 };
2537
2538 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
2539 {
2540         .type = AMD_IP_BLOCK_TYPE_PSP,
2541         .major = 11,
2542         .minor = 0,
2543         .rev = 0,
2544         .funcs = &psp_ip_funcs,
2545 };
2546
2547 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
2548 {
2549         .type = AMD_IP_BLOCK_TYPE_PSP,
2550         .major = 12,
2551         .minor = 0,
2552         .rev = 0,
2553         .funcs = &psp_ip_funcs,
2554 };