drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40 #include "psp_v13_0_4.h"
  41
  42 #include "amdgpu_ras.h"
  43 #include "amdgpu_securedisplay.h"
  44 #include "amdgpu_atomfirmware.h"
  45
  46 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*3)
  47
  48 static int psp_sysfs_init(struct amdgpu_device *adev);
  49 static void psp_sysfs_fini(struct amdgpu_device *adev);
  50
  51 static int psp_load_smu_fw(struct psp_context *psp);
  52 static int psp_rap_terminate(struct psp_context *psp);
  53 static int psp_securedisplay_terminate(struct psp_context *psp);
  54
  55 static int psp_ring_init(struct psp_context *psp,
  56                          enum psp_ring_type ring_type)
  57 {
  58         int ret = 0;
  59         struct psp_ring *ring;
  60         struct amdgpu_device *adev = psp->adev;
  61
  62         ring = &psp->km_ring;
  63
  64         ring->ring_type = ring_type;
  65
  66         /* allocate 4k Page of Local Frame Buffer memory for ring */
  67         ring->ring_size = 0x1000;
  68         ret = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE,
  69                                       AMDGPU_GEM_DOMAIN_VRAM,
  70                                       &adev->firmware.rbuf,
  71                                       &ring->ring_mem_mc_addr,
  72                                       (void **)&ring->ring_mem);
  73         if (ret) {
  74                 ring->ring_size = 0;
  75                 return ret;
  76         }
  77
  78         return 0;
  79 }
  80
  81 /*
  82  * Due to DF Cstate management centralized to PMFW, the firmware
  83  * loading sequence will be updated as below:
  84  *   - Load KDB
  85  *   - Load SYS_DRV
  86  *   - Load tOS
  87  *   - Load PMFW
  88  *   - Setup TMR
  89  *   - Load other non-psp fw
  90  *   - Load ASD
  91  *   - Load XGMI/RAS/HDCP/DTM TA if any
  92  *
  93  * This new sequence is required for
  94  *   - Arcturus and onwards
  95  */
  96 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  97 {
  98         struct amdgpu_device *adev = psp->adev;
  99
 100         if (amdgpu_sriov_vf(adev)) {
 101                 psp->pmfw_centralized_cstate_management = false;
 102                 return;
 103         }
 104
 105         switch (adev->ip_versions[MP0_HWIP][0]) {
 106         case IP_VERSION(11, 0, 0):
 107         case IP_VERSION(11, 0, 4):
 108         case IP_VERSION(11, 0, 5):
 109         case IP_VERSION(11, 0, 7):
 110         case IP_VERSION(11, 0, 9):
 111         case IP_VERSION(11, 0, 11):
 112         case IP_VERSION(11, 0, 12):
 113         case IP_VERSION(11, 0, 13):
 114         case IP_VERSION(13, 0, 0):
 115         case IP_VERSION(13, 0, 2):
 116         case IP_VERSION(13, 0, 7):
 117                 psp->pmfw_centralized_cstate_management = true;
 118                 break;
 119         default:
 120                 psp->pmfw_centralized_cstate_management = false;
 121                 break;
 122         }
 123 }
 124
 125 static int psp_early_init(void *handle)
 126 {
 127         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 128         struct psp_context *psp = &adev->psp;
 129
 130         switch (adev->ip_versions[MP0_HWIP][0]) {
 131         case IP_VERSION(9, 0, 0):
 132                 psp_v3_1_set_psp_funcs(psp);
 133                 psp->autoload_supported = false;
 134                 break;
 135         case IP_VERSION(10, 0, 0):
 136         case IP_VERSION(10, 0, 1):
 137                 psp_v10_0_set_psp_funcs(psp);
 138                 psp->autoload_supported = false;
 139                 break;
 140         case IP_VERSION(11, 0, 2):
 141         case IP_VERSION(11, 0, 4):
 142                 psp_v11_0_set_psp_funcs(psp);
 143                 psp->autoload_supported = false;
 144                 break;
 145         case IP_VERSION(11, 0, 0):
 146         case IP_VERSION(11, 0, 5):
 147         case IP_VERSION(11, 0, 9):
 148         case IP_VERSION(11, 0, 7):
 149         case IP_VERSION(11, 0, 11):
 150         case IP_VERSION(11, 5, 0):
 151         case IP_VERSION(11, 0, 12):
 152         case IP_VERSION(11, 0, 13):
 153                 psp_v11_0_set_psp_funcs(psp);
 154                 psp->autoload_supported = true;
 155                 break;
 156         case IP_VERSION(11, 0, 3):
 157         case IP_VERSION(12, 0, 1):
 158                 psp_v12_0_set_psp_funcs(psp);
 159                 break;
 160         case IP_VERSION(13, 0, 2):
 161                 psp_v13_0_set_psp_funcs(psp);
 162                 break;
 163         case IP_VERSION(13, 0, 1):
 164         case IP_VERSION(13, 0, 3):
 165         case IP_VERSION(13, 0, 5):
 166         case IP_VERSION(13, 0, 8):
 167         case IP_VERSION(13, 0, 10):
 168                 psp_v13_0_set_psp_funcs(psp);
 169                 psp->autoload_supported = true;
 170                 break;
 171         case IP_VERSION(11, 0, 8):
 172                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 173                         psp_v11_0_8_set_psp_funcs(psp);
 174                         psp->autoload_supported = false;
 175                 }
 176                 break;
 177         case IP_VERSION(13, 0, 0):
 178         case IP_VERSION(13, 0, 7):
 179                 psp_v13_0_set_psp_funcs(psp);
 180                 psp->autoload_supported = true;
 181                 break;
 182         case IP_VERSION(13, 0, 4):
 183                 psp_v13_0_4_set_psp_funcs(psp);
 184                 psp->autoload_supported = true;
 185                 break;
 186         default:
 187                 return -EINVAL;
 188         }
 189
 190         psp->adev = adev;
 191
 192         psp_check_pmfw_centralized_cstate_management(psp);
 193
 194         return 0;
 195 }
 196
 197 void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 198 {
 199         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 200                               &mem_ctx->shared_buf);
 201 }
 202
 203 static void psp_free_shared_bufs(struct psp_context *psp)
 204 {
 205         void *tmr_buf;
 206         void **pptr;
 207
 208         /* free TMR memory buffer */
 209         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 210         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 211
 212         /* free xgmi shared memory */
 213         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
 214
 215         /* free ras shared memory */
 216         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
 217
 218         /* free hdcp shared memory */
 219         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
 220
 221         /* free dtm shared memory */
 222         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
 223
 224         /* free rap shared memory */
 225         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
 226
 227         /* free securedisplay shared memory */
 228         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
 229
 230
 231 }
 232
 233 static void psp_memory_training_fini(struct psp_context *psp)
 234 {
 235         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 236
 237         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 238         kfree(ctx->sys_cache);
 239         ctx->sys_cache = NULL;
 240 }
 241
 242 static int psp_memory_training_init(struct psp_context *psp)
 243 {
 244         int ret;
 245         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 246
 247         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 248                 DRM_DEBUG("memory training is not supported!\n");
 249                 return 0;
 250         }
 251
 252         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 253         if (ctx->sys_cache == NULL) {
 254                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 255                 ret = -ENOMEM;
 256                 goto Err_out;
 257         }
 258
 259         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 260                   ctx->train_data_size,
 261                   ctx->p2c_train_data_offset,
 262                   ctx->c2p_train_data_offset);
 263         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 264         return 0;
 265
 266 Err_out:
 267         psp_memory_training_fini(psp);
 268         return ret;
 269 }
 270
 271 /*
 272  * Helper funciton to query psp runtime database entry
 273  *
 274  * @adev: amdgpu_device pointer
 275  * @entry_type: the type of psp runtime database entry
 276  * @db_entry: runtime database entry pointer
 277  *
 278  * Return false if runtime database doesn't exit or entry is invalid
 279  * or true if the specific database entry is found, and copy to @db_entry
 280  */
 281 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 282                                      enum psp_runtime_entry_type entry_type,
 283                                      void *db_entry)
 284 {
 285         uint64_t db_header_pos, db_dir_pos;
 286         struct psp_runtime_data_header db_header = {0};
 287         struct psp_runtime_data_directory db_dir = {0};
 288         bool ret = false;
 289         int i;
 290
 291         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 292         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 293
 294         /* read runtime db header from vram */
 295         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 296                         sizeof(struct psp_runtime_data_header), false);
 297
 298         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 299                 /* runtime db doesn't exist, exit */
 300                 dev_warn(adev->dev, "PSP runtime database doesn't exist\n");
 301                 return false;
 302         }
 303
 304         /* read runtime database entry from vram */
 305         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 306                         sizeof(struct psp_runtime_data_directory), false);
 307
 308         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 309                 /* invalid db entry count, exit */
 310                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 311                 return false;
 312         }
 313
 314         /* look up for requested entry type */
 315         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 316                 if (db_dir.entry_list[i].entry_type == entry_type) {
 317                         switch (entry_type) {
 318                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 319                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 320                                         /* invalid db entry size */
 321                                         dev_warn(adev->dev, "Invalid PSP runtime database boot cfg entry size\n");
 322                                         return false;
 323                                 }
 324                                 /* read runtime database entry */
 325                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 326                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 327                                 ret = true;
 328                                 break;
 329                         case PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS:
 330                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_scpm_entry)) {
 331                                         /* invalid db entry size */
 332                                         dev_warn(adev->dev, "Invalid PSP runtime database scpm entry size\n");
 333                                         return false;
 334                                 }
 335                                 /* read runtime database entry */
 336                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 337                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_scpm_entry), false);
 338                                 ret = true;
 339                                 break;
 340                         default:
 341                                 ret = false;
 342                                 break;
 343                         }
 344                 }
 345         }
 346
 347         return ret;
 348 }
 349
 350 static int psp_init_sriov_microcode(struct psp_context *psp)
 351 {
 352         struct amdgpu_device *adev = psp->adev;
 353         int ret = 0;
 354
 355         switch (adev->ip_versions[MP0_HWIP][0]) {
 356         case IP_VERSION(9, 0, 0):
 357                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 358                 ret = psp_init_cap_microcode(psp, "vega10");
 359                 break;
 360         case IP_VERSION(11, 0, 9):
 361                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 362                 ret = psp_init_cap_microcode(psp, "navi12");
 363                 break;
 364         case IP_VERSION(11, 0, 7):
 365                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 366                 ret = psp_init_cap_microcode(psp, "sienna_cichlid");
 367                 break;
 368         case IP_VERSION(13, 0, 2):
 369                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 370                 ret = psp_init_cap_microcode(psp, "aldebaran");
 371                 ret &= psp_init_ta_microcode(psp, "aldebaran");
 372                 break;
 373         case IP_VERSION(13, 0, 0):
 374                 adev->virt.autoload_ucode_id = 0;
 375                 break;
 376         case IP_VERSION(13, 0, 10):
 377                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
 378                 break;
 379         default:
 380                 BUG();
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 static int psp_sw_init(void *handle)
 387 {
 388         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 389         struct psp_context *psp = &adev->psp;
 390         int ret;
 391         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 392         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 393         struct psp_runtime_scpm_entry scpm_entry;
 394
 395         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 396         if (!psp->cmd) {
 397                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 398                 ret = -ENOMEM;
 399         }
 400
 401         if (amdgpu_sriov_vf(adev))
 402                 ret = psp_init_sriov_microcode(psp);
 403         else
 404                 ret = psp_init_microcode(psp);
 405         if (ret) {
 406                 DRM_ERROR("Failed to load psp firmware!\n");
 407                 return ret;
 408         }
 409
 410         adev->psp.xgmi_context.supports_extended_data =
 411                 !adev->gmc.xgmi.connected_to_cpu &&
 412                         adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2);
 413
 414         memset(&scpm_entry, 0, sizeof(scpm_entry));
 415         if ((psp_get_runtime_db_entry(adev,
 416                                 PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS,
 417                                 &scpm_entry)) &&
 418             (SCPM_DISABLE != scpm_entry.scpm_status)) {
 419                 adev->scpm_enabled = true;
 420                 adev->scpm_status = scpm_entry.scpm_status;
 421         } else {
 422                 adev->scpm_enabled = false;
 423                 adev->scpm_status = SCPM_DISABLE;
 424         }
 425
 426         /* TODO: stop gpu driver services and print alarm if scpm is enabled with error status */
 427
 428         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 429         if (psp_get_runtime_db_entry(adev,
 430                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 431                                 &boot_cfg_entry)) {
 432                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 433                 if ((psp->boot_cfg_bitmask) &
 434                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 435                         /* If psp runtime database exists, then
 436                          * only enable two stage memory training
 437                          * when TWO_STAGE_DRAM_TRAINING bit is set
 438                          * in runtime database */
 439                         mem_training_ctx->enable_mem_training = true;
 440                 }
 441
 442         } else {
 443                 /* If psp runtime database doesn't exist or
 444                  * is invalid, force enable two stage memory
 445                  * training */
 446                 mem_training_ctx->enable_mem_training = true;
 447         }
 448
 449         if (mem_training_ctx->enable_mem_training) {
 450                 ret = psp_memory_training_init(psp);
 451                 if (ret) {
 452                         DRM_ERROR("Failed to initialize memory training!\n");
 453                         return ret;
 454                 }
 455
 456                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 457                 if (ret) {
 458                         DRM_ERROR("Failed to process memory training!\n");
 459                         return ret;
 460                 }
 461         }
 462
 463         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 464             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 465                 ret= psp_sysfs_init(adev);
 466                 if (ret) {
 467                         return ret;
 468                 }
 469         }
 470
 471         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 472                                       amdgpu_sriov_vf(adev) ?
 473                                       AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 474                                       &psp->fw_pri_bo,
 475                                       &psp->fw_pri_mc_addr,
 476                                       &psp->fw_pri_buf);
 477         if (ret)
 478                 return ret;
 479
 480         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 481                                       AMDGPU_GEM_DOMAIN_VRAM,
 482                                       &psp->fence_buf_bo,
 483                                       &psp->fence_buf_mc_addr,
 484                                       &psp->fence_buf);
 485         if (ret)
 486                 goto failed1;
 487
 488         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 489                                       AMDGPU_GEM_DOMAIN_VRAM,
 490                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 491                                       (void **)&psp->cmd_buf_mem);
 492         if (ret)
 493                 goto failed2;
 494
 495         return 0;
 496
 497 failed2:
 498         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 499                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 500 failed1:
 501         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 502                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 503         return ret;
 504 }
 505
 506 static int psp_sw_fini(void *handle)
 507 {
 508         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 509         struct psp_context *psp = &adev->psp;
 510         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 511
 512         psp_memory_training_fini(psp);
 513         if (psp->sos_fw) {
 514                 release_firmware(psp->sos_fw);
 515                 psp->sos_fw = NULL;
 516         }
 517         if (psp->asd_fw) {
 518                 release_firmware(psp->asd_fw);
 519                 psp->asd_fw = NULL;
 520         }
 521         if (psp->ta_fw) {
 522                 release_firmware(psp->ta_fw);
 523                 psp->ta_fw = NULL;
 524         }
 525         if (psp->cap_fw) {
 526                 release_firmware(psp->cap_fw);
 527                 psp->cap_fw = NULL;
 528         }
 529         if (psp->toc_fw) {
 530                 release_firmware(psp->toc_fw);
 531                 psp->toc_fw = NULL;
 532         }
 533         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 534             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 535                 psp_sysfs_fini(adev);
 536
 537         kfree(cmd);
 538         cmd = NULL;
 539
 540         if (psp->km_ring.ring_mem)
 541                 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
 542                                       &psp->km_ring.ring_mem_mc_addr,
 543                                       (void **)&psp->km_ring.ring_mem);
 544
 545         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 546                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 547         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 548                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 549         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 550                               (void **)&psp->cmd_buf_mem);
 551
 552         return 0;
 553 }
 554
 555 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 556                  uint32_t reg_val, uint32_t mask, bool check_changed)
 557 {
 558         uint32_t val;
 559         int i;
 560         struct amdgpu_device *adev = psp->adev;
 561
 562         if (psp->adev->no_hw_access)
 563                 return 0;
 564
 565         for (i = 0; i < adev->usec_timeout; i++) {
 566                 val = RREG32(reg_index);
 567                 if (check_changed) {
 568                         if (val != reg_val)
 569                                 return 0;
 570                 } else {
 571                         if ((val & mask) == reg_val)
 572                                 return 0;
 573                 }
 574                 udelay(1);
 575         }
 576
 577         return -ETIME;
 578 }
 579
 580 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 581 {
 582         switch (cmd_id) {
 583         case GFX_CMD_ID_LOAD_TA:
 584                 return "LOAD_TA";
 585         case GFX_CMD_ID_UNLOAD_TA:
 586                 return "UNLOAD_TA";
 587         case GFX_CMD_ID_INVOKE_CMD:
 588                 return "INVOKE_CMD";
 589         case GFX_CMD_ID_LOAD_ASD:
 590                 return "LOAD_ASD";
 591         case GFX_CMD_ID_SETUP_TMR:
 592                 return "SETUP_TMR";
 593         case GFX_CMD_ID_LOAD_IP_FW:
 594                 return "LOAD_IP_FW";
 595         case GFX_CMD_ID_DESTROY_TMR:
 596                 return "DESTROY_TMR";
 597         case GFX_CMD_ID_SAVE_RESTORE:
 598                 return "SAVE_RESTORE_IP_FW";
 599         case GFX_CMD_ID_SETUP_VMR:
 600                 return "SETUP_VMR";
 601         case GFX_CMD_ID_DESTROY_VMR:
 602                 return "DESTROY_VMR";
 603         case GFX_CMD_ID_PROG_REG:
 604                 return "PROG_REG";
 605         case GFX_CMD_ID_GET_FW_ATTESTATION:
 606                 return "GET_FW_ATTESTATION";
 607         case GFX_CMD_ID_LOAD_TOC:
 608                 return "ID_LOAD_TOC";
 609         case GFX_CMD_ID_AUTOLOAD_RLC:
 610                 return "AUTOLOAD_RLC";
 611         case GFX_CMD_ID_BOOT_CFG:
 612                 return "BOOT_CFG";
 613         default:
 614                 return "UNKNOWN CMD";
 615         }
 616 }
 617
 618 static int
 619 psp_cmd_submit_buf(struct psp_context *psp,
 620                    struct amdgpu_firmware_info *ucode,
 621                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 622 {
 623         int ret;
 624         int index, idx;
 625         int timeout = 20000;
 626         bool ras_intr = false;
 627         bool skip_unsupport = false;
 628
 629         if (psp->adev->no_hw_access)
 630                 return 0;
 631
 632         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
 633                 return 0;
 634
 635         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 636
 637         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 638
 639         index = atomic_inc_return(&psp->fence_value);
 640         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 641         if (ret) {
 642                 atomic_dec(&psp->fence_value);
 643                 goto exit;
 644         }
 645
 646         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 647         while (*((unsigned int *)psp->fence_buf) != index) {
 648                 if (--timeout == 0)
 649                         break;
 650                 /*
 651                  * Shouldn't wait for timeout when err_event_athub occurs,
 652                  * because gpu reset thread triggered and lock resource should
 653                  * be released for psp resume sequence.
 654                  */
 655                 ras_intr = amdgpu_ras_intr_triggered();
 656                 if (ras_intr)
 657                         break;
 658                 usleep_range(10, 100);
 659                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 660         }
 661
 662         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 663         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 664                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 665
 666         memcpy((void*)&cmd->resp, (void*)&psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 667
 668         /* In some cases, psp response status is not 0 even there is no
 669          * problem while the command is submitted. Some version of PSP FW
 670          * doesn't write 0 to that field.
 671          * So here we would like to only print a warning instead of an error
 672          * during psp initialization to avoid breaking hw_init and it doesn't
 673          * return -EINVAL.
 674          */
 675         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 676                 if (ucode)
 677                         DRM_WARN("failed to load ucode %s(0x%X) ",
 678                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 679                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 680                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 681                          psp->cmd_buf_mem->resp.status);
 682                 /* If any firmware (including CAP) load fails under SRIOV, it should
 683                  * return failure to stop the VF from initializing.
 684                  * Also return failure in case of timeout
 685                  */
 686                 if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
 687                         ret = -EINVAL;
 688                         goto exit;
 689                 }
 690         }
 691
 692         if (ucode) {
 693                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 694                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 695         }
 696
 697 exit:
 698         drm_dev_exit(idx);
 699         return ret;
 700 }
 701
 702 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 703 {
 704         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 705
 706         mutex_lock(&psp->mutex);
 707
 708         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 709
 710         return cmd;
 711 }
 712
 713 static void release_psp_cmd_buf(struct psp_context *psp)
 714 {
 715         mutex_unlock(&psp->mutex);
 716 }
 717
 718 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 719                                  struct psp_gfx_cmd_resp *cmd,
 720                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 721 {
 722         struct amdgpu_device *adev = psp->adev;
 723         uint32_t size = amdgpu_bo_size(tmr_bo);
 724         uint64_t tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 725
 726         if (amdgpu_sriov_vf(psp->adev))
 727                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 728         else
 729                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 730         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 731         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 732         cmd->cmd.cmd_setup_tmr.buf_size = size;
 733         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 734         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 735         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 736 }
 737
 738 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 739                                       uint64_t pri_buf_mc, uint32_t size)
 740 {
 741         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 742         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 743         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 744         cmd->cmd.cmd_load_toc.toc_size = size;
 745 }
 746
 747 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 748 static int psp_load_toc(struct psp_context *psp,
 749                         uint32_t *tmr_size)
 750 {
 751         int ret;
 752         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 753
 754         /* Copy toc to psp firmware private buffer */
 755         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 756
 757         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 758
 759         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 760                                  psp->fence_buf_mc_addr);
 761         if (!ret)
 762                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 763
 764         release_psp_cmd_buf(psp);
 765
 766         return ret;
 767 }
 768
 769 /* Set up Trusted Memory Region */
 770 static int psp_tmr_init(struct psp_context *psp)
 771 {
 772         int ret;
 773         int tmr_size;
 774         void *tmr_buf;
 775         void **pptr;
 776
 777         /*
 778          * According to HW engineer, they prefer the TMR address be "naturally
 779          * aligned" , e.g. the start address be an integer divide of TMR size.
 780          *
 781          * Note: this memory need be reserved till the driver
 782          * uninitializes.
 783          */
 784         tmr_size = PSP_TMR_SIZE(psp->adev);
 785
 786         /* For ASICs support RLC autoload, psp will parse the toc
 787          * and calculate the total size of TMR needed */
 788         if (!amdgpu_sriov_vf(psp->adev) &&
 789             psp->toc.start_addr &&
 790             psp->toc.size_bytes &&
 791             psp->fw_pri_buf) {
 792                 ret = psp_load_toc(psp, &tmr_size);
 793                 if (ret) {
 794                         DRM_ERROR("Failed to load toc\n");
 795                         return ret;
 796                 }
 797         }
 798
 799         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 800         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_ALIGNMENT,
 801                                       AMDGPU_GEM_DOMAIN_VRAM,
 802                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 803
 804         return ret;
 805 }
 806
 807 static bool psp_skip_tmr(struct psp_context *psp)
 808 {
 809         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 810         case IP_VERSION(11, 0, 9):
 811         case IP_VERSION(11, 0, 7):
 812         case IP_VERSION(13, 0, 2):
 813         case IP_VERSION(13, 0, 10):
 814                 return true;
 815         default:
 816                 return false;
 817         }
 818 }
 819
 820 static int psp_tmr_load(struct psp_context *psp)
 821 {
 822         int ret;
 823         struct psp_gfx_cmd_resp *cmd;
 824
 825         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 826          * Already set up by host driver.
 827          */
 828         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 829                 return 0;
 830
 831         cmd = acquire_psp_cmd_buf(psp);
 832
 833         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 834         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 835                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 836
 837         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 838                                  psp->fence_buf_mc_addr);
 839
 840         release_psp_cmd_buf(psp);
 841
 842         return ret;
 843 }
 844
 845 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 846                                         struct psp_gfx_cmd_resp *cmd)
 847 {
 848         if (amdgpu_sriov_vf(psp->adev))
 849                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 850         else
 851                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 852 }
 853
 854 static int psp_tmr_unload(struct psp_context *psp)
 855 {
 856         int ret;
 857         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 858
 859         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 860         dev_info(psp->adev->dev, "free PSP TMR buffer\n");
 861
 862         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 863                                  psp->fence_buf_mc_addr);
 864
 865         release_psp_cmd_buf(psp);
 866
 867         return ret;
 868 }
 869
 870 static int psp_tmr_terminate(struct psp_context *psp)
 871 {
 872         return psp_tmr_unload(psp);
 873 }
 874
 875 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 876                                         uint64_t *output_ptr)
 877 {
 878         int ret;
 879         struct psp_gfx_cmd_resp *cmd;
 880
 881         if (!output_ptr)
 882                 return -EINVAL;
 883
 884         if (amdgpu_sriov_vf(psp->adev))
 885                 return 0;
 886
 887         cmd = acquire_psp_cmd_buf(psp);
 888
 889         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 890
 891         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 892                                  psp->fence_buf_mc_addr);
 893
 894         if (!ret) {
 895                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 896                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 897         }
 898
 899         release_psp_cmd_buf(psp);
 900
 901         return ret;
 902 }
 903
 904 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 905 {
 906         struct psp_context *psp = &adev->psp;
 907         struct psp_gfx_cmd_resp *cmd;
 908         int ret;
 909
 910         if (amdgpu_sriov_vf(adev))
 911                 return 0;
 912
 913         cmd = acquire_psp_cmd_buf(psp);
 914
 915         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 916         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 917
 918         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 919         if (!ret) {
 920                 *boot_cfg =
 921                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 922         }
 923
 924         release_psp_cmd_buf(psp);
 925
 926         return ret;
 927 }
 928
 929 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 930 {
 931         int ret;
 932         struct psp_context *psp = &adev->psp;
 933         struct psp_gfx_cmd_resp *cmd;
 934
 935         if (amdgpu_sriov_vf(adev))
 936                 return 0;
 937
 938         cmd = acquire_psp_cmd_buf(psp);
 939
 940         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 941         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 942         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 943         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 944
 945         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 946
 947         release_psp_cmd_buf(psp);
 948
 949         return ret;
 950 }
 951
 952 static int psp_rl_load(struct amdgpu_device *adev)
 953 {
 954         int ret;
 955         struct psp_context *psp = &adev->psp;
 956         struct psp_gfx_cmd_resp *cmd;
 957
 958         if (!is_psp_fw_valid(psp->rl))
 959                 return 0;
 960
 961         cmd = acquire_psp_cmd_buf(psp);
 962
 963         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 964         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
 965
 966         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 967         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
 968         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
 969         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
 970         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
 971
 972         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 973
 974         release_psp_cmd_buf(psp);
 975
 976         return ret;
 977 }
 978
 979 static int psp_asd_initialize(struct psp_context *psp)
 980 {
 981         int ret;
 982
 983         /* If PSP version doesn't match ASD version, asd loading will be failed.
 984          * add workaround to bypass it for sriov now.
 985          * TODO: add version check to make it common
 986          */
 987         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 988                 return 0;
 989
 990         psp->asd_context.mem_context.shared_mc_addr  = 0;
 991         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 992         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
 993
 994         ret = psp_ta_load(psp, &psp->asd_context);
 995         if (!ret)
 996                 psp->asd_context.initialized = true;
 997
 998         return ret;
 999 }
1000
1001 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1002                                        uint32_t session_id)
1003 {
1004         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
1005         cmd->cmd.cmd_unload_ta.session_id = session_id;
1006 }
1007
1008 int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
1009 {
1010         int ret;
1011         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1012
1013         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
1014
1015         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1016
1017         context->resp_status = cmd->resp.status;
1018
1019         release_psp_cmd_buf(psp);
1020
1021         return ret;
1022 }
1023
1024 static int psp_asd_terminate(struct psp_context *psp)
1025 {
1026         int ret;
1027
1028         if (amdgpu_sriov_vf(psp->adev))
1029                 return 0;
1030
1031         if (!psp->asd_context.initialized)
1032                 return 0;
1033
1034         ret = psp_ta_unload(psp, &psp->asd_context);
1035         if (!ret)
1036                 psp->asd_context.initialized = false;
1037
1038         return ret;
1039 }
1040
1041 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1042                 uint32_t id, uint32_t value)
1043 {
1044         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
1045         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
1046         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
1047 }
1048
1049 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
1050                 uint32_t value)
1051 {
1052         struct psp_gfx_cmd_resp *cmd;
1053         int ret = 0;
1054
1055         if (reg >= PSP_REG_LAST)
1056                 return -EINVAL;
1057
1058         cmd = acquire_psp_cmd_buf(psp);
1059
1060         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
1061         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1062         if (ret)
1063                 DRM_ERROR("PSP failed to program reg id %d", reg);
1064
1065         release_psp_cmd_buf(psp);
1066
1067         return ret;
1068 }
1069
1070 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1071                                      uint64_t ta_bin_mc,
1072                                      struct ta_context *context)
1073 {
1074         cmd->cmd_id                             = context->ta_load_type;
1075         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
1076         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
1077         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
1078
1079         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
1080                 lower_32_bits(context->mem_context.shared_mc_addr);
1081         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
1082                 upper_32_bits(context->mem_context.shared_mc_addr);
1083         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
1084 }
1085
1086 int psp_ta_init_shared_buf(struct psp_context *psp,
1087                                   struct ta_mem_context *mem_ctx)
1088 {
1089         /*
1090         * Allocate 16k memory aligned to 4k from Frame Buffer (local
1091         * physical) for ta to host memory
1092         */
1093         return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
1094                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1095                                       &mem_ctx->shared_bo,
1096                                       &mem_ctx->shared_mc_addr,
1097                                       &mem_ctx->shared_buf);
1098 }
1099
1100 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1101                                        uint32_t ta_cmd_id,
1102                                        uint32_t session_id)
1103 {
1104         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
1105         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
1106         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
1107 }
1108
1109 int psp_ta_invoke(struct psp_context *psp,
1110                   uint32_t ta_cmd_id,
1111                   struct ta_context *context)
1112 {
1113         int ret;
1114         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1115
1116         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
1117
1118         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1119                                  psp->fence_buf_mc_addr);
1120
1121         context->resp_status = cmd->resp.status;
1122
1123         release_psp_cmd_buf(psp);
1124
1125         return ret;
1126 }
1127
1128 int psp_ta_load(struct psp_context *psp, struct ta_context *context)
1129 {
1130         int ret;
1131         struct psp_gfx_cmd_resp *cmd;
1132
1133         cmd = acquire_psp_cmd_buf(psp);
1134
1135         psp_copy_fw(psp, context->bin_desc.start_addr,
1136                     context->bin_desc.size_bytes);
1137
1138         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
1139
1140         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1141                                  psp->fence_buf_mc_addr);
1142
1143         context->resp_status = cmd->resp.status;
1144
1145         if (!ret) {
1146                 context->session_id = cmd->resp.session_id;
1147         }
1148
1149         release_psp_cmd_buf(psp);
1150
1151         return ret;
1152 }
1153
1154 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1155 {
1156         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1157 }
1158
1159 int psp_xgmi_terminate(struct psp_context *psp)
1160 {
1161         int ret;
1162         struct amdgpu_device *adev = psp->adev;
1163
1164         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1165         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1166             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1167              adev->gmc.xgmi.connected_to_cpu))
1168                 return 0;
1169
1170         if (!psp->xgmi_context.context.initialized)
1171                 return 0;
1172
1173         ret = psp_ta_unload(psp, &psp->xgmi_context.context);
1174
1175         psp->xgmi_context.context.initialized = false;
1176
1177         return ret;
1178 }
1179
1180 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1181 {
1182         struct ta_xgmi_shared_memory *xgmi_cmd;
1183         int ret;
1184
1185         if (!psp->ta_fw ||
1186             !psp->xgmi_context.context.bin_desc.size_bytes ||
1187             !psp->xgmi_context.context.bin_desc.start_addr)
1188                 return -ENOENT;
1189
1190         if (!load_ta)
1191                 goto invoke;
1192
1193         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1194         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1195
1196         if (!psp->xgmi_context.context.mem_context.shared_buf) {
1197                 ret = psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
1198                 if (ret)
1199                         return ret;
1200         }
1201
1202         /* Load XGMI TA */
1203         ret = psp_ta_load(psp, &psp->xgmi_context.context);
1204         if (!ret)
1205                 psp->xgmi_context.context.initialized = true;
1206         else
1207                 return ret;
1208
1209 invoke:
1210         /* Initialize XGMI session */
1211         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1212         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1213         xgmi_cmd->flag_extend_link_record = set_extended_data;
1214         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1215
1216         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1217
1218         return ret;
1219 }
1220
1221 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1222 {
1223         struct ta_xgmi_shared_memory *xgmi_cmd;
1224         int ret;
1225
1226         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1227         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1228
1229         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1230
1231         /* Invoke xgmi ta to get hive id */
1232         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1233         if (ret)
1234                 return ret;
1235
1236         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1237
1238         return 0;
1239 }
1240
1241 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1242 {
1243         struct ta_xgmi_shared_memory *xgmi_cmd;
1244         int ret;
1245
1246         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1247         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1248
1249         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1250
1251         /* Invoke xgmi ta to get the node id */
1252         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1253         if (ret)
1254                 return ret;
1255
1256         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1257
1258         return 0;
1259 }
1260
1261 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1262 {
1263         return psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1264                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b;
1265 }
1266
1267 /*
1268  * Chips that support extended topology information require the driver to
1269  * reflect topology information in the opposite direction.  This is
1270  * because the TA has already exceeded its link record limit and if the
1271  * TA holds bi-directional information, the driver would have to do
1272  * multiple fetches instead of just two.
1273  */
1274 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1275                                         struct psp_xgmi_node_info node_info)
1276 {
1277         struct amdgpu_device *mirror_adev;
1278         struct amdgpu_hive_info *hive;
1279         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1280         uint64_t dst_node_id = node_info.node_id;
1281         uint8_t dst_num_hops = node_info.num_hops;
1282         uint8_t dst_num_links = node_info.num_links;
1283
1284         hive = amdgpu_get_xgmi_hive(psp->adev);
1285         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1286                 struct psp_xgmi_topology_info *mirror_top_info;
1287                 int j;
1288
1289                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1290                         continue;
1291
1292                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1293                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1294                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1295                                 continue;
1296
1297                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1298                         /*
1299                          * prevent 0 num_links value re-reflection since reflection
1300                          * criteria is based on num_hops (direct or indirect).
1301                          *
1302                          */
1303                         if (dst_num_links)
1304                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1305
1306                         break;
1307                 }
1308
1309                 break;
1310         }
1311
1312         amdgpu_put_xgmi_hive(hive);
1313 }
1314
1315 int psp_xgmi_get_topology_info(struct psp_context *psp,
1316                                int number_devices,
1317                                struct psp_xgmi_topology_info *topology,
1318                                bool get_extended_data)
1319 {
1320         struct ta_xgmi_shared_memory *xgmi_cmd;
1321         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1322         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1323         int i;
1324         int ret;
1325
1326         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1327                 return -EINVAL;
1328
1329         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1330         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1331         xgmi_cmd->flag_extend_link_record = get_extended_data;
1332
1333         /* Fill in the shared memory with topology information as input */
1334         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1335         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1336         topology_info_input->num_nodes = number_devices;
1337
1338         for (i = 0; i < topology_info_input->num_nodes; i++) {
1339                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1340                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1341                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1342                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1343         }
1344
1345         /* Invoke xgmi ta to get the topology information */
1346         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1347         if (ret)
1348                 return ret;
1349
1350         /* Read the output topology information from the shared memory */
1351         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1352         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1353         for (i = 0; i < topology->num_nodes; i++) {
1354                 /* extended data will either be 0 or equal to non-extended data */
1355                 if (topology_info_output->nodes[i].num_hops)
1356                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1357
1358                 /* non-extended data gets everything here so no need to update */
1359                 if (!get_extended_data) {
1360                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1361                         topology->nodes[i].is_sharing_enabled =
1362                                         topology_info_output->nodes[i].is_sharing_enabled;
1363                         topology->nodes[i].sdma_engine =
1364                                         topology_info_output->nodes[i].sdma_engine;
1365                 }
1366
1367         }
1368
1369         /* Invoke xgmi ta again to get the link information */
1370         if (psp_xgmi_peer_link_info_supported(psp)) {
1371                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1372
1373                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1374
1375                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1376
1377                 if (ret)
1378                         return ret;
1379
1380                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1381                 for (i = 0; i < topology->num_nodes; i++) {
1382                         /* accumulate num_links on extended data */
1383                         topology->nodes[i].num_links = get_extended_data ?
1384                                         topology->nodes[i].num_links +
1385                                                         link_info_output->nodes[i].num_links :
1386                                         link_info_output->nodes[i].num_links;
1387
1388                         /* reflect the topology information for bi-directionality */
1389                         if (psp->xgmi_context.supports_extended_data &&
1390                                         get_extended_data && topology->nodes[i].num_hops)
1391                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1392                 }
1393         }
1394
1395         return 0;
1396 }
1397
1398 int psp_xgmi_set_topology_info(struct psp_context *psp,
1399                                int number_devices,
1400                                struct psp_xgmi_topology_info *topology)
1401 {
1402         struct ta_xgmi_shared_memory *xgmi_cmd;
1403         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1404         int i;
1405
1406         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1407                 return -EINVAL;
1408
1409         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1410         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1411
1412         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1413         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1414         topology_info_input->num_nodes = number_devices;
1415
1416         for (i = 0; i < topology_info_input->num_nodes; i++) {
1417                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1418                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1419                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1420                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1421         }
1422
1423         /* Invoke xgmi ta to set topology information */
1424         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1425 }
1426
1427 // ras begin
1428 static void psp_ras_ta_check_status(struct psp_context *psp)
1429 {
1430         struct ta_ras_shared_memory *ras_cmd =
1431                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1432
1433         switch (ras_cmd->ras_status) {
1434         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1435                 dev_warn(psp->adev->dev,
1436                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1437                 break;
1438         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1439                 dev_warn(psp->adev->dev,
1440                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1441                 break;
1442         case TA_RAS_STATUS__SUCCESS:
1443                 break;
1444         case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED:
1445                 if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR)
1446                         dev_warn(psp->adev->dev,
1447                                         "RAS WARNING: Inject error to critical region is not allowed\n");
1448                 break;
1449         default:
1450                 dev_warn(psp->adev->dev,
1451                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1452                 break;
1453         }
1454 }
1455
1456 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1457 {
1458         struct ta_ras_shared_memory *ras_cmd;
1459         int ret;
1460
1461         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1462
1463         /*
1464          * TODO: bypass the loading in sriov for now
1465          */
1466         if (amdgpu_sriov_vf(psp->adev))
1467                 return 0;
1468
1469         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1470
1471         if (amdgpu_ras_intr_triggered())
1472                 return ret;
1473
1474         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1475         {
1476                 DRM_WARN("RAS: Unsupported Interface");
1477                 return -EINVAL;
1478         }
1479
1480         if (!ret) {
1481                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1482                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1483
1484                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1485                 }
1486                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1487                         dev_warn(psp->adev->dev,
1488                                  "RAS internal register access blocked\n");
1489
1490                 psp_ras_ta_check_status(psp);
1491         }
1492
1493         return ret;
1494 }
1495
1496 int psp_ras_enable_features(struct psp_context *psp,
1497                 union ta_ras_cmd_input *info, bool enable)
1498 {
1499         struct ta_ras_shared_memory *ras_cmd;
1500         int ret;
1501
1502         if (!psp->ras_context.context.initialized)
1503                 return -EINVAL;
1504
1505         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1506         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1507
1508         if (enable)
1509                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1510         else
1511                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1512
1513         ras_cmd->ras_in_message = *info;
1514
1515         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1516         if (ret)
1517                 return -EINVAL;
1518
1519         return 0;
1520 }
1521
1522 int psp_ras_terminate(struct psp_context *psp)
1523 {
1524         int ret;
1525
1526         /*
1527          * TODO: bypass the terminate in sriov for now
1528          */
1529         if (amdgpu_sriov_vf(psp->adev))
1530                 return 0;
1531
1532         if (!psp->ras_context.context.initialized)
1533                 return 0;
1534
1535         ret = psp_ta_unload(psp, &psp->ras_context.context);
1536
1537         psp->ras_context.context.initialized = false;
1538
1539         return ret;
1540 }
1541
1542 int psp_ras_initialize(struct psp_context *psp)
1543 {
1544         int ret;
1545         uint32_t boot_cfg = 0xFF;
1546         struct amdgpu_device *adev = psp->adev;
1547         struct ta_ras_shared_memory *ras_cmd;
1548
1549         /*
1550          * TODO: bypass the initialize in sriov for now
1551          */
1552         if (amdgpu_sriov_vf(adev))
1553                 return 0;
1554
1555         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1556             !adev->psp.ras_context.context.bin_desc.start_addr) {
1557                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1558                 return 0;
1559         }
1560
1561         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1562                 /* query GECC enablement status from boot config
1563                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1564                  */
1565                 ret = psp_boot_config_get(adev, &boot_cfg);
1566                 if (ret)
1567                         dev_warn(adev->dev, "PSP get boot config failed\n");
1568
1569                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1570                         if (!boot_cfg) {
1571                                 dev_info(adev->dev, "GECC is disabled\n");
1572                         } else {
1573                                 /* disable GECC in next boot cycle if ras is
1574                                  * disabled by module parameter amdgpu_ras_enable
1575                                  * and/or amdgpu_ras_mask, or boot_config_get call
1576                                  * is failed
1577                                  */
1578                                 ret = psp_boot_config_set(adev, 0);
1579                                 if (ret)
1580                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1581                                 else
1582                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle "
1583                                                  "if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1584                         }
1585                 } else {
1586                         if (1 == boot_cfg) {
1587                                 dev_info(adev->dev, "GECC is enabled\n");
1588                         } else {
1589                                 /* enable GECC in next boot cycle if it is disabled
1590                                  * in boot config, or force enable GECC if failed to
1591                                  * get boot configuration
1592                                  */
1593                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1594                                 if (ret)
1595                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1596                                 else
1597                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1598                         }
1599                 }
1600         }
1601
1602         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1603         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1604
1605         if (!psp->ras_context.context.mem_context.shared_buf) {
1606                 ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1607                 if (ret)
1608                         return ret;
1609         }
1610
1611         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1612         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1613
1614         if (amdgpu_ras_is_poison_mode_supported(adev))
1615                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1616         if (!adev->gmc.xgmi.connected_to_cpu)
1617                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1618
1619         ret = psp_ta_load(psp, &psp->ras_context.context);
1620
1621         if (!ret && !ras_cmd->ras_status)
1622                 psp->ras_context.context.initialized = true;
1623         else {
1624                 if (ras_cmd->ras_status)
1625                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1626
1627                 /* fail to load RAS TA */
1628                 psp->ras_context.context.initialized = false;
1629         }
1630
1631         return ret;
1632 }
1633
1634 int psp_ras_trigger_error(struct psp_context *psp,
1635                           struct ta_ras_trigger_error_input *info)
1636 {
1637         struct ta_ras_shared_memory *ras_cmd;
1638         int ret;
1639
1640         if (!psp->ras_context.context.initialized)
1641                 return -EINVAL;
1642
1643         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1644         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1645
1646         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1647         ras_cmd->ras_in_message.trigger_error = *info;
1648
1649         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1650         if (ret)
1651                 return -EINVAL;
1652
1653         /* If err_event_athub occurs error inject was successful, however
1654            return status from TA is no long reliable */
1655         if (amdgpu_ras_intr_triggered())
1656                 return 0;
1657
1658         if (ras_cmd->ras_status == TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED)
1659                 return -EACCES;
1660         else if (ras_cmd->ras_status)
1661                 return -EINVAL;
1662
1663         return 0;
1664 }
1665 // ras end
1666
1667 // HDCP start
1668 static int psp_hdcp_initialize(struct psp_context *psp)
1669 {
1670         int ret;
1671
1672         /*
1673          * TODO: bypass the initialize in sriov for now
1674          */
1675         if (amdgpu_sriov_vf(psp->adev))
1676                 return 0;
1677
1678         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1679             !psp->hdcp_context.context.bin_desc.start_addr) {
1680                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1681                 return 0;
1682         }
1683
1684         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1685         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1686
1687         if (!psp->hdcp_context.context.initialized) {
1688                 ret = psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1689                 if (ret)
1690                         return ret;
1691         }
1692
1693         ret = psp_ta_load(psp, &psp->hdcp_context.context);
1694         if (!ret) {
1695                 psp->hdcp_context.context.initialized = true;
1696                 mutex_init(&psp->hdcp_context.mutex);
1697         }
1698
1699         return ret;
1700 }
1701
1702 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1703 {
1704         /*
1705          * TODO: bypass the loading in sriov for now
1706          */
1707         if (amdgpu_sriov_vf(psp->adev))
1708                 return 0;
1709
1710         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1711 }
1712
1713 static int psp_hdcp_terminate(struct psp_context *psp)
1714 {
1715         int ret;
1716
1717         /*
1718          * TODO: bypass the terminate in sriov for now
1719          */
1720         if (amdgpu_sriov_vf(psp->adev))
1721                 return 0;
1722
1723         if (!psp->hdcp_context.context.initialized)
1724                 return 0;
1725
1726         ret = psp_ta_unload(psp, &psp->hdcp_context.context);
1727
1728         psp->hdcp_context.context.initialized = false;
1729
1730         return ret;
1731 }
1732 // HDCP end
1733
1734 // DTM start
1735 static int psp_dtm_initialize(struct psp_context *psp)
1736 {
1737         int ret;
1738
1739         /*
1740          * TODO: bypass the initialize in sriov for now
1741          */
1742         if (amdgpu_sriov_vf(psp->adev))
1743                 return 0;
1744
1745         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1746             !psp->dtm_context.context.bin_desc.start_addr) {
1747                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1748                 return 0;
1749         }
1750
1751         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1752         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1753
1754         if (!psp->dtm_context.context.initialized) {
1755                 ret = psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1756                 if (ret)
1757                         return ret;
1758         }
1759
1760         ret = psp_ta_load(psp, &psp->dtm_context.context);
1761         if (!ret) {
1762                 psp->dtm_context.context.initialized = true;
1763                 mutex_init(&psp->dtm_context.mutex);
1764         }
1765
1766         return ret;
1767 }
1768
1769 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1770 {
1771         /*
1772          * TODO: bypass the loading in sriov for now
1773          */
1774         if (amdgpu_sriov_vf(psp->adev))
1775                 return 0;
1776
1777         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1778 }
1779
1780 static int psp_dtm_terminate(struct psp_context *psp)
1781 {
1782         int ret;
1783
1784         /*
1785          * TODO: bypass the terminate in sriov for now
1786          */
1787         if (amdgpu_sriov_vf(psp->adev))
1788                 return 0;
1789
1790         if (!psp->dtm_context.context.initialized)
1791                 return 0;
1792
1793         ret = psp_ta_unload(psp, &psp->dtm_context.context);
1794
1795         psp->dtm_context.context.initialized = false;
1796
1797         return ret;
1798 }
1799 // DTM end
1800
1801 // RAP start
1802 static int psp_rap_initialize(struct psp_context *psp)
1803 {
1804         int ret;
1805         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1806
1807         /*
1808          * TODO: bypass the initialize in sriov for now
1809          */
1810         if (amdgpu_sriov_vf(psp->adev))
1811                 return 0;
1812
1813         if (!psp->rap_context.context.bin_desc.size_bytes ||
1814             !psp->rap_context.context.bin_desc.start_addr) {
1815                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1816                 return 0;
1817         }
1818
1819         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1820         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1821
1822         if (!psp->rap_context.context.initialized) {
1823                 ret = psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1824                 if (ret)
1825                         return ret;
1826         }
1827
1828         ret = psp_ta_load(psp, &psp->rap_context.context);
1829         if (!ret) {
1830                 psp->rap_context.context.initialized = true;
1831                 mutex_init(&psp->rap_context.mutex);
1832         } else
1833                 return ret;
1834
1835         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1836         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1837                 psp_rap_terminate(psp);
1838                 /* free rap shared memory */
1839                 psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1840
1841                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1842                          ret, status);
1843
1844                 return ret;
1845         }
1846
1847         return 0;
1848 }
1849
1850 static int psp_rap_terminate(struct psp_context *psp)
1851 {
1852         int ret;
1853
1854         if (!psp->rap_context.context.initialized)
1855                 return 0;
1856
1857         ret = psp_ta_unload(psp, &psp->rap_context.context);
1858
1859         psp->rap_context.context.initialized = false;
1860
1861         return ret;
1862 }
1863
1864 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1865 {
1866         struct ta_rap_shared_memory *rap_cmd;
1867         int ret = 0;
1868
1869         if (!psp->rap_context.context.initialized)
1870                 return 0;
1871
1872         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1873             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1874                 return -EINVAL;
1875
1876         mutex_lock(&psp->rap_context.mutex);
1877
1878         rap_cmd = (struct ta_rap_shared_memory *)
1879                   psp->rap_context.context.mem_context.shared_buf;
1880         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1881
1882         rap_cmd->cmd_id = ta_cmd_id;
1883         rap_cmd->validation_method_id = METHOD_A;
1884
1885         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1886         if (ret)
1887                 goto out_unlock;
1888
1889         if (status)
1890                 *status = rap_cmd->rap_status;
1891
1892 out_unlock:
1893         mutex_unlock(&psp->rap_context.mutex);
1894
1895         return ret;
1896 }
1897 // RAP end
1898
1899 /* securedisplay start */
1900 static int psp_securedisplay_initialize(struct psp_context *psp)
1901 {
1902         int ret;
1903         struct securedisplay_cmd *securedisplay_cmd;
1904
1905         /*
1906          * TODO: bypass the initialize in sriov for now
1907          */
1908         if (amdgpu_sriov_vf(psp->adev))
1909                 return 0;
1910
1911         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1912             !psp->securedisplay_context.context.bin_desc.start_addr) {
1913                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
1914                 return 0;
1915         }
1916
1917         psp->securedisplay_context.context.mem_context.shared_mem_size =
1918                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
1919         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1920
1921         if (!psp->securedisplay_context.context.initialized) {
1922                 ret = psp_ta_init_shared_buf(psp,
1923                                              &psp->securedisplay_context.context.mem_context);
1924                 if (ret)
1925                         return ret;
1926         }
1927
1928         ret = psp_ta_load(psp, &psp->securedisplay_context.context);
1929         if (!ret) {
1930                 psp->securedisplay_context.context.initialized = true;
1931                 mutex_init(&psp->securedisplay_context.mutex);
1932         } else
1933                 return ret;
1934
1935         mutex_lock(&psp->securedisplay_context.mutex);
1936
1937         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
1938                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1939
1940         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1941
1942         mutex_unlock(&psp->securedisplay_context.mutex);
1943
1944         if (ret) {
1945                 psp_securedisplay_terminate(psp);
1946                 /* free securedisplay shared memory */
1947                 psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
1948                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
1949                 return -EINVAL;
1950         }
1951
1952         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
1953                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
1954                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
1955                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
1956         }
1957
1958         return 0;
1959 }
1960
1961 static int psp_securedisplay_terminate(struct psp_context *psp)
1962 {
1963         int ret;
1964
1965         /*
1966          * TODO:bypass the terminate in sriov for now
1967          */
1968         if (amdgpu_sriov_vf(psp->adev))
1969                 return 0;
1970
1971         if (!psp->securedisplay_context.context.initialized)
1972                 return 0;
1973
1974         ret = psp_ta_unload(psp, &psp->securedisplay_context.context);
1975
1976         psp->securedisplay_context.context.initialized = false;
1977
1978         return ret;
1979 }
1980
1981 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1982 {
1983         int ret;
1984
1985         if (!psp->securedisplay_context.context.initialized)
1986                 return -EINVAL;
1987
1988         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
1989             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
1990                 return -EINVAL;
1991
1992         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
1993
1994         return ret;
1995 }
1996 /* SECUREDISPLAY end */
1997
1998 static int psp_hw_start(struct psp_context *psp)
1999 {
2000         struct amdgpu_device *adev = psp->adev;
2001         int ret;
2002
2003         if (!amdgpu_sriov_vf(adev)) {
2004                 if ((is_psp_fw_valid(psp->kdb)) &&
2005                     (psp->funcs->bootloader_load_kdb != NULL)) {
2006                         ret = psp_bootloader_load_kdb(psp);
2007                         if (ret) {
2008                                 DRM_ERROR("PSP load kdb failed!\n");
2009                                 return ret;
2010                         }
2011                 }
2012
2013                 if ((is_psp_fw_valid(psp->spl)) &&
2014                     (psp->funcs->bootloader_load_spl != NULL)) {
2015                         ret = psp_bootloader_load_spl(psp);
2016                         if (ret) {
2017                                 DRM_ERROR("PSP load spl failed!\n");
2018                                 return ret;
2019                         }
2020                 }
2021
2022                 if ((is_psp_fw_valid(psp->sys)) &&
2023                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
2024                         ret = psp_bootloader_load_sysdrv(psp);
2025                         if (ret) {
2026                                 DRM_ERROR("PSP load sys drv failed!\n");
2027                                 return ret;
2028                         }
2029                 }
2030
2031                 if ((is_psp_fw_valid(psp->soc_drv)) &&
2032                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
2033                         ret = psp_bootloader_load_soc_drv(psp);
2034                         if (ret) {
2035                                 DRM_ERROR("PSP load soc drv failed!\n");
2036                                 return ret;
2037                         }
2038                 }
2039
2040                 if ((is_psp_fw_valid(psp->intf_drv)) &&
2041                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
2042                         ret = psp_bootloader_load_intf_drv(psp);
2043                         if (ret) {
2044                                 DRM_ERROR("PSP load intf drv failed!\n");
2045                                 return ret;
2046                         }
2047                 }
2048
2049                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
2050                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
2051                         ret = psp_bootloader_load_dbg_drv(psp);
2052                         if (ret) {
2053                                 DRM_ERROR("PSP load dbg drv failed!\n");
2054                                 return ret;
2055                         }
2056                 }
2057
2058                 if ((is_psp_fw_valid(psp->ras_drv)) &&
2059                     (psp->funcs->bootloader_load_ras_drv != NULL)) {
2060                         ret = psp_bootloader_load_ras_drv(psp);
2061                         if (ret) {
2062                                 DRM_ERROR("PSP load ras_drv failed!\n");
2063                                 return ret;
2064                         }
2065                 }
2066
2067                 if ((is_psp_fw_valid(psp->sos)) &&
2068                     (psp->funcs->bootloader_load_sos != NULL)) {
2069                         ret = psp_bootloader_load_sos(psp);
2070                         if (ret) {
2071                                 DRM_ERROR("PSP load sos failed!\n");
2072                                 return ret;
2073                         }
2074                 }
2075         }
2076
2077         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2078         if (ret) {
2079                 DRM_ERROR("PSP create ring failed!\n");
2080                 return ret;
2081         }
2082
2083         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2084                 goto skip_pin_bo;
2085
2086         ret = psp_tmr_init(psp);
2087         if (ret) {
2088                 DRM_ERROR("PSP tmr init failed!\n");
2089                 return ret;
2090         }
2091
2092 skip_pin_bo:
2093         /*
2094          * For ASICs with DF Cstate management centralized
2095          * to PMFW, TMR setup should be performed after PMFW
2096          * loaded and before other non-psp firmware loaded.
2097          */
2098         if (psp->pmfw_centralized_cstate_management) {
2099                 ret = psp_load_smu_fw(psp);
2100                 if (ret)
2101                         return ret;
2102         }
2103
2104         ret = psp_tmr_load(psp);
2105         if (ret) {
2106                 DRM_ERROR("PSP load tmr failed!\n");
2107                 return ret;
2108         }
2109
2110         return 0;
2111 }
2112
2113 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2114                            enum psp_gfx_fw_type *type)
2115 {
2116         switch (ucode->ucode_id) {
2117         case AMDGPU_UCODE_ID_CAP:
2118                 *type = GFX_FW_TYPE_CAP;
2119                 break;
2120         case AMDGPU_UCODE_ID_SDMA0:
2121                 *type = GFX_FW_TYPE_SDMA0;
2122                 break;
2123         case AMDGPU_UCODE_ID_SDMA1:
2124                 *type = GFX_FW_TYPE_SDMA1;
2125                 break;
2126         case AMDGPU_UCODE_ID_SDMA2:
2127                 *type = GFX_FW_TYPE_SDMA2;
2128                 break;
2129         case AMDGPU_UCODE_ID_SDMA3:
2130                 *type = GFX_FW_TYPE_SDMA3;
2131                 break;
2132         case AMDGPU_UCODE_ID_SDMA4:
2133                 *type = GFX_FW_TYPE_SDMA4;
2134                 break;
2135         case AMDGPU_UCODE_ID_SDMA5:
2136                 *type = GFX_FW_TYPE_SDMA5;
2137                 break;
2138         case AMDGPU_UCODE_ID_SDMA6:
2139                 *type = GFX_FW_TYPE_SDMA6;
2140                 break;
2141         case AMDGPU_UCODE_ID_SDMA7:
2142                 *type = GFX_FW_TYPE_SDMA7;
2143                 break;
2144         case AMDGPU_UCODE_ID_CP_MES:
2145                 *type = GFX_FW_TYPE_CP_MES;
2146                 break;
2147         case AMDGPU_UCODE_ID_CP_MES_DATA:
2148                 *type = GFX_FW_TYPE_MES_STACK;
2149                 break;
2150         case AMDGPU_UCODE_ID_CP_MES1:
2151                 *type = GFX_FW_TYPE_CP_MES_KIQ;
2152                 break;
2153         case AMDGPU_UCODE_ID_CP_MES1_DATA:
2154                 *type = GFX_FW_TYPE_MES_KIQ_STACK;
2155                 break;
2156         case AMDGPU_UCODE_ID_CP_CE:
2157                 *type = GFX_FW_TYPE_CP_CE;
2158                 break;
2159         case AMDGPU_UCODE_ID_CP_PFP:
2160                 *type = GFX_FW_TYPE_CP_PFP;
2161                 break;
2162         case AMDGPU_UCODE_ID_CP_ME:
2163                 *type = GFX_FW_TYPE_CP_ME;
2164                 break;
2165         case AMDGPU_UCODE_ID_CP_MEC1:
2166                 *type = GFX_FW_TYPE_CP_MEC;
2167                 break;
2168         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2169                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2170                 break;
2171         case AMDGPU_UCODE_ID_CP_MEC2:
2172                 *type = GFX_FW_TYPE_CP_MEC;
2173                 break;
2174         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2175                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2176                 break;
2177         case AMDGPU_UCODE_ID_RLC_P:
2178                 *type = GFX_FW_TYPE_RLC_P;
2179                 break;
2180         case AMDGPU_UCODE_ID_RLC_V:
2181                 *type = GFX_FW_TYPE_RLC_V;
2182                 break;
2183         case AMDGPU_UCODE_ID_RLC_G:
2184                 *type = GFX_FW_TYPE_RLC_G;
2185                 break;
2186         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2187                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2188                 break;
2189         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2190                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2191                 break;
2192         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2193                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2194                 break;
2195         case AMDGPU_UCODE_ID_RLC_IRAM:
2196                 *type = GFX_FW_TYPE_RLC_IRAM;
2197                 break;
2198         case AMDGPU_UCODE_ID_RLC_DRAM:
2199                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2200                 break;
2201         case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
2202                 *type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
2203                 break;
2204         case AMDGPU_UCODE_ID_SE0_TAP_DELAYS:
2205                 *type = GFX_FW_TYPE_SE0_TAP_DELAYS;
2206                 break;
2207         case AMDGPU_UCODE_ID_SE1_TAP_DELAYS:
2208                 *type = GFX_FW_TYPE_SE1_TAP_DELAYS;
2209                 break;
2210         case AMDGPU_UCODE_ID_SE2_TAP_DELAYS:
2211                 *type = GFX_FW_TYPE_SE2_TAP_DELAYS;
2212                 break;
2213         case AMDGPU_UCODE_ID_SE3_TAP_DELAYS:
2214                 *type = GFX_FW_TYPE_SE3_TAP_DELAYS;
2215                 break;
2216         case AMDGPU_UCODE_ID_SMC:
2217                 *type = GFX_FW_TYPE_SMU;
2218                 break;
2219         case AMDGPU_UCODE_ID_PPTABLE:
2220                 *type = GFX_FW_TYPE_PPTABLE;
2221                 break;
2222         case AMDGPU_UCODE_ID_UVD:
2223                 *type = GFX_FW_TYPE_UVD;
2224                 break;
2225         case AMDGPU_UCODE_ID_UVD1:
2226                 *type = GFX_FW_TYPE_UVD1;
2227                 break;
2228         case AMDGPU_UCODE_ID_VCE:
2229                 *type = GFX_FW_TYPE_VCE;
2230                 break;
2231         case AMDGPU_UCODE_ID_VCN:
2232                 *type = GFX_FW_TYPE_VCN;
2233                 break;
2234         case AMDGPU_UCODE_ID_VCN1:
2235                 *type = GFX_FW_TYPE_VCN1;
2236                 break;
2237         case AMDGPU_UCODE_ID_DMCU_ERAM:
2238                 *type = GFX_FW_TYPE_DMCU_ERAM;
2239                 break;
2240         case AMDGPU_UCODE_ID_DMCU_INTV:
2241                 *type = GFX_FW_TYPE_DMCU_ISR;
2242                 break;
2243         case AMDGPU_UCODE_ID_VCN0_RAM:
2244                 *type = GFX_FW_TYPE_VCN0_RAM;
2245                 break;
2246         case AMDGPU_UCODE_ID_VCN1_RAM:
2247                 *type = GFX_FW_TYPE_VCN1_RAM;
2248                 break;
2249         case AMDGPU_UCODE_ID_DMCUB:
2250                 *type = GFX_FW_TYPE_DMUB;
2251                 break;
2252         case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
2253                 *type = GFX_FW_TYPE_SDMA_UCODE_TH0;
2254                 break;
2255         case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
2256                 *type = GFX_FW_TYPE_SDMA_UCODE_TH1;
2257                 break;
2258         case AMDGPU_UCODE_ID_IMU_I:
2259                 *type = GFX_FW_TYPE_IMU_I;
2260                 break;
2261         case AMDGPU_UCODE_ID_IMU_D:
2262                 *type = GFX_FW_TYPE_IMU_D;
2263                 break;
2264         case AMDGPU_UCODE_ID_CP_RS64_PFP:
2265                 *type = GFX_FW_TYPE_RS64_PFP;
2266                 break;
2267         case AMDGPU_UCODE_ID_CP_RS64_ME:
2268                 *type = GFX_FW_TYPE_RS64_ME;
2269                 break;
2270         case AMDGPU_UCODE_ID_CP_RS64_MEC:
2271                 *type = GFX_FW_TYPE_RS64_MEC;
2272                 break;
2273         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
2274                 *type = GFX_FW_TYPE_RS64_PFP_P0_STACK;
2275                 break;
2276         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
2277                 *type = GFX_FW_TYPE_RS64_PFP_P1_STACK;
2278                 break;
2279         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
2280                 *type = GFX_FW_TYPE_RS64_ME_P0_STACK;
2281                 break;
2282         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
2283                 *type = GFX_FW_TYPE_RS64_ME_P1_STACK;
2284                 break;
2285         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
2286                 *type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
2287                 break;
2288         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
2289                 *type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
2290                 break;
2291         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
2292                 *type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
2293                 break;
2294         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
2295                 *type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
2296                 break;
2297         case AMDGPU_UCODE_ID_MAXIMUM:
2298         default:
2299                 return -EINVAL;
2300         }
2301
2302         return 0;
2303 }
2304
2305 static void psp_print_fw_hdr(struct psp_context *psp,
2306                              struct amdgpu_firmware_info *ucode)
2307 {
2308         struct amdgpu_device *adev = psp->adev;
2309         struct common_firmware_header *hdr;
2310
2311         switch (ucode->ucode_id) {
2312         case AMDGPU_UCODE_ID_SDMA0:
2313         case AMDGPU_UCODE_ID_SDMA1:
2314         case AMDGPU_UCODE_ID_SDMA2:
2315         case AMDGPU_UCODE_ID_SDMA3:
2316         case AMDGPU_UCODE_ID_SDMA4:
2317         case AMDGPU_UCODE_ID_SDMA5:
2318         case AMDGPU_UCODE_ID_SDMA6:
2319         case AMDGPU_UCODE_ID_SDMA7:
2320                 hdr = (struct common_firmware_header *)
2321                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2322                 amdgpu_ucode_print_sdma_hdr(hdr);
2323                 break;
2324         case AMDGPU_UCODE_ID_CP_CE:
2325                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2326                 amdgpu_ucode_print_gfx_hdr(hdr);
2327                 break;
2328         case AMDGPU_UCODE_ID_CP_PFP:
2329                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2330                 amdgpu_ucode_print_gfx_hdr(hdr);
2331                 break;
2332         case AMDGPU_UCODE_ID_CP_ME:
2333                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2334                 amdgpu_ucode_print_gfx_hdr(hdr);
2335                 break;
2336         case AMDGPU_UCODE_ID_CP_MEC1:
2337                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2338                 amdgpu_ucode_print_gfx_hdr(hdr);
2339                 break;
2340         case AMDGPU_UCODE_ID_RLC_G:
2341                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2342                 amdgpu_ucode_print_rlc_hdr(hdr);
2343                 break;
2344         case AMDGPU_UCODE_ID_SMC:
2345                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2346                 amdgpu_ucode_print_smc_hdr(hdr);
2347                 break;
2348         default:
2349                 break;
2350         }
2351 }
2352
2353 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2354                                        struct psp_gfx_cmd_resp *cmd)
2355 {
2356         int ret;
2357         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2358
2359         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2360         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2361         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2362         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2363
2364         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2365         if (ret)
2366                 DRM_ERROR("Unknown firmware type\n");
2367
2368         return ret;
2369 }
2370
2371 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2372                                   struct amdgpu_firmware_info *ucode)
2373 {
2374         int ret = 0;
2375         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2376
2377         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2378         if (!ret) {
2379                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2380                                          psp->fence_buf_mc_addr);
2381         }
2382
2383         release_psp_cmd_buf(psp);
2384
2385         return ret;
2386 }
2387
2388 static int psp_load_smu_fw(struct psp_context *psp)
2389 {
2390         int ret;
2391         struct amdgpu_device *adev = psp->adev;
2392         struct amdgpu_firmware_info *ucode =
2393                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2394         struct amdgpu_ras *ras = psp->ras_context.ras;
2395
2396         /*
2397          * Skip SMU FW reloading in case of using BACO for runpm only,
2398          * as SMU is always alive.
2399          */
2400         if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
2401                 return 0;
2402
2403         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2404                 return 0;
2405
2406         if ((amdgpu_in_reset(adev) &&
2407              ras && adev->ras_enabled &&
2408              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2409               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2410                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2411                 if (ret) {
2412                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2413                 }
2414         }
2415
2416         ret = psp_execute_non_psp_fw_load(psp, ucode);
2417
2418         if (ret)
2419                 DRM_ERROR("PSP load smu failed!\n");
2420
2421         return ret;
2422 }
2423
2424 static bool fw_load_skip_check(struct psp_context *psp,
2425                                struct amdgpu_firmware_info *ucode)
2426 {
2427         if (!ucode->fw || !ucode->ucode_size)
2428                 return true;
2429
2430         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2431             (psp_smu_reload_quirk(psp) ||
2432              psp->autoload_supported ||
2433              psp->pmfw_centralized_cstate_management))
2434                 return true;
2435
2436         if (amdgpu_sriov_vf(psp->adev) &&
2437             amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
2438                 return true;
2439
2440         if (psp->autoload_supported &&
2441             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2442              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2443                 /* skip mec JT when autoload is enabled */
2444                 return true;
2445
2446         return false;
2447 }
2448
2449 int psp_load_fw_list(struct psp_context *psp,
2450                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2451 {
2452         int ret = 0, i;
2453         struct amdgpu_firmware_info *ucode;
2454
2455         for (i = 0; i < ucode_count; ++i) {
2456                 ucode = ucode_list[i];
2457                 psp_print_fw_hdr(psp, ucode);
2458                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2459                 if (ret)
2460                         return ret;
2461         }
2462         return ret;
2463 }
2464
2465 static int psp_load_non_psp_fw(struct psp_context *psp)
2466 {
2467         int i, ret;
2468         struct amdgpu_firmware_info *ucode;
2469         struct amdgpu_device *adev = psp->adev;
2470
2471         if (psp->autoload_supported &&
2472             !psp->pmfw_centralized_cstate_management) {
2473                 ret = psp_load_smu_fw(psp);
2474                 if (ret)
2475                         return ret;
2476         }
2477
2478         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2479                 ucode = &adev->firmware.ucode[i];
2480
2481                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2482                     !fw_load_skip_check(psp, ucode)) {
2483                         ret = psp_load_smu_fw(psp);
2484                         if (ret)
2485                                 return ret;
2486                         continue;
2487                 }
2488
2489                 if (fw_load_skip_check(psp, ucode))
2490                         continue;
2491
2492                 if (psp->autoload_supported &&
2493                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2494                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2495                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2496                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2497                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2498                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2499                         /* PSP only receive one SDMA fw for sienna_cichlid,
2500                          * as all four sdma fw are same */
2501                         continue;
2502
2503                 psp_print_fw_hdr(psp, ucode);
2504
2505                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2506                 if (ret)
2507                         return ret;
2508
2509                 /* Start rlc autoload after psp recieved all the gfx firmware */
2510                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2511                     adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
2512                         ret = psp_rlc_autoload_start(psp);
2513                         if (ret) {
2514                                 DRM_ERROR("Failed to start rlc autoload\n");
2515                                 return ret;
2516                         }
2517                 }
2518         }
2519
2520         return 0;
2521 }
2522
2523 static int psp_load_fw(struct amdgpu_device *adev)
2524 {
2525         int ret;
2526         struct psp_context *psp = &adev->psp;
2527
2528         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2529                 /* should not destroy ring, only stop */
2530                 psp_ring_stop(psp, PSP_RING_TYPE__KM);
2531         } else {
2532                 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2533
2534                 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2535                 if (ret) {
2536                         DRM_ERROR("PSP ring init failed!\n");
2537                         goto failed;
2538                 }
2539         }
2540
2541         ret = psp_hw_start(psp);
2542         if (ret)
2543                 goto failed;
2544
2545         ret = psp_load_non_psp_fw(psp);
2546         if (ret)
2547                 goto failed1;
2548
2549         ret = psp_asd_initialize(psp);
2550         if (ret) {
2551                 DRM_ERROR("PSP load asd failed!\n");
2552                 goto failed1;
2553         }
2554
2555         ret = psp_rl_load(adev);
2556         if (ret) {
2557                 DRM_ERROR("PSP load RL failed!\n");
2558                 goto failed1;
2559         }
2560
2561         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2562                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2563                         ret = psp_xgmi_initialize(psp, false, true);
2564                         /* Warning the XGMI seesion initialize failure
2565                         * Instead of stop driver initialization
2566                         */
2567                         if (ret)
2568                                 dev_err(psp->adev->dev,
2569                                         "XGMI: Failed to initialize XGMI session\n");
2570                 }
2571         }
2572
2573         if (psp->ta_fw) {
2574                 ret = psp_ras_initialize(psp);
2575                 if (ret)
2576                         dev_err(psp->adev->dev,
2577                                         "RAS: Failed to initialize RAS\n");
2578
2579                 ret = psp_hdcp_initialize(psp);
2580                 if (ret)
2581                         dev_err(psp->adev->dev,
2582                                 "HDCP: Failed to initialize HDCP\n");
2583
2584                 ret = psp_dtm_initialize(psp);
2585                 if (ret)
2586                         dev_err(psp->adev->dev,
2587                                 "DTM: Failed to initialize DTM\n");
2588
2589                 ret = psp_rap_initialize(psp);
2590                 if (ret)
2591                         dev_err(psp->adev->dev,
2592                                 "RAP: Failed to initialize RAP\n");
2593
2594                 ret = psp_securedisplay_initialize(psp);
2595                 if (ret)
2596                         dev_err(psp->adev->dev,
2597                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2598         }
2599
2600         return 0;
2601
2602 failed1:
2603         psp_free_shared_bufs(psp);
2604 failed:
2605         /*
2606          * all cleanup jobs (xgmi terminate, ras terminate,
2607          * ring destroy, cmd/fence/fw buffers destory,
2608          * psp->cmd destory) are delayed to psp_hw_fini
2609          */
2610         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2611         return ret;
2612 }
2613
2614 static int psp_hw_init(void *handle)
2615 {
2616         int ret;
2617         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2618
2619         mutex_lock(&adev->firmware.mutex);
2620         /*
2621          * This sequence is just used on hw_init only once, no need on
2622          * resume.
2623          */
2624         ret = amdgpu_ucode_init_bo(adev);
2625         if (ret)
2626                 goto failed;
2627
2628         ret = psp_load_fw(adev);
2629         if (ret) {
2630                 DRM_ERROR("PSP firmware loading failed\n");
2631                 goto failed;
2632         }
2633
2634         mutex_unlock(&adev->firmware.mutex);
2635         return 0;
2636
2637 failed:
2638         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2639         mutex_unlock(&adev->firmware.mutex);
2640         return -EINVAL;
2641 }
2642
2643 static int psp_hw_fini(void *handle)
2644 {
2645         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2646         struct psp_context *psp = &adev->psp;
2647
2648         if (psp->ta_fw) {
2649                 psp_ras_terminate(psp);
2650                 psp_securedisplay_terminate(psp);
2651                 psp_rap_terminate(psp);
2652                 psp_dtm_terminate(psp);
2653                 psp_hdcp_terminate(psp);
2654
2655                 if (adev->gmc.xgmi.num_physical_nodes > 1)
2656                         psp_xgmi_terminate(psp);
2657         }
2658
2659         psp_asd_terminate(psp);
2660         psp_tmr_terminate(psp);
2661
2662         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2663
2664         psp_free_shared_bufs(psp);
2665
2666         return 0;
2667 }
2668
2669 static int psp_suspend(void *handle)
2670 {
2671         int ret = 0;
2672         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2673         struct psp_context *psp = &adev->psp;
2674
2675         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2676             psp->xgmi_context.context.initialized) {
2677                 ret = psp_xgmi_terminate(psp);
2678                 if (ret) {
2679                         DRM_ERROR("Failed to terminate xgmi ta\n");
2680                         goto out;
2681                 }
2682         }
2683
2684         if (psp->ta_fw) {
2685                 ret = psp_ras_terminate(psp);
2686                 if (ret) {
2687                         DRM_ERROR("Failed to terminate ras ta\n");
2688                         goto out;
2689                 }
2690                 ret = psp_hdcp_terminate(psp);
2691                 if (ret) {
2692                         DRM_ERROR("Failed to terminate hdcp ta\n");
2693                         goto out;
2694                 }
2695                 ret = psp_dtm_terminate(psp);
2696                 if (ret) {
2697                         DRM_ERROR("Failed to terminate dtm ta\n");
2698                         goto out;
2699                 }
2700                 ret = psp_rap_terminate(psp);
2701                 if (ret) {
2702                         DRM_ERROR("Failed to terminate rap ta\n");
2703                         goto out;
2704                 }
2705                 ret = psp_securedisplay_terminate(psp);
2706                 if (ret) {
2707                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2708                         goto out;
2709                 }
2710         }
2711
2712         ret = psp_asd_terminate(psp);
2713         if (ret) {
2714                 DRM_ERROR("Failed to terminate asd\n");
2715                 goto out;
2716         }
2717
2718         ret = psp_tmr_terminate(psp);
2719         if (ret) {
2720                 DRM_ERROR("Failed to terminate tmr\n");
2721                 goto out;
2722         }
2723
2724         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2725         if (ret) {
2726                 DRM_ERROR("PSP ring stop failed\n");
2727         }
2728
2729 out:
2730         psp_free_shared_bufs(psp);
2731
2732         return ret;
2733 }
2734
2735 static int psp_resume(void *handle)
2736 {
2737         int ret;
2738         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2739         struct psp_context *psp = &adev->psp;
2740
2741         DRM_INFO("PSP is resuming...\n");
2742
2743         if (psp->mem_train_ctx.enable_mem_training) {
2744                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2745                 if (ret) {
2746                         DRM_ERROR("Failed to process memory training!\n");
2747                         return ret;
2748                 }
2749         }
2750
2751         mutex_lock(&adev->firmware.mutex);
2752
2753         ret = psp_hw_start(psp);
2754         if (ret)
2755                 goto failed;
2756
2757         ret = psp_load_non_psp_fw(psp);
2758         if (ret)
2759                 goto failed;
2760
2761         ret = psp_asd_initialize(psp);
2762         if (ret) {
2763                 DRM_ERROR("PSP load asd failed!\n");
2764                 goto failed;
2765         }
2766
2767         ret = psp_rl_load(adev);
2768         if (ret) {
2769                 dev_err(adev->dev, "PSP load RL failed!\n");
2770                 goto failed;
2771         }
2772
2773         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2774                 ret = psp_xgmi_initialize(psp, false, true);
2775                 /* Warning the XGMI seesion initialize failure
2776                  * Instead of stop driver initialization
2777                  */
2778                 if (ret)
2779                         dev_err(psp->adev->dev,
2780                                 "XGMI: Failed to initialize XGMI session\n");
2781         }
2782
2783         if (psp->ta_fw) {
2784                 ret = psp_ras_initialize(psp);
2785                 if (ret)
2786                         dev_err(psp->adev->dev,
2787                                         "RAS: Failed to initialize RAS\n");
2788
2789                 ret = psp_hdcp_initialize(psp);
2790                 if (ret)
2791                         dev_err(psp->adev->dev,
2792                                 "HDCP: Failed to initialize HDCP\n");
2793
2794                 ret = psp_dtm_initialize(psp);
2795                 if (ret)
2796                         dev_err(psp->adev->dev,
2797                                 "DTM: Failed to initialize DTM\n");
2798
2799                 ret = psp_rap_initialize(psp);
2800                 if (ret)
2801                         dev_err(psp->adev->dev,
2802                                 "RAP: Failed to initialize RAP\n");
2803
2804                 ret = psp_securedisplay_initialize(psp);
2805                 if (ret)
2806                         dev_err(psp->adev->dev,
2807                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2808         }
2809
2810         mutex_unlock(&adev->firmware.mutex);
2811
2812         return 0;
2813
2814 failed:
2815         DRM_ERROR("PSP resume failed\n");
2816         mutex_unlock(&adev->firmware.mutex);
2817         return ret;
2818 }
2819
2820 int psp_gpu_reset(struct amdgpu_device *adev)
2821 {
2822         int ret;
2823
2824         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2825                 return 0;
2826
2827         mutex_lock(&adev->psp.mutex);
2828         ret = psp_mode1_reset(&adev->psp);
2829         mutex_unlock(&adev->psp.mutex);
2830
2831         return ret;
2832 }
2833
2834 int psp_rlc_autoload_start(struct psp_context *psp)
2835 {
2836         int ret;
2837         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2838
2839         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2840
2841         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2842                                  psp->fence_buf_mc_addr);
2843
2844         release_psp_cmd_buf(psp);
2845
2846         return ret;
2847 }
2848
2849 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2850                         uint64_t cmd_gpu_addr, int cmd_size)
2851 {
2852         struct amdgpu_firmware_info ucode = {0};
2853
2854         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2855                 AMDGPU_UCODE_ID_VCN0_RAM;
2856         ucode.mc_addr = cmd_gpu_addr;
2857         ucode.ucode_size = cmd_size;
2858
2859         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2860 }
2861
2862 int psp_ring_cmd_submit(struct psp_context *psp,
2863                         uint64_t cmd_buf_mc_addr,
2864                         uint64_t fence_mc_addr,
2865                         int index)
2866 {
2867         unsigned int psp_write_ptr_reg = 0;
2868         struct psp_gfx_rb_frame *write_frame;
2869         struct psp_ring *ring = &psp->km_ring;
2870         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2871         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2872                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2873         struct amdgpu_device *adev = psp->adev;
2874         uint32_t ring_size_dw = ring->ring_size / 4;
2875         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2876
2877         /* KM (GPCOM) prepare write pointer */
2878         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2879
2880         /* Update KM RB frame pointer to new frame */
2881         /* write_frame ptr increments by size of rb_frame in bytes */
2882         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2883         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2884                 write_frame = ring_buffer_start;
2885         else
2886                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2887         /* Check invalid write_frame ptr address */
2888         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2889                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2890                           ring_buffer_start, ring_buffer_end, write_frame);
2891                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2892                 return -EINVAL;
2893         }
2894
2895         /* Initialize KM RB frame */
2896         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2897
2898         /* Update KM RB frame */
2899         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2900         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2901         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2902         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2903         write_frame->fence_value = index;
2904         amdgpu_device_flush_hdp(adev, NULL);
2905
2906         /* Update the write Pointer in DWORDs */
2907         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2908         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2909         return 0;
2910 }
2911
2912 int psp_init_asd_microcode(struct psp_context *psp,
2913                            const char *chip_name)
2914 {
2915         struct amdgpu_device *adev = psp->adev;
2916         char fw_name[PSP_FW_NAME_LEN];
2917         const struct psp_firmware_header_v1_0 *asd_hdr;
2918         int err = 0;
2919
2920         if (!chip_name) {
2921                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2922                 return -EINVAL;
2923         }
2924
2925         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2926         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2927         if (err)
2928                 goto out;
2929
2930         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2931         if (err)
2932                 goto out;
2933
2934         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2935         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2936         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
2937         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2938         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
2939                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2940         return 0;
2941 out:
2942         dev_err(adev->dev, "fail to initialize asd microcode\n");
2943         release_firmware(adev->psp.asd_fw);
2944         adev->psp.asd_fw = NULL;
2945         return err;
2946 }
2947
2948 int psp_init_toc_microcode(struct psp_context *psp,
2949                            const char *chip_name)
2950 {
2951         struct amdgpu_device *adev = psp->adev;
2952         char fw_name[PSP_FW_NAME_LEN];
2953         const struct psp_firmware_header_v1_0 *toc_hdr;
2954         int err = 0;
2955
2956         if (!chip_name) {
2957                 dev_err(adev->dev, "invalid chip name for toc microcode\n");
2958                 return -EINVAL;
2959         }
2960
2961         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
2962         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
2963         if (err)
2964                 goto out;
2965
2966         err = amdgpu_ucode_validate(adev->psp.toc_fw);
2967         if (err)
2968                 goto out;
2969
2970         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
2971         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
2972         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
2973         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
2974         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
2975                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
2976         return 0;
2977 out:
2978         dev_err(adev->dev, "fail to request/validate toc microcode\n");
2979         release_firmware(adev->psp.toc_fw);
2980         adev->psp.toc_fw = NULL;
2981         return err;
2982 }
2983
2984 static int parse_sos_bin_descriptor(struct psp_context *psp,
2985                                    const struct psp_fw_bin_desc *desc,
2986                                    const struct psp_firmware_header_v2_0 *sos_hdr)
2987 {
2988         uint8_t *ucode_start_addr  = NULL;
2989
2990         if (!psp || !desc || !sos_hdr)
2991                 return -EINVAL;
2992
2993         ucode_start_addr  = (uint8_t *)sos_hdr +
2994                             le32_to_cpu(desc->offset_bytes) +
2995                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2996
2997         switch (desc->fw_type) {
2998         case PSP_FW_TYPE_PSP_SOS:
2999                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
3000                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
3001                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
3002                 psp->sos.start_addr        = ucode_start_addr;
3003                 break;
3004         case PSP_FW_TYPE_PSP_SYS_DRV:
3005                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
3006                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
3007                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
3008                 psp->sys.start_addr        = ucode_start_addr;
3009                 break;
3010         case PSP_FW_TYPE_PSP_KDB:
3011                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
3012                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
3013                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
3014                 psp->kdb.start_addr        = ucode_start_addr;
3015                 break;
3016         case PSP_FW_TYPE_PSP_TOC:
3017                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
3018                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
3019                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
3020                 psp->toc.start_addr        = ucode_start_addr;
3021                 break;
3022         case PSP_FW_TYPE_PSP_SPL:
3023                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
3024                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
3025                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
3026                 psp->spl.start_addr        = ucode_start_addr;
3027                 break;
3028         case PSP_FW_TYPE_PSP_RL:
3029                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
3030                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
3031                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
3032                 psp->rl.start_addr         = ucode_start_addr;
3033                 break;
3034         case PSP_FW_TYPE_PSP_SOC_DRV:
3035                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
3036                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
3037                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3038                 psp->soc_drv.start_addr         = ucode_start_addr;
3039                 break;
3040         case PSP_FW_TYPE_PSP_INTF_DRV:
3041                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
3042                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
3043                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
3044                 psp->intf_drv.start_addr        = ucode_start_addr;
3045                 break;
3046         case PSP_FW_TYPE_PSP_DBG_DRV:
3047                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
3048                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
3049                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3050                 psp->dbg_drv.start_addr         = ucode_start_addr;
3051                 break;
3052         case PSP_FW_TYPE_PSP_RAS_DRV:
3053                 psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
3054                 psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
3055                 psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3056                 psp->ras_drv.start_addr         = ucode_start_addr;
3057                 break;
3058         default:
3059                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
3060                 break;
3061         }
3062
3063         return 0;
3064 }
3065
3066 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
3067 {
3068         const struct psp_firmware_header_v1_0 *sos_hdr;
3069         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3070         uint8_t *ucode_array_start_addr;
3071
3072         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3073         ucode_array_start_addr = (uint8_t *)sos_hdr +
3074                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3075
3076         if (adev->gmc.xgmi.connected_to_cpu ||
3077             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
3078                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
3079                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
3080
3081                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
3082                 adev->psp.sys.start_addr = ucode_array_start_addr;
3083
3084                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
3085                 adev->psp.sos.start_addr = ucode_array_start_addr +
3086                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
3087         } else {
3088                 /* Load alternate PSP SOS FW */
3089                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3090
3091                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3092                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3093
3094                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
3095                 adev->psp.sys.start_addr = ucode_array_start_addr +
3096                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
3097
3098                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
3099                 adev->psp.sos.start_addr = ucode_array_start_addr +
3100                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
3101         }
3102
3103         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
3104                 dev_warn(adev->dev, "PSP SOS FW not available");
3105                 return -EINVAL;
3106         }
3107
3108         return 0;
3109 }
3110
3111 int psp_init_sos_microcode(struct psp_context *psp,
3112                            const char *chip_name)
3113 {
3114         struct amdgpu_device *adev = psp->adev;
3115         char fw_name[PSP_FW_NAME_LEN];
3116         const struct psp_firmware_header_v1_0 *sos_hdr;
3117         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3118         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3119         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3120         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3121         int err = 0;
3122         uint8_t *ucode_array_start_addr;
3123         int fw_index = 0;
3124
3125         if (!chip_name) {
3126                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
3127                 return -EINVAL;
3128         }
3129
3130         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3131         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
3132         if (err)
3133                 goto out;
3134
3135         err = amdgpu_ucode_validate(adev->psp.sos_fw);
3136         if (err)
3137                 goto out;
3138
3139         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3140         ucode_array_start_addr = (uint8_t *)sos_hdr +
3141                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3142         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3143
3144         switch (sos_hdr->header.header_version_major) {
3145         case 1:
3146                 err = psp_init_sos_base_fw(adev);
3147                 if (err)
3148                         goto out;
3149
3150                 if (sos_hdr->header.header_version_minor == 1) {
3151                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3152                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3153                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3154                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3155                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3156                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3157                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3158                 }
3159                 if (sos_hdr->header.header_version_minor == 2) {
3160                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3161                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3162                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3163                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3164                 }
3165                 if (sos_hdr->header.header_version_minor == 3) {
3166                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3167                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3168                         adev->psp.toc.start_addr = ucode_array_start_addr +
3169                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3170                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3171                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3172                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3173                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3174                         adev->psp.spl.start_addr = ucode_array_start_addr +
3175                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3176                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3177                         adev->psp.rl.start_addr = ucode_array_start_addr +
3178                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3179                 }
3180                 break;
3181         case 2:
3182                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3183
3184                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3185                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3186                         err = -EINVAL;
3187                         goto out;
3188                 }
3189
3190                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3191                         err = parse_sos_bin_descriptor(psp,
3192                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3193                                                        sos_hdr_v2_0);
3194                         if (err)
3195                                 goto out;
3196                 }
3197                 break;
3198         default:
3199                 dev_err(adev->dev,
3200                         "unsupported psp sos firmware\n");
3201                 err = -EINVAL;
3202                 goto out;
3203         }
3204
3205         return 0;
3206 out:
3207         dev_err(adev->dev,
3208                 "failed to init sos firmware\n");
3209         release_firmware(adev->psp.sos_fw);
3210         adev->psp.sos_fw = NULL;
3211
3212         return err;
3213 }
3214
3215 static int parse_ta_bin_descriptor(struct psp_context *psp,
3216                                    const struct psp_fw_bin_desc *desc,
3217                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3218 {
3219         uint8_t *ucode_start_addr  = NULL;
3220
3221         if (!psp || !desc || !ta_hdr)
3222                 return -EINVAL;
3223
3224         ucode_start_addr  = (uint8_t *)ta_hdr +
3225                             le32_to_cpu(desc->offset_bytes) +
3226                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3227
3228         switch (desc->fw_type) {
3229         case TA_FW_TYPE_PSP_ASD:
3230                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3231                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3232                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3233                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3234                 break;
3235         case TA_FW_TYPE_PSP_XGMI:
3236                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3237                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3238                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3239                 break;
3240         case TA_FW_TYPE_PSP_RAS:
3241                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3242                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3243                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3244                 break;
3245         case TA_FW_TYPE_PSP_HDCP:
3246                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3247                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3248                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3249                 break;
3250         case TA_FW_TYPE_PSP_DTM:
3251                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3252                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3253                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3254                 break;
3255         case TA_FW_TYPE_PSP_RAP:
3256                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3257                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3258                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3259                 break;
3260         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3261                 psp->securedisplay_context.context.bin_desc.fw_version =
3262                         le32_to_cpu(desc->fw_version);
3263                 psp->securedisplay_context.context.bin_desc.size_bytes =
3264                         le32_to_cpu(desc->size_bytes);
3265                 psp->securedisplay_context.context.bin_desc.start_addr =
3266                         ucode_start_addr;
3267                 break;
3268         default:
3269                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3270                 break;
3271         }
3272
3273         return 0;
3274 }
3275
3276 int psp_init_ta_microcode(struct psp_context *psp,
3277                           const char *chip_name)
3278 {
3279         struct amdgpu_device *adev = psp->adev;
3280         char fw_name[PSP_FW_NAME_LEN];
3281         const struct ta_firmware_header_v2_0 *ta_hdr;
3282         int err = 0;
3283         int ta_index = 0;
3284
3285         if (!chip_name) {
3286                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
3287                 return -EINVAL;
3288         }
3289
3290         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3291         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
3292         if (err)
3293                 goto out;
3294
3295         err = amdgpu_ucode_validate(adev->psp.ta_fw);
3296         if (err)
3297                 goto out;
3298
3299         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3300
3301         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
3302                 dev_err(adev->dev, "unsupported TA header version\n");
3303                 err = -EINVAL;
3304                 goto out;
3305         }
3306
3307         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3308                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3309                 err = -EINVAL;
3310                 goto out;
3311         }
3312
3313         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3314                 err = parse_ta_bin_descriptor(psp,
3315                                               &ta_hdr->ta_fw_bin[ta_index],
3316                                               ta_hdr);
3317                 if (err)
3318                         goto out;
3319         }
3320
3321         return 0;
3322 out:
3323         dev_err(adev->dev, "fail to initialize ta microcode\n");
3324         release_firmware(adev->psp.ta_fw);
3325         adev->psp.ta_fw = NULL;
3326         return err;
3327 }
3328
3329 int psp_init_cap_microcode(struct psp_context *psp,
3330                           const char *chip_name)
3331 {
3332         struct amdgpu_device *adev = psp->adev;
3333         char fw_name[PSP_FW_NAME_LEN];
3334         const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
3335         struct amdgpu_firmware_info *info = NULL;
3336         int err = 0;
3337
3338         if (!chip_name) {
3339                 dev_err(adev->dev, "invalid chip name for cap microcode\n");
3340                 return -EINVAL;
3341         }
3342
3343         if (!amdgpu_sriov_vf(adev)) {
3344                 dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
3345                 return -EINVAL;
3346         }
3347
3348         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_cap.bin", chip_name);
3349         err = request_firmware(&adev->psp.cap_fw, fw_name, adev->dev);
3350         if (err) {
3351                 dev_warn(adev->dev, "cap microcode does not exist, skip\n");
3352                 err = 0;
3353                 goto out;
3354         }
3355
3356         err = amdgpu_ucode_validate(adev->psp.cap_fw);
3357         if (err) {
3358                 dev_err(adev->dev, "fail to initialize cap microcode\n");
3359                 goto out;
3360         }
3361
3362         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
3363         info->ucode_id = AMDGPU_UCODE_ID_CAP;
3364         info->fw = adev->psp.cap_fw;
3365         cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
3366                 adev->psp.cap_fw->data;
3367         adev->firmware.fw_size += ALIGN(
3368                         le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
3369         adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
3370         adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
3371         adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
3372
3373         return 0;
3374
3375 out:
3376         release_firmware(adev->psp.cap_fw);
3377         adev->psp.cap_fw = NULL;
3378         return err;
3379 }
3380
3381 static int psp_set_clockgating_state(void *handle,
3382                                      enum amd_clockgating_state state)
3383 {
3384         return 0;
3385 }
3386
3387 static int psp_set_powergating_state(void *handle,
3388                                      enum amd_powergating_state state)
3389 {
3390         return 0;
3391 }
3392
3393 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3394                                          struct device_attribute *attr,
3395                                          char *buf)
3396 {
3397         struct drm_device *ddev = dev_get_drvdata(dev);
3398         struct amdgpu_device *adev = drm_to_adev(ddev);
3399         uint32_t fw_ver;
3400         int ret;
3401
3402         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3403                 DRM_INFO("PSP block is not ready yet.");
3404                 return -EBUSY;
3405         }
3406
3407         mutex_lock(&adev->psp.mutex);
3408         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3409         mutex_unlock(&adev->psp.mutex);
3410
3411         if (ret) {
3412                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3413                 return ret;
3414         }
3415
3416         return sysfs_emit(buf, "%x\n", fw_ver);
3417 }
3418
3419 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3420                                                        struct device_attribute *attr,
3421                                                        const char *buf,
3422                                                        size_t count)
3423 {
3424         struct drm_device *ddev = dev_get_drvdata(dev);
3425         struct amdgpu_device *adev = drm_to_adev(ddev);
3426         int ret, idx;
3427         char fw_name[100];
3428         const struct firmware *usbc_pd_fw;
3429         struct amdgpu_bo *fw_buf_bo = NULL;
3430         uint64_t fw_pri_mc_addr;
3431         void *fw_pri_cpu_addr;
3432
3433         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3434                 DRM_INFO("PSP block is not ready yet.");
3435                 return -EBUSY;
3436         }
3437
3438         if (!drm_dev_enter(ddev, &idx))
3439                 return -ENODEV;
3440
3441         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3442         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3443         if (ret)
3444                 goto fail;
3445
3446         /* LFB address which is aligned to 1MB boundary per PSP request */
3447         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3448                                                 AMDGPU_GEM_DOMAIN_VRAM,
3449                                                 &fw_buf_bo,
3450                                                 &fw_pri_mc_addr,
3451                                                 &fw_pri_cpu_addr);
3452         if (ret)
3453                 goto rel_buf;
3454
3455         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3456
3457         mutex_lock(&adev->psp.mutex);
3458         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3459         mutex_unlock(&adev->psp.mutex);
3460
3461         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3462
3463 rel_buf:
3464         release_firmware(usbc_pd_fw);
3465 fail:
3466         if (ret) {
3467                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3468                 count = ret;
3469         }
3470
3471         drm_dev_exit(idx);
3472         return count;
3473 }
3474
3475 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3476 {
3477         int idx;
3478
3479         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3480                 return;
3481
3482         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3483         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3484
3485         drm_dev_exit(idx);
3486 }
3487
3488 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
3489                    psp_usbc_pd_fw_sysfs_read,
3490                    psp_usbc_pd_fw_sysfs_write);
3491
3492 int is_psp_fw_valid(struct psp_bin_desc bin)
3493 {
3494         return bin.size_bytes;
3495 }
3496
3497 static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj,
3498                                         struct bin_attribute *bin_attr,
3499                                         char *buffer, loff_t pos, size_t count)
3500 {
3501         struct device *dev = kobj_to_dev(kobj);
3502         struct drm_device *ddev = dev_get_drvdata(dev);
3503         struct amdgpu_device *adev = drm_to_adev(ddev);
3504
3505         adev->psp.vbflash_done = false;
3506
3507         /* Safeguard against memory drain */
3508         if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) {
3509                 dev_err(adev->dev, "File size cannot exceed %u", AMD_VBIOS_FILE_MAX_SIZE_B);
3510                 kvfree(adev->psp.vbflash_tmp_buf);
3511                 adev->psp.vbflash_tmp_buf = NULL;
3512                 adev->psp.vbflash_image_size = 0;
3513                 return -ENOMEM;
3514         }
3515
3516         /* TODO Just allocate max for now and optimize to realloc later if needed */
3517         if (!adev->psp.vbflash_tmp_buf) {
3518                 adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL);
3519                 if (!adev->psp.vbflash_tmp_buf)
3520                         return -ENOMEM;
3521         }
3522
3523         mutex_lock(&adev->psp.mutex);
3524         memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count);
3525         adev->psp.vbflash_image_size += count;
3526         mutex_unlock(&adev->psp.mutex);
3527
3528         dev_info(adev->dev, "VBIOS flash write PSP done");
3529
3530         return count;
3531 }
3532
3533 static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj,
3534                                        struct bin_attribute *bin_attr, char *buffer,
3535                                        loff_t pos, size_t count)
3536 {
3537         struct device *dev = kobj_to_dev(kobj);
3538         struct drm_device *ddev = dev_get_drvdata(dev);
3539         struct amdgpu_device *adev = drm_to_adev(ddev);
3540         struct amdgpu_bo *fw_buf_bo = NULL;
3541         uint64_t fw_pri_mc_addr;
3542         void *fw_pri_cpu_addr;
3543         int ret;
3544
3545         dev_info(adev->dev, "VBIOS flash to PSP started");
3546
3547         ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size,
3548                                         AMDGPU_GPU_PAGE_SIZE,
3549                                         AMDGPU_GEM_DOMAIN_VRAM,
3550                                         &fw_buf_bo,
3551                                         &fw_pri_mc_addr,
3552                                         &fw_pri_cpu_addr);
3553         if (ret)
3554                 goto rel_buf;
3555
3556         memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf, adev->psp.vbflash_image_size);
3557
3558         mutex_lock(&adev->psp.mutex);
3559         ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
3560         mutex_unlock(&adev->psp.mutex);
3561
3562         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3563
3564 rel_buf:
3565         kvfree(adev->psp.vbflash_tmp_buf);
3566         adev->psp.vbflash_tmp_buf = NULL;
3567         adev->psp.vbflash_image_size = 0;
3568
3569         if (ret) {
3570                 dev_err(adev->dev, "Failed to load VBIOS FW, err = %d", ret);
3571                 return ret;
3572         }
3573
3574         dev_info(adev->dev, "VBIOS flash to PSP done");
3575         return 0;
3576 }
3577
3578 static ssize_t amdgpu_psp_vbflash_status(struct device *dev,
3579                                          struct device_attribute *attr,
3580                                          char *buf)
3581 {
3582         struct drm_device *ddev = dev_get_drvdata(dev);
3583         struct amdgpu_device *adev = drm_to_adev(ddev);
3584         uint32_t vbflash_status;
3585
3586         vbflash_status = psp_vbflash_status(&adev->psp);
3587         if (!adev->psp.vbflash_done)
3588                 vbflash_status = 0;
3589         else if (adev->psp.vbflash_done && !(vbflash_status & 0x80000000))
3590                 vbflash_status = 1;
3591
3592         return sysfs_emit(buf, "0x%x\n", vbflash_status);
3593 }
3594
3595 static const struct bin_attribute psp_vbflash_bin_attr = {
3596         .attr = {.name = "psp_vbflash", .mode = 0664},
3597         .size = 0,
3598         .write = amdgpu_psp_vbflash_write,
3599         .read = amdgpu_psp_vbflash_read,
3600 };
3601
3602 static DEVICE_ATTR(psp_vbflash_status, 0444, amdgpu_psp_vbflash_status, NULL);
3603
3604 int amdgpu_psp_sysfs_init(struct amdgpu_device *adev)
3605 {
3606         int ret = 0;
3607         struct psp_context *psp = &adev->psp;
3608
3609         if (amdgpu_sriov_vf(adev))
3610                 return -EINVAL;
3611
3612         switch (adev->ip_versions[MP0_HWIP][0]) {
3613         case IP_VERSION(13, 0, 0):
3614         case IP_VERSION(13, 0, 7):
3615                 if (!psp->adev) {
3616                         psp->adev = adev;
3617                         psp_v13_0_set_psp_funcs(psp);
3618                 }
3619                 ret = sysfs_create_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3620                 if (ret)
3621                         dev_err(adev->dev, "Failed to create device file psp_vbflash");
3622                 ret = device_create_file(adev->dev, &dev_attr_psp_vbflash_status);
3623                 if (ret)
3624                         dev_err(adev->dev, "Failed to create device file psp_vbflash_status");
3625                 return ret;
3626         default:
3627                 return 0;
3628         }
3629 }
3630
3631 const struct amd_ip_funcs psp_ip_funcs = {
3632         .name = "psp",
3633         .early_init = psp_early_init,
3634         .late_init = NULL,
3635         .sw_init = psp_sw_init,
3636         .sw_fini = psp_sw_fini,
3637         .hw_init = psp_hw_init,
3638         .hw_fini = psp_hw_fini,
3639         .suspend = psp_suspend,
3640         .resume = psp_resume,
3641         .is_idle = NULL,
3642         .check_soft_reset = NULL,
3643         .wait_for_idle = NULL,
3644         .soft_reset = NULL,
3645         .set_clockgating_state = psp_set_clockgating_state,
3646         .set_powergating_state = psp_set_powergating_state,
3647 };
3648
3649 static int psp_sysfs_init(struct amdgpu_device *adev)
3650 {
3651         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3652
3653         if (ret)
3654                 DRM_ERROR("Failed to create USBC PD FW control file!");
3655
3656         return ret;
3657 }
3658
3659 void amdgpu_psp_sysfs_fini(struct amdgpu_device *adev)
3660 {
3661         sysfs_remove_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3662         device_remove_file(adev->dev, &dev_attr_psp_vbflash_status);
3663 }
3664
3665 static void psp_sysfs_fini(struct amdgpu_device *adev)
3666 {
3667         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3668 }
3669
3670 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
3671 {
3672         .type = AMD_IP_BLOCK_TYPE_PSP,
3673         .major = 3,
3674         .minor = 1,
3675         .rev = 0,
3676         .funcs = &psp_ip_funcs,
3677 };
3678
3679 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
3680 {
3681         .type = AMD_IP_BLOCK_TYPE_PSP,
3682         .major = 10,
3683         .minor = 0,
3684         .rev = 0,
3685         .funcs = &psp_ip_funcs,
3686 };
3687
3688 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
3689 {
3690         .type = AMD_IP_BLOCK_TYPE_PSP,
3691         .major = 11,
3692         .minor = 0,
3693         .rev = 0,
3694         .funcs = &psp_ip_funcs,
3695 };
3696
3697 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3698         .type = AMD_IP_BLOCK_TYPE_PSP,
3699         .major = 11,
3700         .minor = 0,
3701         .rev = 8,
3702         .funcs = &psp_ip_funcs,
3703 };
3704
3705 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
3706 {
3707         .type = AMD_IP_BLOCK_TYPE_PSP,
3708         .major = 12,
3709         .minor = 0,
3710         .rev = 0,
3711         .funcs = &psp_ip_funcs,
3712 };
3713
3714 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3715         .type = AMD_IP_BLOCK_TYPE_PSP,
3716         .major = 13,
3717         .minor = 0,
3718         .rev = 0,
3719         .funcs = &psp_ip_funcs,
3720 };
3721
3722 const struct amdgpu_ip_block_version psp_v13_0_4_ip_block = {
3723         .type = AMD_IP_BLOCK_TYPE_PSP,
3724         .major = 13,
3725         .minor = 0,
3726         .rev = 4,
3727         .funcs = &psp_ip_funcs,
3728 };