drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40 #include "psp_v13_0_4.h"
  41
  42 #include "amdgpu_ras.h"
  43 #include "amdgpu_securedisplay.h"
  44 #include "amdgpu_atomfirmware.h"
  45
  46 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*3)
  47
  48 static int psp_sysfs_init(struct amdgpu_device *adev);
  49 static void psp_sysfs_fini(struct amdgpu_device *adev);
  50
  51 static int psp_load_smu_fw(struct psp_context *psp);
  52 static int psp_rap_terminate(struct psp_context *psp);
  53 static int psp_securedisplay_terminate(struct psp_context *psp);
  54
  55 /*
  56  * Due to DF Cstate management centralized to PMFW, the firmware
  57  * loading sequence will be updated as below:
  58  *   - Load KDB
  59  *   - Load SYS_DRV
  60  *   - Load tOS
  61  *   - Load PMFW
  62  *   - Setup TMR
  63  *   - Load other non-psp fw
  64  *   - Load ASD
  65  *   - Load XGMI/RAS/HDCP/DTM TA if any
  66  *
  67  * This new sequence is required for
  68  *   - Arcturus and onwards
  69  */
  70 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  71 {
  72         struct amdgpu_device *adev = psp->adev;
  73
  74         if (amdgpu_sriov_vf(adev)) {
  75                 psp->pmfw_centralized_cstate_management = false;
  76                 return;
  77         }
  78
  79         switch (adev->ip_versions[MP0_HWIP][0]) {
  80         case IP_VERSION(11, 0, 0):
  81         case IP_VERSION(11, 0, 4):
  82         case IP_VERSION(11, 0, 5):
  83         case IP_VERSION(11, 0, 7):
  84         case IP_VERSION(11, 0, 9):
  85         case IP_VERSION(11, 0, 11):
  86         case IP_VERSION(11, 0, 12):
  87         case IP_VERSION(11, 0, 13):
  88         case IP_VERSION(13, 0, 0):
  89         case IP_VERSION(13, 0, 2):
  90         case IP_VERSION(13, 0, 7):
  91                 psp->pmfw_centralized_cstate_management = true;
  92                 break;
  93         default:
  94                 psp->pmfw_centralized_cstate_management = false;
  95                 break;
  96         }
  97 }
  98
  99 static int psp_early_init(void *handle)
 100 {
 101         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 102         struct psp_context *psp = &adev->psp;
 103
 104         switch (adev->ip_versions[MP0_HWIP][0]) {
 105         case IP_VERSION(9, 0, 0):
 106                 psp_v3_1_set_psp_funcs(psp);
 107                 psp->autoload_supported = false;
 108                 break;
 109         case IP_VERSION(10, 0, 0):
 110         case IP_VERSION(10, 0, 1):
 111                 psp_v10_0_set_psp_funcs(psp);
 112                 psp->autoload_supported = false;
 113                 break;
 114         case IP_VERSION(11, 0, 2):
 115         case IP_VERSION(11, 0, 4):
 116                 psp_v11_0_set_psp_funcs(psp);
 117                 psp->autoload_supported = false;
 118                 break;
 119         case IP_VERSION(11, 0, 0):
 120         case IP_VERSION(11, 0, 5):
 121         case IP_VERSION(11, 0, 9):
 122         case IP_VERSION(11, 0, 7):
 123         case IP_VERSION(11, 0, 11):
 124         case IP_VERSION(11, 5, 0):
 125         case IP_VERSION(11, 0, 12):
 126         case IP_VERSION(11, 0, 13):
 127                 psp_v11_0_set_psp_funcs(psp);
 128                 psp->autoload_supported = true;
 129                 break;
 130         case IP_VERSION(11, 0, 3):
 131         case IP_VERSION(12, 0, 1):
 132                 psp_v12_0_set_psp_funcs(psp);
 133                 break;
 134         case IP_VERSION(13, 0, 2):
 135                 psp_v13_0_set_psp_funcs(psp);
 136                 break;
 137         case IP_VERSION(13, 0, 1):
 138         case IP_VERSION(13, 0, 3):
 139         case IP_VERSION(13, 0, 5):
 140         case IP_VERSION(13, 0, 8):
 141         case IP_VERSION(13, 0, 10):
 142                 psp_v13_0_set_psp_funcs(psp);
 143                 psp->autoload_supported = true;
 144                 break;
 145         case IP_VERSION(11, 0, 8):
 146                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 147                         psp_v11_0_8_set_psp_funcs(psp);
 148                         psp->autoload_supported = false;
 149                 }
 150                 break;
 151         case IP_VERSION(13, 0, 0):
 152         case IP_VERSION(13, 0, 7):
 153                 psp_v13_0_set_psp_funcs(psp);
 154                 psp->autoload_supported = true;
 155                 break;
 156         case IP_VERSION(13, 0, 4):
 157                 psp_v13_0_4_set_psp_funcs(psp);
 158                 psp->autoload_supported = true;
 159                 break;
 160         default:
 161                 return -EINVAL;
 162         }
 163
 164         psp->adev = adev;
 165
 166         psp_check_pmfw_centralized_cstate_management(psp);
 167
 168         return 0;
 169 }
 170
 171 void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 172 {
 173         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 174                               &mem_ctx->shared_buf);
 175 }
 176
 177 static void psp_free_shared_bufs(struct psp_context *psp)
 178 {
 179         void *tmr_buf;
 180         void **pptr;
 181
 182         /* free TMR memory buffer */
 183         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 184         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 185
 186         /* free xgmi shared memory */
 187         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
 188
 189         /* free ras shared memory */
 190         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
 191
 192         /* free hdcp shared memory */
 193         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
 194
 195         /* free dtm shared memory */
 196         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
 197
 198         /* free rap shared memory */
 199         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
 200
 201         /* free securedisplay shared memory */
 202         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
 203
 204
 205 }
 206
 207 static void psp_memory_training_fini(struct psp_context *psp)
 208 {
 209         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 210
 211         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 212         kfree(ctx->sys_cache);
 213         ctx->sys_cache = NULL;
 214 }
 215
 216 static int psp_memory_training_init(struct psp_context *psp)
 217 {
 218         int ret;
 219         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 220
 221         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 222                 DRM_DEBUG("memory training is not supported!\n");
 223                 return 0;
 224         }
 225
 226         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 227         if (ctx->sys_cache == NULL) {
 228                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 229                 ret = -ENOMEM;
 230                 goto Err_out;
 231         }
 232
 233         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 234                   ctx->train_data_size,
 235                   ctx->p2c_train_data_offset,
 236                   ctx->c2p_train_data_offset);
 237         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 238         return 0;
 239
 240 Err_out:
 241         psp_memory_training_fini(psp);
 242         return ret;
 243 }
 244
 245 /*
 246  * Helper funciton to query psp runtime database entry
 247  *
 248  * @adev: amdgpu_device pointer
 249  * @entry_type: the type of psp runtime database entry
 250  * @db_entry: runtime database entry pointer
 251  *
 252  * Return false if runtime database doesn't exit or entry is invalid
 253  * or true if the specific database entry is found, and copy to @db_entry
 254  */
 255 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 256                                      enum psp_runtime_entry_type entry_type,
 257                                      void *db_entry)
 258 {
 259         uint64_t db_header_pos, db_dir_pos;
 260         struct psp_runtime_data_header db_header = {0};
 261         struct psp_runtime_data_directory db_dir = {0};
 262         bool ret = false;
 263         int i;
 264
 265         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 266         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 267
 268         /* read runtime db header from vram */
 269         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 270                         sizeof(struct psp_runtime_data_header), false);
 271
 272         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 273                 /* runtime db doesn't exist, exit */
 274                 dev_warn(adev->dev, "PSP runtime database doesn't exist\n");
 275                 return false;
 276         }
 277
 278         /* read runtime database entry from vram */
 279         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 280                         sizeof(struct psp_runtime_data_directory), false);
 281
 282         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 283                 /* invalid db entry count, exit */
 284                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 285                 return false;
 286         }
 287
 288         /* look up for requested entry type */
 289         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 290                 if (db_dir.entry_list[i].entry_type == entry_type) {
 291                         switch (entry_type) {
 292                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 293                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 294                                         /* invalid db entry size */
 295                                         dev_warn(adev->dev, "Invalid PSP runtime database boot cfg entry size\n");
 296                                         return false;
 297                                 }
 298                                 /* read runtime database entry */
 299                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 300                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 301                                 ret = true;
 302                                 break;
 303                         case PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS:
 304                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_scpm_entry)) {
 305                                         /* invalid db entry size */
 306                                         dev_warn(adev->dev, "Invalid PSP runtime database scpm entry size\n");
 307                                         return false;
 308                                 }
 309                                 /* read runtime database entry */
 310                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 311                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_scpm_entry), false);
 312                                 ret = true;
 313                                 break;
 314                         default:
 315                                 ret = false;
 316                                 break;
 317                         }
 318                 }
 319         }
 320
 321         return ret;
 322 }
 323
 324 static int psp_init_sriov_microcode(struct psp_context *psp)
 325 {
 326         struct amdgpu_device *adev = psp->adev;
 327         int ret = 0;
 328
 329         switch (adev->ip_versions[MP0_HWIP][0]) {
 330         case IP_VERSION(9, 0, 0):
 331                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 332                 ret = psp_init_cap_microcode(psp, "vega10");
 333                 break;
 334         case IP_VERSION(11, 0, 9):
 335                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 336                 ret = psp_init_cap_microcode(psp, "navi12");
 337                 break;
 338         case IP_VERSION(11, 0, 7):
 339                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 340                 ret = psp_init_cap_microcode(psp, "sienna_cichlid");
 341                 break;
 342         case IP_VERSION(13, 0, 2):
 343                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 344                 ret = psp_init_cap_microcode(psp, "aldebaran");
 345                 ret &= psp_init_ta_microcode(psp, "aldebaran");
 346                 break;
 347         case IP_VERSION(13, 0, 0):
 348                 adev->virt.autoload_ucode_id = 0;
 349                 break;
 350         case IP_VERSION(13, 0, 10):
 351                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
 352                 break;
 353         default:
 354                 BUG();
 355                 break;
 356         }
 357         return ret;
 358 }
 359
 360 static int psp_sw_init(void *handle)
 361 {
 362         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 363         struct psp_context *psp = &adev->psp;
 364         int ret;
 365         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 366         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 367         struct psp_runtime_scpm_entry scpm_entry;
 368
 369         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 370         if (!psp->cmd) {
 371                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 372                 ret = -ENOMEM;
 373         }
 374
 375         if (amdgpu_sriov_vf(adev))
 376                 ret = psp_init_sriov_microcode(psp);
 377         else
 378                 ret = psp_init_microcode(psp);
 379         if (ret) {
 380                 DRM_ERROR("Failed to load psp firmware!\n");
 381                 return ret;
 382         }
 383
 384         adev->psp.xgmi_context.supports_extended_data =
 385                 !adev->gmc.xgmi.connected_to_cpu &&
 386                         adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2);
 387
 388         memset(&scpm_entry, 0, sizeof(scpm_entry));
 389         if ((psp_get_runtime_db_entry(adev,
 390                                 PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS,
 391                                 &scpm_entry)) &&
 392             (SCPM_DISABLE != scpm_entry.scpm_status)) {
 393                 adev->scpm_enabled = true;
 394                 adev->scpm_status = scpm_entry.scpm_status;
 395         } else {
 396                 adev->scpm_enabled = false;
 397                 adev->scpm_status = SCPM_DISABLE;
 398         }
 399
 400         /* TODO: stop gpu driver services and print alarm if scpm is enabled with error status */
 401
 402         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 403         if (psp_get_runtime_db_entry(adev,
 404                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 405                                 &boot_cfg_entry)) {
 406                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 407                 if ((psp->boot_cfg_bitmask) &
 408                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 409                         /* If psp runtime database exists, then
 410                          * only enable two stage memory training
 411                          * when TWO_STAGE_DRAM_TRAINING bit is set
 412                          * in runtime database */
 413                         mem_training_ctx->enable_mem_training = true;
 414                 }
 415
 416         } else {
 417                 /* If psp runtime database doesn't exist or
 418                  * is invalid, force enable two stage memory
 419                  * training */
 420                 mem_training_ctx->enable_mem_training = true;
 421         }
 422
 423         if (mem_training_ctx->enable_mem_training) {
 424                 ret = psp_memory_training_init(psp);
 425                 if (ret) {
 426                         DRM_ERROR("Failed to initialize memory training!\n");
 427                         return ret;
 428                 }
 429
 430                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 431                 if (ret) {
 432                         DRM_ERROR("Failed to process memory training!\n");
 433                         return ret;
 434                 }
 435         }
 436
 437         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 438             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 439                 ret= psp_sysfs_init(adev);
 440                 if (ret) {
 441                         return ret;
 442                 }
 443         }
 444
 445         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 446                                       amdgpu_sriov_vf(adev) ?
 447                                       AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 448                                       &psp->fw_pri_bo,
 449                                       &psp->fw_pri_mc_addr,
 450                                       &psp->fw_pri_buf);
 451         if (ret)
 452                 return ret;
 453
 454         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 455                                       AMDGPU_GEM_DOMAIN_VRAM,
 456                                       &psp->fence_buf_bo,
 457                                       &psp->fence_buf_mc_addr,
 458                                       &psp->fence_buf);
 459         if (ret)
 460                 goto failed1;
 461
 462         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 463                                       AMDGPU_GEM_DOMAIN_VRAM,
 464                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 465                                       (void **)&psp->cmd_buf_mem);
 466         if (ret)
 467                 goto failed2;
 468
 469         return 0;
 470
 471 failed2:
 472         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 473                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 474 failed1:
 475         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 476                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 477         return ret;
 478 }
 479
 480 static int psp_sw_fini(void *handle)
 481 {
 482         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 483         struct psp_context *psp = &adev->psp;
 484         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 485
 486         psp_memory_training_fini(psp);
 487         if (psp->sos_fw) {
 488                 release_firmware(psp->sos_fw);
 489                 psp->sos_fw = NULL;
 490         }
 491         if (psp->asd_fw) {
 492                 release_firmware(psp->asd_fw);
 493                 psp->asd_fw = NULL;
 494         }
 495         if (psp->ta_fw) {
 496                 release_firmware(psp->ta_fw);
 497                 psp->ta_fw = NULL;
 498         }
 499         if (psp->cap_fw) {
 500                 release_firmware(psp->cap_fw);
 501                 psp->cap_fw = NULL;
 502         }
 503         if (psp->toc_fw) {
 504                 release_firmware(psp->toc_fw);
 505                 psp->toc_fw = NULL;
 506         }
 507         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 508             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 509                 psp_sysfs_fini(adev);
 510
 511         kfree(cmd);
 512         cmd = NULL;
 513
 514         if (psp->km_ring.ring_mem)
 515                 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
 516                                       &psp->km_ring.ring_mem_mc_addr,
 517                                       (void **)&psp->km_ring.ring_mem);
 518
 519         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 520                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 521         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 522                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 523         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 524                               (void **)&psp->cmd_buf_mem);
 525
 526         return 0;
 527 }
 528
 529 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 530                  uint32_t reg_val, uint32_t mask, bool check_changed)
 531 {
 532         uint32_t val;
 533         int i;
 534         struct amdgpu_device *adev = psp->adev;
 535
 536         if (psp->adev->no_hw_access)
 537                 return 0;
 538
 539         for (i = 0; i < adev->usec_timeout; i++) {
 540                 val = RREG32(reg_index);
 541                 if (check_changed) {
 542                         if (val != reg_val)
 543                                 return 0;
 544                 } else {
 545                         if ((val & mask) == reg_val)
 546                                 return 0;
 547                 }
 548                 udelay(1);
 549         }
 550
 551         return -ETIME;
 552 }
 553
 554 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 555 {
 556         switch (cmd_id) {
 557         case GFX_CMD_ID_LOAD_TA:
 558                 return "LOAD_TA";
 559         case GFX_CMD_ID_UNLOAD_TA:
 560                 return "UNLOAD_TA";
 561         case GFX_CMD_ID_INVOKE_CMD:
 562                 return "INVOKE_CMD";
 563         case GFX_CMD_ID_LOAD_ASD:
 564                 return "LOAD_ASD";
 565         case GFX_CMD_ID_SETUP_TMR:
 566                 return "SETUP_TMR";
 567         case GFX_CMD_ID_LOAD_IP_FW:
 568                 return "LOAD_IP_FW";
 569         case GFX_CMD_ID_DESTROY_TMR:
 570                 return "DESTROY_TMR";
 571         case GFX_CMD_ID_SAVE_RESTORE:
 572                 return "SAVE_RESTORE_IP_FW";
 573         case GFX_CMD_ID_SETUP_VMR:
 574                 return "SETUP_VMR";
 575         case GFX_CMD_ID_DESTROY_VMR:
 576                 return "DESTROY_VMR";
 577         case GFX_CMD_ID_PROG_REG:
 578                 return "PROG_REG";
 579         case GFX_CMD_ID_GET_FW_ATTESTATION:
 580                 return "GET_FW_ATTESTATION";
 581         case GFX_CMD_ID_LOAD_TOC:
 582                 return "ID_LOAD_TOC";
 583         case GFX_CMD_ID_AUTOLOAD_RLC:
 584                 return "AUTOLOAD_RLC";
 585         case GFX_CMD_ID_BOOT_CFG:
 586                 return "BOOT_CFG";
 587         default:
 588                 return "UNKNOWN CMD";
 589         }
 590 }
 591
 592 static int
 593 psp_cmd_submit_buf(struct psp_context *psp,
 594                    struct amdgpu_firmware_info *ucode,
 595                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 596 {
 597         int ret;
 598         int index, idx;
 599         int timeout = 20000;
 600         bool ras_intr = false;
 601         bool skip_unsupport = false;
 602
 603         if (psp->adev->no_hw_access)
 604                 return 0;
 605
 606         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
 607                 return 0;
 608
 609         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 610
 611         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 612
 613         index = atomic_inc_return(&psp->fence_value);
 614         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 615         if (ret) {
 616                 atomic_dec(&psp->fence_value);
 617                 goto exit;
 618         }
 619
 620         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 621         while (*((unsigned int *)psp->fence_buf) != index) {
 622                 if (--timeout == 0)
 623                         break;
 624                 /*
 625                  * Shouldn't wait for timeout when err_event_athub occurs,
 626                  * because gpu reset thread triggered and lock resource should
 627                  * be released for psp resume sequence.
 628                  */
 629                 ras_intr = amdgpu_ras_intr_triggered();
 630                 if (ras_intr)
 631                         break;
 632                 usleep_range(10, 100);
 633                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 634         }
 635
 636         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 637         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 638                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 639
 640         memcpy((void*)&cmd->resp, (void*)&psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 641
 642         /* In some cases, psp response status is not 0 even there is no
 643          * problem while the command is submitted. Some version of PSP FW
 644          * doesn't write 0 to that field.
 645          * So here we would like to only print a warning instead of an error
 646          * during psp initialization to avoid breaking hw_init and it doesn't
 647          * return -EINVAL.
 648          */
 649         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 650                 if (ucode)
 651                         DRM_WARN("failed to load ucode %s(0x%X) ",
 652                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 653                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 654                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 655                          psp->cmd_buf_mem->resp.status);
 656                 /* If any firmware (including CAP) load fails under SRIOV, it should
 657                  * return failure to stop the VF from initializing.
 658                  * Also return failure in case of timeout
 659                  */
 660                 if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
 661                         ret = -EINVAL;
 662                         goto exit;
 663                 }
 664         }
 665
 666         if (ucode) {
 667                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 668                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 669         }
 670
 671 exit:
 672         drm_dev_exit(idx);
 673         return ret;
 674 }
 675
 676 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 677 {
 678         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 679
 680         mutex_lock(&psp->mutex);
 681
 682         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 683
 684         return cmd;
 685 }
 686
 687 static void release_psp_cmd_buf(struct psp_context *psp)
 688 {
 689         mutex_unlock(&psp->mutex);
 690 }
 691
 692 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 693                                  struct psp_gfx_cmd_resp *cmd,
 694                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 695 {
 696         struct amdgpu_device *adev = psp->adev;
 697         uint32_t size = amdgpu_bo_size(tmr_bo);
 698         uint64_t tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 699
 700         if (amdgpu_sriov_vf(psp->adev))
 701                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 702         else
 703                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 704         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 705         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 706         cmd->cmd.cmd_setup_tmr.buf_size = size;
 707         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 708         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 709         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 710 }
 711
 712 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 713                                       uint64_t pri_buf_mc, uint32_t size)
 714 {
 715         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 716         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 717         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 718         cmd->cmd.cmd_load_toc.toc_size = size;
 719 }
 720
 721 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 722 static int psp_load_toc(struct psp_context *psp,
 723                         uint32_t *tmr_size)
 724 {
 725         int ret;
 726         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 727
 728         /* Copy toc to psp firmware private buffer */
 729         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 730
 731         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 732
 733         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 734                                  psp->fence_buf_mc_addr);
 735         if (!ret)
 736                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 737
 738         release_psp_cmd_buf(psp);
 739
 740         return ret;
 741 }
 742
 743 /* Set up Trusted Memory Region */
 744 static int psp_tmr_init(struct psp_context *psp)
 745 {
 746         int ret;
 747         int tmr_size;
 748         void *tmr_buf;
 749         void **pptr;
 750
 751         /*
 752          * According to HW engineer, they prefer the TMR address be "naturally
 753          * aligned" , e.g. the start address be an integer divide of TMR size.
 754          *
 755          * Note: this memory need be reserved till the driver
 756          * uninitializes.
 757          */
 758         tmr_size = PSP_TMR_SIZE(psp->adev);
 759
 760         /* For ASICs support RLC autoload, psp will parse the toc
 761          * and calculate the total size of TMR needed */
 762         if (!amdgpu_sriov_vf(psp->adev) &&
 763             psp->toc.start_addr &&
 764             psp->toc.size_bytes &&
 765             psp->fw_pri_buf) {
 766                 ret = psp_load_toc(psp, &tmr_size);
 767                 if (ret) {
 768                         DRM_ERROR("Failed to load toc\n");
 769                         return ret;
 770                 }
 771         }
 772
 773         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 774         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_ALIGNMENT,
 775                                       AMDGPU_GEM_DOMAIN_VRAM,
 776                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 777
 778         return ret;
 779 }
 780
 781 static bool psp_skip_tmr(struct psp_context *psp)
 782 {
 783         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 784         case IP_VERSION(11, 0, 9):
 785         case IP_VERSION(11, 0, 7):
 786         case IP_VERSION(13, 0, 2):
 787         case IP_VERSION(13, 0, 10):
 788                 return true;
 789         default:
 790                 return false;
 791         }
 792 }
 793
 794 static int psp_tmr_load(struct psp_context *psp)
 795 {
 796         int ret;
 797         struct psp_gfx_cmd_resp *cmd;
 798
 799         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 800          * Already set up by host driver.
 801          */
 802         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 803                 return 0;
 804
 805         cmd = acquire_psp_cmd_buf(psp);
 806
 807         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 808         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 809                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 810
 811         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 812                                  psp->fence_buf_mc_addr);
 813
 814         release_psp_cmd_buf(psp);
 815
 816         return ret;
 817 }
 818
 819 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 820                                         struct psp_gfx_cmd_resp *cmd)
 821 {
 822         if (amdgpu_sriov_vf(psp->adev))
 823                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 824         else
 825                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 826 }
 827
 828 static int psp_tmr_unload(struct psp_context *psp)
 829 {
 830         int ret;
 831         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 832
 833         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 834         dev_info(psp->adev->dev, "free PSP TMR buffer\n");
 835
 836         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 837                                  psp->fence_buf_mc_addr);
 838
 839         release_psp_cmd_buf(psp);
 840
 841         return ret;
 842 }
 843
 844 static int psp_tmr_terminate(struct psp_context *psp)
 845 {
 846         return psp_tmr_unload(psp);
 847 }
 848
 849 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 850                                         uint64_t *output_ptr)
 851 {
 852         int ret;
 853         struct psp_gfx_cmd_resp *cmd;
 854
 855         if (!output_ptr)
 856                 return -EINVAL;
 857
 858         if (amdgpu_sriov_vf(psp->adev))
 859                 return 0;
 860
 861         cmd = acquire_psp_cmd_buf(psp);
 862
 863         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 864
 865         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 866                                  psp->fence_buf_mc_addr);
 867
 868         if (!ret) {
 869                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 870                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 871         }
 872
 873         release_psp_cmd_buf(psp);
 874
 875         return ret;
 876 }
 877
 878 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 879 {
 880         struct psp_context *psp = &adev->psp;
 881         struct psp_gfx_cmd_resp *cmd;
 882         int ret;
 883
 884         if (amdgpu_sriov_vf(adev))
 885                 return 0;
 886
 887         cmd = acquire_psp_cmd_buf(psp);
 888
 889         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 890         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 891
 892         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 893         if (!ret) {
 894                 *boot_cfg =
 895                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 896         }
 897
 898         release_psp_cmd_buf(psp);
 899
 900         return ret;
 901 }
 902
 903 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 904 {
 905         int ret;
 906         struct psp_context *psp = &adev->psp;
 907         struct psp_gfx_cmd_resp *cmd;
 908
 909         if (amdgpu_sriov_vf(adev))
 910                 return 0;
 911
 912         cmd = acquire_psp_cmd_buf(psp);
 913
 914         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 915         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 916         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 917         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 918
 919         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 920
 921         release_psp_cmd_buf(psp);
 922
 923         return ret;
 924 }
 925
 926 static int psp_rl_load(struct amdgpu_device *adev)
 927 {
 928         int ret;
 929         struct psp_context *psp = &adev->psp;
 930         struct psp_gfx_cmd_resp *cmd;
 931
 932         if (!is_psp_fw_valid(psp->rl))
 933                 return 0;
 934
 935         cmd = acquire_psp_cmd_buf(psp);
 936
 937         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 938         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
 939
 940         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 941         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
 942         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
 943         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
 944         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
 945
 946         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 947
 948         release_psp_cmd_buf(psp);
 949
 950         return ret;
 951 }
 952
 953 static int psp_asd_initialize(struct psp_context *psp)
 954 {
 955         int ret;
 956
 957         /* If PSP version doesn't match ASD version, asd loading will be failed.
 958          * add workaround to bypass it for sriov now.
 959          * TODO: add version check to make it common
 960          */
 961         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 962                 return 0;
 963
 964         psp->asd_context.mem_context.shared_mc_addr  = 0;
 965         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 966         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
 967
 968         ret = psp_ta_load(psp, &psp->asd_context);
 969         if (!ret)
 970                 psp->asd_context.initialized = true;
 971
 972         return ret;
 973 }
 974
 975 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 976                                        uint32_t session_id)
 977 {
 978         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 979         cmd->cmd.cmd_unload_ta.session_id = session_id;
 980 }
 981
 982 int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
 983 {
 984         int ret;
 985         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 986
 987         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
 988
 989         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 990
 991         release_psp_cmd_buf(psp);
 992
 993         return ret;
 994 }
 995
 996 static int psp_asd_terminate(struct psp_context *psp)
 997 {
 998         int ret;
 999
1000         if (amdgpu_sriov_vf(psp->adev))
1001                 return 0;
1002
1003         if (!psp->asd_context.initialized)
1004                 return 0;
1005
1006         ret = psp_ta_unload(psp, &psp->asd_context);
1007         if (!ret)
1008                 psp->asd_context.initialized = false;
1009
1010         return ret;
1011 }
1012
1013 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1014                 uint32_t id, uint32_t value)
1015 {
1016         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
1017         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
1018         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
1019 }
1020
1021 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
1022                 uint32_t value)
1023 {
1024         struct psp_gfx_cmd_resp *cmd;
1025         int ret = 0;
1026
1027         if (reg >= PSP_REG_LAST)
1028                 return -EINVAL;
1029
1030         cmd = acquire_psp_cmd_buf(psp);
1031
1032         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
1033         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1034         if (ret)
1035                 DRM_ERROR("PSP failed to program reg id %d", reg);
1036
1037         release_psp_cmd_buf(psp);
1038
1039         return ret;
1040 }
1041
1042 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1043                                      uint64_t ta_bin_mc,
1044                                      struct ta_context *context)
1045 {
1046         cmd->cmd_id                             = context->ta_load_type;
1047         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
1048         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
1049         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
1050
1051         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
1052                 lower_32_bits(context->mem_context.shared_mc_addr);
1053         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
1054                 upper_32_bits(context->mem_context.shared_mc_addr);
1055         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
1056 }
1057
1058 int psp_ta_init_shared_buf(struct psp_context *psp,
1059                                   struct ta_mem_context *mem_ctx)
1060 {
1061         /*
1062         * Allocate 16k memory aligned to 4k from Frame Buffer (local
1063         * physical) for ta to host memory
1064         */
1065         return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
1066                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1067                                       &mem_ctx->shared_bo,
1068                                       &mem_ctx->shared_mc_addr,
1069                                       &mem_ctx->shared_buf);
1070 }
1071
1072 static void psp_prep_ta_invoke_indirect_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1073                                        uint32_t ta_cmd_id,
1074                                        struct ta_context *context)
1075 {
1076         cmd->cmd_id                         = GFX_CMD_ID_INVOKE_CMD;
1077         cmd->cmd.cmd_invoke_cmd.session_id  = context->session_id;
1078         cmd->cmd.cmd_invoke_cmd.ta_cmd_id   = ta_cmd_id;
1079
1080         cmd->cmd.cmd_invoke_cmd.buf.num_desc   = 1;
1081         cmd->cmd.cmd_invoke_cmd.buf.total_size = context->mem_context.shared_mem_size;
1082         cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_size = context->mem_context.shared_mem_size;
1083         cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_phy_addr_lo =
1084                                      lower_32_bits(context->mem_context.shared_mc_addr);
1085         cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_phy_addr_hi =
1086                                      upper_32_bits(context->mem_context.shared_mc_addr);
1087 }
1088
1089 int psp_ta_invoke_indirect(struct psp_context *psp,
1090                   uint32_t ta_cmd_id,
1091                   struct ta_context *context)
1092 {
1093         int ret;
1094         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1095
1096         psp_prep_ta_invoke_indirect_cmd_buf(cmd, ta_cmd_id, context);
1097
1098         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1099                                  psp->fence_buf_mc_addr);
1100
1101         context->resp_status = cmd->resp.status;
1102
1103         release_psp_cmd_buf(psp);
1104
1105         return ret;
1106 }
1107
1108 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1109                                        uint32_t ta_cmd_id,
1110                                        uint32_t session_id)
1111 {
1112         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
1113         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
1114         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
1115 }
1116
1117 int psp_ta_invoke(struct psp_context *psp,
1118                   uint32_t ta_cmd_id,
1119                   struct ta_context *context)
1120 {
1121         int ret;
1122         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1123
1124         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
1125
1126         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1127                                  psp->fence_buf_mc_addr);
1128
1129         context->resp_status = cmd->resp.status;
1130
1131         release_psp_cmd_buf(psp);
1132
1133         return ret;
1134 }
1135
1136 int psp_ta_load(struct psp_context *psp, struct ta_context *context)
1137 {
1138         int ret;
1139         struct psp_gfx_cmd_resp *cmd;
1140
1141         cmd = acquire_psp_cmd_buf(psp);
1142
1143         psp_copy_fw(psp, context->bin_desc.start_addr,
1144                     context->bin_desc.size_bytes);
1145
1146         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
1147
1148         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1149                                  psp->fence_buf_mc_addr);
1150
1151         context->resp_status = cmd->resp.status;
1152
1153         if (!ret) {
1154                 context->session_id = cmd->resp.session_id;
1155         }
1156
1157         release_psp_cmd_buf(psp);
1158
1159         return ret;
1160 }
1161
1162 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1163 {
1164         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1165 }
1166
1167 int psp_xgmi_terminate(struct psp_context *psp)
1168 {
1169         int ret;
1170         struct amdgpu_device *adev = psp->adev;
1171
1172         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1173         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1174             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1175              adev->gmc.xgmi.connected_to_cpu))
1176                 return 0;
1177
1178         if (!psp->xgmi_context.context.initialized)
1179                 return 0;
1180
1181         ret = psp_ta_unload(psp, &psp->xgmi_context.context);
1182
1183         psp->xgmi_context.context.initialized = false;
1184
1185         return ret;
1186 }
1187
1188 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1189 {
1190         struct ta_xgmi_shared_memory *xgmi_cmd;
1191         int ret;
1192
1193         if (!psp->ta_fw ||
1194             !psp->xgmi_context.context.bin_desc.size_bytes ||
1195             !psp->xgmi_context.context.bin_desc.start_addr)
1196                 return -ENOENT;
1197
1198         if (!load_ta)
1199                 goto invoke;
1200
1201         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1202         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1203
1204         if (!psp->xgmi_context.context.mem_context.shared_buf) {
1205                 ret = psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
1206                 if (ret)
1207                         return ret;
1208         }
1209
1210         /* Load XGMI TA */
1211         ret = psp_ta_load(psp, &psp->xgmi_context.context);
1212         if (!ret)
1213                 psp->xgmi_context.context.initialized = true;
1214         else
1215                 return ret;
1216
1217 invoke:
1218         /* Initialize XGMI session */
1219         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1220         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1221         xgmi_cmd->flag_extend_link_record = set_extended_data;
1222         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1223
1224         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1225
1226         return ret;
1227 }
1228
1229 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1230 {
1231         struct ta_xgmi_shared_memory *xgmi_cmd;
1232         int ret;
1233
1234         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1235         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1236
1237         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1238
1239         /* Invoke xgmi ta to get hive id */
1240         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1241         if (ret)
1242                 return ret;
1243
1244         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1245
1246         return 0;
1247 }
1248
1249 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1250 {
1251         struct ta_xgmi_shared_memory *xgmi_cmd;
1252         int ret;
1253
1254         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1255         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1256
1257         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1258
1259         /* Invoke xgmi ta to get the node id */
1260         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1261         if (ret)
1262                 return ret;
1263
1264         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1265
1266         return 0;
1267 }
1268
1269 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1270 {
1271         return psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1272                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b;
1273 }
1274
1275 /*
1276  * Chips that support extended topology information require the driver to
1277  * reflect topology information in the opposite direction.  This is
1278  * because the TA has already exceeded its link record limit and if the
1279  * TA holds bi-directional information, the driver would have to do
1280  * multiple fetches instead of just two.
1281  */
1282 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1283                                         struct psp_xgmi_node_info node_info)
1284 {
1285         struct amdgpu_device *mirror_adev;
1286         struct amdgpu_hive_info *hive;
1287         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1288         uint64_t dst_node_id = node_info.node_id;
1289         uint8_t dst_num_hops = node_info.num_hops;
1290         uint8_t dst_num_links = node_info.num_links;
1291
1292         hive = amdgpu_get_xgmi_hive(psp->adev);
1293         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1294                 struct psp_xgmi_topology_info *mirror_top_info;
1295                 int j;
1296
1297                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1298                         continue;
1299
1300                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1301                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1302                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1303                                 continue;
1304
1305                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1306                         /*
1307                          * prevent 0 num_links value re-reflection since reflection
1308                          * criteria is based on num_hops (direct or indirect).
1309                          *
1310                          */
1311                         if (dst_num_links)
1312                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1313
1314                         break;
1315                 }
1316
1317                 break;
1318         }
1319
1320         amdgpu_put_xgmi_hive(hive);
1321 }
1322
1323 int psp_xgmi_get_topology_info(struct psp_context *psp,
1324                                int number_devices,
1325                                struct psp_xgmi_topology_info *topology,
1326                                bool get_extended_data)
1327 {
1328         struct ta_xgmi_shared_memory *xgmi_cmd;
1329         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1330         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1331         int i;
1332         int ret;
1333
1334         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1335                 return -EINVAL;
1336
1337         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1338         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1339         xgmi_cmd->flag_extend_link_record = get_extended_data;
1340
1341         /* Fill in the shared memory with topology information as input */
1342         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1343         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1344         topology_info_input->num_nodes = number_devices;
1345
1346         for (i = 0; i < topology_info_input->num_nodes; i++) {
1347                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1348                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1349                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1350                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1351         }
1352
1353         /* Invoke xgmi ta to get the topology information */
1354         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1355         if (ret)
1356                 return ret;
1357
1358         /* Read the output topology information from the shared memory */
1359         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1360         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1361         for (i = 0; i < topology->num_nodes; i++) {
1362                 /* extended data will either be 0 or equal to non-extended data */
1363                 if (topology_info_output->nodes[i].num_hops)
1364                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1365
1366                 /* non-extended data gets everything here so no need to update */
1367                 if (!get_extended_data) {
1368                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1369                         topology->nodes[i].is_sharing_enabled =
1370                                         topology_info_output->nodes[i].is_sharing_enabled;
1371                         topology->nodes[i].sdma_engine =
1372                                         topology_info_output->nodes[i].sdma_engine;
1373                 }
1374
1375         }
1376
1377         /* Invoke xgmi ta again to get the link information */
1378         if (psp_xgmi_peer_link_info_supported(psp)) {
1379                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1380
1381                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1382
1383                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1384
1385                 if (ret)
1386                         return ret;
1387
1388                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1389                 for (i = 0; i < topology->num_nodes; i++) {
1390                         /* accumulate num_links on extended data */
1391                         topology->nodes[i].num_links = get_extended_data ?
1392                                         topology->nodes[i].num_links +
1393                                                         link_info_output->nodes[i].num_links :
1394                                         link_info_output->nodes[i].num_links;
1395
1396                         /* reflect the topology information for bi-directionality */
1397                         if (psp->xgmi_context.supports_extended_data &&
1398                                         get_extended_data && topology->nodes[i].num_hops)
1399                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1400                 }
1401         }
1402
1403         return 0;
1404 }
1405
1406 int psp_xgmi_set_topology_info(struct psp_context *psp,
1407                                int number_devices,
1408                                struct psp_xgmi_topology_info *topology)
1409 {
1410         struct ta_xgmi_shared_memory *xgmi_cmd;
1411         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1412         int i;
1413
1414         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1415                 return -EINVAL;
1416
1417         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1418         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1419
1420         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1421         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1422         topology_info_input->num_nodes = number_devices;
1423
1424         for (i = 0; i < topology_info_input->num_nodes; i++) {
1425                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1426                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1427                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1428                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1429         }
1430
1431         /* Invoke xgmi ta to set topology information */
1432         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1433 }
1434
1435 // ras begin
1436 static void psp_ras_ta_check_status(struct psp_context *psp)
1437 {
1438         struct ta_ras_shared_memory *ras_cmd =
1439                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1440
1441         switch (ras_cmd->ras_status) {
1442         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1443                 dev_warn(psp->adev->dev,
1444                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1445                 break;
1446         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1447                 dev_warn(psp->adev->dev,
1448                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1449                 break;
1450         case TA_RAS_STATUS__SUCCESS:
1451                 break;
1452         case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED:
1453                 if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR)
1454                         dev_warn(psp->adev->dev,
1455                                         "RAS WARNING: Inject error to critical region is not allowed\n");
1456                 break;
1457         default:
1458                 dev_warn(psp->adev->dev,
1459                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1460                 break;
1461         }
1462 }
1463
1464 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1465 {
1466         struct ta_ras_shared_memory *ras_cmd;
1467         int ret;
1468
1469         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1470
1471         /*
1472          * TODO: bypass the loading in sriov for now
1473          */
1474         if (amdgpu_sriov_vf(psp->adev))
1475                 return 0;
1476
1477         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1478
1479         if (amdgpu_ras_intr_triggered())
1480                 return ret;
1481
1482         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1483         {
1484                 DRM_WARN("RAS: Unsupported Interface");
1485                 return -EINVAL;
1486         }
1487
1488         if (!ret) {
1489                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1490                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1491
1492                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1493                 }
1494                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1495                         dev_warn(psp->adev->dev,
1496                                  "RAS internal register access blocked\n");
1497
1498                 psp_ras_ta_check_status(psp);
1499         }
1500
1501         return ret;
1502 }
1503
1504 int psp_ras_enable_features(struct psp_context *psp,
1505                 union ta_ras_cmd_input *info, bool enable)
1506 {
1507         struct ta_ras_shared_memory *ras_cmd;
1508         int ret;
1509
1510         if (!psp->ras_context.context.initialized)
1511                 return -EINVAL;
1512
1513         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1514         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1515
1516         if (enable)
1517                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1518         else
1519                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1520
1521         ras_cmd->ras_in_message = *info;
1522
1523         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1524         if (ret)
1525                 return -EINVAL;
1526
1527         return 0;
1528 }
1529
1530 int psp_ras_terminate(struct psp_context *psp)
1531 {
1532         int ret;
1533
1534         /*
1535          * TODO: bypass the terminate in sriov for now
1536          */
1537         if (amdgpu_sriov_vf(psp->adev))
1538                 return 0;
1539
1540         if (!psp->ras_context.context.initialized)
1541                 return 0;
1542
1543         ret = psp_ta_unload(psp, &psp->ras_context.context);
1544
1545         psp->ras_context.context.initialized = false;
1546
1547         return ret;
1548 }
1549
1550 static int psp_ras_initialize(struct psp_context *psp)
1551 {
1552         int ret;
1553         uint32_t boot_cfg = 0xFF;
1554         struct amdgpu_device *adev = psp->adev;
1555         struct ta_ras_shared_memory *ras_cmd;
1556
1557         /*
1558          * TODO: bypass the initialize in sriov for now
1559          */
1560         if (amdgpu_sriov_vf(adev))
1561                 return 0;
1562
1563         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1564             !adev->psp.ras_context.context.bin_desc.start_addr) {
1565                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1566                 return 0;
1567         }
1568
1569         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1570                 /* query GECC enablement status from boot config
1571                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1572                  */
1573                 ret = psp_boot_config_get(adev, &boot_cfg);
1574                 if (ret)
1575                         dev_warn(adev->dev, "PSP get boot config failed\n");
1576
1577                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1578                         if (!boot_cfg) {
1579                                 dev_info(adev->dev, "GECC is disabled\n");
1580                         } else {
1581                                 /* disable GECC in next boot cycle if ras is
1582                                  * disabled by module parameter amdgpu_ras_enable
1583                                  * and/or amdgpu_ras_mask, or boot_config_get call
1584                                  * is failed
1585                                  */
1586                                 ret = psp_boot_config_set(adev, 0);
1587                                 if (ret)
1588                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1589                                 else
1590                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle "
1591                                                  "if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1592                         }
1593                 } else {
1594                         if (1 == boot_cfg) {
1595                                 dev_info(adev->dev, "GECC is enabled\n");
1596                         } else {
1597                                 /* enable GECC in next boot cycle if it is disabled
1598                                  * in boot config, or force enable GECC if failed to
1599                                  * get boot configuration
1600                                  */
1601                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1602                                 if (ret)
1603                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1604                                 else
1605                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1606                         }
1607                 }
1608         }
1609
1610         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1611         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1612
1613         if (!psp->ras_context.context.initialized) {
1614                 ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1615                 if (ret)
1616                         return ret;
1617         }
1618
1619         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1620         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1621
1622         if (amdgpu_ras_is_poison_mode_supported(adev))
1623                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1624         if (!adev->gmc.xgmi.connected_to_cpu)
1625                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1626
1627         ret = psp_ta_load(psp, &psp->ras_context.context);
1628
1629         if (!ret && !ras_cmd->ras_status)
1630                 psp->ras_context.context.initialized = true;
1631         else {
1632                 if (ras_cmd->ras_status)
1633                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1634                 amdgpu_ras_fini(psp->adev);
1635         }
1636
1637         return ret;
1638 }
1639
1640 int psp_ras_trigger_error(struct psp_context *psp,
1641                           struct ta_ras_trigger_error_input *info)
1642 {
1643         struct ta_ras_shared_memory *ras_cmd;
1644         int ret;
1645
1646         if (!psp->ras_context.context.initialized)
1647                 return -EINVAL;
1648
1649         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1650         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1651
1652         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1653         ras_cmd->ras_in_message.trigger_error = *info;
1654
1655         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1656         if (ret)
1657                 return -EINVAL;
1658
1659         /* If err_event_athub occurs error inject was successful, however
1660            return status from TA is no long reliable */
1661         if (amdgpu_ras_intr_triggered())
1662                 return 0;
1663
1664         if (ras_cmd->ras_status == TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED)
1665                 return -EACCES;
1666         else if (ras_cmd->ras_status)
1667                 return -EINVAL;
1668
1669         return 0;
1670 }
1671 // ras end
1672
1673 // HDCP start
1674 static int psp_hdcp_initialize(struct psp_context *psp)
1675 {
1676         int ret;
1677
1678         /*
1679          * TODO: bypass the initialize in sriov for now
1680          */
1681         if (amdgpu_sriov_vf(psp->adev))
1682                 return 0;
1683
1684         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1685             !psp->hdcp_context.context.bin_desc.start_addr) {
1686                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1687                 return 0;
1688         }
1689
1690         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1691         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1692
1693         if (!psp->hdcp_context.context.initialized) {
1694                 ret = psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1695                 if (ret)
1696                         return ret;
1697         }
1698
1699         ret = psp_ta_load(psp, &psp->hdcp_context.context);
1700         if (!ret) {
1701                 psp->hdcp_context.context.initialized = true;
1702                 mutex_init(&psp->hdcp_context.mutex);
1703         }
1704
1705         return ret;
1706 }
1707
1708 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1709 {
1710         /*
1711          * TODO: bypass the loading in sriov for now
1712          */
1713         if (amdgpu_sriov_vf(psp->adev))
1714                 return 0;
1715
1716         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1717 }
1718
1719 static int psp_hdcp_terminate(struct psp_context *psp)
1720 {
1721         int ret;
1722
1723         /*
1724          * TODO: bypass the terminate in sriov for now
1725          */
1726         if (amdgpu_sriov_vf(psp->adev))
1727                 return 0;
1728
1729         if (!psp->hdcp_context.context.initialized)
1730                 return 0;
1731
1732         ret = psp_ta_unload(psp, &psp->hdcp_context.context);
1733
1734         psp->hdcp_context.context.initialized = false;
1735
1736         return ret;
1737 }
1738 // HDCP end
1739
1740 // DTM start
1741 static int psp_dtm_initialize(struct psp_context *psp)
1742 {
1743         int ret;
1744
1745         /*
1746          * TODO: bypass the initialize in sriov for now
1747          */
1748         if (amdgpu_sriov_vf(psp->adev))
1749                 return 0;
1750
1751         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1752             !psp->dtm_context.context.bin_desc.start_addr) {
1753                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1754                 return 0;
1755         }
1756
1757         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1758         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1759
1760         if (!psp->dtm_context.context.initialized) {
1761                 ret = psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1762                 if (ret)
1763                         return ret;
1764         }
1765
1766         ret = psp_ta_load(psp, &psp->dtm_context.context);
1767         if (!ret) {
1768                 psp->dtm_context.context.initialized = true;
1769                 mutex_init(&psp->dtm_context.mutex);
1770         }
1771
1772         return ret;
1773 }
1774
1775 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1776 {
1777         /*
1778          * TODO: bypass the loading in sriov for now
1779          */
1780         if (amdgpu_sriov_vf(psp->adev))
1781                 return 0;
1782
1783         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1784 }
1785
1786 static int psp_dtm_terminate(struct psp_context *psp)
1787 {
1788         int ret;
1789
1790         /*
1791          * TODO: bypass the terminate in sriov for now
1792          */
1793         if (amdgpu_sriov_vf(psp->adev))
1794                 return 0;
1795
1796         if (!psp->dtm_context.context.initialized)
1797                 return 0;
1798
1799         ret = psp_ta_unload(psp, &psp->dtm_context.context);
1800
1801         psp->dtm_context.context.initialized = false;
1802
1803         return ret;
1804 }
1805 // DTM end
1806
1807 // RAP start
1808 static int psp_rap_initialize(struct psp_context *psp)
1809 {
1810         int ret;
1811         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1812
1813         /*
1814          * TODO: bypass the initialize in sriov for now
1815          */
1816         if (amdgpu_sriov_vf(psp->adev))
1817                 return 0;
1818
1819         if (!psp->rap_context.context.bin_desc.size_bytes ||
1820             !psp->rap_context.context.bin_desc.start_addr) {
1821                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1822                 return 0;
1823         }
1824
1825         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1826         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1827
1828         if (!psp->rap_context.context.initialized) {
1829                 ret = psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1830                 if (ret)
1831                         return ret;
1832         }
1833
1834         ret = psp_ta_load(psp, &psp->rap_context.context);
1835         if (!ret) {
1836                 psp->rap_context.context.initialized = true;
1837                 mutex_init(&psp->rap_context.mutex);
1838         } else
1839                 return ret;
1840
1841         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1842         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1843                 psp_rap_terminate(psp);
1844                 /* free rap shared memory */
1845                 psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1846
1847                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1848                          ret, status);
1849
1850                 return ret;
1851         }
1852
1853         return 0;
1854 }
1855
1856 static int psp_rap_terminate(struct psp_context *psp)
1857 {
1858         int ret;
1859
1860         if (!psp->rap_context.context.initialized)
1861                 return 0;
1862
1863         ret = psp_ta_unload(psp, &psp->rap_context.context);
1864
1865         psp->rap_context.context.initialized = false;
1866
1867         return ret;
1868 }
1869
1870 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1871 {
1872         struct ta_rap_shared_memory *rap_cmd;
1873         int ret = 0;
1874
1875         if (!psp->rap_context.context.initialized)
1876                 return 0;
1877
1878         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1879             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1880                 return -EINVAL;
1881
1882         mutex_lock(&psp->rap_context.mutex);
1883
1884         rap_cmd = (struct ta_rap_shared_memory *)
1885                   psp->rap_context.context.mem_context.shared_buf;
1886         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1887
1888         rap_cmd->cmd_id = ta_cmd_id;
1889         rap_cmd->validation_method_id = METHOD_A;
1890
1891         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1892         if (ret)
1893                 goto out_unlock;
1894
1895         if (status)
1896                 *status = rap_cmd->rap_status;
1897
1898 out_unlock:
1899         mutex_unlock(&psp->rap_context.mutex);
1900
1901         return ret;
1902 }
1903 // RAP end
1904
1905 /* securedisplay start */
1906 static int psp_securedisplay_initialize(struct psp_context *psp)
1907 {
1908         int ret;
1909         struct securedisplay_cmd *securedisplay_cmd;
1910
1911         /*
1912          * TODO: bypass the initialize in sriov for now
1913          */
1914         if (amdgpu_sriov_vf(psp->adev))
1915                 return 0;
1916
1917         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1918             !psp->securedisplay_context.context.bin_desc.start_addr) {
1919                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
1920                 return 0;
1921         }
1922
1923         psp->securedisplay_context.context.mem_context.shared_mem_size =
1924                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
1925         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1926
1927         if (!psp->securedisplay_context.context.initialized) {
1928                 ret = psp_ta_init_shared_buf(psp,
1929                                              &psp->securedisplay_context.context.mem_context);
1930                 if (ret)
1931                         return ret;
1932         }
1933
1934         ret = psp_ta_load(psp, &psp->securedisplay_context.context);
1935         if (!ret) {
1936                 psp->securedisplay_context.context.initialized = true;
1937                 mutex_init(&psp->securedisplay_context.mutex);
1938         } else
1939                 return ret;
1940
1941         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
1942                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1943
1944         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1945         if (ret) {
1946                 psp_securedisplay_terminate(psp);
1947                 /* free securedisplay shared memory */
1948                 psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
1949                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
1950                 return -EINVAL;
1951         }
1952
1953         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
1954                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
1955                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
1956                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
1957         }
1958
1959         return 0;
1960 }
1961
1962 static int psp_securedisplay_terminate(struct psp_context *psp)
1963 {
1964         int ret;
1965
1966         /*
1967          * TODO:bypass the terminate in sriov for now
1968          */
1969         if (amdgpu_sriov_vf(psp->adev))
1970                 return 0;
1971
1972         if (!psp->securedisplay_context.context.initialized)
1973                 return 0;
1974
1975         ret = psp_ta_unload(psp, &psp->securedisplay_context.context);
1976
1977         psp->securedisplay_context.context.initialized = false;
1978
1979         return ret;
1980 }
1981
1982 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1983 {
1984         int ret;
1985
1986         if (!psp->securedisplay_context.context.initialized)
1987                 return -EINVAL;
1988
1989         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
1990             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
1991                 return -EINVAL;
1992
1993         mutex_lock(&psp->securedisplay_context.mutex);
1994
1995         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
1996
1997         mutex_unlock(&psp->securedisplay_context.mutex);
1998
1999         return ret;
2000 }
2001 /* SECUREDISPLAY end */
2002
2003 static int psp_hw_start(struct psp_context *psp)
2004 {
2005         struct amdgpu_device *adev = psp->adev;
2006         int ret;
2007
2008         if (!amdgpu_sriov_vf(adev)) {
2009                 if ((is_psp_fw_valid(psp->kdb)) &&
2010                     (psp->funcs->bootloader_load_kdb != NULL)) {
2011                         ret = psp_bootloader_load_kdb(psp);
2012                         if (ret) {
2013                                 DRM_ERROR("PSP load kdb failed!\n");
2014                                 return ret;
2015                         }
2016                 }
2017
2018                 if ((is_psp_fw_valid(psp->spl)) &&
2019                     (psp->funcs->bootloader_load_spl != NULL)) {
2020                         ret = psp_bootloader_load_spl(psp);
2021                         if (ret) {
2022                                 DRM_ERROR("PSP load spl failed!\n");
2023                                 return ret;
2024                         }
2025                 }
2026
2027                 if ((is_psp_fw_valid(psp->sys)) &&
2028                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
2029                         ret = psp_bootloader_load_sysdrv(psp);
2030                         if (ret) {
2031                                 DRM_ERROR("PSP load sys drv failed!\n");
2032                                 return ret;
2033                         }
2034                 }
2035
2036                 if ((is_psp_fw_valid(psp->soc_drv)) &&
2037                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
2038                         ret = psp_bootloader_load_soc_drv(psp);
2039                         if (ret) {
2040                                 DRM_ERROR("PSP load soc drv failed!\n");
2041                                 return ret;
2042                         }
2043                 }
2044
2045                 if ((is_psp_fw_valid(psp->intf_drv)) &&
2046                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
2047                         ret = psp_bootloader_load_intf_drv(psp);
2048                         if (ret) {
2049                                 DRM_ERROR("PSP load intf drv failed!\n");
2050                                 return ret;
2051                         }
2052                 }
2053
2054                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
2055                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
2056                         ret = psp_bootloader_load_dbg_drv(psp);
2057                         if (ret) {
2058                                 DRM_ERROR("PSP load dbg drv failed!\n");
2059                                 return ret;
2060                         }
2061                 }
2062
2063                 if ((is_psp_fw_valid(psp->ras_drv)) &&
2064                     (psp->funcs->bootloader_load_ras_drv != NULL)) {
2065                         ret = psp_bootloader_load_ras_drv(psp);
2066                         if (ret) {
2067                                 DRM_ERROR("PSP load ras_drv failed!\n");
2068                                 return ret;
2069                         }
2070                 }
2071
2072                 if ((is_psp_fw_valid(psp->sos)) &&
2073                     (psp->funcs->bootloader_load_sos != NULL)) {
2074                         ret = psp_bootloader_load_sos(psp);
2075                         if (ret) {
2076                                 DRM_ERROR("PSP load sos failed!\n");
2077                                 return ret;
2078                         }
2079                 }
2080         }
2081
2082         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2083         if (ret) {
2084                 DRM_ERROR("PSP create ring failed!\n");
2085                 return ret;
2086         }
2087
2088         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2089                 goto skip_pin_bo;
2090
2091         ret = psp_tmr_init(psp);
2092         if (ret) {
2093                 DRM_ERROR("PSP tmr init failed!\n");
2094                 return ret;
2095         }
2096
2097 skip_pin_bo:
2098         /*
2099          * For ASICs with DF Cstate management centralized
2100          * to PMFW, TMR setup should be performed after PMFW
2101          * loaded and before other non-psp firmware loaded.
2102          */
2103         if (psp->pmfw_centralized_cstate_management) {
2104                 ret = psp_load_smu_fw(psp);
2105                 if (ret)
2106                         return ret;
2107         }
2108
2109         ret = psp_tmr_load(psp);
2110         if (ret) {
2111                 DRM_ERROR("PSP load tmr failed!\n");
2112                 return ret;
2113         }
2114
2115         return 0;
2116 }
2117
2118 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2119                            enum psp_gfx_fw_type *type)
2120 {
2121         switch (ucode->ucode_id) {
2122         case AMDGPU_UCODE_ID_CAP:
2123                 *type = GFX_FW_TYPE_CAP;
2124                 break;
2125         case AMDGPU_UCODE_ID_SDMA0:
2126                 *type = GFX_FW_TYPE_SDMA0;
2127                 break;
2128         case AMDGPU_UCODE_ID_SDMA1:
2129                 *type = GFX_FW_TYPE_SDMA1;
2130                 break;
2131         case AMDGPU_UCODE_ID_SDMA2:
2132                 *type = GFX_FW_TYPE_SDMA2;
2133                 break;
2134         case AMDGPU_UCODE_ID_SDMA3:
2135                 *type = GFX_FW_TYPE_SDMA3;
2136                 break;
2137         case AMDGPU_UCODE_ID_SDMA4:
2138                 *type = GFX_FW_TYPE_SDMA4;
2139                 break;
2140         case AMDGPU_UCODE_ID_SDMA5:
2141                 *type = GFX_FW_TYPE_SDMA5;
2142                 break;
2143         case AMDGPU_UCODE_ID_SDMA6:
2144                 *type = GFX_FW_TYPE_SDMA6;
2145                 break;
2146         case AMDGPU_UCODE_ID_SDMA7:
2147                 *type = GFX_FW_TYPE_SDMA7;
2148                 break;
2149         case AMDGPU_UCODE_ID_CP_MES:
2150                 *type = GFX_FW_TYPE_CP_MES;
2151                 break;
2152         case AMDGPU_UCODE_ID_CP_MES_DATA:
2153                 *type = GFX_FW_TYPE_MES_STACK;
2154                 break;
2155         case AMDGPU_UCODE_ID_CP_MES1:
2156                 *type = GFX_FW_TYPE_CP_MES_KIQ;
2157                 break;
2158         case AMDGPU_UCODE_ID_CP_MES1_DATA:
2159                 *type = GFX_FW_TYPE_MES_KIQ_STACK;
2160                 break;
2161         case AMDGPU_UCODE_ID_CP_CE:
2162                 *type = GFX_FW_TYPE_CP_CE;
2163                 break;
2164         case AMDGPU_UCODE_ID_CP_PFP:
2165                 *type = GFX_FW_TYPE_CP_PFP;
2166                 break;
2167         case AMDGPU_UCODE_ID_CP_ME:
2168                 *type = GFX_FW_TYPE_CP_ME;
2169                 break;
2170         case AMDGPU_UCODE_ID_CP_MEC1:
2171                 *type = GFX_FW_TYPE_CP_MEC;
2172                 break;
2173         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2174                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2175                 break;
2176         case AMDGPU_UCODE_ID_CP_MEC2:
2177                 *type = GFX_FW_TYPE_CP_MEC;
2178                 break;
2179         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2180                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2181                 break;
2182         case AMDGPU_UCODE_ID_RLC_P:
2183                 *type = GFX_FW_TYPE_RLC_P;
2184                 break;
2185         case AMDGPU_UCODE_ID_RLC_V:
2186                 *type = GFX_FW_TYPE_RLC_V;
2187                 break;
2188         case AMDGPU_UCODE_ID_RLC_G:
2189                 *type = GFX_FW_TYPE_RLC_G;
2190                 break;
2191         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2192                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2193                 break;
2194         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2195                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2196                 break;
2197         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2198                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2199                 break;
2200         case AMDGPU_UCODE_ID_RLC_IRAM:
2201                 *type = GFX_FW_TYPE_RLC_IRAM;
2202                 break;
2203         case AMDGPU_UCODE_ID_RLC_DRAM:
2204                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2205                 break;
2206         case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
2207                 *type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
2208                 break;
2209         case AMDGPU_UCODE_ID_SE0_TAP_DELAYS:
2210                 *type = GFX_FW_TYPE_SE0_TAP_DELAYS;
2211                 break;
2212         case AMDGPU_UCODE_ID_SE1_TAP_DELAYS:
2213                 *type = GFX_FW_TYPE_SE1_TAP_DELAYS;
2214                 break;
2215         case AMDGPU_UCODE_ID_SE2_TAP_DELAYS:
2216                 *type = GFX_FW_TYPE_SE2_TAP_DELAYS;
2217                 break;
2218         case AMDGPU_UCODE_ID_SE3_TAP_DELAYS:
2219                 *type = GFX_FW_TYPE_SE3_TAP_DELAYS;
2220                 break;
2221         case AMDGPU_UCODE_ID_SMC:
2222                 *type = GFX_FW_TYPE_SMU;
2223                 break;
2224         case AMDGPU_UCODE_ID_PPTABLE:
2225                 *type = GFX_FW_TYPE_PPTABLE;
2226                 break;
2227         case AMDGPU_UCODE_ID_UVD:
2228                 *type = GFX_FW_TYPE_UVD;
2229                 break;
2230         case AMDGPU_UCODE_ID_UVD1:
2231                 *type = GFX_FW_TYPE_UVD1;
2232                 break;
2233         case AMDGPU_UCODE_ID_VCE:
2234                 *type = GFX_FW_TYPE_VCE;
2235                 break;
2236         case AMDGPU_UCODE_ID_VCN:
2237                 *type = GFX_FW_TYPE_VCN;
2238                 break;
2239         case AMDGPU_UCODE_ID_VCN1:
2240                 *type = GFX_FW_TYPE_VCN1;
2241                 break;
2242         case AMDGPU_UCODE_ID_DMCU_ERAM:
2243                 *type = GFX_FW_TYPE_DMCU_ERAM;
2244                 break;
2245         case AMDGPU_UCODE_ID_DMCU_INTV:
2246                 *type = GFX_FW_TYPE_DMCU_ISR;
2247                 break;
2248         case AMDGPU_UCODE_ID_VCN0_RAM:
2249                 *type = GFX_FW_TYPE_VCN0_RAM;
2250                 break;
2251         case AMDGPU_UCODE_ID_VCN1_RAM:
2252                 *type = GFX_FW_TYPE_VCN1_RAM;
2253                 break;
2254         case AMDGPU_UCODE_ID_DMCUB:
2255                 *type = GFX_FW_TYPE_DMUB;
2256                 break;
2257         case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
2258                 *type = GFX_FW_TYPE_SDMA_UCODE_TH0;
2259                 break;
2260         case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
2261                 *type = GFX_FW_TYPE_SDMA_UCODE_TH1;
2262                 break;
2263         case AMDGPU_UCODE_ID_IMU_I:
2264                 *type = GFX_FW_TYPE_IMU_I;
2265                 break;
2266         case AMDGPU_UCODE_ID_IMU_D:
2267                 *type = GFX_FW_TYPE_IMU_D;
2268                 break;
2269         case AMDGPU_UCODE_ID_CP_RS64_PFP:
2270                 *type = GFX_FW_TYPE_RS64_PFP;
2271                 break;
2272         case AMDGPU_UCODE_ID_CP_RS64_ME:
2273                 *type = GFX_FW_TYPE_RS64_ME;
2274                 break;
2275         case AMDGPU_UCODE_ID_CP_RS64_MEC:
2276                 *type = GFX_FW_TYPE_RS64_MEC;
2277                 break;
2278         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
2279                 *type = GFX_FW_TYPE_RS64_PFP_P0_STACK;
2280                 break;
2281         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
2282                 *type = GFX_FW_TYPE_RS64_PFP_P1_STACK;
2283                 break;
2284         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
2285                 *type = GFX_FW_TYPE_RS64_ME_P0_STACK;
2286                 break;
2287         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
2288                 *type = GFX_FW_TYPE_RS64_ME_P1_STACK;
2289                 break;
2290         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
2291                 *type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
2292                 break;
2293         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
2294                 *type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
2295                 break;
2296         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
2297                 *type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
2298                 break;
2299         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
2300                 *type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
2301                 break;
2302         case AMDGPU_UCODE_ID_MAXIMUM:
2303         default:
2304                 return -EINVAL;
2305         }
2306
2307         return 0;
2308 }
2309
2310 static void psp_print_fw_hdr(struct psp_context *psp,
2311                              struct amdgpu_firmware_info *ucode)
2312 {
2313         struct amdgpu_device *adev = psp->adev;
2314         struct common_firmware_header *hdr;
2315
2316         switch (ucode->ucode_id) {
2317         case AMDGPU_UCODE_ID_SDMA0:
2318         case AMDGPU_UCODE_ID_SDMA1:
2319         case AMDGPU_UCODE_ID_SDMA2:
2320         case AMDGPU_UCODE_ID_SDMA3:
2321         case AMDGPU_UCODE_ID_SDMA4:
2322         case AMDGPU_UCODE_ID_SDMA5:
2323         case AMDGPU_UCODE_ID_SDMA6:
2324         case AMDGPU_UCODE_ID_SDMA7:
2325                 hdr = (struct common_firmware_header *)
2326                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2327                 amdgpu_ucode_print_sdma_hdr(hdr);
2328                 break;
2329         case AMDGPU_UCODE_ID_CP_CE:
2330                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2331                 amdgpu_ucode_print_gfx_hdr(hdr);
2332                 break;
2333         case AMDGPU_UCODE_ID_CP_PFP:
2334                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2335                 amdgpu_ucode_print_gfx_hdr(hdr);
2336                 break;
2337         case AMDGPU_UCODE_ID_CP_ME:
2338                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2339                 amdgpu_ucode_print_gfx_hdr(hdr);
2340                 break;
2341         case AMDGPU_UCODE_ID_CP_MEC1:
2342                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2343                 amdgpu_ucode_print_gfx_hdr(hdr);
2344                 break;
2345         case AMDGPU_UCODE_ID_RLC_G:
2346                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2347                 amdgpu_ucode_print_rlc_hdr(hdr);
2348                 break;
2349         case AMDGPU_UCODE_ID_SMC:
2350                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2351                 amdgpu_ucode_print_smc_hdr(hdr);
2352                 break;
2353         default:
2354                 break;
2355         }
2356 }
2357
2358 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2359                                        struct psp_gfx_cmd_resp *cmd)
2360 {
2361         int ret;
2362         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2363
2364         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2365         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2366         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2367         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2368
2369         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2370         if (ret)
2371                 DRM_ERROR("Unknown firmware type\n");
2372
2373         return ret;
2374 }
2375
2376 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2377                                   struct amdgpu_firmware_info *ucode)
2378 {
2379         int ret = 0;
2380         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2381
2382         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2383         if (!ret) {
2384                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2385                                          psp->fence_buf_mc_addr);
2386         }
2387
2388         release_psp_cmd_buf(psp);
2389
2390         return ret;
2391 }
2392
2393 static int psp_load_smu_fw(struct psp_context *psp)
2394 {
2395         int ret;
2396         struct amdgpu_device *adev = psp->adev;
2397         struct amdgpu_firmware_info *ucode =
2398                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2399         struct amdgpu_ras *ras = psp->ras_context.ras;
2400
2401         /*
2402          * Skip SMU FW reloading in case of using BACO for runpm only,
2403          * as SMU is always alive.
2404          */
2405         if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
2406                 return 0;
2407
2408         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2409                 return 0;
2410
2411         if ((amdgpu_in_reset(adev) &&
2412              ras && adev->ras_enabled &&
2413              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2414               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2415                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2416                 if (ret) {
2417                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2418                 }
2419         }
2420
2421         ret = psp_execute_non_psp_fw_load(psp, ucode);
2422
2423         if (ret)
2424                 DRM_ERROR("PSP load smu failed!\n");
2425
2426         return ret;
2427 }
2428
2429 static bool fw_load_skip_check(struct psp_context *psp,
2430                                struct amdgpu_firmware_info *ucode)
2431 {
2432         if (!ucode->fw || !ucode->ucode_size)
2433                 return true;
2434
2435         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2436             (psp_smu_reload_quirk(psp) ||
2437              psp->autoload_supported ||
2438              psp->pmfw_centralized_cstate_management))
2439                 return true;
2440
2441         if (amdgpu_sriov_vf(psp->adev) &&
2442             amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
2443                 return true;
2444
2445         if (psp->autoload_supported &&
2446             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2447              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2448                 /* skip mec JT when autoload is enabled */
2449                 return true;
2450
2451         return false;
2452 }
2453
2454 int psp_load_fw_list(struct psp_context *psp,
2455                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2456 {
2457         int ret = 0, i;
2458         struct amdgpu_firmware_info *ucode;
2459
2460         for (i = 0; i < ucode_count; ++i) {
2461                 ucode = ucode_list[i];
2462                 psp_print_fw_hdr(psp, ucode);
2463                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2464                 if (ret)
2465                         return ret;
2466         }
2467         return ret;
2468 }
2469
2470 static int psp_load_non_psp_fw(struct psp_context *psp)
2471 {
2472         int i, ret;
2473         struct amdgpu_firmware_info *ucode;
2474         struct amdgpu_device *adev = psp->adev;
2475
2476         if (psp->autoload_supported &&
2477             !psp->pmfw_centralized_cstate_management) {
2478                 ret = psp_load_smu_fw(psp);
2479                 if (ret)
2480                         return ret;
2481         }
2482
2483         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2484                 ucode = &adev->firmware.ucode[i];
2485
2486                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2487                     !fw_load_skip_check(psp, ucode)) {
2488                         ret = psp_load_smu_fw(psp);
2489                         if (ret)
2490                                 return ret;
2491                         continue;
2492                 }
2493
2494                 if (fw_load_skip_check(psp, ucode))
2495                         continue;
2496
2497                 if (psp->autoload_supported &&
2498                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2499                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2500                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2501                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2502                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2503                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2504                         /* PSP only receive one SDMA fw for sienna_cichlid,
2505                          * as all four sdma fw are same */
2506                         continue;
2507
2508                 psp_print_fw_hdr(psp, ucode);
2509
2510                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2511                 if (ret)
2512                         return ret;
2513
2514                 /* Start rlc autoload after psp recieved all the gfx firmware */
2515                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2516                     adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
2517                         ret = psp_rlc_autoload_start(psp);
2518                         if (ret) {
2519                                 DRM_ERROR("Failed to start rlc autoload\n");
2520                                 return ret;
2521                         }
2522                 }
2523         }
2524
2525         return 0;
2526 }
2527
2528 static int psp_load_fw(struct amdgpu_device *adev)
2529 {
2530         int ret;
2531         struct psp_context *psp = &adev->psp;
2532
2533         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2534                 /* should not destroy ring, only stop */
2535                 psp_ring_stop(psp, PSP_RING_TYPE__KM);
2536         } else {
2537                 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2538
2539                 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2540                 if (ret) {
2541                         DRM_ERROR("PSP ring init failed!\n");
2542                         goto failed;
2543                 }
2544         }
2545
2546         ret = psp_hw_start(psp);
2547         if (ret)
2548                 goto failed;
2549
2550         ret = psp_load_non_psp_fw(psp);
2551         if (ret)
2552                 goto failed1;
2553
2554         ret = psp_asd_initialize(psp);
2555         if (ret) {
2556                 DRM_ERROR("PSP load asd failed!\n");
2557                 goto failed1;
2558         }
2559
2560         ret = psp_rl_load(adev);
2561         if (ret) {
2562                 DRM_ERROR("PSP load RL failed!\n");
2563                 goto failed1;
2564         }
2565
2566         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2567                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2568                         ret = psp_xgmi_initialize(psp, false, true);
2569                         /* Warning the XGMI seesion initialize failure
2570                         * Instead of stop driver initialization
2571                         */
2572                         if (ret)
2573                                 dev_err(psp->adev->dev,
2574                                         "XGMI: Failed to initialize XGMI session\n");
2575                 }
2576         }
2577
2578         if (psp->ta_fw) {
2579                 ret = psp_ras_initialize(psp);
2580                 if (ret)
2581                         dev_err(psp->adev->dev,
2582                                         "RAS: Failed to initialize RAS\n");
2583
2584                 ret = psp_hdcp_initialize(psp);
2585                 if (ret)
2586                         dev_err(psp->adev->dev,
2587                                 "HDCP: Failed to initialize HDCP\n");
2588
2589                 ret = psp_dtm_initialize(psp);
2590                 if (ret)
2591                         dev_err(psp->adev->dev,
2592                                 "DTM: Failed to initialize DTM\n");
2593
2594                 ret = psp_rap_initialize(psp);
2595                 if (ret)
2596                         dev_err(psp->adev->dev,
2597                                 "RAP: Failed to initialize RAP\n");
2598
2599                 ret = psp_securedisplay_initialize(psp);
2600                 if (ret)
2601                         dev_err(psp->adev->dev,
2602                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2603         }
2604
2605         return 0;
2606
2607 failed1:
2608         psp_free_shared_bufs(psp);
2609 failed:
2610         /*
2611          * all cleanup jobs (xgmi terminate, ras terminate,
2612          * ring destroy, cmd/fence/fw buffers destory,
2613          * psp->cmd destory) are delayed to psp_hw_fini
2614          */
2615         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2616         return ret;
2617 }
2618
2619 static int psp_hw_init(void *handle)
2620 {
2621         int ret;
2622         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2623
2624         mutex_lock(&adev->firmware.mutex);
2625         /*
2626          * This sequence is just used on hw_init only once, no need on
2627          * resume.
2628          */
2629         ret = amdgpu_ucode_init_bo(adev);
2630         if (ret)
2631                 goto failed;
2632
2633         ret = psp_load_fw(adev);
2634         if (ret) {
2635                 DRM_ERROR("PSP firmware loading failed\n");
2636                 goto failed;
2637         }
2638
2639         mutex_unlock(&adev->firmware.mutex);
2640         return 0;
2641
2642 failed:
2643         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2644         mutex_unlock(&adev->firmware.mutex);
2645         return -EINVAL;
2646 }
2647
2648 static int psp_hw_fini(void *handle)
2649 {
2650         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2651         struct psp_context *psp = &adev->psp;
2652
2653         if (psp->ta_fw) {
2654                 psp_ras_terminate(psp);
2655                 psp_securedisplay_terminate(psp);
2656                 psp_rap_terminate(psp);
2657                 psp_dtm_terminate(psp);
2658                 psp_hdcp_terminate(psp);
2659
2660                 if (adev->gmc.xgmi.num_physical_nodes > 1)
2661                         psp_xgmi_terminate(psp);
2662         }
2663
2664         psp_asd_terminate(psp);
2665         psp_tmr_terminate(psp);
2666
2667         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2668
2669         psp_free_shared_bufs(psp);
2670
2671         return 0;
2672 }
2673
2674 static int psp_suspend(void *handle)
2675 {
2676         int ret = 0;
2677         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2678         struct psp_context *psp = &adev->psp;
2679
2680         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2681             psp->xgmi_context.context.initialized) {
2682                 ret = psp_xgmi_terminate(psp);
2683                 if (ret) {
2684                         DRM_ERROR("Failed to terminate xgmi ta\n");
2685                         goto out;
2686                 }
2687         }
2688
2689         if (psp->ta_fw) {
2690                 ret = psp_ras_terminate(psp);
2691                 if (ret) {
2692                         DRM_ERROR("Failed to terminate ras ta\n");
2693                         goto out;
2694                 }
2695                 ret = psp_hdcp_terminate(psp);
2696                 if (ret) {
2697                         DRM_ERROR("Failed to terminate hdcp ta\n");
2698                         goto out;
2699                 }
2700                 ret = psp_dtm_terminate(psp);
2701                 if (ret) {
2702                         DRM_ERROR("Failed to terminate dtm ta\n");
2703                         goto out;
2704                 }
2705                 ret = psp_rap_terminate(psp);
2706                 if (ret) {
2707                         DRM_ERROR("Failed to terminate rap ta\n");
2708                         goto out;
2709                 }
2710                 ret = psp_securedisplay_terminate(psp);
2711                 if (ret) {
2712                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2713                         goto out;
2714                 }
2715         }
2716
2717         ret = psp_asd_terminate(psp);
2718         if (ret) {
2719                 DRM_ERROR("Failed to terminate asd\n");
2720                 goto out;
2721         }
2722
2723         ret = psp_tmr_terminate(psp);
2724         if (ret) {
2725                 DRM_ERROR("Failed to terminate tmr\n");
2726                 goto out;
2727         }
2728
2729         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2730         if (ret) {
2731                 DRM_ERROR("PSP ring stop failed\n");
2732         }
2733
2734 out:
2735         psp_free_shared_bufs(psp);
2736
2737         return ret;
2738 }
2739
2740 static int psp_resume(void *handle)
2741 {
2742         int ret;
2743         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2744         struct psp_context *psp = &adev->psp;
2745
2746         DRM_INFO("PSP is resuming...\n");
2747
2748         if (psp->mem_train_ctx.enable_mem_training) {
2749                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2750                 if (ret) {
2751                         DRM_ERROR("Failed to process memory training!\n");
2752                         return ret;
2753                 }
2754         }
2755
2756         mutex_lock(&adev->firmware.mutex);
2757
2758         ret = psp_hw_start(psp);
2759         if (ret)
2760                 goto failed;
2761
2762         ret = psp_load_non_psp_fw(psp);
2763         if (ret)
2764                 goto failed;
2765
2766         ret = psp_asd_initialize(psp);
2767         if (ret) {
2768                 DRM_ERROR("PSP load asd failed!\n");
2769                 goto failed;
2770         }
2771
2772         ret = psp_rl_load(adev);
2773         if (ret) {
2774                 dev_err(adev->dev, "PSP load RL failed!\n");
2775                 goto failed;
2776         }
2777
2778         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2779                 ret = psp_xgmi_initialize(psp, false, true);
2780                 /* Warning the XGMI seesion initialize failure
2781                  * Instead of stop driver initialization
2782                  */
2783                 if (ret)
2784                         dev_err(psp->adev->dev,
2785                                 "XGMI: Failed to initialize XGMI session\n");
2786         }
2787
2788         if (psp->ta_fw) {
2789                 ret = psp_ras_initialize(psp);
2790                 if (ret)
2791                         dev_err(psp->adev->dev,
2792                                         "RAS: Failed to initialize RAS\n");
2793
2794                 ret = psp_hdcp_initialize(psp);
2795                 if (ret)
2796                         dev_err(psp->adev->dev,
2797                                 "HDCP: Failed to initialize HDCP\n");
2798
2799                 ret = psp_dtm_initialize(psp);
2800                 if (ret)
2801                         dev_err(psp->adev->dev,
2802                                 "DTM: Failed to initialize DTM\n");
2803
2804                 ret = psp_rap_initialize(psp);
2805                 if (ret)
2806                         dev_err(psp->adev->dev,
2807                                 "RAP: Failed to initialize RAP\n");
2808
2809                 ret = psp_securedisplay_initialize(psp);
2810                 if (ret)
2811                         dev_err(psp->adev->dev,
2812                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2813         }
2814
2815         mutex_unlock(&adev->firmware.mutex);
2816
2817         return 0;
2818
2819 failed:
2820         DRM_ERROR("PSP resume failed\n");
2821         mutex_unlock(&adev->firmware.mutex);
2822         return ret;
2823 }
2824
2825 int psp_gpu_reset(struct amdgpu_device *adev)
2826 {
2827         int ret;
2828
2829         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2830                 return 0;
2831
2832         mutex_lock(&adev->psp.mutex);
2833         ret = psp_mode1_reset(&adev->psp);
2834         mutex_unlock(&adev->psp.mutex);
2835
2836         return ret;
2837 }
2838
2839 int psp_rlc_autoload_start(struct psp_context *psp)
2840 {
2841         int ret;
2842         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2843
2844         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2845
2846         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2847                                  psp->fence_buf_mc_addr);
2848
2849         release_psp_cmd_buf(psp);
2850
2851         return ret;
2852 }
2853
2854 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2855                         uint64_t cmd_gpu_addr, int cmd_size)
2856 {
2857         struct amdgpu_firmware_info ucode = {0};
2858
2859         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2860                 AMDGPU_UCODE_ID_VCN0_RAM;
2861         ucode.mc_addr = cmd_gpu_addr;
2862         ucode.ucode_size = cmd_size;
2863
2864         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2865 }
2866
2867 int psp_ring_cmd_submit(struct psp_context *psp,
2868                         uint64_t cmd_buf_mc_addr,
2869                         uint64_t fence_mc_addr,
2870                         int index)
2871 {
2872         unsigned int psp_write_ptr_reg = 0;
2873         struct psp_gfx_rb_frame *write_frame;
2874         struct psp_ring *ring = &psp->km_ring;
2875         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2876         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2877                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2878         struct amdgpu_device *adev = psp->adev;
2879         uint32_t ring_size_dw = ring->ring_size / 4;
2880         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2881
2882         /* KM (GPCOM) prepare write pointer */
2883         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2884
2885         /* Update KM RB frame pointer to new frame */
2886         /* write_frame ptr increments by size of rb_frame in bytes */
2887         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2888         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2889                 write_frame = ring_buffer_start;
2890         else
2891                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2892         /* Check invalid write_frame ptr address */
2893         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2894                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2895                           ring_buffer_start, ring_buffer_end, write_frame);
2896                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2897                 return -EINVAL;
2898         }
2899
2900         /* Initialize KM RB frame */
2901         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2902
2903         /* Update KM RB frame */
2904         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2905         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2906         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2907         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2908         write_frame->fence_value = index;
2909         amdgpu_device_flush_hdp(adev, NULL);
2910
2911         /* Update the write Pointer in DWORDs */
2912         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2913         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2914         return 0;
2915 }
2916
2917 int psp_init_asd_microcode(struct psp_context *psp,
2918                            const char *chip_name)
2919 {
2920         struct amdgpu_device *adev = psp->adev;
2921         char fw_name[PSP_FW_NAME_LEN];
2922         const struct psp_firmware_header_v1_0 *asd_hdr;
2923         int err = 0;
2924
2925         if (!chip_name) {
2926                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2927                 return -EINVAL;
2928         }
2929
2930         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2931         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2932         if (err)
2933                 goto out;
2934
2935         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2936         if (err)
2937                 goto out;
2938
2939         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2940         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2941         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
2942         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2943         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
2944                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2945         return 0;
2946 out:
2947         dev_err(adev->dev, "fail to initialize asd microcode\n");
2948         release_firmware(adev->psp.asd_fw);
2949         adev->psp.asd_fw = NULL;
2950         return err;
2951 }
2952
2953 int psp_init_toc_microcode(struct psp_context *psp,
2954                            const char *chip_name)
2955 {
2956         struct amdgpu_device *adev = psp->adev;
2957         char fw_name[PSP_FW_NAME_LEN];
2958         const struct psp_firmware_header_v1_0 *toc_hdr;
2959         int err = 0;
2960
2961         if (!chip_name) {
2962                 dev_err(adev->dev, "invalid chip name for toc microcode\n");
2963                 return -EINVAL;
2964         }
2965
2966         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
2967         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
2968         if (err)
2969                 goto out;
2970
2971         err = amdgpu_ucode_validate(adev->psp.toc_fw);
2972         if (err)
2973                 goto out;
2974
2975         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
2976         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
2977         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
2978         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
2979         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
2980                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
2981         return 0;
2982 out:
2983         dev_err(adev->dev, "fail to request/validate toc microcode\n");
2984         release_firmware(adev->psp.toc_fw);
2985         adev->psp.toc_fw = NULL;
2986         return err;
2987 }
2988
2989 static int parse_sos_bin_descriptor(struct psp_context *psp,
2990                                    const struct psp_fw_bin_desc *desc,
2991                                    const struct psp_firmware_header_v2_0 *sos_hdr)
2992 {
2993         uint8_t *ucode_start_addr  = NULL;
2994
2995         if (!psp || !desc || !sos_hdr)
2996                 return -EINVAL;
2997
2998         ucode_start_addr  = (uint8_t *)sos_hdr +
2999                             le32_to_cpu(desc->offset_bytes) +
3000                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3001
3002         switch (desc->fw_type) {
3003         case PSP_FW_TYPE_PSP_SOS:
3004                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
3005                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
3006                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
3007                 psp->sos.start_addr        = ucode_start_addr;
3008                 break;
3009         case PSP_FW_TYPE_PSP_SYS_DRV:
3010                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
3011                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
3012                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
3013                 psp->sys.start_addr        = ucode_start_addr;
3014                 break;
3015         case PSP_FW_TYPE_PSP_KDB:
3016                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
3017                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
3018                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
3019                 psp->kdb.start_addr        = ucode_start_addr;
3020                 break;
3021         case PSP_FW_TYPE_PSP_TOC:
3022                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
3023                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
3024                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
3025                 psp->toc.start_addr        = ucode_start_addr;
3026                 break;
3027         case PSP_FW_TYPE_PSP_SPL:
3028                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
3029                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
3030                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
3031                 psp->spl.start_addr        = ucode_start_addr;
3032                 break;
3033         case PSP_FW_TYPE_PSP_RL:
3034                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
3035                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
3036                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
3037                 psp->rl.start_addr         = ucode_start_addr;
3038                 break;
3039         case PSP_FW_TYPE_PSP_SOC_DRV:
3040                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
3041                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
3042                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3043                 psp->soc_drv.start_addr         = ucode_start_addr;
3044                 break;
3045         case PSP_FW_TYPE_PSP_INTF_DRV:
3046                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
3047                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
3048                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
3049                 psp->intf_drv.start_addr        = ucode_start_addr;
3050                 break;
3051         case PSP_FW_TYPE_PSP_DBG_DRV:
3052                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
3053                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
3054                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3055                 psp->dbg_drv.start_addr         = ucode_start_addr;
3056                 break;
3057         case PSP_FW_TYPE_PSP_RAS_DRV:
3058                 psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
3059                 psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
3060                 psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3061                 psp->ras_drv.start_addr         = ucode_start_addr;
3062                 break;
3063         default:
3064                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
3065                 break;
3066         }
3067
3068         return 0;
3069 }
3070
3071 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
3072 {
3073         const struct psp_firmware_header_v1_0 *sos_hdr;
3074         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3075         uint8_t *ucode_array_start_addr;
3076
3077         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3078         ucode_array_start_addr = (uint8_t *)sos_hdr +
3079                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3080
3081         if (adev->gmc.xgmi.connected_to_cpu ||
3082             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
3083                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
3084                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
3085
3086                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
3087                 adev->psp.sys.start_addr = ucode_array_start_addr;
3088
3089                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
3090                 adev->psp.sos.start_addr = ucode_array_start_addr +
3091                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
3092         } else {
3093                 /* Load alternate PSP SOS FW */
3094                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3095
3096                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3097                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3098
3099                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
3100                 adev->psp.sys.start_addr = ucode_array_start_addr +
3101                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
3102
3103                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
3104                 adev->psp.sos.start_addr = ucode_array_start_addr +
3105                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
3106         }
3107
3108         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
3109                 dev_warn(adev->dev, "PSP SOS FW not available");
3110                 return -EINVAL;
3111         }
3112
3113         return 0;
3114 }
3115
3116 int psp_init_sos_microcode(struct psp_context *psp,
3117                            const char *chip_name)
3118 {
3119         struct amdgpu_device *adev = psp->adev;
3120         char fw_name[PSP_FW_NAME_LEN];
3121         const struct psp_firmware_header_v1_0 *sos_hdr;
3122         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3123         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3124         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3125         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3126         int err = 0;
3127         uint8_t *ucode_array_start_addr;
3128         int fw_index = 0;
3129
3130         if (!chip_name) {
3131                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
3132                 return -EINVAL;
3133         }
3134
3135         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3136         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
3137         if (err)
3138                 goto out;
3139
3140         err = amdgpu_ucode_validate(adev->psp.sos_fw);
3141         if (err)
3142                 goto out;
3143
3144         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3145         ucode_array_start_addr = (uint8_t *)sos_hdr +
3146                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3147         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3148
3149         switch (sos_hdr->header.header_version_major) {
3150         case 1:
3151                 err = psp_init_sos_base_fw(adev);
3152                 if (err)
3153                         goto out;
3154
3155                 if (sos_hdr->header.header_version_minor == 1) {
3156                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3157                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3158                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3159                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3160                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3161                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3162                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3163                 }
3164                 if (sos_hdr->header.header_version_minor == 2) {
3165                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3166                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3167                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3168                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3169                 }
3170                 if (sos_hdr->header.header_version_minor == 3) {
3171                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3172                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3173                         adev->psp.toc.start_addr = ucode_array_start_addr +
3174                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3175                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3176                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3177                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3178                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3179                         adev->psp.spl.start_addr = ucode_array_start_addr +
3180                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3181                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3182                         adev->psp.rl.start_addr = ucode_array_start_addr +
3183                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3184                 }
3185                 break;
3186         case 2:
3187                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3188
3189                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3190                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3191                         err = -EINVAL;
3192                         goto out;
3193                 }
3194
3195                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3196                         err = parse_sos_bin_descriptor(psp,
3197                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3198                                                        sos_hdr_v2_0);
3199                         if (err)
3200                                 goto out;
3201                 }
3202                 break;
3203         default:
3204                 dev_err(adev->dev,
3205                         "unsupported psp sos firmware\n");
3206                 err = -EINVAL;
3207                 goto out;
3208         }
3209
3210         return 0;
3211 out:
3212         dev_err(adev->dev,
3213                 "failed to init sos firmware\n");
3214         release_firmware(adev->psp.sos_fw);
3215         adev->psp.sos_fw = NULL;
3216
3217         return err;
3218 }
3219
3220 static int parse_ta_bin_descriptor(struct psp_context *psp,
3221                                    const struct psp_fw_bin_desc *desc,
3222                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3223 {
3224         uint8_t *ucode_start_addr  = NULL;
3225
3226         if (!psp || !desc || !ta_hdr)
3227                 return -EINVAL;
3228
3229         ucode_start_addr  = (uint8_t *)ta_hdr +
3230                             le32_to_cpu(desc->offset_bytes) +
3231                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3232
3233         switch (desc->fw_type) {
3234         case TA_FW_TYPE_PSP_ASD:
3235                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3236                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3237                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3238                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3239                 break;
3240         case TA_FW_TYPE_PSP_XGMI:
3241                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3242                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3243                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3244                 break;
3245         case TA_FW_TYPE_PSP_RAS:
3246                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3247                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3248                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3249                 break;
3250         case TA_FW_TYPE_PSP_HDCP:
3251                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3252                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3253                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3254                 break;
3255         case TA_FW_TYPE_PSP_DTM:
3256                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3257                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3258                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3259                 break;
3260         case TA_FW_TYPE_PSP_RAP:
3261                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3262                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3263                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3264                 break;
3265         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3266                 psp->securedisplay_context.context.bin_desc.fw_version =
3267                         le32_to_cpu(desc->fw_version);
3268                 psp->securedisplay_context.context.bin_desc.size_bytes =
3269                         le32_to_cpu(desc->size_bytes);
3270                 psp->securedisplay_context.context.bin_desc.start_addr =
3271                         ucode_start_addr;
3272                 break;
3273         default:
3274                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3275                 break;
3276         }
3277
3278         return 0;
3279 }
3280
3281 int psp_init_ta_microcode(struct psp_context *psp,
3282                           const char *chip_name)
3283 {
3284         struct amdgpu_device *adev = psp->adev;
3285         char fw_name[PSP_FW_NAME_LEN];
3286         const struct ta_firmware_header_v2_0 *ta_hdr;
3287         int err = 0;
3288         int ta_index = 0;
3289
3290         if (!chip_name) {
3291                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
3292                 return -EINVAL;
3293         }
3294
3295         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3296         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
3297         if (err)
3298                 goto out;
3299
3300         err = amdgpu_ucode_validate(adev->psp.ta_fw);
3301         if (err)
3302                 goto out;
3303
3304         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3305
3306         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
3307                 dev_err(adev->dev, "unsupported TA header version\n");
3308                 err = -EINVAL;
3309                 goto out;
3310         }
3311
3312         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3313                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3314                 err = -EINVAL;
3315                 goto out;
3316         }
3317
3318         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3319                 err = parse_ta_bin_descriptor(psp,
3320                                               &ta_hdr->ta_fw_bin[ta_index],
3321                                               ta_hdr);
3322                 if (err)
3323                         goto out;
3324         }
3325
3326         return 0;
3327 out:
3328         dev_err(adev->dev, "fail to initialize ta microcode\n");
3329         release_firmware(adev->psp.ta_fw);
3330         adev->psp.ta_fw = NULL;
3331         return err;
3332 }
3333
3334 int psp_init_cap_microcode(struct psp_context *psp,
3335                           const char *chip_name)
3336 {
3337         struct amdgpu_device *adev = psp->adev;
3338         char fw_name[PSP_FW_NAME_LEN];
3339         const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
3340         struct amdgpu_firmware_info *info = NULL;
3341         int err = 0;
3342
3343         if (!chip_name) {
3344                 dev_err(adev->dev, "invalid chip name for cap microcode\n");
3345                 return -EINVAL;
3346         }
3347
3348         if (!amdgpu_sriov_vf(adev)) {
3349                 dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
3350                 return -EINVAL;
3351         }
3352
3353         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_cap.bin", chip_name);
3354         err = request_firmware(&adev->psp.cap_fw, fw_name, adev->dev);
3355         if (err) {
3356                 dev_warn(adev->dev, "cap microcode does not exist, skip\n");
3357                 err = 0;
3358                 goto out;
3359         }
3360
3361         err = amdgpu_ucode_validate(adev->psp.cap_fw);
3362         if (err) {
3363                 dev_err(adev->dev, "fail to initialize cap microcode\n");
3364                 goto out;
3365         }
3366
3367         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
3368         info->ucode_id = AMDGPU_UCODE_ID_CAP;
3369         info->fw = adev->psp.cap_fw;
3370         cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
3371                 adev->psp.cap_fw->data;
3372         adev->firmware.fw_size += ALIGN(
3373                         le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
3374         adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
3375         adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
3376         adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
3377
3378         return 0;
3379
3380 out:
3381         release_firmware(adev->psp.cap_fw);
3382         adev->psp.cap_fw = NULL;
3383         return err;
3384 }
3385
3386 static int psp_set_clockgating_state(void *handle,
3387                                      enum amd_clockgating_state state)
3388 {
3389         return 0;
3390 }
3391
3392 static int psp_set_powergating_state(void *handle,
3393                                      enum amd_powergating_state state)
3394 {
3395         return 0;
3396 }
3397
3398 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3399                                          struct device_attribute *attr,
3400                                          char *buf)
3401 {
3402         struct drm_device *ddev = dev_get_drvdata(dev);
3403         struct amdgpu_device *adev = drm_to_adev(ddev);
3404         uint32_t fw_ver;
3405         int ret;
3406
3407         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3408                 DRM_INFO("PSP block is not ready yet.");
3409                 return -EBUSY;
3410         }
3411
3412         mutex_lock(&adev->psp.mutex);
3413         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3414         mutex_unlock(&adev->psp.mutex);
3415
3416         if (ret) {
3417                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3418                 return ret;
3419         }
3420
3421         return sysfs_emit(buf, "%x\n", fw_ver);
3422 }
3423
3424 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3425                                                        struct device_attribute *attr,
3426                                                        const char *buf,
3427                                                        size_t count)
3428 {
3429         struct drm_device *ddev = dev_get_drvdata(dev);
3430         struct amdgpu_device *adev = drm_to_adev(ddev);
3431         int ret, idx;
3432         char fw_name[100];
3433         const struct firmware *usbc_pd_fw;
3434         struct amdgpu_bo *fw_buf_bo = NULL;
3435         uint64_t fw_pri_mc_addr;
3436         void *fw_pri_cpu_addr;
3437
3438         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3439                 DRM_INFO("PSP block is not ready yet.");
3440                 return -EBUSY;
3441         }
3442
3443         if (!drm_dev_enter(ddev, &idx))
3444                 return -ENODEV;
3445
3446         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3447         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3448         if (ret)
3449                 goto fail;
3450
3451         /* LFB address which is aligned to 1MB boundary per PSP request */
3452         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3453                                                 AMDGPU_GEM_DOMAIN_VRAM,
3454                                                 &fw_buf_bo,
3455                                                 &fw_pri_mc_addr,
3456                                                 &fw_pri_cpu_addr);
3457         if (ret)
3458                 goto rel_buf;
3459
3460         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3461
3462         mutex_lock(&adev->psp.mutex);
3463         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3464         mutex_unlock(&adev->psp.mutex);
3465
3466         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3467
3468 rel_buf:
3469         release_firmware(usbc_pd_fw);
3470 fail:
3471         if (ret) {
3472                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3473                 count = ret;
3474         }
3475
3476         drm_dev_exit(idx);
3477         return count;
3478 }
3479
3480 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3481 {
3482         int idx;
3483
3484         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3485                 return;
3486
3487         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3488         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3489
3490         drm_dev_exit(idx);
3491 }
3492
3493 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
3494                    psp_usbc_pd_fw_sysfs_read,
3495                    psp_usbc_pd_fw_sysfs_write);
3496
3497 int is_psp_fw_valid(struct psp_bin_desc bin)
3498 {
3499         return bin.size_bytes;
3500 }
3501
3502 static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj,
3503                                         struct bin_attribute *bin_attr,
3504                                         char *buffer, loff_t pos, size_t count)
3505 {
3506         struct device *dev = kobj_to_dev(kobj);
3507         struct drm_device *ddev = dev_get_drvdata(dev);
3508         struct amdgpu_device *adev = drm_to_adev(ddev);
3509
3510         adev->psp.vbflash_done = false;
3511
3512         /* Safeguard against memory drain */
3513         if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) {
3514                 dev_err(adev->dev, "File size cannot exceed %u", AMD_VBIOS_FILE_MAX_SIZE_B);
3515                 kvfree(adev->psp.vbflash_tmp_buf);
3516                 adev->psp.vbflash_tmp_buf = NULL;
3517                 adev->psp.vbflash_image_size = 0;
3518                 return -ENOMEM;
3519         }
3520
3521         /* TODO Just allocate max for now and optimize to realloc later if needed */
3522         if (!adev->psp.vbflash_tmp_buf) {
3523                 adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL);
3524                 if (!adev->psp.vbflash_tmp_buf)
3525                         return -ENOMEM;
3526         }
3527
3528         mutex_lock(&adev->psp.mutex);
3529         memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count);
3530         adev->psp.vbflash_image_size += count;
3531         mutex_unlock(&adev->psp.mutex);
3532
3533         dev_info(adev->dev, "VBIOS flash write PSP done");
3534
3535         return count;
3536 }
3537
3538 static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj,
3539                                        struct bin_attribute *bin_attr, char *buffer,
3540                                        loff_t pos, size_t count)
3541 {
3542         struct device *dev = kobj_to_dev(kobj);
3543         struct drm_device *ddev = dev_get_drvdata(dev);
3544         struct amdgpu_device *adev = drm_to_adev(ddev);
3545         struct amdgpu_bo *fw_buf_bo = NULL;
3546         uint64_t fw_pri_mc_addr;
3547         void *fw_pri_cpu_addr;
3548         int ret;
3549
3550         dev_info(adev->dev, "VBIOS flash to PSP started");
3551
3552         ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size,
3553                                         AMDGPU_GPU_PAGE_SIZE,
3554                                         AMDGPU_GEM_DOMAIN_VRAM,
3555                                         &fw_buf_bo,
3556                                         &fw_pri_mc_addr,
3557                                         &fw_pri_cpu_addr);
3558         if (ret)
3559                 goto rel_buf;
3560
3561         memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf, adev->psp.vbflash_image_size);
3562
3563         mutex_lock(&adev->psp.mutex);
3564         ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
3565         mutex_unlock(&adev->psp.mutex);
3566
3567         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3568
3569 rel_buf:
3570         kvfree(adev->psp.vbflash_tmp_buf);
3571         adev->psp.vbflash_tmp_buf = NULL;
3572         adev->psp.vbflash_image_size = 0;
3573
3574         if (ret) {
3575                 dev_err(adev->dev, "Failed to load VBIOS FW, err = %d", ret);
3576                 return ret;
3577         }
3578
3579         dev_info(adev->dev, "VBIOS flash to PSP done");
3580         return 0;
3581 }
3582
3583 static ssize_t amdgpu_psp_vbflash_status(struct device *dev,
3584                                          struct device_attribute *attr,
3585                                          char *buf)
3586 {
3587         struct drm_device *ddev = dev_get_drvdata(dev);
3588         struct amdgpu_device *adev = drm_to_adev(ddev);
3589         uint32_t vbflash_status;
3590
3591         vbflash_status = psp_vbflash_status(&adev->psp);
3592         if (!adev->psp.vbflash_done)
3593                 vbflash_status = 0;
3594         else if (adev->psp.vbflash_done && !(vbflash_status & 0x80000000))
3595                 vbflash_status = 1;
3596
3597         return sysfs_emit(buf, "0x%x\n", vbflash_status);
3598 }
3599
3600 static const struct bin_attribute psp_vbflash_bin_attr = {
3601         .attr = {.name = "psp_vbflash", .mode = 0664},
3602         .size = 0,
3603         .write = amdgpu_psp_vbflash_write,
3604         .read = amdgpu_psp_vbflash_read,
3605 };
3606
3607 static DEVICE_ATTR(psp_vbflash_status, 0444, amdgpu_psp_vbflash_status, NULL);
3608
3609 int amdgpu_psp_sysfs_init(struct amdgpu_device *adev)
3610 {
3611         int ret = 0;
3612         struct psp_context *psp = &adev->psp;
3613
3614         if (amdgpu_sriov_vf(adev))
3615                 return -EINVAL;
3616
3617         switch (adev->ip_versions[MP0_HWIP][0]) {
3618         case IP_VERSION(13, 0, 0):
3619         case IP_VERSION(13, 0, 7):
3620                 if (!psp->adev) {
3621                         psp->adev = adev;
3622                         psp_v13_0_set_psp_funcs(psp);
3623                 }
3624                 ret = sysfs_create_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3625                 if (ret)
3626                         dev_err(adev->dev, "Failed to create device file psp_vbflash");
3627                 ret = device_create_file(adev->dev, &dev_attr_psp_vbflash_status);
3628                 if (ret)
3629                         dev_err(adev->dev, "Failed to create device file psp_vbflash_status");
3630                 return ret;
3631         default:
3632                 return 0;
3633         }
3634 }
3635
3636 const struct amd_ip_funcs psp_ip_funcs = {
3637         .name = "psp",
3638         .early_init = psp_early_init,
3639         .late_init = NULL,
3640         .sw_init = psp_sw_init,
3641         .sw_fini = psp_sw_fini,
3642         .hw_init = psp_hw_init,
3643         .hw_fini = psp_hw_fini,
3644         .suspend = psp_suspend,
3645         .resume = psp_resume,
3646         .is_idle = NULL,
3647         .check_soft_reset = NULL,
3648         .wait_for_idle = NULL,
3649         .soft_reset = NULL,
3650         .set_clockgating_state = psp_set_clockgating_state,
3651         .set_powergating_state = psp_set_powergating_state,
3652 };
3653
3654 static int psp_sysfs_init(struct amdgpu_device *adev)
3655 {
3656         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3657
3658         if (ret)
3659                 DRM_ERROR("Failed to create USBC PD FW control file!");
3660
3661         return ret;
3662 }
3663
3664 void amdgpu_psp_sysfs_fini(struct amdgpu_device *adev)
3665 {
3666         sysfs_remove_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3667         device_remove_file(adev->dev, &dev_attr_psp_vbflash_status);
3668 }
3669
3670 static void psp_sysfs_fini(struct amdgpu_device *adev)
3671 {
3672         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3673 }
3674
3675 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
3676 {
3677         .type = AMD_IP_BLOCK_TYPE_PSP,
3678         .major = 3,
3679         .minor = 1,
3680         .rev = 0,
3681         .funcs = &psp_ip_funcs,
3682 };
3683
3684 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
3685 {
3686         .type = AMD_IP_BLOCK_TYPE_PSP,
3687         .major = 10,
3688         .minor = 0,
3689         .rev = 0,
3690         .funcs = &psp_ip_funcs,
3691 };
3692
3693 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
3694 {
3695         .type = AMD_IP_BLOCK_TYPE_PSP,
3696         .major = 11,
3697         .minor = 0,
3698         .rev = 0,
3699         .funcs = &psp_ip_funcs,
3700 };
3701
3702 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3703         .type = AMD_IP_BLOCK_TYPE_PSP,
3704         .major = 11,
3705         .minor = 0,
3706         .rev = 8,
3707         .funcs = &psp_ip_funcs,
3708 };
3709
3710 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
3711 {
3712         .type = AMD_IP_BLOCK_TYPE_PSP,
3713         .major = 12,
3714         .minor = 0,
3715         .rev = 0,
3716         .funcs = &psp_ip_funcs,
3717 };
3718
3719 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3720         .type = AMD_IP_BLOCK_TYPE_PSP,
3721         .major = 13,
3722         .minor = 0,
3723         .rev = 0,
3724         .funcs = &psp_ip_funcs,
3725 };
3726
3727 const struct amdgpu_ip_block_version psp_v13_0_4_ip_block = {
3728         .type = AMD_IP_BLOCK_TYPE_PSP,
3729         .major = 13,
3730         .minor = 0,
3731         .rev = 4,
3732         .funcs = &psp_ip_funcs,
3733 };