drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40
  41 #include "amdgpu_ras.h"
  42 #include "amdgpu_securedisplay.h"
  43 #include "amdgpu_atomfirmware.h"
  44
  45 static int psp_sysfs_init(struct amdgpu_device *adev);
  46 static void psp_sysfs_fini(struct amdgpu_device *adev);
  47
  48 static int psp_load_smu_fw(struct psp_context *psp);
  49 static int psp_ta_unload(struct psp_context *psp, struct ta_context *context);
  50 static int psp_ta_load(struct psp_context *psp, struct ta_context *context);
  51 static int psp_rap_terminate(struct psp_context *psp);
  52 static int psp_securedisplay_terminate(struct psp_context *psp);
  53
  54 /*
  55  * Due to DF Cstate management centralized to PMFW, the firmware
  56  * loading sequence will be updated as below:
  57  *   - Load KDB
  58  *   - Load SYS_DRV
  59  *   - Load tOS
  60  *   - Load PMFW
  61  *   - Setup TMR
  62  *   - Load other non-psp fw
  63  *   - Load ASD
  64  *   - Load XGMI/RAS/HDCP/DTM TA if any
  65  *
  66  * This new sequence is required for
  67  *   - Arcturus and onwards
  68  */
  69 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  70 {
  71         struct amdgpu_device *adev = psp->adev;
  72
  73         if (amdgpu_sriov_vf(adev)) {
  74                 psp->pmfw_centralized_cstate_management = false;
  75                 return;
  76         }
  77
  78         switch (adev->ip_versions[MP0_HWIP][0]) {
  79         case IP_VERSION(11, 0, 0):
  80         case IP_VERSION(11, 0, 4):
  81         case IP_VERSION(11, 0, 5):
  82         case IP_VERSION(11, 0, 7):
  83         case IP_VERSION(11, 0, 9):
  84         case IP_VERSION(11, 0, 11):
  85         case IP_VERSION(11, 0, 12):
  86         case IP_VERSION(11, 0, 13):
  87         case IP_VERSION(13, 0, 2):
  88                 psp->pmfw_centralized_cstate_management = true;
  89                 break;
  90         default:
  91                 psp->pmfw_centralized_cstate_management = false;
  92                 break;
  93         }
  94 }
  95
  96 static int psp_early_init(void *handle)
  97 {
  98         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  99         struct psp_context *psp = &adev->psp;
 100
 101         switch (adev->ip_versions[MP0_HWIP][0]) {
 102         case IP_VERSION(9, 0, 0):
 103                 psp_v3_1_set_psp_funcs(psp);
 104                 psp->autoload_supported = false;
 105                 break;
 106         case IP_VERSION(10, 0, 0):
 107         case IP_VERSION(10, 0, 1):
 108                 psp_v10_0_set_psp_funcs(psp);
 109                 psp->autoload_supported = false;
 110                 break;
 111         case IP_VERSION(11, 0, 2):
 112         case IP_VERSION(11, 0, 4):
 113                 psp_v11_0_set_psp_funcs(psp);
 114                 psp->autoload_supported = false;
 115                 break;
 116         case IP_VERSION(11, 0, 0):
 117         case IP_VERSION(11, 0, 5):
 118         case IP_VERSION(11, 0, 9):
 119         case IP_VERSION(11, 0, 7):
 120         case IP_VERSION(11, 0, 11):
 121         case IP_VERSION(11, 5, 0):
 122         case IP_VERSION(11, 0, 12):
 123         case IP_VERSION(11, 0, 13):
 124                 psp_v11_0_set_psp_funcs(psp);
 125                 psp->autoload_supported = true;
 126                 break;
 127         case IP_VERSION(11, 0, 3):
 128         case IP_VERSION(12, 0, 1):
 129                 psp_v12_0_set_psp_funcs(psp);
 130                 break;
 131         case IP_VERSION(13, 0, 2):
 132                 psp_v13_0_set_psp_funcs(psp);
 133                 break;
 134         case IP_VERSION(13, 0, 1):
 135         case IP_VERSION(13, 0, 3):
 136                 psp_v13_0_set_psp_funcs(psp);
 137                 psp->autoload_supported = true;
 138                 break;
 139         case IP_VERSION(11, 0, 8):
 140                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 141                         psp_v11_0_8_set_psp_funcs(psp);
 142                         psp->autoload_supported = false;
 143                 }
 144                 break;
 145         default:
 146                 return -EINVAL;
 147         }
 148
 149         psp->adev = adev;
 150
 151         psp_check_pmfw_centralized_cstate_management(psp);
 152
 153         return 0;
 154 }
 155
 156 static void psp_memory_training_fini(struct psp_context *psp)
 157 {
 158         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 159
 160         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 161         kfree(ctx->sys_cache);
 162         ctx->sys_cache = NULL;
 163 }
 164
 165 static int psp_memory_training_init(struct psp_context *psp)
 166 {
 167         int ret;
 168         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 169
 170         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 171                 DRM_DEBUG("memory training is not supported!\n");
 172                 return 0;
 173         }
 174
 175         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 176         if (ctx->sys_cache == NULL) {
 177                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 178                 ret = -ENOMEM;
 179                 goto Err_out;
 180         }
 181
 182         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 183                   ctx->train_data_size,
 184                   ctx->p2c_train_data_offset,
 185                   ctx->c2p_train_data_offset);
 186         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 187         return 0;
 188
 189 Err_out:
 190         psp_memory_training_fini(psp);
 191         return ret;
 192 }
 193
 194 /*
 195  * Helper funciton to query psp runtime database entry
 196  *
 197  * @adev: amdgpu_device pointer
 198  * @entry_type: the type of psp runtime database entry
 199  * @db_entry: runtime database entry pointer
 200  *
 201  * Return false if runtime database doesn't exit or entry is invalid
 202  * or true if the specific database entry is found, and copy to @db_entry
 203  */
 204 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 205                                      enum psp_runtime_entry_type entry_type,
 206                                      void *db_entry)
 207 {
 208         uint64_t db_header_pos, db_dir_pos;
 209         struct psp_runtime_data_header db_header = {0};
 210         struct psp_runtime_data_directory db_dir = {0};
 211         bool ret = false;
 212         int i;
 213
 214         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 215         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 216
 217         /* read runtime db header from vram */
 218         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 219                         sizeof(struct psp_runtime_data_header), false);
 220
 221         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 222                 /* runtime db doesn't exist, exit */
 223                 dev_warn(adev->dev, "PSP runtime database doesn't exist\n");
 224                 return false;
 225         }
 226
 227         /* read runtime database entry from vram */
 228         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 229                         sizeof(struct psp_runtime_data_directory), false);
 230
 231         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 232                 /* invalid db entry count, exit */
 233                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 234                 return false;
 235         }
 236
 237         /* look up for requested entry type */
 238         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 239                 if (db_dir.entry_list[i].entry_type == entry_type) {
 240                         switch (entry_type) {
 241                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 242                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 243                                         /* invalid db entry size */
 244                                         dev_warn(adev->dev, "Invalid PSP runtime database entry size\n");
 245                                         return false;
 246                                 }
 247                                 /* read runtime database entry */
 248                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 249                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 250                                 ret = true;
 251                                 break;
 252                         default:
 253                                 ret = false;
 254                                 break;
 255                         }
 256                 }
 257         }
 258
 259         return ret;
 260 }
 261
 262 static int psp_sw_init(void *handle)
 263 {
 264         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 265         struct psp_context *psp = &adev->psp;
 266         int ret;
 267         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 268         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 269
 270         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 271         if (!psp->cmd) {
 272                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 273                 ret = -ENOMEM;
 274         }
 275
 276         if (!amdgpu_sriov_vf(adev)) {
 277                 ret = psp_init_microcode(psp);
 278                 if (ret) {
 279                         DRM_ERROR("Failed to load psp firmware!\n");
 280                         return ret;
 281                 }
 282         } else if (amdgpu_sriov_vf(adev) &&
 283                    adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2)) {
 284                 ret = psp_init_ta_microcode(psp, "aldebaran");
 285                 if (ret) {
 286                         DRM_ERROR("Failed to initialize ta microcode!\n");
 287                         return ret;
 288                 }
 289         }
 290
 291         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 292         if (psp_get_runtime_db_entry(adev,
 293                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 294                                 &boot_cfg_entry)) {
 295                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 296                 if ((psp->boot_cfg_bitmask) &
 297                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 298                         /* If psp runtime database exists, then
 299                          * only enable two stage memory training
 300                          * when TWO_STAGE_DRAM_TRAINING bit is set
 301                          * in runtime database */
 302                         mem_training_ctx->enable_mem_training = true;
 303                 }
 304
 305         } else {
 306                 /* If psp runtime database doesn't exist or
 307                  * is invalid, force enable two stage memory
 308                  * training */
 309                 mem_training_ctx->enable_mem_training = true;
 310         }
 311
 312         if (mem_training_ctx->enable_mem_training) {
 313                 ret = psp_memory_training_init(psp);
 314                 if (ret) {
 315                         DRM_ERROR("Failed to initialize memory training!\n");
 316                         return ret;
 317                 }
 318
 319                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 320                 if (ret) {
 321                         DRM_ERROR("Failed to process memory training!\n");
 322                         return ret;
 323                 }
 324         }
 325
 326         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 327             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 328                 ret= psp_sysfs_init(adev);
 329                 if (ret) {
 330                         return ret;
 331                 }
 332         }
 333
 334         return 0;
 335 }
 336
 337 static int psp_sw_fini(void *handle)
 338 {
 339         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 340         struct psp_context *psp = &adev->psp;
 341         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 342
 343         psp_memory_training_fini(psp);
 344         if (psp->sos_fw) {
 345                 release_firmware(psp->sos_fw);
 346                 psp->sos_fw = NULL;
 347         }
 348         if (psp->asd_fw) {
 349                 release_firmware(psp->asd_fw);
 350                 psp->asd_fw = NULL;
 351         }
 352         if (psp->ta_fw) {
 353                 release_firmware(psp->ta_fw);
 354                 psp->ta_fw = NULL;
 355         }
 356
 357         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 358             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 359                 psp_sysfs_fini(adev);
 360
 361         kfree(cmd);
 362         cmd = NULL;
 363
 364         return 0;
 365 }
 366
 367 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 368                  uint32_t reg_val, uint32_t mask, bool check_changed)
 369 {
 370         uint32_t val;
 371         int i;
 372         struct amdgpu_device *adev = psp->adev;
 373
 374         if (psp->adev->no_hw_access)
 375                 return 0;
 376
 377         for (i = 0; i < adev->usec_timeout; i++) {
 378                 val = RREG32(reg_index);
 379                 if (check_changed) {
 380                         if (val != reg_val)
 381                                 return 0;
 382                 } else {
 383                         if ((val & mask) == reg_val)
 384                                 return 0;
 385                 }
 386                 udelay(1);
 387         }
 388
 389         return -ETIME;
 390 }
 391
 392 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 393 {
 394         switch (cmd_id) {
 395         case GFX_CMD_ID_LOAD_TA:
 396                 return "LOAD_TA";
 397         case GFX_CMD_ID_UNLOAD_TA:
 398                 return "UNLOAD_TA";
 399         case GFX_CMD_ID_INVOKE_CMD:
 400                 return "INVOKE_CMD";
 401         case GFX_CMD_ID_LOAD_ASD:
 402                 return "LOAD_ASD";
 403         case GFX_CMD_ID_SETUP_TMR:
 404                 return "SETUP_TMR";
 405         case GFX_CMD_ID_LOAD_IP_FW:
 406                 return "LOAD_IP_FW";
 407         case GFX_CMD_ID_DESTROY_TMR:
 408                 return "DESTROY_TMR";
 409         case GFX_CMD_ID_SAVE_RESTORE:
 410                 return "SAVE_RESTORE_IP_FW";
 411         case GFX_CMD_ID_SETUP_VMR:
 412                 return "SETUP_VMR";
 413         case GFX_CMD_ID_DESTROY_VMR:
 414                 return "DESTROY_VMR";
 415         case GFX_CMD_ID_PROG_REG:
 416                 return "PROG_REG";
 417         case GFX_CMD_ID_GET_FW_ATTESTATION:
 418                 return "GET_FW_ATTESTATION";
 419         case GFX_CMD_ID_LOAD_TOC:
 420                 return "ID_LOAD_TOC";
 421         case GFX_CMD_ID_AUTOLOAD_RLC:
 422                 return "AUTOLOAD_RLC";
 423         case GFX_CMD_ID_BOOT_CFG:
 424                 return "BOOT_CFG";
 425         default:
 426                 return "UNKNOWN CMD";
 427         }
 428 }
 429
 430 static int
 431 psp_cmd_submit_buf(struct psp_context *psp,
 432                    struct amdgpu_firmware_info *ucode,
 433                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 434 {
 435         int ret;
 436         int index, idx;
 437         int timeout = 20000;
 438         bool ras_intr = false;
 439         bool skip_unsupport = false;
 440
 441         if (psp->adev->no_hw_access)
 442                 return 0;
 443
 444         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
 445                 return 0;
 446
 447         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 448
 449         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 450
 451         index = atomic_inc_return(&psp->fence_value);
 452         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 453         if (ret) {
 454                 atomic_dec(&psp->fence_value);
 455                 goto exit;
 456         }
 457
 458         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 459         while (*((unsigned int *)psp->fence_buf) != index) {
 460                 if (--timeout == 0)
 461                         break;
 462                 /*
 463                  * Shouldn't wait for timeout when err_event_athub occurs,
 464                  * because gpu reset thread triggered and lock resource should
 465                  * be released for psp resume sequence.
 466                  */
 467                 ras_intr = amdgpu_ras_intr_triggered();
 468                 if (ras_intr)
 469                         break;
 470                 usleep_range(10, 100);
 471                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 472         }
 473
 474         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 475         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 476                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 477
 478         memcpy((void*)&cmd->resp, (void*)&psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 479
 480         /* In some cases, psp response status is not 0 even there is no
 481          * problem while the command is submitted. Some version of PSP FW
 482          * doesn't write 0 to that field.
 483          * So here we would like to only print a warning instead of an error
 484          * during psp initialization to avoid breaking hw_init and it doesn't
 485          * return -EINVAL.
 486          */
 487         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 488                 if (ucode)
 489                         DRM_WARN("failed to load ucode %s(0x%X) ",
 490                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 491                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 492                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 493                          psp->cmd_buf_mem->resp.status);
 494                 if (!timeout) {
 495                         ret = -EINVAL;
 496                         goto exit;
 497                 }
 498         }
 499
 500         if (ucode) {
 501                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 502                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 503         }
 504
 505 exit:
 506         drm_dev_exit(idx);
 507         return ret;
 508 }
 509
 510 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 511 {
 512         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 513
 514         mutex_lock(&psp->mutex);
 515
 516         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 517
 518         return cmd;
 519 }
 520
 521 static void release_psp_cmd_buf(struct psp_context *psp)
 522 {
 523         mutex_unlock(&psp->mutex);
 524 }
 525
 526 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 527                                  struct psp_gfx_cmd_resp *cmd,
 528                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 529 {
 530         struct amdgpu_device *adev = psp->adev;
 531         uint32_t size = amdgpu_bo_size(tmr_bo);
 532         uint64_t tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 533
 534         if (amdgpu_sriov_vf(psp->adev))
 535                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 536         else
 537                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 538         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 539         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 540         cmd->cmd.cmd_setup_tmr.buf_size = size;
 541         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 542         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 543         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 544 }
 545
 546 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 547                                       uint64_t pri_buf_mc, uint32_t size)
 548 {
 549         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 550         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 551         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 552         cmd->cmd.cmd_load_toc.toc_size = size;
 553 }
 554
 555 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 556 static int psp_load_toc(struct psp_context *psp,
 557                         uint32_t *tmr_size)
 558 {
 559         int ret;
 560         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 561
 562         /* Copy toc to psp firmware private buffer */
 563         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 564
 565         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 566
 567         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 568                                  psp->fence_buf_mc_addr);
 569         if (!ret)
 570                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 571
 572         release_psp_cmd_buf(psp);
 573
 574         return ret;
 575 }
 576
 577 /* Set up Trusted Memory Region */
 578 static int psp_tmr_init(struct psp_context *psp)
 579 {
 580         int ret;
 581         int tmr_size;
 582         void *tmr_buf;
 583         void **pptr;
 584
 585         /*
 586          * According to HW engineer, they prefer the TMR address be "naturally
 587          * aligned" , e.g. the start address be an integer divide of TMR size.
 588          *
 589          * Note: this memory need be reserved till the driver
 590          * uninitializes.
 591          */
 592         tmr_size = PSP_TMR_SIZE(psp->adev);
 593
 594         /* For ASICs support RLC autoload, psp will parse the toc
 595          * and calculate the total size of TMR needed */
 596         if (!amdgpu_sriov_vf(psp->adev) &&
 597             psp->toc.start_addr &&
 598             psp->toc.size_bytes &&
 599             psp->fw_pri_buf) {
 600                 ret = psp_load_toc(psp, &tmr_size);
 601                 if (ret) {
 602                         DRM_ERROR("Failed to load toc\n");
 603                         return ret;
 604                 }
 605         }
 606
 607         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 608         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE(psp->adev),
 609                                       AMDGPU_GEM_DOMAIN_VRAM,
 610                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 611
 612         return ret;
 613 }
 614
 615 static bool psp_skip_tmr(struct psp_context *psp)
 616 {
 617         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 618         case IP_VERSION(11, 0, 9):
 619         case IP_VERSION(11, 0, 7):
 620         case IP_VERSION(13, 0, 2):
 621                 return true;
 622         default:
 623                 return false;
 624         }
 625 }
 626
 627 static int psp_tmr_load(struct psp_context *psp)
 628 {
 629         int ret;
 630         struct psp_gfx_cmd_resp *cmd;
 631
 632         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 633          * Already set up by host driver.
 634          */
 635         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 636                 return 0;
 637
 638         cmd = acquire_psp_cmd_buf(psp);
 639
 640         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 641         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 642                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 643
 644         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 645                                  psp->fence_buf_mc_addr);
 646
 647         release_psp_cmd_buf(psp);
 648
 649         return ret;
 650 }
 651
 652 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 653                                         struct psp_gfx_cmd_resp *cmd)
 654 {
 655         if (amdgpu_sriov_vf(psp->adev))
 656                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 657         else
 658                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 659 }
 660
 661 static int psp_tmr_unload(struct psp_context *psp)
 662 {
 663         int ret;
 664         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 665
 666         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 667         DRM_INFO("free PSP TMR buffer\n");
 668
 669         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 670                                  psp->fence_buf_mc_addr);
 671
 672         release_psp_cmd_buf(psp);
 673
 674         return ret;
 675 }
 676
 677 static int psp_tmr_terminate(struct psp_context *psp)
 678 {
 679         int ret;
 680         void *tmr_buf;
 681         void **pptr;
 682
 683         ret = psp_tmr_unload(psp);
 684         if (ret)
 685                 return ret;
 686
 687         /* free TMR memory buffer */
 688         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 689         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 690
 691         return 0;
 692 }
 693
 694 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 695                                         uint64_t *output_ptr)
 696 {
 697         int ret;
 698         struct psp_gfx_cmd_resp *cmd;
 699
 700         if (!output_ptr)
 701                 return -EINVAL;
 702
 703         if (amdgpu_sriov_vf(psp->adev))
 704                 return 0;
 705
 706         cmd = acquire_psp_cmd_buf(psp);
 707
 708         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 709
 710         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 711                                  psp->fence_buf_mc_addr);
 712
 713         if (!ret) {
 714                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 715                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 716         }
 717
 718         release_psp_cmd_buf(psp);
 719
 720         return ret;
 721 }
 722
 723 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 724 {
 725         struct psp_context *psp = &adev->psp;
 726         struct psp_gfx_cmd_resp *cmd;
 727         int ret;
 728
 729         if (amdgpu_sriov_vf(adev))
 730                 return 0;
 731
 732         cmd = acquire_psp_cmd_buf(psp);
 733
 734         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 735         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 736
 737         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 738         if (!ret) {
 739                 *boot_cfg =
 740                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 741         }
 742
 743         release_psp_cmd_buf(psp);
 744
 745         return ret;
 746 }
 747
 748 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 749 {
 750         int ret;
 751         struct psp_context *psp = &adev->psp;
 752         struct psp_gfx_cmd_resp *cmd;
 753
 754         if (amdgpu_sriov_vf(adev))
 755                 return 0;
 756
 757         cmd = acquire_psp_cmd_buf(psp);
 758
 759         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 760         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 761         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 762         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 763
 764         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 765
 766         release_psp_cmd_buf(psp);
 767
 768         return ret;
 769 }
 770
 771 static int psp_rl_load(struct amdgpu_device *adev)
 772 {
 773         int ret;
 774         struct psp_context *psp = &adev->psp;
 775         struct psp_gfx_cmd_resp *cmd;
 776
 777         if (!is_psp_fw_valid(psp->rl))
 778                 return 0;
 779
 780         cmd = acquire_psp_cmd_buf(psp);
 781
 782         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 783         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
 784
 785         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 786         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
 787         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
 788         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
 789         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
 790
 791         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 792
 793         release_psp_cmd_buf(psp);
 794
 795         return ret;
 796 }
 797
 798 static int psp_asd_load(struct psp_context *psp)
 799 {
 800         return psp_ta_load(psp, &psp->asd_context);
 801 }
 802
 803 static int psp_asd_initialize(struct psp_context *psp)
 804 {
 805         int ret;
 806
 807         /* If PSP version doesn't match ASD version, asd loading will be failed.
 808          * add workaround to bypass it for sriov now.
 809          * TODO: add version check to make it common
 810          */
 811         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 812                 return 0;
 813
 814         psp->asd_context.mem_context.shared_mc_addr  = 0;
 815         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 816         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
 817
 818         ret = psp_asd_load(psp);
 819         if (!ret)
 820                 psp->asd_context.initialized = true;
 821
 822         return ret;
 823 }
 824
 825 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 826                                        uint32_t session_id)
 827 {
 828         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 829         cmd->cmd.cmd_unload_ta.session_id = session_id;
 830 }
 831
 832 static int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
 833 {
 834         int ret;
 835         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 836
 837         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
 838
 839         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 840
 841         release_psp_cmd_buf(psp);
 842
 843         return ret;
 844 }
 845
 846 static int psp_asd_unload(struct psp_context *psp)
 847 {
 848         return psp_ta_unload(psp, &psp->asd_context);
 849 }
 850
 851 static int psp_asd_terminate(struct psp_context *psp)
 852 {
 853         int ret;
 854
 855         if (amdgpu_sriov_vf(psp->adev))
 856                 return 0;
 857
 858         if (!psp->asd_context.initialized)
 859                 return 0;
 860
 861         ret = psp_asd_unload(psp);
 862
 863         if (!ret)
 864                 psp->asd_context.initialized = false;
 865
 866         return ret;
 867 }
 868
 869 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 870                 uint32_t id, uint32_t value)
 871 {
 872         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 873         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 874         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 875 }
 876
 877 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 878                 uint32_t value)
 879 {
 880         struct psp_gfx_cmd_resp *cmd;
 881         int ret = 0;
 882
 883         if (reg >= PSP_REG_LAST)
 884                 return -EINVAL;
 885
 886         cmd = acquire_psp_cmd_buf(psp);
 887
 888         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 889         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 890         if (ret)
 891                 DRM_ERROR("PSP failed to program reg id %d", reg);
 892
 893         release_psp_cmd_buf(psp);
 894
 895         return ret;
 896 }
 897
 898 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 899                                      uint64_t ta_bin_mc,
 900                                      struct ta_context *context)
 901 {
 902         cmd->cmd_id                             = context->ta_load_type;
 903         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 904         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 905         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
 906
 907         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
 908                 lower_32_bits(context->mem_context.shared_mc_addr);
 909         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
 910                 upper_32_bits(context->mem_context.shared_mc_addr);
 911         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
 912 }
 913
 914 static int psp_ta_init_shared_buf(struct psp_context *psp,
 915                                   struct ta_mem_context *mem_ctx)
 916 {
 917         int ret;
 918
 919         /*
 920         * Allocate 16k memory aligned to 4k from Frame Buffer (local
 921         * physical) for ta to host memory
 922         */
 923         ret = amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
 924                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 925                                       &mem_ctx->shared_bo,
 926                                       &mem_ctx->shared_mc_addr,
 927                                       &mem_ctx->shared_buf);
 928
 929         return ret;
 930 }
 931
 932 static void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 933 {
 934         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 935                               &mem_ctx->shared_buf);
 936 }
 937
 938 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 939 {
 940         return psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
 941 }
 942
 943 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 944                                        uint32_t ta_cmd_id,
 945                                        uint32_t session_id)
 946 {
 947         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 948         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 949         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 950 }
 951
 952 static int psp_ta_invoke(struct psp_context *psp,
 953                   uint32_t ta_cmd_id,
 954                   struct ta_context *context)
 955 {
 956         int ret;
 957         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 958
 959         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
 960
 961         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 962                                  psp->fence_buf_mc_addr);
 963
 964         release_psp_cmd_buf(psp);
 965
 966         return ret;
 967 }
 968
 969 static int psp_ta_load(struct psp_context *psp, struct ta_context *context)
 970 {
 971         int ret;
 972         struct psp_gfx_cmd_resp *cmd;
 973
 974         cmd = acquire_psp_cmd_buf(psp);
 975
 976         psp_copy_fw(psp, context->bin_desc.start_addr,
 977                     context->bin_desc.size_bytes);
 978
 979         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
 980
 981         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 982                                  psp->fence_buf_mc_addr);
 983
 984         if (!ret) {
 985                 context->session_id = cmd->resp.session_id;
 986         }
 987
 988         release_psp_cmd_buf(psp);
 989
 990         return ret;
 991 }
 992
 993 static int psp_xgmi_load(struct psp_context *psp)
 994 {
 995         return psp_ta_load(psp, &psp->xgmi_context.context);
 996 }
 997
 998 static int psp_xgmi_unload(struct psp_context *psp)
 999 {
1000         return psp_ta_unload(psp, &psp->xgmi_context.context);
1001 }
1002
1003 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1004 {
1005         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1006 }
1007
1008 int psp_xgmi_terminate(struct psp_context *psp)
1009 {
1010         int ret;
1011         struct amdgpu_device *adev = psp->adev;
1012
1013         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1014         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1015             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1016              adev->gmc.xgmi.connected_to_cpu))
1017                 return 0;
1018
1019         if (!psp->xgmi_context.context.initialized)
1020                 return 0;
1021
1022         ret = psp_xgmi_unload(psp);
1023         if (ret)
1024                 return ret;
1025
1026         psp->xgmi_context.context.initialized = false;
1027
1028         /* free xgmi shared memory */
1029         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
1030
1031         return 0;
1032 }
1033
1034 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1035 {
1036         struct ta_xgmi_shared_memory *xgmi_cmd;
1037         int ret;
1038
1039         if (!psp->ta_fw ||
1040             !psp->xgmi_context.context.bin_desc.size_bytes ||
1041             !psp->xgmi_context.context.bin_desc.start_addr)
1042                 return -ENOENT;
1043
1044         if (!load_ta)
1045                 goto invoke;
1046
1047         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1048         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1049
1050         if (!psp->xgmi_context.context.initialized) {
1051                 ret = psp_xgmi_init_shared_buf(psp);
1052                 if (ret)
1053                         return ret;
1054         }
1055
1056         /* Load XGMI TA */
1057         ret = psp_xgmi_load(psp);
1058         if (!ret)
1059                 psp->xgmi_context.context.initialized = true;
1060         else
1061                 return ret;
1062
1063 invoke:
1064         /* Initialize XGMI session */
1065         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1066         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1067         xgmi_cmd->flag_extend_link_record = set_extended_data;
1068         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1069
1070         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1071
1072         return ret;
1073 }
1074
1075 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1076 {
1077         struct ta_xgmi_shared_memory *xgmi_cmd;
1078         int ret;
1079
1080         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1081         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1082
1083         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1084
1085         /* Invoke xgmi ta to get hive id */
1086         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1087         if (ret)
1088                 return ret;
1089
1090         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1091
1092         return 0;
1093 }
1094
1095 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1096 {
1097         struct ta_xgmi_shared_memory *xgmi_cmd;
1098         int ret;
1099
1100         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1101         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1102
1103         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1104
1105         /* Invoke xgmi ta to get the node id */
1106         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1107         if (ret)
1108                 return ret;
1109
1110         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1111
1112         return 0;
1113 }
1114
1115 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1116 {
1117         return psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1118                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b;
1119 }
1120
1121 /*
1122  * Chips that support extended topology information require the driver to
1123  * reflect topology information in the opposite direction.  This is
1124  * because the TA has already exceeded its link record limit and if the
1125  * TA holds bi-directional information, the driver would have to do
1126  * multiple fetches instead of just two.
1127  */
1128 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1129                                         struct psp_xgmi_node_info node_info)
1130 {
1131         struct amdgpu_device *mirror_adev;
1132         struct amdgpu_hive_info *hive;
1133         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1134         uint64_t dst_node_id = node_info.node_id;
1135         uint8_t dst_num_hops = node_info.num_hops;
1136         uint8_t dst_num_links = node_info.num_links;
1137
1138         hive = amdgpu_get_xgmi_hive(psp->adev);
1139         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1140                 struct psp_xgmi_topology_info *mirror_top_info;
1141                 int j;
1142
1143                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1144                         continue;
1145
1146                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1147                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1148                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1149                                 continue;
1150
1151                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1152                         /*
1153                          * prevent 0 num_links value re-reflection since reflection
1154                          * criteria is based on num_hops (direct or indirect).
1155                          *
1156                          */
1157                         if (dst_num_links)
1158                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1159
1160                         break;
1161                 }
1162
1163                 break;
1164         }
1165 }
1166
1167 int psp_xgmi_get_topology_info(struct psp_context *psp,
1168                                int number_devices,
1169                                struct psp_xgmi_topology_info *topology,
1170                                bool get_extended_data)
1171 {
1172         struct ta_xgmi_shared_memory *xgmi_cmd;
1173         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1174         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1175         int i;
1176         int ret;
1177
1178         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1179                 return -EINVAL;
1180
1181         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1182         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1183         xgmi_cmd->flag_extend_link_record = get_extended_data;
1184
1185         /* Fill in the shared memory with topology information as input */
1186         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1187         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1188         topology_info_input->num_nodes = number_devices;
1189
1190         for (i = 0; i < topology_info_input->num_nodes; i++) {
1191                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1192                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1193                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1194                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1195         }
1196
1197         /* Invoke xgmi ta to get the topology information */
1198         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1199         if (ret)
1200                 return ret;
1201
1202         /* Read the output topology information from the shared memory */
1203         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1204         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1205         for (i = 0; i < topology->num_nodes; i++) {
1206                 /* extended data will either be 0 or equal to non-extended data */
1207                 if (topology_info_output->nodes[i].num_hops)
1208                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1209
1210                 /* non-extended data gets everything here so no need to update */
1211                 if (!get_extended_data) {
1212                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1213                         topology->nodes[i].is_sharing_enabled =
1214                                         topology_info_output->nodes[i].is_sharing_enabled;
1215                         topology->nodes[i].sdma_engine =
1216                                         topology_info_output->nodes[i].sdma_engine;
1217                 }
1218
1219         }
1220
1221         /* Invoke xgmi ta again to get the link information */
1222         if (psp_xgmi_peer_link_info_supported(psp)) {
1223                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1224
1225                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1226
1227                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1228
1229                 if (ret)
1230                         return ret;
1231
1232                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1233                 for (i = 0; i < topology->num_nodes; i++) {
1234                         /* accumulate num_links on extended data */
1235                         topology->nodes[i].num_links = get_extended_data ?
1236                                         topology->nodes[i].num_links +
1237                                                         link_info_output->nodes[i].num_links :
1238                                         link_info_output->nodes[i].num_links;
1239
1240                         /* reflect the topology information for bi-directionality */
1241                         if (psp->xgmi_context.supports_extended_data &&
1242                                         get_extended_data && topology->nodes[i].num_hops)
1243                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1244                 }
1245         }
1246
1247         return 0;
1248 }
1249
1250 int psp_xgmi_set_topology_info(struct psp_context *psp,
1251                                int number_devices,
1252                                struct psp_xgmi_topology_info *topology)
1253 {
1254         struct ta_xgmi_shared_memory *xgmi_cmd;
1255         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1256         int i;
1257
1258         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1259                 return -EINVAL;
1260
1261         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1262         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1263
1264         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1265         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1266         topology_info_input->num_nodes = number_devices;
1267
1268         for (i = 0; i < topology_info_input->num_nodes; i++) {
1269                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1270                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1271                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1272                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1273         }
1274
1275         /* Invoke xgmi ta to set topology information */
1276         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1277 }
1278
1279 // ras begin
1280 static int psp_ras_init_shared_buf(struct psp_context *psp)
1281 {
1282         return psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1283 }
1284
1285 static int psp_ras_load(struct psp_context *psp)
1286 {
1287         return psp_ta_load(psp, &psp->ras_context.context);
1288 }
1289
1290 static int psp_ras_unload(struct psp_context *psp)
1291 {
1292         return psp_ta_unload(psp, &psp->ras_context.context);
1293 }
1294
1295 static void psp_ras_ta_check_status(struct psp_context *psp)
1296 {
1297         struct ta_ras_shared_memory *ras_cmd =
1298                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1299
1300         switch (ras_cmd->ras_status) {
1301         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1302                 dev_warn(psp->adev->dev,
1303                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1304                 break;
1305         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1306                 dev_warn(psp->adev->dev,
1307                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1308                 break;
1309         case TA_RAS_STATUS__SUCCESS:
1310                 break;
1311         default:
1312                 dev_warn(psp->adev->dev,
1313                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1314                 break;
1315         }
1316 }
1317
1318 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1319 {
1320         struct ta_ras_shared_memory *ras_cmd;
1321         int ret;
1322
1323         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1324
1325         /*
1326          * TODO: bypass the loading in sriov for now
1327          */
1328         if (amdgpu_sriov_vf(psp->adev))
1329                 return 0;
1330
1331         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1332
1333         if (amdgpu_ras_intr_triggered())
1334                 return ret;
1335
1336         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1337         {
1338                 DRM_WARN("RAS: Unsupported Interface");
1339                 return -EINVAL;
1340         }
1341
1342         if (!ret) {
1343                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1344                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1345
1346                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1347                 }
1348                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1349                         dev_warn(psp->adev->dev,
1350                                  "RAS internal register access blocked\n");
1351
1352                 psp_ras_ta_check_status(psp);
1353         }
1354
1355         return ret;
1356 }
1357
1358 int psp_ras_enable_features(struct psp_context *psp,
1359                 union ta_ras_cmd_input *info, bool enable)
1360 {
1361         struct ta_ras_shared_memory *ras_cmd;
1362         int ret;
1363
1364         if (!psp->ras_context.context.initialized)
1365                 return -EINVAL;
1366
1367         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1368         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1369
1370         if (enable)
1371                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1372         else
1373                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1374
1375         ras_cmd->ras_in_message = *info;
1376
1377         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1378         if (ret)
1379                 return -EINVAL;
1380
1381         return 0;
1382 }
1383
1384 static int psp_ras_terminate(struct psp_context *psp)
1385 {
1386         int ret;
1387
1388         /*
1389          * TODO: bypass the terminate in sriov for now
1390          */
1391         if (amdgpu_sriov_vf(psp->adev))
1392                 return 0;
1393
1394         if (!psp->ras_context.context.initialized)
1395                 return 0;
1396
1397         ret = psp_ras_unload(psp);
1398         if (ret)
1399                 return ret;
1400
1401         psp->ras_context.context.initialized = false;
1402
1403         /* free ras shared memory */
1404         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
1405
1406         return 0;
1407 }
1408
1409 static int psp_ras_initialize(struct psp_context *psp)
1410 {
1411         int ret;
1412         uint32_t boot_cfg = 0xFF;
1413         struct amdgpu_device *adev = psp->adev;
1414         struct ta_ras_shared_memory *ras_cmd;
1415
1416         /*
1417          * TODO: bypass the initialize in sriov for now
1418          */
1419         if (amdgpu_sriov_vf(adev))
1420                 return 0;
1421
1422         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1423             !adev->psp.ras_context.context.bin_desc.start_addr) {
1424                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1425                 return 0;
1426         }
1427
1428         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1429                 /* query GECC enablement status from boot config
1430                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1431                  */
1432                 ret = psp_boot_config_get(adev, &boot_cfg);
1433                 if (ret)
1434                         dev_warn(adev->dev, "PSP get boot config failed\n");
1435
1436                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1437                         if (!boot_cfg) {
1438                                 dev_info(adev->dev, "GECC is disabled\n");
1439                         } else {
1440                                 /* disable GECC in next boot cycle if ras is
1441                                  * disabled by module parameter amdgpu_ras_enable
1442                                  * and/or amdgpu_ras_mask, or boot_config_get call
1443                                  * is failed
1444                                  */
1445                                 ret = psp_boot_config_set(adev, 0);
1446                                 if (ret)
1447                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1448                                 else
1449                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle "
1450                                                  "if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1451                         }
1452                 } else {
1453                         if (1 == boot_cfg) {
1454                                 dev_info(adev->dev, "GECC is enabled\n");
1455                         } else {
1456                                 /* enable GECC in next boot cycle if it is disabled
1457                                  * in boot config, or force enable GECC if failed to
1458                                  * get boot configuration
1459                                  */
1460                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1461                                 if (ret)
1462                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1463                                 else
1464                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1465                         }
1466                 }
1467         }
1468
1469         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1470         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1471
1472         if (!psp->ras_context.context.initialized) {
1473                 ret = psp_ras_init_shared_buf(psp);
1474                 if (ret)
1475                         return ret;
1476         }
1477
1478         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1479         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1480
1481         if (amdgpu_ras_is_poison_mode_supported(adev))
1482                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1483         if (!adev->gmc.xgmi.connected_to_cpu)
1484                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1485
1486         ret = psp_ras_load(psp);
1487
1488         if (!ret && !ras_cmd->ras_status)
1489                 psp->ras_context.context.initialized = true;
1490         else {
1491                 if (ras_cmd->ras_status)
1492                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1493                 amdgpu_ras_fini(psp->adev);
1494         }
1495
1496         return ret;
1497 }
1498
1499 int psp_ras_trigger_error(struct psp_context *psp,
1500                           struct ta_ras_trigger_error_input *info)
1501 {
1502         struct ta_ras_shared_memory *ras_cmd;
1503         int ret;
1504
1505         if (!psp->ras_context.context.initialized)
1506                 return -EINVAL;
1507
1508         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1509         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1510
1511         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1512         ras_cmd->ras_in_message.trigger_error = *info;
1513
1514         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1515         if (ret)
1516                 return -EINVAL;
1517
1518         /* If err_event_athub occurs error inject was successful, however
1519            return status from TA is no long reliable */
1520         if (amdgpu_ras_intr_triggered())
1521                 return 0;
1522
1523         if (ras_cmd->ras_status)
1524                 return -EINVAL;
1525
1526         return 0;
1527 }
1528 // ras end
1529
1530 // HDCP start
1531 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
1532 {
1533         return psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1534 }
1535
1536 static int psp_hdcp_load(struct psp_context *psp)
1537 {
1538         return psp_ta_load(psp, &psp->hdcp_context.context);
1539 }
1540
1541 static int psp_hdcp_initialize(struct psp_context *psp)
1542 {
1543         int ret;
1544
1545         /*
1546          * TODO: bypass the initialize in sriov for now
1547          */
1548         if (amdgpu_sriov_vf(psp->adev))
1549                 return 0;
1550
1551         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1552             !psp->hdcp_context.context.bin_desc.start_addr) {
1553                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1554                 return 0;
1555         }
1556
1557         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1558         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1559
1560         if (!psp->hdcp_context.context.initialized) {
1561                 ret = psp_hdcp_init_shared_buf(psp);
1562                 if (ret)
1563                         return ret;
1564         }
1565
1566         ret = psp_hdcp_load(psp);
1567         if (!ret) {
1568                 psp->hdcp_context.context.initialized = true;
1569                 mutex_init(&psp->hdcp_context.mutex);
1570         }
1571
1572         return ret;
1573 }
1574
1575 static int psp_hdcp_unload(struct psp_context *psp)
1576 {
1577         return psp_ta_unload(psp, &psp->hdcp_context.context);
1578 }
1579
1580 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1581 {
1582         /*
1583          * TODO: bypass the loading in sriov for now
1584          */
1585         if (amdgpu_sriov_vf(psp->adev))
1586                 return 0;
1587
1588         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1589 }
1590
1591 static int psp_hdcp_terminate(struct psp_context *psp)
1592 {
1593         int ret;
1594
1595         /*
1596          * TODO: bypass the terminate in sriov for now
1597          */
1598         if (amdgpu_sriov_vf(psp->adev))
1599                 return 0;
1600
1601         if (!psp->hdcp_context.context.initialized) {
1602                 if (psp->hdcp_context.context.mem_context.shared_buf)
1603                         goto out;
1604                 else
1605                         return 0;
1606         }
1607
1608         ret = psp_hdcp_unload(psp);
1609         if (ret)
1610                 return ret;
1611
1612         psp->hdcp_context.context.initialized = false;
1613
1614 out:
1615         /* free hdcp shared memory */
1616         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
1617
1618         return 0;
1619 }
1620 // HDCP end
1621
1622 // DTM start
1623 static int psp_dtm_init_shared_buf(struct psp_context *psp)
1624 {
1625         return psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1626 }
1627
1628 static int psp_dtm_load(struct psp_context *psp)
1629 {
1630         return psp_ta_load(psp, &psp->dtm_context.context);
1631 }
1632
1633 static int psp_dtm_initialize(struct psp_context *psp)
1634 {
1635         int ret;
1636
1637         /*
1638          * TODO: bypass the initialize in sriov for now
1639          */
1640         if (amdgpu_sriov_vf(psp->adev))
1641                 return 0;
1642
1643         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1644             !psp->dtm_context.context.bin_desc.start_addr) {
1645                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1646                 return 0;
1647         }
1648
1649         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1650         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1651
1652         if (!psp->dtm_context.context.initialized) {
1653                 ret = psp_dtm_init_shared_buf(psp);
1654                 if (ret)
1655                         return ret;
1656         }
1657
1658         ret = psp_dtm_load(psp);
1659         if (!ret) {
1660                 psp->dtm_context.context.initialized = true;
1661                 mutex_init(&psp->dtm_context.mutex);
1662         }
1663
1664         return ret;
1665 }
1666
1667 static int psp_dtm_unload(struct psp_context *psp)
1668 {
1669         return psp_ta_unload(psp, &psp->dtm_context.context);
1670 }
1671
1672 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1673 {
1674         /*
1675          * TODO: bypass the loading in sriov for now
1676          */
1677         if (amdgpu_sriov_vf(psp->adev))
1678                 return 0;
1679
1680         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1681 }
1682
1683 static int psp_dtm_terminate(struct psp_context *psp)
1684 {
1685         int ret;
1686
1687         /*
1688          * TODO: bypass the terminate in sriov for now
1689          */
1690         if (amdgpu_sriov_vf(psp->adev))
1691                 return 0;
1692
1693         if (!psp->dtm_context.context.initialized) {
1694                 if (psp->dtm_context.context.mem_context.shared_buf)
1695                         goto out;
1696                 else
1697                         return 0;
1698         }
1699
1700         ret = psp_dtm_unload(psp);
1701         if (ret)
1702                 return ret;
1703
1704         psp->dtm_context.context.initialized = false;
1705
1706 out:
1707         /* free dtm shared memory */
1708         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
1709
1710         return 0;
1711 }
1712 // DTM end
1713
1714 // RAP start
1715 static int psp_rap_init_shared_buf(struct psp_context *psp)
1716 {
1717         return psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1718 }
1719
1720 static int psp_rap_load(struct psp_context *psp)
1721 {
1722         return psp_ta_load(psp, &psp->rap_context.context);
1723 }
1724
1725 static int psp_rap_unload(struct psp_context *psp)
1726 {
1727         return psp_ta_unload(psp, &psp->rap_context.context);
1728 }
1729
1730 static int psp_rap_initialize(struct psp_context *psp)
1731 {
1732         int ret;
1733         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1734
1735         /*
1736          * TODO: bypass the initialize in sriov for now
1737          */
1738         if (amdgpu_sriov_vf(psp->adev))
1739                 return 0;
1740
1741         if (!psp->rap_context.context.bin_desc.size_bytes ||
1742             !psp->rap_context.context.bin_desc.start_addr) {
1743                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1744                 return 0;
1745         }
1746
1747         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1748         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1749
1750         if (!psp->rap_context.context.initialized) {
1751                 ret = psp_rap_init_shared_buf(psp);
1752                 if (ret)
1753                         return ret;
1754         }
1755
1756         ret = psp_rap_load(psp);
1757         if (!ret) {
1758                 psp->rap_context.context.initialized = true;
1759                 mutex_init(&psp->rap_context.mutex);
1760         } else
1761                 return ret;
1762
1763         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1764         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1765                 psp_rap_terminate(psp);
1766
1767                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1768                          ret, status);
1769
1770                 return ret;
1771         }
1772
1773         return 0;
1774 }
1775
1776 static int psp_rap_terminate(struct psp_context *psp)
1777 {
1778         int ret;
1779
1780         if (!psp->rap_context.context.initialized)
1781                 return 0;
1782
1783         ret = psp_rap_unload(psp);
1784
1785         psp->rap_context.context.initialized = false;
1786
1787         /* free rap shared memory */
1788         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1789
1790         return ret;
1791 }
1792
1793 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1794 {
1795         struct ta_rap_shared_memory *rap_cmd;
1796         int ret = 0;
1797
1798         if (!psp->rap_context.context.initialized)
1799                 return 0;
1800
1801         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1802             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1803                 return -EINVAL;
1804
1805         mutex_lock(&psp->rap_context.mutex);
1806
1807         rap_cmd = (struct ta_rap_shared_memory *)
1808                   psp->rap_context.context.mem_context.shared_buf;
1809         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1810
1811         rap_cmd->cmd_id = ta_cmd_id;
1812         rap_cmd->validation_method_id = METHOD_A;
1813
1814         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1815         if (ret)
1816                 goto out_unlock;
1817
1818         if (status)
1819                 *status = rap_cmd->rap_status;
1820
1821 out_unlock:
1822         mutex_unlock(&psp->rap_context.mutex);
1823
1824         return ret;
1825 }
1826 // RAP end
1827
1828 /* securedisplay start */
1829 static int psp_securedisplay_init_shared_buf(struct psp_context *psp)
1830 {
1831         return psp_ta_init_shared_buf(
1832                 psp, &psp->securedisplay_context.context.mem_context);
1833 }
1834
1835 static int psp_securedisplay_load(struct psp_context *psp)
1836 {
1837         return psp_ta_load(psp, &psp->securedisplay_context.context);
1838 }
1839
1840 static int psp_securedisplay_unload(struct psp_context *psp)
1841 {
1842         return psp_ta_unload(psp, &psp->securedisplay_context.context);
1843 }
1844
1845 static int psp_securedisplay_initialize(struct psp_context *psp)
1846 {
1847         int ret;
1848         struct securedisplay_cmd *securedisplay_cmd;
1849
1850         /*
1851          * TODO: bypass the initialize in sriov for now
1852          */
1853         if (amdgpu_sriov_vf(psp->adev))
1854                 return 0;
1855
1856         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1857             !psp->securedisplay_context.context.bin_desc.start_addr) {
1858                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
1859                 return 0;
1860         }
1861
1862         psp->securedisplay_context.context.mem_context.shared_mem_size =
1863                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
1864         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1865
1866         if (!psp->securedisplay_context.context.initialized) {
1867                 ret = psp_securedisplay_init_shared_buf(psp);
1868                 if (ret)
1869                         return ret;
1870         }
1871
1872         ret = psp_securedisplay_load(psp);
1873         if (!ret) {
1874                 psp->securedisplay_context.context.initialized = true;
1875                 mutex_init(&psp->securedisplay_context.mutex);
1876         } else
1877                 return ret;
1878
1879         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
1880                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1881
1882         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1883         if (ret) {
1884                 psp_securedisplay_terminate(psp);
1885                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
1886                 return -EINVAL;
1887         }
1888
1889         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
1890                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
1891                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
1892                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
1893         }
1894
1895         return 0;
1896 }
1897
1898 static int psp_securedisplay_terminate(struct psp_context *psp)
1899 {
1900         int ret;
1901
1902         /*
1903          * TODO:bypass the terminate in sriov for now
1904          */
1905         if (amdgpu_sriov_vf(psp->adev))
1906                 return 0;
1907
1908         if (!psp->securedisplay_context.context.initialized)
1909                 return 0;
1910
1911         ret = psp_securedisplay_unload(psp);
1912         if (ret)
1913                 return ret;
1914
1915         psp->securedisplay_context.context.initialized = false;
1916
1917         /* free securedisplay shared memory */
1918         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
1919
1920         return ret;
1921 }
1922
1923 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1924 {
1925         int ret;
1926
1927         if (!psp->securedisplay_context.context.initialized)
1928                 return -EINVAL;
1929
1930         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
1931             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
1932                 return -EINVAL;
1933
1934         mutex_lock(&psp->securedisplay_context.mutex);
1935
1936         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
1937
1938         mutex_unlock(&psp->securedisplay_context.mutex);
1939
1940         return ret;
1941 }
1942 /* SECUREDISPLAY end */
1943
1944 static int psp_hw_start(struct psp_context *psp)
1945 {
1946         struct amdgpu_device *adev = psp->adev;
1947         int ret;
1948
1949         if (!amdgpu_sriov_vf(adev)) {
1950                 if ((is_psp_fw_valid(psp->kdb)) &&
1951                     (psp->funcs->bootloader_load_kdb != NULL)) {
1952                         ret = psp_bootloader_load_kdb(psp);
1953                         if (ret) {
1954                                 DRM_ERROR("PSP load kdb failed!\n");
1955                                 return ret;
1956                         }
1957                 }
1958
1959                 if ((is_psp_fw_valid(psp->spl)) &&
1960                     (psp->funcs->bootloader_load_spl != NULL)) {
1961                         ret = psp_bootloader_load_spl(psp);
1962                         if (ret) {
1963                                 DRM_ERROR("PSP load spl failed!\n");
1964                                 return ret;
1965                         }
1966                 }
1967
1968                 if ((is_psp_fw_valid(psp->sys)) &&
1969                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
1970                         ret = psp_bootloader_load_sysdrv(psp);
1971                         if (ret) {
1972                                 DRM_ERROR("PSP load sys drv failed!\n");
1973                                 return ret;
1974                         }
1975                 }
1976
1977                 if ((is_psp_fw_valid(psp->soc_drv)) &&
1978                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
1979                         ret = psp_bootloader_load_soc_drv(psp);
1980                         if (ret) {
1981                                 DRM_ERROR("PSP load soc drv failed!\n");
1982                                 return ret;
1983                         }
1984                 }
1985
1986                 if ((is_psp_fw_valid(psp->intf_drv)) &&
1987                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
1988                         ret = psp_bootloader_load_intf_drv(psp);
1989                         if (ret) {
1990                                 DRM_ERROR("PSP load intf drv failed!\n");
1991                                 return ret;
1992                         }
1993                 }
1994
1995                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
1996                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
1997                         ret = psp_bootloader_load_dbg_drv(psp);
1998                         if (ret) {
1999                                 DRM_ERROR("PSP load dbg drv failed!\n");
2000                                 return ret;
2001                         }
2002                 }
2003
2004                 if ((is_psp_fw_valid(psp->sos)) &&
2005                     (psp->funcs->bootloader_load_sos != NULL)) {
2006                         ret = psp_bootloader_load_sos(psp);
2007                         if (ret) {
2008                                 DRM_ERROR("PSP load sos failed!\n");
2009                                 return ret;
2010                         }
2011                 }
2012         }
2013
2014         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2015         if (ret) {
2016                 DRM_ERROR("PSP create ring failed!\n");
2017                 return ret;
2018         }
2019
2020         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2021                 goto skip_pin_bo;
2022
2023         ret = psp_tmr_init(psp);
2024         if (ret) {
2025                 DRM_ERROR("PSP tmr init failed!\n");
2026                 return ret;
2027         }
2028
2029 skip_pin_bo:
2030         /*
2031          * For ASICs with DF Cstate management centralized
2032          * to PMFW, TMR setup should be performed after PMFW
2033          * loaded and before other non-psp firmware loaded.
2034          */
2035         if (psp->pmfw_centralized_cstate_management) {
2036                 ret = psp_load_smu_fw(psp);
2037                 if (ret)
2038                         return ret;
2039         }
2040
2041         ret = psp_tmr_load(psp);
2042         if (ret) {
2043                 DRM_ERROR("PSP load tmr failed!\n");
2044                 return ret;
2045         }
2046
2047         return 0;
2048 }
2049
2050 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2051                            enum psp_gfx_fw_type *type)
2052 {
2053         switch (ucode->ucode_id) {
2054         case AMDGPU_UCODE_ID_SDMA0:
2055                 *type = GFX_FW_TYPE_SDMA0;
2056                 break;
2057         case AMDGPU_UCODE_ID_SDMA1:
2058                 *type = GFX_FW_TYPE_SDMA1;
2059                 break;
2060         case AMDGPU_UCODE_ID_SDMA2:
2061                 *type = GFX_FW_TYPE_SDMA2;
2062                 break;
2063         case AMDGPU_UCODE_ID_SDMA3:
2064                 *type = GFX_FW_TYPE_SDMA3;
2065                 break;
2066         case AMDGPU_UCODE_ID_SDMA4:
2067                 *type = GFX_FW_TYPE_SDMA4;
2068                 break;
2069         case AMDGPU_UCODE_ID_SDMA5:
2070                 *type = GFX_FW_TYPE_SDMA5;
2071                 break;
2072         case AMDGPU_UCODE_ID_SDMA6:
2073                 *type = GFX_FW_TYPE_SDMA6;
2074                 break;
2075         case AMDGPU_UCODE_ID_SDMA7:
2076                 *type = GFX_FW_TYPE_SDMA7;
2077                 break;
2078         case AMDGPU_UCODE_ID_CP_MES:
2079                 *type = GFX_FW_TYPE_CP_MES;
2080                 break;
2081         case AMDGPU_UCODE_ID_CP_MES_DATA:
2082                 *type = GFX_FW_TYPE_MES_STACK;
2083                 break;
2084         case AMDGPU_UCODE_ID_CP_CE:
2085                 *type = GFX_FW_TYPE_CP_CE;
2086                 break;
2087         case AMDGPU_UCODE_ID_CP_PFP:
2088                 *type = GFX_FW_TYPE_CP_PFP;
2089                 break;
2090         case AMDGPU_UCODE_ID_CP_ME:
2091                 *type = GFX_FW_TYPE_CP_ME;
2092                 break;
2093         case AMDGPU_UCODE_ID_CP_MEC1:
2094                 *type = GFX_FW_TYPE_CP_MEC;
2095                 break;
2096         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2097                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2098                 break;
2099         case AMDGPU_UCODE_ID_CP_MEC2:
2100                 *type = GFX_FW_TYPE_CP_MEC;
2101                 break;
2102         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2103                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2104                 break;
2105         case AMDGPU_UCODE_ID_RLC_G:
2106                 *type = GFX_FW_TYPE_RLC_G;
2107                 break;
2108         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2109                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2110                 break;
2111         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2112                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2113                 break;
2114         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2115                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2116                 break;
2117         case AMDGPU_UCODE_ID_RLC_IRAM:
2118                 *type = GFX_FW_TYPE_RLC_IRAM;
2119                 break;
2120         case AMDGPU_UCODE_ID_RLC_DRAM:
2121                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2122                 break;
2123         case AMDGPU_UCODE_ID_SMC:
2124                 *type = GFX_FW_TYPE_SMU;
2125                 break;
2126         case AMDGPU_UCODE_ID_UVD:
2127                 *type = GFX_FW_TYPE_UVD;
2128                 break;
2129         case AMDGPU_UCODE_ID_UVD1:
2130                 *type = GFX_FW_TYPE_UVD1;
2131                 break;
2132         case AMDGPU_UCODE_ID_VCE:
2133                 *type = GFX_FW_TYPE_VCE;
2134                 break;
2135         case AMDGPU_UCODE_ID_VCN:
2136                 *type = GFX_FW_TYPE_VCN;
2137                 break;
2138         case AMDGPU_UCODE_ID_VCN1:
2139                 *type = GFX_FW_TYPE_VCN1;
2140                 break;
2141         case AMDGPU_UCODE_ID_DMCU_ERAM:
2142                 *type = GFX_FW_TYPE_DMCU_ERAM;
2143                 break;
2144         case AMDGPU_UCODE_ID_DMCU_INTV:
2145                 *type = GFX_FW_TYPE_DMCU_ISR;
2146                 break;
2147         case AMDGPU_UCODE_ID_VCN0_RAM:
2148                 *type = GFX_FW_TYPE_VCN0_RAM;
2149                 break;
2150         case AMDGPU_UCODE_ID_VCN1_RAM:
2151                 *type = GFX_FW_TYPE_VCN1_RAM;
2152                 break;
2153         case AMDGPU_UCODE_ID_DMCUB:
2154                 *type = GFX_FW_TYPE_DMUB;
2155                 break;
2156         case AMDGPU_UCODE_ID_MAXIMUM:
2157         default:
2158                 return -EINVAL;
2159         }
2160
2161         return 0;
2162 }
2163
2164 static void psp_print_fw_hdr(struct psp_context *psp,
2165                              struct amdgpu_firmware_info *ucode)
2166 {
2167         struct amdgpu_device *adev = psp->adev;
2168         struct common_firmware_header *hdr;
2169
2170         switch (ucode->ucode_id) {
2171         case AMDGPU_UCODE_ID_SDMA0:
2172         case AMDGPU_UCODE_ID_SDMA1:
2173         case AMDGPU_UCODE_ID_SDMA2:
2174         case AMDGPU_UCODE_ID_SDMA3:
2175         case AMDGPU_UCODE_ID_SDMA4:
2176         case AMDGPU_UCODE_ID_SDMA5:
2177         case AMDGPU_UCODE_ID_SDMA6:
2178         case AMDGPU_UCODE_ID_SDMA7:
2179                 hdr = (struct common_firmware_header *)
2180                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2181                 amdgpu_ucode_print_sdma_hdr(hdr);
2182                 break;
2183         case AMDGPU_UCODE_ID_CP_CE:
2184                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2185                 amdgpu_ucode_print_gfx_hdr(hdr);
2186                 break;
2187         case AMDGPU_UCODE_ID_CP_PFP:
2188                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2189                 amdgpu_ucode_print_gfx_hdr(hdr);
2190                 break;
2191         case AMDGPU_UCODE_ID_CP_ME:
2192                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2193                 amdgpu_ucode_print_gfx_hdr(hdr);
2194                 break;
2195         case AMDGPU_UCODE_ID_CP_MEC1:
2196                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2197                 amdgpu_ucode_print_gfx_hdr(hdr);
2198                 break;
2199         case AMDGPU_UCODE_ID_RLC_G:
2200                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2201                 amdgpu_ucode_print_rlc_hdr(hdr);
2202                 break;
2203         case AMDGPU_UCODE_ID_SMC:
2204                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2205                 amdgpu_ucode_print_smc_hdr(hdr);
2206                 break;
2207         default:
2208                 break;
2209         }
2210 }
2211
2212 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2213                                        struct psp_gfx_cmd_resp *cmd)
2214 {
2215         int ret;
2216         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2217
2218         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2219         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2220         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2221         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2222
2223         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2224         if (ret)
2225                 DRM_ERROR("Unknown firmware type\n");
2226
2227         return ret;
2228 }
2229
2230 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2231                                   struct amdgpu_firmware_info *ucode)
2232 {
2233         int ret = 0;
2234         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2235
2236         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2237         if (!ret) {
2238                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2239                                          psp->fence_buf_mc_addr);
2240         }
2241
2242         release_psp_cmd_buf(psp);
2243
2244         return ret;
2245 }
2246
2247 static int psp_load_smu_fw(struct psp_context *psp)
2248 {
2249         int ret;
2250         struct amdgpu_device *adev = psp->adev;
2251         struct amdgpu_firmware_info *ucode =
2252                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2253         struct amdgpu_ras *ras = psp->ras_context.ras;
2254
2255         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2256                 return 0;
2257
2258         if ((amdgpu_in_reset(adev) &&
2259              ras && adev->ras_enabled &&
2260              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2261               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2262                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2263                 if (ret) {
2264                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2265                 }
2266         }
2267
2268         ret = psp_execute_non_psp_fw_load(psp, ucode);
2269
2270         if (ret)
2271                 DRM_ERROR("PSP load smu failed!\n");
2272
2273         return ret;
2274 }
2275
2276 static bool fw_load_skip_check(struct psp_context *psp,
2277                                struct amdgpu_firmware_info *ucode)
2278 {
2279         if (!ucode->fw)
2280                 return true;
2281
2282         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2283             (psp_smu_reload_quirk(psp) ||
2284              psp->autoload_supported ||
2285              psp->pmfw_centralized_cstate_management))
2286                 return true;
2287
2288         if (amdgpu_sriov_vf(psp->adev) &&
2289            (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
2290             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
2291             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
2292             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
2293             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
2294             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
2295             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
2296             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
2297             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
2298             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
2299             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
2300             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
2301             || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
2302                 /*skip ucode loading in SRIOV VF */
2303                 return true;
2304
2305         if (psp->autoload_supported &&
2306             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2307              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2308                 /* skip mec JT when autoload is enabled */
2309                 return true;
2310
2311         return false;
2312 }
2313
2314 int psp_load_fw_list(struct psp_context *psp,
2315                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2316 {
2317         int ret = 0, i;
2318         struct amdgpu_firmware_info *ucode;
2319
2320         for (i = 0; i < ucode_count; ++i) {
2321                 ucode = ucode_list[i];
2322                 psp_print_fw_hdr(psp, ucode);
2323                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2324                 if (ret)
2325                         return ret;
2326         }
2327         return ret;
2328 }
2329
2330 static int psp_load_non_psp_fw(struct psp_context *psp)
2331 {
2332         int i, ret;
2333         struct amdgpu_firmware_info *ucode;
2334         struct amdgpu_device *adev = psp->adev;
2335
2336         if (psp->autoload_supported &&
2337             !psp->pmfw_centralized_cstate_management) {
2338                 ret = psp_load_smu_fw(psp);
2339                 if (ret)
2340                         return ret;
2341         }
2342
2343         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2344                 ucode = &adev->firmware.ucode[i];
2345
2346                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2347                     !fw_load_skip_check(psp, ucode)) {
2348                         ret = psp_load_smu_fw(psp);
2349                         if (ret)
2350                                 return ret;
2351                         continue;
2352                 }
2353
2354                 if (fw_load_skip_check(psp, ucode))
2355                         continue;
2356
2357                 if (psp->autoload_supported &&
2358                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2359                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2360                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2361                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2362                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2363                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2364                         /* PSP only receive one SDMA fw for sienna_cichlid,
2365                          * as all four sdma fw are same */
2366                         continue;
2367
2368                 psp_print_fw_hdr(psp, ucode);
2369
2370                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2371                 if (ret)
2372                         return ret;
2373
2374                 /* Start rlc autoload after psp recieved all the gfx firmware */
2375                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2376                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
2377                         ret = psp_rlc_autoload_start(psp);
2378                         if (ret) {
2379                                 DRM_ERROR("Failed to start rlc autoload\n");
2380                                 return ret;
2381                         }
2382                 }
2383         }
2384
2385         return 0;
2386 }
2387
2388 static int psp_load_fw(struct amdgpu_device *adev)
2389 {
2390         int ret;
2391         struct psp_context *psp = &adev->psp;
2392
2393         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2394                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
2395                 goto skip_memalloc;
2396         }
2397
2398         if (amdgpu_sriov_vf(adev)) {
2399                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
2400                                                 AMDGPU_GEM_DOMAIN_VRAM,
2401                                                 &psp->fw_pri_bo,
2402                                                 &psp->fw_pri_mc_addr,
2403                                                 &psp->fw_pri_buf);
2404         } else {
2405                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
2406                                                 AMDGPU_GEM_DOMAIN_GTT,
2407                                                 &psp->fw_pri_bo,
2408                                                 &psp->fw_pri_mc_addr,
2409                                                 &psp->fw_pri_buf);
2410         }
2411
2412         if (ret)
2413                 goto failed;
2414
2415         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
2416                                         AMDGPU_GEM_DOMAIN_VRAM,
2417                                         &psp->fence_buf_bo,
2418                                         &psp->fence_buf_mc_addr,
2419                                         &psp->fence_buf);
2420         if (ret)
2421                 goto failed;
2422
2423         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
2424                                       AMDGPU_GEM_DOMAIN_VRAM,
2425                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2426                                       (void **)&psp->cmd_buf_mem);
2427         if (ret)
2428                 goto failed;
2429
2430         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2431
2432         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2433         if (ret) {
2434                 DRM_ERROR("PSP ring init failed!\n");
2435                 goto failed;
2436         }
2437
2438 skip_memalloc:
2439         ret = psp_hw_start(psp);
2440         if (ret)
2441                 goto failed;
2442
2443         ret = psp_load_non_psp_fw(psp);
2444         if (ret)
2445                 goto failed;
2446
2447         ret = psp_asd_initialize(psp);
2448         if (ret) {
2449                 DRM_ERROR("PSP load asd failed!\n");
2450                 return ret;
2451         }
2452
2453         ret = psp_rl_load(adev);
2454         if (ret) {
2455                 DRM_ERROR("PSP load RL failed!\n");
2456                 return ret;
2457         }
2458
2459         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2460                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2461                         ret = psp_xgmi_initialize(psp, false, true);
2462                         /* Warning the XGMI seesion initialize failure
2463                         * Instead of stop driver initialization
2464                         */
2465                         if (ret)
2466                                 dev_err(psp->adev->dev,
2467                                         "XGMI: Failed to initialize XGMI session\n");
2468                 }
2469         }
2470
2471         if (psp->ta_fw) {
2472                 ret = psp_ras_initialize(psp);
2473                 if (ret)
2474                         dev_err(psp->adev->dev,
2475                                         "RAS: Failed to initialize RAS\n");
2476
2477                 ret = psp_hdcp_initialize(psp);
2478                 if (ret)
2479                         dev_err(psp->adev->dev,
2480                                 "HDCP: Failed to initialize HDCP\n");
2481
2482                 ret = psp_dtm_initialize(psp);
2483                 if (ret)
2484                         dev_err(psp->adev->dev,
2485                                 "DTM: Failed to initialize DTM\n");
2486
2487                 ret = psp_rap_initialize(psp);
2488                 if (ret)
2489                         dev_err(psp->adev->dev,
2490                                 "RAP: Failed to initialize RAP\n");
2491
2492                 ret = psp_securedisplay_initialize(psp);
2493                 if (ret)
2494                         dev_err(psp->adev->dev,
2495                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2496         }
2497
2498         return 0;
2499
2500 failed:
2501         /*
2502          * all cleanup jobs (xgmi terminate, ras terminate,
2503          * ring destroy, cmd/fence/fw buffers destory,
2504          * psp->cmd destory) are delayed to psp_hw_fini
2505          */
2506         return ret;
2507 }
2508
2509 static int psp_hw_init(void *handle)
2510 {
2511         int ret;
2512         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2513
2514         mutex_lock(&adev->firmware.mutex);
2515         /*
2516          * This sequence is just used on hw_init only once, no need on
2517          * resume.
2518          */
2519         ret = amdgpu_ucode_init_bo(adev);
2520         if (ret)
2521                 goto failed;
2522
2523         ret = psp_load_fw(adev);
2524         if (ret) {
2525                 DRM_ERROR("PSP firmware loading failed\n");
2526                 goto failed;
2527         }
2528
2529         mutex_unlock(&adev->firmware.mutex);
2530         return 0;
2531
2532 failed:
2533         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2534         mutex_unlock(&adev->firmware.mutex);
2535         return -EINVAL;
2536 }
2537
2538 static int psp_hw_fini(void *handle)
2539 {
2540         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2541         struct psp_context *psp = &adev->psp;
2542
2543         if (psp->ta_fw) {
2544                 psp_ras_terminate(psp);
2545                 psp_securedisplay_terminate(psp);
2546                 psp_rap_terminate(psp);
2547                 psp_dtm_terminate(psp);
2548                 psp_hdcp_terminate(psp);
2549         }
2550
2551         psp_asd_terminate(psp);
2552
2553         psp_tmr_terminate(psp);
2554         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2555
2556         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
2557                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
2558         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
2559                               &psp->fence_buf_mc_addr, &psp->fence_buf);
2560         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
2561                               (void **)&psp->cmd_buf_mem);
2562
2563         return 0;
2564 }
2565
2566 static int psp_suspend(void *handle)
2567 {
2568         int ret;
2569         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2570         struct psp_context *psp = &adev->psp;
2571
2572         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2573             psp->xgmi_context.context.initialized) {
2574                 ret = psp_xgmi_terminate(psp);
2575                 if (ret) {
2576                         DRM_ERROR("Failed to terminate xgmi ta\n");
2577                         return ret;
2578                 }
2579         }
2580
2581         if (psp->ta_fw) {
2582                 ret = psp_ras_terminate(psp);
2583                 if (ret) {
2584                         DRM_ERROR("Failed to terminate ras ta\n");
2585                         return ret;
2586                 }
2587                 ret = psp_hdcp_terminate(psp);
2588                 if (ret) {
2589                         DRM_ERROR("Failed to terminate hdcp ta\n");
2590                         return ret;
2591                 }
2592                 ret = psp_dtm_terminate(psp);
2593                 if (ret) {
2594                         DRM_ERROR("Failed to terminate dtm ta\n");
2595                         return ret;
2596                 }
2597                 ret = psp_rap_terminate(psp);
2598                 if (ret) {
2599                         DRM_ERROR("Failed to terminate rap ta\n");
2600                         return ret;
2601                 }
2602                 ret = psp_securedisplay_terminate(psp);
2603                 if (ret) {
2604                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2605                         return ret;
2606                 }
2607         }
2608
2609         ret = psp_asd_terminate(psp);
2610         if (ret) {
2611                 DRM_ERROR("Failed to terminate asd\n");
2612                 return ret;
2613         }
2614
2615         ret = psp_tmr_terminate(psp);
2616         if (ret) {
2617                 DRM_ERROR("Failed to terminate tmr\n");
2618                 return ret;
2619         }
2620
2621         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2622         if (ret) {
2623                 DRM_ERROR("PSP ring stop failed\n");
2624                 return ret;
2625         }
2626
2627         return 0;
2628 }
2629
2630 static int psp_resume(void *handle)
2631 {
2632         int ret;
2633         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2634         struct psp_context *psp = &adev->psp;
2635
2636         DRM_INFO("PSP is resuming...\n");
2637
2638         if (psp->mem_train_ctx.enable_mem_training) {
2639                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2640                 if (ret) {
2641                         DRM_ERROR("Failed to process memory training!\n");
2642                         return ret;
2643                 }
2644         }
2645
2646         mutex_lock(&adev->firmware.mutex);
2647
2648         ret = psp_hw_start(psp);
2649         if (ret)
2650                 goto failed;
2651
2652         ret = psp_load_non_psp_fw(psp);
2653         if (ret)
2654                 goto failed;
2655
2656         ret = psp_asd_initialize(psp);
2657         if (ret) {
2658                 DRM_ERROR("PSP load asd failed!\n");
2659                 goto failed;
2660         }
2661
2662         ret = psp_rl_load(adev);
2663         if (ret) {
2664                 dev_err(adev->dev, "PSP load RL failed!\n");
2665                 goto failed;
2666         }
2667
2668         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2669                 ret = psp_xgmi_initialize(psp, false, true);
2670                 /* Warning the XGMI seesion initialize failure
2671                  * Instead of stop driver initialization
2672                  */
2673                 if (ret)
2674                         dev_err(psp->adev->dev,
2675                                 "XGMI: Failed to initialize XGMI session\n");
2676         }
2677
2678         if (psp->ta_fw) {
2679                 ret = psp_ras_initialize(psp);
2680                 if (ret)
2681                         dev_err(psp->adev->dev,
2682                                         "RAS: Failed to initialize RAS\n");
2683
2684                 ret = psp_hdcp_initialize(psp);
2685                 if (ret)
2686                         dev_err(psp->adev->dev,
2687                                 "HDCP: Failed to initialize HDCP\n");
2688
2689                 ret = psp_dtm_initialize(psp);
2690                 if (ret)
2691                         dev_err(psp->adev->dev,
2692                                 "DTM: Failed to initialize DTM\n");
2693
2694                 ret = psp_rap_initialize(psp);
2695                 if (ret)
2696                         dev_err(psp->adev->dev,
2697                                 "RAP: Failed to initialize RAP\n");
2698
2699                 ret = psp_securedisplay_initialize(psp);
2700                 if (ret)
2701                         dev_err(psp->adev->dev,
2702                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2703         }
2704
2705         mutex_unlock(&adev->firmware.mutex);
2706
2707         return 0;
2708
2709 failed:
2710         DRM_ERROR("PSP resume failed\n");
2711         mutex_unlock(&adev->firmware.mutex);
2712         return ret;
2713 }
2714
2715 int psp_gpu_reset(struct amdgpu_device *adev)
2716 {
2717         int ret;
2718
2719         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2720                 return 0;
2721
2722         mutex_lock(&adev->psp.mutex);
2723         ret = psp_mode1_reset(&adev->psp);
2724         mutex_unlock(&adev->psp.mutex);
2725
2726         return ret;
2727 }
2728
2729 int psp_rlc_autoload_start(struct psp_context *psp)
2730 {
2731         int ret;
2732         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2733
2734         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2735
2736         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2737                                  psp->fence_buf_mc_addr);
2738
2739         release_psp_cmd_buf(psp);
2740
2741         return ret;
2742 }
2743
2744 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2745                         uint64_t cmd_gpu_addr, int cmd_size)
2746 {
2747         struct amdgpu_firmware_info ucode = {0};
2748
2749         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2750                 AMDGPU_UCODE_ID_VCN0_RAM;
2751         ucode.mc_addr = cmd_gpu_addr;
2752         ucode.ucode_size = cmd_size;
2753
2754         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2755 }
2756
2757 int psp_ring_cmd_submit(struct psp_context *psp,
2758                         uint64_t cmd_buf_mc_addr,
2759                         uint64_t fence_mc_addr,
2760                         int index)
2761 {
2762         unsigned int psp_write_ptr_reg = 0;
2763         struct psp_gfx_rb_frame *write_frame;
2764         struct psp_ring *ring = &psp->km_ring;
2765         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2766         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2767                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2768         struct amdgpu_device *adev = psp->adev;
2769         uint32_t ring_size_dw = ring->ring_size / 4;
2770         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2771
2772         /* KM (GPCOM) prepare write pointer */
2773         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2774
2775         /* Update KM RB frame pointer to new frame */
2776         /* write_frame ptr increments by size of rb_frame in bytes */
2777         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2778         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2779                 write_frame = ring_buffer_start;
2780         else
2781                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2782         /* Check invalid write_frame ptr address */
2783         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2784                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2785                           ring_buffer_start, ring_buffer_end, write_frame);
2786                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2787                 return -EINVAL;
2788         }
2789
2790         /* Initialize KM RB frame */
2791         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2792
2793         /* Update KM RB frame */
2794         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2795         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2796         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2797         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2798         write_frame->fence_value = index;
2799         amdgpu_device_flush_hdp(adev, NULL);
2800
2801         /* Update the write Pointer in DWORDs */
2802         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2803         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2804         return 0;
2805 }
2806
2807 int psp_init_asd_microcode(struct psp_context *psp,
2808                            const char *chip_name)
2809 {
2810         struct amdgpu_device *adev = psp->adev;
2811         char fw_name[PSP_FW_NAME_LEN];
2812         const struct psp_firmware_header_v1_0 *asd_hdr;
2813         int err = 0;
2814
2815         if (!chip_name) {
2816                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2817                 return -EINVAL;
2818         }
2819
2820         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2821         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2822         if (err)
2823                 goto out;
2824
2825         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2826         if (err)
2827                 goto out;
2828
2829         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2830         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2831         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
2832         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2833         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
2834                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2835         return 0;
2836 out:
2837         dev_err(adev->dev, "fail to initialize asd microcode\n");
2838         release_firmware(adev->psp.asd_fw);
2839         adev->psp.asd_fw = NULL;
2840         return err;
2841 }
2842
2843 int psp_init_toc_microcode(struct psp_context *psp,
2844                            const char *chip_name)
2845 {
2846         struct amdgpu_device *adev = psp->adev;
2847         char fw_name[PSP_FW_NAME_LEN];
2848         const struct psp_firmware_header_v1_0 *toc_hdr;
2849         int err = 0;
2850
2851         if (!chip_name) {
2852                 dev_err(adev->dev, "invalid chip name for toc microcode\n");
2853                 return -EINVAL;
2854         }
2855
2856         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
2857         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
2858         if (err)
2859                 goto out;
2860
2861         err = amdgpu_ucode_validate(adev->psp.toc_fw);
2862         if (err)
2863                 goto out;
2864
2865         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
2866         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
2867         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
2868         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
2869         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
2870                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
2871         return 0;
2872 out:
2873         dev_err(adev->dev, "fail to request/validate toc microcode\n");
2874         release_firmware(adev->psp.toc_fw);
2875         adev->psp.toc_fw = NULL;
2876         return err;
2877 }
2878
2879 static int parse_sos_bin_descriptor(struct psp_context *psp,
2880                                    const struct psp_fw_bin_desc *desc,
2881                                    const struct psp_firmware_header_v2_0 *sos_hdr)
2882 {
2883         uint8_t *ucode_start_addr  = NULL;
2884
2885         if (!psp || !desc || !sos_hdr)
2886                 return -EINVAL;
2887
2888         ucode_start_addr  = (uint8_t *)sos_hdr +
2889                             le32_to_cpu(desc->offset_bytes) +
2890                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2891
2892         switch (desc->fw_type) {
2893         case PSP_FW_TYPE_PSP_SOS:
2894                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
2895                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
2896                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
2897                 psp->sos.start_addr        = ucode_start_addr;
2898                 break;
2899         case PSP_FW_TYPE_PSP_SYS_DRV:
2900                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
2901                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
2902                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
2903                 psp->sys.start_addr        = ucode_start_addr;
2904                 break;
2905         case PSP_FW_TYPE_PSP_KDB:
2906                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
2907                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
2908                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
2909                 psp->kdb.start_addr        = ucode_start_addr;
2910                 break;
2911         case PSP_FW_TYPE_PSP_TOC:
2912                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
2913                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
2914                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
2915                 psp->toc.start_addr        = ucode_start_addr;
2916                 break;
2917         case PSP_FW_TYPE_PSP_SPL:
2918                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
2919                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
2920                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
2921                 psp->spl.start_addr        = ucode_start_addr;
2922                 break;
2923         case PSP_FW_TYPE_PSP_RL:
2924                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
2925                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
2926                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
2927                 psp->rl.start_addr         = ucode_start_addr;
2928                 break;
2929         case PSP_FW_TYPE_PSP_SOC_DRV:
2930                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
2931                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
2932                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
2933                 psp->soc_drv.start_addr         = ucode_start_addr;
2934                 break;
2935         case PSP_FW_TYPE_PSP_INTF_DRV:
2936                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
2937                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
2938                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
2939                 psp->intf_drv.start_addr        = ucode_start_addr;
2940                 break;
2941         case PSP_FW_TYPE_PSP_DBG_DRV:
2942                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
2943                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
2944                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
2945                 psp->dbg_drv.start_addr         = ucode_start_addr;
2946                 break;
2947         default:
2948                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
2949                 break;
2950         }
2951
2952         return 0;
2953 }
2954
2955 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
2956 {
2957         const struct psp_firmware_header_v1_0 *sos_hdr;
2958         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2959         uint8_t *ucode_array_start_addr;
2960
2961         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
2962         ucode_array_start_addr = (uint8_t *)sos_hdr +
2963                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2964
2965         if (adev->gmc.xgmi.connected_to_cpu ||
2966             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
2967                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
2968                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
2969
2970                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
2971                 adev->psp.sys.start_addr = ucode_array_start_addr;
2972
2973                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
2974                 adev->psp.sos.start_addr = ucode_array_start_addr +
2975                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
2976                 adev->psp.xgmi_context.supports_extended_data = false;
2977         } else {
2978                 /* Load alternate PSP SOS FW */
2979                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
2980
2981                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
2982                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
2983
2984                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
2985                 adev->psp.sys.start_addr = ucode_array_start_addr +
2986                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
2987
2988                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
2989                 adev->psp.sos.start_addr = ucode_array_start_addr +
2990                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
2991                 adev->psp.xgmi_context.supports_extended_data = true;
2992         }
2993
2994         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
2995                 dev_warn(adev->dev, "PSP SOS FW not available");
2996                 return -EINVAL;
2997         }
2998
2999         return 0;
3000 }
3001
3002 int psp_init_sos_microcode(struct psp_context *psp,
3003                            const char *chip_name)
3004 {
3005         struct amdgpu_device *adev = psp->adev;
3006         char fw_name[PSP_FW_NAME_LEN];
3007         const struct psp_firmware_header_v1_0 *sos_hdr;
3008         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3009         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3010         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3011         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3012         int err = 0;
3013         uint8_t *ucode_array_start_addr;
3014         int fw_index = 0;
3015
3016         if (!chip_name) {
3017                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
3018                 return -EINVAL;
3019         }
3020
3021         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3022         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
3023         if (err)
3024                 goto out;
3025
3026         err = amdgpu_ucode_validate(adev->psp.sos_fw);
3027         if (err)
3028                 goto out;
3029
3030         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3031         ucode_array_start_addr = (uint8_t *)sos_hdr +
3032                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3033         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3034
3035         switch (sos_hdr->header.header_version_major) {
3036         case 1:
3037                 err = psp_init_sos_base_fw(adev);
3038                 if (err)
3039                         goto out;
3040
3041                 if (sos_hdr->header.header_version_minor == 1) {
3042                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3043                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3044                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3045                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3046                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3047                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3048                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3049                 }
3050                 if (sos_hdr->header.header_version_minor == 2) {
3051                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3052                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3053                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3054                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3055                 }
3056                 if (sos_hdr->header.header_version_minor == 3) {
3057                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3058                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3059                         adev->psp.toc.start_addr = ucode_array_start_addr +
3060                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3061                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3062                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3063                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3064                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3065                         adev->psp.spl.start_addr = ucode_array_start_addr +
3066                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3067                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3068                         adev->psp.rl.start_addr = ucode_array_start_addr +
3069                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3070                 }
3071                 break;
3072         case 2:
3073                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3074
3075                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3076                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3077                         err = -EINVAL;
3078                         goto out;
3079                 }
3080
3081                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3082                         err = parse_sos_bin_descriptor(psp,
3083                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3084                                                        sos_hdr_v2_0);
3085                         if (err)
3086                                 goto out;
3087                 }
3088                 break;
3089         default:
3090                 dev_err(adev->dev,
3091                         "unsupported psp sos firmware\n");
3092                 err = -EINVAL;
3093                 goto out;
3094         }
3095
3096         return 0;
3097 out:
3098         dev_err(adev->dev,
3099                 "failed to init sos firmware\n");
3100         release_firmware(adev->psp.sos_fw);
3101         adev->psp.sos_fw = NULL;
3102
3103         return err;
3104 }
3105
3106 static int parse_ta_bin_descriptor(struct psp_context *psp,
3107                                    const struct psp_fw_bin_desc *desc,
3108                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3109 {
3110         uint8_t *ucode_start_addr  = NULL;
3111
3112         if (!psp || !desc || !ta_hdr)
3113                 return -EINVAL;
3114
3115         ucode_start_addr  = (uint8_t *)ta_hdr +
3116                             le32_to_cpu(desc->offset_bytes) +
3117                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3118
3119         switch (desc->fw_type) {
3120         case TA_FW_TYPE_PSP_ASD:
3121                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3122                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3123                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3124                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3125                 break;
3126         case TA_FW_TYPE_PSP_XGMI:
3127                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3128                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3129                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3130                 break;
3131         case TA_FW_TYPE_PSP_RAS:
3132                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3133                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3134                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3135                 break;
3136         case TA_FW_TYPE_PSP_HDCP:
3137                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3138                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3139                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3140                 break;
3141         case TA_FW_TYPE_PSP_DTM:
3142                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3143                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3144                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3145                 break;
3146         case TA_FW_TYPE_PSP_RAP:
3147                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3148                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3149                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3150                 break;
3151         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3152                 psp->securedisplay_context.context.bin_desc.fw_version =
3153                         le32_to_cpu(desc->fw_version);
3154                 psp->securedisplay_context.context.bin_desc.size_bytes =
3155                         le32_to_cpu(desc->size_bytes);
3156                 psp->securedisplay_context.context.bin_desc.start_addr =
3157                         ucode_start_addr;
3158                 break;
3159         default:
3160                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3161                 break;
3162         }
3163
3164         return 0;
3165 }
3166
3167 int psp_init_ta_microcode(struct psp_context *psp,
3168                           const char *chip_name)
3169 {
3170         struct amdgpu_device *adev = psp->adev;
3171         char fw_name[PSP_FW_NAME_LEN];
3172         const struct ta_firmware_header_v2_0 *ta_hdr;
3173         int err = 0;
3174         int ta_index = 0;
3175
3176         if (!chip_name) {
3177                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
3178                 return -EINVAL;
3179         }
3180
3181         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3182         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
3183         if (err)
3184                 goto out;
3185
3186         err = amdgpu_ucode_validate(adev->psp.ta_fw);
3187         if (err)
3188                 goto out;
3189
3190         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3191
3192         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
3193                 dev_err(adev->dev, "unsupported TA header version\n");
3194                 err = -EINVAL;
3195                 goto out;
3196         }
3197
3198         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3199                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3200                 err = -EINVAL;
3201                 goto out;
3202         }
3203
3204         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3205                 err = parse_ta_bin_descriptor(psp,
3206                                               &ta_hdr->ta_fw_bin[ta_index],
3207                                               ta_hdr);
3208                 if (err)
3209                         goto out;
3210         }
3211
3212         return 0;
3213 out:
3214         dev_err(adev->dev, "fail to initialize ta microcode\n");
3215         release_firmware(adev->psp.ta_fw);
3216         adev->psp.ta_fw = NULL;
3217         return err;
3218 }
3219
3220 static int psp_set_clockgating_state(void *handle,
3221                                      enum amd_clockgating_state state)
3222 {
3223         return 0;
3224 }
3225
3226 static int psp_set_powergating_state(void *handle,
3227                                      enum amd_powergating_state state)
3228 {
3229         return 0;
3230 }
3231
3232 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3233                                          struct device_attribute *attr,
3234                                          char *buf)
3235 {
3236         struct drm_device *ddev = dev_get_drvdata(dev);
3237         struct amdgpu_device *adev = drm_to_adev(ddev);
3238         uint32_t fw_ver;
3239         int ret;
3240
3241         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3242                 DRM_INFO("PSP block is not ready yet.");
3243                 return -EBUSY;
3244         }
3245
3246         mutex_lock(&adev->psp.mutex);
3247         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3248         mutex_unlock(&adev->psp.mutex);
3249
3250         if (ret) {
3251                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3252                 return ret;
3253         }
3254
3255         return sysfs_emit(buf, "%x\n", fw_ver);
3256 }
3257
3258 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3259                                                        struct device_attribute *attr,
3260                                                        const char *buf,
3261                                                        size_t count)
3262 {
3263         struct drm_device *ddev = dev_get_drvdata(dev);
3264         struct amdgpu_device *adev = drm_to_adev(ddev);
3265         int ret, idx;
3266         char fw_name[100];
3267         const struct firmware *usbc_pd_fw;
3268         struct amdgpu_bo *fw_buf_bo = NULL;
3269         uint64_t fw_pri_mc_addr;
3270         void *fw_pri_cpu_addr;
3271
3272         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3273                 DRM_INFO("PSP block is not ready yet.");
3274                 return -EBUSY;
3275         }
3276
3277         if (!drm_dev_enter(ddev, &idx))
3278                 return -ENODEV;
3279
3280         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3281         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3282         if (ret)
3283                 goto fail;
3284
3285         /* LFB address which is aligned to 1MB boundary per PSP request */
3286         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3287                                                 AMDGPU_GEM_DOMAIN_VRAM,
3288                                                 &fw_buf_bo,
3289                                                 &fw_pri_mc_addr,
3290                                                 &fw_pri_cpu_addr);
3291         if (ret)
3292                 goto rel_buf;
3293
3294         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3295
3296         mutex_lock(&adev->psp.mutex);
3297         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3298         mutex_unlock(&adev->psp.mutex);
3299
3300         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3301
3302 rel_buf:
3303         release_firmware(usbc_pd_fw);
3304 fail:
3305         if (ret) {
3306                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3307                 count = ret;
3308         }
3309
3310         drm_dev_exit(idx);
3311         return count;
3312 }
3313
3314 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3315 {
3316         int idx;
3317
3318         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3319                 return;
3320
3321         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3322         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3323
3324         drm_dev_exit(idx);
3325 }
3326
3327 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
3328                    psp_usbc_pd_fw_sysfs_read,
3329                    psp_usbc_pd_fw_sysfs_write);
3330
3331 int is_psp_fw_valid(struct psp_bin_desc bin)
3332 {
3333         return bin.size_bytes;
3334 }
3335
3336 const struct amd_ip_funcs psp_ip_funcs = {
3337         .name = "psp",
3338         .early_init = psp_early_init,
3339         .late_init = NULL,
3340         .sw_init = psp_sw_init,
3341         .sw_fini = psp_sw_fini,
3342         .hw_init = psp_hw_init,
3343         .hw_fini = psp_hw_fini,
3344         .suspend = psp_suspend,
3345         .resume = psp_resume,
3346         .is_idle = NULL,
3347         .check_soft_reset = NULL,
3348         .wait_for_idle = NULL,
3349         .soft_reset = NULL,
3350         .set_clockgating_state = psp_set_clockgating_state,
3351         .set_powergating_state = psp_set_powergating_state,
3352 };
3353
3354 static int psp_sysfs_init(struct amdgpu_device *adev)
3355 {
3356         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3357
3358         if (ret)
3359                 DRM_ERROR("Failed to create USBC PD FW control file!");
3360
3361         return ret;
3362 }
3363
3364 static void psp_sysfs_fini(struct amdgpu_device *adev)
3365 {
3366         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3367 }
3368
3369 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
3370 {
3371         .type = AMD_IP_BLOCK_TYPE_PSP,
3372         .major = 3,
3373         .minor = 1,
3374         .rev = 0,
3375         .funcs = &psp_ip_funcs,
3376 };
3377
3378 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
3379 {
3380         .type = AMD_IP_BLOCK_TYPE_PSP,
3381         .major = 10,
3382         .minor = 0,
3383         .rev = 0,
3384         .funcs = &psp_ip_funcs,
3385 };
3386
3387 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
3388 {
3389         .type = AMD_IP_BLOCK_TYPE_PSP,
3390         .major = 11,
3391         .minor = 0,
3392         .rev = 0,
3393         .funcs = &psp_ip_funcs,
3394 };
3395
3396 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3397         .type = AMD_IP_BLOCK_TYPE_PSP,
3398         .major = 11,
3399         .minor = 0,
3400         .rev = 8,
3401         .funcs = &psp_ip_funcs,
3402 };
3403
3404 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
3405 {
3406         .type = AMD_IP_BLOCK_TYPE_PSP,
3407         .major = 12,
3408         .minor = 0,
3409         .rev = 0,
3410         .funcs = &psp_ip_funcs,
3411 };
3412
3413 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3414         .type = AMD_IP_BLOCK_TYPE_PSP,
3415         .major = 13,
3416         .minor = 0,
3417         .rev = 0,
3418         .funcs = &psp_ip_funcs,
3419 };