drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40 #include "psp_v13_0_4.h"
  41 #include "psp_v14_0.h"
  42
  43 #include "amdgpu_ras.h"
  44 #include "amdgpu_securedisplay.h"
  45 #include "amdgpu_atomfirmware.h"
  46
  47 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*3)
  48
  49 static int psp_load_smu_fw(struct psp_context *psp);
  50 static int psp_rap_terminate(struct psp_context *psp);
  51 static int psp_securedisplay_terminate(struct psp_context *psp);
  52
  53 static int psp_ring_init(struct psp_context *psp,
  54                          enum psp_ring_type ring_type)
  55 {
  56         int ret = 0;
  57         struct psp_ring *ring;
  58         struct amdgpu_device *adev = psp->adev;
  59
  60         ring = &psp->km_ring;
  61
  62         ring->ring_type = ring_type;
  63
  64         /* allocate 4k Page of Local Frame Buffer memory for ring */
  65         ring->ring_size = 0x1000;
  66         ret = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE,
  67                                       AMDGPU_GEM_DOMAIN_VRAM |
  68                                       AMDGPU_GEM_DOMAIN_GTT,
  69                                       &adev->firmware.rbuf,
  70                                       &ring->ring_mem_mc_addr,
  71                                       (void **)&ring->ring_mem);
  72         if (ret) {
  73                 ring->ring_size = 0;
  74                 return ret;
  75         }
  76
  77         return 0;
  78 }
  79
  80 /*
  81  * Due to DF Cstate management centralized to PMFW, the firmware
  82  * loading sequence will be updated as below:
  83  *   - Load KDB
  84  *   - Load SYS_DRV
  85  *   - Load tOS
  86  *   - Load PMFW
  87  *   - Setup TMR
  88  *   - Load other non-psp fw
  89  *   - Load ASD
  90  *   - Load XGMI/RAS/HDCP/DTM TA if any
  91  *
  92  * This new sequence is required for
  93  *   - Arcturus and onwards
  94  */
  95 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  96 {
  97         struct amdgpu_device *adev = psp->adev;
  98
  99         if (amdgpu_sriov_vf(adev)) {
 100                 psp->pmfw_centralized_cstate_management = false;
 101                 return;
 102         }
 103
 104         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
 105         case IP_VERSION(11, 0, 0):
 106         case IP_VERSION(11, 0, 4):
 107         case IP_VERSION(11, 0, 5):
 108         case IP_VERSION(11, 0, 7):
 109         case IP_VERSION(11, 0, 9):
 110         case IP_VERSION(11, 0, 11):
 111         case IP_VERSION(11, 0, 12):
 112         case IP_VERSION(11, 0, 13):
 113         case IP_VERSION(13, 0, 0):
 114         case IP_VERSION(13, 0, 2):
 115         case IP_VERSION(13, 0, 7):
 116                 psp->pmfw_centralized_cstate_management = true;
 117                 break;
 118         default:
 119                 psp->pmfw_centralized_cstate_management = false;
 120                 break;
 121         }
 122 }
 123
 124 static int psp_init_sriov_microcode(struct psp_context *psp)
 125 {
 126         struct amdgpu_device *adev = psp->adev;
 127         char ucode_prefix[30];
 128         int ret = 0;
 129
 130         amdgpu_ucode_ip_version_decode(adev, MP0_HWIP, ucode_prefix, sizeof(ucode_prefix));
 131
 132         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
 133         case IP_VERSION(9, 0, 0):
 134         case IP_VERSION(11, 0, 7):
 135         case IP_VERSION(11, 0, 9):
 136                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 137                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 138                 break;
 139         case IP_VERSION(13, 0, 2):
 140                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 141                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 142                 ret &= psp_init_ta_microcode(psp, ucode_prefix);
 143                 break;
 144         case IP_VERSION(13, 0, 0):
 145                 adev->virt.autoload_ucode_id = 0;
 146                 break;
 147         case IP_VERSION(13, 0, 6):
 148         case IP_VERSION(13, 0, 14):
 149                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 150                 ret &= psp_init_ta_microcode(psp, ucode_prefix);
 151                 break;
 152         case IP_VERSION(13, 0, 10):
 153                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
 154                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 155                 break;
 156         default:
 157                 return -EINVAL;
 158         }
 159         return ret;
 160 }
 161
 162 static int psp_early_init(void *handle)
 163 {
 164         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 165         struct psp_context *psp = &adev->psp;
 166
 167         psp->autoload_supported = true;
 168         psp->boot_time_tmr = true;
 169
 170         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
 171         case IP_VERSION(9, 0, 0):
 172                 psp_v3_1_set_psp_funcs(psp);
 173                 psp->autoload_supported = false;
 174                 psp->boot_time_tmr = false;
 175                 break;
 176         case IP_VERSION(10, 0, 0):
 177         case IP_VERSION(10, 0, 1):
 178                 psp_v10_0_set_psp_funcs(psp);
 179                 psp->autoload_supported = false;
 180                 psp->boot_time_tmr = false;
 181                 break;
 182         case IP_VERSION(11, 0, 2):
 183         case IP_VERSION(11, 0, 4):
 184                 psp_v11_0_set_psp_funcs(psp);
 185                 psp->autoload_supported = false;
 186                 psp->boot_time_tmr = false;
 187                 break;
 188         case IP_VERSION(11, 0, 0):
 189         case IP_VERSION(11, 0, 7):
 190                 adev->psp.sup_pd_fw_up = !amdgpu_sriov_vf(adev);
 191                 fallthrough;
 192         case IP_VERSION(11, 0, 5):
 193         case IP_VERSION(11, 0, 9):
 194         case IP_VERSION(11, 0, 11):
 195         case IP_VERSION(11, 5, 0):
 196         case IP_VERSION(11, 0, 12):
 197         case IP_VERSION(11, 0, 13):
 198                 psp_v11_0_set_psp_funcs(psp);
 199                 psp->boot_time_tmr = false;
 200                 break;
 201         case IP_VERSION(11, 0, 3):
 202         case IP_VERSION(12, 0, 1):
 203                 psp_v12_0_set_psp_funcs(psp);
 204                 psp->autoload_supported = false;
 205                 psp->boot_time_tmr = false;
 206                 break;
 207         case IP_VERSION(13, 0, 2):
 208                 psp->boot_time_tmr = false;
 209                 fallthrough;
 210         case IP_VERSION(13, 0, 6):
 211         case IP_VERSION(13, 0, 14):
 212                 psp_v13_0_set_psp_funcs(psp);
 213                 psp->autoload_supported = false;
 214                 break;
 215         case IP_VERSION(13, 0, 1):
 216         case IP_VERSION(13, 0, 3):
 217         case IP_VERSION(13, 0, 5):
 218         case IP_VERSION(13, 0, 8):
 219         case IP_VERSION(13, 0, 11):
 220         case IP_VERSION(14, 0, 0):
 221         case IP_VERSION(14, 0, 1):
 222         case IP_VERSION(14, 0, 4):
 223                 psp_v13_0_set_psp_funcs(psp);
 224                 psp->boot_time_tmr = false;
 225                 break;
 226         case IP_VERSION(11, 0, 8):
 227                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 228                         psp_v11_0_8_set_psp_funcs(psp);
 229                 }
 230                 psp->autoload_supported = false;
 231                 psp->boot_time_tmr = false;
 232                 break;
 233         case IP_VERSION(13, 0, 0):
 234         case IP_VERSION(13, 0, 7):
 235         case IP_VERSION(13, 0, 10):
 236                 psp_v13_0_set_psp_funcs(psp);
 237                 adev->psp.sup_ifwi_up = !amdgpu_sriov_vf(adev);
 238                 psp->boot_time_tmr = false;
 239                 break;
 240         case IP_VERSION(13, 0, 4):
 241                 psp_v13_0_4_set_psp_funcs(psp);
 242                 psp->boot_time_tmr = false;
 243                 break;
 244         case IP_VERSION(14, 0, 2):
 245         case IP_VERSION(14, 0, 3):
 246                 psp_v14_0_set_psp_funcs(psp);
 247                 break;
 248         default:
 249                 return -EINVAL;
 250         }
 251
 252         psp->adev = adev;
 253
 254         adev->psp_timeout = 20000;
 255
 256         psp_check_pmfw_centralized_cstate_management(psp);
 257
 258         if (amdgpu_sriov_vf(adev))
 259                 return psp_init_sriov_microcode(psp);
 260         else
 261                 return psp_init_microcode(psp);
 262 }
 263
 264 void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 265 {
 266         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 267                               &mem_ctx->shared_buf);
 268         mem_ctx->shared_bo = NULL;
 269 }
 270
 271 static void psp_free_shared_bufs(struct psp_context *psp)
 272 {
 273         void *tmr_buf;
 274         void **pptr;
 275
 276         /* free TMR memory buffer */
 277         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 278         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 279         psp->tmr_bo = NULL;
 280
 281         /* free xgmi shared memory */
 282         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
 283
 284         /* free ras shared memory */
 285         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
 286
 287         /* free hdcp shared memory */
 288         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
 289
 290         /* free dtm shared memory */
 291         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
 292
 293         /* free rap shared memory */
 294         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
 295
 296         /* free securedisplay shared memory */
 297         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
 298
 299
 300 }
 301
 302 static void psp_memory_training_fini(struct psp_context *psp)
 303 {
 304         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 305
 306         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 307         kfree(ctx->sys_cache);
 308         ctx->sys_cache = NULL;
 309 }
 310
 311 static int psp_memory_training_init(struct psp_context *psp)
 312 {
 313         int ret;
 314         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 315
 316         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 317                 dev_dbg(psp->adev->dev, "memory training is not supported!\n");
 318                 return 0;
 319         }
 320
 321         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 322         if (ctx->sys_cache == NULL) {
 323                 dev_err(psp->adev->dev, "alloc mem_train_ctx.sys_cache failed!\n");
 324                 ret = -ENOMEM;
 325                 goto Err_out;
 326         }
 327
 328         dev_dbg(psp->adev->dev,
 329                 "train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 330                 ctx->train_data_size,
 331                 ctx->p2c_train_data_offset,
 332                 ctx->c2p_train_data_offset);
 333         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 334         return 0;
 335
 336 Err_out:
 337         psp_memory_training_fini(psp);
 338         return ret;
 339 }
 340
 341 /*
 342  * Helper funciton to query psp runtime database entry
 343  *
 344  * @adev: amdgpu_device pointer
 345  * @entry_type: the type of psp runtime database entry
 346  * @db_entry: runtime database entry pointer
 347  *
 348  * Return false if runtime database doesn't exit or entry is invalid
 349  * or true if the specific database entry is found, and copy to @db_entry
 350  */
 351 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 352                                      enum psp_runtime_entry_type entry_type,
 353                                      void *db_entry)
 354 {
 355         uint64_t db_header_pos, db_dir_pos;
 356         struct psp_runtime_data_header db_header = {0};
 357         struct psp_runtime_data_directory db_dir = {0};
 358         bool ret = false;
 359         int i;
 360
 361         if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
 362             amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14))
 363                 return false;
 364
 365         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 366         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 367
 368         /* read runtime db header from vram */
 369         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 370                         sizeof(struct psp_runtime_data_header), false);
 371
 372         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 373                 /* runtime db doesn't exist, exit */
 374                 dev_dbg(adev->dev, "PSP runtime database doesn't exist\n");
 375                 return false;
 376         }
 377
 378         /* read runtime database entry from vram */
 379         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 380                         sizeof(struct psp_runtime_data_directory), false);
 381
 382         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 383                 /* invalid db entry count, exit */
 384                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 385                 return false;
 386         }
 387
 388         /* look up for requested entry type */
 389         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 390                 if (db_dir.entry_list[i].entry_type == entry_type) {
 391                         switch (entry_type) {
 392                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 393                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 394                                         /* invalid db entry size */
 395                                         dev_warn(adev->dev, "Invalid PSP runtime database boot cfg entry size\n");
 396                                         return false;
 397                                 }
 398                                 /* read runtime database entry */
 399                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 400                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 401                                 ret = true;
 402                                 break;
 403                         case PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS:
 404                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_scpm_entry)) {
 405                                         /* invalid db entry size */
 406                                         dev_warn(adev->dev, "Invalid PSP runtime database scpm entry size\n");
 407                                         return false;
 408                                 }
 409                                 /* read runtime database entry */
 410                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 411                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_scpm_entry), false);
 412                                 ret = true;
 413                                 break;
 414                         default:
 415                                 ret = false;
 416                                 break;
 417                         }
 418                 }
 419         }
 420
 421         return ret;
 422 }
 423
 424 static int psp_sw_init(void *handle)
 425 {
 426         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 427         struct psp_context *psp = &adev->psp;
 428         int ret;
 429         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 430         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 431         struct psp_runtime_scpm_entry scpm_entry;
 432
 433         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 434         if (!psp->cmd) {
 435                 dev_err(adev->dev, "Failed to allocate memory to command buffer!\n");
 436                 ret = -ENOMEM;
 437         }
 438
 439         adev->psp.xgmi_context.supports_extended_data =
 440                 !adev->gmc.xgmi.connected_to_cpu &&
 441                 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 2);
 442
 443         memset(&scpm_entry, 0, sizeof(scpm_entry));
 444         if ((psp_get_runtime_db_entry(adev,
 445                                 PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS,
 446                                 &scpm_entry)) &&
 447             (scpm_entry.scpm_status != SCPM_DISABLE)) {
 448                 adev->scpm_enabled = true;
 449                 adev->scpm_status = scpm_entry.scpm_status;
 450         } else {
 451                 adev->scpm_enabled = false;
 452                 adev->scpm_status = SCPM_DISABLE;
 453         }
 454
 455         /* TODO: stop gpu driver services and print alarm if scpm is enabled with error status */
 456
 457         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 458         if (psp_get_runtime_db_entry(adev,
 459                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 460                                 &boot_cfg_entry)) {
 461                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 462                 if ((psp->boot_cfg_bitmask) &
 463                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 464                         /* If psp runtime database exists, then
 465                          * only enable two stage memory training
 466                          * when TWO_STAGE_DRAM_TRAINING bit is set
 467                          * in runtime database
 468                          */
 469                         mem_training_ctx->enable_mem_training = true;
 470                 }
 471
 472         } else {
 473                 /* If psp runtime database doesn't exist or is
 474                  * invalid, force enable two stage memory training
 475                  */
 476                 mem_training_ctx->enable_mem_training = true;
 477         }
 478
 479         if (mem_training_ctx->enable_mem_training) {
 480                 ret = psp_memory_training_init(psp);
 481                 if (ret) {
 482                         dev_err(adev->dev, "Failed to initialize memory training!\n");
 483                         return ret;
 484                 }
 485
 486                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 487                 if (ret) {
 488                         dev_err(adev->dev, "Failed to process memory training!\n");
 489                         return ret;
 490                 }
 491         }
 492
 493         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 494                                       (amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf) ?
 495                                       AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 496                                       &psp->fw_pri_bo,
 497                                       &psp->fw_pri_mc_addr,
 498                                       &psp->fw_pri_buf);
 499         if (ret)
 500                 return ret;
 501
 502         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 503                                       AMDGPU_GEM_DOMAIN_VRAM |
 504                                       AMDGPU_GEM_DOMAIN_GTT,
 505                                       &psp->fence_buf_bo,
 506                                       &psp->fence_buf_mc_addr,
 507                                       &psp->fence_buf);
 508         if (ret)
 509                 goto failed1;
 510
 511         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 512                                       AMDGPU_GEM_DOMAIN_VRAM |
 513                                       AMDGPU_GEM_DOMAIN_GTT,
 514                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 515                                       (void **)&psp->cmd_buf_mem);
 516         if (ret)
 517                 goto failed2;
 518
 519         return 0;
 520
 521 failed2:
 522         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 523                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 524 failed1:
 525         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 526                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 527         return ret;
 528 }
 529
 530 static int psp_sw_fini(void *handle)
 531 {
 532         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 533         struct psp_context *psp = &adev->psp;
 534         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 535
 536         psp_memory_training_fini(psp);
 537
 538         amdgpu_ucode_release(&psp->sos_fw);
 539         amdgpu_ucode_release(&psp->asd_fw);
 540         amdgpu_ucode_release(&psp->ta_fw);
 541         amdgpu_ucode_release(&psp->cap_fw);
 542         amdgpu_ucode_release(&psp->toc_fw);
 543
 544         kfree(cmd);
 545         cmd = NULL;
 546
 547         psp_free_shared_bufs(psp);
 548
 549         if (psp->km_ring.ring_mem)
 550                 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
 551                                       &psp->km_ring.ring_mem_mc_addr,
 552                                       (void **)&psp->km_ring.ring_mem);
 553
 554         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 555                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 556         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 557                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 558         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 559                               (void **)&psp->cmd_buf_mem);
 560
 561         return 0;
 562 }
 563
 564 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 565                  uint32_t reg_val, uint32_t mask, bool check_changed)
 566 {
 567         uint32_t val;
 568         int i;
 569         struct amdgpu_device *adev = psp->adev;
 570
 571         if (psp->adev->no_hw_access)
 572                 return 0;
 573
 574         for (i = 0; i < adev->usec_timeout; i++) {
 575                 val = RREG32(reg_index);
 576                 if (check_changed) {
 577                         if (val != reg_val)
 578                                 return 0;
 579                 } else {
 580                         if ((val & mask) == reg_val)
 581                                 return 0;
 582                 }
 583                 udelay(1);
 584         }
 585
 586         return -ETIME;
 587 }
 588
 589 int psp_wait_for_spirom_update(struct psp_context *psp, uint32_t reg_index,
 590                                uint32_t reg_val, uint32_t mask, uint32_t msec_timeout)
 591 {
 592         uint32_t val;
 593         int i;
 594         struct amdgpu_device *adev = psp->adev;
 595
 596         if (psp->adev->no_hw_access)
 597                 return 0;
 598
 599         for (i = 0; i < msec_timeout; i++) {
 600                 val = RREG32(reg_index);
 601                 if ((val & mask) == reg_val)
 602                         return 0;
 603                 msleep(1);
 604         }
 605
 606         return -ETIME;
 607 }
 608
 609 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 610 {
 611         switch (cmd_id) {
 612         case GFX_CMD_ID_LOAD_TA:
 613                 return "LOAD_TA";
 614         case GFX_CMD_ID_UNLOAD_TA:
 615                 return "UNLOAD_TA";
 616         case GFX_CMD_ID_INVOKE_CMD:
 617                 return "INVOKE_CMD";
 618         case GFX_CMD_ID_LOAD_ASD:
 619                 return "LOAD_ASD";
 620         case GFX_CMD_ID_SETUP_TMR:
 621                 return "SETUP_TMR";
 622         case GFX_CMD_ID_LOAD_IP_FW:
 623                 return "LOAD_IP_FW";
 624         case GFX_CMD_ID_DESTROY_TMR:
 625                 return "DESTROY_TMR";
 626         case GFX_CMD_ID_SAVE_RESTORE:
 627                 return "SAVE_RESTORE_IP_FW";
 628         case GFX_CMD_ID_SETUP_VMR:
 629                 return "SETUP_VMR";
 630         case GFX_CMD_ID_DESTROY_VMR:
 631                 return "DESTROY_VMR";
 632         case GFX_CMD_ID_PROG_REG:
 633                 return "PROG_REG";
 634         case GFX_CMD_ID_GET_FW_ATTESTATION:
 635                 return "GET_FW_ATTESTATION";
 636         case GFX_CMD_ID_LOAD_TOC:
 637                 return "ID_LOAD_TOC";
 638         case GFX_CMD_ID_AUTOLOAD_RLC:
 639                 return "AUTOLOAD_RLC";
 640         case GFX_CMD_ID_BOOT_CFG:
 641                 return "BOOT_CFG";
 642         default:
 643                 return "UNKNOWN CMD";
 644         }
 645 }
 646
 647 static bool psp_err_warn(struct psp_context *psp)
 648 {
 649         struct psp_gfx_cmd_resp *cmd = psp->cmd_buf_mem;
 650
 651         /* This response indicates reg list is already loaded */
 652         if (amdgpu_ip_version(psp->adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 2) &&
 653             cmd->cmd_id == GFX_CMD_ID_LOAD_IP_FW &&
 654             cmd->cmd.cmd_load_ip_fw.fw_type == GFX_FW_TYPE_REG_LIST &&
 655             cmd->resp.status == TEE_ERROR_CANCEL)
 656                 return false;
 657
 658         return true;
 659 }
 660
 661 static int
 662 psp_cmd_submit_buf(struct psp_context *psp,
 663                    struct amdgpu_firmware_info *ucode,
 664                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 665 {
 666         int ret;
 667         int index;
 668         int timeout = psp->adev->psp_timeout;
 669         bool ras_intr = false;
 670         bool skip_unsupport = false;
 671
 672         if (psp->adev->no_hw_access)
 673                 return 0;
 674
 675         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 676
 677         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 678
 679         index = atomic_inc_return(&psp->fence_value);
 680         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 681         if (ret) {
 682                 atomic_dec(&psp->fence_value);
 683                 goto exit;
 684         }
 685
 686         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 687         while (*((unsigned int *)psp->fence_buf) != index) {
 688                 if (--timeout == 0)
 689                         break;
 690                 /*
 691                  * Shouldn't wait for timeout when err_event_athub occurs,
 692                  * because gpu reset thread triggered and lock resource should
 693                  * be released for psp resume sequence.
 694                  */
 695                 ras_intr = amdgpu_ras_intr_triggered();
 696                 if (ras_intr)
 697                         break;
 698                 usleep_range(10, 100);
 699                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 700         }
 701
 702         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 703         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 704                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 705
 706         memcpy(&cmd->resp, &psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 707
 708         /* In some cases, psp response status is not 0 even there is no
 709          * problem while the command is submitted. Some version of PSP FW
 710          * doesn't write 0 to that field.
 711          * So here we would like to only print a warning instead of an error
 712          * during psp initialization to avoid breaking hw_init and it doesn't
 713          * return -EINVAL.
 714          */
 715         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 716                 if (ucode)
 717                         dev_warn(psp->adev->dev,
 718                                  "failed to load ucode %s(0x%X) ",
 719                                  amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 720                 if (psp_err_warn(psp))
 721                         dev_warn(
 722                                 psp->adev->dev,
 723                                 "psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 724                                 psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id),
 725                                 psp->cmd_buf_mem->cmd_id,
 726                                 psp->cmd_buf_mem->resp.status);
 727                 /* If any firmware (including CAP) load fails under SRIOV, it should
 728                  * return failure to stop the VF from initializing.
 729                  * Also return failure in case of timeout
 730                  */
 731                 if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
 732                         ret = -EINVAL;
 733                         goto exit;
 734                 }
 735         }
 736
 737         if (ucode) {
 738                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 739                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 740         }
 741
 742 exit:
 743         return ret;
 744 }
 745
 746 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 747 {
 748         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 749
 750         mutex_lock(&psp->mutex);
 751
 752         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 753
 754         return cmd;
 755 }
 756
 757 static void release_psp_cmd_buf(struct psp_context *psp)
 758 {
 759         mutex_unlock(&psp->mutex);
 760 }
 761
 762 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 763                                  struct psp_gfx_cmd_resp *cmd,
 764                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 765 {
 766         struct amdgpu_device *adev = psp->adev;
 767         uint32_t size = 0;
 768         uint64_t tmr_pa = 0;
 769
 770         if (tmr_bo) {
 771                 size = amdgpu_bo_size(tmr_bo);
 772                 tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 773         }
 774
 775         if (amdgpu_sriov_vf(psp->adev))
 776                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 777         else
 778                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 779         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 780         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 781         cmd->cmd.cmd_setup_tmr.buf_size = size;
 782         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 783         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 784         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 785 }
 786
 787 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 788                                       uint64_t pri_buf_mc, uint32_t size)
 789 {
 790         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 791         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 792         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 793         cmd->cmd.cmd_load_toc.toc_size = size;
 794 }
 795
 796 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 797 static int psp_load_toc(struct psp_context *psp,
 798                         uint32_t *tmr_size)
 799 {
 800         int ret;
 801         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 802
 803         /* Copy toc to psp firmware private buffer */
 804         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 805
 806         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 807
 808         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 809                                  psp->fence_buf_mc_addr);
 810         if (!ret)
 811                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 812
 813         release_psp_cmd_buf(psp);
 814
 815         return ret;
 816 }
 817
 818 /* Set up Trusted Memory Region */
 819 static int psp_tmr_init(struct psp_context *psp)
 820 {
 821         int ret = 0;
 822         int tmr_size;
 823         void *tmr_buf;
 824         void **pptr;
 825
 826         /*
 827          * According to HW engineer, they prefer the TMR address be "naturally
 828          * aligned" , e.g. the start address be an integer divide of TMR size.
 829          *
 830          * Note: this memory need be reserved till the driver
 831          * uninitializes.
 832          */
 833         tmr_size = PSP_TMR_SIZE(psp->adev);
 834
 835         /* For ASICs support RLC autoload, psp will parse the toc
 836          * and calculate the total size of TMR needed
 837          */
 838         if (!amdgpu_sriov_vf(psp->adev) &&
 839             psp->toc.start_addr &&
 840             psp->toc.size_bytes &&
 841             psp->fw_pri_buf) {
 842                 ret = psp_load_toc(psp, &tmr_size);
 843                 if (ret) {
 844                         dev_err(psp->adev->dev, "Failed to load toc\n");
 845                         return ret;
 846                 }
 847         }
 848
 849         if (!psp->tmr_bo && !psp->boot_time_tmr) {
 850                 pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 851                 ret = amdgpu_bo_create_kernel(psp->adev, tmr_size,
 852                                               PSP_TMR_ALIGNMENT,
 853                                               AMDGPU_HAS_VRAM(psp->adev) ?
 854                                               AMDGPU_GEM_DOMAIN_VRAM :
 855                                               AMDGPU_GEM_DOMAIN_GTT,
 856                                               &psp->tmr_bo, &psp->tmr_mc_addr,
 857                                               pptr);
 858         }
 859
 860         return ret;
 861 }
 862
 863 static bool psp_skip_tmr(struct psp_context *psp)
 864 {
 865         switch (amdgpu_ip_version(psp->adev, MP0_HWIP, 0)) {
 866         case IP_VERSION(11, 0, 9):
 867         case IP_VERSION(11, 0, 7):
 868         case IP_VERSION(13, 0, 2):
 869         case IP_VERSION(13, 0, 6):
 870         case IP_VERSION(13, 0, 10):
 871         case IP_VERSION(13, 0, 14):
 872                 return true;
 873         default:
 874                 return false;
 875         }
 876 }
 877
 878 static int psp_tmr_load(struct psp_context *psp)
 879 {
 880         int ret;
 881         struct psp_gfx_cmd_resp *cmd;
 882
 883         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 884          * Already set up by host driver.
 885          */
 886         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 887                 return 0;
 888
 889         cmd = acquire_psp_cmd_buf(psp);
 890
 891         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 892         if (psp->tmr_bo)
 893                 dev_info(psp->adev->dev, "reserve 0x%lx from 0x%llx for PSP TMR\n",
 894                          amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 895
 896         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 897                                  psp->fence_buf_mc_addr);
 898
 899         release_psp_cmd_buf(psp);
 900
 901         return ret;
 902 }
 903
 904 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 905                                         struct psp_gfx_cmd_resp *cmd)
 906 {
 907         if (amdgpu_sriov_vf(psp->adev))
 908                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 909         else
 910                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 911 }
 912
 913 static int psp_tmr_unload(struct psp_context *psp)
 914 {
 915         int ret;
 916         struct psp_gfx_cmd_resp *cmd;
 917
 918         /* skip TMR unload for Navi12 and CHIP_SIENNA_CICHLID SRIOV,
 919          * as TMR is not loaded at all
 920          */
 921         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 922                 return 0;
 923
 924         cmd = acquire_psp_cmd_buf(psp);
 925
 926         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 927         dev_dbg(psp->adev->dev, "free PSP TMR buffer\n");
 928
 929         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 930                                  psp->fence_buf_mc_addr);
 931
 932         release_psp_cmd_buf(psp);
 933
 934         return ret;
 935 }
 936
 937 static int psp_tmr_terminate(struct psp_context *psp)
 938 {
 939         return psp_tmr_unload(psp);
 940 }
 941
 942 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 943                                         uint64_t *output_ptr)
 944 {
 945         int ret;
 946         struct psp_gfx_cmd_resp *cmd;
 947
 948         if (!output_ptr)
 949                 return -EINVAL;
 950
 951         if (amdgpu_sriov_vf(psp->adev))
 952                 return 0;
 953
 954         cmd = acquire_psp_cmd_buf(psp);
 955
 956         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 957
 958         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 959                                  psp->fence_buf_mc_addr);
 960
 961         if (!ret) {
 962                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 963                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 964         }
 965
 966         release_psp_cmd_buf(psp);
 967
 968         return ret;
 969 }
 970
 971 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 972 {
 973         struct psp_context *psp = &adev->psp;
 974         struct psp_gfx_cmd_resp *cmd;
 975         int ret;
 976
 977         if (amdgpu_sriov_vf(adev))
 978                 return 0;
 979
 980         cmd = acquire_psp_cmd_buf(psp);
 981
 982         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 983         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 984
 985         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 986         if (!ret) {
 987                 *boot_cfg =
 988                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 989         }
 990
 991         release_psp_cmd_buf(psp);
 992
 993         return ret;
 994 }
 995
 996 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 997 {
 998         int ret;
 999         struct psp_context *psp = &adev->psp;
1000         struct psp_gfx_cmd_resp *cmd;
1001
1002         if (amdgpu_sriov_vf(adev))
1003                 return 0;
1004
1005         cmd = acquire_psp_cmd_buf(psp);
1006
1007         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
1008         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
1009         cmd->cmd.boot_cfg.boot_config = boot_cfg;
1010         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
1011
1012         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1013
1014         release_psp_cmd_buf(psp);
1015
1016         return ret;
1017 }
1018
1019 static int psp_rl_load(struct amdgpu_device *adev)
1020 {
1021         int ret;
1022         struct psp_context *psp = &adev->psp;
1023         struct psp_gfx_cmd_resp *cmd;
1024
1025         if (!is_psp_fw_valid(psp->rl))
1026                 return 0;
1027
1028         cmd = acquire_psp_cmd_buf(psp);
1029
1030         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1031         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
1032
1033         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1034         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
1035         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
1036         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
1037         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
1038
1039         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1040
1041         release_psp_cmd_buf(psp);
1042
1043         return ret;
1044 }
1045
1046 int psp_spatial_partition(struct psp_context *psp, int mode)
1047 {
1048         struct psp_gfx_cmd_resp *cmd;
1049         int ret;
1050
1051         if (amdgpu_sriov_vf(psp->adev))
1052                 return 0;
1053
1054         cmd = acquire_psp_cmd_buf(psp);
1055
1056         cmd->cmd_id = GFX_CMD_ID_SRIOV_SPATIAL_PART;
1057         cmd->cmd.cmd_spatial_part.mode = mode;
1058
1059         dev_info(psp->adev->dev, "Requesting %d partitions through PSP", mode);
1060         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1061
1062         release_psp_cmd_buf(psp);
1063
1064         return ret;
1065 }
1066
1067 static int psp_asd_initialize(struct psp_context *psp)
1068 {
1069         int ret;
1070
1071         /* If PSP version doesn't match ASD version, asd loading will be failed.
1072          * add workaround to bypass it for sriov now.
1073          * TODO: add version check to make it common
1074          */
1075         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
1076                 return 0;
1077
1078         /* bypass asd if display hardware is not available */
1079         if (!amdgpu_device_has_display_hardware(psp->adev) &&
1080             amdgpu_ip_version(psp->adev, MP0_HWIP, 0) >= IP_VERSION(13, 0, 10))
1081                 return 0;
1082
1083         psp->asd_context.mem_context.shared_mc_addr  = 0;
1084         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
1085         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
1086
1087         ret = psp_ta_load(psp, &psp->asd_context);
1088         if (!ret)
1089                 psp->asd_context.initialized = true;
1090
1091         return ret;
1092 }
1093
1094 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1095                                        uint32_t session_id)
1096 {
1097         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
1098         cmd->cmd.cmd_unload_ta.session_id = session_id;
1099 }
1100
1101 int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
1102 {
1103         int ret;
1104         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1105
1106         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
1107
1108         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1109
1110         context->resp_status = cmd->resp.status;
1111
1112         release_psp_cmd_buf(psp);
1113
1114         return ret;
1115 }
1116
1117 static int psp_asd_terminate(struct psp_context *psp)
1118 {
1119         int ret;
1120
1121         if (amdgpu_sriov_vf(psp->adev))
1122                 return 0;
1123
1124         if (!psp->asd_context.initialized)
1125                 return 0;
1126
1127         ret = psp_ta_unload(psp, &psp->asd_context);
1128         if (!ret)
1129                 psp->asd_context.initialized = false;
1130
1131         return ret;
1132 }
1133
1134 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1135                 uint32_t id, uint32_t value)
1136 {
1137         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
1138         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
1139         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
1140 }
1141
1142 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
1143                 uint32_t value)
1144 {
1145         struct psp_gfx_cmd_resp *cmd;
1146         int ret = 0;
1147
1148         if (reg >= PSP_REG_LAST)
1149                 return -EINVAL;
1150
1151         cmd = acquire_psp_cmd_buf(psp);
1152
1153         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
1154         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1155         if (ret)
1156                 dev_err(psp->adev->dev, "PSP failed to program reg id %d\n", reg);
1157
1158         release_psp_cmd_buf(psp);
1159
1160         return ret;
1161 }
1162
1163 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1164                                      uint64_t ta_bin_mc,
1165                                      struct ta_context *context)
1166 {
1167         cmd->cmd_id                             = context->ta_load_type;
1168         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
1169         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
1170         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
1171
1172         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
1173                 lower_32_bits(context->mem_context.shared_mc_addr);
1174         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
1175                 upper_32_bits(context->mem_context.shared_mc_addr);
1176         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
1177 }
1178
1179 int psp_ta_init_shared_buf(struct psp_context *psp,
1180                                   struct ta_mem_context *mem_ctx)
1181 {
1182         /*
1183          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1184          * physical) for ta to host memory
1185          */
1186         return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
1187                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM |
1188                                       AMDGPU_GEM_DOMAIN_GTT,
1189                                       &mem_ctx->shared_bo,
1190                                       &mem_ctx->shared_mc_addr,
1191                                       &mem_ctx->shared_buf);
1192 }
1193
1194 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1195                                        uint32_t ta_cmd_id,
1196                                        uint32_t session_id)
1197 {
1198         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
1199         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
1200         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
1201 }
1202
1203 int psp_ta_invoke(struct psp_context *psp,
1204                   uint32_t ta_cmd_id,
1205                   struct ta_context *context)
1206 {
1207         int ret;
1208         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1209
1210         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
1211
1212         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1213                                  psp->fence_buf_mc_addr);
1214
1215         context->resp_status = cmd->resp.status;
1216
1217         release_psp_cmd_buf(psp);
1218
1219         return ret;
1220 }
1221
1222 int psp_ta_load(struct psp_context *psp, struct ta_context *context)
1223 {
1224         int ret;
1225         struct psp_gfx_cmd_resp *cmd;
1226
1227         cmd = acquire_psp_cmd_buf(psp);
1228
1229         psp_copy_fw(psp, context->bin_desc.start_addr,
1230                     context->bin_desc.size_bytes);
1231
1232         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
1233
1234         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1235                                  psp->fence_buf_mc_addr);
1236
1237         context->resp_status = cmd->resp.status;
1238
1239         if (!ret)
1240                 context->session_id = cmd->resp.session_id;
1241
1242         release_psp_cmd_buf(psp);
1243
1244         return ret;
1245 }
1246
1247 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1248 {
1249         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1250 }
1251
1252 int psp_xgmi_terminate(struct psp_context *psp)
1253 {
1254         int ret;
1255         struct amdgpu_device *adev = psp->adev;
1256
1257         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1258         if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(11, 0, 4) ||
1259             (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 2) &&
1260              adev->gmc.xgmi.connected_to_cpu))
1261                 return 0;
1262
1263         if (!psp->xgmi_context.context.initialized)
1264                 return 0;
1265
1266         ret = psp_ta_unload(psp, &psp->xgmi_context.context);
1267
1268         psp->xgmi_context.context.initialized = false;
1269
1270         return ret;
1271 }
1272
1273 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1274 {
1275         struct ta_xgmi_shared_memory *xgmi_cmd;
1276         int ret;
1277
1278         if (!psp->ta_fw ||
1279             !psp->xgmi_context.context.bin_desc.size_bytes ||
1280             !psp->xgmi_context.context.bin_desc.start_addr)
1281                 return -ENOENT;
1282
1283         if (!load_ta)
1284                 goto invoke;
1285
1286         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1287         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1288
1289         if (!psp->xgmi_context.context.mem_context.shared_buf) {
1290                 ret = psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
1291                 if (ret)
1292                         return ret;
1293         }
1294
1295         /* Load XGMI TA */
1296         ret = psp_ta_load(psp, &psp->xgmi_context.context);
1297         if (!ret)
1298                 psp->xgmi_context.context.initialized = true;
1299         else
1300                 return ret;
1301
1302 invoke:
1303         /* Initialize XGMI session */
1304         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1305         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1306         xgmi_cmd->flag_extend_link_record = set_extended_data;
1307         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1308
1309         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1310         /* note down the capbility flag for XGMI TA */
1311         psp->xgmi_context.xgmi_ta_caps = xgmi_cmd->caps_flag;
1312
1313         return ret;
1314 }
1315
1316 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1317 {
1318         struct ta_xgmi_shared_memory *xgmi_cmd;
1319         int ret;
1320
1321         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1322         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1323
1324         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1325
1326         /* Invoke xgmi ta to get hive id */
1327         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1328         if (ret)
1329                 return ret;
1330
1331         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1332
1333         return 0;
1334 }
1335
1336 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1337 {
1338         struct ta_xgmi_shared_memory *xgmi_cmd;
1339         int ret;
1340
1341         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1342         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1343
1344         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1345
1346         /* Invoke xgmi ta to get the node id */
1347         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1348         if (ret)
1349                 return ret;
1350
1351         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1352
1353         return 0;
1354 }
1355
1356 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1357 {
1358         return (amdgpu_ip_version(psp->adev, MP0_HWIP, 0) ==
1359                         IP_VERSION(13, 0, 2) &&
1360                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b) ||
1361                amdgpu_ip_version(psp->adev, MP0_HWIP, 0) >=
1362                        IP_VERSION(13, 0, 6);
1363 }
1364
1365 /*
1366  * Chips that support extended topology information require the driver to
1367  * reflect topology information in the opposite direction.  This is
1368  * because the TA has already exceeded its link record limit and if the
1369  * TA holds bi-directional information, the driver would have to do
1370  * multiple fetches instead of just two.
1371  */
1372 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1373                                         struct psp_xgmi_node_info node_info)
1374 {
1375         struct amdgpu_device *mirror_adev;
1376         struct amdgpu_hive_info *hive;
1377         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1378         uint64_t dst_node_id = node_info.node_id;
1379         uint8_t dst_num_hops = node_info.num_hops;
1380         uint8_t dst_num_links = node_info.num_links;
1381
1382         hive = amdgpu_get_xgmi_hive(psp->adev);
1383         if (WARN_ON(!hive))
1384                 return;
1385
1386         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1387                 struct psp_xgmi_topology_info *mirror_top_info;
1388                 int j;
1389
1390                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1391                         continue;
1392
1393                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1394                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1395                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1396                                 continue;
1397
1398                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1399                         /*
1400                          * prevent 0 num_links value re-reflection since reflection
1401                          * criteria is based on num_hops (direct or indirect).
1402                          *
1403                          */
1404                         if (dst_num_links)
1405                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1406
1407                         break;
1408                 }
1409
1410                 break;
1411         }
1412
1413         amdgpu_put_xgmi_hive(hive);
1414 }
1415
1416 int psp_xgmi_get_topology_info(struct psp_context *psp,
1417                                int number_devices,
1418                                struct psp_xgmi_topology_info *topology,
1419                                bool get_extended_data)
1420 {
1421         struct ta_xgmi_shared_memory *xgmi_cmd;
1422         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1423         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1424         int i;
1425         int ret;
1426
1427         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1428                 return -EINVAL;
1429
1430         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1431         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1432         xgmi_cmd->flag_extend_link_record = get_extended_data;
1433
1434         /* Fill in the shared memory with topology information as input */
1435         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1436         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_TOPOLOGY_INFO;
1437         topology_info_input->num_nodes = number_devices;
1438
1439         for (i = 0; i < topology_info_input->num_nodes; i++) {
1440                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1441                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1442                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1443                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1444         }
1445
1446         /* Invoke xgmi ta to get the topology information */
1447         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_TOPOLOGY_INFO);
1448         if (ret)
1449                 return ret;
1450
1451         /* Read the output topology information from the shared memory */
1452         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1453         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1454         for (i = 0; i < topology->num_nodes; i++) {
1455                 /* extended data will either be 0 or equal to non-extended data */
1456                 if (topology_info_output->nodes[i].num_hops)
1457                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1458
1459                 /* non-extended data gets everything here so no need to update */
1460                 if (!get_extended_data) {
1461                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1462                         topology->nodes[i].is_sharing_enabled =
1463                                         topology_info_output->nodes[i].is_sharing_enabled;
1464                         topology->nodes[i].sdma_engine =
1465                                         topology_info_output->nodes[i].sdma_engine;
1466                 }
1467
1468         }
1469
1470         /* Invoke xgmi ta again to get the link information */
1471         if (psp_xgmi_peer_link_info_supported(psp)) {
1472                 struct ta_xgmi_cmd_get_peer_link_info *link_info_output;
1473                 struct ta_xgmi_cmd_get_extend_peer_link_info *link_extend_info_output;
1474                 bool requires_reflection =
1475                         (psp->xgmi_context.supports_extended_data &&
1476                          get_extended_data) ||
1477                         amdgpu_ip_version(psp->adev, MP0_HWIP, 0) ==
1478                                 IP_VERSION(13, 0, 6) ||
1479                         amdgpu_ip_version(psp->adev, MP0_HWIP, 0) ==
1480                                 IP_VERSION(13, 0, 14);
1481                 bool ta_port_num_support = amdgpu_sriov_vf(psp->adev) ? 0 :
1482                                 psp->xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG;
1483
1484                 /* popluate the shared output buffer rather than the cmd input buffer
1485                  * with node_ids as the input for GET_PEER_LINKS command execution.
1486                  * This is required for GET_PEER_LINKS per xgmi ta implementation.
1487                  * The same requirement for GET_EXTEND_PEER_LINKS command.
1488                  */
1489                 if (ta_port_num_support) {
1490                         link_extend_info_output = &xgmi_cmd->xgmi_out_message.get_extend_link_info;
1491
1492                         for (i = 0; i < topology->num_nodes; i++)
1493                                 link_extend_info_output->nodes[i].node_id = topology->nodes[i].node_id;
1494
1495                         link_extend_info_output->num_nodes = topology->num_nodes;
1496                         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_EXTEND_PEER_LINKS;
1497                 } else {
1498                         link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1499
1500                         for (i = 0; i < topology->num_nodes; i++)
1501                                 link_info_output->nodes[i].node_id = topology->nodes[i].node_id;
1502
1503                         link_info_output->num_nodes = topology->num_nodes;
1504                         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1505                 }
1506
1507                 ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1508                 if (ret)
1509                         return ret;
1510
1511                 for (i = 0; i < topology->num_nodes; i++) {
1512                         uint8_t node_num_links = ta_port_num_support ?
1513                                 link_extend_info_output->nodes[i].num_links : link_info_output->nodes[i].num_links;
1514                         /* accumulate num_links on extended data */
1515                         if (get_extended_data) {
1516                                 topology->nodes[i].num_links = topology->nodes[i].num_links + node_num_links;
1517                         } else {
1518                                 topology->nodes[i].num_links = (requires_reflection && topology->nodes[i].num_links) ?
1519                                                                 topology->nodes[i].num_links : node_num_links;
1520                         }
1521                         /* popluate the connected port num info if supported and available */
1522                         if (ta_port_num_support && topology->nodes[i].num_links) {
1523                                 memcpy(topology->nodes[i].port_num, link_extend_info_output->nodes[i].port_num,
1524                                        sizeof(struct xgmi_connected_port_num) * TA_XGMI__MAX_PORT_NUM);
1525                         }
1526
1527                         /* reflect the topology information for bi-directionality */
1528                         if (requires_reflection && topology->nodes[i].num_hops)
1529                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1530                 }
1531         }
1532
1533         return 0;
1534 }
1535
1536 int psp_xgmi_set_topology_info(struct psp_context *psp,
1537                                int number_devices,
1538                                struct psp_xgmi_topology_info *topology)
1539 {
1540         struct ta_xgmi_shared_memory *xgmi_cmd;
1541         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1542         int i;
1543
1544         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1545                 return -EINVAL;
1546
1547         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1548         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1549
1550         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1551         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1552         topology_info_input->num_nodes = number_devices;
1553
1554         for (i = 0; i < topology_info_input->num_nodes; i++) {
1555                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1556                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1557                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1558                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1559         }
1560
1561         /* Invoke xgmi ta to set topology information */
1562         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1563 }
1564
1565 // ras begin
1566 static void psp_ras_ta_check_status(struct psp_context *psp)
1567 {
1568         struct ta_ras_shared_memory *ras_cmd =
1569                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1570
1571         switch (ras_cmd->ras_status) {
1572         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1573                 dev_warn(psp->adev->dev,
1574                          "RAS WARNING: cmd failed due to unsupported ip\n");
1575                 break;
1576         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1577                 dev_warn(psp->adev->dev,
1578                          "RAS WARNING: cmd failed due to unsupported error injection\n");
1579                 break;
1580         case TA_RAS_STATUS__SUCCESS:
1581                 break;
1582         case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED:
1583                 if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR)
1584                         dev_warn(psp->adev->dev,
1585                                  "RAS WARNING: Inject error to critical region is not allowed\n");
1586                 break;
1587         default:
1588                 dev_warn(psp->adev->dev,
1589                          "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1590                 break;
1591         }
1592 }
1593
1594 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1595 {
1596         struct ta_ras_shared_memory *ras_cmd;
1597         int ret;
1598
1599         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1600
1601         /*
1602          * TODO: bypass the loading in sriov for now
1603          */
1604         if (amdgpu_sriov_vf(psp->adev))
1605                 return 0;
1606
1607         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1608
1609         if (amdgpu_ras_intr_triggered())
1610                 return ret;
1611
1612         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER) {
1613                 dev_warn(psp->adev->dev, "RAS: Unsupported Interface\n");
1614                 return -EINVAL;
1615         }
1616
1617         if (!ret) {
1618                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1619                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1620
1621                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1622                 } else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1623                         dev_warn(psp->adev->dev,
1624                                  "RAS internal register access blocked\n");
1625
1626                 psp_ras_ta_check_status(psp);
1627         }
1628
1629         return ret;
1630 }
1631
1632 int psp_ras_enable_features(struct psp_context *psp,
1633                 union ta_ras_cmd_input *info, bool enable)
1634 {
1635         struct ta_ras_shared_memory *ras_cmd;
1636         int ret;
1637
1638         if (!psp->ras_context.context.initialized)
1639                 return -EINVAL;
1640
1641         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1642         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1643
1644         if (enable)
1645                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1646         else
1647                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1648
1649         ras_cmd->ras_in_message = *info;
1650
1651         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1652         if (ret)
1653                 return -EINVAL;
1654
1655         return 0;
1656 }
1657
1658 int psp_ras_terminate(struct psp_context *psp)
1659 {
1660         int ret;
1661
1662         /*
1663          * TODO: bypass the terminate in sriov for now
1664          */
1665         if (amdgpu_sriov_vf(psp->adev))
1666                 return 0;
1667
1668         if (!psp->ras_context.context.initialized)
1669                 return 0;
1670
1671         ret = psp_ta_unload(psp, &psp->ras_context.context);
1672
1673         psp->ras_context.context.initialized = false;
1674
1675         return ret;
1676 }
1677
1678 int psp_ras_initialize(struct psp_context *psp)
1679 {
1680         int ret;
1681         uint32_t boot_cfg = 0xFF;
1682         struct amdgpu_device *adev = psp->adev;
1683         struct ta_ras_shared_memory *ras_cmd;
1684
1685         /*
1686          * TODO: bypass the initialize in sriov for now
1687          */
1688         if (amdgpu_sriov_vf(adev))
1689                 return 0;
1690
1691         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1692             !adev->psp.ras_context.context.bin_desc.start_addr) {
1693                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1694                 return 0;
1695         }
1696
1697         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1698                 /* query GECC enablement status from boot config
1699                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1700                  */
1701                 ret = psp_boot_config_get(adev, &boot_cfg);
1702                 if (ret)
1703                         dev_warn(adev->dev, "PSP get boot config failed\n");
1704
1705                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1706                         if (!boot_cfg) {
1707                                 dev_info(adev->dev, "GECC is disabled\n");
1708                         } else {
1709                                 /* disable GECC in next boot cycle if ras is
1710                                  * disabled by module parameter amdgpu_ras_enable
1711                                  * and/or amdgpu_ras_mask, or boot_config_get call
1712                                  * is failed
1713                                  */
1714                                 ret = psp_boot_config_set(adev, 0);
1715                                 if (ret)
1716                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1717                                 else
1718                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1719                         }
1720                 } else {
1721                         if (boot_cfg == 1) {
1722                                 dev_info(adev->dev, "GECC is enabled\n");
1723                         } else {
1724                                 /* enable GECC in next boot cycle if it is disabled
1725                                  * in boot config, or force enable GECC if failed to
1726                                  * get boot configuration
1727                                  */
1728                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1729                                 if (ret)
1730                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1731                                 else
1732                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1733                         }
1734                 }
1735         }
1736
1737         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1738         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1739
1740         if (!psp->ras_context.context.mem_context.shared_buf) {
1741                 ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1742                 if (ret)
1743                         return ret;
1744         }
1745
1746         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1747         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1748
1749         if (amdgpu_ras_is_poison_mode_supported(adev))
1750                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1751         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1752                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1753         ras_cmd->ras_in_message.init_flags.xcc_mask =
1754                 adev->gfx.xcc_mask;
1755         ras_cmd->ras_in_message.init_flags.channel_dis_num = hweight32(adev->gmc.m_half_use) * 2;
1756
1757         ret = psp_ta_load(psp, &psp->ras_context.context);
1758
1759         if (!ret && !ras_cmd->ras_status)
1760                 psp->ras_context.context.initialized = true;
1761         else {
1762                 if (ras_cmd->ras_status)
1763                         dev_warn(adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1764
1765                 /* fail to load RAS TA */
1766                 psp->ras_context.context.initialized = false;
1767         }
1768
1769         return ret;
1770 }
1771
1772 int psp_ras_trigger_error(struct psp_context *psp,
1773                           struct ta_ras_trigger_error_input *info, uint32_t instance_mask)
1774 {
1775         struct ta_ras_shared_memory *ras_cmd;
1776         struct amdgpu_device *adev = psp->adev;
1777         int ret;
1778         uint32_t dev_mask;
1779
1780         if (!psp->ras_context.context.initialized)
1781                 return -EINVAL;
1782
1783         switch (info->block_id) {
1784         case TA_RAS_BLOCK__GFX:
1785                 dev_mask = GET_MASK(GC, instance_mask);
1786                 break;
1787         case TA_RAS_BLOCK__SDMA:
1788                 dev_mask = GET_MASK(SDMA0, instance_mask);
1789                 break;
1790         case TA_RAS_BLOCK__VCN:
1791         case TA_RAS_BLOCK__JPEG:
1792                 dev_mask = GET_MASK(VCN, instance_mask);
1793                 break;
1794         default:
1795                 dev_mask = instance_mask;
1796                 break;
1797         }
1798
1799         /* reuse sub_block_index for backward compatibility */
1800         dev_mask <<= AMDGPU_RAS_INST_SHIFT;
1801         dev_mask &= AMDGPU_RAS_INST_MASK;
1802         info->sub_block_index |= dev_mask;
1803
1804         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1805         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1806
1807         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1808         ras_cmd->ras_in_message.trigger_error = *info;
1809
1810         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1811         if (ret)
1812                 return -EINVAL;
1813
1814         /* If err_event_athub occurs error inject was successful, however
1815          *  return status from TA is no long reliable
1816          */
1817         if (amdgpu_ras_intr_triggered())
1818                 return 0;
1819
1820         if (ras_cmd->ras_status == TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED)
1821                 return -EACCES;
1822         else if (ras_cmd->ras_status)
1823                 return -EINVAL;
1824
1825         return 0;
1826 }
1827
1828 int psp_ras_query_address(struct psp_context *psp,
1829                           struct ta_ras_query_address_input *addr_in,
1830                           struct ta_ras_query_address_output *addr_out)
1831 {
1832         struct ta_ras_shared_memory *ras_cmd;
1833         int ret;
1834
1835         if (!psp->ras_context.context.initialized)
1836                 return -EINVAL;
1837
1838         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1839         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1840
1841         ras_cmd->cmd_id = TA_RAS_COMMAND__QUERY_ADDRESS;
1842         ras_cmd->ras_in_message.address = *addr_in;
1843
1844         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1845         if (ret || ras_cmd->ras_status || psp->cmd_buf_mem->resp.status)
1846                 return -EINVAL;
1847
1848         *addr_out = ras_cmd->ras_out_message.address;
1849
1850         return 0;
1851 }
1852 // ras end
1853
1854 // HDCP start
1855 static int psp_hdcp_initialize(struct psp_context *psp)
1856 {
1857         int ret;
1858
1859         /*
1860          * TODO: bypass the initialize in sriov for now
1861          */
1862         if (amdgpu_sriov_vf(psp->adev))
1863                 return 0;
1864
1865         /* bypass hdcp initialization if dmu is harvested */
1866         if (!amdgpu_device_has_display_hardware(psp->adev))
1867                 return 0;
1868
1869         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1870             !psp->hdcp_context.context.bin_desc.start_addr) {
1871                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1872                 return 0;
1873         }
1874
1875         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1876         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1877
1878         if (!psp->hdcp_context.context.mem_context.shared_buf) {
1879                 ret = psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1880                 if (ret)
1881                         return ret;
1882         }
1883
1884         ret = psp_ta_load(psp, &psp->hdcp_context.context);
1885         if (!ret) {
1886                 psp->hdcp_context.context.initialized = true;
1887                 mutex_init(&psp->hdcp_context.mutex);
1888         }
1889
1890         return ret;
1891 }
1892
1893 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1894 {
1895         /*
1896          * TODO: bypass the loading in sriov for now
1897          */
1898         if (amdgpu_sriov_vf(psp->adev))
1899                 return 0;
1900
1901         if (!psp->hdcp_context.context.initialized)
1902                 return 0;
1903
1904         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1905 }
1906
1907 static int psp_hdcp_terminate(struct psp_context *psp)
1908 {
1909         int ret;
1910
1911         /*
1912          * TODO: bypass the terminate in sriov for now
1913          */
1914         if (amdgpu_sriov_vf(psp->adev))
1915                 return 0;
1916
1917         if (!psp->hdcp_context.context.initialized)
1918                 return 0;
1919
1920         ret = psp_ta_unload(psp, &psp->hdcp_context.context);
1921
1922         psp->hdcp_context.context.initialized = false;
1923
1924         return ret;
1925 }
1926 // HDCP end
1927
1928 // DTM start
1929 static int psp_dtm_initialize(struct psp_context *psp)
1930 {
1931         int ret;
1932
1933         /*
1934          * TODO: bypass the initialize in sriov for now
1935          */
1936         if (amdgpu_sriov_vf(psp->adev))
1937                 return 0;
1938
1939         /* bypass dtm initialization if dmu is harvested */
1940         if (!amdgpu_device_has_display_hardware(psp->adev))
1941                 return 0;
1942
1943         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1944             !psp->dtm_context.context.bin_desc.start_addr) {
1945                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1946                 return 0;
1947         }
1948
1949         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1950         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1951
1952         if (!psp->dtm_context.context.mem_context.shared_buf) {
1953                 ret = psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1954                 if (ret)
1955                         return ret;
1956         }
1957
1958         ret = psp_ta_load(psp, &psp->dtm_context.context);
1959         if (!ret) {
1960                 psp->dtm_context.context.initialized = true;
1961                 mutex_init(&psp->dtm_context.mutex);
1962         }
1963
1964         return ret;
1965 }
1966
1967 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1968 {
1969         /*
1970          * TODO: bypass the loading in sriov for now
1971          */
1972         if (amdgpu_sriov_vf(psp->adev))
1973                 return 0;
1974
1975         if (!psp->dtm_context.context.initialized)
1976                 return 0;
1977
1978         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1979 }
1980
1981 static int psp_dtm_terminate(struct psp_context *psp)
1982 {
1983         int ret;
1984
1985         /*
1986          * TODO: bypass the terminate in sriov for now
1987          */
1988         if (amdgpu_sriov_vf(psp->adev))
1989                 return 0;
1990
1991         if (!psp->dtm_context.context.initialized)
1992                 return 0;
1993
1994         ret = psp_ta_unload(psp, &psp->dtm_context.context);
1995
1996         psp->dtm_context.context.initialized = false;
1997
1998         return ret;
1999 }
2000 // DTM end
2001
2002 // RAP start
2003 static int psp_rap_initialize(struct psp_context *psp)
2004 {
2005         int ret;
2006         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
2007
2008         /*
2009          * TODO: bypass the initialize in sriov for now
2010          */
2011         if (amdgpu_sriov_vf(psp->adev))
2012                 return 0;
2013
2014         if (!psp->rap_context.context.bin_desc.size_bytes ||
2015             !psp->rap_context.context.bin_desc.start_addr) {
2016                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
2017                 return 0;
2018         }
2019
2020         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
2021         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
2022
2023         if (!psp->rap_context.context.mem_context.shared_buf) {
2024                 ret = psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
2025                 if (ret)
2026                         return ret;
2027         }
2028
2029         ret = psp_ta_load(psp, &psp->rap_context.context);
2030         if (!ret) {
2031                 psp->rap_context.context.initialized = true;
2032                 mutex_init(&psp->rap_context.mutex);
2033         } else
2034                 return ret;
2035
2036         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
2037         if (ret || status != TA_RAP_STATUS__SUCCESS) {
2038                 psp_rap_terminate(psp);
2039                 /* free rap shared memory */
2040                 psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
2041
2042                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
2043                          ret, status);
2044
2045                 return ret;
2046         }
2047
2048         return 0;
2049 }
2050
2051 static int psp_rap_terminate(struct psp_context *psp)
2052 {
2053         int ret;
2054
2055         if (!psp->rap_context.context.initialized)
2056                 return 0;
2057
2058         ret = psp_ta_unload(psp, &psp->rap_context.context);
2059
2060         psp->rap_context.context.initialized = false;
2061
2062         return ret;
2063 }
2064
2065 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
2066 {
2067         struct ta_rap_shared_memory *rap_cmd;
2068         int ret = 0;
2069
2070         if (!psp->rap_context.context.initialized)
2071                 return 0;
2072
2073         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
2074             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
2075                 return -EINVAL;
2076
2077         mutex_lock(&psp->rap_context.mutex);
2078
2079         rap_cmd = (struct ta_rap_shared_memory *)
2080                   psp->rap_context.context.mem_context.shared_buf;
2081         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
2082
2083         rap_cmd->cmd_id = ta_cmd_id;
2084         rap_cmd->validation_method_id = METHOD_A;
2085
2086         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
2087         if (ret)
2088                 goto out_unlock;
2089
2090         if (status)
2091                 *status = rap_cmd->rap_status;
2092
2093 out_unlock:
2094         mutex_unlock(&psp->rap_context.mutex);
2095
2096         return ret;
2097 }
2098 // RAP end
2099
2100 /* securedisplay start */
2101 static int psp_securedisplay_initialize(struct psp_context *psp)
2102 {
2103         int ret;
2104         struct ta_securedisplay_cmd *securedisplay_cmd;
2105
2106         /*
2107          * TODO: bypass the initialize in sriov for now
2108          */
2109         if (amdgpu_sriov_vf(psp->adev))
2110                 return 0;
2111
2112         /* bypass securedisplay initialization if dmu is harvested */
2113         if (!amdgpu_device_has_display_hardware(psp->adev))
2114                 return 0;
2115
2116         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
2117             !psp->securedisplay_context.context.bin_desc.start_addr) {
2118                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
2119                 return 0;
2120         }
2121
2122         psp->securedisplay_context.context.mem_context.shared_mem_size =
2123                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
2124         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
2125
2126         if (!psp->securedisplay_context.context.initialized) {
2127                 ret = psp_ta_init_shared_buf(psp,
2128                                              &psp->securedisplay_context.context.mem_context);
2129                 if (ret)
2130                         return ret;
2131         }
2132
2133         ret = psp_ta_load(psp, &psp->securedisplay_context.context);
2134         if (!ret) {
2135                 psp->securedisplay_context.context.initialized = true;
2136                 mutex_init(&psp->securedisplay_context.mutex);
2137         } else
2138                 return ret;
2139
2140         mutex_lock(&psp->securedisplay_context.mutex);
2141
2142         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
2143                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
2144
2145         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
2146
2147         mutex_unlock(&psp->securedisplay_context.mutex);
2148
2149         if (ret) {
2150                 psp_securedisplay_terminate(psp);
2151                 /* free securedisplay shared memory */
2152                 psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
2153                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
2154                 return -EINVAL;
2155         }
2156
2157         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
2158                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
2159                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
2160                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
2161                 /* don't try again */
2162                 psp->securedisplay_context.context.bin_desc.size_bytes = 0;
2163         }
2164
2165         return 0;
2166 }
2167
2168 static int psp_securedisplay_terminate(struct psp_context *psp)
2169 {
2170         int ret;
2171
2172         /*
2173          * TODO:bypass the terminate in sriov for now
2174          */
2175         if (amdgpu_sriov_vf(psp->adev))
2176                 return 0;
2177
2178         if (!psp->securedisplay_context.context.initialized)
2179                 return 0;
2180
2181         ret = psp_ta_unload(psp, &psp->securedisplay_context.context);
2182
2183         psp->securedisplay_context.context.initialized = false;
2184
2185         return ret;
2186 }
2187
2188 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
2189 {
2190         int ret;
2191
2192         if (!psp->securedisplay_context.context.initialized)
2193                 return -EINVAL;
2194
2195         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
2196             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
2197                 return -EINVAL;
2198
2199         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
2200
2201         return ret;
2202 }
2203 /* SECUREDISPLAY end */
2204
2205 int amdgpu_psp_wait_for_bootloader(struct amdgpu_device *adev)
2206 {
2207         struct psp_context *psp = &adev->psp;
2208         int ret = 0;
2209
2210         if (!amdgpu_sriov_vf(adev) && psp->funcs && psp->funcs->wait_for_bootloader != NULL)
2211                 ret = psp->funcs->wait_for_bootloader(psp);
2212
2213         return ret;
2214 }
2215
2216 bool amdgpu_psp_get_ras_capability(struct psp_context *psp)
2217 {
2218         if (psp->funcs &&
2219             psp->funcs->get_ras_capability) {
2220                 return psp->funcs->get_ras_capability(psp);
2221         } else {
2222                 return false;
2223         }
2224 }
2225
2226 static int psp_hw_start(struct psp_context *psp)
2227 {
2228         struct amdgpu_device *adev = psp->adev;
2229         int ret;
2230
2231         if (!amdgpu_sriov_vf(adev)) {
2232                 if ((is_psp_fw_valid(psp->kdb)) &&
2233                     (psp->funcs->bootloader_load_kdb != NULL)) {
2234                         ret = psp_bootloader_load_kdb(psp);
2235                         if (ret) {
2236                                 dev_err(adev->dev, "PSP load kdb failed!\n");
2237                                 return ret;
2238                         }
2239                 }
2240
2241                 if ((is_psp_fw_valid(psp->spl)) &&
2242                     (psp->funcs->bootloader_load_spl != NULL)) {
2243                         ret = psp_bootloader_load_spl(psp);
2244                         if (ret) {
2245                                 dev_err(adev->dev, "PSP load spl failed!\n");
2246                                 return ret;
2247                         }
2248                 }
2249
2250                 if ((is_psp_fw_valid(psp->sys)) &&
2251                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
2252                         ret = psp_bootloader_load_sysdrv(psp);
2253                         if (ret) {
2254                                 dev_err(adev->dev, "PSP load sys drv failed!\n");
2255                                 return ret;
2256                         }
2257                 }
2258
2259                 if ((is_psp_fw_valid(psp->soc_drv)) &&
2260                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
2261                         ret = psp_bootloader_load_soc_drv(psp);
2262                         if (ret) {
2263                                 dev_err(adev->dev, "PSP load soc drv failed!\n");
2264                                 return ret;
2265                         }
2266                 }
2267
2268                 if ((is_psp_fw_valid(psp->intf_drv)) &&
2269                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
2270                         ret = psp_bootloader_load_intf_drv(psp);
2271                         if (ret) {
2272                                 dev_err(adev->dev, "PSP load intf drv failed!\n");
2273                                 return ret;
2274                         }
2275                 }
2276
2277                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
2278                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
2279                         ret = psp_bootloader_load_dbg_drv(psp);
2280                         if (ret) {
2281                                 dev_err(adev->dev, "PSP load dbg drv failed!\n");
2282                                 return ret;
2283                         }
2284                 }
2285
2286                 if ((is_psp_fw_valid(psp->ras_drv)) &&
2287                     (psp->funcs->bootloader_load_ras_drv != NULL)) {
2288                         ret = psp_bootloader_load_ras_drv(psp);
2289                         if (ret) {
2290                                 dev_err(adev->dev, "PSP load ras_drv failed!\n");
2291                                 return ret;
2292                         }
2293                 }
2294
2295                 if ((is_psp_fw_valid(psp->ipkeymgr_drv)) &&
2296                     (psp->funcs->bootloader_load_ipkeymgr_drv != NULL)) {
2297                         ret = psp_bootloader_load_ipkeymgr_drv(psp);
2298                         if (ret) {
2299                                 dev_err(adev->dev, "PSP load ipkeymgr_drv failed!\n");
2300                                 return ret;
2301                         }
2302                 }
2303
2304                 if ((is_psp_fw_valid(psp->sos)) &&
2305                     (psp->funcs->bootloader_load_sos != NULL)) {
2306                         ret = psp_bootloader_load_sos(psp);
2307                         if (ret) {
2308                                 dev_err(adev->dev, "PSP load sos failed!\n");
2309                                 return ret;
2310                         }
2311                 }
2312         }
2313
2314         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2315         if (ret) {
2316                 dev_err(adev->dev, "PSP create ring failed!\n");
2317                 return ret;
2318         }
2319
2320         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2321                 goto skip_pin_bo;
2322
2323         if (!psp->boot_time_tmr || psp->autoload_supported) {
2324                 ret = psp_tmr_init(psp);
2325                 if (ret) {
2326                         dev_err(adev->dev, "PSP tmr init failed!\n");
2327                         return ret;
2328                 }
2329         }
2330
2331 skip_pin_bo:
2332         /*
2333          * For ASICs with DF Cstate management centralized
2334          * to PMFW, TMR setup should be performed after PMFW
2335          * loaded and before other non-psp firmware loaded.
2336          */
2337         if (psp->pmfw_centralized_cstate_management) {
2338                 ret = psp_load_smu_fw(psp);
2339                 if (ret)
2340                         return ret;
2341         }
2342
2343         if (!psp->boot_time_tmr || !psp->autoload_supported) {
2344                 ret = psp_tmr_load(psp);
2345                 if (ret) {
2346                         dev_err(adev->dev, "PSP load tmr failed!\n");
2347                         return ret;
2348                 }
2349         }
2350
2351         return 0;
2352 }
2353
2354 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2355                            enum psp_gfx_fw_type *type)
2356 {
2357         switch (ucode->ucode_id) {
2358         case AMDGPU_UCODE_ID_CAP:
2359                 *type = GFX_FW_TYPE_CAP;
2360                 break;
2361         case AMDGPU_UCODE_ID_SDMA0:
2362                 *type = GFX_FW_TYPE_SDMA0;
2363                 break;
2364         case AMDGPU_UCODE_ID_SDMA1:
2365                 *type = GFX_FW_TYPE_SDMA1;
2366                 break;
2367         case AMDGPU_UCODE_ID_SDMA2:
2368                 *type = GFX_FW_TYPE_SDMA2;
2369                 break;
2370         case AMDGPU_UCODE_ID_SDMA3:
2371                 *type = GFX_FW_TYPE_SDMA3;
2372                 break;
2373         case AMDGPU_UCODE_ID_SDMA4:
2374                 *type = GFX_FW_TYPE_SDMA4;
2375                 break;
2376         case AMDGPU_UCODE_ID_SDMA5:
2377                 *type = GFX_FW_TYPE_SDMA5;
2378                 break;
2379         case AMDGPU_UCODE_ID_SDMA6:
2380                 *type = GFX_FW_TYPE_SDMA6;
2381                 break;
2382         case AMDGPU_UCODE_ID_SDMA7:
2383                 *type = GFX_FW_TYPE_SDMA7;
2384                 break;
2385         case AMDGPU_UCODE_ID_CP_MES:
2386                 *type = GFX_FW_TYPE_CP_MES;
2387                 break;
2388         case AMDGPU_UCODE_ID_CP_MES_DATA:
2389                 *type = GFX_FW_TYPE_MES_STACK;
2390                 break;
2391         case AMDGPU_UCODE_ID_CP_MES1:
2392                 *type = GFX_FW_TYPE_CP_MES_KIQ;
2393                 break;
2394         case AMDGPU_UCODE_ID_CP_MES1_DATA:
2395                 *type = GFX_FW_TYPE_MES_KIQ_STACK;
2396                 break;
2397         case AMDGPU_UCODE_ID_CP_CE:
2398                 *type = GFX_FW_TYPE_CP_CE;
2399                 break;
2400         case AMDGPU_UCODE_ID_CP_PFP:
2401                 *type = GFX_FW_TYPE_CP_PFP;
2402                 break;
2403         case AMDGPU_UCODE_ID_CP_ME:
2404                 *type = GFX_FW_TYPE_CP_ME;
2405                 break;
2406         case AMDGPU_UCODE_ID_CP_MEC1:
2407                 *type = GFX_FW_TYPE_CP_MEC;
2408                 break;
2409         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2410                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2411                 break;
2412         case AMDGPU_UCODE_ID_CP_MEC2:
2413                 *type = GFX_FW_TYPE_CP_MEC;
2414                 break;
2415         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2416                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2417                 break;
2418         case AMDGPU_UCODE_ID_RLC_P:
2419                 *type = GFX_FW_TYPE_RLC_P;
2420                 break;
2421         case AMDGPU_UCODE_ID_RLC_V:
2422                 *type = GFX_FW_TYPE_RLC_V;
2423                 break;
2424         case AMDGPU_UCODE_ID_RLC_G:
2425                 *type = GFX_FW_TYPE_RLC_G;
2426                 break;
2427         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2428                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2429                 break;
2430         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2431                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2432                 break;
2433         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2434                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2435                 break;
2436         case AMDGPU_UCODE_ID_RLC_IRAM:
2437                 *type = GFX_FW_TYPE_RLC_IRAM;
2438                 break;
2439         case AMDGPU_UCODE_ID_RLC_DRAM:
2440                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2441                 break;
2442         case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
2443                 *type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
2444                 break;
2445         case AMDGPU_UCODE_ID_SE0_TAP_DELAYS:
2446                 *type = GFX_FW_TYPE_SE0_TAP_DELAYS;
2447                 break;
2448         case AMDGPU_UCODE_ID_SE1_TAP_DELAYS:
2449                 *type = GFX_FW_TYPE_SE1_TAP_DELAYS;
2450                 break;
2451         case AMDGPU_UCODE_ID_SE2_TAP_DELAYS:
2452                 *type = GFX_FW_TYPE_SE2_TAP_DELAYS;
2453                 break;
2454         case AMDGPU_UCODE_ID_SE3_TAP_DELAYS:
2455                 *type = GFX_FW_TYPE_SE3_TAP_DELAYS;
2456                 break;
2457         case AMDGPU_UCODE_ID_SMC:
2458                 *type = GFX_FW_TYPE_SMU;
2459                 break;
2460         case AMDGPU_UCODE_ID_PPTABLE:
2461                 *type = GFX_FW_TYPE_PPTABLE;
2462                 break;
2463         case AMDGPU_UCODE_ID_UVD:
2464                 *type = GFX_FW_TYPE_UVD;
2465                 break;
2466         case AMDGPU_UCODE_ID_UVD1:
2467                 *type = GFX_FW_TYPE_UVD1;
2468                 break;
2469         case AMDGPU_UCODE_ID_VCE:
2470                 *type = GFX_FW_TYPE_VCE;
2471                 break;
2472         case AMDGPU_UCODE_ID_VCN:
2473                 *type = GFX_FW_TYPE_VCN;
2474                 break;
2475         case AMDGPU_UCODE_ID_VCN1:
2476                 *type = GFX_FW_TYPE_VCN1;
2477                 break;
2478         case AMDGPU_UCODE_ID_DMCU_ERAM:
2479                 *type = GFX_FW_TYPE_DMCU_ERAM;
2480                 break;
2481         case AMDGPU_UCODE_ID_DMCU_INTV:
2482                 *type = GFX_FW_TYPE_DMCU_ISR;
2483                 break;
2484         case AMDGPU_UCODE_ID_VCN0_RAM:
2485                 *type = GFX_FW_TYPE_VCN0_RAM;
2486                 break;
2487         case AMDGPU_UCODE_ID_VCN1_RAM:
2488                 *type = GFX_FW_TYPE_VCN1_RAM;
2489                 break;
2490         case AMDGPU_UCODE_ID_DMCUB:
2491                 *type = GFX_FW_TYPE_DMUB;
2492                 break;
2493         case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
2494         case AMDGPU_UCODE_ID_SDMA_RS64:
2495                 *type = GFX_FW_TYPE_SDMA_UCODE_TH0;
2496                 break;
2497         case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
2498                 *type = GFX_FW_TYPE_SDMA_UCODE_TH1;
2499                 break;
2500         case AMDGPU_UCODE_ID_IMU_I:
2501                 *type = GFX_FW_TYPE_IMU_I;
2502                 break;
2503         case AMDGPU_UCODE_ID_IMU_D:
2504                 *type = GFX_FW_TYPE_IMU_D;
2505                 break;
2506         case AMDGPU_UCODE_ID_CP_RS64_PFP:
2507                 *type = GFX_FW_TYPE_RS64_PFP;
2508                 break;
2509         case AMDGPU_UCODE_ID_CP_RS64_ME:
2510                 *type = GFX_FW_TYPE_RS64_ME;
2511                 break;
2512         case AMDGPU_UCODE_ID_CP_RS64_MEC:
2513                 *type = GFX_FW_TYPE_RS64_MEC;
2514                 break;
2515         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
2516                 *type = GFX_FW_TYPE_RS64_PFP_P0_STACK;
2517                 break;
2518         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
2519                 *type = GFX_FW_TYPE_RS64_PFP_P1_STACK;
2520                 break;
2521         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
2522                 *type = GFX_FW_TYPE_RS64_ME_P0_STACK;
2523                 break;
2524         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
2525                 *type = GFX_FW_TYPE_RS64_ME_P1_STACK;
2526                 break;
2527         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
2528                 *type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
2529                 break;
2530         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
2531                 *type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
2532                 break;
2533         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
2534                 *type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
2535                 break;
2536         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
2537                 *type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
2538                 break;
2539         case AMDGPU_UCODE_ID_VPE_CTX:
2540                 *type = GFX_FW_TYPE_VPEC_FW1;
2541                 break;
2542         case AMDGPU_UCODE_ID_VPE_CTL:
2543                 *type = GFX_FW_TYPE_VPEC_FW2;
2544                 break;
2545         case AMDGPU_UCODE_ID_VPE:
2546                 *type = GFX_FW_TYPE_VPE;
2547                 break;
2548         case AMDGPU_UCODE_ID_UMSCH_MM_UCODE:
2549                 *type = GFX_FW_TYPE_UMSCH_UCODE;
2550                 break;
2551         case AMDGPU_UCODE_ID_UMSCH_MM_DATA:
2552                 *type = GFX_FW_TYPE_UMSCH_DATA;
2553                 break;
2554         case AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER:
2555                 *type = GFX_FW_TYPE_UMSCH_CMD_BUFFER;
2556                 break;
2557         case AMDGPU_UCODE_ID_P2S_TABLE:
2558                 *type = GFX_FW_TYPE_P2S_TABLE;
2559                 break;
2560         case AMDGPU_UCODE_ID_JPEG_RAM:
2561                 *type = GFX_FW_TYPE_JPEG_RAM;
2562                 break;
2563         case AMDGPU_UCODE_ID_ISP:
2564                 *type = GFX_FW_TYPE_ISP;
2565                 break;
2566         case AMDGPU_UCODE_ID_MAXIMUM:
2567         default:
2568                 return -EINVAL;
2569         }
2570
2571         return 0;
2572 }
2573
2574 static void psp_print_fw_hdr(struct psp_context *psp,
2575                              struct amdgpu_firmware_info *ucode)
2576 {
2577         struct amdgpu_device *adev = psp->adev;
2578         struct common_firmware_header *hdr;
2579
2580         switch (ucode->ucode_id) {
2581         case AMDGPU_UCODE_ID_SDMA0:
2582         case AMDGPU_UCODE_ID_SDMA1:
2583         case AMDGPU_UCODE_ID_SDMA2:
2584         case AMDGPU_UCODE_ID_SDMA3:
2585         case AMDGPU_UCODE_ID_SDMA4:
2586         case AMDGPU_UCODE_ID_SDMA5:
2587         case AMDGPU_UCODE_ID_SDMA6:
2588         case AMDGPU_UCODE_ID_SDMA7:
2589                 hdr = (struct common_firmware_header *)
2590                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2591                 amdgpu_ucode_print_sdma_hdr(hdr);
2592                 break;
2593         case AMDGPU_UCODE_ID_CP_CE:
2594                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2595                 amdgpu_ucode_print_gfx_hdr(hdr);
2596                 break;
2597         case AMDGPU_UCODE_ID_CP_PFP:
2598                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2599                 amdgpu_ucode_print_gfx_hdr(hdr);
2600                 break;
2601         case AMDGPU_UCODE_ID_CP_ME:
2602                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2603                 amdgpu_ucode_print_gfx_hdr(hdr);
2604                 break;
2605         case AMDGPU_UCODE_ID_CP_MEC1:
2606                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2607                 amdgpu_ucode_print_gfx_hdr(hdr);
2608                 break;
2609         case AMDGPU_UCODE_ID_RLC_G:
2610                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2611                 amdgpu_ucode_print_rlc_hdr(hdr);
2612                 break;
2613         case AMDGPU_UCODE_ID_SMC:
2614                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2615                 amdgpu_ucode_print_smc_hdr(hdr);
2616                 break;
2617         default:
2618                 break;
2619         }
2620 }
2621
2622 static int psp_prep_load_ip_fw_cmd_buf(struct psp_context *psp,
2623                                        struct amdgpu_firmware_info *ucode,
2624                                        struct psp_gfx_cmd_resp *cmd)
2625 {
2626         int ret;
2627         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2628
2629         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2630         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2631         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2632         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2633
2634         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2635         if (ret)
2636                 dev_err(psp->adev->dev, "Unknown firmware type\n");
2637
2638         return ret;
2639 }
2640
2641 int psp_execute_ip_fw_load(struct psp_context *psp,
2642                            struct amdgpu_firmware_info *ucode)
2643 {
2644         int ret = 0;
2645         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2646
2647         ret = psp_prep_load_ip_fw_cmd_buf(psp, ucode, cmd);
2648         if (!ret) {
2649                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2650                                          psp->fence_buf_mc_addr);
2651         }
2652
2653         release_psp_cmd_buf(psp);
2654
2655         return ret;
2656 }
2657
2658 static int psp_load_p2s_table(struct psp_context *psp)
2659 {
2660         int ret;
2661         struct amdgpu_device *adev = psp->adev;
2662         struct amdgpu_firmware_info *ucode =
2663                 &adev->firmware.ucode[AMDGPU_UCODE_ID_P2S_TABLE];
2664
2665         if (adev->in_runpm && ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
2666                                 (adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)))
2667                 return 0;
2668
2669         if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
2670             amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)) {
2671                 uint32_t supp_vers = adev->flags & AMD_IS_APU ? 0x0036013D :
2672                                                                 0x0036003C;
2673                 if (psp->sos.fw_version < supp_vers)
2674                         return 0;
2675         }
2676
2677         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2678                 return 0;
2679
2680         ret = psp_execute_ip_fw_load(psp, ucode);
2681
2682         return ret;
2683 }
2684
2685 static int psp_load_smu_fw(struct psp_context *psp)
2686 {
2687         int ret;
2688         struct amdgpu_device *adev = psp->adev;
2689         struct amdgpu_firmware_info *ucode =
2690                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2691         struct amdgpu_ras *ras = psp->ras_context.ras;
2692
2693         /*
2694          * Skip SMU FW reloading in case of using BACO for runpm only,
2695          * as SMU is always alive.
2696          */
2697         if (adev->in_runpm && ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
2698                                 (adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)))
2699                 return 0;
2700
2701         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2702                 return 0;
2703
2704         if ((amdgpu_in_reset(adev) && ras && adev->ras_enabled &&
2705              (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(11, 0, 4) ||
2706               amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(11, 0, 2)))) {
2707                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2708                 if (ret)
2709                         dev_err(adev->dev, "Failed to set MP1 state prepare for reload\n");
2710         }
2711
2712         ret = psp_execute_ip_fw_load(psp, ucode);
2713
2714         if (ret)
2715                 dev_err(adev->dev, "PSP load smu failed!\n");
2716
2717         return ret;
2718 }
2719
2720 static bool fw_load_skip_check(struct psp_context *psp,
2721                                struct amdgpu_firmware_info *ucode)
2722 {
2723         if (!ucode->fw || !ucode->ucode_size)
2724                 return true;
2725
2726         if (ucode->ucode_id == AMDGPU_UCODE_ID_P2S_TABLE)
2727                 return true;
2728
2729         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2730             (psp_smu_reload_quirk(psp) ||
2731              psp->autoload_supported ||
2732              psp->pmfw_centralized_cstate_management))
2733                 return true;
2734
2735         if (amdgpu_sriov_vf(psp->adev) &&
2736             amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
2737                 return true;
2738
2739         if (psp->autoload_supported &&
2740             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2741              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2742                 /* skip mec JT when autoload is enabled */
2743                 return true;
2744
2745         return false;
2746 }
2747
2748 int psp_load_fw_list(struct psp_context *psp,
2749                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2750 {
2751         int ret = 0, i;
2752         struct amdgpu_firmware_info *ucode;
2753
2754         for (i = 0; i < ucode_count; ++i) {
2755                 ucode = ucode_list[i];
2756                 psp_print_fw_hdr(psp, ucode);
2757                 ret = psp_execute_ip_fw_load(psp, ucode);
2758                 if (ret)
2759                         return ret;
2760         }
2761         return ret;
2762 }
2763
2764 static int psp_load_non_psp_fw(struct psp_context *psp)
2765 {
2766         int i, ret;
2767         struct amdgpu_firmware_info *ucode;
2768         struct amdgpu_device *adev = psp->adev;
2769
2770         if (psp->autoload_supported &&
2771             !psp->pmfw_centralized_cstate_management) {
2772                 ret = psp_load_smu_fw(psp);
2773                 if (ret)
2774                         return ret;
2775         }
2776
2777         /* Load P2S table first if it's available */
2778         psp_load_p2s_table(psp);
2779
2780         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2781                 ucode = &adev->firmware.ucode[i];
2782
2783                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2784                     !fw_load_skip_check(psp, ucode)) {
2785                         ret = psp_load_smu_fw(psp);
2786                         if (ret)
2787                                 return ret;
2788                         continue;
2789                 }
2790
2791                 if (fw_load_skip_check(psp, ucode))
2792                         continue;
2793
2794                 if (psp->autoload_supported &&
2795                     (amdgpu_ip_version(adev, MP0_HWIP, 0) ==
2796                              IP_VERSION(11, 0, 7) ||
2797                      amdgpu_ip_version(adev, MP0_HWIP, 0) ==
2798                              IP_VERSION(11, 0, 11) ||
2799                      amdgpu_ip_version(adev, MP0_HWIP, 0) ==
2800                              IP_VERSION(11, 0, 12)) &&
2801                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2802                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2803                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2804                         /* PSP only receive one SDMA fw for sienna_cichlid,
2805                          * as all four sdma fw are same
2806                          */
2807                         continue;
2808
2809                 psp_print_fw_hdr(psp, ucode);
2810
2811                 ret = psp_execute_ip_fw_load(psp, ucode);
2812                 if (ret)
2813                         return ret;
2814
2815                 /* Start rlc autoload after psp recieved all the gfx firmware */
2816                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2817                     adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
2818                         ret = psp_rlc_autoload_start(psp);
2819                         if (ret) {
2820                                 dev_err(adev->dev, "Failed to start rlc autoload\n");
2821                                 return ret;
2822                         }
2823                 }
2824         }
2825
2826         return 0;
2827 }
2828
2829 static int psp_load_fw(struct amdgpu_device *adev)
2830 {
2831         int ret;
2832         struct psp_context *psp = &adev->psp;
2833
2834         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2835                 /* should not destroy ring, only stop */
2836                 psp_ring_stop(psp, PSP_RING_TYPE__KM);
2837         } else {
2838                 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2839
2840                 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2841                 if (ret) {
2842                         dev_err(adev->dev, "PSP ring init failed!\n");
2843                         goto failed;
2844                 }
2845         }
2846
2847         ret = psp_hw_start(psp);
2848         if (ret)
2849                 goto failed;
2850
2851         ret = psp_load_non_psp_fw(psp);
2852         if (ret)
2853                 goto failed1;
2854
2855         ret = psp_asd_initialize(psp);
2856         if (ret) {
2857                 dev_err(adev->dev, "PSP load asd failed!\n");
2858                 goto failed1;
2859         }
2860
2861         ret = psp_rl_load(adev);
2862         if (ret) {
2863                 dev_err(adev->dev, "PSP load RL failed!\n");
2864                 goto failed1;
2865         }
2866
2867         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2868                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2869                         ret = psp_xgmi_initialize(psp, false, true);
2870                         /* Warning the XGMI seesion initialize failure
2871                          * Instead of stop driver initialization
2872                          */
2873                         if (ret)
2874                                 dev_err(psp->adev->dev,
2875                                         "XGMI: Failed to initialize XGMI session\n");
2876                 }
2877         }
2878
2879         if (psp->ta_fw) {
2880                 ret = psp_ras_initialize(psp);
2881                 if (ret)
2882                         dev_err(psp->adev->dev,
2883                                 "RAS: Failed to initialize RAS\n");
2884
2885                 ret = psp_hdcp_initialize(psp);
2886                 if (ret)
2887                         dev_err(psp->adev->dev,
2888                                 "HDCP: Failed to initialize HDCP\n");
2889
2890                 ret = psp_dtm_initialize(psp);
2891                 if (ret)
2892                         dev_err(psp->adev->dev,
2893                                 "DTM: Failed to initialize DTM\n");
2894
2895                 ret = psp_rap_initialize(psp);
2896                 if (ret)
2897                         dev_err(psp->adev->dev,
2898                                 "RAP: Failed to initialize RAP\n");
2899
2900                 ret = psp_securedisplay_initialize(psp);
2901                 if (ret)
2902                         dev_err(psp->adev->dev,
2903                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2904         }
2905
2906         return 0;
2907
2908 failed1:
2909         psp_free_shared_bufs(psp);
2910 failed:
2911         /*
2912          * all cleanup jobs (xgmi terminate, ras terminate,
2913          * ring destroy, cmd/fence/fw buffers destory,
2914          * psp->cmd destory) are delayed to psp_hw_fini
2915          */
2916         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2917         return ret;
2918 }
2919
2920 static int psp_hw_init(void *handle)
2921 {
2922         int ret;
2923         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2924
2925         mutex_lock(&adev->firmware.mutex);
2926         /*
2927          * This sequence is just used on hw_init only once, no need on
2928          * resume.
2929          */
2930         ret = amdgpu_ucode_init_bo(adev);
2931         if (ret)
2932                 goto failed;
2933
2934         ret = psp_load_fw(adev);
2935         if (ret) {
2936                 dev_err(adev->dev, "PSP firmware loading failed\n");
2937                 goto failed;
2938         }
2939
2940         mutex_unlock(&adev->firmware.mutex);
2941         return 0;
2942
2943 failed:
2944         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2945         mutex_unlock(&adev->firmware.mutex);
2946         return -EINVAL;
2947 }
2948
2949 static int psp_hw_fini(void *handle)
2950 {
2951         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2952         struct psp_context *psp = &adev->psp;
2953
2954         if (psp->ta_fw) {
2955                 psp_ras_terminate(psp);
2956                 psp_securedisplay_terminate(psp);
2957                 psp_rap_terminate(psp);
2958                 psp_dtm_terminate(psp);
2959                 psp_hdcp_terminate(psp);
2960
2961                 if (adev->gmc.xgmi.num_physical_nodes > 1)
2962                         psp_xgmi_terminate(psp);
2963         }
2964
2965         psp_asd_terminate(psp);
2966         psp_tmr_terminate(psp);
2967
2968         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2969
2970         return 0;
2971 }
2972
2973 static int psp_suspend(void *handle)
2974 {
2975         int ret = 0;
2976         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2977         struct psp_context *psp = &adev->psp;
2978
2979         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2980             psp->xgmi_context.context.initialized) {
2981                 ret = psp_xgmi_terminate(psp);
2982                 if (ret) {
2983                         dev_err(adev->dev, "Failed to terminate xgmi ta\n");
2984                         goto out;
2985                 }
2986         }
2987
2988         if (psp->ta_fw) {
2989                 ret = psp_ras_terminate(psp);
2990                 if (ret) {
2991                         dev_err(adev->dev, "Failed to terminate ras ta\n");
2992                         goto out;
2993                 }
2994                 ret = psp_hdcp_terminate(psp);
2995                 if (ret) {
2996                         dev_err(adev->dev, "Failed to terminate hdcp ta\n");
2997                         goto out;
2998                 }
2999                 ret = psp_dtm_terminate(psp);
3000                 if (ret) {
3001                         dev_err(adev->dev, "Failed to terminate dtm ta\n");
3002                         goto out;
3003                 }
3004                 ret = psp_rap_terminate(psp);
3005                 if (ret) {
3006                         dev_err(adev->dev, "Failed to terminate rap ta\n");
3007                         goto out;
3008                 }
3009                 ret = psp_securedisplay_terminate(psp);
3010                 if (ret) {
3011                         dev_err(adev->dev, "Failed to terminate securedisplay ta\n");
3012                         goto out;
3013                 }
3014         }
3015
3016         ret = psp_asd_terminate(psp);
3017         if (ret) {
3018                 dev_err(adev->dev, "Failed to terminate asd\n");
3019                 goto out;
3020         }
3021
3022         ret = psp_tmr_terminate(psp);
3023         if (ret) {
3024                 dev_err(adev->dev, "Failed to terminate tmr\n");
3025                 goto out;
3026         }
3027
3028         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
3029         if (ret)
3030                 dev_err(adev->dev, "PSP ring stop failed\n");
3031
3032 out:
3033         return ret;
3034 }
3035
3036 static int psp_resume(void *handle)
3037 {
3038         int ret;
3039         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3040         struct psp_context *psp = &adev->psp;
3041
3042         dev_info(adev->dev, "PSP is resuming...\n");
3043
3044         if (psp->mem_train_ctx.enable_mem_training) {
3045                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
3046                 if (ret) {
3047                         dev_err(adev->dev, "Failed to process memory training!\n");
3048                         return ret;
3049                 }
3050         }
3051
3052         mutex_lock(&adev->firmware.mutex);
3053
3054         ret = psp_hw_start(psp);
3055         if (ret)
3056                 goto failed;
3057
3058         ret = psp_load_non_psp_fw(psp);
3059         if (ret)
3060                 goto failed;
3061
3062         ret = psp_asd_initialize(psp);
3063         if (ret) {
3064                 dev_err(adev->dev, "PSP load asd failed!\n");
3065                 goto failed;
3066         }
3067
3068         ret = psp_rl_load(adev);
3069         if (ret) {
3070                 dev_err(adev->dev, "PSP load RL failed!\n");
3071                 goto failed;
3072         }
3073
3074         if (adev->gmc.xgmi.num_physical_nodes > 1) {
3075                 ret = psp_xgmi_initialize(psp, false, true);
3076                 /* Warning the XGMI seesion initialize failure
3077                  * Instead of stop driver initialization
3078                  */
3079                 if (ret)
3080                         dev_err(psp->adev->dev,
3081                                 "XGMI: Failed to initialize XGMI session\n");
3082         }
3083
3084         if (psp->ta_fw) {
3085                 ret = psp_ras_initialize(psp);
3086                 if (ret)
3087                         dev_err(psp->adev->dev,
3088                                 "RAS: Failed to initialize RAS\n");
3089
3090                 ret = psp_hdcp_initialize(psp);
3091                 if (ret)
3092                         dev_err(psp->adev->dev,
3093                                 "HDCP: Failed to initialize HDCP\n");
3094
3095                 ret = psp_dtm_initialize(psp);
3096                 if (ret)
3097                         dev_err(psp->adev->dev,
3098                                 "DTM: Failed to initialize DTM\n");
3099
3100                 ret = psp_rap_initialize(psp);
3101                 if (ret)
3102                         dev_err(psp->adev->dev,
3103                                 "RAP: Failed to initialize RAP\n");
3104
3105                 ret = psp_securedisplay_initialize(psp);
3106                 if (ret)
3107                         dev_err(psp->adev->dev,
3108                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
3109         }
3110
3111         mutex_unlock(&adev->firmware.mutex);
3112
3113         return 0;
3114
3115 failed:
3116         dev_err(adev->dev, "PSP resume failed\n");
3117         mutex_unlock(&adev->firmware.mutex);
3118         return ret;
3119 }
3120
3121 int psp_gpu_reset(struct amdgpu_device *adev)
3122 {
3123         int ret;
3124
3125         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
3126                 return 0;
3127
3128         mutex_lock(&adev->psp.mutex);
3129         ret = psp_mode1_reset(&adev->psp);
3130         mutex_unlock(&adev->psp.mutex);
3131
3132         return ret;
3133 }
3134
3135 int psp_rlc_autoload_start(struct psp_context *psp)
3136 {
3137         int ret;
3138         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
3139
3140         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
3141
3142         ret = psp_cmd_submit_buf(psp, NULL, cmd,
3143                                  psp->fence_buf_mc_addr);
3144
3145         release_psp_cmd_buf(psp);
3146
3147         return ret;
3148 }
3149
3150 int psp_ring_cmd_submit(struct psp_context *psp,
3151                         uint64_t cmd_buf_mc_addr,
3152                         uint64_t fence_mc_addr,
3153                         int index)
3154 {
3155         unsigned int psp_write_ptr_reg = 0;
3156         struct psp_gfx_rb_frame *write_frame;
3157         struct psp_ring *ring = &psp->km_ring;
3158         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
3159         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
3160                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
3161         struct amdgpu_device *adev = psp->adev;
3162         uint32_t ring_size_dw = ring->ring_size / 4;
3163         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
3164
3165         /* KM (GPCOM) prepare write pointer */
3166         psp_write_ptr_reg = psp_ring_get_wptr(psp);
3167
3168         /* Update KM RB frame pointer to new frame */
3169         /* write_frame ptr increments by size of rb_frame in bytes */
3170         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
3171         if ((psp_write_ptr_reg % ring_size_dw) == 0)
3172                 write_frame = ring_buffer_start;
3173         else
3174                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
3175         /* Check invalid write_frame ptr address */
3176         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
3177                 dev_err(adev->dev,
3178                         "ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
3179                         ring_buffer_start, ring_buffer_end, write_frame);
3180                 dev_err(adev->dev,
3181                         "write_frame is pointing to address out of bounds\n");
3182                 return -EINVAL;
3183         }
3184
3185         /* Initialize KM RB frame */
3186         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
3187
3188         /* Update KM RB frame */
3189         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
3190         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
3191         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
3192         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
3193         write_frame->fence_value = index;
3194         amdgpu_device_flush_hdp(adev, NULL);
3195
3196         /* Update the write Pointer in DWORDs */
3197         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
3198         psp_ring_set_wptr(psp, psp_write_ptr_reg);
3199         return 0;
3200 }
3201
3202 int psp_init_asd_microcode(struct psp_context *psp, const char *chip_name)
3203 {
3204         struct amdgpu_device *adev = psp->adev;
3205         const struct psp_firmware_header_v1_0 *asd_hdr;
3206         int err = 0;
3207
3208         err = amdgpu_ucode_request(adev, &adev->psp.asd_fw, "amdgpu/%s_asd.bin", chip_name);
3209         if (err)
3210                 goto out;
3211
3212         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
3213         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
3214         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
3215         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
3216         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
3217                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
3218         return 0;
3219 out:
3220         amdgpu_ucode_release(&adev->psp.asd_fw);
3221         return err;
3222 }
3223
3224 int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name)
3225 {
3226         struct amdgpu_device *adev = psp->adev;
3227         const struct psp_firmware_header_v1_0 *toc_hdr;
3228         int err = 0;
3229
3230         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, "amdgpu/%s_toc.bin", chip_name);
3231         if (err)
3232                 goto out;
3233
3234         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
3235         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
3236         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
3237         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
3238         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
3239                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
3240         return 0;
3241 out:
3242         amdgpu_ucode_release(&adev->psp.toc_fw);
3243         return err;
3244 }
3245
3246 static int parse_sos_bin_descriptor(struct psp_context *psp,
3247                                    const struct psp_fw_bin_desc *desc,
3248                                    const struct psp_firmware_header_v2_0 *sos_hdr)
3249 {
3250         uint8_t *ucode_start_addr  = NULL;
3251
3252         if (!psp || !desc || !sos_hdr)
3253                 return -EINVAL;
3254
3255         ucode_start_addr  = (uint8_t *)sos_hdr +
3256                             le32_to_cpu(desc->offset_bytes) +
3257                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3258
3259         switch (desc->fw_type) {
3260         case PSP_FW_TYPE_PSP_SOS:
3261                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
3262                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
3263                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
3264                 psp->sos.start_addr        = ucode_start_addr;
3265                 break;
3266         case PSP_FW_TYPE_PSP_SYS_DRV:
3267                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
3268                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
3269                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
3270                 psp->sys.start_addr        = ucode_start_addr;
3271                 break;
3272         case PSP_FW_TYPE_PSP_KDB:
3273                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
3274                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
3275                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
3276                 psp->kdb.start_addr        = ucode_start_addr;
3277                 break;
3278         case PSP_FW_TYPE_PSP_TOC:
3279                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
3280                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
3281                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
3282                 psp->toc.start_addr        = ucode_start_addr;
3283                 break;
3284         case PSP_FW_TYPE_PSP_SPL:
3285                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
3286                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
3287                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
3288                 psp->spl.start_addr        = ucode_start_addr;
3289                 break;
3290         case PSP_FW_TYPE_PSP_RL:
3291                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
3292                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
3293                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
3294                 psp->rl.start_addr         = ucode_start_addr;
3295                 break;
3296         case PSP_FW_TYPE_PSP_SOC_DRV:
3297                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
3298                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
3299                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3300                 psp->soc_drv.start_addr         = ucode_start_addr;
3301                 break;
3302         case PSP_FW_TYPE_PSP_INTF_DRV:
3303                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
3304                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
3305                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
3306                 psp->intf_drv.start_addr        = ucode_start_addr;
3307                 break;
3308         case PSP_FW_TYPE_PSP_DBG_DRV:
3309                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
3310                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
3311                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3312                 psp->dbg_drv.start_addr         = ucode_start_addr;
3313                 break;
3314         case PSP_FW_TYPE_PSP_RAS_DRV:
3315                 psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
3316                 psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
3317                 psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3318                 psp->ras_drv.start_addr         = ucode_start_addr;
3319                 break;
3320         case PSP_FW_TYPE_PSP_IPKEYMGR_DRV:
3321                 psp->ipkeymgr_drv.fw_version         = le32_to_cpu(desc->fw_version);
3322                 psp->ipkeymgr_drv.feature_version    = le32_to_cpu(desc->fw_version);
3323                 psp->ipkeymgr_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3324                 psp->ipkeymgr_drv.start_addr         = ucode_start_addr;
3325                 break;
3326         default:
3327                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
3328                 break;
3329         }
3330
3331         return 0;
3332 }
3333
3334 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
3335 {
3336         const struct psp_firmware_header_v1_0 *sos_hdr;
3337         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3338         uint8_t *ucode_array_start_addr;
3339
3340         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3341         ucode_array_start_addr = (uint8_t *)sos_hdr +
3342                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3343
3344         if (adev->gmc.xgmi.connected_to_cpu ||
3345             (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(13, 0, 2))) {
3346                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
3347                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
3348
3349                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
3350                 adev->psp.sys.start_addr = ucode_array_start_addr;
3351
3352                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
3353                 adev->psp.sos.start_addr = ucode_array_start_addr +
3354                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
3355         } else {
3356                 /* Load alternate PSP SOS FW */
3357                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3358
3359                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3360                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3361
3362                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
3363                 adev->psp.sys.start_addr = ucode_array_start_addr +
3364                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
3365
3366                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
3367                 adev->psp.sos.start_addr = ucode_array_start_addr +
3368                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
3369         }
3370
3371         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
3372                 dev_warn(adev->dev, "PSP SOS FW not available");
3373                 return -EINVAL;
3374         }
3375
3376         return 0;
3377 }
3378
3379 int psp_init_sos_microcode(struct psp_context *psp, const char *chip_name)
3380 {
3381         struct amdgpu_device *adev = psp->adev;
3382         const struct psp_firmware_header_v1_0 *sos_hdr;
3383         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3384         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3385         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3386         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3387         int err = 0;
3388         uint8_t *ucode_array_start_addr;
3389         int fw_index = 0;
3390
3391         err = amdgpu_ucode_request(adev, &adev->psp.sos_fw, "amdgpu/%s_sos.bin", chip_name);
3392         if (err)
3393                 goto out;
3394
3395         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3396         ucode_array_start_addr = (uint8_t *)sos_hdr +
3397                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3398         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3399
3400         switch (sos_hdr->header.header_version_major) {
3401         case 1:
3402                 err = psp_init_sos_base_fw(adev);
3403                 if (err)
3404                         goto out;
3405
3406                 if (sos_hdr->header.header_version_minor == 1) {
3407                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3408                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3409                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3410                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3411                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3412                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3413                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3414                 }
3415                 if (sos_hdr->header.header_version_minor == 2) {
3416                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3417                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3418                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3419                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3420                 }
3421                 if (sos_hdr->header.header_version_minor == 3) {
3422                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3423                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3424                         adev->psp.toc.start_addr = ucode_array_start_addr +
3425                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3426                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3427                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3428                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3429                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3430                         adev->psp.spl.start_addr = ucode_array_start_addr +
3431                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3432                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3433                         adev->psp.rl.start_addr = ucode_array_start_addr +
3434                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3435                 }
3436                 break;
3437         case 2:
3438                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3439
3440                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3441                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3442                         err = -EINVAL;
3443                         goto out;
3444                 }
3445
3446                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3447                         err = parse_sos_bin_descriptor(psp,
3448                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3449                                                        sos_hdr_v2_0);
3450                         if (err)
3451                                 goto out;
3452                 }
3453                 break;
3454         default:
3455                 dev_err(adev->dev,
3456                         "unsupported psp sos firmware\n");
3457                 err = -EINVAL;
3458                 goto out;
3459         }
3460
3461         return 0;
3462 out:
3463         amdgpu_ucode_release(&adev->psp.sos_fw);
3464
3465         return err;
3466 }
3467
3468 static int parse_ta_bin_descriptor(struct psp_context *psp,
3469                                    const struct psp_fw_bin_desc *desc,
3470                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3471 {
3472         uint8_t *ucode_start_addr  = NULL;
3473
3474         if (!psp || !desc || !ta_hdr)
3475                 return -EINVAL;
3476
3477         ucode_start_addr  = (uint8_t *)ta_hdr +
3478                             le32_to_cpu(desc->offset_bytes) +
3479                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3480
3481         switch (desc->fw_type) {
3482         case TA_FW_TYPE_PSP_ASD:
3483                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3484                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3485                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3486                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3487                 break;
3488         case TA_FW_TYPE_PSP_XGMI:
3489                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3490                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3491                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3492                 break;
3493         case TA_FW_TYPE_PSP_RAS:
3494                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3495                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3496                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3497                 break;
3498         case TA_FW_TYPE_PSP_HDCP:
3499                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3500                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3501                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3502                 break;
3503         case TA_FW_TYPE_PSP_DTM:
3504                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3505                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3506                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3507                 break;
3508         case TA_FW_TYPE_PSP_RAP:
3509                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3510                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3511                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3512                 break;
3513         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3514                 psp->securedisplay_context.context.bin_desc.fw_version =
3515                         le32_to_cpu(desc->fw_version);
3516                 psp->securedisplay_context.context.bin_desc.size_bytes =
3517                         le32_to_cpu(desc->size_bytes);
3518                 psp->securedisplay_context.context.bin_desc.start_addr =
3519                         ucode_start_addr;
3520                 break;
3521         default:
3522                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3523                 break;
3524         }
3525
3526         return 0;
3527 }
3528
3529 static int parse_ta_v1_microcode(struct psp_context *psp)
3530 {
3531         const struct ta_firmware_header_v1_0 *ta_hdr;
3532         struct amdgpu_device *adev = psp->adev;
3533
3534         ta_hdr = (const struct ta_firmware_header_v1_0 *) adev->psp.ta_fw->data;
3535
3536         if (le16_to_cpu(ta_hdr->header.header_version_major) != 1)
3537                 return -EINVAL;
3538
3539         adev->psp.xgmi_context.context.bin_desc.fw_version =
3540                 le32_to_cpu(ta_hdr->xgmi.fw_version);
3541         adev->psp.xgmi_context.context.bin_desc.size_bytes =
3542                 le32_to_cpu(ta_hdr->xgmi.size_bytes);
3543         adev->psp.xgmi_context.context.bin_desc.start_addr =
3544                 (uint8_t *)ta_hdr +
3545                 le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3546
3547         adev->psp.ras_context.context.bin_desc.fw_version =
3548                 le32_to_cpu(ta_hdr->ras.fw_version);
3549         adev->psp.ras_context.context.bin_desc.size_bytes =
3550                 le32_to_cpu(ta_hdr->ras.size_bytes);
3551         adev->psp.ras_context.context.bin_desc.start_addr =
3552                 (uint8_t *)adev->psp.xgmi_context.context.bin_desc.start_addr +
3553                 le32_to_cpu(ta_hdr->ras.offset_bytes);
3554
3555         adev->psp.hdcp_context.context.bin_desc.fw_version =
3556                 le32_to_cpu(ta_hdr->hdcp.fw_version);
3557         adev->psp.hdcp_context.context.bin_desc.size_bytes =
3558                 le32_to_cpu(ta_hdr->hdcp.size_bytes);
3559         adev->psp.hdcp_context.context.bin_desc.start_addr =
3560                 (uint8_t *)ta_hdr +
3561                 le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3562
3563         adev->psp.dtm_context.context.bin_desc.fw_version =
3564                 le32_to_cpu(ta_hdr->dtm.fw_version);
3565         adev->psp.dtm_context.context.bin_desc.size_bytes =
3566                 le32_to_cpu(ta_hdr->dtm.size_bytes);
3567         adev->psp.dtm_context.context.bin_desc.start_addr =
3568                 (uint8_t *)adev->psp.hdcp_context.context.bin_desc.start_addr +
3569                 le32_to_cpu(ta_hdr->dtm.offset_bytes);
3570
3571         adev->psp.securedisplay_context.context.bin_desc.fw_version =
3572                 le32_to_cpu(ta_hdr->securedisplay.fw_version);
3573         adev->psp.securedisplay_context.context.bin_desc.size_bytes =
3574                 le32_to_cpu(ta_hdr->securedisplay.size_bytes);
3575         adev->psp.securedisplay_context.context.bin_desc.start_addr =
3576                 (uint8_t *)adev->psp.hdcp_context.context.bin_desc.start_addr +
3577                 le32_to_cpu(ta_hdr->securedisplay.offset_bytes);
3578
3579         adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version);
3580
3581         return 0;
3582 }
3583
3584 static int parse_ta_v2_microcode(struct psp_context *psp)
3585 {
3586         const struct ta_firmware_header_v2_0 *ta_hdr;
3587         struct amdgpu_device *adev = psp->adev;
3588         int err = 0;
3589         int ta_index = 0;
3590
3591         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3592
3593         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2)
3594                 return -EINVAL;
3595
3596         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3597                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3598                 return -EINVAL;
3599         }
3600
3601         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3602                 err = parse_ta_bin_descriptor(psp,
3603                                               &ta_hdr->ta_fw_bin[ta_index],
3604                                               ta_hdr);
3605                 if (err)
3606                         return err;
3607         }
3608
3609         return 0;
3610 }
3611
3612 int psp_init_ta_microcode(struct psp_context *psp, const char *chip_name)
3613 {
3614         const struct common_firmware_header *hdr;
3615         struct amdgpu_device *adev = psp->adev;
3616         int err;
3617
3618         err = amdgpu_ucode_request(adev, &adev->psp.ta_fw, "amdgpu/%s_ta.bin", chip_name);
3619         if (err)
3620                 return err;
3621
3622         hdr = (const struct common_firmware_header *)adev->psp.ta_fw->data;
3623         switch (le16_to_cpu(hdr->header_version_major)) {
3624         case 1:
3625                 err = parse_ta_v1_microcode(psp);
3626                 break;
3627         case 2:
3628                 err = parse_ta_v2_microcode(psp);
3629                 break;
3630         default:
3631                 dev_err(adev->dev, "unsupported TA header version\n");
3632                 err = -EINVAL;
3633         }
3634
3635         if (err)
3636                 amdgpu_ucode_release(&adev->psp.ta_fw);
3637
3638         return err;
3639 }
3640
3641 int psp_init_cap_microcode(struct psp_context *psp, const char *chip_name)
3642 {
3643         struct amdgpu_device *adev = psp->adev;
3644         const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
3645         struct amdgpu_firmware_info *info = NULL;
3646         int err = 0;
3647
3648         if (!amdgpu_sriov_vf(adev)) {
3649                 dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
3650                 return -EINVAL;
3651         }
3652
3653         err = amdgpu_ucode_request(adev, &adev->psp.cap_fw, "amdgpu/%s_cap.bin", chip_name);
3654         if (err) {
3655                 if (err == -ENODEV) {
3656                         dev_warn(adev->dev, "cap microcode does not exist, skip\n");
3657                         err = 0;
3658                         goto out;
3659                 }
3660                 dev_err(adev->dev, "fail to initialize cap microcode\n");
3661         }
3662
3663         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
3664         info->ucode_id = AMDGPU_UCODE_ID_CAP;
3665         info->fw = adev->psp.cap_fw;
3666         cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
3667                 adev->psp.cap_fw->data;
3668         adev->firmware.fw_size += ALIGN(
3669                         le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
3670         adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
3671         adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
3672         adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
3673
3674         return 0;
3675
3676 out:
3677         amdgpu_ucode_release(&adev->psp.cap_fw);
3678         return err;
3679 }
3680
3681 static int psp_set_clockgating_state(void *handle,
3682                                      enum amd_clockgating_state state)
3683 {
3684         return 0;
3685 }
3686
3687 static int psp_set_powergating_state(void *handle,
3688                                      enum amd_powergating_state state)
3689 {
3690         return 0;
3691 }
3692
3693 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3694                                          struct device_attribute *attr,
3695                                          char *buf)
3696 {
3697         struct drm_device *ddev = dev_get_drvdata(dev);
3698         struct amdgpu_device *adev = drm_to_adev(ddev);
3699         uint32_t fw_ver;
3700         int ret;
3701
3702         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3703                 dev_info(adev->dev, "PSP block is not ready yet\n.");
3704                 return -EBUSY;
3705         }
3706
3707         mutex_lock(&adev->psp.mutex);
3708         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3709         mutex_unlock(&adev->psp.mutex);
3710
3711         if (ret) {
3712                 dev_err(adev->dev, "Failed to read USBC PD FW, err = %d\n", ret);
3713                 return ret;
3714         }
3715
3716         return sysfs_emit(buf, "%x\n", fw_ver);
3717 }
3718
3719 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3720                                                        struct device_attribute *attr,
3721                                                        const char *buf,
3722                                                        size_t count)
3723 {
3724         struct drm_device *ddev = dev_get_drvdata(dev);
3725         struct amdgpu_device *adev = drm_to_adev(ddev);
3726         int ret, idx;
3727         const struct firmware *usbc_pd_fw;
3728         struct amdgpu_bo *fw_buf_bo = NULL;
3729         uint64_t fw_pri_mc_addr;
3730         void *fw_pri_cpu_addr;
3731
3732         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3733                 dev_err(adev->dev, "PSP block is not ready yet.");
3734                 return -EBUSY;
3735         }
3736
3737         if (!drm_dev_enter(ddev, &idx))
3738                 return -ENODEV;
3739
3740         ret = amdgpu_ucode_request(adev, &usbc_pd_fw, "amdgpu/%s", buf);
3741         if (ret)
3742                 goto fail;
3743
3744         /* LFB address which is aligned to 1MB boundary per PSP request */
3745         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3746                                       AMDGPU_GEM_DOMAIN_VRAM |
3747                                       AMDGPU_GEM_DOMAIN_GTT,
3748                                       &fw_buf_bo, &fw_pri_mc_addr,
3749                                       &fw_pri_cpu_addr);
3750         if (ret)
3751                 goto rel_buf;
3752
3753         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3754
3755         mutex_lock(&adev->psp.mutex);
3756         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3757         mutex_unlock(&adev->psp.mutex);
3758
3759         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3760
3761 rel_buf:
3762         amdgpu_ucode_release(&usbc_pd_fw);
3763 fail:
3764         if (ret) {
3765                 dev_err(adev->dev, "Failed to load USBC PD FW, err = %d", ret);
3766                 count = ret;
3767         }
3768
3769         drm_dev_exit(idx);
3770         return count;
3771 }
3772
3773 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3774 {
3775         int idx;
3776
3777         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3778                 return;
3779
3780         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3781         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3782
3783         drm_dev_exit(idx);
3784 }
3785
3786 /**
3787  * DOC: usbc_pd_fw
3788  * Reading from this file will retrieve the USB-C PD firmware version. Writing to
3789  * this file will trigger the update process.
3790  */
3791 static DEVICE_ATTR(usbc_pd_fw, 0644,
3792                    psp_usbc_pd_fw_sysfs_read,
3793                    psp_usbc_pd_fw_sysfs_write);
3794
3795 int is_psp_fw_valid(struct psp_bin_desc bin)
3796 {
3797         return bin.size_bytes;
3798 }
3799
3800 static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj,
3801                                         struct bin_attribute *bin_attr,
3802                                         char *buffer, loff_t pos, size_t count)
3803 {
3804         struct device *dev = kobj_to_dev(kobj);
3805         struct drm_device *ddev = dev_get_drvdata(dev);
3806         struct amdgpu_device *adev = drm_to_adev(ddev);
3807
3808         adev->psp.vbflash_done = false;
3809
3810         /* Safeguard against memory drain */
3811         if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) {
3812                 dev_err(adev->dev, "File size cannot exceed %u\n", AMD_VBIOS_FILE_MAX_SIZE_B);
3813                 kvfree(adev->psp.vbflash_tmp_buf);
3814                 adev->psp.vbflash_tmp_buf = NULL;
3815                 adev->psp.vbflash_image_size = 0;
3816                 return -ENOMEM;
3817         }
3818
3819         /* TODO Just allocate max for now and optimize to realloc later if needed */
3820         if (!adev->psp.vbflash_tmp_buf) {
3821                 adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL);
3822                 if (!adev->psp.vbflash_tmp_buf)
3823                         return -ENOMEM;
3824         }
3825
3826         mutex_lock(&adev->psp.mutex);
3827         memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count);
3828         adev->psp.vbflash_image_size += count;
3829         mutex_unlock(&adev->psp.mutex);
3830
3831         dev_dbg(adev->dev, "IFWI staged for update\n");
3832
3833         return count;
3834 }
3835
3836 static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj,
3837                                        struct bin_attribute *bin_attr, char *buffer,
3838                                        loff_t pos, size_t count)
3839 {
3840         struct device *dev = kobj_to_dev(kobj);
3841         struct drm_device *ddev = dev_get_drvdata(dev);
3842         struct amdgpu_device *adev = drm_to_adev(ddev);
3843         struct amdgpu_bo *fw_buf_bo = NULL;
3844         uint64_t fw_pri_mc_addr;
3845         void *fw_pri_cpu_addr;
3846         int ret;
3847
3848         if (adev->psp.vbflash_image_size == 0)
3849                 return -EINVAL;
3850
3851         dev_dbg(adev->dev, "PSP IFWI flash process initiated\n");
3852
3853         ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size,
3854                                         AMDGPU_GPU_PAGE_SIZE,
3855                                         AMDGPU_GEM_DOMAIN_VRAM,
3856                                         &fw_buf_bo,
3857                                         &fw_pri_mc_addr,
3858                                         &fw_pri_cpu_addr);
3859         if (ret)
3860                 goto rel_buf;
3861
3862         memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf, adev->psp.vbflash_image_size);
3863
3864         mutex_lock(&adev->psp.mutex);
3865         ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
3866         mutex_unlock(&adev->psp.mutex);
3867
3868         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3869
3870 rel_buf:
3871         kvfree(adev->psp.vbflash_tmp_buf);
3872         adev->psp.vbflash_tmp_buf = NULL;
3873         adev->psp.vbflash_image_size = 0;
3874
3875         if (ret) {
3876                 dev_err(adev->dev, "Failed to load IFWI, err = %d\n", ret);
3877                 return ret;
3878         }
3879
3880         dev_dbg(adev->dev, "PSP IFWI flash process done\n");
3881         return 0;
3882 }
3883
3884 /**
3885  * DOC: psp_vbflash
3886  * Writing to this file will stage an IFWI for update. Reading from this file
3887  * will trigger the update process.
3888  */
3889 static struct bin_attribute psp_vbflash_bin_attr = {
3890         .attr = {.name = "psp_vbflash", .mode = 0660},
3891         .size = 0,
3892         .write = amdgpu_psp_vbflash_write,
3893         .read = amdgpu_psp_vbflash_read,
3894 };
3895
3896 /**
3897  * DOC: psp_vbflash_status
3898  * The status of the flash process.
3899  * 0: IFWI flash not complete.
3900  * 1: IFWI flash complete.
3901  */
3902 static ssize_t amdgpu_psp_vbflash_status(struct device *dev,
3903                                          struct device_attribute *attr,
3904                                          char *buf)
3905 {
3906         struct drm_device *ddev = dev_get_drvdata(dev);
3907         struct amdgpu_device *adev = drm_to_adev(ddev);
3908         uint32_t vbflash_status;
3909
3910         vbflash_status = psp_vbflash_status(&adev->psp);
3911         if (!adev->psp.vbflash_done)
3912                 vbflash_status = 0;
3913         else if (adev->psp.vbflash_done && !(vbflash_status & 0x80000000))
3914                 vbflash_status = 1;
3915
3916         return sysfs_emit(buf, "0x%x\n", vbflash_status);
3917 }
3918 static DEVICE_ATTR(psp_vbflash_status, 0440, amdgpu_psp_vbflash_status, NULL);
3919
3920 static struct bin_attribute *bin_flash_attrs[] = {
3921         &psp_vbflash_bin_attr,
3922         NULL
3923 };
3924
3925 static struct attribute *flash_attrs[] = {
3926         &dev_attr_psp_vbflash_status.attr,
3927         &dev_attr_usbc_pd_fw.attr,
3928         NULL
3929 };
3930
3931 static umode_t amdgpu_flash_attr_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
3932 {
3933         struct device *dev = kobj_to_dev(kobj);
3934         struct drm_device *ddev = dev_get_drvdata(dev);
3935         struct amdgpu_device *adev = drm_to_adev(ddev);
3936
3937         if (attr == &dev_attr_usbc_pd_fw.attr)
3938                 return adev->psp.sup_pd_fw_up ? 0660 : 0;
3939
3940         return adev->psp.sup_ifwi_up ? 0440 : 0;
3941 }
3942
3943 static umode_t amdgpu_bin_flash_attr_is_visible(struct kobject *kobj,
3944                                                 struct bin_attribute *attr,
3945                                                 int idx)
3946 {
3947         struct device *dev = kobj_to_dev(kobj);
3948         struct drm_device *ddev = dev_get_drvdata(dev);
3949         struct amdgpu_device *adev = drm_to_adev(ddev);
3950
3951         return adev->psp.sup_ifwi_up ? 0660 : 0;
3952 }
3953
3954 const struct attribute_group amdgpu_flash_attr_group = {
3955         .attrs = flash_attrs,
3956         .bin_attrs = bin_flash_attrs,
3957         .is_bin_visible = amdgpu_bin_flash_attr_is_visible,
3958         .is_visible = amdgpu_flash_attr_is_visible,
3959 };
3960
3961 const struct amd_ip_funcs psp_ip_funcs = {
3962         .name = "psp",
3963         .early_init = psp_early_init,
3964         .late_init = NULL,
3965         .sw_init = psp_sw_init,
3966         .sw_fini = psp_sw_fini,
3967         .hw_init = psp_hw_init,
3968         .hw_fini = psp_hw_fini,
3969         .suspend = psp_suspend,
3970         .resume = psp_resume,
3971         .is_idle = NULL,
3972         .check_soft_reset = NULL,
3973         .wait_for_idle = NULL,
3974         .soft_reset = NULL,
3975         .set_clockgating_state = psp_set_clockgating_state,
3976         .set_powergating_state = psp_set_powergating_state,
3977 };
3978
3979 const struct amdgpu_ip_block_version psp_v3_1_ip_block = {
3980         .type = AMD_IP_BLOCK_TYPE_PSP,
3981         .major = 3,
3982         .minor = 1,
3983         .rev = 0,
3984         .funcs = &psp_ip_funcs,
3985 };
3986
3987 const struct amdgpu_ip_block_version psp_v10_0_ip_block = {
3988         .type = AMD_IP_BLOCK_TYPE_PSP,
3989         .major = 10,
3990         .minor = 0,
3991         .rev = 0,
3992         .funcs = &psp_ip_funcs,
3993 };
3994
3995 const struct amdgpu_ip_block_version psp_v11_0_ip_block = {
3996         .type = AMD_IP_BLOCK_TYPE_PSP,
3997         .major = 11,
3998         .minor = 0,
3999         .rev = 0,
4000         .funcs = &psp_ip_funcs,
4001 };
4002
4003 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
4004         .type = AMD_IP_BLOCK_TYPE_PSP,
4005         .major = 11,
4006         .minor = 0,
4007         .rev = 8,
4008         .funcs = &psp_ip_funcs,
4009 };
4010
4011 const struct amdgpu_ip_block_version psp_v12_0_ip_block = {
4012         .type = AMD_IP_BLOCK_TYPE_PSP,
4013         .major = 12,
4014         .minor = 0,
4015         .rev = 0,
4016         .funcs = &psp_ip_funcs,
4017 };
4018
4019 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
4020         .type = AMD_IP_BLOCK_TYPE_PSP,
4021         .major = 13,
4022         .minor = 0,
4023         .rev = 0,
4024         .funcs = &psp_ip_funcs,
4025 };
4026
4027 const struct amdgpu_ip_block_version psp_v13_0_4_ip_block = {
4028         .type = AMD_IP_BLOCK_TYPE_PSP,
4029         .major = 13,
4030         .minor = 0,
4031         .rev = 4,
4032         .funcs = &psp_ip_funcs,
4033 };
4034
4035 const struct amdgpu_ip_block_version psp_v14_0_ip_block = {
4036         .type = AMD_IP_BLOCK_TYPE_PSP,
4037         .major = 14,
4038         .minor = 0,
4039         .rev = 0,
4040         .funcs = &psp_ip_funcs,
4041 };