drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40 #include "psp_v13_0_4.h"
  41
  42 #include "amdgpu_ras.h"
  43 #include "amdgpu_securedisplay.h"
  44 #include "amdgpu_atomfirmware.h"
  45
  46 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*3)
  47
  48 static int psp_sysfs_init(struct amdgpu_device *adev);
  49 static void psp_sysfs_fini(struct amdgpu_device *adev);
  50
  51 static int psp_load_smu_fw(struct psp_context *psp);
  52 static int psp_rap_terminate(struct psp_context *psp);
  53 static int psp_securedisplay_terminate(struct psp_context *psp);
  54
  55 static int psp_ring_init(struct psp_context *psp,
  56                          enum psp_ring_type ring_type)
  57 {
  58         int ret = 0;
  59         struct psp_ring *ring;
  60         struct amdgpu_device *adev = psp->adev;
  61
  62         ring = &psp->km_ring;
  63
  64         ring->ring_type = ring_type;
  65
  66         /* allocate 4k Page of Local Frame Buffer memory for ring */
  67         ring->ring_size = 0x1000;
  68         ret = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE,
  69                                       AMDGPU_GEM_DOMAIN_VRAM |
  70                                       AMDGPU_GEM_DOMAIN_GTT,
  71                                       &adev->firmware.rbuf,
  72                                       &ring->ring_mem_mc_addr,
  73                                       (void **)&ring->ring_mem);
  74         if (ret) {
  75                 ring->ring_size = 0;
  76                 return ret;
  77         }
  78
  79         return 0;
  80 }
  81
  82 /*
  83  * Due to DF Cstate management centralized to PMFW, the firmware
  84  * loading sequence will be updated as below:
  85  *   - Load KDB
  86  *   - Load SYS_DRV
  87  *   - Load tOS
  88  *   - Load PMFW
  89  *   - Setup TMR
  90  *   - Load other non-psp fw
  91  *   - Load ASD
  92  *   - Load XGMI/RAS/HDCP/DTM TA if any
  93  *
  94  * This new sequence is required for
  95  *   - Arcturus and onwards
  96  */
  97 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  98 {
  99         struct amdgpu_device *adev = psp->adev;
 100
 101         if (amdgpu_sriov_vf(adev)) {
 102                 psp->pmfw_centralized_cstate_management = false;
 103                 return;
 104         }
 105
 106         switch (adev->ip_versions[MP0_HWIP][0]) {
 107         case IP_VERSION(11, 0, 0):
 108         case IP_VERSION(11, 0, 4):
 109         case IP_VERSION(11, 0, 5):
 110         case IP_VERSION(11, 0, 7):
 111         case IP_VERSION(11, 0, 9):
 112         case IP_VERSION(11, 0, 11):
 113         case IP_VERSION(11, 0, 12):
 114         case IP_VERSION(11, 0, 13):
 115         case IP_VERSION(13, 0, 0):
 116         case IP_VERSION(13, 0, 2):
 117         case IP_VERSION(13, 0, 7):
 118                 psp->pmfw_centralized_cstate_management = true;
 119                 break;
 120         default:
 121                 psp->pmfw_centralized_cstate_management = false;
 122                 break;
 123         }
 124 }
 125
 126 static int psp_early_init(void *handle)
 127 {
 128         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 129         struct psp_context *psp = &adev->psp;
 130
 131         switch (adev->ip_versions[MP0_HWIP][0]) {
 132         case IP_VERSION(9, 0, 0):
 133                 psp_v3_1_set_psp_funcs(psp);
 134                 psp->autoload_supported = false;
 135                 break;
 136         case IP_VERSION(10, 0, 0):
 137         case IP_VERSION(10, 0, 1):
 138                 psp_v10_0_set_psp_funcs(psp);
 139                 psp->autoload_supported = false;
 140                 break;
 141         case IP_VERSION(11, 0, 2):
 142         case IP_VERSION(11, 0, 4):
 143                 psp_v11_0_set_psp_funcs(psp);
 144                 psp->autoload_supported = false;
 145                 break;
 146         case IP_VERSION(11, 0, 0):
 147         case IP_VERSION(11, 0, 5):
 148         case IP_VERSION(11, 0, 9):
 149         case IP_VERSION(11, 0, 7):
 150         case IP_VERSION(11, 0, 11):
 151         case IP_VERSION(11, 5, 0):
 152         case IP_VERSION(11, 0, 12):
 153         case IP_VERSION(11, 0, 13):
 154                 psp_v11_0_set_psp_funcs(psp);
 155                 psp->autoload_supported = true;
 156                 break;
 157         case IP_VERSION(11, 0, 3):
 158         case IP_VERSION(12, 0, 1):
 159                 psp_v12_0_set_psp_funcs(psp);
 160                 break;
 161         case IP_VERSION(13, 0, 2):
 162                 psp_v13_0_set_psp_funcs(psp);
 163                 break;
 164         case IP_VERSION(13, 0, 1):
 165         case IP_VERSION(13, 0, 3):
 166         case IP_VERSION(13, 0, 5):
 167         case IP_VERSION(13, 0, 8):
 168         case IP_VERSION(13, 0, 10):
 169         case IP_VERSION(13, 0, 11):
 170                 psp_v13_0_set_psp_funcs(psp);
 171                 psp->autoload_supported = true;
 172                 break;
 173         case IP_VERSION(11, 0, 8):
 174                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 175                         psp_v11_0_8_set_psp_funcs(psp);
 176                         psp->autoload_supported = false;
 177                 }
 178                 break;
 179         case IP_VERSION(13, 0, 0):
 180         case IP_VERSION(13, 0, 7):
 181                 psp_v13_0_set_psp_funcs(psp);
 182                 psp->autoload_supported = true;
 183                 break;
 184         case IP_VERSION(13, 0, 4):
 185                 psp_v13_0_4_set_psp_funcs(psp);
 186                 psp->autoload_supported = true;
 187                 break;
 188         default:
 189                 return -EINVAL;
 190         }
 191
 192         psp->adev = adev;
 193
 194         psp_check_pmfw_centralized_cstate_management(psp);
 195
 196         return 0;
 197 }
 198
 199 void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 200 {
 201         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 202                               &mem_ctx->shared_buf);
 203         mem_ctx->shared_bo = NULL;
 204 }
 205
 206 static void psp_free_shared_bufs(struct psp_context *psp)
 207 {
 208         void *tmr_buf;
 209         void **pptr;
 210
 211         /* free TMR memory buffer */
 212         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 213         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 214         psp->tmr_bo = NULL;
 215
 216         /* free xgmi shared memory */
 217         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
 218
 219         /* free ras shared memory */
 220         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
 221
 222         /* free hdcp shared memory */
 223         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
 224
 225         /* free dtm shared memory */
 226         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
 227
 228         /* free rap shared memory */
 229         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
 230
 231         /* free securedisplay shared memory */
 232         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
 233
 234
 235 }
 236
 237 static void psp_memory_training_fini(struct psp_context *psp)
 238 {
 239         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 240
 241         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 242         kfree(ctx->sys_cache);
 243         ctx->sys_cache = NULL;
 244 }
 245
 246 static int psp_memory_training_init(struct psp_context *psp)
 247 {
 248         int ret;
 249         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 250
 251         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 252                 DRM_DEBUG("memory training is not supported!\n");
 253                 return 0;
 254         }
 255
 256         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 257         if (ctx->sys_cache == NULL) {
 258                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 259                 ret = -ENOMEM;
 260                 goto Err_out;
 261         }
 262
 263         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 264                   ctx->train_data_size,
 265                   ctx->p2c_train_data_offset,
 266                   ctx->c2p_train_data_offset);
 267         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 268         return 0;
 269
 270 Err_out:
 271         psp_memory_training_fini(psp);
 272         return ret;
 273 }
 274
 275 /*
 276  * Helper funciton to query psp runtime database entry
 277  *
 278  * @adev: amdgpu_device pointer
 279  * @entry_type: the type of psp runtime database entry
 280  * @db_entry: runtime database entry pointer
 281  *
 282  * Return false if runtime database doesn't exit or entry is invalid
 283  * or true if the specific database entry is found, and copy to @db_entry
 284  */
 285 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 286                                      enum psp_runtime_entry_type entry_type,
 287                                      void *db_entry)
 288 {
 289         uint64_t db_header_pos, db_dir_pos;
 290         struct psp_runtime_data_header db_header = {0};
 291         struct psp_runtime_data_directory db_dir = {0};
 292         bool ret = false;
 293         int i;
 294
 295         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 296         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 297
 298         /* read runtime db header from vram */
 299         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 300                         sizeof(struct psp_runtime_data_header), false);
 301
 302         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 303                 /* runtime db doesn't exist, exit */
 304                 dev_warn(adev->dev, "PSP runtime database doesn't exist\n");
 305                 return false;
 306         }
 307
 308         /* read runtime database entry from vram */
 309         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 310                         sizeof(struct psp_runtime_data_directory), false);
 311
 312         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 313                 /* invalid db entry count, exit */
 314                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 315                 return false;
 316         }
 317
 318         /* look up for requested entry type */
 319         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 320                 if (db_dir.entry_list[i].entry_type == entry_type) {
 321                         switch (entry_type) {
 322                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 323                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 324                                         /* invalid db entry size */
 325                                         dev_warn(adev->dev, "Invalid PSP runtime database boot cfg entry size\n");
 326                                         return false;
 327                                 }
 328                                 /* read runtime database entry */
 329                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 330                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 331                                 ret = true;
 332                                 break;
 333                         case PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS:
 334                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_scpm_entry)) {
 335                                         /* invalid db entry size */
 336                                         dev_warn(adev->dev, "Invalid PSP runtime database scpm entry size\n");
 337                                         return false;
 338                                 }
 339                                 /* read runtime database entry */
 340                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 341                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_scpm_entry), false);
 342                                 ret = true;
 343                                 break;
 344                         default:
 345                                 ret = false;
 346                                 break;
 347                         }
 348                 }
 349         }
 350
 351         return ret;
 352 }
 353
 354 static int psp_init_sriov_microcode(struct psp_context *psp)
 355 {
 356         struct amdgpu_device *adev = psp->adev;
 357         int ret = 0;
 358
 359         switch (adev->ip_versions[MP0_HWIP][0]) {
 360         case IP_VERSION(9, 0, 0):
 361                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 362                 ret = psp_init_cap_microcode(psp, "vega10");
 363                 break;
 364         case IP_VERSION(11, 0, 9):
 365                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 366                 ret = psp_init_cap_microcode(psp, "navi12");
 367                 break;
 368         case IP_VERSION(11, 0, 7):
 369                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 370                 ret = psp_init_cap_microcode(psp, "sienna_cichlid");
 371                 break;
 372         case IP_VERSION(13, 0, 2):
 373                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 374                 ret = psp_init_cap_microcode(psp, "aldebaran");
 375                 ret &= psp_init_ta_microcode(psp, "aldebaran");
 376                 break;
 377         case IP_VERSION(13, 0, 0):
 378                 adev->virt.autoload_ucode_id = 0;
 379                 break;
 380         case IP_VERSION(13, 0, 10):
 381                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
 382                 break;
 383         default:
 384                 BUG();
 385                 break;
 386         }
 387         return ret;
 388 }
 389
 390 static int psp_sw_init(void *handle)
 391 {
 392         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 393         struct psp_context *psp = &adev->psp;
 394         int ret;
 395         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 396         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 397         struct psp_runtime_scpm_entry scpm_entry;
 398
 399         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 400         if (!psp->cmd) {
 401                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 402                 ret = -ENOMEM;
 403         }
 404
 405         if (amdgpu_sriov_vf(adev))
 406                 ret = psp_init_sriov_microcode(psp);
 407         else
 408                 ret = psp_init_microcode(psp);
 409         if (ret) {
 410                 DRM_ERROR("Failed to load psp firmware!\n");
 411                 return ret;
 412         }
 413
 414         adev->psp.xgmi_context.supports_extended_data =
 415                 !adev->gmc.xgmi.connected_to_cpu &&
 416                         adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2);
 417
 418         memset(&scpm_entry, 0, sizeof(scpm_entry));
 419         if ((psp_get_runtime_db_entry(adev,
 420                                 PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS,
 421                                 &scpm_entry)) &&
 422             (SCPM_DISABLE != scpm_entry.scpm_status)) {
 423                 adev->scpm_enabled = true;
 424                 adev->scpm_status = scpm_entry.scpm_status;
 425         } else {
 426                 adev->scpm_enabled = false;
 427                 adev->scpm_status = SCPM_DISABLE;
 428         }
 429
 430         /* TODO: stop gpu driver services and print alarm if scpm is enabled with error status */
 431
 432         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 433         if (psp_get_runtime_db_entry(adev,
 434                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 435                                 &boot_cfg_entry)) {
 436                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 437                 if ((psp->boot_cfg_bitmask) &
 438                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 439                         /* If psp runtime database exists, then
 440                          * only enable two stage memory training
 441                          * when TWO_STAGE_DRAM_TRAINING bit is set
 442                          * in runtime database */
 443                         mem_training_ctx->enable_mem_training = true;
 444                 }
 445
 446         } else {
 447                 /* If psp runtime database doesn't exist or
 448                  * is invalid, force enable two stage memory
 449                  * training */
 450                 mem_training_ctx->enable_mem_training = true;
 451         }
 452
 453         if (mem_training_ctx->enable_mem_training) {
 454                 ret = psp_memory_training_init(psp);
 455                 if (ret) {
 456                         DRM_ERROR("Failed to initialize memory training!\n");
 457                         return ret;
 458                 }
 459
 460                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 461                 if (ret) {
 462                         DRM_ERROR("Failed to process memory training!\n");
 463                         return ret;
 464                 }
 465         }
 466
 467         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 468             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 469                 ret= psp_sysfs_init(adev);
 470                 if (ret) {
 471                         return ret;
 472                 }
 473         }
 474
 475         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 476                                       amdgpu_sriov_vf(adev) ?
 477                                       AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 478                                       &psp->fw_pri_bo,
 479                                       &psp->fw_pri_mc_addr,
 480                                       &psp->fw_pri_buf);
 481         if (ret)
 482                 return ret;
 483
 484         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 485                                       AMDGPU_GEM_DOMAIN_VRAM,
 486                                       &psp->fence_buf_bo,
 487                                       &psp->fence_buf_mc_addr,
 488                                       &psp->fence_buf);
 489         if (ret)
 490                 goto failed1;
 491
 492         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 493                                       AMDGPU_GEM_DOMAIN_VRAM,
 494                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 495                                       (void **)&psp->cmd_buf_mem);
 496         if (ret)
 497                 goto failed2;
 498
 499         return 0;
 500
 501 failed2:
 502         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 503                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 504 failed1:
 505         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 506                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 507         return ret;
 508 }
 509
 510 static int psp_sw_fini(void *handle)
 511 {
 512         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 513         struct psp_context *psp = &adev->psp;
 514         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 515
 516         psp_memory_training_fini(psp);
 517
 518         release_firmware(psp->sos_fw);
 519         psp->sos_fw = NULL;
 520
 521         release_firmware(psp->asd_fw);
 522         psp->asd_fw = NULL;
 523
 524         release_firmware(psp->ta_fw);
 525         psp->ta_fw = NULL;
 526
 527         release_firmware(psp->cap_fw);
 528         psp->cap_fw = NULL;
 529
 530         release_firmware(psp->toc_fw);
 531         psp->toc_fw = NULL;
 532
 533         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 534             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 535                 psp_sysfs_fini(adev);
 536
 537         kfree(cmd);
 538         cmd = NULL;
 539
 540         if (psp->km_ring.ring_mem)
 541                 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
 542                                       &psp->km_ring.ring_mem_mc_addr,
 543                                       (void **)&psp->km_ring.ring_mem);
 544
 545         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 546                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 547         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 548                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 549         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 550                               (void **)&psp->cmd_buf_mem);
 551
 552         return 0;
 553 }
 554
 555 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 556                  uint32_t reg_val, uint32_t mask, bool check_changed)
 557 {
 558         uint32_t val;
 559         int i;
 560         struct amdgpu_device *adev = psp->adev;
 561
 562         if (psp->adev->no_hw_access)
 563                 return 0;
 564
 565         for (i = 0; i < adev->usec_timeout; i++) {
 566                 val = RREG32(reg_index);
 567                 if (check_changed) {
 568                         if (val != reg_val)
 569                                 return 0;
 570                 } else {
 571                         if ((val & mask) == reg_val)
 572                                 return 0;
 573                 }
 574                 udelay(1);
 575         }
 576
 577         return -ETIME;
 578 }
 579
 580 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 581 {
 582         switch (cmd_id) {
 583         case GFX_CMD_ID_LOAD_TA:
 584                 return "LOAD_TA";
 585         case GFX_CMD_ID_UNLOAD_TA:
 586                 return "UNLOAD_TA";
 587         case GFX_CMD_ID_INVOKE_CMD:
 588                 return "INVOKE_CMD";
 589         case GFX_CMD_ID_LOAD_ASD:
 590                 return "LOAD_ASD";
 591         case GFX_CMD_ID_SETUP_TMR:
 592                 return "SETUP_TMR";
 593         case GFX_CMD_ID_LOAD_IP_FW:
 594                 return "LOAD_IP_FW";
 595         case GFX_CMD_ID_DESTROY_TMR:
 596                 return "DESTROY_TMR";
 597         case GFX_CMD_ID_SAVE_RESTORE:
 598                 return "SAVE_RESTORE_IP_FW";
 599         case GFX_CMD_ID_SETUP_VMR:
 600                 return "SETUP_VMR";
 601         case GFX_CMD_ID_DESTROY_VMR:
 602                 return "DESTROY_VMR";
 603         case GFX_CMD_ID_PROG_REG:
 604                 return "PROG_REG";
 605         case GFX_CMD_ID_GET_FW_ATTESTATION:
 606                 return "GET_FW_ATTESTATION";
 607         case GFX_CMD_ID_LOAD_TOC:
 608                 return "ID_LOAD_TOC";
 609         case GFX_CMD_ID_AUTOLOAD_RLC:
 610                 return "AUTOLOAD_RLC";
 611         case GFX_CMD_ID_BOOT_CFG:
 612                 return "BOOT_CFG";
 613         default:
 614                 return "UNKNOWN CMD";
 615         }
 616 }
 617
 618 static int
 619 psp_cmd_submit_buf(struct psp_context *psp,
 620                    struct amdgpu_firmware_info *ucode,
 621                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 622 {
 623         int ret;
 624         int index, idx;
 625         int timeout = 20000;
 626         bool ras_intr = false;
 627         bool skip_unsupport = false;
 628
 629         if (psp->adev->no_hw_access)
 630                 return 0;
 631
 632         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
 633                 return 0;
 634
 635         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 636
 637         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 638
 639         index = atomic_inc_return(&psp->fence_value);
 640         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 641         if (ret) {
 642                 atomic_dec(&psp->fence_value);
 643                 goto exit;
 644         }
 645
 646         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 647         while (*((unsigned int *)psp->fence_buf) != index) {
 648                 if (--timeout == 0)
 649                         break;
 650                 /*
 651                  * Shouldn't wait for timeout when err_event_athub occurs,
 652                  * because gpu reset thread triggered and lock resource should
 653                  * be released for psp resume sequence.
 654                  */
 655                 ras_intr = amdgpu_ras_intr_triggered();
 656                 if (ras_intr)
 657                         break;
 658                 usleep_range(10, 100);
 659                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 660         }
 661
 662         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 663         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 664                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 665
 666         memcpy((void*)&cmd->resp, (void*)&psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 667
 668         /* In some cases, psp response status is not 0 even there is no
 669          * problem while the command is submitted. Some version of PSP FW
 670          * doesn't write 0 to that field.
 671          * So here we would like to only print a warning instead of an error
 672          * during psp initialization to avoid breaking hw_init and it doesn't
 673          * return -EINVAL.
 674          */
 675         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 676                 if (ucode)
 677                         DRM_WARN("failed to load ucode %s(0x%X) ",
 678                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 679                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 680                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 681                          psp->cmd_buf_mem->resp.status);
 682                 /* If any firmware (including CAP) load fails under SRIOV, it should
 683                  * return failure to stop the VF from initializing.
 684                  * Also return failure in case of timeout
 685                  */
 686                 if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
 687                         ret = -EINVAL;
 688                         goto exit;
 689                 }
 690         }
 691
 692         if (ucode) {
 693                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 694                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 695         }
 696
 697 exit:
 698         drm_dev_exit(idx);
 699         return ret;
 700 }
 701
 702 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 703 {
 704         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 705
 706         mutex_lock(&psp->mutex);
 707
 708         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 709
 710         return cmd;
 711 }
 712
 713 static void release_psp_cmd_buf(struct psp_context *psp)
 714 {
 715         mutex_unlock(&psp->mutex);
 716 }
 717
 718 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 719                                  struct psp_gfx_cmd_resp *cmd,
 720                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 721 {
 722         struct amdgpu_device *adev = psp->adev;
 723         uint32_t size = amdgpu_bo_size(tmr_bo);
 724         uint64_t tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 725
 726         if (amdgpu_sriov_vf(psp->adev))
 727                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 728         else
 729                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 730         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 731         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 732         cmd->cmd.cmd_setup_tmr.buf_size = size;
 733         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 734         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 735         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 736 }
 737
 738 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 739                                       uint64_t pri_buf_mc, uint32_t size)
 740 {
 741         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 742         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 743         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 744         cmd->cmd.cmd_load_toc.toc_size = size;
 745 }
 746
 747 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 748 static int psp_load_toc(struct psp_context *psp,
 749                         uint32_t *tmr_size)
 750 {
 751         int ret;
 752         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 753
 754         /* Copy toc to psp firmware private buffer */
 755         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 756
 757         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 758
 759         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 760                                  psp->fence_buf_mc_addr);
 761         if (!ret)
 762                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 763
 764         release_psp_cmd_buf(psp);
 765
 766         return ret;
 767 }
 768
 769 /* Set up Trusted Memory Region */
 770 static int psp_tmr_init(struct psp_context *psp)
 771 {
 772         int ret = 0;
 773         int tmr_size;
 774         void *tmr_buf;
 775         void **pptr;
 776
 777         /*
 778          * According to HW engineer, they prefer the TMR address be "naturally
 779          * aligned" , e.g. the start address be an integer divide of TMR size.
 780          *
 781          * Note: this memory need be reserved till the driver
 782          * uninitializes.
 783          */
 784         tmr_size = PSP_TMR_SIZE(psp->adev);
 785
 786         /* For ASICs support RLC autoload, psp will parse the toc
 787          * and calculate the total size of TMR needed */
 788         if (!amdgpu_sriov_vf(psp->adev) &&
 789             psp->toc.start_addr &&
 790             psp->toc.size_bytes &&
 791             psp->fw_pri_buf) {
 792                 ret = psp_load_toc(psp, &tmr_size);
 793                 if (ret) {
 794                         DRM_ERROR("Failed to load toc\n");
 795                         return ret;
 796                 }
 797         }
 798
 799         if (!psp->tmr_bo) {
 800                 pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 801                 ret = amdgpu_bo_create_kernel(psp->adev, tmr_size,
 802                                               PSP_TMR_ALIGNMENT,
 803                                               AMDGPU_HAS_VRAM(psp->adev) ?
 804                                               AMDGPU_GEM_DOMAIN_VRAM :
 805                                               AMDGPU_GEM_DOMAIN_GTT,
 806                                               &psp->tmr_bo, &psp->tmr_mc_addr,
 807                                               pptr);
 808         }
 809
 810         return ret;
 811 }
 812
 813 static bool psp_skip_tmr(struct psp_context *psp)
 814 {
 815         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 816         case IP_VERSION(11, 0, 9):
 817         case IP_VERSION(11, 0, 7):
 818         case IP_VERSION(13, 0, 2):
 819         case IP_VERSION(13, 0, 10):
 820                 return true;
 821         default:
 822                 return false;
 823         }
 824 }
 825
 826 static int psp_tmr_load(struct psp_context *psp)
 827 {
 828         int ret;
 829         struct psp_gfx_cmd_resp *cmd;
 830
 831         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 832          * Already set up by host driver.
 833          */
 834         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 835                 return 0;
 836
 837         cmd = acquire_psp_cmd_buf(psp);
 838
 839         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 840         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 841                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 842
 843         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 844                                  psp->fence_buf_mc_addr);
 845
 846         release_psp_cmd_buf(psp);
 847
 848         return ret;
 849 }
 850
 851 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 852                                         struct psp_gfx_cmd_resp *cmd)
 853 {
 854         if (amdgpu_sriov_vf(psp->adev))
 855                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 856         else
 857                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 858 }
 859
 860 static int psp_tmr_unload(struct psp_context *psp)
 861 {
 862         int ret;
 863         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 864
 865         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 866         dev_dbg(psp->adev->dev, "free PSP TMR buffer\n");
 867
 868         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 869                                  psp->fence_buf_mc_addr);
 870
 871         release_psp_cmd_buf(psp);
 872
 873         return ret;
 874 }
 875
 876 static int psp_tmr_terminate(struct psp_context *psp)
 877 {
 878         return psp_tmr_unload(psp);
 879 }
 880
 881 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 882                                         uint64_t *output_ptr)
 883 {
 884         int ret;
 885         struct psp_gfx_cmd_resp *cmd;
 886
 887         if (!output_ptr)
 888                 return -EINVAL;
 889
 890         if (amdgpu_sriov_vf(psp->adev))
 891                 return 0;
 892
 893         cmd = acquire_psp_cmd_buf(psp);
 894
 895         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 896
 897         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 898                                  psp->fence_buf_mc_addr);
 899
 900         if (!ret) {
 901                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 902                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 903         }
 904
 905         release_psp_cmd_buf(psp);
 906
 907         return ret;
 908 }
 909
 910 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 911 {
 912         struct psp_context *psp = &adev->psp;
 913         struct psp_gfx_cmd_resp *cmd;
 914         int ret;
 915
 916         if (amdgpu_sriov_vf(adev))
 917                 return 0;
 918
 919         cmd = acquire_psp_cmd_buf(psp);
 920
 921         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 922         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 923
 924         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 925         if (!ret) {
 926                 *boot_cfg =
 927                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 928         }
 929
 930         release_psp_cmd_buf(psp);
 931
 932         return ret;
 933 }
 934
 935 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 936 {
 937         int ret;
 938         struct psp_context *psp = &adev->psp;
 939         struct psp_gfx_cmd_resp *cmd;
 940
 941         if (amdgpu_sriov_vf(adev))
 942                 return 0;
 943
 944         cmd = acquire_psp_cmd_buf(psp);
 945
 946         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 947         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 948         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 949         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 950
 951         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 952
 953         release_psp_cmd_buf(psp);
 954
 955         return ret;
 956 }
 957
 958 static int psp_rl_load(struct amdgpu_device *adev)
 959 {
 960         int ret;
 961         struct psp_context *psp = &adev->psp;
 962         struct psp_gfx_cmd_resp *cmd;
 963
 964         if (!is_psp_fw_valid(psp->rl))
 965                 return 0;
 966
 967         cmd = acquire_psp_cmd_buf(psp);
 968
 969         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 970         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
 971
 972         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 973         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
 974         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
 975         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
 976         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
 977
 978         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 979
 980         release_psp_cmd_buf(psp);
 981
 982         return ret;
 983 }
 984
 985 static int psp_asd_initialize(struct psp_context *psp)
 986 {
 987         int ret;
 988
 989         /* If PSP version doesn't match ASD version, asd loading will be failed.
 990          * add workaround to bypass it for sriov now.
 991          * TODO: add version check to make it common
 992          */
 993         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 994                 return 0;
 995
 996         psp->asd_context.mem_context.shared_mc_addr  = 0;
 997         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 998         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
 999
1000         ret = psp_ta_load(psp, &psp->asd_context);
1001         if (!ret)
1002                 psp->asd_context.initialized = true;
1003
1004         return ret;
1005 }
1006
1007 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1008                                        uint32_t session_id)
1009 {
1010         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
1011         cmd->cmd.cmd_unload_ta.session_id = session_id;
1012 }
1013
1014 int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
1015 {
1016         int ret;
1017         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1018
1019         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
1020
1021         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1022
1023         context->resp_status = cmd->resp.status;
1024
1025         release_psp_cmd_buf(psp);
1026
1027         return ret;
1028 }
1029
1030 static int psp_asd_terminate(struct psp_context *psp)
1031 {
1032         int ret;
1033
1034         if (amdgpu_sriov_vf(psp->adev))
1035                 return 0;
1036
1037         if (!psp->asd_context.initialized)
1038                 return 0;
1039
1040         ret = psp_ta_unload(psp, &psp->asd_context);
1041         if (!ret)
1042                 psp->asd_context.initialized = false;
1043
1044         return ret;
1045 }
1046
1047 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1048                 uint32_t id, uint32_t value)
1049 {
1050         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
1051         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
1052         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
1053 }
1054
1055 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
1056                 uint32_t value)
1057 {
1058         struct psp_gfx_cmd_resp *cmd;
1059         int ret = 0;
1060
1061         if (reg >= PSP_REG_LAST)
1062                 return -EINVAL;
1063
1064         cmd = acquire_psp_cmd_buf(psp);
1065
1066         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
1067         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1068         if (ret)
1069                 DRM_ERROR("PSP failed to program reg id %d", reg);
1070
1071         release_psp_cmd_buf(psp);
1072
1073         return ret;
1074 }
1075
1076 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1077                                      uint64_t ta_bin_mc,
1078                                      struct ta_context *context)
1079 {
1080         cmd->cmd_id                             = context->ta_load_type;
1081         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
1082         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
1083         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
1084
1085         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
1086                 lower_32_bits(context->mem_context.shared_mc_addr);
1087         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
1088                 upper_32_bits(context->mem_context.shared_mc_addr);
1089         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
1090 }
1091
1092 int psp_ta_init_shared_buf(struct psp_context *psp,
1093                                   struct ta_mem_context *mem_ctx)
1094 {
1095         /*
1096         * Allocate 16k memory aligned to 4k from Frame Buffer (local
1097         * physical) for ta to host memory
1098         */
1099         return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
1100                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM |
1101                                       AMDGPU_GEM_DOMAIN_GTT,
1102                                       &mem_ctx->shared_bo,
1103                                       &mem_ctx->shared_mc_addr,
1104                                       &mem_ctx->shared_buf);
1105 }
1106
1107 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1108                                        uint32_t ta_cmd_id,
1109                                        uint32_t session_id)
1110 {
1111         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
1112         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
1113         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
1114 }
1115
1116 int psp_ta_invoke(struct psp_context *psp,
1117                   uint32_t ta_cmd_id,
1118                   struct ta_context *context)
1119 {
1120         int ret;
1121         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1122
1123         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
1124
1125         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1126                                  psp->fence_buf_mc_addr);
1127
1128         context->resp_status = cmd->resp.status;
1129
1130         release_psp_cmd_buf(psp);
1131
1132         return ret;
1133 }
1134
1135 int psp_ta_load(struct psp_context *psp, struct ta_context *context)
1136 {
1137         int ret;
1138         struct psp_gfx_cmd_resp *cmd;
1139
1140         cmd = acquire_psp_cmd_buf(psp);
1141
1142         psp_copy_fw(psp, context->bin_desc.start_addr,
1143                     context->bin_desc.size_bytes);
1144
1145         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
1146
1147         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1148                                  psp->fence_buf_mc_addr);
1149
1150         context->resp_status = cmd->resp.status;
1151
1152         if (!ret) {
1153                 context->session_id = cmd->resp.session_id;
1154         }
1155
1156         release_psp_cmd_buf(psp);
1157
1158         return ret;
1159 }
1160
1161 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1162 {
1163         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1164 }
1165
1166 int psp_xgmi_terminate(struct psp_context *psp)
1167 {
1168         int ret;
1169         struct amdgpu_device *adev = psp->adev;
1170
1171         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1172         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1173             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1174              adev->gmc.xgmi.connected_to_cpu))
1175                 return 0;
1176
1177         if (!psp->xgmi_context.context.initialized)
1178                 return 0;
1179
1180         ret = psp_ta_unload(psp, &psp->xgmi_context.context);
1181
1182         psp->xgmi_context.context.initialized = false;
1183
1184         return ret;
1185 }
1186
1187 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1188 {
1189         struct ta_xgmi_shared_memory *xgmi_cmd;
1190         int ret;
1191
1192         if (!psp->ta_fw ||
1193             !psp->xgmi_context.context.bin_desc.size_bytes ||
1194             !psp->xgmi_context.context.bin_desc.start_addr)
1195                 return -ENOENT;
1196
1197         if (!load_ta)
1198                 goto invoke;
1199
1200         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1201         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1202
1203         if (!psp->xgmi_context.context.mem_context.shared_buf) {
1204                 ret = psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
1205                 if (ret)
1206                         return ret;
1207         }
1208
1209         /* Load XGMI TA */
1210         ret = psp_ta_load(psp, &psp->xgmi_context.context);
1211         if (!ret)
1212                 psp->xgmi_context.context.initialized = true;
1213         else
1214                 return ret;
1215
1216 invoke:
1217         /* Initialize XGMI session */
1218         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1219         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1220         xgmi_cmd->flag_extend_link_record = set_extended_data;
1221         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1222
1223         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1224
1225         return ret;
1226 }
1227
1228 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1229 {
1230         struct ta_xgmi_shared_memory *xgmi_cmd;
1231         int ret;
1232
1233         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1234         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1235
1236         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1237
1238         /* Invoke xgmi ta to get hive id */
1239         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1240         if (ret)
1241                 return ret;
1242
1243         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1244
1245         return 0;
1246 }
1247
1248 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1249 {
1250         struct ta_xgmi_shared_memory *xgmi_cmd;
1251         int ret;
1252
1253         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1254         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1255
1256         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1257
1258         /* Invoke xgmi ta to get the node id */
1259         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1260         if (ret)
1261                 return ret;
1262
1263         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1264
1265         return 0;
1266 }
1267
1268 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1269 {
1270         return psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1271                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b;
1272 }
1273
1274 /*
1275  * Chips that support extended topology information require the driver to
1276  * reflect topology information in the opposite direction.  This is
1277  * because the TA has already exceeded its link record limit and if the
1278  * TA holds bi-directional information, the driver would have to do
1279  * multiple fetches instead of just two.
1280  */
1281 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1282                                         struct psp_xgmi_node_info node_info)
1283 {
1284         struct amdgpu_device *mirror_adev;
1285         struct amdgpu_hive_info *hive;
1286         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1287         uint64_t dst_node_id = node_info.node_id;
1288         uint8_t dst_num_hops = node_info.num_hops;
1289         uint8_t dst_num_links = node_info.num_links;
1290
1291         hive = amdgpu_get_xgmi_hive(psp->adev);
1292         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1293                 struct psp_xgmi_topology_info *mirror_top_info;
1294                 int j;
1295
1296                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1297                         continue;
1298
1299                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1300                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1301                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1302                                 continue;
1303
1304                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1305                         /*
1306                          * prevent 0 num_links value re-reflection since reflection
1307                          * criteria is based on num_hops (direct or indirect).
1308                          *
1309                          */
1310                         if (dst_num_links)
1311                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1312
1313                         break;
1314                 }
1315
1316                 break;
1317         }
1318
1319         amdgpu_put_xgmi_hive(hive);
1320 }
1321
1322 int psp_xgmi_get_topology_info(struct psp_context *psp,
1323                                int number_devices,
1324                                struct psp_xgmi_topology_info *topology,
1325                                bool get_extended_data)
1326 {
1327         struct ta_xgmi_shared_memory *xgmi_cmd;
1328         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1329         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1330         int i;
1331         int ret;
1332
1333         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1334                 return -EINVAL;
1335
1336         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1337         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1338         xgmi_cmd->flag_extend_link_record = get_extended_data;
1339
1340         /* Fill in the shared memory with topology information as input */
1341         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1342         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1343         topology_info_input->num_nodes = number_devices;
1344
1345         for (i = 0; i < topology_info_input->num_nodes; i++) {
1346                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1347                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1348                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1349                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1350         }
1351
1352         /* Invoke xgmi ta to get the topology information */
1353         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1354         if (ret)
1355                 return ret;
1356
1357         /* Read the output topology information from the shared memory */
1358         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1359         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1360         for (i = 0; i < topology->num_nodes; i++) {
1361                 /* extended data will either be 0 or equal to non-extended data */
1362                 if (topology_info_output->nodes[i].num_hops)
1363                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1364
1365                 /* non-extended data gets everything here so no need to update */
1366                 if (!get_extended_data) {
1367                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1368                         topology->nodes[i].is_sharing_enabled =
1369                                         topology_info_output->nodes[i].is_sharing_enabled;
1370                         topology->nodes[i].sdma_engine =
1371                                         topology_info_output->nodes[i].sdma_engine;
1372                 }
1373
1374         }
1375
1376         /* Invoke xgmi ta again to get the link information */
1377         if (psp_xgmi_peer_link_info_supported(psp)) {
1378                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1379
1380                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1381
1382                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1383
1384                 if (ret)
1385                         return ret;
1386
1387                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1388                 for (i = 0; i < topology->num_nodes; i++) {
1389                         /* accumulate num_links on extended data */
1390                         topology->nodes[i].num_links = get_extended_data ?
1391                                         topology->nodes[i].num_links +
1392                                                         link_info_output->nodes[i].num_links :
1393                                         link_info_output->nodes[i].num_links;
1394
1395                         /* reflect the topology information for bi-directionality */
1396                         if (psp->xgmi_context.supports_extended_data &&
1397                                         get_extended_data && topology->nodes[i].num_hops)
1398                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1399                 }
1400         }
1401
1402         return 0;
1403 }
1404
1405 int psp_xgmi_set_topology_info(struct psp_context *psp,
1406                                int number_devices,
1407                                struct psp_xgmi_topology_info *topology)
1408 {
1409         struct ta_xgmi_shared_memory *xgmi_cmd;
1410         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1411         int i;
1412
1413         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1414                 return -EINVAL;
1415
1416         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1417         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1418
1419         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1420         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1421         topology_info_input->num_nodes = number_devices;
1422
1423         for (i = 0; i < topology_info_input->num_nodes; i++) {
1424                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1425                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1426                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1427                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1428         }
1429
1430         /* Invoke xgmi ta to set topology information */
1431         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1432 }
1433
1434 // ras begin
1435 static void psp_ras_ta_check_status(struct psp_context *psp)
1436 {
1437         struct ta_ras_shared_memory *ras_cmd =
1438                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1439
1440         switch (ras_cmd->ras_status) {
1441         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1442                 dev_warn(psp->adev->dev,
1443                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1444                 break;
1445         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1446                 dev_warn(psp->adev->dev,
1447                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1448                 break;
1449         case TA_RAS_STATUS__SUCCESS:
1450                 break;
1451         case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED:
1452                 if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR)
1453                         dev_warn(psp->adev->dev,
1454                                         "RAS WARNING: Inject error to critical region is not allowed\n");
1455                 break;
1456         default:
1457                 dev_warn(psp->adev->dev,
1458                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1459                 break;
1460         }
1461 }
1462
1463 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1464 {
1465         struct ta_ras_shared_memory *ras_cmd;
1466         int ret;
1467
1468         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1469
1470         /*
1471          * TODO: bypass the loading in sriov for now
1472          */
1473         if (amdgpu_sriov_vf(psp->adev))
1474                 return 0;
1475
1476         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1477
1478         if (amdgpu_ras_intr_triggered())
1479                 return ret;
1480
1481         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
1482         {
1483                 DRM_WARN("RAS: Unsupported Interface");
1484                 return -EINVAL;
1485         }
1486
1487         if (!ret) {
1488                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1489                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1490
1491                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1492                 }
1493                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1494                         dev_warn(psp->adev->dev,
1495                                  "RAS internal register access blocked\n");
1496
1497                 psp_ras_ta_check_status(psp);
1498         }
1499
1500         return ret;
1501 }
1502
1503 int psp_ras_enable_features(struct psp_context *psp,
1504                 union ta_ras_cmd_input *info, bool enable)
1505 {
1506         struct ta_ras_shared_memory *ras_cmd;
1507         int ret;
1508
1509         if (!psp->ras_context.context.initialized)
1510                 return -EINVAL;
1511
1512         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1513         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1514
1515         if (enable)
1516                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1517         else
1518                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1519
1520         ras_cmd->ras_in_message = *info;
1521
1522         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1523         if (ret)
1524                 return -EINVAL;
1525
1526         return 0;
1527 }
1528
1529 int psp_ras_terminate(struct psp_context *psp)
1530 {
1531         int ret;
1532
1533         /*
1534          * TODO: bypass the terminate in sriov for now
1535          */
1536         if (amdgpu_sriov_vf(psp->adev))
1537                 return 0;
1538
1539         if (!psp->ras_context.context.initialized)
1540                 return 0;
1541
1542         ret = psp_ta_unload(psp, &psp->ras_context.context);
1543
1544         psp->ras_context.context.initialized = false;
1545
1546         return ret;
1547 }
1548
1549 int psp_ras_initialize(struct psp_context *psp)
1550 {
1551         int ret;
1552         uint32_t boot_cfg = 0xFF;
1553         struct amdgpu_device *adev = psp->adev;
1554         struct ta_ras_shared_memory *ras_cmd;
1555
1556         /*
1557          * TODO: bypass the initialize in sriov for now
1558          */
1559         if (amdgpu_sriov_vf(adev))
1560                 return 0;
1561
1562         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1563             !adev->psp.ras_context.context.bin_desc.start_addr) {
1564                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1565                 return 0;
1566         }
1567
1568         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1569                 /* query GECC enablement status from boot config
1570                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1571                  */
1572                 ret = psp_boot_config_get(adev, &boot_cfg);
1573                 if (ret)
1574                         dev_warn(adev->dev, "PSP get boot config failed\n");
1575
1576                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1577                         if (!boot_cfg) {
1578                                 dev_info(adev->dev, "GECC is disabled\n");
1579                         } else {
1580                                 /* disable GECC in next boot cycle if ras is
1581                                  * disabled by module parameter amdgpu_ras_enable
1582                                  * and/or amdgpu_ras_mask, or boot_config_get call
1583                                  * is failed
1584                                  */
1585                                 ret = psp_boot_config_set(adev, 0);
1586                                 if (ret)
1587                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1588                                 else
1589                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle "
1590                                                  "if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1591                         }
1592                 } else {
1593                         if (1 == boot_cfg) {
1594                                 dev_info(adev->dev, "GECC is enabled\n");
1595                         } else {
1596                                 /* enable GECC in next boot cycle if it is disabled
1597                                  * in boot config, or force enable GECC if failed to
1598                                  * get boot configuration
1599                                  */
1600                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1601                                 if (ret)
1602                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1603                                 else
1604                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1605                         }
1606                 }
1607         }
1608
1609         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1610         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1611
1612         if (!psp->ras_context.context.mem_context.shared_buf) {
1613                 ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1614                 if (ret)
1615                         return ret;
1616         }
1617
1618         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1619         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1620
1621         if (amdgpu_ras_is_poison_mode_supported(adev))
1622                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1623         if (!adev->gmc.xgmi.connected_to_cpu)
1624                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1625
1626         ret = psp_ta_load(psp, &psp->ras_context.context);
1627
1628         if (!ret && !ras_cmd->ras_status)
1629                 psp->ras_context.context.initialized = true;
1630         else {
1631                 if (ras_cmd->ras_status)
1632                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1633
1634                 /* fail to load RAS TA */
1635                 psp->ras_context.context.initialized = false;
1636         }
1637
1638         return ret;
1639 }
1640
1641 int psp_ras_trigger_error(struct psp_context *psp,
1642                           struct ta_ras_trigger_error_input *info)
1643 {
1644         struct ta_ras_shared_memory *ras_cmd;
1645         int ret;
1646
1647         if (!psp->ras_context.context.initialized)
1648                 return -EINVAL;
1649
1650         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1651         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1652
1653         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1654         ras_cmd->ras_in_message.trigger_error = *info;
1655
1656         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1657         if (ret)
1658                 return -EINVAL;
1659
1660         /* If err_event_athub occurs error inject was successful, however
1661            return status from TA is no long reliable */
1662         if (amdgpu_ras_intr_triggered())
1663                 return 0;
1664
1665         if (ras_cmd->ras_status == TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED)
1666                 return -EACCES;
1667         else if (ras_cmd->ras_status)
1668                 return -EINVAL;
1669
1670         return 0;
1671 }
1672 // ras end
1673
1674 // HDCP start
1675 static int psp_hdcp_initialize(struct psp_context *psp)
1676 {
1677         int ret;
1678
1679         /*
1680          * TODO: bypass the initialize in sriov for now
1681          */
1682         if (amdgpu_sriov_vf(psp->adev))
1683                 return 0;
1684
1685         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1686             !psp->hdcp_context.context.bin_desc.start_addr) {
1687                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1688                 return 0;
1689         }
1690
1691         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1692         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1693
1694         if (!psp->hdcp_context.context.initialized) {
1695                 ret = psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1696                 if (ret)
1697                         return ret;
1698         }
1699
1700         ret = psp_ta_load(psp, &psp->hdcp_context.context);
1701         if (!ret) {
1702                 psp->hdcp_context.context.initialized = true;
1703                 mutex_init(&psp->hdcp_context.mutex);
1704         }
1705
1706         return ret;
1707 }
1708
1709 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1710 {
1711         /*
1712          * TODO: bypass the loading in sriov for now
1713          */
1714         if (amdgpu_sriov_vf(psp->adev))
1715                 return 0;
1716
1717         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1718 }
1719
1720 static int psp_hdcp_terminate(struct psp_context *psp)
1721 {
1722         int ret;
1723
1724         /*
1725          * TODO: bypass the terminate in sriov for now
1726          */
1727         if (amdgpu_sriov_vf(psp->adev))
1728                 return 0;
1729
1730         if (!psp->hdcp_context.context.initialized)
1731                 return 0;
1732
1733         ret = psp_ta_unload(psp, &psp->hdcp_context.context);
1734
1735         psp->hdcp_context.context.initialized = false;
1736
1737         return ret;
1738 }
1739 // HDCP end
1740
1741 // DTM start
1742 static int psp_dtm_initialize(struct psp_context *psp)
1743 {
1744         int ret;
1745
1746         /*
1747          * TODO: bypass the initialize in sriov for now
1748          */
1749         if (amdgpu_sriov_vf(psp->adev))
1750                 return 0;
1751
1752         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1753             !psp->dtm_context.context.bin_desc.start_addr) {
1754                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1755                 return 0;
1756         }
1757
1758         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1759         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1760
1761         if (!psp->dtm_context.context.initialized) {
1762                 ret = psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1763                 if (ret)
1764                         return ret;
1765         }
1766
1767         ret = psp_ta_load(psp, &psp->dtm_context.context);
1768         if (!ret) {
1769                 psp->dtm_context.context.initialized = true;
1770                 mutex_init(&psp->dtm_context.mutex);
1771         }
1772
1773         return ret;
1774 }
1775
1776 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1777 {
1778         /*
1779          * TODO: bypass the loading in sriov for now
1780          */
1781         if (amdgpu_sriov_vf(psp->adev))
1782                 return 0;
1783
1784         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1785 }
1786
1787 static int psp_dtm_terminate(struct psp_context *psp)
1788 {
1789         int ret;
1790
1791         /*
1792          * TODO: bypass the terminate in sriov for now
1793          */
1794         if (amdgpu_sriov_vf(psp->adev))
1795                 return 0;
1796
1797         if (!psp->dtm_context.context.initialized)
1798                 return 0;
1799
1800         ret = psp_ta_unload(psp, &psp->dtm_context.context);
1801
1802         psp->dtm_context.context.initialized = false;
1803
1804         return ret;
1805 }
1806 // DTM end
1807
1808 // RAP start
1809 static int psp_rap_initialize(struct psp_context *psp)
1810 {
1811         int ret;
1812         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1813
1814         /*
1815          * TODO: bypass the initialize in sriov for now
1816          */
1817         if (amdgpu_sriov_vf(psp->adev))
1818                 return 0;
1819
1820         if (!psp->rap_context.context.bin_desc.size_bytes ||
1821             !psp->rap_context.context.bin_desc.start_addr) {
1822                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1823                 return 0;
1824         }
1825
1826         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1827         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1828
1829         if (!psp->rap_context.context.initialized) {
1830                 ret = psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1831                 if (ret)
1832                         return ret;
1833         }
1834
1835         ret = psp_ta_load(psp, &psp->rap_context.context);
1836         if (!ret) {
1837                 psp->rap_context.context.initialized = true;
1838                 mutex_init(&psp->rap_context.mutex);
1839         } else
1840                 return ret;
1841
1842         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1843         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1844                 psp_rap_terminate(psp);
1845                 /* free rap shared memory */
1846                 psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1847
1848                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1849                          ret, status);
1850
1851                 return ret;
1852         }
1853
1854         return 0;
1855 }
1856
1857 static int psp_rap_terminate(struct psp_context *psp)
1858 {
1859         int ret;
1860
1861         if (!psp->rap_context.context.initialized)
1862                 return 0;
1863
1864         ret = psp_ta_unload(psp, &psp->rap_context.context);
1865
1866         psp->rap_context.context.initialized = false;
1867
1868         return ret;
1869 }
1870
1871 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1872 {
1873         struct ta_rap_shared_memory *rap_cmd;
1874         int ret = 0;
1875
1876         if (!psp->rap_context.context.initialized)
1877                 return 0;
1878
1879         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1880             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1881                 return -EINVAL;
1882
1883         mutex_lock(&psp->rap_context.mutex);
1884
1885         rap_cmd = (struct ta_rap_shared_memory *)
1886                   psp->rap_context.context.mem_context.shared_buf;
1887         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1888
1889         rap_cmd->cmd_id = ta_cmd_id;
1890         rap_cmd->validation_method_id = METHOD_A;
1891
1892         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1893         if (ret)
1894                 goto out_unlock;
1895
1896         if (status)
1897                 *status = rap_cmd->rap_status;
1898
1899 out_unlock:
1900         mutex_unlock(&psp->rap_context.mutex);
1901
1902         return ret;
1903 }
1904 // RAP end
1905
1906 /* securedisplay start */
1907 static int psp_securedisplay_initialize(struct psp_context *psp)
1908 {
1909         int ret;
1910         struct ta_securedisplay_cmd *securedisplay_cmd;
1911
1912         /*
1913          * TODO: bypass the initialize in sriov for now
1914          */
1915         if (amdgpu_sriov_vf(psp->adev))
1916                 return 0;
1917
1918         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1919             !psp->securedisplay_context.context.bin_desc.start_addr) {
1920                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
1921                 return 0;
1922         }
1923
1924         psp->securedisplay_context.context.mem_context.shared_mem_size =
1925                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
1926         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1927
1928         if (!psp->securedisplay_context.context.initialized) {
1929                 ret = psp_ta_init_shared_buf(psp,
1930                                              &psp->securedisplay_context.context.mem_context);
1931                 if (ret)
1932                         return ret;
1933         }
1934
1935         ret = psp_ta_load(psp, &psp->securedisplay_context.context);
1936         if (!ret) {
1937                 psp->securedisplay_context.context.initialized = true;
1938                 mutex_init(&psp->securedisplay_context.mutex);
1939         } else
1940                 return ret;
1941
1942         mutex_lock(&psp->securedisplay_context.mutex);
1943
1944         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
1945                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1946
1947         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
1948
1949         mutex_unlock(&psp->securedisplay_context.mutex);
1950
1951         if (ret) {
1952                 psp_securedisplay_terminate(psp);
1953                 /* free securedisplay shared memory */
1954                 psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
1955                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
1956                 return -EINVAL;
1957         }
1958
1959         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
1960                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
1961                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
1962                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
1963         }
1964
1965         return 0;
1966 }
1967
1968 static int psp_securedisplay_terminate(struct psp_context *psp)
1969 {
1970         int ret;
1971
1972         /*
1973          * TODO:bypass the terminate in sriov for now
1974          */
1975         if (amdgpu_sriov_vf(psp->adev))
1976                 return 0;
1977
1978         if (!psp->securedisplay_context.context.initialized)
1979                 return 0;
1980
1981         ret = psp_ta_unload(psp, &psp->securedisplay_context.context);
1982
1983         psp->securedisplay_context.context.initialized = false;
1984
1985         return ret;
1986 }
1987
1988 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1989 {
1990         int ret;
1991
1992         if (!psp->securedisplay_context.context.initialized)
1993                 return -EINVAL;
1994
1995         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
1996             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
1997                 return -EINVAL;
1998
1999         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
2000
2001         return ret;
2002 }
2003 /* SECUREDISPLAY end */
2004
2005 static int psp_hw_start(struct psp_context *psp)
2006 {
2007         struct amdgpu_device *adev = psp->adev;
2008         int ret;
2009
2010         if (!amdgpu_sriov_vf(adev)) {
2011                 if ((is_psp_fw_valid(psp->kdb)) &&
2012                     (psp->funcs->bootloader_load_kdb != NULL)) {
2013                         ret = psp_bootloader_load_kdb(psp);
2014                         if (ret) {
2015                                 DRM_ERROR("PSP load kdb failed!\n");
2016                                 return ret;
2017                         }
2018                 }
2019
2020                 if ((is_psp_fw_valid(psp->spl)) &&
2021                     (psp->funcs->bootloader_load_spl != NULL)) {
2022                         ret = psp_bootloader_load_spl(psp);
2023                         if (ret) {
2024                                 DRM_ERROR("PSP load spl failed!\n");
2025                                 return ret;
2026                         }
2027                 }
2028
2029                 if ((is_psp_fw_valid(psp->sys)) &&
2030                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
2031                         ret = psp_bootloader_load_sysdrv(psp);
2032                         if (ret) {
2033                                 DRM_ERROR("PSP load sys drv failed!\n");
2034                                 return ret;
2035                         }
2036                 }
2037
2038                 if ((is_psp_fw_valid(psp->soc_drv)) &&
2039                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
2040                         ret = psp_bootloader_load_soc_drv(psp);
2041                         if (ret) {
2042                                 DRM_ERROR("PSP load soc drv failed!\n");
2043                                 return ret;
2044                         }
2045                 }
2046
2047                 if ((is_psp_fw_valid(psp->intf_drv)) &&
2048                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
2049                         ret = psp_bootloader_load_intf_drv(psp);
2050                         if (ret) {
2051                                 DRM_ERROR("PSP load intf drv failed!\n");
2052                                 return ret;
2053                         }
2054                 }
2055
2056                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
2057                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
2058                         ret = psp_bootloader_load_dbg_drv(psp);
2059                         if (ret) {
2060                                 DRM_ERROR("PSP load dbg drv failed!\n");
2061                                 return ret;
2062                         }
2063                 }
2064
2065                 if ((is_psp_fw_valid(psp->ras_drv)) &&
2066                     (psp->funcs->bootloader_load_ras_drv != NULL)) {
2067                         ret = psp_bootloader_load_ras_drv(psp);
2068                         if (ret) {
2069                                 DRM_ERROR("PSP load ras_drv failed!\n");
2070                                 return ret;
2071                         }
2072                 }
2073
2074                 if ((is_psp_fw_valid(psp->sos)) &&
2075                     (psp->funcs->bootloader_load_sos != NULL)) {
2076                         ret = psp_bootloader_load_sos(psp);
2077                         if (ret) {
2078                                 DRM_ERROR("PSP load sos failed!\n");
2079                                 return ret;
2080                         }
2081                 }
2082         }
2083
2084         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2085         if (ret) {
2086                 DRM_ERROR("PSP create ring failed!\n");
2087                 return ret;
2088         }
2089
2090         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2091                 goto skip_pin_bo;
2092
2093         ret = psp_tmr_init(psp);
2094         if (ret) {
2095                 DRM_ERROR("PSP tmr init failed!\n");
2096                 return ret;
2097         }
2098
2099 skip_pin_bo:
2100         /*
2101          * For ASICs with DF Cstate management centralized
2102          * to PMFW, TMR setup should be performed after PMFW
2103          * loaded and before other non-psp firmware loaded.
2104          */
2105         if (psp->pmfw_centralized_cstate_management) {
2106                 ret = psp_load_smu_fw(psp);
2107                 if (ret)
2108                         return ret;
2109         }
2110
2111         ret = psp_tmr_load(psp);
2112         if (ret) {
2113                 DRM_ERROR("PSP load tmr failed!\n");
2114                 return ret;
2115         }
2116
2117         return 0;
2118 }
2119
2120 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2121                            enum psp_gfx_fw_type *type)
2122 {
2123         switch (ucode->ucode_id) {
2124         case AMDGPU_UCODE_ID_CAP:
2125                 *type = GFX_FW_TYPE_CAP;
2126                 break;
2127         case AMDGPU_UCODE_ID_SDMA0:
2128                 *type = GFX_FW_TYPE_SDMA0;
2129                 break;
2130         case AMDGPU_UCODE_ID_SDMA1:
2131                 *type = GFX_FW_TYPE_SDMA1;
2132                 break;
2133         case AMDGPU_UCODE_ID_SDMA2:
2134                 *type = GFX_FW_TYPE_SDMA2;
2135                 break;
2136         case AMDGPU_UCODE_ID_SDMA3:
2137                 *type = GFX_FW_TYPE_SDMA3;
2138                 break;
2139         case AMDGPU_UCODE_ID_SDMA4:
2140                 *type = GFX_FW_TYPE_SDMA4;
2141                 break;
2142         case AMDGPU_UCODE_ID_SDMA5:
2143                 *type = GFX_FW_TYPE_SDMA5;
2144                 break;
2145         case AMDGPU_UCODE_ID_SDMA6:
2146                 *type = GFX_FW_TYPE_SDMA6;
2147                 break;
2148         case AMDGPU_UCODE_ID_SDMA7:
2149                 *type = GFX_FW_TYPE_SDMA7;
2150                 break;
2151         case AMDGPU_UCODE_ID_CP_MES:
2152                 *type = GFX_FW_TYPE_CP_MES;
2153                 break;
2154         case AMDGPU_UCODE_ID_CP_MES_DATA:
2155                 *type = GFX_FW_TYPE_MES_STACK;
2156                 break;
2157         case AMDGPU_UCODE_ID_CP_MES1:
2158                 *type = GFX_FW_TYPE_CP_MES_KIQ;
2159                 break;
2160         case AMDGPU_UCODE_ID_CP_MES1_DATA:
2161                 *type = GFX_FW_TYPE_MES_KIQ_STACK;
2162                 break;
2163         case AMDGPU_UCODE_ID_CP_CE:
2164                 *type = GFX_FW_TYPE_CP_CE;
2165                 break;
2166         case AMDGPU_UCODE_ID_CP_PFP:
2167                 *type = GFX_FW_TYPE_CP_PFP;
2168                 break;
2169         case AMDGPU_UCODE_ID_CP_ME:
2170                 *type = GFX_FW_TYPE_CP_ME;
2171                 break;
2172         case AMDGPU_UCODE_ID_CP_MEC1:
2173                 *type = GFX_FW_TYPE_CP_MEC;
2174                 break;
2175         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2176                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2177                 break;
2178         case AMDGPU_UCODE_ID_CP_MEC2:
2179                 *type = GFX_FW_TYPE_CP_MEC;
2180                 break;
2181         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2182                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2183                 break;
2184         case AMDGPU_UCODE_ID_RLC_P:
2185                 *type = GFX_FW_TYPE_RLC_P;
2186                 break;
2187         case AMDGPU_UCODE_ID_RLC_V:
2188                 *type = GFX_FW_TYPE_RLC_V;
2189                 break;
2190         case AMDGPU_UCODE_ID_RLC_G:
2191                 *type = GFX_FW_TYPE_RLC_G;
2192                 break;
2193         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2194                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2195                 break;
2196         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2197                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2198                 break;
2199         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2200                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2201                 break;
2202         case AMDGPU_UCODE_ID_RLC_IRAM:
2203                 *type = GFX_FW_TYPE_RLC_IRAM;
2204                 break;
2205         case AMDGPU_UCODE_ID_RLC_DRAM:
2206                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2207                 break;
2208         case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
2209                 *type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
2210                 break;
2211         case AMDGPU_UCODE_ID_SE0_TAP_DELAYS:
2212                 *type = GFX_FW_TYPE_SE0_TAP_DELAYS;
2213                 break;
2214         case AMDGPU_UCODE_ID_SE1_TAP_DELAYS:
2215                 *type = GFX_FW_TYPE_SE1_TAP_DELAYS;
2216                 break;
2217         case AMDGPU_UCODE_ID_SE2_TAP_DELAYS:
2218                 *type = GFX_FW_TYPE_SE2_TAP_DELAYS;
2219                 break;
2220         case AMDGPU_UCODE_ID_SE3_TAP_DELAYS:
2221                 *type = GFX_FW_TYPE_SE3_TAP_DELAYS;
2222                 break;
2223         case AMDGPU_UCODE_ID_SMC:
2224                 *type = GFX_FW_TYPE_SMU;
2225                 break;
2226         case AMDGPU_UCODE_ID_PPTABLE:
2227                 *type = GFX_FW_TYPE_PPTABLE;
2228                 break;
2229         case AMDGPU_UCODE_ID_UVD:
2230                 *type = GFX_FW_TYPE_UVD;
2231                 break;
2232         case AMDGPU_UCODE_ID_UVD1:
2233                 *type = GFX_FW_TYPE_UVD1;
2234                 break;
2235         case AMDGPU_UCODE_ID_VCE:
2236                 *type = GFX_FW_TYPE_VCE;
2237                 break;
2238         case AMDGPU_UCODE_ID_VCN:
2239                 *type = GFX_FW_TYPE_VCN;
2240                 break;
2241         case AMDGPU_UCODE_ID_VCN1:
2242                 *type = GFX_FW_TYPE_VCN1;
2243                 break;
2244         case AMDGPU_UCODE_ID_DMCU_ERAM:
2245                 *type = GFX_FW_TYPE_DMCU_ERAM;
2246                 break;
2247         case AMDGPU_UCODE_ID_DMCU_INTV:
2248                 *type = GFX_FW_TYPE_DMCU_ISR;
2249                 break;
2250         case AMDGPU_UCODE_ID_VCN0_RAM:
2251                 *type = GFX_FW_TYPE_VCN0_RAM;
2252                 break;
2253         case AMDGPU_UCODE_ID_VCN1_RAM:
2254                 *type = GFX_FW_TYPE_VCN1_RAM;
2255                 break;
2256         case AMDGPU_UCODE_ID_DMCUB:
2257                 *type = GFX_FW_TYPE_DMUB;
2258                 break;
2259         case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
2260                 *type = GFX_FW_TYPE_SDMA_UCODE_TH0;
2261                 break;
2262         case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
2263                 *type = GFX_FW_TYPE_SDMA_UCODE_TH1;
2264                 break;
2265         case AMDGPU_UCODE_ID_IMU_I:
2266                 *type = GFX_FW_TYPE_IMU_I;
2267                 break;
2268         case AMDGPU_UCODE_ID_IMU_D:
2269                 *type = GFX_FW_TYPE_IMU_D;
2270                 break;
2271         case AMDGPU_UCODE_ID_CP_RS64_PFP:
2272                 *type = GFX_FW_TYPE_RS64_PFP;
2273                 break;
2274         case AMDGPU_UCODE_ID_CP_RS64_ME:
2275                 *type = GFX_FW_TYPE_RS64_ME;
2276                 break;
2277         case AMDGPU_UCODE_ID_CP_RS64_MEC:
2278                 *type = GFX_FW_TYPE_RS64_MEC;
2279                 break;
2280         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
2281                 *type = GFX_FW_TYPE_RS64_PFP_P0_STACK;
2282                 break;
2283         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
2284                 *type = GFX_FW_TYPE_RS64_PFP_P1_STACK;
2285                 break;
2286         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
2287                 *type = GFX_FW_TYPE_RS64_ME_P0_STACK;
2288                 break;
2289         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
2290                 *type = GFX_FW_TYPE_RS64_ME_P1_STACK;
2291                 break;
2292         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
2293                 *type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
2294                 break;
2295         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
2296                 *type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
2297                 break;
2298         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
2299                 *type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
2300                 break;
2301         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
2302                 *type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
2303                 break;
2304         case AMDGPU_UCODE_ID_MAXIMUM:
2305         default:
2306                 return -EINVAL;
2307         }
2308
2309         return 0;
2310 }
2311
2312 static void psp_print_fw_hdr(struct psp_context *psp,
2313                              struct amdgpu_firmware_info *ucode)
2314 {
2315         struct amdgpu_device *adev = psp->adev;
2316         struct common_firmware_header *hdr;
2317
2318         switch (ucode->ucode_id) {
2319         case AMDGPU_UCODE_ID_SDMA0:
2320         case AMDGPU_UCODE_ID_SDMA1:
2321         case AMDGPU_UCODE_ID_SDMA2:
2322         case AMDGPU_UCODE_ID_SDMA3:
2323         case AMDGPU_UCODE_ID_SDMA4:
2324         case AMDGPU_UCODE_ID_SDMA5:
2325         case AMDGPU_UCODE_ID_SDMA6:
2326         case AMDGPU_UCODE_ID_SDMA7:
2327                 hdr = (struct common_firmware_header *)
2328                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2329                 amdgpu_ucode_print_sdma_hdr(hdr);
2330                 break;
2331         case AMDGPU_UCODE_ID_CP_CE:
2332                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2333                 amdgpu_ucode_print_gfx_hdr(hdr);
2334                 break;
2335         case AMDGPU_UCODE_ID_CP_PFP:
2336                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2337                 amdgpu_ucode_print_gfx_hdr(hdr);
2338                 break;
2339         case AMDGPU_UCODE_ID_CP_ME:
2340                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2341                 amdgpu_ucode_print_gfx_hdr(hdr);
2342                 break;
2343         case AMDGPU_UCODE_ID_CP_MEC1:
2344                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2345                 amdgpu_ucode_print_gfx_hdr(hdr);
2346                 break;
2347         case AMDGPU_UCODE_ID_RLC_G:
2348                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2349                 amdgpu_ucode_print_rlc_hdr(hdr);
2350                 break;
2351         case AMDGPU_UCODE_ID_SMC:
2352                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2353                 amdgpu_ucode_print_smc_hdr(hdr);
2354                 break;
2355         default:
2356                 break;
2357         }
2358 }
2359
2360 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2361                                        struct psp_gfx_cmd_resp *cmd)
2362 {
2363         int ret;
2364         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2365
2366         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2367         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2368         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2369         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2370
2371         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2372         if (ret)
2373                 DRM_ERROR("Unknown firmware type\n");
2374
2375         return ret;
2376 }
2377
2378 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2379                                   struct amdgpu_firmware_info *ucode)
2380 {
2381         int ret = 0;
2382         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2383
2384         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2385         if (!ret) {
2386                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2387                                          psp->fence_buf_mc_addr);
2388         }
2389
2390         release_psp_cmd_buf(psp);
2391
2392         return ret;
2393 }
2394
2395 static int psp_load_smu_fw(struct psp_context *psp)
2396 {
2397         int ret;
2398         struct amdgpu_device *adev = psp->adev;
2399         struct amdgpu_firmware_info *ucode =
2400                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2401         struct amdgpu_ras *ras = psp->ras_context.ras;
2402
2403         /*
2404          * Skip SMU FW reloading in case of using BACO for runpm only,
2405          * as SMU is always alive.
2406          */
2407         if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
2408                 return 0;
2409
2410         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2411                 return 0;
2412
2413         if ((amdgpu_in_reset(adev) &&
2414              ras && adev->ras_enabled &&
2415              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2416               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2417                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2418                 if (ret) {
2419                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2420                 }
2421         }
2422
2423         ret = psp_execute_non_psp_fw_load(psp, ucode);
2424
2425         if (ret)
2426                 DRM_ERROR("PSP load smu failed!\n");
2427
2428         return ret;
2429 }
2430
2431 static bool fw_load_skip_check(struct psp_context *psp,
2432                                struct amdgpu_firmware_info *ucode)
2433 {
2434         if (!ucode->fw || !ucode->ucode_size)
2435                 return true;
2436
2437         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2438             (psp_smu_reload_quirk(psp) ||
2439              psp->autoload_supported ||
2440              psp->pmfw_centralized_cstate_management))
2441                 return true;
2442
2443         if (amdgpu_sriov_vf(psp->adev) &&
2444             amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
2445                 return true;
2446
2447         if (psp->autoload_supported &&
2448             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2449              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2450                 /* skip mec JT when autoload is enabled */
2451                 return true;
2452
2453         return false;
2454 }
2455
2456 int psp_load_fw_list(struct psp_context *psp,
2457                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2458 {
2459         int ret = 0, i;
2460         struct amdgpu_firmware_info *ucode;
2461
2462         for (i = 0; i < ucode_count; ++i) {
2463                 ucode = ucode_list[i];
2464                 psp_print_fw_hdr(psp, ucode);
2465                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2466                 if (ret)
2467                         return ret;
2468         }
2469         return ret;
2470 }
2471
2472 static int psp_load_non_psp_fw(struct psp_context *psp)
2473 {
2474         int i, ret;
2475         struct amdgpu_firmware_info *ucode;
2476         struct amdgpu_device *adev = psp->adev;
2477
2478         if (psp->autoload_supported &&
2479             !psp->pmfw_centralized_cstate_management) {
2480                 ret = psp_load_smu_fw(psp);
2481                 if (ret)
2482                         return ret;
2483         }
2484
2485         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2486                 ucode = &adev->firmware.ucode[i];
2487
2488                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2489                     !fw_load_skip_check(psp, ucode)) {
2490                         ret = psp_load_smu_fw(psp);
2491                         if (ret)
2492                                 return ret;
2493                         continue;
2494                 }
2495
2496                 if (fw_load_skip_check(psp, ucode))
2497                         continue;
2498
2499                 if (psp->autoload_supported &&
2500                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2501                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2502                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2503                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2504                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2505                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2506                         /* PSP only receive one SDMA fw for sienna_cichlid,
2507                          * as all four sdma fw are same */
2508                         continue;
2509
2510                 psp_print_fw_hdr(psp, ucode);
2511
2512                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2513                 if (ret)
2514                         return ret;
2515
2516                 /* Start rlc autoload after psp recieved all the gfx firmware */
2517                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2518                     adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
2519                         ret = psp_rlc_autoload_start(psp);
2520                         if (ret) {
2521                                 DRM_ERROR("Failed to start rlc autoload\n");
2522                                 return ret;
2523                         }
2524                 }
2525         }
2526
2527         return 0;
2528 }
2529
2530 static int psp_load_fw(struct amdgpu_device *adev)
2531 {
2532         int ret;
2533         struct psp_context *psp = &adev->psp;
2534
2535         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2536                 /* should not destroy ring, only stop */
2537                 psp_ring_stop(psp, PSP_RING_TYPE__KM);
2538         } else {
2539                 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2540
2541                 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2542                 if (ret) {
2543                         DRM_ERROR("PSP ring init failed!\n");
2544                         goto failed;
2545                 }
2546         }
2547
2548         ret = psp_hw_start(psp);
2549         if (ret)
2550                 goto failed;
2551
2552         ret = psp_load_non_psp_fw(psp);
2553         if (ret)
2554                 goto failed1;
2555
2556         ret = psp_asd_initialize(psp);
2557         if (ret) {
2558                 DRM_ERROR("PSP load asd failed!\n");
2559                 goto failed1;
2560         }
2561
2562         ret = psp_rl_load(adev);
2563         if (ret) {
2564                 DRM_ERROR("PSP load RL failed!\n");
2565                 goto failed1;
2566         }
2567
2568         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2569                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2570                         ret = psp_xgmi_initialize(psp, false, true);
2571                         /* Warning the XGMI seesion initialize failure
2572                         * Instead of stop driver initialization
2573                         */
2574                         if (ret)
2575                                 dev_err(psp->adev->dev,
2576                                         "XGMI: Failed to initialize XGMI session\n");
2577                 }
2578         }
2579
2580         if (psp->ta_fw) {
2581                 ret = psp_ras_initialize(psp);
2582                 if (ret)
2583                         dev_err(psp->adev->dev,
2584                                         "RAS: Failed to initialize RAS\n");
2585
2586                 ret = psp_hdcp_initialize(psp);
2587                 if (ret)
2588                         dev_err(psp->adev->dev,
2589                                 "HDCP: Failed to initialize HDCP\n");
2590
2591                 ret = psp_dtm_initialize(psp);
2592                 if (ret)
2593                         dev_err(psp->adev->dev,
2594                                 "DTM: Failed to initialize DTM\n");
2595
2596                 ret = psp_rap_initialize(psp);
2597                 if (ret)
2598                         dev_err(psp->adev->dev,
2599                                 "RAP: Failed to initialize RAP\n");
2600
2601                 ret = psp_securedisplay_initialize(psp);
2602                 if (ret)
2603                         dev_err(psp->adev->dev,
2604                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2605         }
2606
2607         return 0;
2608
2609 failed1:
2610         psp_free_shared_bufs(psp);
2611 failed:
2612         /*
2613          * all cleanup jobs (xgmi terminate, ras terminate,
2614          * ring destroy, cmd/fence/fw buffers destory,
2615          * psp->cmd destory) are delayed to psp_hw_fini
2616          */
2617         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2618         return ret;
2619 }
2620
2621 static int psp_hw_init(void *handle)
2622 {
2623         int ret;
2624         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2625
2626         mutex_lock(&adev->firmware.mutex);
2627         /*
2628          * This sequence is just used on hw_init only once, no need on
2629          * resume.
2630          */
2631         ret = amdgpu_ucode_init_bo(adev);
2632         if (ret)
2633                 goto failed;
2634
2635         ret = psp_load_fw(adev);
2636         if (ret) {
2637                 DRM_ERROR("PSP firmware loading failed\n");
2638                 goto failed;
2639         }
2640
2641         mutex_unlock(&adev->firmware.mutex);
2642         return 0;
2643
2644 failed:
2645         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2646         mutex_unlock(&adev->firmware.mutex);
2647         return -EINVAL;
2648 }
2649
2650 static int psp_hw_fini(void *handle)
2651 {
2652         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2653         struct psp_context *psp = &adev->psp;
2654
2655         if (psp->ta_fw) {
2656                 psp_ras_terminate(psp);
2657                 psp_securedisplay_terminate(psp);
2658                 psp_rap_terminate(psp);
2659                 psp_dtm_terminate(psp);
2660                 psp_hdcp_terminate(psp);
2661
2662                 if (adev->gmc.xgmi.num_physical_nodes > 1)
2663                         psp_xgmi_terminate(psp);
2664         }
2665
2666         psp_asd_terminate(psp);
2667         psp_tmr_terminate(psp);
2668
2669         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2670
2671         psp_free_shared_bufs(psp);
2672
2673         return 0;
2674 }
2675
2676 static int psp_suspend(void *handle)
2677 {
2678         int ret = 0;
2679         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2680         struct psp_context *psp = &adev->psp;
2681
2682         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2683             psp->xgmi_context.context.initialized) {
2684                 ret = psp_xgmi_terminate(psp);
2685                 if (ret) {
2686                         DRM_ERROR("Failed to terminate xgmi ta\n");
2687                         goto out;
2688                 }
2689         }
2690
2691         if (psp->ta_fw) {
2692                 ret = psp_ras_terminate(psp);
2693                 if (ret) {
2694                         DRM_ERROR("Failed to terminate ras ta\n");
2695                         goto out;
2696                 }
2697                 ret = psp_hdcp_terminate(psp);
2698                 if (ret) {
2699                         DRM_ERROR("Failed to terminate hdcp ta\n");
2700                         goto out;
2701                 }
2702                 ret = psp_dtm_terminate(psp);
2703                 if (ret) {
2704                         DRM_ERROR("Failed to terminate dtm ta\n");
2705                         goto out;
2706                 }
2707                 ret = psp_rap_terminate(psp);
2708                 if (ret) {
2709                         DRM_ERROR("Failed to terminate rap ta\n");
2710                         goto out;
2711                 }
2712                 ret = psp_securedisplay_terminate(psp);
2713                 if (ret) {
2714                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2715                         goto out;
2716                 }
2717         }
2718
2719         ret = psp_asd_terminate(psp);
2720         if (ret) {
2721                 DRM_ERROR("Failed to terminate asd\n");
2722                 goto out;
2723         }
2724
2725         ret = psp_tmr_terminate(psp);
2726         if (ret) {
2727                 DRM_ERROR("Failed to terminate tmr\n");
2728                 goto out;
2729         }
2730
2731         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2732         if (ret) {
2733                 DRM_ERROR("PSP ring stop failed\n");
2734         }
2735
2736 out:
2737         return ret;
2738 }
2739
2740 static int psp_resume(void *handle)
2741 {
2742         int ret;
2743         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2744         struct psp_context *psp = &adev->psp;
2745
2746         DRM_INFO("PSP is resuming...\n");
2747
2748         if (psp->mem_train_ctx.enable_mem_training) {
2749                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2750                 if (ret) {
2751                         DRM_ERROR("Failed to process memory training!\n");
2752                         return ret;
2753                 }
2754         }
2755
2756         mutex_lock(&adev->firmware.mutex);
2757
2758         ret = psp_hw_start(psp);
2759         if (ret)
2760                 goto failed;
2761
2762         ret = psp_load_non_psp_fw(psp);
2763         if (ret)
2764                 goto failed;
2765
2766         ret = psp_asd_initialize(psp);
2767         if (ret) {
2768                 DRM_ERROR("PSP load asd failed!\n");
2769                 goto failed;
2770         }
2771
2772         ret = psp_rl_load(adev);
2773         if (ret) {
2774                 dev_err(adev->dev, "PSP load RL failed!\n");
2775                 goto failed;
2776         }
2777
2778         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2779                 ret = psp_xgmi_initialize(psp, false, true);
2780                 /* Warning the XGMI seesion initialize failure
2781                  * Instead of stop driver initialization
2782                  */
2783                 if (ret)
2784                         dev_err(psp->adev->dev,
2785                                 "XGMI: Failed to initialize XGMI session\n");
2786         }
2787
2788         if (psp->ta_fw) {
2789                 ret = psp_ras_initialize(psp);
2790                 if (ret)
2791                         dev_err(psp->adev->dev,
2792                                         "RAS: Failed to initialize RAS\n");
2793
2794                 ret = psp_hdcp_initialize(psp);
2795                 if (ret)
2796                         dev_err(psp->adev->dev,
2797                                 "HDCP: Failed to initialize HDCP\n");
2798
2799                 ret = psp_dtm_initialize(psp);
2800                 if (ret)
2801                         dev_err(psp->adev->dev,
2802                                 "DTM: Failed to initialize DTM\n");
2803
2804                 ret = psp_rap_initialize(psp);
2805                 if (ret)
2806                         dev_err(psp->adev->dev,
2807                                 "RAP: Failed to initialize RAP\n");
2808
2809                 ret = psp_securedisplay_initialize(psp);
2810                 if (ret)
2811                         dev_err(psp->adev->dev,
2812                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2813         }
2814
2815         mutex_unlock(&adev->firmware.mutex);
2816
2817         return 0;
2818
2819 failed:
2820         DRM_ERROR("PSP resume failed\n");
2821         mutex_unlock(&adev->firmware.mutex);
2822         return ret;
2823 }
2824
2825 int psp_gpu_reset(struct amdgpu_device *adev)
2826 {
2827         int ret;
2828
2829         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2830                 return 0;
2831
2832         mutex_lock(&adev->psp.mutex);
2833         ret = psp_mode1_reset(&adev->psp);
2834         mutex_unlock(&adev->psp.mutex);
2835
2836         return ret;
2837 }
2838
2839 int psp_rlc_autoload_start(struct psp_context *psp)
2840 {
2841         int ret;
2842         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2843
2844         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2845
2846         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2847                                  psp->fence_buf_mc_addr);
2848
2849         release_psp_cmd_buf(psp);
2850
2851         return ret;
2852 }
2853
2854 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2855                         uint64_t cmd_gpu_addr, int cmd_size)
2856 {
2857         struct amdgpu_firmware_info ucode = {0};
2858
2859         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2860                 AMDGPU_UCODE_ID_VCN0_RAM;
2861         ucode.mc_addr = cmd_gpu_addr;
2862         ucode.ucode_size = cmd_size;
2863
2864         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2865 }
2866
2867 int psp_ring_cmd_submit(struct psp_context *psp,
2868                         uint64_t cmd_buf_mc_addr,
2869                         uint64_t fence_mc_addr,
2870                         int index)
2871 {
2872         unsigned int psp_write_ptr_reg = 0;
2873         struct psp_gfx_rb_frame *write_frame;
2874         struct psp_ring *ring = &psp->km_ring;
2875         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2876         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2877                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2878         struct amdgpu_device *adev = psp->adev;
2879         uint32_t ring_size_dw = ring->ring_size / 4;
2880         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2881
2882         /* KM (GPCOM) prepare write pointer */
2883         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2884
2885         /* Update KM RB frame pointer to new frame */
2886         /* write_frame ptr increments by size of rb_frame in bytes */
2887         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2888         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2889                 write_frame = ring_buffer_start;
2890         else
2891                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2892         /* Check invalid write_frame ptr address */
2893         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2894                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2895                           ring_buffer_start, ring_buffer_end, write_frame);
2896                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2897                 return -EINVAL;
2898         }
2899
2900         /* Initialize KM RB frame */
2901         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2902
2903         /* Update KM RB frame */
2904         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2905         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2906         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2907         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2908         write_frame->fence_value = index;
2909         amdgpu_device_flush_hdp(adev, NULL);
2910
2911         /* Update the write Pointer in DWORDs */
2912         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2913         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2914         return 0;
2915 }
2916
2917 int psp_init_asd_microcode(struct psp_context *psp,
2918                            const char *chip_name)
2919 {
2920         struct amdgpu_device *adev = psp->adev;
2921         char fw_name[PSP_FW_NAME_LEN];
2922         const struct psp_firmware_header_v1_0 *asd_hdr;
2923         int err = 0;
2924
2925         if (!chip_name) {
2926                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2927                 return -EINVAL;
2928         }
2929
2930         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2931         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2932         if (err)
2933                 goto out;
2934
2935         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2936         if (err)
2937                 goto out;
2938
2939         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2940         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2941         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
2942         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2943         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
2944                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2945         return 0;
2946 out:
2947         dev_err(adev->dev, "fail to initialize asd microcode\n");
2948         release_firmware(adev->psp.asd_fw);
2949         adev->psp.asd_fw = NULL;
2950         return err;
2951 }
2952
2953 int psp_init_toc_microcode(struct psp_context *psp,
2954                            const char *chip_name)
2955 {
2956         struct amdgpu_device *adev = psp->adev;
2957         char fw_name[PSP_FW_NAME_LEN];
2958         const struct psp_firmware_header_v1_0 *toc_hdr;
2959         int err = 0;
2960
2961         if (!chip_name) {
2962                 dev_err(adev->dev, "invalid chip name for toc microcode\n");
2963                 return -EINVAL;
2964         }
2965
2966         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
2967         err = request_firmware(&adev->psp.toc_fw, fw_name, adev->dev);
2968         if (err)
2969                 goto out;
2970
2971         err = amdgpu_ucode_validate(adev->psp.toc_fw);
2972         if (err)
2973                 goto out;
2974
2975         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
2976         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
2977         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
2978         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
2979         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
2980                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
2981         return 0;
2982 out:
2983         dev_err(adev->dev, "fail to request/validate toc microcode\n");
2984         release_firmware(adev->psp.toc_fw);
2985         adev->psp.toc_fw = NULL;
2986         return err;
2987 }
2988
2989 static int parse_sos_bin_descriptor(struct psp_context *psp,
2990                                    const struct psp_fw_bin_desc *desc,
2991                                    const struct psp_firmware_header_v2_0 *sos_hdr)
2992 {
2993         uint8_t *ucode_start_addr  = NULL;
2994
2995         if (!psp || !desc || !sos_hdr)
2996                 return -EINVAL;
2997
2998         ucode_start_addr  = (uint8_t *)sos_hdr +
2999                             le32_to_cpu(desc->offset_bytes) +
3000                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3001
3002         switch (desc->fw_type) {
3003         case PSP_FW_TYPE_PSP_SOS:
3004                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
3005                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
3006                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
3007                 psp->sos.start_addr        = ucode_start_addr;
3008                 break;
3009         case PSP_FW_TYPE_PSP_SYS_DRV:
3010                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
3011                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
3012                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
3013                 psp->sys.start_addr        = ucode_start_addr;
3014                 break;
3015         case PSP_FW_TYPE_PSP_KDB:
3016                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
3017                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
3018                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
3019                 psp->kdb.start_addr        = ucode_start_addr;
3020                 break;
3021         case PSP_FW_TYPE_PSP_TOC:
3022                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
3023                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
3024                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
3025                 psp->toc.start_addr        = ucode_start_addr;
3026                 break;
3027         case PSP_FW_TYPE_PSP_SPL:
3028                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
3029                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
3030                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
3031                 psp->spl.start_addr        = ucode_start_addr;
3032                 break;
3033         case PSP_FW_TYPE_PSP_RL:
3034                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
3035                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
3036                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
3037                 psp->rl.start_addr         = ucode_start_addr;
3038                 break;
3039         case PSP_FW_TYPE_PSP_SOC_DRV:
3040                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
3041                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
3042                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3043                 psp->soc_drv.start_addr         = ucode_start_addr;
3044                 break;
3045         case PSP_FW_TYPE_PSP_INTF_DRV:
3046                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
3047                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
3048                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
3049                 psp->intf_drv.start_addr        = ucode_start_addr;
3050                 break;
3051         case PSP_FW_TYPE_PSP_DBG_DRV:
3052                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
3053                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
3054                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3055                 psp->dbg_drv.start_addr         = ucode_start_addr;
3056                 break;
3057         case PSP_FW_TYPE_PSP_RAS_DRV:
3058                 psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
3059                 psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
3060                 psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3061                 psp->ras_drv.start_addr         = ucode_start_addr;
3062                 break;
3063         default:
3064                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
3065                 break;
3066         }
3067
3068         return 0;
3069 }
3070
3071 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
3072 {
3073         const struct psp_firmware_header_v1_0 *sos_hdr;
3074         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3075         uint8_t *ucode_array_start_addr;
3076
3077         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3078         ucode_array_start_addr = (uint8_t *)sos_hdr +
3079                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3080
3081         if (adev->gmc.xgmi.connected_to_cpu ||
3082             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
3083                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
3084                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
3085
3086                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
3087                 adev->psp.sys.start_addr = ucode_array_start_addr;
3088
3089                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
3090                 adev->psp.sos.start_addr = ucode_array_start_addr +
3091                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
3092         } else {
3093                 /* Load alternate PSP SOS FW */
3094                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3095
3096                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3097                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3098
3099                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
3100                 adev->psp.sys.start_addr = ucode_array_start_addr +
3101                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
3102
3103                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
3104                 adev->psp.sos.start_addr = ucode_array_start_addr +
3105                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
3106         }
3107
3108         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
3109                 dev_warn(adev->dev, "PSP SOS FW not available");
3110                 return -EINVAL;
3111         }
3112
3113         return 0;
3114 }
3115
3116 int psp_init_sos_microcode(struct psp_context *psp,
3117                            const char *chip_name)
3118 {
3119         struct amdgpu_device *adev = psp->adev;
3120         char fw_name[PSP_FW_NAME_LEN];
3121         const struct psp_firmware_header_v1_0 *sos_hdr;
3122         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3123         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3124         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3125         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3126         int err = 0;
3127         uint8_t *ucode_array_start_addr;
3128         int fw_index = 0;
3129
3130         if (!chip_name) {
3131                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
3132                 return -EINVAL;
3133         }
3134
3135         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3136         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
3137         if (err)
3138                 goto out;
3139
3140         err = amdgpu_ucode_validate(adev->psp.sos_fw);
3141         if (err)
3142                 goto out;
3143
3144         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3145         ucode_array_start_addr = (uint8_t *)sos_hdr +
3146                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3147         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3148
3149         switch (sos_hdr->header.header_version_major) {
3150         case 1:
3151                 err = psp_init_sos_base_fw(adev);
3152                 if (err)
3153                         goto out;
3154
3155                 if (sos_hdr->header.header_version_minor == 1) {
3156                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3157                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3158                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3159                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3160                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3161                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3162                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3163                 }
3164                 if (sos_hdr->header.header_version_minor == 2) {
3165                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3166                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3167                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3168                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3169                 }
3170                 if (sos_hdr->header.header_version_minor == 3) {
3171                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3172                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3173                         adev->psp.toc.start_addr = ucode_array_start_addr +
3174                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3175                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3176                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3177                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3178                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3179                         adev->psp.spl.start_addr = ucode_array_start_addr +
3180                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3181                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3182                         adev->psp.rl.start_addr = ucode_array_start_addr +
3183                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3184                 }
3185                 break;
3186         case 2:
3187                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3188
3189                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3190                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3191                         err = -EINVAL;
3192                         goto out;
3193                 }
3194
3195                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3196                         err = parse_sos_bin_descriptor(psp,
3197                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3198                                                        sos_hdr_v2_0);
3199                         if (err)
3200                                 goto out;
3201                 }
3202                 break;
3203         default:
3204                 dev_err(adev->dev,
3205                         "unsupported psp sos firmware\n");
3206                 err = -EINVAL;
3207                 goto out;
3208         }
3209
3210         return 0;
3211 out:
3212         dev_err(adev->dev,
3213                 "failed to init sos firmware\n");
3214         release_firmware(adev->psp.sos_fw);
3215         adev->psp.sos_fw = NULL;
3216
3217         return err;
3218 }
3219
3220 static int parse_ta_bin_descriptor(struct psp_context *psp,
3221                                    const struct psp_fw_bin_desc *desc,
3222                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3223 {
3224         uint8_t *ucode_start_addr  = NULL;
3225
3226         if (!psp || !desc || !ta_hdr)
3227                 return -EINVAL;
3228
3229         ucode_start_addr  = (uint8_t *)ta_hdr +
3230                             le32_to_cpu(desc->offset_bytes) +
3231                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3232
3233         switch (desc->fw_type) {
3234         case TA_FW_TYPE_PSP_ASD:
3235                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3236                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3237                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3238                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3239                 break;
3240         case TA_FW_TYPE_PSP_XGMI:
3241                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3242                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3243                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3244                 break;
3245         case TA_FW_TYPE_PSP_RAS:
3246                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3247                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3248                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3249                 break;
3250         case TA_FW_TYPE_PSP_HDCP:
3251                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3252                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3253                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3254                 break;
3255         case TA_FW_TYPE_PSP_DTM:
3256                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3257                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3258                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3259                 break;
3260         case TA_FW_TYPE_PSP_RAP:
3261                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3262                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3263                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3264                 break;
3265         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3266                 psp->securedisplay_context.context.bin_desc.fw_version =
3267                         le32_to_cpu(desc->fw_version);
3268                 psp->securedisplay_context.context.bin_desc.size_bytes =
3269                         le32_to_cpu(desc->size_bytes);
3270                 psp->securedisplay_context.context.bin_desc.start_addr =
3271                         ucode_start_addr;
3272                 break;
3273         default:
3274                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3275                 break;
3276         }
3277
3278         return 0;
3279 }
3280
3281 int psp_init_ta_microcode(struct psp_context *psp,
3282                           const char *chip_name)
3283 {
3284         struct amdgpu_device *adev = psp->adev;
3285         char fw_name[PSP_FW_NAME_LEN];
3286         const struct ta_firmware_header_v2_0 *ta_hdr;
3287         int err = 0;
3288         int ta_index = 0;
3289
3290         if (!chip_name) {
3291                 dev_err(adev->dev, "invalid chip name for ta microcode\n");
3292                 return -EINVAL;
3293         }
3294
3295         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3296         err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
3297         if (err)
3298                 goto out;
3299
3300         err = amdgpu_ucode_validate(adev->psp.ta_fw);
3301         if (err)
3302                 goto out;
3303
3304         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3305
3306         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2) {
3307                 dev_err(adev->dev, "unsupported TA header version\n");
3308                 err = -EINVAL;
3309                 goto out;
3310         }
3311
3312         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3313                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3314                 err = -EINVAL;
3315                 goto out;
3316         }
3317
3318         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3319                 err = parse_ta_bin_descriptor(psp,
3320                                               &ta_hdr->ta_fw_bin[ta_index],
3321                                               ta_hdr);
3322                 if (err)
3323                         goto out;
3324         }
3325
3326         return 0;
3327 out:
3328         dev_err(adev->dev, "fail to initialize ta microcode\n");
3329         release_firmware(adev->psp.ta_fw);
3330         adev->psp.ta_fw = NULL;
3331         return err;
3332 }
3333
3334 int psp_init_cap_microcode(struct psp_context *psp,
3335                           const char *chip_name)
3336 {
3337         struct amdgpu_device *adev = psp->adev;
3338         char fw_name[PSP_FW_NAME_LEN];
3339         const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
3340         struct amdgpu_firmware_info *info = NULL;
3341         int err = 0;
3342
3343         if (!chip_name) {
3344                 dev_err(adev->dev, "invalid chip name for cap microcode\n");
3345                 return -EINVAL;
3346         }
3347
3348         if (!amdgpu_sriov_vf(adev)) {
3349                 dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
3350                 return -EINVAL;
3351         }
3352
3353         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_cap.bin", chip_name);
3354         err = request_firmware(&adev->psp.cap_fw, fw_name, adev->dev);
3355         if (err) {
3356                 dev_warn(adev->dev, "cap microcode does not exist, skip\n");
3357                 err = 0;
3358                 goto out;
3359         }
3360
3361         err = amdgpu_ucode_validate(adev->psp.cap_fw);
3362         if (err) {
3363                 dev_err(adev->dev, "fail to initialize cap microcode\n");
3364                 goto out;
3365         }
3366
3367         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
3368         info->ucode_id = AMDGPU_UCODE_ID_CAP;
3369         info->fw = adev->psp.cap_fw;
3370         cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
3371                 adev->psp.cap_fw->data;
3372         adev->firmware.fw_size += ALIGN(
3373                         le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
3374         adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
3375         adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
3376         adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
3377
3378         return 0;
3379
3380 out:
3381         release_firmware(adev->psp.cap_fw);
3382         adev->psp.cap_fw = NULL;
3383         return err;
3384 }
3385
3386 static int psp_set_clockgating_state(void *handle,
3387                                      enum amd_clockgating_state state)
3388 {
3389         return 0;
3390 }
3391
3392 static int psp_set_powergating_state(void *handle,
3393                                      enum amd_powergating_state state)
3394 {
3395         return 0;
3396 }
3397
3398 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3399                                          struct device_attribute *attr,
3400                                          char *buf)
3401 {
3402         struct drm_device *ddev = dev_get_drvdata(dev);
3403         struct amdgpu_device *adev = drm_to_adev(ddev);
3404         uint32_t fw_ver;
3405         int ret;
3406
3407         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3408                 DRM_INFO("PSP block is not ready yet.");
3409                 return -EBUSY;
3410         }
3411
3412         mutex_lock(&adev->psp.mutex);
3413         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3414         mutex_unlock(&adev->psp.mutex);
3415
3416         if (ret) {
3417                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3418                 return ret;
3419         }
3420
3421         return sysfs_emit(buf, "%x\n", fw_ver);
3422 }
3423
3424 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3425                                                        struct device_attribute *attr,
3426                                                        const char *buf,
3427                                                        size_t count)
3428 {
3429         struct drm_device *ddev = dev_get_drvdata(dev);
3430         struct amdgpu_device *adev = drm_to_adev(ddev);
3431         int ret, idx;
3432         char fw_name[100];
3433         const struct firmware *usbc_pd_fw;
3434         struct amdgpu_bo *fw_buf_bo = NULL;
3435         uint64_t fw_pri_mc_addr;
3436         void *fw_pri_cpu_addr;
3437
3438         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3439                 DRM_INFO("PSP block is not ready yet.");
3440                 return -EBUSY;
3441         }
3442
3443         if (!drm_dev_enter(ddev, &idx))
3444                 return -ENODEV;
3445
3446         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3447         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3448         if (ret)
3449                 goto fail;
3450
3451         /* LFB address which is aligned to 1MB boundary per PSP request */
3452         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3453                                       AMDGPU_GEM_DOMAIN_VRAM |
3454                                       AMDGPU_GEM_DOMAIN_GTT,
3455                                       &fw_buf_bo, &fw_pri_mc_addr,
3456                                       &fw_pri_cpu_addr);
3457         if (ret)
3458                 goto rel_buf;
3459
3460         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3461
3462         mutex_lock(&adev->psp.mutex);
3463         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3464         mutex_unlock(&adev->psp.mutex);
3465
3466         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3467
3468 rel_buf:
3469         release_firmware(usbc_pd_fw);
3470 fail:
3471         if (ret) {
3472                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3473                 count = ret;
3474         }
3475
3476         drm_dev_exit(idx);
3477         return count;
3478 }
3479
3480 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3481 {
3482         int idx;
3483
3484         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3485                 return;
3486
3487         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3488         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3489
3490         drm_dev_exit(idx);
3491 }
3492
3493 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
3494                    psp_usbc_pd_fw_sysfs_read,
3495                    psp_usbc_pd_fw_sysfs_write);
3496
3497 int is_psp_fw_valid(struct psp_bin_desc bin)
3498 {
3499         return bin.size_bytes;
3500 }
3501
3502 static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj,
3503                                         struct bin_attribute *bin_attr,
3504                                         char *buffer, loff_t pos, size_t count)
3505 {
3506         struct device *dev = kobj_to_dev(kobj);
3507         struct drm_device *ddev = dev_get_drvdata(dev);
3508         struct amdgpu_device *adev = drm_to_adev(ddev);
3509
3510         adev->psp.vbflash_done = false;
3511
3512         /* Safeguard against memory drain */
3513         if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) {
3514                 dev_err(adev->dev, "File size cannot exceed %u", AMD_VBIOS_FILE_MAX_SIZE_B);
3515                 kvfree(adev->psp.vbflash_tmp_buf);
3516                 adev->psp.vbflash_tmp_buf = NULL;
3517                 adev->psp.vbflash_image_size = 0;
3518                 return -ENOMEM;
3519         }
3520
3521         /* TODO Just allocate max for now and optimize to realloc later if needed */
3522         if (!adev->psp.vbflash_tmp_buf) {
3523                 adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL);
3524                 if (!adev->psp.vbflash_tmp_buf)
3525                         return -ENOMEM;
3526         }
3527
3528         mutex_lock(&adev->psp.mutex);
3529         memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count);
3530         adev->psp.vbflash_image_size += count;
3531         mutex_unlock(&adev->psp.mutex);
3532
3533         dev_info(adev->dev, "VBIOS flash write PSP done");
3534
3535         return count;
3536 }
3537
3538 static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj,
3539                                        struct bin_attribute *bin_attr, char *buffer,
3540                                        loff_t pos, size_t count)
3541 {
3542         struct device *dev = kobj_to_dev(kobj);
3543         struct drm_device *ddev = dev_get_drvdata(dev);
3544         struct amdgpu_device *adev = drm_to_adev(ddev);
3545         struct amdgpu_bo *fw_buf_bo = NULL;
3546         uint64_t fw_pri_mc_addr;
3547         void *fw_pri_cpu_addr;
3548         int ret;
3549
3550         dev_info(adev->dev, "VBIOS flash to PSP started");
3551
3552         ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size,
3553                                         AMDGPU_GPU_PAGE_SIZE,
3554                                         AMDGPU_GEM_DOMAIN_VRAM,
3555                                         &fw_buf_bo,
3556                                         &fw_pri_mc_addr,
3557                                         &fw_pri_cpu_addr);
3558         if (ret)
3559                 goto rel_buf;
3560
3561         memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf, adev->psp.vbflash_image_size);
3562
3563         mutex_lock(&adev->psp.mutex);
3564         ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
3565         mutex_unlock(&adev->psp.mutex);
3566
3567         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3568
3569 rel_buf:
3570         kvfree(adev->psp.vbflash_tmp_buf);
3571         adev->psp.vbflash_tmp_buf = NULL;
3572         adev->psp.vbflash_image_size = 0;
3573
3574         if (ret) {
3575                 dev_err(adev->dev, "Failed to load VBIOS FW, err = %d", ret);
3576                 return ret;
3577         }
3578
3579         dev_info(adev->dev, "VBIOS flash to PSP done");
3580         return 0;
3581 }
3582
3583 static ssize_t amdgpu_psp_vbflash_status(struct device *dev,
3584                                          struct device_attribute *attr,
3585                                          char *buf)
3586 {
3587         struct drm_device *ddev = dev_get_drvdata(dev);
3588         struct amdgpu_device *adev = drm_to_adev(ddev);
3589         uint32_t vbflash_status;
3590
3591         vbflash_status = psp_vbflash_status(&adev->psp);
3592         if (!adev->psp.vbflash_done)
3593                 vbflash_status = 0;
3594         else if (adev->psp.vbflash_done && !(vbflash_status & 0x80000000))
3595                 vbflash_status = 1;
3596
3597         return sysfs_emit(buf, "0x%x\n", vbflash_status);
3598 }
3599
3600 static const struct bin_attribute psp_vbflash_bin_attr = {
3601         .attr = {.name = "psp_vbflash", .mode = 0664},
3602         .size = 0,
3603         .write = amdgpu_psp_vbflash_write,
3604         .read = amdgpu_psp_vbflash_read,
3605 };
3606
3607 static DEVICE_ATTR(psp_vbflash_status, 0444, amdgpu_psp_vbflash_status, NULL);
3608
3609 int amdgpu_psp_sysfs_init(struct amdgpu_device *adev)
3610 {
3611         int ret = 0;
3612         struct psp_context *psp = &adev->psp;
3613
3614         if (amdgpu_sriov_vf(adev))
3615                 return -EINVAL;
3616
3617         switch (adev->ip_versions[MP0_HWIP][0]) {
3618         case IP_VERSION(13, 0, 0):
3619         case IP_VERSION(13, 0, 7):
3620                 if (!psp->adev) {
3621                         psp->adev = adev;
3622                         psp_v13_0_set_psp_funcs(psp);
3623                 }
3624                 ret = sysfs_create_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3625                 if (ret)
3626                         dev_err(adev->dev, "Failed to create device file psp_vbflash");
3627                 ret = device_create_file(adev->dev, &dev_attr_psp_vbflash_status);
3628                 if (ret)
3629                         dev_err(adev->dev, "Failed to create device file psp_vbflash_status");
3630                 return ret;
3631         default:
3632                 return 0;
3633         }
3634 }
3635
3636 const struct amd_ip_funcs psp_ip_funcs = {
3637         .name = "psp",
3638         .early_init = psp_early_init,
3639         .late_init = NULL,
3640         .sw_init = psp_sw_init,
3641         .sw_fini = psp_sw_fini,
3642         .hw_init = psp_hw_init,
3643         .hw_fini = psp_hw_fini,
3644         .suspend = psp_suspend,
3645         .resume = psp_resume,
3646         .is_idle = NULL,
3647         .check_soft_reset = NULL,
3648         .wait_for_idle = NULL,
3649         .soft_reset = NULL,
3650         .set_clockgating_state = psp_set_clockgating_state,
3651         .set_powergating_state = psp_set_powergating_state,
3652 };
3653
3654 static int psp_sysfs_init(struct amdgpu_device *adev)
3655 {
3656         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3657
3658         if (ret)
3659                 DRM_ERROR("Failed to create USBC PD FW control file!");
3660
3661         return ret;
3662 }
3663
3664 void amdgpu_psp_sysfs_fini(struct amdgpu_device *adev)
3665 {
3666         sysfs_remove_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3667         device_remove_file(adev->dev, &dev_attr_psp_vbflash_status);
3668 }
3669
3670 static void psp_sysfs_fini(struct amdgpu_device *adev)
3671 {
3672         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3673 }
3674
3675 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
3676 {
3677         .type = AMD_IP_BLOCK_TYPE_PSP,
3678         .major = 3,
3679         .minor = 1,
3680         .rev = 0,
3681         .funcs = &psp_ip_funcs,
3682 };
3683
3684 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
3685 {
3686         .type = AMD_IP_BLOCK_TYPE_PSP,
3687         .major = 10,
3688         .minor = 0,
3689         .rev = 0,
3690         .funcs = &psp_ip_funcs,
3691 };
3692
3693 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
3694 {
3695         .type = AMD_IP_BLOCK_TYPE_PSP,
3696         .major = 11,
3697         .minor = 0,
3698         .rev = 0,
3699         .funcs = &psp_ip_funcs,
3700 };
3701
3702 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3703         .type = AMD_IP_BLOCK_TYPE_PSP,
3704         .major = 11,
3705         .minor = 0,
3706         .rev = 8,
3707         .funcs = &psp_ip_funcs,
3708 };
3709
3710 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
3711 {
3712         .type = AMD_IP_BLOCK_TYPE_PSP,
3713         .major = 12,
3714         .minor = 0,
3715         .rev = 0,
3716         .funcs = &psp_ip_funcs,
3717 };
3718
3719 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3720         .type = AMD_IP_BLOCK_TYPE_PSP,
3721         .major = 13,
3722         .minor = 0,
3723         .rev = 0,
3724         .funcs = &psp_ip_funcs,
3725 };
3726
3727 const struct amdgpu_ip_block_version psp_v13_0_4_ip_block = {
3728         .type = AMD_IP_BLOCK_TYPE_PSP,
3729         .major = 13,
3730         .minor = 0,
3731         .rev = 4,
3732         .funcs = &psp_ip_funcs,
3733 };