drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <linux/dma-mapping.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "soc15_common.h"
  33 #include "psp_v3_1.h"
  34 #include "psp_v10_0.h"
  35 #include "psp_v11_0.h"
  36 #include "psp_v12_0.h"
  37
  38 #include "amdgpu_ras.h"
  39
  40 static int psp_sysfs_init(struct amdgpu_device *adev);
  41 static void psp_sysfs_fini(struct amdgpu_device *adev);
  42
  43 static int psp_load_smu_fw(struct psp_context *psp);
  44
  45 /*
  46  * Due to DF Cstate management centralized to PMFW, the firmware
  47  * loading sequence will be updated as below:
  48  *   - Load KDB
  49  *   - Load SYS_DRV
  50  *   - Load tOS
  51  *   - Load PMFW
  52  *   - Setup TMR
  53  *   - Load other non-psp fw
  54  *   - Load ASD
  55  *   - Load XGMI/RAS/HDCP/DTM TA if any
  56  *
  57  * This new sequence is required for
  58  *   - Arcturus
  59  *   - Navi12 and onwards
  60  */
  61 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  62 {
  63         struct amdgpu_device *adev = psp->adev;
  64
  65         psp->pmfw_centralized_cstate_management = false;
  66
  67         if (amdgpu_sriov_vf(adev))
  68                 return;
  69
  70         if (adev->flags & AMD_IS_APU)
  71                 return;
  72
  73         if ((adev->asic_type == CHIP_ARCTURUS) ||
  74             (adev->asic_type >= CHIP_NAVI12))
  75                 psp->pmfw_centralized_cstate_management = true;
  76 }
  77
  78 static int psp_early_init(void *handle)
  79 {
  80         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  81         struct psp_context *psp = &adev->psp;
  82
  83         switch (adev->asic_type) {
  84         case CHIP_VEGA10:
  85         case CHIP_VEGA12:
  86                 psp_v3_1_set_psp_funcs(psp);
  87                 psp->autoload_supported = false;
  88                 break;
  89         case CHIP_RAVEN:
  90                 psp_v10_0_set_psp_funcs(psp);
  91                 psp->autoload_supported = false;
  92                 break;
  93         case CHIP_VEGA20:
  94         case CHIP_ARCTURUS:
  95                 psp_v11_0_set_psp_funcs(psp);
  96                 psp->autoload_supported = false;
  97                 break;
  98         case CHIP_NAVI10:
  99         case CHIP_NAVI14:
 100         case CHIP_NAVI12:
 101         case CHIP_SIENNA_CICHLID:
 102                 psp_v11_0_set_psp_funcs(psp);
 103                 psp->autoload_supported = true;
 104                 break;
 105         case CHIP_RENOIR:
 106                 psp_v12_0_set_psp_funcs(psp);
 107                 break;
 108         default:
 109                 return -EINVAL;
 110         }
 111
 112         psp->adev = adev;
 113
 114         psp_check_pmfw_centralized_cstate_management(psp);
 115
 116         return 0;
 117 }
 118
 119 static void psp_memory_training_fini(struct psp_context *psp)
 120 {
 121         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 122
 123         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 124         kfree(ctx->sys_cache);
 125         ctx->sys_cache = NULL;
 126 }
 127
 128 static int psp_memory_training_init(struct psp_context *psp)
 129 {
 130         int ret;
 131         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 132
 133         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 134                 DRM_DEBUG("memory training is not supported!\n");
 135                 return 0;
 136         }
 137
 138         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 139         if (ctx->sys_cache == NULL) {
 140                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 141                 ret = -ENOMEM;
 142                 goto Err_out;
 143         }
 144
 145         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 146                   ctx->train_data_size,
 147                   ctx->p2c_train_data_offset,
 148                   ctx->c2p_train_data_offset);
 149         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 150         return 0;
 151
 152 Err_out:
 153         psp_memory_training_fini(psp);
 154         return ret;
 155 }
 156
 157 static int psp_sw_init(void *handle)
 158 {
 159         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 160         struct psp_context *psp = &adev->psp;
 161         int ret;
 162
 163         ret = psp_init_microcode(psp);
 164         if (ret) {
 165                 DRM_ERROR("Failed to load psp firmware!\n");
 166                 return ret;
 167         }
 168
 169         ret = psp_memory_training_init(psp);
 170         if (ret) {
 171                 DRM_ERROR("Failed to initialize memory training!\n");
 172                 return ret;
 173         }
 174         ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 175         if (ret) {
 176                 DRM_ERROR("Failed to process memory training!\n");
 177                 return ret;
 178         }
 179
 180         if (adev->asic_type == CHIP_NAVI10) {
 181                 ret= psp_sysfs_init(adev);
 182                 if (ret) {
 183                         return ret;
 184                 }
 185         }
 186
 187         return 0;
 188 }
 189
 190 static int psp_sw_fini(void *handle)
 191 {
 192         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 193
 194         psp_memory_training_fini(&adev->psp);
 195         release_firmware(adev->psp.sos_fw);
 196         adev->psp.sos_fw = NULL;
 197         release_firmware(adev->psp.asd_fw);
 198         adev->psp.asd_fw = NULL;
 199         release_firmware(adev->psp.ta_fw);
 200         adev->psp.ta_fw = NULL;
 201
 202         if (adev->asic_type == CHIP_NAVI10)
 203                 psp_sysfs_fini(adev);
 204
 205         return 0;
 206 }
 207
 208 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 209                  uint32_t reg_val, uint32_t mask, bool check_changed)
 210 {
 211         uint32_t val;
 212         int i;
 213         struct amdgpu_device *adev = psp->adev;
 214
 215         for (i = 0; i < adev->usec_timeout; i++) {
 216                 val = RREG32(reg_index);
 217                 if (check_changed) {
 218                         if (val != reg_val)
 219                                 return 0;
 220                 } else {
 221                         if ((val & mask) == reg_val)
 222                                 return 0;
 223                 }
 224                 udelay(1);
 225         }
 226
 227         return -ETIME;
 228 }
 229
 230 static int
 231 psp_cmd_submit_buf(struct psp_context *psp,
 232                    struct amdgpu_firmware_info *ucode,
 233                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 234 {
 235         int ret;
 236         int index;
 237         int timeout = 2000;
 238         bool ras_intr = false;
 239         bool skip_unsupport = false;
 240
 241         mutex_lock(&psp->mutex);
 242
 243         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 244
 245         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 246
 247         index = atomic_inc_return(&psp->fence_value);
 248         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 249         if (ret) {
 250                 atomic_dec(&psp->fence_value);
 251                 mutex_unlock(&psp->mutex);
 252                 return ret;
 253         }
 254
 255         amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 256         while (*((unsigned int *)psp->fence_buf) != index) {
 257                 if (--timeout == 0)
 258                         break;
 259                 /*
 260                  * Shouldn't wait for timeout when err_event_athub occurs,
 261                  * because gpu reset thread triggered and lock resource should
 262                  * be released for psp resume sequence.
 263                  */
 264                 ras_intr = amdgpu_ras_intr_triggered();
 265                 if (ras_intr)
 266                         break;
 267                 msleep(1);
 268                 amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 269         }
 270
 271         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 272         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 273                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 274
 275         /* In some cases, psp response status is not 0 even there is no
 276          * problem while the command is submitted. Some version of PSP FW
 277          * doesn't write 0 to that field.
 278          * So here we would like to only print a warning instead of an error
 279          * during psp initialization to avoid breaking hw_init and it doesn't
 280          * return -EINVAL.
 281          */
 282         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 283                 if (ucode)
 284                         DRM_WARN("failed to load ucode id (%d) ",
 285                                   ucode->ucode_id);
 286                 DRM_WARN("psp command (0x%X) failed and response status is (0x%X)\n",
 287                          psp->cmd_buf_mem->cmd_id,
 288                          psp->cmd_buf_mem->resp.status);
 289                 if (!timeout) {
 290                         mutex_unlock(&psp->mutex);
 291                         return -EINVAL;
 292                 }
 293         }
 294
 295         /* get xGMI session id from response buffer */
 296         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 297
 298         if (ucode) {
 299                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 300                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 301         }
 302         mutex_unlock(&psp->mutex);
 303
 304         return ret;
 305 }
 306
 307 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 308                                  struct psp_gfx_cmd_resp *cmd,
 309                                  uint64_t tmr_mc, uint32_t size)
 310 {
 311         if (amdgpu_sriov_vf(psp->adev))
 312                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 313         else
 314                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 315         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 316         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 317         cmd->cmd.cmd_setup_tmr.buf_size = size;
 318 }
 319
 320 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 321                                       uint64_t pri_buf_mc, uint32_t size)
 322 {
 323         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 324         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 325         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 326         cmd->cmd.cmd_load_toc.toc_size = size;
 327 }
 328
 329 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 330 static int psp_load_toc(struct psp_context *psp,
 331                         uint32_t *tmr_size)
 332 {
 333         int ret;
 334         struct psp_gfx_cmd_resp *cmd;
 335
 336         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 337         if (!cmd)
 338                 return -ENOMEM;
 339         /* Copy toc to psp firmware private buffer */
 340         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 341         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 342
 343         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 344
 345         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 346                                  psp->fence_buf_mc_addr);
 347         if (!ret)
 348                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 349         kfree(cmd);
 350         return ret;
 351 }
 352
 353 /* Set up Trusted Memory Region */
 354 static int psp_tmr_init(struct psp_context *psp)
 355 {
 356         int ret;
 357         int tmr_size;
 358         void *tmr_buf;
 359         void **pptr;
 360
 361         /*
 362          * According to HW engineer, they prefer the TMR address be "naturally
 363          * aligned" , e.g. the start address be an integer divide of TMR size.
 364          *
 365          * Note: this memory need be reserved till the driver
 366          * uninitializes.
 367          */
 368         tmr_size = PSP_TMR_SIZE;
 369
 370         /* For ASICs support RLC autoload, psp will parse the toc
 371          * and calculate the total size of TMR needed */
 372         if (!amdgpu_sriov_vf(psp->adev) &&
 373             psp->toc_start_addr &&
 374             psp->toc_bin_size &&
 375             psp->fw_pri_buf) {
 376                 ret = psp_load_toc(psp, &tmr_size);
 377                 if (ret) {
 378                         DRM_ERROR("Failed to load toc\n");
 379                         return ret;
 380                 }
 381         }
 382
 383         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 384         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 385                                       AMDGPU_GEM_DOMAIN_VRAM,
 386                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 387
 388         return ret;
 389 }
 390
 391 static int psp_clear_vf_fw(struct psp_context *psp)
 392 {
 393         int ret;
 394         struct psp_gfx_cmd_resp *cmd;
 395
 396         if (!amdgpu_sriov_vf(psp->adev) || psp->adev->asic_type != CHIP_NAVI12)
 397                 return 0;
 398
 399         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 400         if (!cmd)
 401                 return -ENOMEM;
 402
 403         cmd->cmd_id = GFX_CMD_ID_CLEAR_VF_FW;
 404
 405         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 406         kfree(cmd);
 407
 408         return ret;
 409 }
 410
 411 static int psp_tmr_load(struct psp_context *psp)
 412 {
 413         int ret;
 414         struct psp_gfx_cmd_resp *cmd;
 415
 416         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 417         if (!cmd)
 418                 return -ENOMEM;
 419
 420         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 421                              amdgpu_bo_size(psp->tmr_bo));
 422         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 423                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 424
 425         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 426                                  psp->fence_buf_mc_addr);
 427
 428         kfree(cmd);
 429
 430         return ret;
 431 }
 432
 433 static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 434                                 uint64_t asd_mc, uint32_t size)
 435 {
 436         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 437         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 438         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 439         cmd->cmd.cmd_load_ta.app_len = size;
 440
 441         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = 0;
 442         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = 0;
 443         cmd->cmd.cmd_load_ta.cmd_buf_len = 0;
 444 }
 445
 446 static int psp_asd_load(struct psp_context *psp)
 447 {
 448         int ret;
 449         struct psp_gfx_cmd_resp *cmd;
 450
 451         /* If PSP version doesn't match ASD version, asd loading will be failed.
 452          * add workaround to bypass it for sriov now.
 453          * TODO: add version check to make it common
 454          */
 455         if (amdgpu_sriov_vf(psp->adev) || (psp->adev->asic_type == CHIP_SIENNA_CICHLID))
 456                 return 0;
 457
 458         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 459         if (!cmd)
 460                 return -ENOMEM;
 461
 462         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 463         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 464
 465         psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 466                                   psp->asd_ucode_size);
 467
 468         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 469                                  psp->fence_buf_mc_addr);
 470         if (!ret) {
 471                 psp->asd_context.asd_initialized = true;
 472                 psp->asd_context.session_id = cmd->resp.session_id;
 473         }
 474
 475         kfree(cmd);
 476
 477         return ret;
 478 }
 479
 480 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 481                                        uint32_t session_id)
 482 {
 483         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 484         cmd->cmd.cmd_unload_ta.session_id = session_id;
 485 }
 486
 487 static int psp_asd_unload(struct psp_context *psp)
 488 {
 489         int ret;
 490         struct psp_gfx_cmd_resp *cmd;
 491
 492         if (amdgpu_sriov_vf(psp->adev))
 493                 return 0;
 494
 495         if (!psp->asd_context.asd_initialized)
 496                 return 0;
 497
 498         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 499         if (!cmd)
 500                 return -ENOMEM;
 501
 502         psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id);
 503
 504         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 505                                  psp->fence_buf_mc_addr);
 506         if (!ret)
 507                 psp->asd_context.asd_initialized = false;
 508
 509         kfree(cmd);
 510
 511         return ret;
 512 }
 513
 514 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 515                 uint32_t id, uint32_t value)
 516 {
 517         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 518         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 519         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 520 }
 521
 522 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 523                 uint32_t value)
 524 {
 525         struct psp_gfx_cmd_resp *cmd = NULL;
 526         int ret = 0;
 527
 528         if (reg >= PSP_REG_LAST)
 529                 return -EINVAL;
 530
 531         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 532         if (!cmd)
 533                 return -ENOMEM;
 534
 535         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 536         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 537
 538         kfree(cmd);
 539         return ret;
 540 }
 541
 542 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 543                                      uint64_t ta_bin_mc,
 544                                      uint32_t ta_bin_size,
 545                                      uint64_t ta_shared_mc,
 546                                      uint32_t ta_shared_size)
 547 {
 548         cmd->cmd_id                             = GFX_CMD_ID_LOAD_TA;
 549         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
 550         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
 551         cmd->cmd.cmd_load_ta.app_len            = ta_bin_size;
 552
 553         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ta_shared_mc);
 554         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ta_shared_mc);
 555         cmd->cmd.cmd_load_ta.cmd_buf_len         = ta_shared_size;
 556 }
 557
 558 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 559 {
 560         int ret;
 561
 562         /*
 563          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 564          * physical) for xgmi ta <-> Driver
 565          */
 566         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 567                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 568                                       &psp->xgmi_context.xgmi_shared_bo,
 569                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 570                                       &psp->xgmi_context.xgmi_shared_buf);
 571
 572         return ret;
 573 }
 574
 575 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 576                                        uint32_t ta_cmd_id,
 577                                        uint32_t session_id)
 578 {
 579         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
 580         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
 581         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
 582 }
 583
 584 static int psp_ta_invoke(struct psp_context *psp,
 585                   uint32_t ta_cmd_id,
 586                   uint32_t session_id)
 587 {
 588         int ret;
 589         struct psp_gfx_cmd_resp *cmd;
 590
 591         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 592         if (!cmd)
 593                 return -ENOMEM;
 594
 595         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id);
 596
 597         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 598                                  psp->fence_buf_mc_addr);
 599
 600         kfree(cmd);
 601
 602         return ret;
 603 }
 604
 605 static int psp_xgmi_load(struct psp_context *psp)
 606 {
 607         int ret;
 608         struct psp_gfx_cmd_resp *cmd;
 609
 610         /*
 611          * TODO: bypass the loading in sriov for now
 612          */
 613
 614         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 615         if (!cmd)
 616                 return -ENOMEM;
 617
 618         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 619         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 620
 621         psp_prep_ta_load_cmd_buf(cmd,
 622                                  psp->fw_pri_mc_addr,
 623                                  psp->ta_xgmi_ucode_size,
 624                                  psp->xgmi_context.xgmi_shared_mc_addr,
 625                                  PSP_XGMI_SHARED_MEM_SIZE);
 626
 627         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 628                                  psp->fence_buf_mc_addr);
 629
 630         if (!ret) {
 631                 psp->xgmi_context.initialized = 1;
 632                 psp->xgmi_context.session_id = cmd->resp.session_id;
 633         }
 634
 635         kfree(cmd);
 636
 637         return ret;
 638 }
 639
 640 static int psp_xgmi_unload(struct psp_context *psp)
 641 {
 642         int ret;
 643         struct psp_gfx_cmd_resp *cmd;
 644         struct amdgpu_device *adev = psp->adev;
 645
 646         /* XGMI TA unload currently is not supported on Arcturus */
 647         if (adev->asic_type == CHIP_ARCTURUS)
 648                 return 0;
 649
 650         /*
 651          * TODO: bypass the unloading in sriov for now
 652          */
 653
 654         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 655         if (!cmd)
 656                 return -ENOMEM;
 657
 658         psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 659
 660         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 661                                  psp->fence_buf_mc_addr);
 662
 663         kfree(cmd);
 664
 665         return ret;
 666 }
 667
 668 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 669 {
 670         return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id);
 671 }
 672
 673 int psp_xgmi_terminate(struct psp_context *psp)
 674 {
 675         int ret;
 676
 677         if (!psp->xgmi_context.initialized)
 678                 return 0;
 679
 680         ret = psp_xgmi_unload(psp);
 681         if (ret)
 682                 return ret;
 683
 684         psp->xgmi_context.initialized = 0;
 685
 686         /* free xgmi shared memory */
 687         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 688                         &psp->xgmi_context.xgmi_shared_mc_addr,
 689                         &psp->xgmi_context.xgmi_shared_buf);
 690
 691         return 0;
 692 }
 693
 694 int psp_xgmi_initialize(struct psp_context *psp)
 695 {
 696         struct ta_xgmi_shared_memory *xgmi_cmd;
 697         int ret;
 698
 699         if (!psp->adev->psp.ta_fw ||
 700             !psp->adev->psp.ta_xgmi_ucode_size ||
 701             !psp->adev->psp.ta_xgmi_start_addr)
 702                 return -ENOENT;
 703
 704         if (!psp->xgmi_context.initialized) {
 705                 ret = psp_xgmi_init_shared_buf(psp);
 706                 if (ret)
 707                         return ret;
 708         }
 709
 710         /* Load XGMI TA */
 711         ret = psp_xgmi_load(psp);
 712         if (ret)
 713                 return ret;
 714
 715         /* Initialize XGMI session */
 716         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 717         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 718         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 719
 720         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 721
 722         return ret;
 723 }
 724
 725 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
 726 {
 727         struct ta_xgmi_shared_memory *xgmi_cmd;
 728         int ret;
 729
 730         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 731         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 732
 733         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
 734
 735         /* Invoke xgmi ta to get hive id */
 736         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 737         if (ret)
 738                 return ret;
 739
 740         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
 741
 742         return 0;
 743 }
 744
 745 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
 746 {
 747         struct ta_xgmi_shared_memory *xgmi_cmd;
 748         int ret;
 749
 750         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 751         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 752
 753         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
 754
 755         /* Invoke xgmi ta to get the node id */
 756         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 757         if (ret)
 758                 return ret;
 759
 760         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
 761
 762         return 0;
 763 }
 764
 765 int psp_xgmi_get_topology_info(struct psp_context *psp,
 766                                int number_devices,
 767                                struct psp_xgmi_topology_info *topology)
 768 {
 769         struct ta_xgmi_shared_memory *xgmi_cmd;
 770         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 771         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
 772         int i;
 773         int ret;
 774
 775         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 776                 return -EINVAL;
 777
 778         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 779         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 780
 781         /* Fill in the shared memory with topology information as input */
 782         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 783         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
 784         topology_info_input->num_nodes = number_devices;
 785
 786         for (i = 0; i < topology_info_input->num_nodes; i++) {
 787                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 788                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 789                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
 790                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 791         }
 792
 793         /* Invoke xgmi ta to get the topology information */
 794         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
 795         if (ret)
 796                 return ret;
 797
 798         /* Read the output topology information from the shared memory */
 799         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
 800         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
 801         for (i = 0; i < topology->num_nodes; i++) {
 802                 topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
 803                 topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
 804                 topology->nodes[i].is_sharing_enabled = topology_info_output->nodes[i].is_sharing_enabled;
 805                 topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine;
 806         }
 807
 808         return 0;
 809 }
 810
 811 int psp_xgmi_set_topology_info(struct psp_context *psp,
 812                                int number_devices,
 813                                struct psp_xgmi_topology_info *topology)
 814 {
 815         struct ta_xgmi_shared_memory *xgmi_cmd;
 816         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
 817         int i;
 818
 819         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
 820                 return -EINVAL;
 821
 822         xgmi_cmd = (struct ta_xgmi_shared_memory*)psp->xgmi_context.xgmi_shared_buf;
 823         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 824
 825         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
 826         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
 827         topology_info_input->num_nodes = number_devices;
 828
 829         for (i = 0; i < topology_info_input->num_nodes; i++) {
 830                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 831                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
 832                 topology_info_input->nodes[i].is_sharing_enabled = 1;
 833                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 834         }
 835
 836         /* Invoke xgmi ta to set topology information */
 837         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
 838 }
 839
 840 // ras begin
 841 static int psp_ras_init_shared_buf(struct psp_context *psp)
 842 {
 843         int ret;
 844
 845         /*
 846          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 847          * physical) for ras ta <-> Driver
 848          */
 849         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 850                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 851                         &psp->ras.ras_shared_bo,
 852                         &psp->ras.ras_shared_mc_addr,
 853                         &psp->ras.ras_shared_buf);
 854
 855         return ret;
 856 }
 857
 858 static int psp_ras_load(struct psp_context *psp)
 859 {
 860         int ret;
 861         struct psp_gfx_cmd_resp *cmd;
 862
 863         /*
 864          * TODO: bypass the loading in sriov for now
 865          */
 866         if (amdgpu_sriov_vf(psp->adev))
 867                 return 0;
 868
 869         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 870         if (!cmd)
 871                 return -ENOMEM;
 872
 873         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 874         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 875
 876         psp_prep_ta_load_cmd_buf(cmd,
 877                                  psp->fw_pri_mc_addr,
 878                                  psp->ta_ras_ucode_size,
 879                                  psp->ras.ras_shared_mc_addr,
 880                                  PSP_RAS_SHARED_MEM_SIZE);
 881
 882         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 883                         psp->fence_buf_mc_addr);
 884
 885         if (!ret) {
 886                 psp->ras.ras_initialized = true;
 887                 psp->ras.session_id = cmd->resp.session_id;
 888         }
 889
 890         kfree(cmd);
 891
 892         return ret;
 893 }
 894
 895 static int psp_ras_unload(struct psp_context *psp)
 896 {
 897         int ret;
 898         struct psp_gfx_cmd_resp *cmd;
 899
 900         /*
 901          * TODO: bypass the unloading in sriov for now
 902          */
 903         if (amdgpu_sriov_vf(psp->adev))
 904                 return 0;
 905
 906         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 907         if (!cmd)
 908                 return -ENOMEM;
 909
 910         psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 911
 912         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 913                         psp->fence_buf_mc_addr);
 914
 915         kfree(cmd);
 916
 917         return ret;
 918 }
 919
 920 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 921 {
 922         struct ta_ras_shared_memory *ras_cmd;
 923         int ret;
 924
 925         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 926
 927         /*
 928          * TODO: bypass the loading in sriov for now
 929          */
 930         if (amdgpu_sriov_vf(psp->adev))
 931                 return 0;
 932
 933         ret = psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id);
 934
 935         if (amdgpu_ras_intr_triggered())
 936                 return ret;
 937
 938         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER)
 939         {
 940                 DRM_WARN("RAS: Unsupported Interface");
 941                 return -EINVAL;
 942         }
 943
 944         if (!ret) {
 945                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
 946                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
 947
 948                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
 949                 }
 950                 else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
 951                         dev_warn(psp->adev->dev,
 952                                  "RAS internal register access blocked\n");
 953         }
 954
 955         return ret;
 956 }
 957
 958 int psp_ras_enable_features(struct psp_context *psp,
 959                 union ta_ras_cmd_input *info, bool enable)
 960 {
 961         struct ta_ras_shared_memory *ras_cmd;
 962         int ret;
 963
 964         if (!psp->ras.ras_initialized)
 965                 return -EINVAL;
 966
 967         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 968         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 969
 970         if (enable)
 971                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 972         else
 973                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 974
 975         ras_cmd->ras_in_message = *info;
 976
 977         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 978         if (ret)
 979                 return -EINVAL;
 980
 981         return ras_cmd->ras_status;
 982 }
 983
 984 static int psp_ras_terminate(struct psp_context *psp)
 985 {
 986         int ret;
 987
 988         /*
 989          * TODO: bypass the terminate in sriov for now
 990          */
 991         if (amdgpu_sriov_vf(psp->adev))
 992                 return 0;
 993
 994         if (!psp->ras.ras_initialized)
 995                 return 0;
 996
 997         ret = psp_ras_unload(psp);
 998         if (ret)
 999                 return ret;
1000
1001         psp->ras.ras_initialized = false;
1002
1003         /* free ras shared memory */
1004         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
1005                         &psp->ras.ras_shared_mc_addr,
1006                         &psp->ras.ras_shared_buf);
1007
1008         return 0;
1009 }
1010
1011 static int psp_ras_initialize(struct psp_context *psp)
1012 {
1013         int ret;
1014
1015         /*
1016          * TODO: bypass the initialize in sriov for now
1017          */
1018         if (amdgpu_sriov_vf(psp->adev))
1019                 return 0;
1020
1021         if (!psp->adev->psp.ta_ras_ucode_size ||
1022             !psp->adev->psp.ta_ras_start_addr) {
1023                 dev_info(psp->adev->dev, "RAS: optional ras ta ucode is not available\n");
1024                 return 0;
1025         }
1026
1027         if (!psp->ras.ras_initialized) {
1028                 ret = psp_ras_init_shared_buf(psp);
1029                 if (ret)
1030                         return ret;
1031         }
1032
1033         ret = psp_ras_load(psp);
1034         if (ret)
1035                 return ret;
1036
1037         return 0;
1038 }
1039
1040 int psp_ras_trigger_error(struct psp_context *psp,
1041                           struct ta_ras_trigger_error_input *info)
1042 {
1043         struct ta_ras_shared_memory *ras_cmd;
1044         int ret;
1045
1046         if (!psp->ras.ras_initialized)
1047                 return -EINVAL;
1048
1049         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1050         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1051
1052         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1053         ras_cmd->ras_in_message.trigger_error = *info;
1054
1055         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1056         if (ret)
1057                 return -EINVAL;
1058
1059         /* If err_event_athub occurs error inject was successful, however
1060            return status from TA is no long reliable */
1061         if (amdgpu_ras_intr_triggered())
1062                 return 0;
1063
1064         return ras_cmd->ras_status;
1065 }
1066 // ras end
1067
1068 // HDCP start
1069 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
1070 {
1071         int ret;
1072
1073         /*
1074          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1075          * physical) for hdcp ta <-> Driver
1076          */
1077         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
1078                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1079                                       &psp->hdcp_context.hdcp_shared_bo,
1080                                       &psp->hdcp_context.hdcp_shared_mc_addr,
1081                                       &psp->hdcp_context.hdcp_shared_buf);
1082
1083         return ret;
1084 }
1085
1086 static int psp_hdcp_load(struct psp_context *psp)
1087 {
1088         int ret;
1089         struct psp_gfx_cmd_resp *cmd;
1090
1091         /*
1092          * TODO: bypass the loading in sriov for now
1093          */
1094         if (amdgpu_sriov_vf(psp->adev))
1095                 return 0;
1096
1097         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1098         if (!cmd)
1099                 return -ENOMEM;
1100
1101         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1102         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
1103                psp->ta_hdcp_ucode_size);
1104
1105         psp_prep_ta_load_cmd_buf(cmd,
1106                                  psp->fw_pri_mc_addr,
1107                                  psp->ta_hdcp_ucode_size,
1108                                  psp->hdcp_context.hdcp_shared_mc_addr,
1109                                  PSP_HDCP_SHARED_MEM_SIZE);
1110
1111         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1112
1113         if (!ret) {
1114                 psp->hdcp_context.hdcp_initialized = true;
1115                 psp->hdcp_context.session_id = cmd->resp.session_id;
1116                 mutex_init(&psp->hdcp_context.mutex);
1117         }
1118
1119         kfree(cmd);
1120
1121         return ret;
1122 }
1123 static int psp_hdcp_initialize(struct psp_context *psp)
1124 {
1125         int ret;
1126
1127         /*
1128          * TODO: bypass the initialize in sriov for now
1129          */
1130         if (amdgpu_sriov_vf(psp->adev))
1131                 return 0;
1132
1133         if (!psp->adev->psp.ta_hdcp_ucode_size ||
1134             !psp->adev->psp.ta_hdcp_start_addr) {
1135                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1136                 return 0;
1137         }
1138
1139         if (!psp->hdcp_context.hdcp_initialized) {
1140                 ret = psp_hdcp_init_shared_buf(psp);
1141                 if (ret)
1142                         return ret;
1143         }
1144
1145         ret = psp_hdcp_load(psp);
1146         if (ret)
1147                 return ret;
1148
1149         return 0;
1150 }
1151
1152 static int psp_hdcp_unload(struct psp_context *psp)
1153 {
1154         int ret;
1155         struct psp_gfx_cmd_resp *cmd;
1156
1157         /*
1158          * TODO: bypass the unloading in sriov for now
1159          */
1160         if (amdgpu_sriov_vf(psp->adev))
1161                 return 0;
1162
1163         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1164         if (!cmd)
1165                 return -ENOMEM;
1166
1167         psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
1168
1169         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1170
1171         kfree(cmd);
1172
1173         return ret;
1174 }
1175
1176 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1177 {
1178         /*
1179          * TODO: bypass the loading in sriov for now
1180          */
1181         if (amdgpu_sriov_vf(psp->adev))
1182                 return 0;
1183
1184         return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id);
1185 }
1186
1187 static int psp_hdcp_terminate(struct psp_context *psp)
1188 {
1189         int ret;
1190
1191         /*
1192          * TODO: bypass the terminate in sriov for now
1193          */
1194         if (amdgpu_sriov_vf(psp->adev))
1195                 return 0;
1196
1197         if (!psp->hdcp_context.hdcp_initialized)
1198                 return 0;
1199
1200         ret = psp_hdcp_unload(psp);
1201         if (ret)
1202                 return ret;
1203
1204         psp->hdcp_context.hdcp_initialized = false;
1205
1206         /* free hdcp shared memory */
1207         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
1208                               &psp->hdcp_context.hdcp_shared_mc_addr,
1209                               &psp->hdcp_context.hdcp_shared_buf);
1210
1211         return 0;
1212 }
1213 // HDCP end
1214
1215 // DTM start
1216 static int psp_dtm_init_shared_buf(struct psp_context *psp)
1217 {
1218         int ret;
1219
1220         /*
1221          * Allocate 16k memory aligned to 4k from Frame Buffer (local
1222          * physical) for dtm ta <-> Driver
1223          */
1224         ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE,
1225                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1226                                       &psp->dtm_context.dtm_shared_bo,
1227                                       &psp->dtm_context.dtm_shared_mc_addr,
1228                                       &psp->dtm_context.dtm_shared_buf);
1229
1230         return ret;
1231 }
1232
1233 static int psp_dtm_load(struct psp_context *psp)
1234 {
1235         int ret;
1236         struct psp_gfx_cmd_resp *cmd;
1237
1238         /*
1239          * TODO: bypass the loading in sriov for now
1240          */
1241         if (amdgpu_sriov_vf(psp->adev))
1242                 return 0;
1243
1244         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1245         if (!cmd)
1246                 return -ENOMEM;
1247
1248         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1249         memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size);
1250
1251         psp_prep_ta_load_cmd_buf(cmd,
1252                                  psp->fw_pri_mc_addr,
1253                                  psp->ta_dtm_ucode_size,
1254                                  psp->dtm_context.dtm_shared_mc_addr,
1255                                  PSP_DTM_SHARED_MEM_SIZE);
1256
1257         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1258
1259         if (!ret) {
1260                 psp->dtm_context.dtm_initialized = true;
1261                 psp->dtm_context.session_id = cmd->resp.session_id;
1262                 mutex_init(&psp->dtm_context.mutex);
1263         }
1264
1265         kfree(cmd);
1266
1267         return ret;
1268 }
1269
1270 static int psp_dtm_initialize(struct psp_context *psp)
1271 {
1272         int ret;
1273
1274         /*
1275          * TODO: bypass the initialize in sriov for now
1276          */
1277         if (amdgpu_sriov_vf(psp->adev))
1278                 return 0;
1279
1280         if (!psp->adev->psp.ta_dtm_ucode_size ||
1281             !psp->adev->psp.ta_dtm_start_addr) {
1282                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1283                 return 0;
1284         }
1285
1286         if (!psp->dtm_context.dtm_initialized) {
1287                 ret = psp_dtm_init_shared_buf(psp);
1288                 if (ret)
1289                         return ret;
1290         }
1291
1292         ret = psp_dtm_load(psp);
1293         if (ret)
1294                 return ret;
1295
1296         return 0;
1297 }
1298
1299 static int psp_dtm_unload(struct psp_context *psp)
1300 {
1301         int ret;
1302         struct psp_gfx_cmd_resp *cmd;
1303
1304         /*
1305          * TODO: bypass the unloading in sriov for now
1306          */
1307         if (amdgpu_sriov_vf(psp->adev))
1308                 return 0;
1309
1310         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1311         if (!cmd)
1312                 return -ENOMEM;
1313
1314         psp_prep_ta_unload_cmd_buf(cmd, psp->dtm_context.session_id);
1315
1316         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1317
1318         kfree(cmd);
1319
1320         return ret;
1321 }
1322
1323 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1324 {
1325         /*
1326          * TODO: bypass the loading in sriov for now
1327          */
1328         if (amdgpu_sriov_vf(psp->adev))
1329                 return 0;
1330
1331         return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id);
1332 }
1333
1334 static int psp_dtm_terminate(struct psp_context *psp)
1335 {
1336         int ret;
1337
1338         /*
1339          * TODO: bypass the terminate in sriov for now
1340          */
1341         if (amdgpu_sriov_vf(psp->adev))
1342                 return 0;
1343
1344         if (!psp->dtm_context.dtm_initialized)
1345                 return 0;
1346
1347         ret = psp_dtm_unload(psp);
1348         if (ret)
1349                 return ret;
1350
1351         psp->dtm_context.dtm_initialized = false;
1352
1353         /* free hdcp shared memory */
1354         amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo,
1355                               &psp->dtm_context.dtm_shared_mc_addr,
1356                               &psp->dtm_context.dtm_shared_buf);
1357
1358         return 0;
1359 }
1360 // DTM end
1361
1362 static int psp_hw_start(struct psp_context *psp)
1363 {
1364         struct amdgpu_device *adev = psp->adev;
1365         int ret;
1366
1367         if (!amdgpu_sriov_vf(adev)) {
1368                 if (psp->kdb_bin_size &&
1369                     (psp->funcs->bootloader_load_kdb != NULL)) {
1370                         ret = psp_bootloader_load_kdb(psp);
1371                         if (ret) {
1372                                 DRM_ERROR("PSP load kdb failed!\n");
1373                                 return ret;
1374                         }
1375                 }
1376
1377                 if (psp->spl_bin_size) {
1378                         ret = psp_bootloader_load_spl(psp);
1379                         if (ret) {
1380                                 DRM_ERROR("PSP load spl failed!\n");
1381                                 return ret;
1382                         }
1383                 }
1384
1385                 ret = psp_bootloader_load_sysdrv(psp);
1386                 if (ret) {
1387                         DRM_ERROR("PSP load sysdrv failed!\n");
1388                         return ret;
1389                 }
1390
1391                 ret = psp_bootloader_load_sos(psp);
1392                 if (ret) {
1393                         DRM_ERROR("PSP load sos failed!\n");
1394                         return ret;
1395                 }
1396         }
1397
1398         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
1399         if (ret) {
1400                 DRM_ERROR("PSP create ring failed!\n");
1401                 return ret;
1402         }
1403
1404         ret = psp_clear_vf_fw(psp);
1405         if (ret) {
1406                 DRM_ERROR("PSP clear vf fw!\n");
1407                 return ret;
1408         }
1409
1410         ret = psp_tmr_init(psp);
1411         if (ret) {
1412                 DRM_ERROR("PSP tmr init failed!\n");
1413                 return ret;
1414         }
1415
1416         /*
1417          * For ASICs with DF Cstate management centralized
1418          * to PMFW, TMR setup should be performed after PMFW
1419          * loaded and before other non-psp firmware loaded.
1420          */
1421         if (psp->pmfw_centralized_cstate_management) {
1422                 ret = psp_load_smu_fw(psp);
1423                 if (ret)
1424                         return ret;
1425         }
1426
1427         ret = psp_tmr_load(psp);
1428         if (ret) {
1429                 DRM_ERROR("PSP load tmr failed!\n");
1430                 return ret;
1431         }
1432
1433         return 0;
1434 }
1435
1436 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1437                            enum psp_gfx_fw_type *type)
1438 {
1439         switch (ucode->ucode_id) {
1440         case AMDGPU_UCODE_ID_SDMA0:
1441                 *type = GFX_FW_TYPE_SDMA0;
1442                 break;
1443         case AMDGPU_UCODE_ID_SDMA1:
1444                 *type = GFX_FW_TYPE_SDMA1;
1445                 break;
1446         case AMDGPU_UCODE_ID_SDMA2:
1447                 *type = GFX_FW_TYPE_SDMA2;
1448                 break;
1449         case AMDGPU_UCODE_ID_SDMA3:
1450                 *type = GFX_FW_TYPE_SDMA3;
1451                 break;
1452         case AMDGPU_UCODE_ID_SDMA4:
1453                 *type = GFX_FW_TYPE_SDMA4;
1454                 break;
1455         case AMDGPU_UCODE_ID_SDMA5:
1456                 *type = GFX_FW_TYPE_SDMA5;
1457                 break;
1458         case AMDGPU_UCODE_ID_SDMA6:
1459                 *type = GFX_FW_TYPE_SDMA6;
1460                 break;
1461         case AMDGPU_UCODE_ID_SDMA7:
1462                 *type = GFX_FW_TYPE_SDMA7;
1463                 break;
1464         case AMDGPU_UCODE_ID_CP_MES:
1465                 *type = GFX_FW_TYPE_CP_MES;
1466                 break;
1467         case AMDGPU_UCODE_ID_CP_MES_DATA:
1468                 *type = GFX_FW_TYPE_MES_STACK;
1469                 break;
1470         case AMDGPU_UCODE_ID_CP_CE:
1471                 *type = GFX_FW_TYPE_CP_CE;
1472                 break;
1473         case AMDGPU_UCODE_ID_CP_PFP:
1474                 *type = GFX_FW_TYPE_CP_PFP;
1475                 break;
1476         case AMDGPU_UCODE_ID_CP_ME:
1477                 *type = GFX_FW_TYPE_CP_ME;
1478                 break;
1479         case AMDGPU_UCODE_ID_CP_MEC1:
1480                 *type = GFX_FW_TYPE_CP_MEC;
1481                 break;
1482         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1483                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1484                 break;
1485         case AMDGPU_UCODE_ID_CP_MEC2:
1486                 *type = GFX_FW_TYPE_CP_MEC;
1487                 break;
1488         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1489                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1490                 break;
1491         case AMDGPU_UCODE_ID_RLC_G:
1492                 *type = GFX_FW_TYPE_RLC_G;
1493                 break;
1494         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1495                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1496                 break;
1497         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1498                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1499                 break;
1500         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1501                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1502                 break;
1503         case AMDGPU_UCODE_ID_SMC:
1504                 *type = GFX_FW_TYPE_SMU;
1505                 break;
1506         case AMDGPU_UCODE_ID_UVD:
1507                 *type = GFX_FW_TYPE_UVD;
1508                 break;
1509         case AMDGPU_UCODE_ID_UVD1:
1510                 *type = GFX_FW_TYPE_UVD1;
1511                 break;
1512         case AMDGPU_UCODE_ID_VCE:
1513                 *type = GFX_FW_TYPE_VCE;
1514                 break;
1515         case AMDGPU_UCODE_ID_VCN:
1516                 *type = GFX_FW_TYPE_VCN;
1517                 break;
1518         case AMDGPU_UCODE_ID_VCN1:
1519                 *type = GFX_FW_TYPE_VCN1;
1520                 break;
1521         case AMDGPU_UCODE_ID_DMCU_ERAM:
1522                 *type = GFX_FW_TYPE_DMCU_ERAM;
1523                 break;
1524         case AMDGPU_UCODE_ID_DMCU_INTV:
1525                 *type = GFX_FW_TYPE_DMCU_ISR;
1526                 break;
1527         case AMDGPU_UCODE_ID_VCN0_RAM:
1528                 *type = GFX_FW_TYPE_VCN0_RAM;
1529                 break;
1530         case AMDGPU_UCODE_ID_VCN1_RAM:
1531                 *type = GFX_FW_TYPE_VCN1_RAM;
1532                 break;
1533         case AMDGPU_UCODE_ID_DMCUB:
1534                 *type = GFX_FW_TYPE_DMUB;
1535                 break;
1536         case AMDGPU_UCODE_ID_MAXIMUM:
1537         default:
1538                 return -EINVAL;
1539         }
1540
1541         return 0;
1542 }
1543
1544 static void psp_print_fw_hdr(struct psp_context *psp,
1545                              struct amdgpu_firmware_info *ucode)
1546 {
1547         struct amdgpu_device *adev = psp->adev;
1548         struct common_firmware_header *hdr;
1549
1550         switch (ucode->ucode_id) {
1551         case AMDGPU_UCODE_ID_SDMA0:
1552         case AMDGPU_UCODE_ID_SDMA1:
1553         case AMDGPU_UCODE_ID_SDMA2:
1554         case AMDGPU_UCODE_ID_SDMA3:
1555         case AMDGPU_UCODE_ID_SDMA4:
1556         case AMDGPU_UCODE_ID_SDMA5:
1557         case AMDGPU_UCODE_ID_SDMA6:
1558         case AMDGPU_UCODE_ID_SDMA7:
1559                 hdr = (struct common_firmware_header *)
1560                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1561                 amdgpu_ucode_print_sdma_hdr(hdr);
1562                 break;
1563         case AMDGPU_UCODE_ID_CP_CE:
1564                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1565                 amdgpu_ucode_print_gfx_hdr(hdr);
1566                 break;
1567         case AMDGPU_UCODE_ID_CP_PFP:
1568                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1569                 amdgpu_ucode_print_gfx_hdr(hdr);
1570                 break;
1571         case AMDGPU_UCODE_ID_CP_ME:
1572                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1573                 amdgpu_ucode_print_gfx_hdr(hdr);
1574                 break;
1575         case AMDGPU_UCODE_ID_CP_MEC1:
1576                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1577                 amdgpu_ucode_print_gfx_hdr(hdr);
1578                 break;
1579         case AMDGPU_UCODE_ID_RLC_G:
1580                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1581                 amdgpu_ucode_print_rlc_hdr(hdr);
1582                 break;
1583         case AMDGPU_UCODE_ID_SMC:
1584                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1585                 amdgpu_ucode_print_smc_hdr(hdr);
1586                 break;
1587         default:
1588                 break;
1589         }
1590 }
1591
1592 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1593                                        struct psp_gfx_cmd_resp *cmd)
1594 {
1595         int ret;
1596         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1597
1598         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1599
1600         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1601         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1602         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1603         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1604
1605         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1606         if (ret)
1607                 DRM_ERROR("Unknown firmware type\n");
1608
1609         return ret;
1610 }
1611
1612 static int psp_execute_np_fw_load(struct psp_context *psp,
1613                                   struct amdgpu_firmware_info *ucode)
1614 {
1615         int ret = 0;
1616
1617         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1618         if (ret)
1619                 return ret;
1620
1621         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1622                                  psp->fence_buf_mc_addr);
1623
1624         return ret;
1625 }
1626
1627 static int psp_load_smu_fw(struct psp_context *psp)
1628 {
1629         int ret;
1630         struct amdgpu_device* adev = psp->adev;
1631         struct amdgpu_firmware_info *ucode =
1632                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1633         struct amdgpu_ras *ras = psp->ras.ras;
1634
1635         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
1636                 return 0;
1637
1638
1639         if (adev->in_gpu_reset && ras && ras->supported) {
1640                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
1641                 if (ret) {
1642                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
1643                 }
1644         }
1645
1646         ret = psp_execute_np_fw_load(psp, ucode);
1647
1648         if (ret)
1649                 DRM_ERROR("PSP load smu failed!\n");
1650
1651         return ret;
1652 }
1653
1654 static bool fw_load_skip_check(struct psp_context *psp,
1655                                struct amdgpu_firmware_info *ucode)
1656 {
1657         if (!ucode->fw)
1658                 return true;
1659
1660         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1661             (psp_smu_reload_quirk(psp) ||
1662              psp->autoload_supported ||
1663              psp->pmfw_centralized_cstate_management))
1664                 return true;
1665
1666         if (amdgpu_sriov_vf(psp->adev) &&
1667            (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1668             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1669             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1670             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1671             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1672             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1673             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1674             || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1675             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
1676             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
1677             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
1678             || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
1679             || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
1680                 /*skip ucode loading in SRIOV VF */
1681                 return true;
1682
1683         if (psp->autoload_supported &&
1684             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1685              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1686                 /* skip mec JT when autoload is enabled */
1687                 return true;
1688
1689         return false;
1690 }
1691
1692 static int psp_np_fw_load(struct psp_context *psp)
1693 {
1694         int i, ret;
1695         struct amdgpu_firmware_info *ucode;
1696         struct amdgpu_device* adev = psp->adev;
1697
1698         if (psp->autoload_supported &&
1699             !psp->pmfw_centralized_cstate_management) {
1700                 ret = psp_load_smu_fw(psp);
1701                 if (ret)
1702                         return ret;
1703         }
1704
1705         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1706                 ucode = &adev->firmware.ucode[i];
1707
1708                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1709                     !fw_load_skip_check(psp, ucode)) {
1710                         ret = psp_load_smu_fw(psp);
1711                         if (ret)
1712                                 return ret;
1713                         continue;
1714                 }
1715
1716                 if (fw_load_skip_check(psp, ucode))
1717                         continue;
1718
1719                 if (psp->autoload_supported &&
1720                     adev->asic_type == CHIP_SIENNA_CICHLID &&
1721                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
1722                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
1723                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
1724                         /* PSP only receive one SDMA fw for sienna_cichlid,
1725                          * as all four sdma fw are same */
1726                         continue;
1727
1728                 psp_print_fw_hdr(psp, ucode);
1729
1730                 ret = psp_execute_np_fw_load(psp, ucode);
1731                 if (ret)
1732                         return ret;
1733
1734                 /* Start rlc autoload after psp recieved all the gfx firmware */
1735                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
1736                     AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
1737                         ret = psp_rlc_autoload_start(psp);
1738                         if (ret) {
1739                                 DRM_ERROR("Failed to start rlc autoload\n");
1740                                 return ret;
1741                         }
1742                 }
1743         }
1744
1745         return 0;
1746 }
1747
1748 static int psp_load_fw(struct amdgpu_device *adev)
1749 {
1750         int ret;
1751         struct psp_context *psp = &adev->psp;
1752
1753         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1754                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1755                 goto skip_memalloc;
1756         }
1757
1758         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1759         if (!psp->cmd)
1760                 return -ENOMEM;
1761
1762         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1763                                         AMDGPU_GEM_DOMAIN_GTT,
1764                                         &psp->fw_pri_bo,
1765                                         &psp->fw_pri_mc_addr,
1766                                         &psp->fw_pri_buf);
1767         if (ret)
1768                 goto failed;
1769
1770         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1771                                         AMDGPU_GEM_DOMAIN_VRAM,
1772                                         &psp->fence_buf_bo,
1773                                         &psp->fence_buf_mc_addr,
1774                                         &psp->fence_buf);
1775         if (ret)
1776                 goto failed;
1777
1778         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1779                                       AMDGPU_GEM_DOMAIN_VRAM,
1780                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1781                                       (void **)&psp->cmd_buf_mem);
1782         if (ret)
1783                 goto failed;
1784
1785         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1786
1787         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1788         if (ret) {
1789                 DRM_ERROR("PSP ring init failed!\n");
1790                 goto failed;
1791         }
1792
1793 skip_memalloc:
1794         ret = psp_hw_start(psp);
1795         if (ret)
1796                 goto failed;
1797
1798         ret = psp_np_fw_load(psp);
1799         if (ret)
1800                 goto failed;
1801
1802         ret = psp_asd_load(psp);
1803         if (ret) {
1804                 DRM_ERROR("PSP load asd failed!\n");
1805                 return ret;
1806         }
1807
1808         if (psp->adev->psp.ta_fw) {
1809                 ret = psp_ras_initialize(psp);
1810                 if (ret)
1811                         dev_err(psp->adev->dev,
1812                                         "RAS: Failed to initialize RAS\n");
1813
1814                 ret = psp_hdcp_initialize(psp);
1815                 if (ret)
1816                         dev_err(psp->adev->dev,
1817                                 "HDCP: Failed to initialize HDCP\n");
1818
1819                 ret = psp_dtm_initialize(psp);
1820                 if (ret)
1821                         dev_err(psp->adev->dev,
1822                                 "DTM: Failed to initialize DTM\n");
1823         }
1824
1825         return 0;
1826
1827 failed:
1828         /*
1829          * all cleanup jobs (xgmi terminate, ras terminate,
1830          * ring destroy, cmd/fence/fw buffers destory,
1831          * psp->cmd destory) are delayed to psp_hw_fini
1832          */
1833         return ret;
1834 }
1835
1836 static int psp_hw_init(void *handle)
1837 {
1838         int ret;
1839         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1840
1841         mutex_lock(&adev->firmware.mutex);
1842         /*
1843          * This sequence is just used on hw_init only once, no need on
1844          * resume.
1845          */
1846         ret = amdgpu_ucode_init_bo(adev);
1847         if (ret)
1848                 goto failed;
1849
1850         ret = psp_load_fw(adev);
1851         if (ret) {
1852                 DRM_ERROR("PSP firmware loading failed\n");
1853                 goto failed;
1854         }
1855
1856         mutex_unlock(&adev->firmware.mutex);
1857         return 0;
1858
1859 failed:
1860         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1861         mutex_unlock(&adev->firmware.mutex);
1862         return -EINVAL;
1863 }
1864
1865 static int psp_hw_fini(void *handle)
1866 {
1867         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1868         struct psp_context *psp = &adev->psp;
1869         void *tmr_buf;
1870         void **pptr;
1871         int ret;
1872
1873         if (psp->adev->psp.ta_fw) {
1874                 psp_ras_terminate(psp);
1875                 psp_dtm_terminate(psp);
1876                 psp_hdcp_terminate(psp);
1877         }
1878
1879         psp_asd_unload(psp);
1880         ret = psp_clear_vf_fw(psp);
1881         if (ret) {
1882                 DRM_ERROR("PSP clear vf fw!\n");
1883                 return ret;
1884         }
1885
1886         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1887
1888         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1889         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1890         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1891                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1892         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1893                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1894         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1895                               (void **)&psp->cmd_buf_mem);
1896
1897         kfree(psp->cmd);
1898         psp->cmd = NULL;
1899
1900         return 0;
1901 }
1902
1903 static int psp_suspend(void *handle)
1904 {
1905         int ret;
1906         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1907         struct psp_context *psp = &adev->psp;
1908
1909         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1910             psp->xgmi_context.initialized == 1) {
1911                 ret = psp_xgmi_terminate(psp);
1912                 if (ret) {
1913                         DRM_ERROR("Failed to terminate xgmi ta\n");
1914                         return ret;
1915                 }
1916         }
1917
1918         if (psp->adev->psp.ta_fw) {
1919                 ret = psp_ras_terminate(psp);
1920                 if (ret) {
1921                         DRM_ERROR("Failed to terminate ras ta\n");
1922                         return ret;
1923                 }
1924                 ret = psp_hdcp_terminate(psp);
1925                 if (ret) {
1926                         DRM_ERROR("Failed to terminate hdcp ta\n");
1927                         return ret;
1928                 }
1929                 ret = psp_dtm_terminate(psp);
1930                 if (ret) {
1931                         DRM_ERROR("Failed to terminate dtm ta\n");
1932                         return ret;
1933                 }
1934         }
1935
1936         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1937         if (ret) {
1938                 DRM_ERROR("PSP ring stop failed\n");
1939                 return ret;
1940         }
1941
1942         return 0;
1943 }
1944
1945 static int psp_resume(void *handle)
1946 {
1947         int ret;
1948         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1949         struct psp_context *psp = &adev->psp;
1950
1951         DRM_INFO("PSP is resuming...\n");
1952
1953         ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
1954         if (ret) {
1955                 DRM_ERROR("Failed to process memory training!\n");
1956                 return ret;
1957         }
1958
1959         mutex_lock(&adev->firmware.mutex);
1960
1961         ret = psp_hw_start(psp);
1962         if (ret)
1963                 goto failed;
1964
1965         ret = psp_np_fw_load(psp);
1966         if (ret)
1967                 goto failed;
1968
1969         ret = psp_asd_load(psp);
1970         if (ret) {
1971                 DRM_ERROR("PSP load asd failed!\n");
1972                 goto failed;
1973         }
1974
1975         if (adev->gmc.xgmi.num_physical_nodes > 1) {
1976                 ret = psp_xgmi_initialize(psp);
1977                 /* Warning the XGMI seesion initialize failure
1978                  * Instead of stop driver initialization
1979                  */
1980                 if (ret)
1981                         dev_err(psp->adev->dev,
1982                                 "XGMI: Failed to initialize XGMI session\n");
1983         }
1984
1985         if (psp->adev->psp.ta_fw) {
1986                 ret = psp_ras_initialize(psp);
1987                 if (ret)
1988                         dev_err(psp->adev->dev,
1989                                         "RAS: Failed to initialize RAS\n");
1990
1991                 ret = psp_hdcp_initialize(psp);
1992                 if (ret)
1993                         dev_err(psp->adev->dev,
1994                                 "HDCP: Failed to initialize HDCP\n");
1995
1996                 ret = psp_dtm_initialize(psp);
1997                 if (ret)
1998                         dev_err(psp->adev->dev,
1999                                 "DTM: Failed to initialize DTM\n");
2000         }
2001
2002         mutex_unlock(&adev->firmware.mutex);
2003
2004         return 0;
2005
2006 failed:
2007         DRM_ERROR("PSP resume failed\n");
2008         mutex_unlock(&adev->firmware.mutex);
2009         return ret;
2010 }
2011
2012 int psp_gpu_reset(struct amdgpu_device *adev)
2013 {
2014         int ret;
2015
2016         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2017                 return 0;
2018
2019         mutex_lock(&adev->psp.mutex);
2020         ret = psp_mode1_reset(&adev->psp);
2021         mutex_unlock(&adev->psp.mutex);
2022
2023         return ret;
2024 }
2025
2026 int psp_rlc_autoload_start(struct psp_context *psp)
2027 {
2028         int ret;
2029         struct psp_gfx_cmd_resp *cmd;
2030
2031         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
2032         if (!cmd)
2033                 return -ENOMEM;
2034
2035         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2036
2037         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2038                                  psp->fence_buf_mc_addr);
2039         kfree(cmd);
2040         return ret;
2041 }
2042
2043 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2044                         uint64_t cmd_gpu_addr, int cmd_size)
2045 {
2046         struct amdgpu_firmware_info ucode = {0};
2047
2048         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2049                 AMDGPU_UCODE_ID_VCN0_RAM;
2050         ucode.mc_addr = cmd_gpu_addr;
2051         ucode.ucode_size = cmd_size;
2052
2053         return psp_execute_np_fw_load(&adev->psp, &ucode);
2054 }
2055
2056 int psp_ring_cmd_submit(struct psp_context *psp,
2057                         uint64_t cmd_buf_mc_addr,
2058                         uint64_t fence_mc_addr,
2059                         int index)
2060 {
2061         unsigned int psp_write_ptr_reg = 0;
2062         struct psp_gfx_rb_frame *write_frame;
2063         struct psp_ring *ring = &psp->km_ring;
2064         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2065         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2066                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2067         struct amdgpu_device *adev = psp->adev;
2068         uint32_t ring_size_dw = ring->ring_size / 4;
2069         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2070
2071         /* KM (GPCOM) prepare write pointer */
2072         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2073
2074         /* Update KM RB frame pointer to new frame */
2075         /* write_frame ptr increments by size of rb_frame in bytes */
2076         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2077         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2078                 write_frame = ring_buffer_start;
2079         else
2080                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2081         /* Check invalid write_frame ptr address */
2082         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2083                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2084                           ring_buffer_start, ring_buffer_end, write_frame);
2085                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2086                 return -EINVAL;
2087         }
2088
2089         /* Initialize KM RB frame */
2090         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2091
2092         /* Update KM RB frame */
2093         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2094         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2095         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2096         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2097         write_frame->fence_value = index;
2098         amdgpu_asic_flush_hdp(adev, NULL);
2099
2100         /* Update the write Pointer in DWORDs */
2101         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2102         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2103         return 0;
2104 }
2105
2106 int psp_init_asd_microcode(struct psp_context *psp,
2107                            const char *chip_name)
2108 {
2109         struct amdgpu_device *adev = psp->adev;
2110         char fw_name[30];
2111         const struct psp_firmware_header_v1_0 *asd_hdr;
2112         int err = 0;
2113
2114         if (!chip_name) {
2115                 dev_err(adev->dev, "invalid chip name for asd microcode\n");
2116                 return -EINVAL;
2117         }
2118
2119         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
2120         err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
2121         if (err)
2122                 goto out;
2123
2124         err = amdgpu_ucode_validate(adev->psp.asd_fw);
2125         if (err)
2126                 goto out;
2127
2128         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
2129         adev->psp.asd_fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
2130         adev->psp.asd_feature_version = le32_to_cpu(asd_hdr->ucode_feature_version);
2131         adev->psp.asd_ucode_size = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
2132         adev->psp.asd_start_addr = (uint8_t *)asd_hdr +
2133                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
2134         return 0;
2135 out:
2136         dev_err(adev->dev, "fail to initialize asd microcode\n");
2137         release_firmware(adev->psp.asd_fw);
2138         adev->psp.asd_fw = NULL;
2139         return err;
2140 }
2141
2142 int psp_init_sos_microcode(struct psp_context *psp,
2143                            const char *chip_name)
2144 {
2145         struct amdgpu_device *adev = psp->adev;
2146         char fw_name[30];
2147         const struct psp_firmware_header_v1_0 *sos_hdr;
2148         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
2149         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
2150         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
2151         int err = 0;
2152
2153         if (!chip_name) {
2154                 dev_err(adev->dev, "invalid chip name for sos microcode\n");
2155                 return -EINVAL;
2156         }
2157
2158         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
2159         err = request_firmware(&adev->psp.sos_fw, fw_name, adev->dev);
2160         if (err)
2161                 goto out;
2162
2163         err = amdgpu_ucode_validate(adev->psp.sos_fw);
2164         if (err)
2165                 goto out;
2166
2167         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
2168         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
2169
2170         switch (sos_hdr->header.header_version_major) {
2171         case 1:
2172                 adev->psp.sos_fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
2173                 adev->psp.sos_feature_version = le32_to_cpu(sos_hdr->ucode_feature_version);
2174                 adev->psp.sos_bin_size = le32_to_cpu(sos_hdr->sos_size_bytes);
2175                 adev->psp.sys_bin_size = le32_to_cpu(sos_hdr->sos_offset_bytes);
2176                 adev->psp.sys_start_addr = (uint8_t *)sos_hdr +
2177                                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
2178                 adev->psp.sos_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2179                                 le32_to_cpu(sos_hdr->sos_offset_bytes);
2180                 if (sos_hdr->header.header_version_minor == 1) {
2181                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
2182                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_1->toc_size_bytes);
2183                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2184                                         le32_to_cpu(sos_hdr_v1_1->toc_offset_bytes);
2185                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_1->kdb_size_bytes);
2186                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2187                                         le32_to_cpu(sos_hdr_v1_1->kdb_offset_bytes);
2188                 }
2189                 if (sos_hdr->header.header_version_minor == 2) {
2190                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
2191                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_2->kdb_size_bytes);
2192                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2193                                                     le32_to_cpu(sos_hdr_v1_2->kdb_offset_bytes);
2194                 }
2195                 if (sos_hdr->header.header_version_minor == 3) {
2196                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
2197                         adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.toc_size_bytes);
2198                         adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2199                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc_offset_bytes);
2200                         adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_size_bytes);
2201                         adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2202                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb_offset_bytes);
2203                         adev->psp.spl_bin_size = le32_to_cpu(sos_hdr_v1_3->spl_size_bytes);
2204                         adev->psp.spl_start_addr = (uint8_t *)adev->psp.sys_start_addr +
2205                                 le32_to_cpu(sos_hdr_v1_3->spl_offset_bytes);
2206                 }
2207                 break;
2208         default:
2209                 dev_err(adev->dev,
2210                         "unsupported psp sos firmware\n");
2211                 err = -EINVAL;
2212                 goto out;
2213         }
2214
2215         return 0;
2216 out:
2217         dev_err(adev->dev,
2218                 "failed to init sos firmware\n");
2219         release_firmware(adev->psp.sos_fw);
2220         adev->psp.sos_fw = NULL;
2221
2222         return err;
2223 }
2224
2225 static int psp_set_clockgating_state(void *handle,
2226                                      enum amd_clockgating_state state)
2227 {
2228         return 0;
2229 }
2230
2231 static int psp_set_powergating_state(void *handle,
2232                                      enum amd_powergating_state state)
2233 {
2234         return 0;
2235 }
2236
2237 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
2238                                          struct device_attribute *attr,
2239                                          char *buf)
2240 {
2241         struct drm_device *ddev = dev_get_drvdata(dev);
2242         struct amdgpu_device *adev = ddev->dev_private;
2243         uint32_t fw_ver;
2244         int ret;
2245
2246         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2247                 DRM_INFO("PSP block is not ready yet.");
2248                 return -EBUSY;
2249         }
2250
2251         mutex_lock(&adev->psp.mutex);
2252         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
2253         mutex_unlock(&adev->psp.mutex);
2254
2255         if (ret) {
2256                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
2257                 return ret;
2258         }
2259
2260         return snprintf(buf, PAGE_SIZE, "%x\n", fw_ver);
2261 }
2262
2263 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
2264                                                        struct device_attribute *attr,
2265                                                        const char *buf,
2266                                                        size_t count)
2267 {
2268         struct drm_device *ddev = dev_get_drvdata(dev);
2269         struct amdgpu_device *adev = ddev->dev_private;
2270         void *cpu_addr;
2271         dma_addr_t dma_addr;
2272         int ret;
2273         char fw_name[100];
2274         const struct firmware *usbc_pd_fw;
2275
2276         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
2277                 DRM_INFO("PSP block is not ready yet.");
2278                 return -EBUSY;
2279         }
2280
2281         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
2282         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
2283         if (ret)
2284                 goto fail;
2285
2286         /* We need contiguous physical mem to place the FW  for psp to access */
2287         cpu_addr = dma_alloc_coherent(adev->dev, usbc_pd_fw->size, &dma_addr, GFP_KERNEL);
2288
2289         ret = dma_mapping_error(adev->dev, dma_addr);
2290         if (ret)
2291                 goto rel_buf;
2292
2293         memcpy_toio(cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
2294
2295         /*
2296          * x86 specific workaround.
2297          * Without it the buffer is invisible in PSP.
2298          *
2299          * TODO Remove once PSP starts snooping CPU cache
2300          */
2301 #ifdef CONFIG_X86
2302         clflush_cache_range(cpu_addr, (usbc_pd_fw->size & ~(L1_CACHE_BYTES - 1)));
2303 #endif
2304
2305         mutex_lock(&adev->psp.mutex);
2306         ret = psp_load_usbc_pd_fw(&adev->psp, dma_addr);
2307         mutex_unlock(&adev->psp.mutex);
2308
2309 rel_buf:
2310         dma_free_coherent(adev->dev, usbc_pd_fw->size, cpu_addr, dma_addr);
2311         release_firmware(usbc_pd_fw);
2312
2313 fail:
2314         if (ret) {
2315                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
2316                 return ret;
2317         }
2318
2319         return count;
2320 }
2321
2322 static DEVICE_ATTR(usbc_pd_fw, S_IRUGO | S_IWUSR,
2323                    psp_usbc_pd_fw_sysfs_read,
2324                    psp_usbc_pd_fw_sysfs_write);
2325
2326
2327
2328 const struct amd_ip_funcs psp_ip_funcs = {
2329         .name = "psp",
2330         .early_init = psp_early_init,
2331         .late_init = NULL,
2332         .sw_init = psp_sw_init,
2333         .sw_fini = psp_sw_fini,
2334         .hw_init = psp_hw_init,
2335         .hw_fini = psp_hw_fini,
2336         .suspend = psp_suspend,
2337         .resume = psp_resume,
2338         .is_idle = NULL,
2339         .check_soft_reset = NULL,
2340         .wait_for_idle = NULL,
2341         .soft_reset = NULL,
2342         .set_clockgating_state = psp_set_clockgating_state,
2343         .set_powergating_state = psp_set_powergating_state,
2344 };
2345
2346 static int psp_sysfs_init(struct amdgpu_device *adev)
2347 {
2348         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
2349
2350         if (ret)
2351                 DRM_ERROR("Failed to create USBC PD FW control file!");
2352
2353         return ret;
2354 }
2355
2356 static void psp_sysfs_fini(struct amdgpu_device *adev)
2357 {
2358         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
2359 }
2360
2361 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
2362 {
2363         .type = AMD_IP_BLOCK_TYPE_PSP,
2364         .major = 3,
2365         .minor = 1,
2366         .rev = 0,
2367         .funcs = &psp_ip_funcs,
2368 };
2369
2370 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
2371 {
2372         .type = AMD_IP_BLOCK_TYPE_PSP,
2373         .major = 10,
2374         .minor = 0,
2375         .rev = 0,
2376         .funcs = &psp_ip_funcs,
2377 };
2378
2379 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
2380 {
2381         .type = AMD_IP_BLOCK_TYPE_PSP,
2382         .major = 11,
2383         .minor = 0,
2384         .rev = 0,
2385         .funcs = &psp_ip_funcs,
2386 };
2387
2388 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
2389 {
2390         .type = AMD_IP_BLOCK_TYPE_PSP,
2391         .major = 12,
2392         .minor = 0,
2393         .rev = 0,
2394         .funcs = &psp_ip_funcs,
2395 };