drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 #include "amdgpu_ras.h"
  38
  39 static void psp_set_funcs(struct amdgpu_device *adev);
  40
  41 static int psp_early_init(void *handle)
  42 {
  43         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  44         struct psp_context *psp = &adev->psp;
  45
  46         psp_set_funcs(adev);
  47
  48         switch (adev->asic_type) {
  49         case CHIP_VEGA10:
  50         case CHIP_VEGA12:
  51                 psp_v3_1_set_psp_funcs(psp);
  52                 psp->autoload_supported = false;
  53                 break;
  54         case CHIP_RAVEN:
  55                 psp_v10_0_set_psp_funcs(psp);
  56                 psp->autoload_supported = false;
  57                 break;
  58         case CHIP_VEGA20:
  59         case CHIP_ARCTURUS:
  60                 psp_v11_0_set_psp_funcs(psp);
  61                 psp->autoload_supported = false;
  62                 break;
  63         case CHIP_NAVI10:
  64         case CHIP_NAVI14:
  65         case CHIP_NAVI12:
  66                 psp_v11_0_set_psp_funcs(psp);
  67                 psp->autoload_supported = true;
  68                 break;
  69         case CHIP_RENOIR:
  70                 psp_v12_0_set_psp_funcs(psp);
  71                 break;
  72         default:
  73                 return -EINVAL;
  74         }
  75
  76         psp->adev = adev;
  77
  78         return 0;
  79 }
  80
  81 static int psp_sw_init(void *handle)
  82 {
  83         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  84         struct psp_context *psp = &adev->psp;
  85         int ret;
  86
  87         ret = psp_init_microcode(psp);
  88         if (ret) {
  89                 DRM_ERROR("Failed to load psp firmware!\n");
  90                 return ret;
  91         }
  92
  93         ret = psp_mem_training_init(psp);
  94         if (ret) {
  95                 DRM_ERROR("Failed to initialize memory training!\n");
  96                 return ret;
  97         }
  98         ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
  99         if (ret) {
 100                 DRM_ERROR("Failed to process memory training!\n");
 101                 return ret;
 102         }
 103
 104         return 0;
 105 }
 106
 107 static int psp_sw_fini(void *handle)
 108 {
 109         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 110
 111         psp_mem_training_fini(&adev->psp);
 112         release_firmware(adev->psp.sos_fw);
 113         adev->psp.sos_fw = NULL;
 114         release_firmware(adev->psp.asd_fw);
 115         adev->psp.asd_fw = NULL;
 116         if (adev->psp.ta_fw) {
 117                 release_firmware(adev->psp.ta_fw);
 118                 adev->psp.ta_fw = NULL;
 119         }
 120         return 0;
 121 }
 122
 123 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 124                  uint32_t reg_val, uint32_t mask, bool check_changed)
 125 {
 126         uint32_t val;
 127         int i;
 128         struct amdgpu_device *adev = psp->adev;
 129
 130         for (i = 0; i < adev->usec_timeout; i++) {
 131                 val = RREG32(reg_index);
 132                 if (check_changed) {
 133                         if (val != reg_val)
 134                                 return 0;
 135                 } else {
 136                         if ((val & mask) == reg_val)
 137                                 return 0;
 138                 }
 139                 udelay(1);
 140         }
 141
 142         return -ETIME;
 143 }
 144
 145 static int
 146 psp_cmd_submit_buf(struct psp_context *psp,
 147                    struct amdgpu_firmware_info *ucode,
 148                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 149 {
 150         int ret;
 151         int index;
 152         int timeout = 2000;
 153
 154         mutex_lock(&psp->mutex);
 155
 156         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 157
 158         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 159
 160         index = atomic_inc_return(&psp->fence_value);
 161         ret = psp_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 162         if (ret) {
 163                 atomic_dec(&psp->fence_value);
 164                 mutex_unlock(&psp->mutex);
 165                 return ret;
 166         }
 167
 168         amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 169         while (*((unsigned int *)psp->fence_buf) != index) {
 170                 if (--timeout == 0)
 171                         break;
 172                 /*
 173                  * Shouldn't wait for timeout when err_event_athub occurs,
 174                  * because gpu reset thread triggered and lock resource should
 175                  * be released for psp resume sequence.
 176                  */
 177                 if (amdgpu_ras_intr_triggered())
 178                         break;
 179                 msleep(1);
 180                 amdgpu_asic_invalidate_hdp(psp->adev, NULL);
 181         }
 182
 183         /* In some cases, psp response status is not 0 even there is no
 184          * problem while the command is submitted. Some version of PSP FW
 185          * doesn't write 0 to that field.
 186          * So here we would like to only print a warning instead of an error
 187          * during psp initialization to avoid breaking hw_init and it doesn't
 188          * return -EINVAL.
 189          */
 190         if (psp->cmd_buf_mem->resp.status || !timeout) {
 191                 if (ucode)
 192                         DRM_WARN("failed to load ucode id (%d) ",
 193                                   ucode->ucode_id);
 194                 DRM_DEBUG_DRIVER("psp command (0x%X) failed and response status is (0x%X)\n",
 195                          psp->cmd_buf_mem->cmd_id,
 196                          psp->cmd_buf_mem->resp.status & GFX_CMD_STATUS_MASK);
 197                 if (!timeout) {
 198                         mutex_unlock(&psp->mutex);
 199                         return -EINVAL;
 200                 }
 201         }
 202
 203         /* get xGMI session id from response buffer */
 204         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 205
 206         if (ucode) {
 207                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 208                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 209         }
 210         mutex_unlock(&psp->mutex);
 211
 212         return ret;
 213 }
 214
 215 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 216                                  struct psp_gfx_cmd_resp *cmd,
 217                                  uint64_t tmr_mc, uint32_t size)
 218 {
 219         if (psp_support_vmr_ring(psp))
 220                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 221         else
 222                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 223         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 224         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 225         cmd->cmd.cmd_setup_tmr.buf_size = size;
 226 }
 227
 228 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 229                                       uint64_t pri_buf_mc, uint32_t size)
 230 {
 231         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 232         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 233         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 234         cmd->cmd.cmd_load_toc.toc_size = size;
 235 }
 236
 237 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 238 static int psp_load_toc(struct psp_context *psp,
 239                         uint32_t *tmr_size)
 240 {
 241         int ret;
 242         struct psp_gfx_cmd_resp *cmd;
 243
 244         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 245         if (!cmd)
 246                 return -ENOMEM;
 247         /* Copy toc to psp firmware private buffer */
 248         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 249         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 250
 251         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 252
 253         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 254                                  psp->fence_buf_mc_addr);
 255         if (!ret)
 256                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 257         kfree(cmd);
 258         return ret;
 259 }
 260
 261 /* Set up Trusted Memory Region */
 262 static int psp_tmr_init(struct psp_context *psp)
 263 {
 264         int ret;
 265         int tmr_size;
 266         void *tmr_buf;
 267         void **pptr;
 268
 269         /*
 270          * According to HW engineer, they prefer the TMR address be "naturally
 271          * aligned" , e.g. the start address be an integer divide of TMR size.
 272          *
 273          * Note: this memory need be reserved till the driver
 274          * uninitializes.
 275          */
 276         tmr_size = PSP_TMR_SIZE;
 277
 278         /* For ASICs support RLC autoload, psp will parse the toc
 279          * and calculate the total size of TMR needed */
 280         if (!amdgpu_sriov_vf(psp->adev) &&
 281             psp->toc_start_addr &&
 282             psp->toc_bin_size &&
 283             psp->fw_pri_buf) {
 284                 ret = psp_load_toc(psp, &tmr_size);
 285                 if (ret) {
 286                         DRM_ERROR("Failed to load toc\n");
 287                         return ret;
 288                 }
 289         }
 290
 291         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 292         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 293                                       AMDGPU_GEM_DOMAIN_VRAM,
 294                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 295
 296         return ret;
 297 }
 298
 299 static int psp_tmr_load(struct psp_context *psp)
 300 {
 301         int ret;
 302         struct psp_gfx_cmd_resp *cmd;
 303
 304         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 305         if (!cmd)
 306                 return -ENOMEM;
 307
 308         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 309                              amdgpu_bo_size(psp->tmr_bo));
 310         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 311                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 312
 313         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 314                                  psp->fence_buf_mc_addr);
 315
 316         kfree(cmd);
 317
 318         return ret;
 319 }
 320
 321 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 322                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 323                                  uint32_t size, uint32_t shared_size)
 324 {
 325         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 326         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 327         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 328         cmd->cmd.cmd_load_ta.app_len = size;
 329
 330         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 331         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 332         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 333 }
 334
 335 static int psp_asd_init(struct psp_context *psp)
 336 {
 337         int ret;
 338
 339         /*
 340          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 341          * physical) for shared ASD <-> Driver
 342          */
 343         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 344                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 345                                       &psp->asd_shared_bo,
 346                                       &psp->asd_shared_mc_addr,
 347                                       &psp->asd_shared_buf);
 348
 349         return ret;
 350 }
 351
 352 static int psp_asd_load(struct psp_context *psp)
 353 {
 354         int ret;
 355         struct psp_gfx_cmd_resp *cmd;
 356
 357         /* If PSP version doesn't match ASD version, asd loading will be failed.
 358          * add workaround to bypass it for sriov now.
 359          * TODO: add version check to make it common
 360          */
 361         if (amdgpu_sriov_vf(psp->adev))
 362                 return 0;
 363
 364         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 365         if (!cmd)
 366                 return -ENOMEM;
 367
 368         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 369         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 370
 371         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 372                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 373
 374         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 375                                  psp->fence_buf_mc_addr);
 376
 377         kfree(cmd);
 378
 379         return ret;
 380 }
 381
 382 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 383                 uint32_t id, uint32_t value)
 384 {
 385         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 386         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 387         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 388 }
 389
 390 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 391                 uint32_t value)
 392 {
 393         struct psp_gfx_cmd_resp *cmd = NULL;
 394         int ret = 0;
 395
 396         if (reg >= PSP_REG_LAST)
 397                 return -EINVAL;
 398
 399         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 400         if (!cmd)
 401                 return -ENOMEM;
 402
 403         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 404         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 405
 406         kfree(cmd);
 407         return ret;
 408 }
 409
 410 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 411                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 412                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 413 {
 414         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 415         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 416         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 417         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 418
 419         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 420         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 421         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 422 }
 423
 424 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 425 {
 426         int ret;
 427
 428         /*
 429          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 430          * physical) for xgmi ta <-> Driver
 431          */
 432         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 433                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 434                                       &psp->xgmi_context.xgmi_shared_bo,
 435                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 436                                       &psp->xgmi_context.xgmi_shared_buf);
 437
 438         return ret;
 439 }
 440
 441 static int psp_xgmi_load(struct psp_context *psp)
 442 {
 443         int ret;
 444         struct psp_gfx_cmd_resp *cmd;
 445
 446         /*
 447          * TODO: bypass the loading in sriov for now
 448          */
 449         if (amdgpu_sriov_vf(psp->adev))
 450                 return 0;
 451
 452         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 453         if (!cmd)
 454                 return -ENOMEM;
 455
 456         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 457         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 458
 459         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 460                                       psp->xgmi_context.xgmi_shared_mc_addr,
 461                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 462
 463         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 464                                  psp->fence_buf_mc_addr);
 465
 466         if (!ret) {
 467                 psp->xgmi_context.initialized = 1;
 468                 psp->xgmi_context.session_id = cmd->resp.session_id;
 469         }
 470
 471         kfree(cmd);
 472
 473         return ret;
 474 }
 475
 476 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 477                                             uint32_t xgmi_session_id)
 478 {
 479         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 480         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 481 }
 482
 483 static int psp_xgmi_unload(struct psp_context *psp)
 484 {
 485         int ret;
 486         struct psp_gfx_cmd_resp *cmd;
 487
 488         /*
 489          * TODO: bypass the unloading in sriov for now
 490          */
 491         if (amdgpu_sriov_vf(psp->adev))
 492                 return 0;
 493
 494         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 495         if (!cmd)
 496                 return -ENOMEM;
 497
 498         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 499
 500         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 501                                  psp->fence_buf_mc_addr);
 502
 503         kfree(cmd);
 504
 505         return ret;
 506 }
 507
 508 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 509                                             uint32_t ta_cmd_id,
 510                                             uint32_t xgmi_session_id)
 511 {
 512         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 513         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 514         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 515         /* Note: cmd_invoke_cmd.buf is not used for now */
 516 }
 517
 518 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 519 {
 520         int ret;
 521         struct psp_gfx_cmd_resp *cmd;
 522
 523         /*
 524          * TODO: bypass the loading in sriov for now
 525         */
 526         if (amdgpu_sriov_vf(psp->adev))
 527                 return 0;
 528
 529         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 530         if (!cmd)
 531                 return -ENOMEM;
 532
 533         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 534                                         psp->xgmi_context.session_id);
 535
 536         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 537                                  psp->fence_buf_mc_addr);
 538
 539         kfree(cmd);
 540
 541         return ret;
 542 }
 543
 544 static int psp_xgmi_terminate(struct psp_context *psp)
 545 {
 546         int ret;
 547
 548         if (!psp->xgmi_context.initialized)
 549                 return 0;
 550
 551         ret = psp_xgmi_unload(psp);
 552         if (ret)
 553                 return ret;
 554
 555         psp->xgmi_context.initialized = 0;
 556
 557         /* free xgmi shared memory */
 558         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 559                         &psp->xgmi_context.xgmi_shared_mc_addr,
 560                         &psp->xgmi_context.xgmi_shared_buf);
 561
 562         return 0;
 563 }
 564
 565 static int psp_xgmi_initialize(struct psp_context *psp)
 566 {
 567         struct ta_xgmi_shared_memory *xgmi_cmd;
 568         int ret;
 569
 570         if (!psp->adev->psp.ta_fw)
 571                 return -ENOENT;
 572
 573         if (!psp->xgmi_context.initialized) {
 574                 ret = psp_xgmi_init_shared_buf(psp);
 575                 if (ret)
 576                         return ret;
 577         }
 578
 579         /* Load XGMI TA */
 580         ret = psp_xgmi_load(psp);
 581         if (ret)
 582                 return ret;
 583
 584         /* Initialize XGMI session */
 585         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 586         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 587         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 588
 589         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 590
 591         return ret;
 592 }
 593
 594 // ras begin
 595 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 596                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 597                 uint32_t ras_ta_size, uint32_t shared_size)
 598 {
 599         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 600         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 601         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 602         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 603
 604         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 605         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 606         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 607 }
 608
 609 static int psp_ras_init_shared_buf(struct psp_context *psp)
 610 {
 611         int ret;
 612
 613         /*
 614          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 615          * physical) for ras ta <-> Driver
 616          */
 617         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 618                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 619                         &psp->ras.ras_shared_bo,
 620                         &psp->ras.ras_shared_mc_addr,
 621                         &psp->ras.ras_shared_buf);
 622
 623         return ret;
 624 }
 625
 626 static int psp_ras_load(struct psp_context *psp)
 627 {
 628         int ret;
 629         struct psp_gfx_cmd_resp *cmd;
 630
 631         /*
 632          * TODO: bypass the loading in sriov for now
 633          */
 634         if (amdgpu_sriov_vf(psp->adev))
 635                 return 0;
 636
 637         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 638         if (!cmd)
 639                 return -ENOMEM;
 640
 641         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 642         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 643
 644         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 645                         psp->ras.ras_shared_mc_addr,
 646                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 647
 648         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 649                         psp->fence_buf_mc_addr);
 650
 651         if (!ret) {
 652                 psp->ras.ras_initialized = 1;
 653                 psp->ras.session_id = cmd->resp.session_id;
 654         }
 655
 656         kfree(cmd);
 657
 658         return ret;
 659 }
 660
 661 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 662                                                 uint32_t ras_session_id)
 663 {
 664         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 665         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 666 }
 667
 668 static int psp_ras_unload(struct psp_context *psp)
 669 {
 670         int ret;
 671         struct psp_gfx_cmd_resp *cmd;
 672
 673         /*
 674          * TODO: bypass the unloading in sriov for now
 675          */
 676         if (amdgpu_sriov_vf(psp->adev))
 677                 return 0;
 678
 679         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 680         if (!cmd)
 681                 return -ENOMEM;
 682
 683         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 684
 685         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 686                         psp->fence_buf_mc_addr);
 687
 688         kfree(cmd);
 689
 690         return ret;
 691 }
 692
 693 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 694                 uint32_t ta_cmd_id,
 695                 uint32_t ras_session_id)
 696 {
 697         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 698         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 699         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 700         /* Note: cmd_invoke_cmd.buf is not used for now */
 701 }
 702
 703 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 704 {
 705         int ret;
 706         struct psp_gfx_cmd_resp *cmd;
 707
 708         /*
 709          * TODO: bypass the loading in sriov for now
 710          */
 711         if (amdgpu_sriov_vf(psp->adev))
 712                 return 0;
 713
 714         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 715         if (!cmd)
 716                 return -ENOMEM;
 717
 718         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 719                         psp->ras.session_id);
 720
 721         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 722                         psp->fence_buf_mc_addr);
 723
 724         kfree(cmd);
 725
 726         return ret;
 727 }
 728
 729 int psp_ras_enable_features(struct psp_context *psp,
 730                 union ta_ras_cmd_input *info, bool enable)
 731 {
 732         struct ta_ras_shared_memory *ras_cmd;
 733         int ret;
 734
 735         if (!psp->ras.ras_initialized)
 736                 return -EINVAL;
 737
 738         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 739         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 740
 741         if (enable)
 742                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 743         else
 744                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 745
 746         ras_cmd->ras_in_message = *info;
 747
 748         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 749         if (ret)
 750                 return -EINVAL;
 751
 752         return ras_cmd->ras_status;
 753 }
 754
 755 static int psp_ras_terminate(struct psp_context *psp)
 756 {
 757         int ret;
 758
 759         if (!psp->ras.ras_initialized)
 760                 return 0;
 761
 762         ret = psp_ras_unload(psp);
 763         if (ret)
 764                 return ret;
 765
 766         psp->ras.ras_initialized = 0;
 767
 768         /* free ras shared memory */
 769         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 770                         &psp->ras.ras_shared_mc_addr,
 771                         &psp->ras.ras_shared_buf);
 772
 773         return 0;
 774 }
 775
 776 static int psp_ras_initialize(struct psp_context *psp)
 777 {
 778         int ret;
 779
 780         if (!psp->ras.ras_initialized) {
 781                 ret = psp_ras_init_shared_buf(psp);
 782                 if (ret)
 783                         return ret;
 784         }
 785
 786         ret = psp_ras_load(psp);
 787         if (ret)
 788                 return ret;
 789
 790         return 0;
 791 }
 792 // ras end
 793
 794 // HDCP start
 795 static void psp_prep_hdcp_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 796                                           uint64_t hdcp_ta_mc,
 797                                           uint64_t hdcp_mc_shared,
 798                                           uint32_t hdcp_ta_size,
 799                                           uint32_t shared_size)
 800 {
 801         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 802         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(hdcp_ta_mc);
 803         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(hdcp_ta_mc);
 804         cmd->cmd.cmd_load_ta.app_len = hdcp_ta_size;
 805
 806         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
 807                 lower_32_bits(hdcp_mc_shared);
 808         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
 809                 upper_32_bits(hdcp_mc_shared);
 810         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 811 }
 812
 813 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
 814 {
 815         int ret;
 816
 817         /*
 818          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 819          * physical) for hdcp ta <-> Driver
 820          */
 821         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
 822                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 823                                       &psp->hdcp_context.hdcp_shared_bo,
 824                                       &psp->hdcp_context.hdcp_shared_mc_addr,
 825                                       &psp->hdcp_context.hdcp_shared_buf);
 826
 827         return ret;
 828 }
 829
 830 static int psp_hdcp_load(struct psp_context *psp)
 831 {
 832         int ret;
 833         struct psp_gfx_cmd_resp *cmd;
 834
 835         /*
 836          * TODO: bypass the loading in sriov for now
 837          */
 838         if (amdgpu_sriov_vf(psp->adev))
 839                 return 0;
 840
 841         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 842         if (!cmd)
 843                 return -ENOMEM;
 844
 845         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 846         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
 847                psp->ta_hdcp_ucode_size);
 848
 849         psp_prep_hdcp_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 850                                       psp->hdcp_context.hdcp_shared_mc_addr,
 851                                       psp->ta_hdcp_ucode_size,
 852                                       PSP_HDCP_SHARED_MEM_SIZE);
 853
 854         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 855
 856         if (!ret) {
 857                 psp->hdcp_context.hdcp_initialized = 1;
 858                 psp->hdcp_context.session_id = cmd->resp.session_id;
 859         }
 860
 861         kfree(cmd);
 862
 863         return ret;
 864 }
 865 static int psp_hdcp_initialize(struct psp_context *psp)
 866 {
 867         int ret;
 868
 869         if (!psp->hdcp_context.hdcp_initialized) {
 870                 ret = psp_hdcp_init_shared_buf(psp);
 871                 if (ret)
 872                         return ret;
 873         }
 874
 875         ret = psp_hdcp_load(psp);
 876         if (ret)
 877                 return ret;
 878
 879         return 0;
 880 }
 881 static void psp_prep_hdcp_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 882                                             uint32_t hdcp_session_id)
 883 {
 884         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 885         cmd->cmd.cmd_unload_ta.session_id = hdcp_session_id;
 886 }
 887
 888 static int psp_hdcp_unload(struct psp_context *psp)
 889 {
 890         int ret;
 891         struct psp_gfx_cmd_resp *cmd;
 892
 893         /*
 894          * TODO: bypass the unloading in sriov for now
 895          */
 896         if (amdgpu_sriov_vf(psp->adev))
 897                 return 0;
 898
 899         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 900         if (!cmd)
 901                 return -ENOMEM;
 902
 903         psp_prep_hdcp_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
 904
 905         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 906
 907         kfree(cmd);
 908
 909         return ret;
 910 }
 911
 912 static void psp_prep_hdcp_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 913                                             uint32_t ta_cmd_id,
 914                                             uint32_t hdcp_session_id)
 915 {
 916         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 917         cmd->cmd.cmd_invoke_cmd.session_id = hdcp_session_id;
 918         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 919         /* Note: cmd_invoke_cmd.buf is not used for now */
 920 }
 921
 922 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 923 {
 924         int ret;
 925         struct psp_gfx_cmd_resp *cmd;
 926
 927         /*
 928          * TODO: bypass the loading in sriov for now
 929          */
 930         if (amdgpu_sriov_vf(psp->adev))
 931                 return 0;
 932
 933         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 934         if (!cmd)
 935                 return -ENOMEM;
 936
 937         psp_prep_hdcp_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 938                                         psp->hdcp_context.session_id);
 939
 940         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 941
 942         kfree(cmd);
 943
 944         return ret;
 945 }
 946
 947 static int psp_hdcp_terminate(struct psp_context *psp)
 948 {
 949         int ret;
 950
 951         if (!psp->hdcp_context.hdcp_initialized)
 952                 return 0;
 953
 954         ret = psp_hdcp_unload(psp);
 955         if (ret)
 956                 return ret;
 957
 958         psp->hdcp_context.hdcp_initialized = 0;
 959
 960         /* free hdcp shared memory */
 961         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
 962                               &psp->hdcp_context.hdcp_shared_mc_addr,
 963                               &psp->hdcp_context.hdcp_shared_buf);
 964
 965         return 0;
 966 }
 967 // HDCP end
 968
 969 // DTM start
 970 static void psp_prep_dtm_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 971                                          uint64_t dtm_ta_mc,
 972                                          uint64_t dtm_mc_shared,
 973                                          uint32_t dtm_ta_size,
 974                                          uint32_t shared_size)
 975 {
 976         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 977         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(dtm_ta_mc);
 978         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(dtm_ta_mc);
 979         cmd->cmd.cmd_load_ta.app_len = dtm_ta_size;
 980
 981         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(dtm_mc_shared);
 982         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(dtm_mc_shared);
 983         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 984 }
 985
 986 static int psp_dtm_init_shared_buf(struct psp_context *psp)
 987 {
 988         int ret;
 989
 990         /*
 991          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 992          * physical) for dtm ta <-> Driver
 993          */
 994         ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE,
 995                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 996                                       &psp->dtm_context.dtm_shared_bo,
 997                                       &psp->dtm_context.dtm_shared_mc_addr,
 998                                       &psp->dtm_context.dtm_shared_buf);
 999
1000         return ret;
1001 }
1002
1003 static int psp_dtm_load(struct psp_context *psp)
1004 {
1005         int ret;
1006         struct psp_gfx_cmd_resp *cmd;
1007
1008         /*
1009          * TODO: bypass the loading in sriov for now
1010          */
1011         if (amdgpu_sriov_vf(psp->adev))
1012                 return 0;
1013
1014         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1015         if (!cmd)
1016                 return -ENOMEM;
1017
1018         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1019         memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size);
1020
1021         psp_prep_dtm_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
1022                                      psp->dtm_context.dtm_shared_mc_addr,
1023                                      psp->ta_dtm_ucode_size,
1024                                      PSP_DTM_SHARED_MEM_SIZE);
1025
1026         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1027
1028         if (!ret) {
1029                 psp->dtm_context.dtm_initialized = 1;
1030                 psp->dtm_context.session_id = cmd->resp.session_id;
1031         }
1032
1033         kfree(cmd);
1034
1035         return ret;
1036 }
1037
1038 static int psp_dtm_initialize(struct psp_context *psp)
1039 {
1040         int ret;
1041
1042         if (!psp->dtm_context.dtm_initialized) {
1043                 ret = psp_dtm_init_shared_buf(psp);
1044                 if (ret)
1045                         return ret;
1046         }
1047
1048         ret = psp_dtm_load(psp);
1049         if (ret)
1050                 return ret;
1051
1052         return 0;
1053 }
1054
1055 static void psp_prep_dtm_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1056                                            uint32_t ta_cmd_id,
1057                                            uint32_t dtm_session_id)
1058 {
1059         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
1060         cmd->cmd.cmd_invoke_cmd.session_id = dtm_session_id;
1061         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
1062         /* Note: cmd_invoke_cmd.buf is not used for now */
1063 }
1064
1065 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1066 {
1067         int ret;
1068         struct psp_gfx_cmd_resp *cmd;
1069
1070         /*
1071          * TODO: bypass the loading in sriov for now
1072          */
1073         if (amdgpu_sriov_vf(psp->adev))
1074                 return 0;
1075
1076         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1077         if (!cmd)
1078                 return -ENOMEM;
1079
1080         psp_prep_dtm_ta_invoke_cmd_buf(cmd, ta_cmd_id,
1081                                        psp->dtm_context.session_id);
1082
1083         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1084
1085         kfree(cmd);
1086
1087         return ret;
1088 }
1089
1090 static int psp_dtm_terminate(struct psp_context *psp)
1091 {
1092         int ret;
1093
1094         if (!psp->dtm_context.dtm_initialized)
1095                 return 0;
1096
1097         ret = psp_hdcp_unload(psp);
1098         if (ret)
1099                 return ret;
1100
1101         psp->dtm_context.dtm_initialized = 0;
1102
1103         /* free hdcp shared memory */
1104         amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo,
1105                               &psp->dtm_context.dtm_shared_mc_addr,
1106                               &psp->dtm_context.dtm_shared_buf);
1107
1108         return 0;
1109 }
1110 // DTM end
1111
1112 static int psp_hw_start(struct psp_context *psp)
1113 {
1114         struct amdgpu_device *adev = psp->adev;
1115         int ret;
1116
1117         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
1118                 if (psp->kdb_bin_size &&
1119                     (psp->funcs->bootloader_load_kdb != NULL)) {
1120                         ret = psp_bootloader_load_kdb(psp);
1121                         if (ret) {
1122                                 DRM_ERROR("PSP load kdb failed!\n");
1123                                 return ret;
1124                         }
1125                 }
1126
1127                 ret = psp_bootloader_load_sysdrv(psp);
1128                 if (ret) {
1129                         DRM_ERROR("PSP load sysdrv failed!\n");
1130                         return ret;
1131                 }
1132
1133                 ret = psp_bootloader_load_sos(psp);
1134                 if (ret) {
1135                         DRM_ERROR("PSP load sos failed!\n");
1136                         return ret;
1137                 }
1138         }
1139
1140         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
1141         if (ret) {
1142                 DRM_ERROR("PSP create ring failed!\n");
1143                 return ret;
1144         }
1145
1146         ret = psp_tmr_init(psp);
1147         if (ret) {
1148                 DRM_ERROR("PSP tmr init failed!\n");
1149                 return ret;
1150         }
1151
1152         ret = psp_tmr_load(psp);
1153         if (ret) {
1154                 DRM_ERROR("PSP load tmr failed!\n");
1155                 return ret;
1156         }
1157
1158         ret = psp_asd_init(psp);
1159         if (ret) {
1160                 DRM_ERROR("PSP asd init failed!\n");
1161                 return ret;
1162         }
1163
1164         ret = psp_asd_load(psp);
1165         if (ret) {
1166                 DRM_ERROR("PSP load asd failed!\n");
1167                 return ret;
1168         }
1169
1170         if (adev->gmc.xgmi.num_physical_nodes > 1) {
1171                 ret = psp_xgmi_initialize(psp);
1172                 /* Warning the XGMI seesion initialize failure
1173                  * Instead of stop driver initialization
1174                  */
1175                 if (ret)
1176                         dev_err(psp->adev->dev,
1177                                 "XGMI: Failed to initialize XGMI session\n");
1178         }
1179
1180         if (psp->adev->psp.ta_fw) {
1181                 ret = psp_ras_initialize(psp);
1182                 if (ret)
1183                         dev_err(psp->adev->dev,
1184                                         "RAS: Failed to initialize RAS\n");
1185
1186                 ret = psp_hdcp_initialize(psp);
1187                 if (ret)
1188                         dev_err(psp->adev->dev,
1189                                 "HDCP: Failed to initialize HDCP\n");
1190
1191                 ret = psp_dtm_initialize(psp);
1192                 if (ret)
1193                         dev_err(psp->adev->dev,
1194                                 "DTM: Failed to initialize DTM\n");
1195         }
1196
1197         return 0;
1198 }
1199
1200 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1201                            enum psp_gfx_fw_type *type)
1202 {
1203         switch (ucode->ucode_id) {
1204         case AMDGPU_UCODE_ID_SDMA0:
1205                 *type = GFX_FW_TYPE_SDMA0;
1206                 break;
1207         case AMDGPU_UCODE_ID_SDMA1:
1208                 *type = GFX_FW_TYPE_SDMA1;
1209                 break;
1210         case AMDGPU_UCODE_ID_SDMA2:
1211                 *type = GFX_FW_TYPE_SDMA2;
1212                 break;
1213         case AMDGPU_UCODE_ID_SDMA3:
1214                 *type = GFX_FW_TYPE_SDMA3;
1215                 break;
1216         case AMDGPU_UCODE_ID_SDMA4:
1217                 *type = GFX_FW_TYPE_SDMA4;
1218                 break;
1219         case AMDGPU_UCODE_ID_SDMA5:
1220                 *type = GFX_FW_TYPE_SDMA5;
1221                 break;
1222         case AMDGPU_UCODE_ID_SDMA6:
1223                 *type = GFX_FW_TYPE_SDMA6;
1224                 break;
1225         case AMDGPU_UCODE_ID_SDMA7:
1226                 *type = GFX_FW_TYPE_SDMA7;
1227                 break;
1228         case AMDGPU_UCODE_ID_CP_CE:
1229                 *type = GFX_FW_TYPE_CP_CE;
1230                 break;
1231         case AMDGPU_UCODE_ID_CP_PFP:
1232                 *type = GFX_FW_TYPE_CP_PFP;
1233                 break;
1234         case AMDGPU_UCODE_ID_CP_ME:
1235                 *type = GFX_FW_TYPE_CP_ME;
1236                 break;
1237         case AMDGPU_UCODE_ID_CP_MEC1:
1238                 *type = GFX_FW_TYPE_CP_MEC;
1239                 break;
1240         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1241                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1242                 break;
1243         case AMDGPU_UCODE_ID_CP_MEC2:
1244                 *type = GFX_FW_TYPE_CP_MEC;
1245                 break;
1246         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1247                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1248                 break;
1249         case AMDGPU_UCODE_ID_RLC_G:
1250                 *type = GFX_FW_TYPE_RLC_G;
1251                 break;
1252         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1253                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1254                 break;
1255         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1256                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1257                 break;
1258         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1259                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1260                 break;
1261         case AMDGPU_UCODE_ID_SMC:
1262                 *type = GFX_FW_TYPE_SMU;
1263                 break;
1264         case AMDGPU_UCODE_ID_UVD:
1265                 *type = GFX_FW_TYPE_UVD;
1266                 break;
1267         case AMDGPU_UCODE_ID_UVD1:
1268                 *type = GFX_FW_TYPE_UVD1;
1269                 break;
1270         case AMDGPU_UCODE_ID_VCE:
1271                 *type = GFX_FW_TYPE_VCE;
1272                 break;
1273         case AMDGPU_UCODE_ID_VCN:
1274                 *type = GFX_FW_TYPE_VCN;
1275                 break;
1276         case AMDGPU_UCODE_ID_DMCU_ERAM:
1277                 *type = GFX_FW_TYPE_DMCU_ERAM;
1278                 break;
1279         case AMDGPU_UCODE_ID_DMCU_INTV:
1280                 *type = GFX_FW_TYPE_DMCU_ISR;
1281                 break;
1282         case AMDGPU_UCODE_ID_VCN0_RAM:
1283                 *type = GFX_FW_TYPE_VCN0_RAM;
1284                 break;
1285         case AMDGPU_UCODE_ID_VCN1_RAM:
1286                 *type = GFX_FW_TYPE_VCN1_RAM;
1287                 break;
1288         case AMDGPU_UCODE_ID_MAXIMUM:
1289         default:
1290                 return -EINVAL;
1291         }
1292
1293         return 0;
1294 }
1295
1296 static void psp_print_fw_hdr(struct psp_context *psp,
1297                              struct amdgpu_firmware_info *ucode)
1298 {
1299         struct amdgpu_device *adev = psp->adev;
1300         struct common_firmware_header *hdr;
1301
1302         switch (ucode->ucode_id) {
1303         case AMDGPU_UCODE_ID_SDMA0:
1304         case AMDGPU_UCODE_ID_SDMA1:
1305         case AMDGPU_UCODE_ID_SDMA2:
1306         case AMDGPU_UCODE_ID_SDMA3:
1307         case AMDGPU_UCODE_ID_SDMA4:
1308         case AMDGPU_UCODE_ID_SDMA5:
1309         case AMDGPU_UCODE_ID_SDMA6:
1310         case AMDGPU_UCODE_ID_SDMA7:
1311                 hdr = (struct common_firmware_header *)
1312                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1313                 amdgpu_ucode_print_sdma_hdr(hdr);
1314                 break;
1315         case AMDGPU_UCODE_ID_CP_CE:
1316                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1317                 amdgpu_ucode_print_gfx_hdr(hdr);
1318                 break;
1319         case AMDGPU_UCODE_ID_CP_PFP:
1320                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1321                 amdgpu_ucode_print_gfx_hdr(hdr);
1322                 break;
1323         case AMDGPU_UCODE_ID_CP_ME:
1324                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1325                 amdgpu_ucode_print_gfx_hdr(hdr);
1326                 break;
1327         case AMDGPU_UCODE_ID_CP_MEC1:
1328                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1329                 amdgpu_ucode_print_gfx_hdr(hdr);
1330                 break;
1331         case AMDGPU_UCODE_ID_RLC_G:
1332                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1333                 amdgpu_ucode_print_rlc_hdr(hdr);
1334                 break;
1335         case AMDGPU_UCODE_ID_SMC:
1336                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1337                 amdgpu_ucode_print_smc_hdr(hdr);
1338                 break;
1339         default:
1340                 break;
1341         }
1342 }
1343
1344 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1345                                        struct psp_gfx_cmd_resp *cmd)
1346 {
1347         int ret;
1348         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1349
1350         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1351
1352         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1353         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1354         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1355         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1356
1357         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1358         if (ret)
1359                 DRM_ERROR("Unknown firmware type\n");
1360
1361         return ret;
1362 }
1363
1364 static int psp_execute_np_fw_load(struct psp_context *psp,
1365                                struct amdgpu_firmware_info *ucode)
1366 {
1367         int ret = 0;
1368
1369         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1370         if (ret)
1371                 return ret;
1372
1373         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1374                                  psp->fence_buf_mc_addr);
1375
1376         return ret;
1377 }
1378
1379 static int psp_np_fw_load(struct psp_context *psp)
1380 {
1381         int i, ret;
1382         struct amdgpu_firmware_info *ucode;
1383         struct amdgpu_device* adev = psp->adev;
1384
1385         if (psp->autoload_supported) {
1386                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1387                 if (!ucode->fw)
1388                         goto out;
1389
1390                 ret = psp_execute_np_fw_load(psp, ucode);
1391                 if (ret)
1392                         return ret;
1393         }
1394
1395 out:
1396         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1397                 ucode = &adev->firmware.ucode[i];
1398                 if (!ucode->fw)
1399                         continue;
1400
1401                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1402                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1403                         continue;
1404
1405                 if (amdgpu_sriov_vf(adev) &&
1406                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1407                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1408                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1409                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1410                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1411                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1412                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1413                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1414                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
1415                         /*skip ucode loading in SRIOV VF */
1416                         continue;
1417
1418                 if (psp->autoload_supported &&
1419                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1420                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1421                         /* skip mec JT when autoload is enabled */
1422                         continue;
1423
1424                 psp_print_fw_hdr(psp, ucode);
1425
1426                 ret = psp_execute_np_fw_load(psp, ucode);
1427                 if (ret)
1428                         return ret;
1429
1430                 /* Start rlc autoload after psp recieved all the gfx firmware */
1431                 if (psp->autoload_supported && ucode->ucode_id ==
1432                         AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
1433                         ret = psp_rlc_autoload(psp);
1434                         if (ret) {
1435                                 DRM_ERROR("Failed to start rlc autoload\n");
1436                                 return ret;
1437                         }
1438                 }
1439 #if 0
1440                 /* check if firmware loaded sucessfully */
1441                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1442                         return -EINVAL;
1443 #endif
1444         }
1445
1446         return 0;
1447 }
1448
1449 static int psp_load_fw(struct amdgpu_device *adev)
1450 {
1451         int ret;
1452         struct psp_context *psp = &adev->psp;
1453
1454         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1455                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1456                 goto skip_memalloc;
1457         }
1458
1459         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1460         if (!psp->cmd)
1461                 return -ENOMEM;
1462
1463         /* this fw pri bo is not used under SRIOV */
1464         if (!amdgpu_sriov_vf(psp->adev)) {
1465                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1466                                               AMDGPU_GEM_DOMAIN_GTT,
1467                                               &psp->fw_pri_bo,
1468                                               &psp->fw_pri_mc_addr,
1469                                               &psp->fw_pri_buf);
1470                 if (ret)
1471                         goto failed;
1472         }
1473
1474         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1475                                         AMDGPU_GEM_DOMAIN_VRAM,
1476                                         &psp->fence_buf_bo,
1477                                         &psp->fence_buf_mc_addr,
1478                                         &psp->fence_buf);
1479         if (ret)
1480                 goto failed;
1481
1482         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1483                                       AMDGPU_GEM_DOMAIN_VRAM,
1484                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1485                                       (void **)&psp->cmd_buf_mem);
1486         if (ret)
1487                 goto failed;
1488
1489         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1490
1491         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1492         if (ret) {
1493                 DRM_ERROR("PSP ring init failed!\n");
1494                 goto failed;
1495         }
1496
1497 skip_memalloc:
1498         ret = psp_hw_start(psp);
1499         if (ret)
1500                 goto failed;
1501
1502         ret = psp_np_fw_load(psp);
1503         if (ret)
1504                 goto failed;
1505
1506         return 0;
1507
1508 failed:
1509         /*
1510          * all cleanup jobs (xgmi terminate, ras terminate,
1511          * ring destroy, cmd/fence/fw buffers destory,
1512          * psp->cmd destory) are delayed to psp_hw_fini
1513          */
1514         return ret;
1515 }
1516
1517 static int psp_hw_init(void *handle)
1518 {
1519         int ret;
1520         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1521
1522         mutex_lock(&adev->firmware.mutex);
1523         /*
1524          * This sequence is just used on hw_init only once, no need on
1525          * resume.
1526          */
1527         ret = amdgpu_ucode_init_bo(adev);
1528         if (ret)
1529                 goto failed;
1530
1531         ret = psp_load_fw(adev);
1532         if (ret) {
1533                 DRM_ERROR("PSP firmware loading failed\n");
1534                 goto failed;
1535         }
1536
1537         mutex_unlock(&adev->firmware.mutex);
1538         return 0;
1539
1540 failed:
1541         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1542         mutex_unlock(&adev->firmware.mutex);
1543         return -EINVAL;
1544 }
1545
1546 static int psp_hw_fini(void *handle)
1547 {
1548         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1549         struct psp_context *psp = &adev->psp;
1550         void *tmr_buf;
1551         void **pptr;
1552
1553         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1554             psp->xgmi_context.initialized == 1)
1555                 psp_xgmi_terminate(psp);
1556
1557         if (psp->adev->psp.ta_fw) {
1558                 psp_ras_terminate(psp);
1559                 psp_dtm_terminate(psp);
1560                 psp_hdcp_terminate(psp);
1561         }
1562
1563         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1564
1565         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1566         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1567         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1568                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1569         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1570                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1571         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1572                               &psp->asd_shared_buf);
1573         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1574                               (void **)&psp->cmd_buf_mem);
1575
1576         kfree(psp->cmd);
1577         psp->cmd = NULL;
1578
1579         return 0;
1580 }
1581
1582 static int psp_suspend(void *handle)
1583 {
1584         int ret;
1585         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1586         struct psp_context *psp = &adev->psp;
1587
1588         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1589             psp->xgmi_context.initialized == 1) {
1590                 ret = psp_xgmi_terminate(psp);
1591                 if (ret) {
1592                         DRM_ERROR("Failed to terminate xgmi ta\n");
1593                         return ret;
1594                 }
1595         }
1596
1597         if (psp->adev->psp.ta_fw) {
1598                 ret = psp_ras_terminate(psp);
1599                 if (ret) {
1600                         DRM_ERROR("Failed to terminate ras ta\n");
1601                         return ret;
1602                 }
1603                 ret = psp_hdcp_terminate(psp);
1604                 if (ret) {
1605                         DRM_ERROR("Failed to terminate hdcp ta\n");
1606                         return ret;
1607                 }
1608                 ret = psp_dtm_terminate(psp);
1609                 if (ret) {
1610                         DRM_ERROR("Failed to terminate dtm ta\n");
1611                         return ret;
1612                 }
1613         }
1614
1615         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1616         if (ret) {
1617                 DRM_ERROR("PSP ring stop failed\n");
1618                 return ret;
1619         }
1620
1621         return 0;
1622 }
1623
1624 static int psp_resume(void *handle)
1625 {
1626         int ret;
1627         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1628         struct psp_context *psp = &adev->psp;
1629
1630         DRM_INFO("PSP is resuming...\n");
1631
1632         ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
1633         if (ret) {
1634                 DRM_ERROR("Failed to process memory training!\n");
1635                 return ret;
1636         }
1637
1638         mutex_lock(&adev->firmware.mutex);
1639
1640         ret = psp_hw_start(psp);
1641         if (ret)
1642                 goto failed;
1643
1644         ret = psp_np_fw_load(psp);
1645         if (ret)
1646                 goto failed;
1647
1648         mutex_unlock(&adev->firmware.mutex);
1649
1650         return 0;
1651
1652 failed:
1653         DRM_ERROR("PSP resume failed\n");
1654         mutex_unlock(&adev->firmware.mutex);
1655         return ret;
1656 }
1657
1658 int psp_gpu_reset(struct amdgpu_device *adev)
1659 {
1660         int ret;
1661
1662         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1663                 return 0;
1664
1665         mutex_lock(&adev->psp.mutex);
1666         ret = psp_mode1_reset(&adev->psp);
1667         mutex_unlock(&adev->psp.mutex);
1668
1669         return ret;
1670 }
1671
1672 int psp_rlc_autoload_start(struct psp_context *psp)
1673 {
1674         int ret;
1675         struct psp_gfx_cmd_resp *cmd;
1676
1677         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1678         if (!cmd)
1679                 return -ENOMEM;
1680
1681         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1682
1683         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1684                                  psp->fence_buf_mc_addr);
1685         kfree(cmd);
1686         return ret;
1687 }
1688
1689 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1690                         uint64_t cmd_gpu_addr, int cmd_size)
1691 {
1692         struct amdgpu_firmware_info ucode = {0};
1693
1694         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1695                 AMDGPU_UCODE_ID_VCN0_RAM;
1696         ucode.mc_addr = cmd_gpu_addr;
1697         ucode.ucode_size = cmd_size;
1698
1699         return psp_execute_np_fw_load(&adev->psp, &ucode);
1700 }
1701
1702 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1703                                         enum AMDGPU_UCODE_ID ucode_type)
1704 {
1705         struct amdgpu_firmware_info *ucode = NULL;
1706
1707         if (!adev->firmware.fw_size)
1708                 return false;
1709
1710         ucode = &adev->firmware.ucode[ucode_type];
1711         if (!ucode->fw || !ucode->ucode_size)
1712                 return false;
1713
1714         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1715 }
1716
1717 static int psp_set_clockgating_state(void *handle,
1718                                      enum amd_clockgating_state state)
1719 {
1720         return 0;
1721 }
1722
1723 static int psp_set_powergating_state(void *handle,
1724                                      enum amd_powergating_state state)
1725 {
1726         return 0;
1727 }
1728
1729 const struct amd_ip_funcs psp_ip_funcs = {
1730         .name = "psp",
1731         .early_init = psp_early_init,
1732         .late_init = NULL,
1733         .sw_init = psp_sw_init,
1734         .sw_fini = psp_sw_fini,
1735         .hw_init = psp_hw_init,
1736         .hw_fini = psp_hw_fini,
1737         .suspend = psp_suspend,
1738         .resume = psp_resume,
1739         .is_idle = NULL,
1740         .check_soft_reset = NULL,
1741         .wait_for_idle = NULL,
1742         .soft_reset = NULL,
1743         .set_clockgating_state = psp_set_clockgating_state,
1744         .set_powergating_state = psp_set_powergating_state,
1745 };
1746
1747 static const struct amdgpu_psp_funcs psp_funcs = {
1748         .check_fw_loading_status = psp_check_fw_loading_status,
1749 };
1750
1751 static void psp_set_funcs(struct amdgpu_device *adev)
1752 {
1753         if (NULL == adev->firmware.funcs)
1754                 adev->firmware.funcs = &psp_funcs;
1755 }
1756
1757 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1758 {
1759         .type = AMD_IP_BLOCK_TYPE_PSP,
1760         .major = 3,
1761         .minor = 1,
1762         .rev = 0,
1763         .funcs = &psp_ip_funcs,
1764 };
1765
1766 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1767 {
1768         .type = AMD_IP_BLOCK_TYPE_PSP,
1769         .major = 10,
1770         .minor = 0,
1771         .rev = 0,
1772         .funcs = &psp_ip_funcs,
1773 };
1774
1775 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1776 {
1777         .type = AMD_IP_BLOCK_TYPE_PSP,
1778         .major = 11,
1779         .minor = 0,
1780         .rev = 0,
1781         .funcs = &psp_ip_funcs,
1782 };
1783
1784 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1785 {
1786         .type = AMD_IP_BLOCK_TYPE_PSP,
1787         .major = 12,
1788         .minor = 0,
1789         .rev = 0,
1790         .funcs = &psp_ip_funcs,
1791 };