drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drmP.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35
  36 static void psp_set_funcs(struct amdgpu_device *adev);
  37
  38 static int psp_early_init(void *handle)
  39 {
  40         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  41         struct psp_context *psp = &adev->psp;
  42
  43         psp_set_funcs(adev);
  44
  45         switch (adev->asic_type) {
  46         case CHIP_VEGA10:
  47         case CHIP_VEGA12:
  48                 psp_v3_1_set_psp_funcs(psp);
  49                 break;
  50         case CHIP_RAVEN:
  51                 psp_v10_0_set_psp_funcs(psp);
  52                 break;
  53         case CHIP_VEGA20:
  54                 psp_v11_0_set_psp_funcs(psp);
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         psp->adev = adev;
  61
  62         return 0;
  63 }
  64
  65 static int psp_sw_init(void *handle)
  66 {
  67         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  68         struct psp_context *psp = &adev->psp;
  69         int ret;
  70
  71         ret = psp_init_microcode(psp);
  72         if (ret) {
  73                 DRM_ERROR("Failed to load psp firmware!\n");
  74                 return ret;
  75         }
  76
  77         return 0;
  78 }
  79
  80 static int psp_sw_fini(void *handle)
  81 {
  82         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  83
  84         release_firmware(adev->psp.sos_fw);
  85         adev->psp.sos_fw = NULL;
  86         release_firmware(adev->psp.asd_fw);
  87         adev->psp.asd_fw = NULL;
  88         if (adev->psp.ta_fw) {
  89                 release_firmware(adev->psp.ta_fw);
  90                 adev->psp.ta_fw = NULL;
  91         }
  92         return 0;
  93 }
  94
  95 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
  96                  uint32_t reg_val, uint32_t mask, bool check_changed)
  97 {
  98         uint32_t val;
  99         int i;
 100         struct amdgpu_device *adev = psp->adev;
 101
 102         for (i = 0; i < adev->usec_timeout; i++) {
 103                 val = RREG32(reg_index);
 104                 if (check_changed) {
 105                         if (val != reg_val)
 106                                 return 0;
 107                 } else {
 108                         if ((val & mask) == reg_val)
 109                                 return 0;
 110                 }
 111                 udelay(1);
 112         }
 113
 114         return -ETIME;
 115 }
 116
 117 static int
 118 psp_cmd_submit_buf(struct psp_context *psp,
 119                    struct amdgpu_firmware_info *ucode,
 120                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 121 {
 122         int ret;
 123         int index;
 124         int timeout = 2000;
 125
 126         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 127
 128         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 129
 130         index = atomic_inc_return(&psp->fence_value);
 131         ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr,
 132                              fence_mc_addr, index);
 133         if (ret) {
 134                 atomic_dec(&psp->fence_value);
 135                 return ret;
 136         }
 137
 138         while (*((unsigned int *)psp->fence_buf) != index) {
 139                 if (--timeout == 0)
 140                         break;
 141                 msleep(1);
 142         }
 143
 144         /* In some cases, psp response status is not 0 even there is no
 145          * problem while the command is submitted. Some version of PSP FW
 146          * doesn't write 0 to that field.
 147          * So here we would like to only print a warning instead of an error
 148          * during psp initialization to avoid breaking hw_init and it doesn't
 149          * return -EINVAL.
 150          */
 151         if (psp->cmd_buf_mem->resp.status || !timeout) {
 152                 if (ucode)
 153                         DRM_WARN("failed to load ucode id (%d) ",
 154                                   ucode->ucode_id);
 155                 DRM_WARN("psp command failed and response status is (%d)\n",
 156                           psp->cmd_buf_mem->resp.status);
 157                 if (!timeout)
 158                         return -EINVAL;
 159         }
 160
 161         /* get xGMI session id from response buffer */
 162         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 163
 164         if (ucode) {
 165                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 166                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 167         }
 168
 169         return ret;
 170 }
 171
 172 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 173                                  struct psp_gfx_cmd_resp *cmd,
 174                                  uint64_t tmr_mc, uint32_t size)
 175 {
 176         if (psp_support_vmr_ring(psp))
 177                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 178         else
 179                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 180         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 181         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 182         cmd->cmd.cmd_setup_tmr.buf_size = size;
 183 }
 184
 185 /* Set up Trusted Memory Region */
 186 static int psp_tmr_init(struct psp_context *psp)
 187 {
 188         int ret;
 189
 190         /*
 191          * According to HW engineer, they prefer the TMR address be "naturally
 192          * aligned" , e.g. the start address be an integer divide of TMR size.
 193          *
 194          * Note: this memory need be reserved till the driver
 195          * uninitializes.
 196          */
 197         ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
 198                                       AMDGPU_GEM_DOMAIN_VRAM,
 199                                       &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 200
 201         return ret;
 202 }
 203
 204 static int psp_tmr_load(struct psp_context *psp)
 205 {
 206         int ret;
 207         struct psp_gfx_cmd_resp *cmd;
 208
 209         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 210         if (!cmd)
 211                 return -ENOMEM;
 212
 213         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, PSP_TMR_SIZE);
 214         DRM_INFO("reserve 0x%x from 0x%llx for PSP TMR SIZE\n",
 215                         PSP_TMR_SIZE, psp->tmr_mc_addr);
 216
 217         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 218                                  psp->fence_buf_mc_addr);
 219         if (ret)
 220                 goto failed;
 221
 222         kfree(cmd);
 223
 224         return 0;
 225
 226 failed:
 227         kfree(cmd);
 228         return ret;
 229 }
 230
 231 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 232                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 233                                  uint32_t size, uint32_t shared_size)
 234 {
 235         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 236         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 237         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 238         cmd->cmd.cmd_load_ta.app_len = size;
 239
 240         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 241         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 242         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 243 }
 244
 245 static int psp_asd_init(struct psp_context *psp)
 246 {
 247         int ret;
 248
 249         /*
 250          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 251          * physical) for shared ASD <-> Driver
 252          */
 253         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 254                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 255                                       &psp->asd_shared_bo,
 256                                       &psp->asd_shared_mc_addr,
 257                                       &psp->asd_shared_buf);
 258
 259         return ret;
 260 }
 261
 262 static int psp_asd_load(struct psp_context *psp)
 263 {
 264         int ret;
 265         struct psp_gfx_cmd_resp *cmd;
 266
 267         /* If PSP version doesn't match ASD version, asd loading will be failed.
 268          * add workaround to bypass it for sriov now.
 269          * TODO: add version check to make it common
 270          */
 271         if (amdgpu_sriov_vf(psp->adev))
 272                 return 0;
 273
 274         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 275         if (!cmd)
 276                 return -ENOMEM;
 277
 278         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 279         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 280
 281         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 282                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 283
 284         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 285                                  psp->fence_buf_mc_addr);
 286
 287         kfree(cmd);
 288
 289         return ret;
 290 }
 291
 292 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 293                 uint32_t id, uint32_t value)
 294 {
 295         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 296         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 297         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 298 }
 299
 300 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 301                 uint32_t value)
 302 {
 303         struct psp_gfx_cmd_resp *cmd = NULL;
 304         int ret = 0;
 305
 306         if (reg >= PSP_REG_LAST)
 307                 return -EINVAL;
 308
 309         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 310         if (!cmd)
 311                 return -ENOMEM;
 312
 313         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 314         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 315
 316         kfree(cmd);
 317         return ret;
 318 }
 319
 320 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 321                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 322                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 323 {
 324         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 325         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 326         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 327         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 328
 329         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 330         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 331         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 332 }
 333
 334 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 335 {
 336         int ret;
 337
 338         /*
 339          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 340          * physical) for xgmi ta <-> Driver
 341          */
 342         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 343                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 344                                       &psp->xgmi_context.xgmi_shared_bo,
 345                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 346                                       &psp->xgmi_context.xgmi_shared_buf);
 347
 348         return ret;
 349 }
 350
 351 static int psp_xgmi_load(struct psp_context *psp)
 352 {
 353         int ret;
 354         struct psp_gfx_cmd_resp *cmd;
 355
 356         /*
 357          * TODO: bypass the loading in sriov for now
 358          */
 359         if (amdgpu_sriov_vf(psp->adev))
 360                 return 0;
 361
 362         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 363         if (!cmd)
 364                 return -ENOMEM;
 365
 366         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 367         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 368
 369         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 370                                       psp->xgmi_context.xgmi_shared_mc_addr,
 371                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 372
 373         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 374                                  psp->fence_buf_mc_addr);
 375
 376         if (!ret) {
 377                 psp->xgmi_context.initialized = 1;
 378                 psp->xgmi_context.session_id = cmd->resp.session_id;
 379         }
 380
 381         kfree(cmd);
 382
 383         return ret;
 384 }
 385
 386 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 387                                             uint32_t xgmi_session_id)
 388 {
 389         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 390         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 391 }
 392
 393 static int psp_xgmi_unload(struct psp_context *psp)
 394 {
 395         int ret;
 396         struct psp_gfx_cmd_resp *cmd;
 397
 398         /*
 399          * TODO: bypass the unloading in sriov for now
 400          */
 401         if (amdgpu_sriov_vf(psp->adev))
 402                 return 0;
 403
 404         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 405         if (!cmd)
 406                 return -ENOMEM;
 407
 408         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 409
 410         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 411                                  psp->fence_buf_mc_addr);
 412
 413         kfree(cmd);
 414
 415         return ret;
 416 }
 417
 418 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 419                                             uint32_t ta_cmd_id,
 420                                             uint32_t xgmi_session_id)
 421 {
 422         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 423         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 424         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 425         /* Note: cmd_invoke_cmd.buf is not used for now */
 426 }
 427
 428 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 429 {
 430         int ret;
 431         struct psp_gfx_cmd_resp *cmd;
 432
 433         /*
 434          * TODO: bypass the loading in sriov for now
 435         */
 436         if (amdgpu_sriov_vf(psp->adev))
 437                 return 0;
 438
 439         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 440         if (!cmd)
 441                 return -ENOMEM;
 442
 443         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 444                                         psp->xgmi_context.session_id);
 445
 446         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 447                                  psp->fence_buf_mc_addr);
 448
 449         kfree(cmd);
 450
 451         return ret;
 452 }
 453
 454 static int psp_xgmi_terminate(struct psp_context *psp)
 455 {
 456         int ret;
 457
 458         if (!psp->xgmi_context.initialized)
 459                 return 0;
 460
 461         ret = psp_xgmi_unload(psp);
 462         if (ret)
 463                 return ret;
 464
 465         psp->xgmi_context.initialized = 0;
 466
 467         /* free xgmi shared memory */
 468         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 469                         &psp->xgmi_context.xgmi_shared_mc_addr,
 470                         &psp->xgmi_context.xgmi_shared_buf);
 471
 472         return 0;
 473 }
 474
 475 static int psp_xgmi_initialize(struct psp_context *psp)
 476 {
 477         struct ta_xgmi_shared_memory *xgmi_cmd;
 478         int ret;
 479
 480         if (!psp->adev->psp.ta_fw)
 481                 return -ENOENT;
 482
 483         if (!psp->xgmi_context.initialized) {
 484                 ret = psp_xgmi_init_shared_buf(psp);
 485                 if (ret)
 486                         return ret;
 487         }
 488
 489         /* Load XGMI TA */
 490         ret = psp_xgmi_load(psp);
 491         if (ret)
 492                 return ret;
 493
 494         /* Initialize XGMI session */
 495         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 496         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 497         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 498
 499         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 500
 501         return ret;
 502 }
 503
 504 // ras begin
 505 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 506                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 507                 uint32_t ras_ta_size, uint32_t shared_size)
 508 {
 509         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 510         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 511         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 512         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 513
 514         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 515         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 516         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 517 }
 518
 519 static int psp_ras_init_shared_buf(struct psp_context *psp)
 520 {
 521         int ret;
 522
 523         /*
 524          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 525          * physical) for ras ta <-> Driver
 526          */
 527         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 528                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 529                         &psp->ras.ras_shared_bo,
 530                         &psp->ras.ras_shared_mc_addr,
 531                         &psp->ras.ras_shared_buf);
 532
 533         return ret;
 534 }
 535
 536 static int psp_ras_load(struct psp_context *psp)
 537 {
 538         int ret;
 539         struct psp_gfx_cmd_resp *cmd;
 540
 541         /*
 542          * TODO: bypass the loading in sriov for now
 543          */
 544         if (amdgpu_sriov_vf(psp->adev))
 545                 return 0;
 546
 547         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 548         if (!cmd)
 549                 return -ENOMEM;
 550
 551         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 552         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 553
 554         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 555                         psp->ras.ras_shared_mc_addr,
 556                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 557
 558         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 559                         psp->fence_buf_mc_addr);
 560
 561         if (!ret) {
 562                 psp->ras.ras_initialized = 1;
 563                 psp->ras.session_id = cmd->resp.session_id;
 564         }
 565
 566         kfree(cmd);
 567
 568         return ret;
 569 }
 570
 571 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 572                                                 uint32_t ras_session_id)
 573 {
 574         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 575         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 576 }
 577
 578 static int psp_ras_unload(struct psp_context *psp)
 579 {
 580         int ret;
 581         struct psp_gfx_cmd_resp *cmd;
 582
 583         /*
 584          * TODO: bypass the unloading in sriov for now
 585          */
 586         if (amdgpu_sriov_vf(psp->adev))
 587                 return 0;
 588
 589         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 590         if (!cmd)
 591                 return -ENOMEM;
 592
 593         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 594
 595         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 596                         psp->fence_buf_mc_addr);
 597
 598         kfree(cmd);
 599
 600         return ret;
 601 }
 602
 603 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 604                 uint32_t ta_cmd_id,
 605                 uint32_t ras_session_id)
 606 {
 607         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 608         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 609         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 610         /* Note: cmd_invoke_cmd.buf is not used for now */
 611 }
 612
 613 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 614 {
 615         int ret;
 616         struct psp_gfx_cmd_resp *cmd;
 617
 618         /*
 619          * TODO: bypass the loading in sriov for now
 620          */
 621         if (amdgpu_sriov_vf(psp->adev))
 622                 return 0;
 623
 624         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 625         if (!cmd)
 626                 return -ENOMEM;
 627
 628         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 629                         psp->ras.session_id);
 630
 631         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 632                         psp->fence_buf_mc_addr);
 633
 634         kfree(cmd);
 635
 636         return ret;
 637 }
 638
 639 int psp_ras_enable_features(struct psp_context *psp,
 640                 union ta_ras_cmd_input *info, bool enable)
 641 {
 642         struct ta_ras_shared_memory *ras_cmd;
 643         int ret;
 644
 645         if (!psp->ras.ras_initialized)
 646                 return -EINVAL;
 647
 648         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 649         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 650
 651         if (enable)
 652                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 653         else
 654                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 655
 656         ras_cmd->ras_in_message = *info;
 657
 658         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 659         if (ret)
 660                 return -EINVAL;
 661
 662         return ras_cmd->ras_status;
 663 }
 664
 665 static int psp_ras_terminate(struct psp_context *psp)
 666 {
 667         int ret;
 668
 669         if (!psp->ras.ras_initialized)
 670                 return 0;
 671
 672         ret = psp_ras_unload(psp);
 673         if (ret)
 674                 return ret;
 675
 676         psp->ras.ras_initialized = 0;
 677
 678         /* free ras shared memory */
 679         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 680                         &psp->ras.ras_shared_mc_addr,
 681                         &psp->ras.ras_shared_buf);
 682
 683         return 0;
 684 }
 685
 686 static int psp_ras_initialize(struct psp_context *psp)
 687 {
 688         int ret;
 689
 690         if (!psp->ras.ras_initialized) {
 691                 ret = psp_ras_init_shared_buf(psp);
 692                 if (ret)
 693                         return ret;
 694         }
 695
 696         ret = psp_ras_load(psp);
 697         if (ret)
 698                 return ret;
 699
 700         return 0;
 701 }
 702 // ras end
 703
 704 static int psp_hw_start(struct psp_context *psp)
 705 {
 706         struct amdgpu_device *adev = psp->adev;
 707         int ret;
 708
 709         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 710                 ret = psp_bootloader_load_sysdrv(psp);
 711                 if (ret) {
 712                         DRM_ERROR("PSP load sysdrv failed!\n");
 713                         return ret;
 714                 }
 715
 716                 ret = psp_bootloader_load_sos(psp);
 717                 if (ret) {
 718                         DRM_ERROR("PSP load sos failed!\n");
 719                         return ret;
 720                 }
 721         }
 722
 723         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 724         if (ret) {
 725                 DRM_ERROR("PSP create ring failed!\n");
 726                 return ret;
 727         }
 728
 729         ret = psp_tmr_load(psp);
 730         if (ret) {
 731                 DRM_ERROR("PSP load tmr failed!\n");
 732                 return ret;
 733         }
 734
 735         ret = psp_asd_load(psp);
 736         if (ret) {
 737                 DRM_ERROR("PSP load asd failed!\n");
 738                 return ret;
 739         }
 740
 741         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 742                 ret = psp_xgmi_initialize(psp);
 743                 /* Warning the XGMI seesion initialize failure
 744                  * Instead of stop driver initialization
 745                  */
 746                 if (ret)
 747                         dev_err(psp->adev->dev,
 748                                 "XGMI: Failed to initialize XGMI session\n");
 749         }
 750
 751
 752         if (psp->adev->psp.ta_fw) {
 753                 ret = psp_ras_initialize(psp);
 754                 if (ret)
 755                         dev_err(psp->adev->dev,
 756                                         "RAS: Failed to initialize RAS\n");
 757         }
 758
 759         return 0;
 760 }
 761
 762 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 763                            enum psp_gfx_fw_type *type)
 764 {
 765         switch (ucode->ucode_id) {
 766         case AMDGPU_UCODE_ID_SDMA0:
 767                 *type = GFX_FW_TYPE_SDMA0;
 768                 break;
 769         case AMDGPU_UCODE_ID_SDMA1:
 770                 *type = GFX_FW_TYPE_SDMA1;
 771                 break;
 772         case AMDGPU_UCODE_ID_CP_CE:
 773                 *type = GFX_FW_TYPE_CP_CE;
 774                 break;
 775         case AMDGPU_UCODE_ID_CP_PFP:
 776                 *type = GFX_FW_TYPE_CP_PFP;
 777                 break;
 778         case AMDGPU_UCODE_ID_CP_ME:
 779                 *type = GFX_FW_TYPE_CP_ME;
 780                 break;
 781         case AMDGPU_UCODE_ID_CP_MEC1:
 782                 *type = GFX_FW_TYPE_CP_MEC;
 783                 break;
 784         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 785                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 786                 break;
 787         case AMDGPU_UCODE_ID_CP_MEC2:
 788                 *type = GFX_FW_TYPE_CP_MEC;
 789                 break;
 790         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 791                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 792                 break;
 793         case AMDGPU_UCODE_ID_RLC_G:
 794                 *type = GFX_FW_TYPE_RLC_G;
 795                 break;
 796         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 797                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 798                 break;
 799         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 800                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 801                 break;
 802         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 803                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 804                 break;
 805         case AMDGPU_UCODE_ID_SMC:
 806                 *type = GFX_FW_TYPE_SMU;
 807                 break;
 808         case AMDGPU_UCODE_ID_UVD:
 809                 *type = GFX_FW_TYPE_UVD;
 810                 break;
 811         case AMDGPU_UCODE_ID_UVD1:
 812                 *type = GFX_FW_TYPE_UVD1;
 813                 break;
 814         case AMDGPU_UCODE_ID_VCE:
 815                 *type = GFX_FW_TYPE_VCE;
 816                 break;
 817         case AMDGPU_UCODE_ID_VCN:
 818                 *type = GFX_FW_TYPE_VCN;
 819                 break;
 820         case AMDGPU_UCODE_ID_DMCU_ERAM:
 821                 *type = GFX_FW_TYPE_DMCU_ERAM;
 822                 break;
 823         case AMDGPU_UCODE_ID_DMCU_INTV:
 824                 *type = GFX_FW_TYPE_DMCU_ISR;
 825                 break;
 826         case AMDGPU_UCODE_ID_MAXIMUM:
 827         default:
 828                 return -EINVAL;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
 835                                        struct psp_gfx_cmd_resp *cmd)
 836 {
 837         int ret;
 838         uint64_t fw_mem_mc_addr = ucode->mc_addr;
 839
 840         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 841
 842         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 843         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
 844         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
 845         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 846
 847         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
 848         if (ret)
 849                 DRM_ERROR("Unknown firmware type\n");
 850
 851         return ret;
 852 }
 853
 854 static int psp_np_fw_load(struct psp_context *psp)
 855 {
 856         int i, ret;
 857         struct amdgpu_firmware_info *ucode;
 858         struct amdgpu_device* adev = psp->adev;
 859
 860         for (i = 0; i < adev->firmware.max_ucodes; i++) {
 861                 ucode = &adev->firmware.ucode[i];
 862                 if (!ucode->fw)
 863                         continue;
 864
 865                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
 866                     psp_smu_reload_quirk(psp))
 867                         continue;
 868                 if (amdgpu_sriov_vf(adev) &&
 869                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
 870                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
 871                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
 872                         /*skip ucode loading in SRIOV VF */
 873                         continue;
 874
 875                 ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
 876                 if (ret)
 877                         return ret;
 878
 879                 ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
 880                                          psp->fence_buf_mc_addr);
 881                 if (ret)
 882                         return ret;
 883
 884 #if 0
 885                 /* check if firmware loaded sucessfully */
 886                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
 887                         return -EINVAL;
 888 #endif
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int psp_load_fw(struct amdgpu_device *adev)
 895 {
 896         int ret;
 897         struct psp_context *psp = &adev->psp;
 898
 899         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
 900                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
 901                 goto skip_memalloc;
 902         }
 903
 904         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 905         if (!psp->cmd)
 906                 return -ENOMEM;
 907
 908         /* this fw pri bo is not used under SRIOV */
 909         if (!amdgpu_sriov_vf(psp->adev)) {
 910                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 911                                               AMDGPU_GEM_DOMAIN_GTT,
 912                                               &psp->fw_pri_bo,
 913                                               &psp->fw_pri_mc_addr,
 914                                               &psp->fw_pri_buf);
 915                 if (ret)
 916                         goto failed;
 917         }
 918
 919         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 920                                         AMDGPU_GEM_DOMAIN_VRAM,
 921                                         &psp->fence_buf_bo,
 922                                         &psp->fence_buf_mc_addr,
 923                                         &psp->fence_buf);
 924         if (ret)
 925                 goto failed;
 926
 927         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 928                                       AMDGPU_GEM_DOMAIN_VRAM,
 929                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 930                                       (void **)&psp->cmd_buf_mem);
 931         if (ret)
 932                 goto failed;
 933
 934         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
 935
 936         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
 937         if (ret) {
 938                 DRM_ERROR("PSP ring init failed!\n");
 939                 goto failed;
 940         }
 941
 942         ret = psp_tmr_init(psp);
 943         if (ret) {
 944                 DRM_ERROR("PSP tmr init failed!\n");
 945                 goto failed;
 946         }
 947
 948         ret = psp_asd_init(psp);
 949         if (ret) {
 950                 DRM_ERROR("PSP asd init failed!\n");
 951                 goto failed;
 952         }
 953
 954 skip_memalloc:
 955         ret = psp_hw_start(psp);
 956         if (ret)
 957                 goto failed;
 958
 959         ret = psp_np_fw_load(psp);
 960         if (ret)
 961                 goto failed;
 962
 963         return 0;
 964
 965 failed:
 966         /*
 967          * all cleanup jobs (xgmi terminate, ras terminate,
 968          * ring destroy, cmd/fence/fw buffers destory,
 969          * psp->cmd destory) are delayed to psp_hw_fini
 970          */
 971         return ret;
 972 }
 973
 974 static int psp_hw_init(void *handle)
 975 {
 976         int ret;
 977         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 978
 979         mutex_lock(&adev->firmware.mutex);
 980         /*
 981          * This sequence is just used on hw_init only once, no need on
 982          * resume.
 983          */
 984         ret = amdgpu_ucode_init_bo(adev);
 985         if (ret)
 986                 goto failed;
 987
 988         ret = psp_load_fw(adev);
 989         if (ret) {
 990                 DRM_ERROR("PSP firmware loading failed\n");
 991                 goto failed;
 992         }
 993
 994         mutex_unlock(&adev->firmware.mutex);
 995         return 0;
 996
 997 failed:
 998         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
 999         mutex_unlock(&adev->firmware.mutex);
1000         return -EINVAL;
1001 }
1002
1003 static int psp_hw_fini(void *handle)
1004 {
1005         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1006         struct psp_context *psp = &adev->psp;
1007
1008         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1009             psp->xgmi_context.initialized == 1)
1010                 psp_xgmi_terminate(psp);
1011
1012         if (psp->adev->psp.ta_fw)
1013                 psp_ras_terminate(psp);
1014
1015         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1016
1017         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
1018         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1019                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1020         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1021                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1022         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1023                               &psp->asd_shared_buf);
1024         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1025                               (void **)&psp->cmd_buf_mem);
1026
1027         kfree(psp->cmd);
1028         psp->cmd = NULL;
1029
1030         return 0;
1031 }
1032
1033 static int psp_suspend(void *handle)
1034 {
1035         int ret;
1036         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1037         struct psp_context *psp = &adev->psp;
1038
1039         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1040             psp->xgmi_context.initialized == 1) {
1041                 ret = psp_xgmi_terminate(psp);
1042                 if (ret) {
1043                         DRM_ERROR("Failed to terminate xgmi ta\n");
1044                         return ret;
1045                 }
1046         }
1047
1048         if (psp->adev->psp.ta_fw) {
1049                 ret = psp_ras_terminate(psp);
1050                 if (ret) {
1051                         DRM_ERROR("Failed to terminate ras ta\n");
1052                         return ret;
1053                 }
1054         }
1055
1056         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1057         if (ret) {
1058                 DRM_ERROR("PSP ring stop failed\n");
1059                 return ret;
1060         }
1061
1062         return 0;
1063 }
1064
1065 static int psp_resume(void *handle)
1066 {
1067         int ret;
1068         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1069         struct psp_context *psp = &adev->psp;
1070
1071         DRM_INFO("PSP is resuming...\n");
1072
1073         mutex_lock(&adev->firmware.mutex);
1074
1075         ret = psp_hw_start(psp);
1076         if (ret)
1077                 goto failed;
1078
1079         ret = psp_np_fw_load(psp);
1080         if (ret)
1081                 goto failed;
1082
1083         mutex_unlock(&adev->firmware.mutex);
1084
1085         return 0;
1086
1087 failed:
1088         DRM_ERROR("PSP resume failed\n");
1089         mutex_unlock(&adev->firmware.mutex);
1090         return ret;
1091 }
1092
1093 int psp_gpu_reset(struct amdgpu_device *adev)
1094 {
1095         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1096                 return 0;
1097
1098         return psp_mode1_reset(&adev->psp);
1099 }
1100
1101 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1102                                         enum AMDGPU_UCODE_ID ucode_type)
1103 {
1104         struct amdgpu_firmware_info *ucode = NULL;
1105
1106         if (!adev->firmware.fw_size)
1107                 return false;
1108
1109         ucode = &adev->firmware.ucode[ucode_type];
1110         if (!ucode->fw || !ucode->ucode_size)
1111                 return false;
1112
1113         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1114 }
1115
1116 static int psp_set_clockgating_state(void *handle,
1117                                      enum amd_clockgating_state state)
1118 {
1119         return 0;
1120 }
1121
1122 static int psp_set_powergating_state(void *handle,
1123                                      enum amd_powergating_state state)
1124 {
1125         return 0;
1126 }
1127
1128 const struct amd_ip_funcs psp_ip_funcs = {
1129         .name = "psp",
1130         .early_init = psp_early_init,
1131         .late_init = NULL,
1132         .sw_init = psp_sw_init,
1133         .sw_fini = psp_sw_fini,
1134         .hw_init = psp_hw_init,
1135         .hw_fini = psp_hw_fini,
1136         .suspend = psp_suspend,
1137         .resume = psp_resume,
1138         .is_idle = NULL,
1139         .check_soft_reset = NULL,
1140         .wait_for_idle = NULL,
1141         .soft_reset = NULL,
1142         .set_clockgating_state = psp_set_clockgating_state,
1143         .set_powergating_state = psp_set_powergating_state,
1144 };
1145
1146 static const struct amdgpu_psp_funcs psp_funcs = {
1147         .check_fw_loading_status = psp_check_fw_loading_status,
1148 };
1149
1150 static void psp_set_funcs(struct amdgpu_device *adev)
1151 {
1152         if (NULL == adev->firmware.funcs)
1153                 adev->firmware.funcs = &psp_funcs;
1154 }
1155
1156 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1157 {
1158         .type = AMD_IP_BLOCK_TYPE_PSP,
1159         .major = 3,
1160         .minor = 1,
1161         .rev = 0,
1162         .funcs = &psp_ip_funcs,
1163 };
1164
1165 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1166 {
1167         .type = AMD_IP_BLOCK_TYPE_PSP,
1168         .major = 10,
1169         .minor = 0,
1170         .rev = 0,
1171         .funcs = &psp_ip_funcs,
1172 };
1173
1174 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1175 {
1176         .type = AMD_IP_BLOCK_TYPE_PSP,
1177         .major = 11,
1178         .minor = 0,
1179         .rev = 0,
1180         .funcs = &psp_ip_funcs,
1181 };