drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c

   1 /*
   2  * Copyright 2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_vcn.h"
  27 #include "amdgpu_pm.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "vcn_v2_0.h"
  31 #include "mmsch_v3_0.h"
  32
  33 #include "vcn/vcn_3_0_0_offset.h"
  34 #include "vcn/vcn_3_0_0_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
  36
  37 #include <drm/drm_drv.h>
  38
  39 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET                        0x27
  40 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET                    0x0f
  41 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET                  0x10
  42 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET                  0x11
  43 #define mmUVD_NO_OP_INTERNAL_OFFSET                             0x29
  44 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET                       0x66
  45 #define mmUVD_SCRATCH9_INTERNAL_OFFSET                          0xc01d
  46
  47 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET                   0x431
  48 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET          0x3b4
  49 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET         0x3b5
  50 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET                       0x25c
  51
  52 #define VCN_INSTANCES_SIENNA_CICHLID                            2
  53 #define DEC_SW_RING_ENABLED                                     FALSE
  54
  55 #define RDECODE_MSG_CREATE                                      0x00000000
  56 #define RDECODE_MESSAGE_CREATE                                  0x00000001
  57
  58 static int amdgpu_ih_clientid_vcns[] = {
  59         SOC15_IH_CLIENTID_VCN,
  60         SOC15_IH_CLIENTID_VCN1
  61 };
  62
  63 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
  64 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  65 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  66 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
  67 static int vcn_v3_0_set_powergating_state(void *handle,
  68                         enum amd_powergating_state state);
  69 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
  70                         int inst_idx, struct dpg_pause_state *new_state);
  71
  72 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
  73 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
  74
  75 /**
  76  * vcn_v3_0_early_init - set function pointers
  77  *
  78  * @handle: amdgpu_device pointer
  79  *
  80  * Set ring and irq function pointers
  81  */
  82 static int vcn_v3_0_early_init(void *handle)
  83 {
  84         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  85
  86         if (amdgpu_sriov_vf(adev)) {
  87                 adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
  88                 adev->vcn.harvest_config = 0;
  89                 adev->vcn.num_enc_rings = 1;
  90
  91         } else {
  92                 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
  93                                                  AMDGPU_VCN_HARVEST_VCN1))
  94                         /* both instances are harvested, disable the block */
  95                         return -ENOENT;
  96
  97                 if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 0, 33))
  98                         adev->vcn.num_enc_rings = 0;
  99                 else
 100                         adev->vcn.num_enc_rings = 2;
 101         }
 102
 103         vcn_v3_0_set_dec_ring_funcs(adev);
 104         vcn_v3_0_set_enc_ring_funcs(adev);
 105         vcn_v3_0_set_irq_funcs(adev);
 106
 107         return 0;
 108 }
 109
 110 /**
 111  * vcn_v3_0_sw_init - sw init for VCN block
 112  *
 113  * @handle: amdgpu_device pointer
 114  *
 115  * Load firmware and sw initialization
 116  */
 117 static int vcn_v3_0_sw_init(void *handle)
 118 {
 119         struct amdgpu_ring *ring;
 120         int i, j, r;
 121         int vcn_doorbell_index = 0;
 122         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 123
 124         r = amdgpu_vcn_sw_init(adev);
 125         if (r)
 126                 return r;
 127
 128         amdgpu_vcn_setup_ucode(adev);
 129
 130         r = amdgpu_vcn_resume(adev);
 131         if (r)
 132                 return r;
 133
 134         /*
 135          * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
 136          * Formula:
 137          *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
 138          *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
 139          *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
 140          */
 141         if (amdgpu_sriov_vf(adev)) {
 142                 vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
 143                 /* get DWORD offset */
 144                 vcn_doorbell_index = vcn_doorbell_index << 1;
 145         }
 146
 147         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 148                 volatile struct amdgpu_fw_shared *fw_shared;
 149
 150                 if (adev->vcn.harvest_config & (1 << i))
 151                         continue;
 152
 153                 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
 154                 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
 155                 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
 156                 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
 157                 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
 158                 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
 159
 160                 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
 161                 adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
 162                 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
 163                 adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
 164                 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
 165                 adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
 166                 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
 167                 adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
 168                 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
 169                 adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
 170
 171                 /* VCN DEC TRAP */
 172                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 173                                 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
 174                 if (r)
 175                         return r;
 176
 177                 atomic_set(&adev->vcn.inst[i].sched_score, 0);
 178
 179                 ring = &adev->vcn.inst[i].ring_dec;
 180                 ring->use_doorbell = true;
 181                 if (amdgpu_sriov_vf(adev)) {
 182                         ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
 183                 } else {
 184                         ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
 185                 }
 186                 sprintf(ring->name, "vcn_dec_%d", i);
 187                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 188                                      AMDGPU_RING_PRIO_DEFAULT,
 189                                      &adev->vcn.inst[i].sched_score);
 190                 if (r)
 191                         return r;
 192
 193                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
 194                         enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
 195
 196                         /* VCN ENC TRAP */
 197                         r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 198                                 j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
 199                         if (r)
 200                                 return r;
 201
 202                         ring = &adev->vcn.inst[i].ring_enc[j];
 203                         ring->use_doorbell = true;
 204                         if (amdgpu_sriov_vf(adev)) {
 205                                 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
 206                         } else {
 207                                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
 208                         }
 209                         sprintf(ring->name, "vcn_enc_%d.%d", i, j);
 210                         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 211                                              hw_prio, &adev->vcn.inst[i].sched_score);
 212                         if (r)
 213                                 return r;
 214                 }
 215
 216                 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
 217                 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
 218                                              cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
 219                                              cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
 220                 fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
 221         }
 222
 223         if (amdgpu_sriov_vf(adev)) {
 224                 r = amdgpu_virt_alloc_mm_table(adev);
 225                 if (r)
 226                         return r;
 227         }
 228         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 229                 adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
 230
 231         return 0;
 232 }
 233
 234 /**
 235  * vcn_v3_0_sw_fini - sw fini for VCN block
 236  *
 237  * @handle: amdgpu_device pointer
 238  *
 239  * VCN suspend and free up sw allocation
 240  */
 241 static int vcn_v3_0_sw_fini(void *handle)
 242 {
 243         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 244         int i, r, idx;
 245
 246         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 247                 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 248                         volatile struct amdgpu_fw_shared *fw_shared;
 249
 250                         if (adev->vcn.harvest_config & (1 << i))
 251                                 continue;
 252                         fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
 253                         fw_shared->present_flag_0 = 0;
 254                         fw_shared->sw_ring.is_enabled = false;
 255                 }
 256
 257                 drm_dev_exit(idx);
 258         }
 259
 260         if (amdgpu_sriov_vf(adev))
 261                 amdgpu_virt_free_mm_table(adev);
 262
 263         r = amdgpu_vcn_suspend(adev);
 264         if (r)
 265                 return r;
 266
 267         r = amdgpu_vcn_sw_fini(adev);
 268
 269         return r;
 270 }
 271
 272 /**
 273  * vcn_v3_0_hw_init - start and test VCN block
 274  *
 275  * @handle: amdgpu_device pointer
 276  *
 277  * Initialize the hardware, boot up the VCPU and do some testing
 278  */
 279 static int vcn_v3_0_hw_init(void *handle)
 280 {
 281         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 282         struct amdgpu_ring *ring;
 283         int i, j, r;
 284
 285         if (amdgpu_sriov_vf(adev)) {
 286                 r = vcn_v3_0_start_sriov(adev);
 287                 if (r)
 288                         goto done;
 289
 290                 /* initialize VCN dec and enc ring buffers */
 291                 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 292                         if (adev->vcn.harvest_config & (1 << i))
 293                                 continue;
 294
 295                         ring = &adev->vcn.inst[i].ring_dec;
 296                         if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {
 297                                 ring->sched.ready = false;
 298                                 dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
 299                         } else {
 300                                 ring->wptr = 0;
 301                                 ring->wptr_old = 0;
 302                                 vcn_v3_0_dec_ring_set_wptr(ring);
 303                                 ring->sched.ready = true;
 304                         }
 305
 306                         for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
 307                                 ring = &adev->vcn.inst[i].ring_enc[j];
 308                                 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
 309                                         ring->sched.ready = false;
 310                                         dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
 311                                 } else {
 312                                         ring->wptr = 0;
 313                                         ring->wptr_old = 0;
 314                                         vcn_v3_0_enc_ring_set_wptr(ring);
 315                                         ring->sched.ready = true;
 316                                 }
 317                         }
 318                 }
 319         } else {
 320                 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 321                         if (adev->vcn.harvest_config & (1 << i))
 322                                 continue;
 323
 324                         ring = &adev->vcn.inst[i].ring_dec;
 325
 326                         adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 327                                                      ring->doorbell_index, i);
 328
 329                         r = amdgpu_ring_test_helper(ring);
 330                         if (r)
 331                                 goto done;
 332
 333                         for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
 334                                 ring = &adev->vcn.inst[i].ring_enc[j];
 335                                 r = amdgpu_ring_test_helper(ring);
 336                                 if (r)
 337                                         goto done;
 338                         }
 339                 }
 340         }
 341
 342 done:
 343         if (!r)
 344                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 345                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 346
 347         return r;
 348 }
 349
 350 /**
 351  * vcn_v3_0_hw_fini - stop the hardware block
 352  *
 353  * @handle: amdgpu_device pointer
 354  *
 355  * Stop the VCN block, mark ring as not ready any more
 356  */
 357 static int vcn_v3_0_hw_fini(void *handle)
 358 {
 359         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 360         int i;
 361
 362         cancel_delayed_work_sync(&adev->vcn.idle_work);
 363
 364         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 365                 if (adev->vcn.harvest_config & (1 << i))
 366                         continue;
 367
 368                 if (!amdgpu_sriov_vf(adev)) {
 369                         if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 370                                         (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 371                                          RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
 372                                 vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 373                         }
 374                 }
 375         }
 376
 377         return 0;
 378 }
 379
 380 /**
 381  * vcn_v3_0_suspend - suspend VCN block
 382  *
 383  * @handle: amdgpu_device pointer
 384  *
 385  * HW fini and suspend VCN block
 386  */
 387 static int vcn_v3_0_suspend(void *handle)
 388 {
 389         int r;
 390         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 391
 392         r = vcn_v3_0_hw_fini(adev);
 393         if (r)
 394                 return r;
 395
 396         r = amdgpu_vcn_suspend(adev);
 397
 398         return r;
 399 }
 400
 401 /**
 402  * vcn_v3_0_resume - resume VCN block
 403  *
 404  * @handle: amdgpu_device pointer
 405  *
 406  * Resume firmware and hw init VCN block
 407  */
 408 static int vcn_v3_0_resume(void *handle)
 409 {
 410         int r;
 411         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 412
 413         r = amdgpu_vcn_resume(adev);
 414         if (r)
 415                 return r;
 416
 417         r = vcn_v3_0_hw_init(adev);
 418
 419         return r;
 420 }
 421
 422 /**
 423  * vcn_v3_0_mc_resume - memory controller programming
 424  *
 425  * @adev: amdgpu_device pointer
 426  * @inst: instance number
 427  *
 428  * Let the VCN memory controller know it's offsets
 429  */
 430 static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
 431 {
 432         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 433         uint32_t offset;
 434
 435         /* cache window 0: fw */
 436         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 437                 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 438                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
 439                 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 440                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
 441                 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
 442                 offset = 0;
 443         } else {
 444                 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 445                         lower_32_bits(adev->vcn.inst[inst].gpu_addr));
 446                 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 447                         upper_32_bits(adev->vcn.inst[inst].gpu_addr));
 448                 offset = size;
 449                 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
 450                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 451         }
 452         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
 453
 454         /* cache window 1: stack */
 455         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 456                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 457         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 458                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 459         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
 460         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 461
 462         /* cache window 2: context */
 463         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 464                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 465         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 466                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 467         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
 468         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 469
 470         /* non-cache window */
 471         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 472                 lower_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
 473         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 474                 upper_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
 475         WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
 476         WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
 477                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
 478 }
 479
 480 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 481 {
 482         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 483         uint32_t offset;
 484
 485         /* cache window 0: fw */
 486         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 487                 if (!indirect) {
 488                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 489                                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 490                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
 491                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 492                                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 493                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
 494                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 495                                 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 496                 } else {
 497                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 498                                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 499                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 500                                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 501                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 502                                 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 503                 }
 504                 offset = 0;
 505         } else {
 506                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 507                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 508                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 509                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 510                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 511                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 512                 offset = size;
 513                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 514                         VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
 515                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 516         }
 517
 518         if (!indirect)
 519                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 520                         VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 521         else
 522                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 523                         VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 524
 525         /* cache window 1: stack */
 526         if (!indirect) {
 527                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 528                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 529                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 530                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 531                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 532                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 533                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 534                         VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 535         } else {
 536                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 537                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 538                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 539                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 540                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 541                         VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 542         }
 543         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 544                         VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 545
 546         /* cache window 2: context */
 547         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 548                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 549                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 550         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 551                         VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 552                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 553         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 554                         VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 555         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 556                         VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 557
 558         /* non-cache window */
 559         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 560                         VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 561                         lower_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
 562         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 563                         VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 564                         upper_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
 565         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 566                         VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 567         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 568                         VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
 569                         AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
 570
 571         /* VCN global tiling registers */
 572         WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
 573                 UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 574 }
 575
 576 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
 577 {
 578         uint32_t data = 0;
 579
 580         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 581                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 582                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 583                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 584                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 585                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 586                         | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
 587                         | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 588                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 589                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 590                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 591                         | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 592                         | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
 593                         | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 594                         | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 595
 596                 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
 597                 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
 598                         UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
 599         } else {
 600                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 601                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 602                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 603                         | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 604                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 605                         | 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
 606                         | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 607                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 608                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 609                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 610                         | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 611                         | 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
 612                         | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 613                         | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 614                 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
 615                 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);
 616         }
 617
 618         data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
 619         data &= ~0x103;
 620         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 621                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 622                         UVD_POWER_STATUS__UVD_PG_EN_MASK;
 623
 624         WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
 625 }
 626
 627 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
 628 {
 629         uint32_t data;
 630
 631         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 632                 /* Before power off, this indicator has to be turned on */
 633                 data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
 634                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 635                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 636                 WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
 637
 638                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 639                         | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 640                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 641                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 642                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 643                         | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
 644                         | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 645                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 646                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 647                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 648                         | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 649                         | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
 650                         | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 651                         | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 652                 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
 653
 654                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 655                         | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
 656                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 657                         | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
 658                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 659                         | 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
 660                         | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
 661                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 662                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 663                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
 664                         | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
 665                         | 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
 666                         | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
 667                         | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
 668                 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
 669         }
 670 }
 671
 672 /**
 673  * vcn_v3_0_disable_clock_gating - disable VCN clock gating
 674  *
 675  * @adev: amdgpu_device pointer
 676  * @inst: instance number
 677  *
 678  * Disable clock gating for VCN block
 679  */
 680 static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
 681 {
 682         uint32_t data;
 683
 684         /* VCN disable CGC */
 685         data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
 686         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 687                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 688         else
 689                 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 690         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 691         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 692         WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
 693
 694         data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
 695         data &= ~(UVD_CGC_GATE__SYS_MASK
 696                 | UVD_CGC_GATE__UDEC_MASK
 697                 | UVD_CGC_GATE__MPEG2_MASK
 698                 | UVD_CGC_GATE__REGS_MASK
 699                 | UVD_CGC_GATE__RBC_MASK
 700                 | UVD_CGC_GATE__LMI_MC_MASK
 701                 | UVD_CGC_GATE__LMI_UMC_MASK
 702                 | UVD_CGC_GATE__IDCT_MASK
 703                 | UVD_CGC_GATE__MPRD_MASK
 704                 | UVD_CGC_GATE__MPC_MASK
 705                 | UVD_CGC_GATE__LBSI_MASK
 706                 | UVD_CGC_GATE__LRBBM_MASK
 707                 | UVD_CGC_GATE__UDEC_RE_MASK
 708                 | UVD_CGC_GATE__UDEC_CM_MASK
 709                 | UVD_CGC_GATE__UDEC_IT_MASK
 710                 | UVD_CGC_GATE__UDEC_DB_MASK
 711                 | UVD_CGC_GATE__UDEC_MP_MASK
 712                 | UVD_CGC_GATE__WCB_MASK
 713                 | UVD_CGC_GATE__VCPU_MASK
 714                 | UVD_CGC_GATE__MMSCH_MASK);
 715
 716         WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
 717
 718         SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);
 719
 720         data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
 721         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 722                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 723                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 724                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 725                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 726                 | UVD_CGC_CTRL__SYS_MODE_MASK
 727                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 728                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 729                 | UVD_CGC_CTRL__REGS_MODE_MASK
 730                 | UVD_CGC_CTRL__RBC_MODE_MASK
 731                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 732                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 733                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 734                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 735                 | UVD_CGC_CTRL__MPC_MODE_MASK
 736                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 737                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 738                 | UVD_CGC_CTRL__WCB_MODE_MASK
 739                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 740                 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
 741         WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
 742
 743         data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
 744         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 745                 | UVD_SUVD_CGC_GATE__SIT_MASK
 746                 | UVD_SUVD_CGC_GATE__SMP_MASK
 747                 | UVD_SUVD_CGC_GATE__SCM_MASK
 748                 | UVD_SUVD_CGC_GATE__SDB_MASK
 749                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 750                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 751                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 752                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 753                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 754                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 755                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 756                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 757                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 758                 | UVD_SUVD_CGC_GATE__ENT_MASK
 759                 | UVD_SUVD_CGC_GATE__IME_MASK
 760                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 761                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 762                 | UVD_SUVD_CGC_GATE__SITE_MASK
 763                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 764                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 765                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 766                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 767                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK
 768                 | UVD_SUVD_CGC_GATE__EFC_MASK
 769                 | UVD_SUVD_CGC_GATE__SAOE_MASK
 770                 | UVD_SUVD_CGC_GATE__SRE_AV1_MASK
 771                 | UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
 772                 | UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
 773                 | UVD_SUVD_CGC_GATE__SCM_AV1_MASK
 774                 | UVD_SUVD_CGC_GATE__SMPA_MASK);
 775         WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
 776
 777         data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
 778         data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
 779                 | UVD_SUVD_CGC_GATE2__MPBE1_MASK
 780                 | UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
 781                 | UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
 782                 | UVD_SUVD_CGC_GATE2__MPC1_MASK);
 783         WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
 784
 785         data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
 786         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 787                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 788                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 789                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 790                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 791                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 792                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 793                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 794                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
 795                 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
 796                 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
 797                 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
 798                 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
 799                 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
 800                 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
 801                 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
 802                 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
 803                 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
 804                 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
 805         WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
 806 }
 807
 808 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
 809                 uint8_t sram_sel, int inst_idx, uint8_t indirect)
 810 {
 811         uint32_t reg_data = 0;
 812
 813         /* enable sw clock gating control */
 814         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 815                 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 816         else
 817                 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 818         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 819         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 820         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 821                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 822                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 823                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 824                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 825                  UVD_CGC_CTRL__SYS_MODE_MASK |
 826                  UVD_CGC_CTRL__UDEC_MODE_MASK |
 827                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
 828                  UVD_CGC_CTRL__REGS_MODE_MASK |
 829                  UVD_CGC_CTRL__RBC_MODE_MASK |
 830                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 831                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 832                  UVD_CGC_CTRL__IDCT_MODE_MASK |
 833                  UVD_CGC_CTRL__MPRD_MODE_MASK |
 834                  UVD_CGC_CTRL__MPC_MODE_MASK |
 835                  UVD_CGC_CTRL__LBSI_MODE_MASK |
 836                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
 837                  UVD_CGC_CTRL__WCB_MODE_MASK |
 838                  UVD_CGC_CTRL__VCPU_MODE_MASK |
 839                  UVD_CGC_CTRL__MMSCH_MODE_MASK);
 840         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 841                 VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
 842
 843         /* turn off clock gating */
 844         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 845                 VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
 846
 847         /* turn on SUVD clock gating */
 848         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 849                 VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
 850
 851         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
 852         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 853                 VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
 854 }
 855
 856 /**
 857  * vcn_v3_0_enable_clock_gating - enable VCN clock gating
 858  *
 859  * @adev: amdgpu_device pointer
 860  * @inst: instance number
 861  *
 862  * Enable clock gating for VCN block
 863  */
 864 static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
 865 {
 866         uint32_t data;
 867
 868         /* enable VCN CGC */
 869         data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
 870         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 871                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 872         else
 873                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 874         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 875         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 876         WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
 877
 878         data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
 879         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 880                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 881                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 882                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 883                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 884                 | UVD_CGC_CTRL__SYS_MODE_MASK
 885                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 886                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 887                 | UVD_CGC_CTRL__REGS_MODE_MASK
 888                 | UVD_CGC_CTRL__RBC_MODE_MASK
 889                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 890                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 891                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 892                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 893                 | UVD_CGC_CTRL__MPC_MODE_MASK
 894                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 895                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 896                 | UVD_CGC_CTRL__WCB_MODE_MASK
 897                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 898                 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
 899         WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
 900
 901         data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
 902         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 903                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 904                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 905                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 906                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 907                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 908                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 909                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 910                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
 911                 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
 912                 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
 913                 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
 914                 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
 915                 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
 916                 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
 917                 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
 918                 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
 919                 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
 920                 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
 921         WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
 922 }
 923
 924 static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 925 {
 926         volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
 927         struct amdgpu_ring *ring;
 928         uint32_t rb_bufsz, tmp;
 929
 930         /* disable register anti-hang mechanism */
 931         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
 932                 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 933         /* enable dynamic power gating mode */
 934         tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
 935         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 936         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 937         WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
 938
 939         if (indirect)
 940                 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 941
 942         /* enable clock gating */
 943         vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
 944
 945         /* enable VCPU clock */
 946         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 947         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 948         tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
 949         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 950                 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
 951
 952         /* disable master interupt */
 953         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 954                 VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
 955
 956         /* setup mmUVD_LMI_CTRL */
 957         tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 958                 UVD_LMI_CTRL__REQ_MODE_MASK |
 959                 UVD_LMI_CTRL__CRC_RESET_MASK |
 960                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 961                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 962                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 963                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 964                 0x00100000L);
 965         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 966                 VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
 967
 968         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 969                 VCN, inst_idx, mmUVD_MPC_CNTL),
 970                 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
 971
 972         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 973                 VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
 974                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 975                  (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 976                  (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 977                  (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
 978
 979         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 980                 VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
 981                  ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 982                  (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 983                  (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 984                  (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
 985
 986         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 987                 VCN, inst_idx, mmUVD_MPC_SET_MUX),
 988                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 989                  (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 990                  (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
 991
 992         vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
 993
 994         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 995                 VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
 996         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 997                 VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
 998
 999         /* enable LMI MC and UMC channels */
1000         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1001                 VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1002
1003         /* unblock VCPU register access */
1004         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1005                 VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1006
1007         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1008         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1009         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1010                 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1011
1012         /* enable master interrupt */
1013         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1014                 VCN, inst_idx, mmUVD_MASTINT_EN),
1015                 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1016
1017         /* add nop to workaround PSP size check */
1018         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1019                 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1020
1021         if (indirect)
1022                 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1023                         (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1024                                 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
1025
1026         ring = &adev->vcn.inst[inst_idx].ring_dec;
1027         /* force RBC into idle state */
1028         rb_bufsz = order_base_2(ring->ring_size);
1029         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1030         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1031         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1032         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1033         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1034         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1035
1036         /* Stall DPG before WPTR/RPTR reset */
1037         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1038                 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1039                 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1040         fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1041
1042         /* set the write pointer delay */
1043         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1044
1045         /* set the wb address */
1046         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1047                 (upper_32_bits(ring->gpu_addr) >> 2));
1048
1049         /* programm the RB_BASE for ring buffer */
1050         WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1051                 lower_32_bits(ring->gpu_addr));
1052         WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1053                 upper_32_bits(ring->gpu_addr));
1054
1055         /* Initialize the ring buffer's read and write pointers */
1056         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1057
1058         WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1059
1060         ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1061         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1062                 lower_32_bits(ring->wptr));
1063
1064         /* Reset FW shared memory RBC WPTR/RPTR */
1065         fw_shared->rb.rptr = 0;
1066         fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1067
1068         /*resetting done, fw can check RB ring */
1069         fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1070
1071         /* Unstall DPG */
1072         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1073                 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1074
1075         return 0;
1076 }
1077
1078 static int vcn_v3_0_start(struct amdgpu_device *adev)
1079 {
1080         volatile struct amdgpu_fw_shared *fw_shared;
1081         struct amdgpu_ring *ring;
1082         uint32_t rb_bufsz, tmp;
1083         int i, j, k, r;
1084
1085         if (adev->pm.dpm_enabled)
1086                 amdgpu_dpm_enable_uvd(adev, true);
1087
1088         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1089                 if (adev->vcn.harvest_config & (1 << i))
1090                         continue;
1091
1092                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG){
1093                         r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1094                         continue;
1095                 }
1096
1097                 /* disable VCN power gating */
1098                 vcn_v3_0_disable_static_power_gating(adev, i);
1099
1100                 /* set VCN status busy */
1101                 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1102                 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1103
1104                 /*SW clock gating */
1105                 vcn_v3_0_disable_clock_gating(adev, i);
1106
1107                 /* enable VCPU clock */
1108                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1109                         UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1110
1111                 /* disable master interrupt */
1112                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1113                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
1114
1115                 /* enable LMI MC and UMC channels */
1116                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1117                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1118
1119                 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1120                 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1121                 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1122                 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1123
1124                 /* setup mmUVD_LMI_CTRL */
1125                 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1126                 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1127                         UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1128                         UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1129                         UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1130                         UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1131
1132                 /* setup mmUVD_MPC_CNTL */
1133                 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1134                 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1135                 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1136                 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1137
1138                 /* setup UVD_MPC_SET_MUXA0 */
1139                 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1140                         ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1141                         (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1142                         (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1143                         (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1144
1145                 /* setup UVD_MPC_SET_MUXB0 */
1146                 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1147                         ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1148                         (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1149                         (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1150                         (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1151
1152                 /* setup mmUVD_MPC_SET_MUX */
1153                 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1154                         ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1155                         (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1156                         (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1157
1158                 vcn_v3_0_mc_resume(adev, i);
1159
1160                 /* VCN global tiling registers */
1161                 WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1162                         adev->gfx.config.gb_addr_config);
1163
1164                 /* unblock VCPU register access */
1165                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1166                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1167
1168                 /* release VCPU reset to boot */
1169                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1170                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
1171
1172                 for (j = 0; j < 10; ++j) {
1173                         uint32_t status;
1174
1175                         for (k = 0; k < 100; ++k) {
1176                                 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1177                                 if (status & 2)
1178                                         break;
1179                                 mdelay(10);
1180                         }
1181                         r = 0;
1182                         if (status & 2)
1183                                 break;
1184
1185                         DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1186                         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1187                                 UVD_VCPU_CNTL__BLK_RST_MASK,
1188                                 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1189                         mdelay(10);
1190                         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1191                                 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1192
1193                         mdelay(10);
1194                         r = -1;
1195                 }
1196
1197                 if (r) {
1198                         DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1199                         return r;
1200                 }
1201
1202                 /* enable master interrupt */
1203                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1204                         UVD_MASTINT_EN__VCPU_EN_MASK,
1205                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
1206
1207                 /* clear the busy bit of VCN_STATUS */
1208                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1209                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1210
1211                 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1212
1213                 ring = &adev->vcn.inst[i].ring_dec;
1214                 /* force RBC into idle state */
1215                 rb_bufsz = order_base_2(ring->ring_size);
1216                 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1217                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1218                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1219                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1220                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1221                 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1222
1223                 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
1224                 fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1225
1226                 /* programm the RB_BASE for ring buffer */
1227                 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1228                         lower_32_bits(ring->gpu_addr));
1229                 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1230                         upper_32_bits(ring->gpu_addr));
1231
1232                 /* Initialize the ring buffer's read and write pointers */
1233                 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1234
1235                 WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1236                 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1237                 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1238                         lower_32_bits(ring->wptr));
1239                 fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1240                 fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1241
1242                 if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1243                         fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1244                         ring = &adev->vcn.inst[i].ring_enc[0];
1245                         WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1246                         WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1247                         WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1248                         WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1249                         WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1250                         fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1251
1252                         fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1253                         ring = &adev->vcn.inst[i].ring_enc[1];
1254                         WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1255                         WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1256                         WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1257                         WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1258                         WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1259                         fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1260                 }
1261         }
1262
1263         return 0;
1264 }
1265
1266 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1267 {
1268         int i, j;
1269         struct amdgpu_ring *ring;
1270         uint64_t cache_addr;
1271         uint64_t rb_addr;
1272         uint64_t ctx_addr;
1273         uint32_t param, resp, expected;
1274         uint32_t offset, cache_size;
1275         uint32_t tmp, timeout;
1276
1277         struct amdgpu_mm_table *table = &adev->virt.mm_table;
1278         uint32_t *table_loc;
1279         uint32_t table_size;
1280         uint32_t size, size_dw;
1281
1282         struct mmsch_v3_0_cmd_direct_write
1283                 direct_wt = { {0} };
1284         struct mmsch_v3_0_cmd_direct_read_modify_write
1285                 direct_rd_mod_wt = { {0} };
1286         struct mmsch_v3_0_cmd_end end = { {0} };
1287         struct mmsch_v3_0_init_header header;
1288
1289         direct_wt.cmd_header.command_type =
1290                 MMSCH_COMMAND__DIRECT_REG_WRITE;
1291         direct_rd_mod_wt.cmd_header.command_type =
1292                 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1293         end.cmd_header.command_type =
1294                 MMSCH_COMMAND__END;
1295
1296         header.version = MMSCH_VERSION;
1297         header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1298         for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1299                 header.inst[i].init_status = 0;
1300                 header.inst[i].table_offset = 0;
1301                 header.inst[i].table_size = 0;
1302         }
1303
1304         table_loc = (uint32_t *)table->cpu_addr;
1305         table_loc += header.total_size;
1306         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1307                 if (adev->vcn.harvest_config & (1 << i))
1308                         continue;
1309
1310                 table_size = 0;
1311
1312                 MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1313                         mmUVD_STATUS),
1314                         ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1315
1316                 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1317
1318                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1319                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1320                                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1321                                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1322                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1323                                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1324                                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1325                         offset = 0;
1326                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1327                                 mmUVD_VCPU_CACHE_OFFSET0),
1328                                 0);
1329                 } else {
1330                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1331                                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1332                                 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1333                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1334                                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1335                                 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1336                         offset = cache_size;
1337                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1338                                 mmUVD_VCPU_CACHE_OFFSET0),
1339                                 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1340                 }
1341
1342                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1343                         mmUVD_VCPU_CACHE_SIZE0),
1344                         cache_size);
1345
1346                 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1347                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1348                         mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1349                         lower_32_bits(cache_addr));
1350                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1351                         mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1352                         upper_32_bits(cache_addr));
1353                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1354                         mmUVD_VCPU_CACHE_OFFSET1),
1355                         0);
1356                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1357                         mmUVD_VCPU_CACHE_SIZE1),
1358                         AMDGPU_VCN_STACK_SIZE);
1359
1360                 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1361                         AMDGPU_VCN_STACK_SIZE;
1362                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1363                         mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1364                         lower_32_bits(cache_addr));
1365                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1366                         mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1367                         upper_32_bits(cache_addr));
1368                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1369                         mmUVD_VCPU_CACHE_OFFSET2),
1370                         0);
1371                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1372                         mmUVD_VCPU_CACHE_SIZE2),
1373                         AMDGPU_VCN_CONTEXT_SIZE);
1374
1375                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1376                         ring = &adev->vcn.inst[i].ring_enc[j];
1377                         ring->wptr = 0;
1378                         rb_addr = ring->gpu_addr;
1379                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1380                                 mmUVD_RB_BASE_LO),
1381                                 lower_32_bits(rb_addr));
1382                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1383                                 mmUVD_RB_BASE_HI),
1384                                 upper_32_bits(rb_addr));
1385                         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1386                                 mmUVD_RB_SIZE),
1387                                 ring->ring_size / 4);
1388                 }
1389
1390                 ring = &adev->vcn.inst[i].ring_dec;
1391                 ring->wptr = 0;
1392                 rb_addr = ring->gpu_addr;
1393                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1394                         mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1395                         lower_32_bits(rb_addr));
1396                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1397                         mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1398                         upper_32_bits(rb_addr));
1399                 /* force RBC into idle state */
1400                 tmp = order_base_2(ring->ring_size);
1401                 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1402                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1403                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1404                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1405                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1406                 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1407                         mmUVD_RBC_RB_CNTL),
1408                         tmp);
1409
1410                 /* add end packet */
1411                 MMSCH_V3_0_INSERT_END();
1412
1413                 /* refine header */
1414                 header.inst[i].init_status = 0;
1415                 header.inst[i].table_offset = header.total_size;
1416                 header.inst[i].table_size = table_size;
1417                 header.total_size += table_size;
1418         }
1419
1420         /* Update init table header in memory */
1421         size = sizeof(struct mmsch_v3_0_init_header);
1422         table_loc = (uint32_t *)table->cpu_addr;
1423         memcpy((void *)table_loc, &header, size);
1424
1425         /* message MMSCH (in VCN[0]) to initialize this client
1426          * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1427          * of memory descriptor location
1428          */
1429         ctx_addr = table->gpu_addr;
1430         WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1431         WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1432
1433         /* 2, update vmid of descriptor */
1434         tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1435         tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1436         /* use domain0 for MM scheduler */
1437         tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1438         WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1439
1440         /* 3, notify mmsch about the size of this descriptor */
1441         size = header.total_size;
1442         WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1443
1444         /* 4, set resp to zero */
1445         WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1446
1447         /* 5, kick off the initialization and wait until
1448          * MMSCH_VF_MAILBOX_RESP becomes non-zero
1449          */
1450         param = 0x10000001;
1451         WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1452         tmp = 0;
1453         timeout = 1000;
1454         resp = 0;
1455         expected = param + 1;
1456         while (resp != expected) {
1457                 resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1458                 if (resp == expected)
1459                         break;
1460
1461                 udelay(10);
1462                 tmp = tmp + 10;
1463                 if (tmp >= timeout) {
1464                         DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1465                                 " waiting for mmMMSCH_VF_MAILBOX_RESP "\
1466                                 "(expected=0x%08x, readback=0x%08x)\n",
1467                                 tmp, expected, resp);
1468                         return -EBUSY;
1469                 }
1470         }
1471
1472         return 0;
1473 }
1474
1475 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1476 {
1477         uint32_t tmp;
1478
1479         /* Wait for power status to be 1 */
1480         SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1481                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1482
1483         /* wait for read ptr to be equal to write ptr */
1484         tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1485         SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1486
1487         tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1488         SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1489
1490         tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1491         SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1492
1493         SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1494                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1495
1496         /* disable dynamic power gating mode */
1497         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1498                 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1499
1500         return 0;
1501 }
1502
1503 static int vcn_v3_0_stop(struct amdgpu_device *adev)
1504 {
1505         uint32_t tmp;
1506         int i, r = 0;
1507
1508         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1509                 if (adev->vcn.harvest_config & (1 << i))
1510                         continue;
1511
1512                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1513                         r = vcn_v3_0_stop_dpg_mode(adev, i);
1514                         continue;
1515                 }
1516
1517                 /* wait for vcn idle */
1518                 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1519                 if (r)
1520                         return r;
1521
1522                 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1523                         UVD_LMI_STATUS__READ_CLEAN_MASK |
1524                         UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1525                         UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1526                 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1527                 if (r)
1528                         return r;
1529
1530                 /* disable LMI UMC channel */
1531                 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1532                 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1533                 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1534                 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1535                         UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1536                 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1537                 if (r)
1538                         return r;
1539
1540                 /* block VCPU register access */
1541                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1542                         UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1543                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1544
1545                 /* reset VCPU */
1546                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1547                         UVD_VCPU_CNTL__BLK_RST_MASK,
1548                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
1549
1550                 /* disable VCPU clock */
1551                 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1552                         ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1553
1554                 /* apply soft reset */
1555                 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1556                 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1557                 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1558                 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1559                 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1560                 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1561
1562                 /* clear status */
1563                 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1564
1565                 /* apply HW clock gating */
1566                 vcn_v3_0_enable_clock_gating(adev, i);
1567
1568                 /* enable VCN power gating */
1569                 vcn_v3_0_enable_static_power_gating(adev, i);
1570         }
1571
1572         if (adev->pm.dpm_enabled)
1573                 amdgpu_dpm_enable_uvd(adev, false);
1574
1575         return 0;
1576 }
1577
1578 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1579                    int inst_idx, struct dpg_pause_state *new_state)
1580 {
1581         volatile struct amdgpu_fw_shared *fw_shared;
1582         struct amdgpu_ring *ring;
1583         uint32_t reg_data = 0;
1584         int ret_code;
1585
1586         /* pause/unpause if state is changed */
1587         if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1588                 DRM_DEBUG("dpg pause state changed %d -> %d",
1589                         adev->vcn.inst[inst_idx].pause_state.fw_based,  new_state->fw_based);
1590                 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1591                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1592
1593                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1594                         ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1595                                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1596
1597                         if (!ret_code) {
1598                                 /* pause DPG */
1599                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1600                                 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1601
1602                                 /* wait for ACK */
1603                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1604                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1605                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1606
1607                                 /* Stall DPG before WPTR/RPTR reset */
1608                                 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1609                                         UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1610                                         ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1611
1612                                 if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1613                                         /* Restore */
1614                                         fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
1615                                         fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1616                                         ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1617                                         ring->wptr = 0;
1618                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1619                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1620                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1621                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1622                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1623                                         fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1624
1625                                         fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1626                                         ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1627                                         ring->wptr = 0;
1628                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1629                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1630                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1631                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1632                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1633                                         fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1634
1635                                         /* restore wptr/rptr with pointers saved in FW shared memory*/
1636                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1637                                         WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1638                                 }
1639
1640                                 /* Unstall DPG */
1641                                 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1642                                         0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1643
1644                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1645                                         UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1646                         }
1647                 } else {
1648                         /* unpause dpg, no need to wait */
1649                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1650                         WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1651                 }
1652                 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1653         }
1654
1655         return 0;
1656 }
1657
1658 /**
1659  * vcn_v3_0_dec_ring_get_rptr - get read pointer
1660  *
1661  * @ring: amdgpu_ring pointer
1662  *
1663  * Returns the current hardware read pointer
1664  */
1665 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1666 {
1667         struct amdgpu_device *adev = ring->adev;
1668
1669         return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1670 }
1671
1672 /**
1673  * vcn_v3_0_dec_ring_get_wptr - get write pointer
1674  *
1675  * @ring: amdgpu_ring pointer
1676  *
1677  * Returns the current hardware write pointer
1678  */
1679 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1680 {
1681         struct amdgpu_device *adev = ring->adev;
1682
1683         if (ring->use_doorbell)
1684                 return adev->wb.wb[ring->wptr_offs];
1685         else
1686                 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1687 }
1688
1689 /**
1690  * vcn_v3_0_dec_ring_set_wptr - set write pointer
1691  *
1692  * @ring: amdgpu_ring pointer
1693  *
1694  * Commits the write pointer to the hardware
1695  */
1696 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1697 {
1698         struct amdgpu_device *adev = ring->adev;
1699         volatile struct amdgpu_fw_shared *fw_shared;
1700
1701         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1702                 /*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1703                 fw_shared = adev->vcn.inst[ring->me].fw_shared_cpu_addr;
1704                 fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1705                 WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1706                         lower_32_bits(ring->wptr));
1707         }
1708
1709         if (ring->use_doorbell) {
1710                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1711                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1712         } else {
1713                 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1714         }
1715 }
1716
1717 static void vcn_v3_0_dec_sw_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1718                                 u64 seq, uint32_t flags)
1719 {
1720         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1721
1722         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_FENCE);
1723         amdgpu_ring_write(ring, addr);
1724         amdgpu_ring_write(ring, upper_32_bits(addr));
1725         amdgpu_ring_write(ring, seq);
1726         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_TRAP);
1727 }
1728
1729 static void vcn_v3_0_dec_sw_ring_insert_end(struct amdgpu_ring *ring)
1730 {
1731         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_END);
1732 }
1733
1734 static void vcn_v3_0_dec_sw_ring_emit_ib(struct amdgpu_ring *ring,
1735                                struct amdgpu_job *job,
1736                                struct amdgpu_ib *ib,
1737                                uint32_t flags)
1738 {
1739         uint32_t vmid = AMDGPU_JOB_GET_VMID(job);
1740
1741         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_IB);
1742         amdgpu_ring_write(ring, vmid);
1743         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1744         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1745         amdgpu_ring_write(ring, ib->length_dw);
1746 }
1747
1748 static void vcn_v3_0_dec_sw_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1749                                 uint32_t val, uint32_t mask)
1750 {
1751         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WAIT);
1752         amdgpu_ring_write(ring, reg << 2);
1753         amdgpu_ring_write(ring, mask);
1754         amdgpu_ring_write(ring, val);
1755 }
1756
1757 static void vcn_v3_0_dec_sw_ring_emit_vm_flush(struct amdgpu_ring *ring,
1758                                 uint32_t vmid, uint64_t pd_addr)
1759 {
1760         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1761         uint32_t data0, data1, mask;
1762
1763         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1764
1765         /* wait for register write */
1766         data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1767         data1 = lower_32_bits(pd_addr);
1768         mask = 0xffffffff;
1769         vcn_v3_0_dec_sw_ring_emit_reg_wait(ring, data0, data1, mask);
1770 }
1771
1772 static void vcn_v3_0_dec_sw_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1773 {
1774         amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WRITE);
1775         amdgpu_ring_write(ring, reg << 2);
1776         amdgpu_ring_write(ring, val);
1777 }
1778
1779 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1780         .type = AMDGPU_RING_TYPE_VCN_DEC,
1781         .align_mask = 0x3f,
1782         .nop = VCN_DEC_SW_CMD_NO_OP,
1783         .vmhub = AMDGPU_MMHUB_0,
1784         .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1785         .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1786         .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1787         .emit_frame_size =
1788                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1789                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1790                 4 + /* vcn_v3_0_dec_sw_ring_emit_vm_flush */
1791                 5 + 5 + /* vcn_v3_0_dec_sw_ring_emit_fdec_swe x2 vm fdec_swe */
1792                 1, /* vcn_v3_0_dec_sw_ring_insert_end */
1793         .emit_ib_size = 5, /* vcn_v3_0_dec_sw_ring_emit_ib */
1794         .emit_ib = vcn_v3_0_dec_sw_ring_emit_ib,
1795         .emit_fence = vcn_v3_0_dec_sw_ring_emit_fence,
1796         .emit_vm_flush = vcn_v3_0_dec_sw_ring_emit_vm_flush,
1797         .test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1798         .test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1799         .insert_nop = amdgpu_ring_insert_nop,
1800         .insert_end = vcn_v3_0_dec_sw_ring_insert_end,
1801         .pad_ib = amdgpu_ring_generic_pad_ib,
1802         .begin_use = amdgpu_vcn_ring_begin_use,
1803         .end_use = amdgpu_vcn_ring_end_use,
1804         .emit_wreg = vcn_v3_0_dec_sw_ring_emit_wreg,
1805         .emit_reg_wait = vcn_v3_0_dec_sw_ring_emit_reg_wait,
1806         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1807 };
1808
1809 static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p)
1810 {
1811         struct drm_gpu_scheduler **scheds;
1812
1813         /* The create msg must be in the first IB submitted */
1814         if (atomic_read(&p->entity->fence_seq))
1815                 return -EINVAL;
1816
1817         scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1818                 [AMDGPU_RING_PRIO_DEFAULT].sched;
1819         drm_sched_entity_modify_sched(p->entity, scheds, 1);
1820         return 0;
1821 }
1822
1823 static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
1824 {
1825         struct ttm_operation_ctx ctx = { false, false };
1826         struct amdgpu_bo_va_mapping *map;
1827         uint32_t *msg, num_buffers;
1828         struct amdgpu_bo *bo;
1829         uint64_t start, end;
1830         unsigned int i;
1831         void * ptr;
1832         int r;
1833
1834         addr &= AMDGPU_GMC_HOLE_MASK;
1835         r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1836         if (r) {
1837                 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1838                 return r;
1839         }
1840
1841         start = map->start * AMDGPU_GPU_PAGE_SIZE;
1842         end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1843         if (addr & 0x7) {
1844                 DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1845                 return -EINVAL;
1846         }
1847
1848         bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1849         amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1850         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1851         if (r) {
1852                 DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1853                 return r;
1854         }
1855
1856         r = amdgpu_bo_kmap(bo, &ptr);
1857         if (r) {
1858                 DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1859                 return r;
1860         }
1861
1862         msg = ptr + addr - start;
1863
1864         /* Check length */
1865         if (msg[1] > end - addr) {
1866                 r = -EINVAL;
1867                 goto out;
1868         }
1869
1870         if (msg[3] != RDECODE_MSG_CREATE)
1871                 goto out;
1872
1873         num_buffers = msg[2];
1874         for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1875                 uint32_t offset, size, *create;
1876
1877                 if (msg[0] != RDECODE_MESSAGE_CREATE)
1878                         continue;
1879
1880                 offset = msg[1];
1881                 size = msg[2];
1882
1883                 if (offset + size > end) {
1884                         r = -EINVAL;
1885                         goto out;
1886                 }
1887
1888                 create = ptr + addr + offset - start;
1889
1890                 /* H246, HEVC and VP9 can run on any instance */
1891                 if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1892                         continue;
1893
1894                 r = vcn_v3_0_limit_sched(p);
1895                 if (r)
1896                         goto out;
1897         }
1898
1899 out:
1900         amdgpu_bo_kunmap(bo);
1901         return r;
1902 }
1903
1904 static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1905                                            uint32_t ib_idx)
1906 {
1907         struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
1908         struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
1909         uint32_t msg_lo = 0, msg_hi = 0;
1910         unsigned i;
1911         int r;
1912
1913         /* The first instance can decode anything */
1914         if (!ring->me)
1915                 return 0;
1916
1917         for (i = 0; i < ib->length_dw; i += 2) {
1918                 uint32_t reg = amdgpu_get_ib_value(p, ib_idx, i);
1919                 uint32_t val = amdgpu_get_ib_value(p, ib_idx, i + 1);
1920
1921                 if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1922                         msg_lo = val;
1923                 } else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1924                         msg_hi = val;
1925                 } else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
1926                            val == 0) {
1927                         r = vcn_v3_0_dec_msg(p, ((u64)msg_hi) << 32 | msg_lo);
1928                         if (r)
1929                                 return r;
1930                 }
1931         }
1932         return 0;
1933 }
1934
1935 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1936         .type = AMDGPU_RING_TYPE_VCN_DEC,
1937         .align_mask = 0xf,
1938         .vmhub = AMDGPU_MMHUB_0,
1939         .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1940         .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1941         .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1942         .patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
1943         .emit_frame_size =
1944                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1945                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1946                 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1947                 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1948                 6,
1949         .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1950         .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1951         .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1952         .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1953         .test_ring = vcn_v2_0_dec_ring_test_ring,
1954         .test_ib = amdgpu_vcn_dec_ring_test_ib,
1955         .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1956         .insert_start = vcn_v2_0_dec_ring_insert_start,
1957         .insert_end = vcn_v2_0_dec_ring_insert_end,
1958         .pad_ib = amdgpu_ring_generic_pad_ib,
1959         .begin_use = amdgpu_vcn_ring_begin_use,
1960         .end_use = amdgpu_vcn_ring_end_use,
1961         .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1962         .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1963         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1964 };
1965
1966 /**
1967  * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1968  *
1969  * @ring: amdgpu_ring pointer
1970  *
1971  * Returns the current hardware enc read pointer
1972  */
1973 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1974 {
1975         struct amdgpu_device *adev = ring->adev;
1976
1977         if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1978                 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1979         else
1980                 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1981 }
1982
1983 /**
1984  * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1985  *
1986  * @ring: amdgpu_ring pointer
1987  *
1988  * Returns the current hardware enc write pointer
1989  */
1990 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1991 {
1992         struct amdgpu_device *adev = ring->adev;
1993
1994         if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1995                 if (ring->use_doorbell)
1996                         return adev->wb.wb[ring->wptr_offs];
1997                 else
1998                         return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1999         } else {
2000                 if (ring->use_doorbell)
2001                         return adev->wb.wb[ring->wptr_offs];
2002                 else
2003                         return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
2004         }
2005 }
2006
2007 /**
2008  * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
2009  *
2010  * @ring: amdgpu_ring pointer
2011  *
2012  * Commits the enc write pointer to the hardware
2013  */
2014 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
2015 {
2016         struct amdgpu_device *adev = ring->adev;
2017
2018         if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
2019                 if (ring->use_doorbell) {
2020                         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
2021                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2022                 } else {
2023                         WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
2024                 }
2025         } else {
2026                 if (ring->use_doorbell) {
2027                         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
2028                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2029                 } else {
2030                         WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
2031                 }
2032         }
2033 }
2034
2035 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
2036         .type = AMDGPU_RING_TYPE_VCN_ENC,
2037         .align_mask = 0x3f,
2038         .nop = VCN_ENC_CMD_NO_OP,
2039         .vmhub = AMDGPU_MMHUB_0,
2040         .get_rptr = vcn_v3_0_enc_ring_get_rptr,
2041         .get_wptr = vcn_v3_0_enc_ring_get_wptr,
2042         .set_wptr = vcn_v3_0_enc_ring_set_wptr,
2043         .emit_frame_size =
2044                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2045                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2046                 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2047                 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2048                 1, /* vcn_v2_0_enc_ring_insert_end */
2049         .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2050         .emit_ib = vcn_v2_0_enc_ring_emit_ib,
2051         .emit_fence = vcn_v2_0_enc_ring_emit_fence,
2052         .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2053         .test_ring = amdgpu_vcn_enc_ring_test_ring,
2054         .test_ib = amdgpu_vcn_enc_ring_test_ib,
2055         .insert_nop = amdgpu_ring_insert_nop,
2056         .insert_end = vcn_v2_0_enc_ring_insert_end,
2057         .pad_ib = amdgpu_ring_generic_pad_ib,
2058         .begin_use = amdgpu_vcn_ring_begin_use,
2059         .end_use = amdgpu_vcn_ring_end_use,
2060         .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2061         .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2062         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2063 };
2064
2065 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2066 {
2067         int i;
2068
2069         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2070                 if (adev->vcn.harvest_config & (1 << i))
2071                         continue;
2072
2073                 if (!DEC_SW_RING_ENABLED)
2074                         adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2075                 else
2076                         adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2077                 adev->vcn.inst[i].ring_dec.me = i;
2078                 DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
2079                           DEC_SW_RING_ENABLED?"(Software Ring)":"");
2080         }
2081 }
2082
2083 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2084 {
2085         int i, j;
2086
2087         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2088                 if (adev->vcn.harvest_config & (1 << i))
2089                         continue;
2090
2091                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
2092                         adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2093                         adev->vcn.inst[i].ring_enc[j].me = i;
2094                 }
2095                 if (adev->vcn.num_enc_rings > 0)
2096                         DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
2097         }
2098 }
2099
2100 static bool vcn_v3_0_is_idle(void *handle)
2101 {
2102         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2103         int i, ret = 1;
2104
2105         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2106                 if (adev->vcn.harvest_config & (1 << i))
2107                         continue;
2108
2109                 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2110         }
2111
2112         return ret;
2113 }
2114
2115 static int vcn_v3_0_wait_for_idle(void *handle)
2116 {
2117         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2118         int i, ret = 0;
2119
2120         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2121                 if (adev->vcn.harvest_config & (1 << i))
2122                         continue;
2123
2124                 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2125                         UVD_STATUS__IDLE);
2126                 if (ret)
2127                         return ret;
2128         }
2129
2130         return ret;
2131 }
2132
2133 static int vcn_v3_0_set_clockgating_state(void *handle,
2134                                           enum amd_clockgating_state state)
2135 {
2136         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2137         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
2138         int i;
2139
2140         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2141                 if (adev->vcn.harvest_config & (1 << i))
2142                         continue;
2143
2144                 if (enable) {
2145                         if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2146                                 return -EBUSY;
2147                         vcn_v3_0_enable_clock_gating(adev, i);
2148                 } else {
2149                         vcn_v3_0_disable_clock_gating(adev, i);
2150                 }
2151         }
2152
2153         return 0;
2154 }
2155
2156 static int vcn_v3_0_set_powergating_state(void *handle,
2157                                           enum amd_powergating_state state)
2158 {
2159         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2160         int ret;
2161
2162         /* for SRIOV, guest should not control VCN Power-gating
2163          * MMSCH FW should control Power-gating and clock-gating
2164          * guest should avoid touching CGC and PG
2165          */
2166         if (amdgpu_sriov_vf(adev)) {
2167                 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2168                 return 0;
2169         }
2170
2171         if(state == adev->vcn.cur_state)
2172                 return 0;
2173
2174         if (state == AMD_PG_STATE_GATE)
2175                 ret = vcn_v3_0_stop(adev);
2176         else
2177                 ret = vcn_v3_0_start(adev);
2178
2179         if(!ret)
2180                 adev->vcn.cur_state = state;
2181
2182         return ret;
2183 }
2184
2185 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2186                                         struct amdgpu_irq_src *source,
2187                                         unsigned type,
2188                                         enum amdgpu_interrupt_state state)
2189 {
2190         return 0;
2191 }
2192
2193 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2194                                       struct amdgpu_irq_src *source,
2195                                       struct amdgpu_iv_entry *entry)
2196 {
2197         uint32_t ip_instance;
2198
2199         switch (entry->client_id) {
2200         case SOC15_IH_CLIENTID_VCN:
2201                 ip_instance = 0;
2202                 break;
2203         case SOC15_IH_CLIENTID_VCN1:
2204                 ip_instance = 1;
2205                 break;
2206         default:
2207                 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2208                 return 0;
2209         }
2210
2211         DRM_DEBUG("IH: VCN TRAP\n");
2212
2213         switch (entry->src_id) {
2214         case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2215                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2216                 break;
2217         case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2218                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2219                 break;
2220         case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2221                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2222                 break;
2223         default:
2224                 DRM_ERROR("Unhandled interrupt: %d %d\n",
2225                           entry->src_id, entry->src_data[0]);
2226                 break;
2227         }
2228
2229         return 0;
2230 }
2231
2232 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2233         .set = vcn_v3_0_set_interrupt_state,
2234         .process = vcn_v3_0_process_interrupt,
2235 };
2236
2237 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2238 {
2239         int i;
2240
2241         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2242                 if (adev->vcn.harvest_config & (1 << i))
2243                         continue;
2244
2245                 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2246                 adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2247         }
2248 }
2249
2250 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2251         .name = "vcn_v3_0",
2252         .early_init = vcn_v3_0_early_init,
2253         .late_init = NULL,
2254         .sw_init = vcn_v3_0_sw_init,
2255         .sw_fini = vcn_v3_0_sw_fini,
2256         .hw_init = vcn_v3_0_hw_init,
2257         .hw_fini = vcn_v3_0_hw_fini,
2258         .suspend = vcn_v3_0_suspend,
2259         .resume = vcn_v3_0_resume,
2260         .is_idle = vcn_v3_0_is_idle,
2261         .wait_for_idle = vcn_v3_0_wait_for_idle,
2262         .check_soft_reset = NULL,
2263         .pre_soft_reset = NULL,
2264         .soft_reset = NULL,
2265         .post_soft_reset = NULL,
2266         .set_clockgating_state = vcn_v3_0_set_clockgating_state,
2267         .set_powergating_state = vcn_v3_0_set_powergating_state,
2268 };
2269
2270 const struct amdgpu_ip_block_version vcn_v3_0_ip_block =
2271 {
2272         .type = AMD_IP_BLOCK_TYPE_VCN,
2273         .major = 3,
2274         .minor = 0,
2275         .rev = 0,
2276         .funcs = &vcn_v3_0_ip_funcs,
2277 };