drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_vcn.h"
  27 #include "amdgpu_pm.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "soc15_hw_ip.h"
  31 #include "vcn_v2_0.h"
  32 #include "vcn_sw_ring.h"
  33
  34 #include "vcn/vcn_4_0_0_offset.h"
  35 #include "vcn/vcn_4_0_0_sh_mask.h"
  36 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  37
  38 #include <drm/drm_drv.h>
  39
  40 #define mmUVD_DPG_LMA_CTL                                                       regUVD_DPG_LMA_CTL
  41 #define mmUVD_DPG_LMA_CTL_BASE_IDX                                              regUVD_DPG_LMA_CTL_BASE_IDX
  42 #define mmUVD_DPG_LMA_DATA                                                      regUVD_DPG_LMA_DATA
  43 #define mmUVD_DPG_LMA_DATA_BASE_IDX                                             regUVD_DPG_LMA_DATA_BASE_IDX
  44
  45 #define VCN_VID_SOC_ADDRESS_2_0                                                 0x1fb00
  46 #define VCN1_VID_SOC_ADDRESS_3_0                                                0x48300
  47
  48 bool unifiedQ_enabled = false;
  49
  50 static int amdgpu_ih_clientid_vcns[] = {
  51         SOC15_IH_CLIENTID_VCN,
  52         SOC15_IH_CLIENTID_VCN1
  53 };
  54
  55 static void vcn_v4_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  56 static void vcn_v4_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  57 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
  58 static int vcn_v4_0_set_powergating_state(void *handle,
  59         enum amd_powergating_state state);
  60 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,
  61         int inst_idx, struct dpg_pause_state *new_state);
  62
  63 /**
  64  * vcn_v4_0_early_init - set function pointers
  65  *
  66  * @handle: amdgpu_device pointer
  67  *
  68  * Set ring and irq function pointers
  69  */
  70 static int vcn_v4_0_early_init(void *handle)
  71 {
  72         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  73
  74         if (unifiedQ_enabled) {
  75                 adev->vcn.num_vcn_inst = 1;
  76                 adev->vcn.num_enc_rings = 1;
  77         } else {
  78                 adev->vcn.num_enc_rings = 2;
  79         }
  80
  81         if (!unifiedQ_enabled)
  82                 vcn_v4_0_set_dec_ring_funcs(adev);
  83
  84         vcn_v4_0_set_enc_ring_funcs(adev);
  85         vcn_v4_0_set_irq_funcs(adev);
  86
  87         return 0;
  88 }
  89
  90 static void amdgpu_vcn_setup_unified_queue_ucode(struct amdgpu_device *adev)
  91 {
  92         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
  93                 const struct common_firmware_header *hdr;
  94
  95                 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
  96                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
  97                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
  98                 adev->firmware.fw_size +=
  99                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
 100                 DRM_INFO("PSP loading VCN firmware\n");
 101         }
 102 }
 103
 104 /**
 105  * vcn_v4_0_sw_init - sw init for VCN block
 106  *
 107  * @handle: amdgpu_device pointer
 108  *
 109  * Load firmware and sw initialization
 110  */
 111 static int vcn_v4_0_sw_init(void *handle)
 112 {
 113         struct amdgpu_ring *ring;
 114         int i, j, r;
 115         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 116
 117         r = amdgpu_vcn_sw_init(adev);
 118         if (r)
 119                 return r;
 120
 121         if (unifiedQ_enabled)
 122                 amdgpu_vcn_setup_unified_queue_ucode(adev);
 123         else
 124                 amdgpu_vcn_setup_ucode(adev);
 125
 126         r = amdgpu_vcn_resume(adev);
 127         if (r)
 128                 return r;
 129
 130         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 131                 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 132                 if (adev->vcn.harvest_config & (1 << i))
 133                         continue;
 134                 /* VCN DEC TRAP */
 135                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 136                                 VCN_4_0__SRCID__UVD_TRAP, &adev->vcn.inst[i].irq);
 137                 if (r)
 138                         return r;
 139
 140                 atomic_set(&adev->vcn.inst[i].sched_score, 0);
 141                 if (!unifiedQ_enabled) {
 142                         ring = &adev->vcn.inst[i].ring_dec;
 143                         ring->use_doorbell = true;
 144
 145                         /* VCN4 doorbell layout
 146                          * 1: VCN_JPEG_DB_CTRL UVD_JRBC_RB_WPTR; (jpeg)
 147                          * 2: VCN_RB1_DB_CTRL  UVD_RB_WPTR; (decode/encode for unified queue)
 148                          * 3: VCN_RB2_DB_CTRL  UVD_RB_WPTR2; (encode only for swqueue)
 149                          * 4: VCN_RB3_DB_CTRL  UVD_RB_WPTR3; (Reserved)
 150                          * 5: VCN_RB4_DB_CTRL  UVD_RB_WPTR4; (decode only for swqueue)
 151                          */
 152
 153                         ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1)
 154                                                 + 5 + 8 * i;
 155
 156                         sprintf(ring->name, "vcn_dec_%d", i);
 157                         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 158                                              AMDGPU_RING_PRIO_DEFAULT,
 159                                              &adev->vcn.inst[i].sched_score);
 160                         if (r)
 161                                 return r;
 162                 }
 163                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
 164                         /* VCN ENC TRAP */
 165                         r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 166                                 j + VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
 167                         if (r)
 168                                 return r;
 169
 170                         ring = &adev->vcn.inst[i].ring_enc[j];
 171                         ring->use_doorbell = true;
 172
 173                         ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
 174
 175                         if (unifiedQ_enabled) {
 176                                 sprintf(ring->name, "vcn_unified%d", i);
 177                                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 178                                      AMDGPU_RING_PRIO_DEFAULT, NULL);
 179                         } else {
 180                                 enum amdgpu_ring_priority_level hw_prio;
 181
 182                                 hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
 183                                 sprintf(ring->name, "vcn_enc_%d.%d", i, j);
 184                                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 185                                              hw_prio, &adev->vcn.inst[i].sched_score);
 186                         }
 187                         if (r)
 188                                 return r;
 189                 }
 190
 191                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 192                 fw_shared->present_flag_0 = 0;
 193
 194                 if (unifiedQ_enabled) {
 195                         fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
 196                         fw_shared->sq.is_enabled = 1;
 197                 }
 198
 199                 if (amdgpu_vcnfw_log)
 200                         amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
 201         }
 202
 203         if (!unifiedQ_enabled) {
 204                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 205                         adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
 206         }
 207         return 0;
 208 }
 209
 210 /**
 211  * vcn_v4_0_sw_fini - sw fini for VCN block
 212  *
 213  * @handle: amdgpu_device pointer
 214  *
 215  * VCN suspend and free up sw allocation
 216  */
 217 static int vcn_v4_0_sw_fini(void *handle)
 218 {
 219         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 220         int i, r, idx;
 221
 222         if (drm_dev_enter(&adev->ddev, &idx)) {
 223                 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 224                         volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 225
 226                         if (adev->vcn.harvest_config & (1 << i))
 227                                 continue;
 228
 229                         fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 230                         fw_shared->present_flag_0 = 0;
 231                         fw_shared->sq.is_enabled = 0;
 232                 }
 233
 234                 drm_dev_exit(idx);
 235         }
 236
 237         r = amdgpu_vcn_suspend(adev);
 238         if (r)
 239                 return r;
 240
 241         r = amdgpu_vcn_sw_fini(adev);
 242
 243         return r;
 244 }
 245
 246 /**
 247  * vcn_v4_0_hw_init - start and test VCN block
 248  *
 249  * @handle: amdgpu_device pointer
 250  *
 251  * Initialize the hardware, boot up the VCPU and do some testing
 252  */
 253 static int vcn_v4_0_hw_init(void *handle)
 254 {
 255         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 256         struct amdgpu_ring *ring;
 257         int i, j, r;
 258
 259         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 260                 if (adev->vcn.harvest_config & (1 << i))
 261                         continue;
 262                 if (unifiedQ_enabled)
 263                         ring = &adev->vcn.inst[i].ring_enc[0];
 264                 else
 265                         ring = &adev->vcn.inst[i].ring_dec;
 266
 267                 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 268                                 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
 269
 270                 r = amdgpu_ring_test_helper(ring);
 271                 if (r)
 272                         goto done;
 273
 274                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
 275                         ring = &adev->vcn.inst[i].ring_enc[j];
 276                         r = amdgpu_ring_test_helper(ring);
 277                         if (r)
 278                                 goto done;
 279                 }
 280         }
 281
 282 done:
 283         if (!r)
 284                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 285                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 286
 287         return r;
 288 }
 289
 290 /**
 291  * vcn_v4_0_hw_fini - stop the hardware block
 292  *
 293  * @handle: amdgpu_device pointer
 294  *
 295  * Stop the VCN block, mark ring as not ready any more
 296  */
 297 static int vcn_v4_0_hw_fini(void *handle)
 298 {
 299         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 300         int i;
 301
 302         cancel_delayed_work_sync(&adev->vcn.idle_work);
 303
 304         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 305                 if (adev->vcn.harvest_config & (1 << i))
 306                         continue;
 307
 308                 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 309                         (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 310                                 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
 311                         vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 312                 }
 313         }
 314
 315         return 0;
 316 }
 317
 318 /**
 319  * vcn_v4_0_suspend - suspend VCN block
 320  *
 321  * @handle: amdgpu_device pointer
 322  *
 323  * HW fini and suspend VCN block
 324  */
 325 static int vcn_v4_0_suspend(void *handle)
 326 {
 327         int r;
 328         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 329
 330         r = vcn_v4_0_hw_fini(adev);
 331         if (r)
 332                 return r;
 333
 334         r = amdgpu_vcn_suspend(adev);
 335
 336         return r;
 337 }
 338
 339 /**
 340  * vcn_v4_0_resume - resume VCN block
 341  *
 342  * @handle: amdgpu_device pointer
 343  *
 344  * Resume firmware and hw init VCN block
 345  */
 346 static int vcn_v4_0_resume(void *handle)
 347 {
 348         int r;
 349         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 350
 351         r = amdgpu_vcn_resume(adev);
 352         if (r)
 353                 return r;
 354
 355         r = vcn_v4_0_hw_init(adev);
 356
 357         return r;
 358 }
 359
 360 /**
 361  * vcn_v4_0_mc_resume - memory controller programming
 362  *
 363  * @adev: amdgpu_device pointer
 364  * @inst: instance number
 365  *
 366  * Let the VCN memory controller know it's offsets
 367  */
 368 static void vcn_v4_0_mc_resume(struct amdgpu_device *adev, int inst)
 369 {
 370         uint32_t offset, size;
 371         const struct common_firmware_header *hdr;
 372
 373         hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
 374         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 375
 376         /* cache window 0: fw */
 377         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 378                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 379                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
 380                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 381                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
 382                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
 383                 offset = 0;
 384         } else {
 385                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 386                         lower_32_bits(adev->vcn.inst[inst].gpu_addr));
 387                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 388                         upper_32_bits(adev->vcn.inst[inst].gpu_addr));
 389                 offset = size;
 390                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 391         }
 392         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
 393
 394         /* cache window 1: stack */
 395         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 396                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 397         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 398                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 399         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
 400         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 401
 402         /* cache window 2: context */
 403         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 404                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 405         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 406                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 407         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
 408         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 409
 410         /* non-cache window */
 411         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 412                 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 413         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 414                 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 415         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
 416         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
 417                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
 418 }
 419
 420 /**
 421  * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
 422  *
 423  * @adev: amdgpu_device pointer
 424  * @inst_idx: instance number index
 425  * @indirect: indirectly write sram
 426  *
 427  * Let the VCN memory controller know it's offsets with dpg mode
 428  */
 429 static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 430 {
 431         uint32_t offset, size;
 432         const struct common_firmware_header *hdr;
 433         hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
 434         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 435
 436         /* cache window 0: fw */
 437         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 438                 if (!indirect) {
 439                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 440                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 441                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
 442                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 443                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 444                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
 445                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 446                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 447                 } else {
 448                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 449                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 450                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 451                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 452                         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 453                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 454                 }
 455                 offset = 0;
 456         } else {
 457                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 458                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 459                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 460                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 461                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 462                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 463                 offset = size;
 464                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 465                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
 466                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 467
 468         }
 469
 470         if (!indirect)
 471                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 472                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 473         else
 474                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 475                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 476
 477         /* cache window 1: stack */
 478         if (!indirect) {
 479                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 480                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 481                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 482                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 483                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 484                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 485                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 486                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 487         } else {
 488                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 489                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 490                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 491                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 492                 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 493                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 494         }
 495         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 496                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 497
 498         /* cache window 2: context */
 499         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 500                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 501                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 502         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 503                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 504                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 505         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 506                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 507         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 508                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 509
 510         /* non-cache window */
 511         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 512                         VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 513                         lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 514         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 515                         VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 516                         upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 517         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 518                         VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 519         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 520                         VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
 521                         AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
 522
 523         /* VCN global tiling registers */
 524         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 525                 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 526 }
 527
 528 /**
 529  * vcn_v4_0_disable_static_power_gating - disable VCN static power gating
 530  *
 531  * @adev: amdgpu_device pointer
 532  * @inst: instance number
 533  *
 534  * Disable static power gating for VCN block
 535  */
 536 static void vcn_v4_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
 537 {
 538         uint32_t data = 0;
 539
 540         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 541                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 542                         | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
 543                         | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 544                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 545                         | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
 546                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 547                         | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
 548                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 549                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 550                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 551                         | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 552                         | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
 553                         | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 554                         | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 555
 556                 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
 557                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
 558                         UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
 559         } else {
 560                 uint32_t value;
 561
 562                 value = (inst) ? 0x2200800 : 0;
 563                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 564                         | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
 565                         | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 566                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 567                         | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
 568                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 569                         | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
 570                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 571                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 572                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 573                         | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 574                         | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
 575                         | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 576                         | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 577
 578                 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
 579                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value,  0x3F3FFFFF);
 580         }
 581
 582         data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 583         data &= ~0x103;
 584         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 585                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 586                         UVD_POWER_STATUS__UVD_PG_EN_MASK;
 587
 588         WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 589
 590         return;
 591 }
 592
 593 /**
 594  * vcn_v4_0_enable_static_power_gating - enable VCN static power gating
 595  *
 596  * @adev: amdgpu_device pointer
 597  * @inst: instance number
 598  *
 599  * Enable static power gating for VCN block
 600  */
 601 static void vcn_v4_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
 602 {
 603         uint32_t data;
 604
 605         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 606                 /* Before power off, this indicator has to be turned on */
 607                 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 608                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 609                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 610                 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 611
 612                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 613                         | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
 614                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 615                         | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
 616                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 617                         | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
 618                         | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
 619                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 620                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 621                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 622                         | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
 623                         | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
 624                         | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
 625                         | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
 626                 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
 627
 628                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 629                         | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
 630                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 631                         | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
 632                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 633                         | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
 634                         | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
 635                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 636                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 637                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
 638                         | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
 639                         | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
 640                         | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
 641                         | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
 642                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
 643         }
 644
 645         return;
 646 }
 647
 648 /**
 649  * vcn_v4_0_disable_clock_gating - disable VCN clock gating
 650  *
 651  * @adev: amdgpu_device pointer
 652  * @inst: instance number
 653  *
 654  * Disable clock gating for VCN block
 655  */
 656 static void vcn_v4_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
 657 {
 658         uint32_t data;
 659
 660         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 661                 return;
 662
 663         /* VCN disable CGC */
 664         data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 665         data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 666         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 667         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 668         WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 669
 670         data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
 671         data &= ~(UVD_CGC_GATE__SYS_MASK
 672                 | UVD_CGC_GATE__UDEC_MASK
 673                 | UVD_CGC_GATE__MPEG2_MASK
 674                 | UVD_CGC_GATE__REGS_MASK
 675                 | UVD_CGC_GATE__RBC_MASK
 676                 | UVD_CGC_GATE__LMI_MC_MASK
 677                 | UVD_CGC_GATE__LMI_UMC_MASK
 678                 | UVD_CGC_GATE__IDCT_MASK
 679                 | UVD_CGC_GATE__MPRD_MASK
 680                 | UVD_CGC_GATE__MPC_MASK
 681                 | UVD_CGC_GATE__LBSI_MASK
 682                 | UVD_CGC_GATE__LRBBM_MASK
 683                 | UVD_CGC_GATE__UDEC_RE_MASK
 684                 | UVD_CGC_GATE__UDEC_CM_MASK
 685                 | UVD_CGC_GATE__UDEC_IT_MASK
 686                 | UVD_CGC_GATE__UDEC_DB_MASK
 687                 | UVD_CGC_GATE__UDEC_MP_MASK
 688                 | UVD_CGC_GATE__WCB_MASK
 689                 | UVD_CGC_GATE__VCPU_MASK
 690                 | UVD_CGC_GATE__MMSCH_MASK);
 691
 692         WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
 693         SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0,  0xFFFFFFFF);
 694
 695         data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 696         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 697                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 698                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 699                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 700                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 701                 | UVD_CGC_CTRL__SYS_MODE_MASK
 702                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 703                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 704                 | UVD_CGC_CTRL__REGS_MODE_MASK
 705                 | UVD_CGC_CTRL__RBC_MODE_MASK
 706                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 707                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 708                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 709                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 710                 | UVD_CGC_CTRL__MPC_MODE_MASK
 711                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 712                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 713                 | UVD_CGC_CTRL__WCB_MODE_MASK
 714                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 715                 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
 716         WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 717
 718         data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
 719         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 720                 | UVD_SUVD_CGC_GATE__SIT_MASK
 721                 | UVD_SUVD_CGC_GATE__SMP_MASK
 722                 | UVD_SUVD_CGC_GATE__SCM_MASK
 723                 | UVD_SUVD_CGC_GATE__SDB_MASK
 724                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 725                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 726                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 727                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 728                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 729                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 730                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 731                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 732                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 733                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 734                 | UVD_SUVD_CGC_GATE__ENT_MASK
 735                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 736                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 737                 | UVD_SUVD_CGC_GATE__SITE_MASK
 738                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 739                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 740                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 741                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 742                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 743         WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
 744
 745         data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
 746         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 747                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 748                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 749                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 750                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 751                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 752                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 753                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 754                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 755                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 756         WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
 757 }
 758
 759 /**
 760  * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
 761  *
 762  * @adev: amdgpu_device pointer
 763  * @sram_sel: sram select
 764  * @inst_idx: instance number index
 765  * @indirect: indirectly write sram
 766  *
 767  * Disable clock gating for VCN block with dpg mode
 768  */
 769 static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
 770       int inst_idx, uint8_t indirect)
 771 {
 772         uint32_t reg_data = 0;
 773
 774         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 775                 return;
 776
 777         /* enable sw clock gating control */
 778         reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 779         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 780         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 781         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 782                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 783                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 784                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 785                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 786                  UVD_CGC_CTRL__SYS_MODE_MASK |
 787                  UVD_CGC_CTRL__UDEC_MODE_MASK |
 788                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
 789                  UVD_CGC_CTRL__REGS_MODE_MASK |
 790                  UVD_CGC_CTRL__RBC_MODE_MASK |
 791                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 792                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 793                  UVD_CGC_CTRL__IDCT_MODE_MASK |
 794                  UVD_CGC_CTRL__MPRD_MODE_MASK |
 795                  UVD_CGC_CTRL__MPC_MODE_MASK |
 796                  UVD_CGC_CTRL__LBSI_MODE_MASK |
 797                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
 798                  UVD_CGC_CTRL__WCB_MODE_MASK |
 799                  UVD_CGC_CTRL__VCPU_MODE_MASK);
 800         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 801                 VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
 802
 803         /* turn off clock gating */
 804         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 805                 VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
 806
 807         /* turn on SUVD clock gating */
 808         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 809                 VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
 810
 811         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
 812         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 813                 VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
 814 }
 815
 816 /**
 817  * vcn_v4_0_enable_clock_gating - enable VCN clock gating
 818  *
 819  * @adev: amdgpu_device pointer
 820  * @inst: instance number
 821  *
 822  * Enable clock gating for VCN block
 823  */
 824 static void vcn_v4_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
 825 {
 826         uint32_t data;
 827
 828         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 829                 return;
 830
 831         /* enable VCN CGC */
 832         data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 833         data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 834         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 835         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 836         WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 837
 838         data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 839         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 840                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 841                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 842                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 843                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 844                 | UVD_CGC_CTRL__SYS_MODE_MASK
 845                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 846                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 847                 | UVD_CGC_CTRL__REGS_MODE_MASK
 848                 | UVD_CGC_CTRL__RBC_MODE_MASK
 849                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 850                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 851                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 852                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 853                 | UVD_CGC_CTRL__MPC_MODE_MASK
 854                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 855                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 856                 | UVD_CGC_CTRL__WCB_MODE_MASK
 857                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 858                 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
 859         WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 860
 861         data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
 862         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 863                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 864                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 865                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 866                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 867                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 868                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 869                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 870                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 871                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 872         WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
 873
 874         return;
 875 }
 876
 877 /**
 878  * vcn_v4_0_start_dpg_mode - VCN start with dpg mode
 879  *
 880  * @adev: amdgpu_device pointer
 881  * @inst_idx: instance number index
 882  * @indirect: indirectly write sram
 883  *
 884  * Start VCN block with dpg mode
 885  */
 886 static int vcn_v4_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 887 {
 888         volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
 889         struct amdgpu_ring *ring;
 890         uint32_t tmp;
 891         int i;
 892
 893         /* disable register anti-hang mechanism */
 894         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
 895                 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 896         /* enable dynamic power gating mode */
 897         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
 898         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 899         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 900         WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
 901
 902         if (indirect)
 903                 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 904
 905         /* enable clock gating */
 906         vcn_v4_0_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
 907
 908         /* enable VCPU clock */
 909         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 910         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
 911         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 912                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 913
 914         /* disable master interupt */
 915         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 916                 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
 917
 918         /* setup regUVD_LMI_CTRL */
 919         tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 920                 UVD_LMI_CTRL__REQ_MODE_MASK |
 921                 UVD_LMI_CTRL__CRC_RESET_MASK |
 922                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 923                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 924                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 925                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 926                 0x00100000L);
 927         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 928                 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
 929
 930         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 931                 VCN, inst_idx, regUVD_MPC_CNTL),
 932                 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
 933
 934         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 935                 VCN, inst_idx, regUVD_MPC_SET_MUXA0),
 936                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 937                  (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 938                  (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 939                  (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
 940
 941         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 942                 VCN, inst_idx, regUVD_MPC_SET_MUXB0),
 943                  ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 944                  (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 945                  (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 946                  (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
 947
 948         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 949                 VCN, inst_idx, regUVD_MPC_SET_MUX),
 950                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 951                  (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 952                  (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
 953
 954         vcn_v4_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
 955
 956         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 957         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 958         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 959                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 960
 961         /* enable LMI MC and UMC channels */
 962         tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
 963         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 964                 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
 965
 966         /* enable master interrupt */
 967         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 968                 VCN, inst_idx, regUVD_MASTINT_EN),
 969                 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
 970
 971
 972         if (indirect)
 973                 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
 974                         (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
 975                                 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
 976
 977         if (unifiedQ_enabled) {
 978                 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
 979                 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
 980         } else
 981                 ring = &adev->vcn.inst[inst_idx].ring_dec;
 982
 983         WREG32_SOC15(VCN, inst_idx, regVCN_RB4_DB_CTRL,
 984                 ring->doorbell_index << VCN_RB4_DB_CTRL__OFFSET__SHIFT |
 985                 VCN_RB4_DB_CTRL__EN_MASK);
 986
 987         /* program the RB_BASE for ring buffer */
 988         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO4,
 989                 lower_32_bits(ring->gpu_addr));
 990         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI4,
 991                 upper_32_bits(ring->gpu_addr));
 992
 993         WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE4, ring->ring_size / sizeof(uint32_t));
 994
 995         /* reseting ring, fw should not check RB ring */
 996         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
 997         tmp &= ~(VCN_RB_ENABLE__RB4_EN_MASK);
 998         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
 999
1000         /* Initialize the ring buffer's read and write pointers */
1001         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR4);
1002         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR4, tmp);
1003         ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR4);
1004
1005         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1006         tmp |= VCN_RB_ENABLE__RB4_EN_MASK;
1007         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1008
1009         WREG32_SOC15(VCN, inst_idx, regUVD_SCRATCH2, 0);
1010
1011         if (unifiedQ_enabled)
1012                 fw_shared->sq.queue_mode &= ~FW_QUEUE_RING_RESET;
1013
1014         for (i = 0; i < adev->vcn.num_enc_rings; i++) {
1015                 ring = &adev->vcn.inst[inst_idx].ring_enc[i];
1016
1017                 if (i) {
1018                         ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1019
1020                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO2, ring->gpu_addr);
1021                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1022                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE2, ring->ring_size / 4);
1023                         tmp= RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR2);
1024                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR2, tmp);
1025                         ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR2);
1026
1027                         WREG32_SOC15(VCN, inst_idx, regVCN_RB2_DB_CTRL,
1028                                 ring->doorbell_index << VCN_RB2_DB_CTRL__OFFSET__SHIFT |
1029                                 VCN_RB2_DB_CTRL__EN_MASK);
1030                 } else {
1031                         ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1032
1033                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
1034                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1035                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
1036                         tmp= RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
1037                         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
1038                         ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1039
1040                         WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
1041                                 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1042                                 VCN_RB1_DB_CTRL__EN_MASK);
1043                 }
1044         }
1045         return 0;
1046 }
1047
1048
1049 /**
1050  * vcn_v4_0_start - VCN start
1051  *
1052  * @adev: amdgpu_device pointer
1053  *
1054  * Start VCN block
1055  */
1056 static int vcn_v4_0_start(struct amdgpu_device *adev)
1057 {
1058         volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1059         struct amdgpu_ring *ring;
1060         uint32_t tmp;
1061         int i, j, k, r;
1062
1063         if (adev->pm.dpm_enabled)
1064                 amdgpu_dpm_enable_uvd(adev, true);
1065
1066         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1067                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1068                         r = vcn_v4_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1069                         continue;
1070                 }
1071
1072                 /* disable VCN power gating */
1073                 vcn_v4_0_disable_static_power_gating(adev, i);
1074
1075                 /* set VCN status busy */
1076                 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1077                 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1078
1079                 /*SW clock gating */
1080                 vcn_v4_0_disable_clock_gating(adev, i);
1081
1082                 /* enable VCPU clock */
1083                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1084                         UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1085
1086                 /* disable master interrupt */
1087                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1088                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
1089
1090                 /* enable LMI MC and UMC channels */
1091                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1092                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1093
1094                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1095                 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1096                 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1097                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1098
1099                 /* setup regUVD_LMI_CTRL */
1100                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1101                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1102                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1103                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1104                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1105                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1106
1107                 /* setup regUVD_MPC_CNTL */
1108                 tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1109                 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1110                 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1111                 WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1112
1113                 /* setup UVD_MPC_SET_MUXA0 */
1114                 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1115                         ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1116                         (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1117                         (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1118                         (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1119
1120                 /* setup UVD_MPC_SET_MUXB0 */
1121                 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1122                         ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1123                         (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1124                         (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1125                         (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1126
1127                 /* setup UVD_MPC_SET_MUX */
1128                 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1129                         ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1130                         (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1131                         (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1132
1133                 vcn_v4_0_mc_resume(adev, i);
1134
1135                 /* VCN global tiling registers */
1136                 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1137                         adev->gfx.config.gb_addr_config);
1138
1139                 /* unblock VCPU register access */
1140                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1141                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1142
1143                 /* release VCPU reset to boot */
1144                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1145                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
1146
1147                 for (j = 0; j < 10; ++j) {
1148                         uint32_t status;
1149
1150                         for (k = 0; k < 100; ++k) {
1151                                 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1152                                 if (status & 2)
1153                                         break;
1154                                 mdelay(10);
1155                                 if (amdgpu_emu_mode==1)
1156                                         msleep(1);
1157                         }
1158
1159                         if (amdgpu_emu_mode==1) {
1160                                 if (status & 2) {
1161                                         r = 0;
1162                                         break;
1163                                 }
1164                         } else {
1165                                 r = 0;
1166                                 if (status & 2)
1167                                         break;
1168
1169                                 dev_err(adev->dev, "VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1170                                         WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1171                                         UVD_VCPU_CNTL__BLK_RST_MASK,
1172                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
1173                                 mdelay(10);
1174                                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1175                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
1176
1177                                 mdelay(10);
1178                                 r = -1;
1179                         }
1180                 }
1181
1182                 if (r) {
1183                         dev_err(adev->dev, "VCN[%d] decode not responding, giving up!!!\n", i);
1184                         return r;
1185                 }
1186
1187                 /* enable master interrupt */
1188                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1189                         UVD_MASTINT_EN__VCPU_EN_MASK,
1190                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
1191
1192                 /* clear the busy bit of VCN_STATUS */
1193                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1194                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1195
1196                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1197                 if (unifiedQ_enabled) {
1198                         ring = &adev->vcn.inst[i].ring_enc[0];
1199                         fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1200                 } else {
1201                         ring = &adev->vcn.inst[i].ring_dec;
1202
1203                         WREG32_SOC15(VCN, i, regVCN_RB4_DB_CTRL,
1204                                 ring->doorbell_index << VCN_RB4_DB_CTRL__OFFSET__SHIFT |
1205                                 VCN_RB4_DB_CTRL__EN_MASK);
1206
1207                         /* program the RB_BASE for ring buffer */
1208                         WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO4,
1209                                 lower_32_bits(ring->gpu_addr));
1210                         WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI4,
1211                                 upper_32_bits(ring->gpu_addr));
1212
1213                         WREG32_SOC15(VCN, i, regUVD_RB_SIZE4, ring->ring_size / sizeof(uint32_t));
1214
1215                         /* resetting ring, fw should not check RB ring */
1216                         tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1217                         tmp &= ~(VCN_RB_ENABLE__RB4_EN_MASK);
1218                         WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1219
1220                         /* Initialize the ring buffer's read and write pointers */
1221                         tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR4);
1222                         WREG32_SOC15(VCN, i, regUVD_RB_WPTR4, tmp);
1223                         ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR4);
1224
1225                         tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1226                         tmp |= VCN_RB_ENABLE__RB4_EN_MASK;
1227                         WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1228
1229                         ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_RPTR4);
1230                 }
1231                 ring = &adev->vcn.inst[i].ring_enc[0];
1232                 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1233                         ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1234                         VCN_RB1_DB_CTRL__EN_MASK);
1235                 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1236                 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1237                 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1238                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1239                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1240                 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1241                 if (unifiedQ_enabled)
1242                         fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1243                 else {
1244                         ring = &adev->vcn.inst[i].ring_enc[1];
1245                         WREG32_SOC15(VCN, i, regVCN_RB2_DB_CTRL,
1246                                 ring->doorbell_index << VCN_RB2_DB_CTRL__OFFSET__SHIFT |
1247                                 VCN_RB2_DB_CTRL__EN_MASK);
1248                         tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR2);
1249                         WREG32_SOC15(VCN, i, regUVD_RB_WPTR2, tmp);
1250                         ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR2);
1251                         WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO2, ring->gpu_addr);
1252                         WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1253                         WREG32_SOC15(VCN, i, regUVD_RB_SIZE2, ring->ring_size / 4);
1254                 }
1255         }
1256
1257         return 0;
1258 }
1259
1260 /**
1261  * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1262  *
1263  * @adev: amdgpu_device pointer
1264  * @inst_idx: instance number index
1265  *
1266  * Stop VCN block with dpg mode
1267  */
1268 static int vcn_v4_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1269 {
1270         uint32_t tmp;
1271
1272         /* Wait for power status to be 1 */
1273         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1274                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1275
1276         /* wait for read ptr to be equal to write ptr */
1277         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1278         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1279
1280         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR2);
1281         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1282
1283         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR4);
1284         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR4, tmp, 0xFFFFFFFF);
1285
1286         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1287                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1288
1289         /* disable dynamic power gating mode */
1290         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1291                 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1292         return 0;
1293 }
1294
1295 /**
1296  * vcn_v4_0_stop - VCN stop
1297  *
1298  * @adev: amdgpu_device pointer
1299  *
1300  * Stop VCN block
1301  */
1302 static int vcn_v4_0_stop(struct amdgpu_device *adev)
1303 {
1304         uint32_t tmp;
1305         int i, r = 0;
1306
1307         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1308                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1309                         r = vcn_v4_0_stop_dpg_mode(adev, i);
1310                         continue;
1311                 }
1312
1313                 /* wait for vcn idle */
1314                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1315                 if (r)
1316                         return r;
1317
1318                 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1319                         UVD_LMI_STATUS__READ_CLEAN_MASK |
1320                         UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1321                         UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1322                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1323                 if (r)
1324                         return r;
1325
1326                 /* disable LMI UMC channel */
1327                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1328                 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1329                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1330                 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1331                         UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1332                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1333                 if (r)
1334                         return r;
1335
1336                 /* block VCPU register access */
1337                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1338                                 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1339                                 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1340
1341                 /* reset VCPU */
1342                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1343                                 UVD_VCPU_CNTL__BLK_RST_MASK,
1344                                 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1345
1346                 /* disable VCPU clock */
1347                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1348                                 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1349
1350                 /* apply soft reset */
1351                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1352                 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1353                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1354                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1355                 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1356                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1357
1358                 /* clear status */
1359                 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1360
1361                 /* apply HW clock gating */
1362                 vcn_v4_0_enable_clock_gating(adev, i);
1363
1364                 /* enable VCN power gating */
1365                 vcn_v4_0_enable_static_power_gating(adev, i);
1366         }
1367
1368         if (adev->pm.dpm_enabled)
1369                 amdgpu_dpm_enable_uvd(adev, false);
1370
1371         return 0;
1372 }
1373
1374 /**
1375  * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1376  *
1377  * @adev: amdgpu_device pointer
1378  * @inst_idx: instance number index
1379  * @new_state: pause state
1380  *
1381  * Pause dpg mode for VCN block
1382  */
1383 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
1384       struct dpg_pause_state *new_state)
1385 {
1386         uint32_t reg_data = 0;
1387         int ret_code;
1388
1389         /* pause/unpause if state is changed */
1390         if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1391                 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1392                         adev->vcn.inst[inst_idx].pause_state.fw_based,  new_state->fw_based);
1393                 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1394                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1395
1396                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1397                         ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1398                                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1399
1400                         if (!ret_code) {
1401                                 /* pause DPG */
1402                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1403                                 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1404
1405                                 /* wait for ACK */
1406                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1407                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1408                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1409
1410                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1411                                         UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1412                         }
1413                 } else {
1414                         /* unpause dpg, no need to wait */
1415                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1416                         WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1417                         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1418                                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1419                 }
1420                 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1421         }
1422
1423         return 0;
1424 }
1425
1426 /**
1427  * vcn_v4_0_dec_ring_get_rptr - get read pointer
1428  *
1429  * @ring: amdgpu_ring pointer
1430  *
1431  * Returns the current hardware read pointer
1432  */
1433 static uint64_t vcn_v4_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1434 {
1435         struct amdgpu_device *adev = ring->adev;
1436
1437         return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR4);
1438 }
1439
1440 /**
1441  * vcn_v4_0_dec_ring_get_wptr - get write pointer
1442  *
1443  * @ring: amdgpu_ring pointer
1444  *
1445  * Returns the current hardware write pointer
1446  */
1447 static uint64_t vcn_v4_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1448 {
1449         struct amdgpu_device *adev = ring->adev;
1450
1451         if (ring->use_doorbell)
1452                 return *ring->wptr_cpu_addr;
1453         else
1454                 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR4);
1455 }
1456
1457 /**
1458  * vcn_v4_0_dec_ring_set_wptr - set write pointer
1459  *
1460  * @ring: amdgpu_ring pointer
1461  *
1462  * Commits the write pointer to the hardware
1463  */
1464 static void vcn_v4_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1465 {
1466         struct amdgpu_device *adev = ring->adev;
1467
1468         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1469                 WREG32_SOC15(VCN, ring->me, regUVD_SCRATCH2,
1470                         lower_32_bits(ring->wptr));
1471         }
1472
1473         if (ring->use_doorbell) {
1474                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1475                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1476         } else {
1477                 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR4, lower_32_bits(ring->wptr));
1478         }
1479 }
1480
1481 static const struct amdgpu_ring_funcs vcn_v4_0_dec_sw_ring_vm_funcs = {
1482         .type = AMDGPU_RING_TYPE_VCN_DEC,
1483         .align_mask = 0x3f,
1484         .nop = VCN_DEC_SW_CMD_NO_OP,
1485         .vmhub = AMDGPU_MMHUB_0,
1486         .get_rptr = vcn_v4_0_dec_ring_get_rptr,
1487         .get_wptr = vcn_v4_0_dec_ring_get_wptr,
1488         .set_wptr = vcn_v4_0_dec_ring_set_wptr,
1489         .emit_frame_size =
1490                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1491                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1492                 VCN_SW_RING_EMIT_FRAME_SIZE,
1493         .emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1494         .emit_ib = vcn_dec_sw_ring_emit_ib,
1495         .emit_fence = vcn_dec_sw_ring_emit_fence,
1496         .emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1497         .test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1498         .test_ib = amdgpu_vcn_dec_sw_ring_test_ib,
1499         .insert_nop = amdgpu_ring_insert_nop,
1500         .insert_end = vcn_dec_sw_ring_insert_end,
1501         .pad_ib = amdgpu_ring_generic_pad_ib,
1502         .begin_use = amdgpu_vcn_ring_begin_use,
1503         .end_use = amdgpu_vcn_ring_end_use,
1504         .emit_wreg = vcn_dec_sw_ring_emit_wreg,
1505         .emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1506         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1507 };
1508
1509 /**
1510  * vcn_v4_0_enc_ring_get_rptr - get enc read pointer
1511  *
1512  * @ring: amdgpu_ring pointer
1513  *
1514  * Returns the current hardware enc read pointer
1515  */
1516 static uint64_t vcn_v4_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1517 {
1518         struct amdgpu_device *adev = ring->adev;
1519
1520         if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1521                 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1522         else
1523                 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR2);
1524 }
1525
1526 /**
1527  * vcn_v4_0_enc_ring_get_wptr - get enc write pointer
1528  *
1529  * @ring: amdgpu_ring pointer
1530  *
1531  * Returns the current hardware enc write pointer
1532  */
1533 static uint64_t vcn_v4_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1534 {
1535         struct amdgpu_device *adev = ring->adev;
1536
1537         if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1538                 if (ring->use_doorbell)
1539                         return *ring->wptr_cpu_addr;
1540                 else
1541                         return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1542         } else {
1543                 if (ring->use_doorbell)
1544                         return *ring->wptr_cpu_addr;
1545                 else
1546                         return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR2);
1547         }
1548 }
1549
1550 /**
1551  * vcn_v4_0_enc_ring_set_wptr - set enc write pointer
1552  *
1553  * @ring: amdgpu_ring pointer
1554  *
1555  * Commits the enc write pointer to the hardware
1556  */
1557 static void vcn_v4_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1558 {
1559         struct amdgpu_device *adev = ring->adev;
1560
1561         if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1562                 if (ring->use_doorbell) {
1563                         *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1564                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1565                 } else {
1566                         WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1567                 }
1568         } else {
1569                 if (ring->use_doorbell) {
1570                         *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1571                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1572                 } else {
1573                         WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1574                 }
1575         }
1576 }
1577
1578 static const struct amdgpu_ring_funcs vcn_v4_0_enc_ring_vm_funcs = {
1579         .type = AMDGPU_RING_TYPE_VCN_ENC,
1580         .align_mask = 0x3f,
1581         .nop = VCN_ENC_CMD_NO_OP,
1582         .vmhub = AMDGPU_MMHUB_0,
1583         .get_rptr = vcn_v4_0_enc_ring_get_rptr,
1584         .get_wptr = vcn_v4_0_enc_ring_get_wptr,
1585         .set_wptr = vcn_v4_0_enc_ring_set_wptr,
1586         .emit_frame_size =
1587                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1588                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1589                 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1590                 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1591                 1, /* vcn_v2_0_enc_ring_insert_end */
1592         .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1593         .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1594         .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1595         .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1596         .test_ring = amdgpu_vcn_enc_ring_test_ring,
1597         .test_ib = amdgpu_vcn_enc_ring_test_ib,
1598         .insert_nop = amdgpu_ring_insert_nop,
1599         .insert_end = vcn_v2_0_enc_ring_insert_end,
1600         .pad_ib = amdgpu_ring_generic_pad_ib,
1601         .begin_use = amdgpu_vcn_ring_begin_use,
1602         .end_use = amdgpu_vcn_ring_end_use,
1603         .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1604         .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1605         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1606 };
1607
1608 /**
1609  * vcn_v4_0_set_dec_ring_funcs - set dec ring functions
1610  *
1611  * @adev: amdgpu_device pointer
1612  *
1613  * Set decode ring functions
1614  */
1615 static void vcn_v4_0_set_dec_ring_funcs(struct amdgpu_device *adev)
1616 {
1617         int i;
1618
1619         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1620                 if (adev->vcn.harvest_config & (1 << i))
1621                         continue;
1622
1623                 adev->vcn.inst[i].ring_dec.funcs = &vcn_v4_0_dec_sw_ring_vm_funcs;
1624                 adev->vcn.inst[i].ring_dec.me = i;
1625                 DRM_INFO("VCN(%d) decode software ring is enabled in VM mode\n", i);
1626         }
1627 }
1628
1629 /**
1630  * vcn_v4_0_set_enc_ring_funcs - set enc ring functions
1631  *
1632  * @adev: amdgpu_device pointer
1633  *
1634  * Set encode ring functions
1635  */
1636 static void vcn_v4_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1637 {
1638         int i, j;
1639
1640         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1641                 if (adev->vcn.harvest_config & (1 << i))
1642                         continue;
1643
1644                 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1645                         adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v4_0_enc_ring_vm_funcs;
1646                         adev->vcn.inst[i].ring_enc[j].me = i;
1647                 }
1648                 DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
1649         }
1650 }
1651
1652 /**
1653  * vcn_v4_0_is_idle - check VCN block is idle
1654  *
1655  * @handle: amdgpu_device pointer
1656  *
1657  * Check whether VCN block is idle
1658  */
1659 static bool vcn_v4_0_is_idle(void *handle)
1660 {
1661         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1662         int i, ret = 1;
1663
1664         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1665                 if (adev->vcn.harvest_config & (1 << i))
1666                         continue;
1667
1668                 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1669         }
1670
1671         return ret;
1672 }
1673
1674 /**
1675  * vcn_v4_0_wait_for_idle - wait for VCN block idle
1676  *
1677  * @handle: amdgpu_device pointer
1678  *
1679  * Wait for VCN block idle
1680  */
1681 static int vcn_v4_0_wait_for_idle(void *handle)
1682 {
1683         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1684         int i, ret = 0;
1685
1686         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1687                 if (adev->vcn.harvest_config & (1 << i))
1688                         continue;
1689
1690                 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1691                         UVD_STATUS__IDLE);
1692                 if (ret)
1693                         return ret;
1694         }
1695
1696         return ret;
1697 }
1698
1699 /**
1700  * vcn_v4_0_set_clockgating_state - set VCN block clockgating state
1701  *
1702  * @handle: amdgpu_device pointer
1703  * @state: clock gating state
1704  *
1705  * Set VCN block clockgating state
1706  */
1707 static int vcn_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1708 {
1709         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1710         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1711         int i;
1712
1713         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1714                 if (adev->vcn.harvest_config & (1 << i))
1715                         continue;
1716
1717                 if (enable) {
1718                         if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1719                                 return -EBUSY;
1720                         vcn_v4_0_enable_clock_gating(adev, i);
1721                 } else {
1722                         vcn_v4_0_disable_clock_gating(adev, i);
1723                 }
1724         }
1725
1726         return 0;
1727 }
1728
1729 /**
1730  * vcn_v4_0_set_powergating_state - set VCN block powergating state
1731  *
1732  * @handle: amdgpu_device pointer
1733  * @state: power gating state
1734  *
1735  * Set VCN block powergating state
1736  */
1737 static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1738 {
1739         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1740         int ret;
1741
1742         if(state == adev->vcn.cur_state)
1743                 return 0;
1744
1745         if (state == AMD_PG_STATE_GATE)
1746                 ret = vcn_v4_0_stop(adev);
1747         else
1748                 ret = vcn_v4_0_start(adev);
1749
1750         if(!ret)
1751                 adev->vcn.cur_state = state;
1752
1753         return ret;
1754 }
1755
1756 /**
1757  * vcn_v4_0_set_interrupt_state - set VCN block interrupt state
1758  *
1759  * @adev: amdgpu_device pointer
1760  * @source: interrupt sources
1761  * @type: interrupt types
1762  * @state: interrupt states
1763  *
1764  * Set VCN block interrupt state
1765  */
1766 static int vcn_v4_0_set_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1767       unsigned type, enum amdgpu_interrupt_state state)
1768 {
1769         return 0;
1770 }
1771
1772 /**
1773  * vcn_v4_0_process_interrupt - process VCN block interrupt
1774  *
1775  * @adev: amdgpu_device pointer
1776  * @source: interrupt sources
1777  * @entry: interrupt entry from clients and sources
1778  *
1779  * Process VCN block interrupt
1780  */
1781 static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1782       struct amdgpu_iv_entry *entry)
1783 {
1784         uint32_t ip_instance;
1785
1786         switch (entry->client_id) {
1787         case SOC15_IH_CLIENTID_VCN:
1788                 ip_instance = 0;
1789                 break;
1790         case SOC15_IH_CLIENTID_VCN1:
1791                 ip_instance = 1;
1792                 break;
1793         default:
1794                 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1795                 return 0;
1796         }
1797
1798         DRM_DEBUG("IH: VCN TRAP\n");
1799
1800         switch (entry->src_id) {
1801         case VCN_4_0__SRCID__UVD_TRAP:
1802                 if (!unifiedQ_enabled) {
1803                         amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
1804                         break;
1805                 }
1806                 break;
1807         case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1808                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1809                 break;
1810         case VCN_4_0__SRCID__UVD_ENC_LOW_LATENCY:
1811                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
1812                 break;
1813         default:
1814                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1815                           entry->src_id, entry->src_data[0]);
1816                 break;
1817         }
1818
1819         return 0;
1820 }
1821
1822 static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
1823         .set = vcn_v4_0_set_interrupt_state,
1824         .process = vcn_v4_0_process_interrupt,
1825 };
1826
1827 /**
1828  * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
1829  *
1830  * @adev: amdgpu_device pointer
1831  *
1832  * Set VCN block interrupt irq functions
1833  */
1834 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
1835 {
1836         int i;
1837
1838         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1839                 if (adev->vcn.harvest_config & (1 << i))
1840                         continue;
1841
1842                 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1843                 adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
1844         }
1845 }
1846
1847 static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
1848         .name = "vcn_v4_0",
1849         .early_init = vcn_v4_0_early_init,
1850         .late_init = NULL,
1851         .sw_init = vcn_v4_0_sw_init,
1852         .sw_fini = vcn_v4_0_sw_fini,
1853         .hw_init = vcn_v4_0_hw_init,
1854         .hw_fini = vcn_v4_0_hw_fini,
1855         .suspend = vcn_v4_0_suspend,
1856         .resume = vcn_v4_0_resume,
1857         .is_idle = vcn_v4_0_is_idle,
1858         .wait_for_idle = vcn_v4_0_wait_for_idle,
1859         .check_soft_reset = NULL,
1860         .pre_soft_reset = NULL,
1861         .soft_reset = NULL,
1862         .post_soft_reset = NULL,
1863         .set_clockgating_state = vcn_v4_0_set_clockgating_state,
1864         .set_powergating_state = vcn_v4_0_set_powergating_state,
1865 };
1866
1867 const struct amdgpu_ip_block_version vcn_v4_0_ip_block =
1868 {
1869         .type = AMD_IP_BLOCK_TYPE_VCN,
1870         .major = 4,
1871         .minor = 0,
1872         .rev = 0,
1873         .funcs = &vcn_v4_0_ip_funcs,
1874 };