drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_vcn.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "amdgpu_pm.h"
  31 #include "amdgpu_psp.h"
  32
  33 #include "vcn/vcn_2_0_0_offset.h"
  34 #include "vcn/vcn_2_0_0_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
  36
  37 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET                        0x1fd
  38 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET                    0x503
  39 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET                  0x504
  40 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET                  0x505
  41 #define mmUVD_NO_OP_INTERNAL_OFFSET                             0x53f
  42 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET                       0x54a
  43 #define mmUVD_SCRATCH9_INTERNAL_OFFSET                          0xc01d
  44
  45 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET                   0x1e1
  46 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET         0x5a6
  47 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET          0x5a7
  48 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET                       0x1e2
  49
  50 #define mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET                         0x1bfff
  51 #define mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET                            0x4029
  52 #define mmUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET                          0x402a
  53 #define mmUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET                          0x402b
  54 #define mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET          0x40ea
  55 #define mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET         0x40eb
  56 #define mmUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET                          0x40cf
  57 #define mmUVD_LMI_JPEG_VMID_INTERNAL_OFFSET                             0x40d1
  58 #define mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET                 0x40e8
  59 #define mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET                0x40e9
  60 #define mmUVD_JRBC_IB_SIZE_INTERNAL_OFFSET                              0x4082
  61 #define mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET          0x40ec
  62 #define mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET         0x40ed
  63 #define mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET                     0x4085
  64 #define mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET                          0x4084
  65 #define mmUVD_JRBC_STATUS_INTERNAL_OFFSET                               0x4089
  66 #define mmUVD_JPEG_PITCH_INTERNAL_OFFSET                                0x401f
  67
  68 #define JRBC_DEC_EXTERNAL_REG_WRITE_ADDR                                0x18000
  69
  70 #define mmUVD_RBC_XX_IB_REG_CHECK                                       0x026b
  71 #define mmUVD_RBC_XX_IB_REG_CHECK_BASE_IDX                              1
  72 #define mmUVD_REG_XX_MASK                                               0x026c
  73 #define mmUVD_REG_XX_MASK_BASE_IDX                                      1
  74
  75 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  76 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  77 static void vcn_v2_0_set_jpeg_ring_funcs(struct amdgpu_device *adev);
  78 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev);
  79 static int vcn_v2_0_set_powergating_state(void *handle,
  80                                 enum amd_powergating_state state);
  81 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
  82                                 struct dpg_pause_state *new_state);
  83
  84 /**
  85  * vcn_v2_0_early_init - set function pointers
  86  *
  87  * @handle: amdgpu_device pointer
  88  *
  89  * Set ring and irq function pointers
  90  */
  91 static int vcn_v2_0_early_init(void *handle)
  92 {
  93         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  94
  95         adev->vcn.num_vcn_inst = 1;
  96         adev->vcn.num_enc_rings = 2;
  97
  98         vcn_v2_0_set_dec_ring_funcs(adev);
  99         vcn_v2_0_set_enc_ring_funcs(adev);
 100         vcn_v2_0_set_jpeg_ring_funcs(adev);
 101         vcn_v2_0_set_irq_funcs(adev);
 102
 103         return 0;
 104 }
 105
 106 /**
 107  * vcn_v2_0_sw_init - sw init for VCN block
 108  *
 109  * @handle: amdgpu_device pointer
 110  *
 111  * Load firmware and sw initialization
 112  */
 113 static int vcn_v2_0_sw_init(void *handle)
 114 {
 115         struct amdgpu_ring *ring;
 116         int i, r;
 117         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 118
 119         /* VCN DEC TRAP */
 120         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 121                               VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT,
 122                               &adev->vcn.inst->irq);
 123         if (r)
 124                 return r;
 125
 126         /* VCN ENC TRAP */
 127         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 128                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 129                                       i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
 130                                       &adev->vcn.inst->irq);
 131                 if (r)
 132                         return r;
 133         }
 134
 135         /* VCN JPEG TRAP */
 136         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 137                               VCN_2_0__SRCID__JPEG_DECODE, &adev->vcn.inst->irq);
 138         if (r)
 139                 return r;
 140
 141         r = amdgpu_vcn_sw_init(adev);
 142         if (r)
 143                 return r;
 144
 145         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 146                 const struct common_firmware_header *hdr;
 147                 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
 148                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
 149                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
 150                 adev->firmware.fw_size +=
 151                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
 152                 DRM_INFO("PSP loading VCN firmware\n");
 153         }
 154
 155         r = amdgpu_vcn_resume(adev);
 156         if (r)
 157                 return r;
 158
 159         ring = &adev->vcn.inst->ring_dec;
 160
 161         ring->use_doorbell = true;
 162         ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
 163
 164         sprintf(ring->name, "vcn_dec");
 165         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0);
 166         if (r)
 167                 return r;
 168
 169         adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
 170         adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
 171         adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
 172         adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
 173         adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
 174         adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
 175
 176         adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
 177         adev->vcn.inst->external.scratch9 = SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
 178         adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
 179         adev->vcn.inst->external.data0 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
 180         adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
 181         adev->vcn.inst->external.data1 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
 182         adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
 183         adev->vcn.inst->external.cmd = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
 184         adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
 185         adev->vcn.inst->external.nop = SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
 186
 187         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 188                 ring = &adev->vcn.inst->ring_enc[i];
 189                 ring->use_doorbell = true;
 190                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i;
 191                 sprintf(ring->name, "vcn_enc%d", i);
 192                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0);
 193                 if (r)
 194                         return r;
 195         }
 196
 197         ring = &adev->vcn.inst->ring_jpeg;
 198         ring->use_doorbell = true;
 199         ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
 200         sprintf(ring->name, "vcn_jpeg");
 201         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0);
 202         if (r)
 203                 return r;
 204
 205         adev->vcn.pause_dpg_mode = vcn_v2_0_pause_dpg_mode;
 206
 207         adev->vcn.internal.jpeg_pitch = mmUVD_JPEG_PITCH_INTERNAL_OFFSET;
 208         adev->vcn.inst->external.jpeg_pitch = SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_PITCH);
 209
 210         return 0;
 211 }
 212
 213 /**
 214  * vcn_v2_0_sw_fini - sw fini for VCN block
 215  *
 216  * @handle: amdgpu_device pointer
 217  *
 218  * VCN suspend and free up sw allocation
 219  */
 220 static int vcn_v2_0_sw_fini(void *handle)
 221 {
 222         int r;
 223         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 224
 225         r = amdgpu_vcn_suspend(adev);
 226         if (r)
 227                 return r;
 228
 229         r = amdgpu_vcn_sw_fini(adev);
 230
 231         return r;
 232 }
 233
 234 /**
 235  * vcn_v2_0_hw_init - start and test VCN block
 236  *
 237  * @handle: amdgpu_device pointer
 238  *
 239  * Initialize the hardware, boot up the VCPU and do some testing
 240  */
 241 static int vcn_v2_0_hw_init(void *handle)
 242 {
 243         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 244         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 245         int i, r;
 246
 247         adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 248                                              ring->doorbell_index, 0);
 249
 250         r = amdgpu_ring_test_helper(ring);
 251         if (r)
 252                 goto done;
 253
 254         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 255                 ring = &adev->vcn.inst->ring_enc[i];
 256                 r = amdgpu_ring_test_helper(ring);
 257                 if (r)
 258                         goto done;
 259         }
 260
 261         ring = &adev->vcn.inst->ring_jpeg;
 262         r = amdgpu_ring_test_helper(ring);
 263         if (r)
 264                 goto done;
 265
 266 done:
 267         if (!r)
 268                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 269                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 270
 271         return r;
 272 }
 273
 274 /**
 275  * vcn_v2_0_hw_fini - stop the hardware block
 276  *
 277  * @handle: amdgpu_device pointer
 278  *
 279  * Stop the VCN block, mark ring as not ready any more
 280  */
 281 static int vcn_v2_0_hw_fini(void *handle)
 282 {
 283         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 284         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 285         int i;
 286
 287         if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 288             (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 289               RREG32_SOC15(VCN, 0, mmUVD_STATUS)))
 290                 vcn_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 291
 292         ring->sched.ready = false;
 293
 294         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 295                 ring = &adev->vcn.inst->ring_enc[i];
 296                 ring->sched.ready = false;
 297         }
 298
 299         ring = &adev->vcn.inst->ring_jpeg;
 300         ring->sched.ready = false;
 301
 302         return 0;
 303 }
 304
 305 /**
 306  * vcn_v2_0_suspend - suspend VCN block
 307  *
 308  * @handle: amdgpu_device pointer
 309  *
 310  * HW fini and suspend VCN block
 311  */
 312 static int vcn_v2_0_suspend(void *handle)
 313 {
 314         int r;
 315         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 316
 317         r = vcn_v2_0_hw_fini(adev);
 318         if (r)
 319                 return r;
 320
 321         r = amdgpu_vcn_suspend(adev);
 322
 323         return r;
 324 }
 325
 326 /**
 327  * vcn_v2_0_resume - resume VCN block
 328  *
 329  * @handle: amdgpu_device pointer
 330  *
 331  * Resume firmware and hw init VCN block
 332  */
 333 static int vcn_v2_0_resume(void *handle)
 334 {
 335         int r;
 336         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 337
 338         r = amdgpu_vcn_resume(adev);
 339         if (r)
 340                 return r;
 341
 342         r = vcn_v2_0_hw_init(adev);
 343
 344         return r;
 345 }
 346
 347 /**
 348  * vcn_v2_0_mc_resume - memory controller programming
 349  *
 350  * @adev: amdgpu_device pointer
 351  *
 352  * Let the VCN memory controller know it's offsets
 353  */
 354 static void vcn_v2_0_mc_resume(struct amdgpu_device *adev)
 355 {
 356         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 357         uint32_t offset;
 358
 359         /* cache window 0: fw */
 360         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 361                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 362                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
 363                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 364                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
 365                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
 366                 offset = 0;
 367         } else {
 368                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 369                         lower_32_bits(adev->vcn.inst->gpu_addr));
 370                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 371                         upper_32_bits(adev->vcn.inst->gpu_addr));
 372                 offset = size;
 373                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 374                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 375         }
 376
 377         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 378
 379         /* cache window 1: stack */
 380         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 381                 lower_32_bits(adev->vcn.inst->gpu_addr + offset));
 382         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 383                 upper_32_bits(adev->vcn.inst->gpu_addr + offset));
 384         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 385         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 386
 387         /* cache window 2: context */
 388         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 389                 lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 390         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 391                 upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 392         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 393         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 394
 395         WREG32_SOC15(UVD, 0, mmUVD_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 396         WREG32_SOC15(UVD, 0, mmJPEG_DEC_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 397 }
 398
 399 static void vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device *adev, bool indirect)
 400 {
 401         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 402         uint32_t offset;
 403
 404         /* cache window 0: fw */
 405         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 406                 if (!indirect) {
 407                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 408                                 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 409                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo), 0, indirect);
 410                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 411                                 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 412                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi), 0, indirect);
 413                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 414                                 UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 415                 } else {
 416                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 417                                 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 418                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 419                                 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 420                         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 421                                 UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 422                 }
 423                 offset = 0;
 424         } else {
 425                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 426                         UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 427                         lower_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
 428                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 429                         UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 430                         upper_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
 431                 offset = size;
 432                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 433                         UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
 434                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 435         }
 436
 437         if (!indirect)
 438                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 439                         UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 440         else
 441                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 442                         UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 443
 444         /* cache window 1: stack */
 445         if (!indirect) {
 446                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 447                         UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 448                         lower_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
 449                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 450                         UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 451                         upper_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
 452                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 453                         UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 454         } else {
 455                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 456                         UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 457                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 458                         UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 459                 WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 460                         UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 461         }
 462         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 463                 UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 464
 465         /* cache window 2: context */
 466         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 467                 UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 468                 lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 469         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 470                 UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 471                 upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 472         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 473                 UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 474         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 475                 UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 476
 477         /* non-cache window */
 478         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 479                 UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 0, 0, indirect);
 480         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 481                 UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 0, 0, indirect);
 482         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 483                 UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 484         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 485                 UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0), 0, 0, indirect);
 486
 487         /* VCN global tiling registers */
 488         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 489                 UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 490 }
 491
 492 /**
 493  * vcn_v2_0_disable_clock_gating - disable VCN clock gating
 494  *
 495  * @adev: amdgpu_device pointer
 496  * @sw: enable SW clock gating
 497  *
 498  * Disable clock gating for VCN block
 499  */
 500 static void vcn_v2_0_disable_clock_gating(struct amdgpu_device *adev)
 501 {
 502         uint32_t data;
 503
 504         /* UVD disable CGC */
 505         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 506         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 507                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 508         else
 509                 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 510         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 511         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 512         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 513
 514         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 515         data &= ~(UVD_CGC_GATE__SYS_MASK
 516                 | UVD_CGC_GATE__UDEC_MASK
 517                 | UVD_CGC_GATE__MPEG2_MASK
 518                 | UVD_CGC_GATE__REGS_MASK
 519                 | UVD_CGC_GATE__RBC_MASK
 520                 | UVD_CGC_GATE__LMI_MC_MASK
 521                 | UVD_CGC_GATE__LMI_UMC_MASK
 522                 | UVD_CGC_GATE__IDCT_MASK
 523                 | UVD_CGC_GATE__MPRD_MASK
 524                 | UVD_CGC_GATE__MPC_MASK
 525                 | UVD_CGC_GATE__LBSI_MASK
 526                 | UVD_CGC_GATE__LRBBM_MASK
 527                 | UVD_CGC_GATE__UDEC_RE_MASK
 528                 | UVD_CGC_GATE__UDEC_CM_MASK
 529                 | UVD_CGC_GATE__UDEC_IT_MASK
 530                 | UVD_CGC_GATE__UDEC_DB_MASK
 531                 | UVD_CGC_GATE__UDEC_MP_MASK
 532                 | UVD_CGC_GATE__WCB_MASK
 533                 | UVD_CGC_GATE__VCPU_MASK
 534                 | UVD_CGC_GATE__SCPU_MASK);
 535         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 536
 537         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 538         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 539                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 540                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 541                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 542                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 543                 | UVD_CGC_CTRL__SYS_MODE_MASK
 544                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 545                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 546                 | UVD_CGC_CTRL__REGS_MODE_MASK
 547                 | UVD_CGC_CTRL__RBC_MODE_MASK
 548                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 549                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 550                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 551                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 552                 | UVD_CGC_CTRL__MPC_MODE_MASK
 553                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 554                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 555                 | UVD_CGC_CTRL__WCB_MODE_MASK
 556                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 557                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 558         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 559
 560         /* turn on */
 561         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 562         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 563                 | UVD_SUVD_CGC_GATE__SIT_MASK
 564                 | UVD_SUVD_CGC_GATE__SMP_MASK
 565                 | UVD_SUVD_CGC_GATE__SCM_MASK
 566                 | UVD_SUVD_CGC_GATE__SDB_MASK
 567                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 568                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 569                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 570                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 571                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 572                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 573                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 574                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 575                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 576                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 577                 | UVD_SUVD_CGC_GATE__ENT_MASK
 578                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 579                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 580                 | UVD_SUVD_CGC_GATE__SITE_MASK
 581                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 582                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 583                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 584                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 585                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 586         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 587
 588         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 589         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 590                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 591                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 592                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 593                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 594                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 595                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 596                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 597                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 598                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 599         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 600 }
 601
 602 static void vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
 603                 uint8_t sram_sel, uint8_t indirect)
 604 {
 605         uint32_t reg_data = 0;
 606
 607         /* enable sw clock gating control */
 608         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 609                 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 610         else
 611                 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 612         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 613         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 614         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 615                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 616                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 617                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 618                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 619                  UVD_CGC_CTRL__SYS_MODE_MASK |
 620                  UVD_CGC_CTRL__UDEC_MODE_MASK |
 621                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
 622                  UVD_CGC_CTRL__REGS_MODE_MASK |
 623                  UVD_CGC_CTRL__RBC_MODE_MASK |
 624                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 625                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 626                  UVD_CGC_CTRL__IDCT_MODE_MASK |
 627                  UVD_CGC_CTRL__MPRD_MODE_MASK |
 628                  UVD_CGC_CTRL__MPC_MODE_MASK |
 629                  UVD_CGC_CTRL__LBSI_MODE_MASK |
 630                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
 631                  UVD_CGC_CTRL__WCB_MODE_MASK |
 632                  UVD_CGC_CTRL__VCPU_MODE_MASK |
 633                  UVD_CGC_CTRL__SCPU_MODE_MASK);
 634         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 635                 UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
 636
 637         /* turn off clock gating */
 638         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 639                 UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
 640
 641         /* turn on SUVD clock gating */
 642         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 643                 UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
 644
 645         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
 646         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 647                 UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
 648 }
 649
 650 /**
 651  * jpeg_v2_0_start - start JPEG block
 652  *
 653  * @adev: amdgpu_device pointer
 654  *
 655  * Setup and start the JPEG block
 656  */
 657 static int jpeg_v2_0_start(struct amdgpu_device *adev)
 658 {
 659         struct amdgpu_ring *ring = &adev->vcn.inst->ring_jpeg;
 660         uint32_t tmp;
 661         int r = 0;
 662
 663         /* disable power gating */
 664         tmp = 1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
 665         WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_PGFSM_CONFIG), tmp);
 666
 667         SOC15_WAIT_ON_RREG(VCN, 0,
 668                 mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS_UVDJ_PWR_ON,
 669                 UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK, r);
 670
 671         if (r) {
 672                 DRM_ERROR("amdgpu: JPEG disable power gating failed\n");
 673                 return r;
 674         }
 675
 676         /* Removing the anti hang mechanism to indicate the UVDJ tile is ON */
 677         tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS)) & ~0x1;
 678         WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS), tmp);
 679
 680         /* JPEG disable CGC */
 681         tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 682         tmp |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 683         tmp |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 684         tmp |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 685         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, tmp);
 686
 687         tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 688         tmp &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
 689                 | JPEG_CGC_GATE__JPEG2_DEC_MASK
 690                 | JPEG_CGC_GATE__JPEG_ENC_MASK
 691                 | JPEG_CGC_GATE__JMCIF_MASK
 692                 | JPEG_CGC_GATE__JRBBM_MASK);
 693         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, tmp);
 694
 695         /* enable JMI channel */
 696         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_JMI_CNTL), 0,
 697                 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 698
 699         /* enable System Interrupt for JRBC */
 700         WREG32_P(SOC15_REG_OFFSET(VCN, 0, mmJPEG_SYS_INT_EN),
 701                 JPEG_SYS_INT_EN__DJRBC_MASK,
 702                 ~JPEG_SYS_INT_EN__DJRBC_MASK);
 703
 704         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
 705         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
 706         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 707                 lower_32_bits(ring->gpu_addr));
 708         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 709                 upper_32_bits(ring->gpu_addr));
 710         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, 0);
 711         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, 0);
 712         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, 0x00000002L);
 713         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_SIZE, ring->ring_size / 4);
 714         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
 715
 716         return 0;
 717 }
 718
 719 /**
 720  * jpeg_v2_0_stop - stop JPEG block
 721  *
 722  * @adev: amdgpu_device pointer
 723  *
 724  * stop the JPEG block
 725  */
 726 static int jpeg_v2_0_stop(struct amdgpu_device *adev)
 727 {
 728         uint32_t tmp;
 729         int r = 0;
 730
 731         /* reset JMI */
 732         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_JMI_CNTL),
 733                 UVD_JMI_CNTL__SOFT_RESET_MASK,
 734                 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 735
 736         /* enable JPEG CGC */
 737         tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 738         tmp |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 739         tmp |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 740         tmp |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 741         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, tmp);
 742
 743
 744         tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 745         tmp |= (JPEG_CGC_GATE__JPEG_DEC_MASK
 746                 |JPEG_CGC_GATE__JPEG2_DEC_MASK
 747                 |JPEG_CGC_GATE__JPEG_ENC_MASK
 748                 |JPEG_CGC_GATE__JMCIF_MASK
 749                 |JPEG_CGC_GATE__JRBBM_MASK);
 750         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, tmp);
 751
 752         /* enable power gating */
 753         tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS));
 754         tmp &= ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK;
 755         tmp |=  0x1; //UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_TILES_OFF;
 756         WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS), tmp);
 757
 758         tmp = 2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
 759         WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_PGFSM_CONFIG), tmp);
 760
 761         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
 762                 (2 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT),
 763                 UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK, r);
 764
 765         if (r) {
 766                 DRM_ERROR("amdgpu: JPEG enable power gating failed\n");
 767                 return r;
 768         }
 769
 770         return r;
 771 }
 772
 773 /**
 774  * vcn_v2_0_enable_clock_gating - enable VCN clock gating
 775  *
 776  * @adev: amdgpu_device pointer
 777  * @sw: enable SW clock gating
 778  *
 779  * Enable clock gating for VCN block
 780  */
 781 static void vcn_v2_0_enable_clock_gating(struct amdgpu_device *adev)
 782 {
 783         uint32_t data = 0;
 784
 785         /* enable UVD CGC */
 786         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 787         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 788                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 789         else
 790                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 791         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 792         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 793         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 794
 795         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 796         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 797                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 798                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 799                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 800                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 801                 | UVD_CGC_CTRL__SYS_MODE_MASK
 802                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 803                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 804                 | UVD_CGC_CTRL__REGS_MODE_MASK
 805                 | UVD_CGC_CTRL__RBC_MODE_MASK
 806                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 807                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 808                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 809                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 810                 | UVD_CGC_CTRL__MPC_MODE_MASK
 811                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 812                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 813                 | UVD_CGC_CTRL__WCB_MODE_MASK
 814                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 815                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 816         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 817
 818         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 819         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 820                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 821                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 822                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 823                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 824                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 825                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 826                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 827                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 828                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 829         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 830 }
 831
 832 static void vcn_v2_0_disable_static_power_gating(struct amdgpu_device *adev)
 833 {
 834         uint32_t data = 0;
 835         int ret;
 836
 837         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 838                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 839                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 840                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 841                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 842                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 843                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 844                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 845                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 846                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 847                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 848
 849                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 850                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
 851                         UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0, 0xFFFFF, ret);
 852         } else {
 853                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 854                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 855                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 856                         | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 857                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 858                         | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 859                         | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 860                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 861                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 862                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 863                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 864                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFF, ret);
 865         }
 866
 867         /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS,
 868          * UVDU_PWR_STATUS are 0 (power on) */
 869
 870         data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 871         data &= ~0x103;
 872         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 873                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 874                         UVD_POWER_STATUS__UVD_PG_EN_MASK;
 875
 876         WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 877 }
 878
 879 static void vcn_v2_0_enable_static_power_gating(struct amdgpu_device *adev)
 880 {
 881         uint32_t data = 0;
 882         int ret;
 883
 884         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 885                 /* Before power off, this indicator has to be turned on */
 886                 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 887                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 888                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 889                 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 890
 891
 892                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 893                         | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 894                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 895                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 896                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 897                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 898                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 899                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 900                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 901                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
 902
 903                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 904
 905                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 906                         | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
 907                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 908                         | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
 909                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 910                         | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
 911                         | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
 912                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 913                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 914                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT);
 915                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFF, ret);
 916         }
 917 }
 918
 919 static int vcn_v2_0_start_dpg_mode(struct amdgpu_device *adev, bool indirect)
 920 {
 921         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 922         uint32_t rb_bufsz, tmp;
 923
 924         vcn_v2_0_enable_static_power_gating(adev);
 925
 926         /* enable dynamic power gating mode */
 927         tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
 928         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 929         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 930         WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
 931
 932         if (indirect)
 933                 adev->vcn.dpg_sram_curr_addr = (uint32_t*)adev->vcn.dpg_sram_cpu_addr;
 934
 935         /* enable clock gating */
 936         vcn_v2_0_clock_gating_dpg_mode(adev, 0, indirect);
 937
 938         /* enable VCPU clock */
 939         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 940         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 941         tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
 942         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 943                 UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
 944
 945         /* disable master interupt */
 946         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 947                 UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
 948
 949         /* setup mmUVD_LMI_CTRL */
 950         tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 951                 UVD_LMI_CTRL__REQ_MODE_MASK |
 952                 UVD_LMI_CTRL__CRC_RESET_MASK |
 953                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 954                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 955                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 956                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 957                 0x00100000L);
 958         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 959                 UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
 960
 961         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 962                 UVD, 0, mmUVD_MPC_CNTL),
 963                 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
 964
 965         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 966                 UVD, 0, mmUVD_MPC_SET_MUXA0),
 967                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 968                  (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 969                  (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 970                  (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
 971
 972         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 973                 UVD, 0, mmUVD_MPC_SET_MUXB0),
 974                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 975                  (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 976                  (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 977                  (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
 978
 979         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 980                 UVD, 0, mmUVD_MPC_SET_MUX),
 981                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 982                  (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 983                  (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
 984
 985         vcn_v2_0_mc_resume_dpg_mode(adev, indirect);
 986
 987         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 988                 UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
 989         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 990                 UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
 991
 992         /* release VCPU reset to boot */
 993         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 994                 UVD, 0, mmUVD_SOFT_RESET), 0, 0, indirect);
 995
 996         /* enable LMI MC and UMC channels */
 997         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
 998                 UVD, 0, mmUVD_LMI_CTRL2),
 999                 0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT, 0, indirect);
1000
1001         /* enable master interrupt */
1002         WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
1003                 UVD, 0, mmUVD_MASTINT_EN),
1004                 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1005
1006         if (indirect)
1007                 psp_update_vcn_sram(adev, 0, adev->vcn.dpg_sram_gpu_addr,
1008                                     (uint32_t)((uintptr_t)adev->vcn.dpg_sram_curr_addr -
1009                                                (uintptr_t)adev->vcn.dpg_sram_cpu_addr));
1010
1011         /* force RBC into idle state */
1012         rb_bufsz = order_base_2(ring->ring_size);
1013         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1014         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1015         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1016         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1017         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1018         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1019
1020         /* set the write pointer delay */
1021         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
1022
1023         /* set the wb address */
1024         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
1025                 (upper_32_bits(ring->gpu_addr) >> 2));
1026
1027         /* programm the RB_BASE for ring buffer */
1028         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1029                 lower_32_bits(ring->gpu_addr));
1030         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1031                 upper_32_bits(ring->gpu_addr));
1032
1033         /* Initialize the ring buffer's read and write pointers */
1034         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1035
1036         WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
1037
1038         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1039         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1040                 lower_32_bits(ring->wptr));
1041
1042         return 0;
1043 }
1044
1045 static int vcn_v2_0_start(struct amdgpu_device *adev)
1046 {
1047         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
1048         uint32_t rb_bufsz, tmp;
1049         uint32_t lmi_swap_cntl;
1050         int i, j, r;
1051
1052         if (adev->pm.dpm_enabled)
1053                 amdgpu_dpm_enable_uvd(adev, true);
1054
1055         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1056                 r = vcn_v2_0_start_dpg_mode(adev, adev->vcn.indirect_sram);
1057                 if (r)
1058                         return r;
1059                 goto jpeg;
1060         }
1061
1062         vcn_v2_0_disable_static_power_gating(adev);
1063
1064         /* set uvd status busy */
1065         tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1066         WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
1067
1068         /*SW clock gating */
1069         vcn_v2_0_disable_clock_gating(adev);
1070
1071         /* enable VCPU clock */
1072         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
1073                 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1074
1075         /* disable master interrupt */
1076         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
1077                 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1078
1079         /* setup mmUVD_LMI_CTRL */
1080         tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
1081         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp |
1082                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1083                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1084                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1085                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1086
1087         /* setup mmUVD_MPC_CNTL */
1088         tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
1089         tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1090         tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1091         WREG32_SOC15(VCN, 0, mmUVD_MPC_CNTL, tmp);
1092
1093         /* setup UVD_MPC_SET_MUXA0 */
1094         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
1095                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1096                 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1097                 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1098                 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1099
1100         /* setup UVD_MPC_SET_MUXB0 */
1101         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
1102                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1103                 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1104                 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1105                 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1106
1107         /* setup mmUVD_MPC_SET_MUX */
1108         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
1109                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1110                 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1111                 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1112
1113         vcn_v2_0_mc_resume(adev);
1114
1115         /* release VCPU reset to boot */
1116         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
1117                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1118
1119         /* enable LMI MC and UMC channels */
1120         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
1121                 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1122
1123         tmp = RREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET);
1124         tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1125         tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1126         WREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET, tmp);
1127
1128         /* disable byte swapping */
1129         lmi_swap_cntl = 0;
1130 #ifdef __BIG_ENDIAN
1131         /* swap (8 in 32) RB and IB */
1132         lmi_swap_cntl = 0xa;
1133 #endif
1134         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
1135
1136         for (i = 0; i < 10; ++i) {
1137                 uint32_t status;
1138
1139                 for (j = 0; j < 100; ++j) {
1140                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
1141                         if (status & 2)
1142                                 break;
1143                         mdelay(10);
1144                 }
1145                 r = 0;
1146                 if (status & 2)
1147                         break;
1148
1149                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1150                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1151                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1152                         ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1153                 mdelay(10);
1154                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
1155                         ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1156                 mdelay(10);
1157                 r = -1;
1158         }
1159
1160         if (r) {
1161                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
1162                 return r;
1163         }
1164
1165         /* enable master interrupt */
1166         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
1167                 UVD_MASTINT_EN__VCPU_EN_MASK,
1168                 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1169
1170         /* clear the busy bit of VCN_STATUS */
1171         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
1172                 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1173
1174         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_VMID, 0);
1175
1176         /* force RBC into idle state */
1177         rb_bufsz = order_base_2(ring->ring_size);
1178         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1179         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1180         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1181         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1182         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1183         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1184
1185         /* programm the RB_BASE for ring buffer */
1186         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1187                 lower_32_bits(ring->gpu_addr));
1188         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1189                 upper_32_bits(ring->gpu_addr));
1190
1191         /* Initialize the ring buffer's read and write pointers */
1192         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1193
1194         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1195         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1196                         lower_32_bits(ring->wptr));
1197
1198         ring = &adev->vcn.inst->ring_enc[0];
1199         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1200         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1201         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1202         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1203         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1204
1205         ring = &adev->vcn.inst->ring_enc[1];
1206         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1207         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1208         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1209         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1210         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1211
1212 jpeg:
1213         r = jpeg_v2_0_start(adev);
1214
1215         return r;
1216 }
1217
1218 static int vcn_v2_0_stop_dpg_mode(struct amdgpu_device *adev)
1219 {
1220         int ret_code = 0;
1221         uint32_t tmp;
1222
1223         /* Wait for power status to be 1 */
1224         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1225                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1226
1227         /* wait for read ptr to be equal to write ptr */
1228         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1229         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1230
1231         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1232         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF, ret_code);
1233
1234         tmp = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1235         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_JRBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1236
1237         tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1238         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1239
1240         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1241                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1242
1243         /* disable dynamic power gating mode */
1244         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1245                         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1246
1247         return 0;
1248 }
1249
1250 static int vcn_v2_0_stop(struct amdgpu_device *adev)
1251 {
1252         uint32_t tmp;
1253         int r;
1254
1255         r = jpeg_v2_0_stop(adev);
1256         if (r)
1257                 return r;
1258
1259         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1260                 r = vcn_v2_0_stop_dpg_mode(adev);
1261                 if (r)
1262                         return r;
1263                 goto power_off;
1264         }
1265
1266         /* wait for uvd idle */
1267         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7, r);
1268         if (r)
1269                 return r;
1270
1271         tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1272                 UVD_LMI_STATUS__READ_CLEAN_MASK |
1273                 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1274                 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1275         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp, r);
1276         if (r)
1277                 return r;
1278
1279         /* stall UMC channel */
1280         tmp = RREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2);
1281         tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1282         WREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2, tmp);
1283
1284         tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1285                 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1286         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp, r);
1287         if (r)
1288                 return r;
1289
1290         /* disable VCPU clock */
1291         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
1292                 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1293
1294         /* reset LMI UMC */
1295         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1296                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
1297                 ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
1298
1299         /* reset LMI */
1300         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1301                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
1302                 ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
1303
1304         /* reset VCPU */
1305         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1306                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1307                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1308
1309         /* clear status */
1310         WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
1311
1312         vcn_v2_0_enable_clock_gating(adev);
1313         vcn_v2_0_enable_static_power_gating(adev);
1314
1315 power_off:
1316         if (adev->pm.dpm_enabled)
1317                 amdgpu_dpm_enable_uvd(adev, false);
1318
1319         return 0;
1320 }
1321
1322 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
1323                                 struct dpg_pause_state *new_state)
1324 {
1325         struct amdgpu_ring *ring;
1326         uint32_t reg_data = 0;
1327         int ret_code;
1328
1329         /* pause/unpause if state is changed */
1330         if (adev->vcn.pause_state.fw_based != new_state->fw_based) {
1331                 DRM_DEBUG("dpg pause state changed %d -> %d",
1332                         adev->vcn.pause_state.fw_based, new_state->fw_based);
1333                 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1334                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1335
1336                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1337                         ret_code = 0;
1338                         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1,
1339                                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1340
1341                         if (!ret_code) {
1342                                 /* pause DPG */
1343                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1344                                 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1345
1346                                 /* wait for ACK */
1347                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1348                                            UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1349                                            UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
1350
1351                                 /* Restore */
1352                                 ring = &adev->vcn.inst->ring_enc[0];
1353                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1354                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1355                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1356                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1357                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1358
1359                                 ring = &adev->vcn.inst->ring_enc[1];
1360                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1361                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1362                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1363                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1364                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1365
1366                                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1367                                            RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1368
1369                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1370                                            UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1371                                            UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1372                         }
1373                 } else {
1374                         /* unpause dpg, no need to wait */
1375                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1376                         WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1377                 }
1378                 adev->vcn.pause_state.fw_based = new_state->fw_based;
1379         }
1380
1381         return 0;
1382 }
1383
1384 static bool vcn_v2_0_is_idle(void *handle)
1385 {
1386         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1387
1388         return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
1389 }
1390
1391 static int vcn_v2_0_wait_for_idle(void *handle)
1392 {
1393         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1394         int ret = 0;
1395
1396         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
1397                 UVD_STATUS__IDLE, ret);
1398
1399         return ret;
1400 }
1401
1402 static int vcn_v2_0_set_clockgating_state(void *handle,
1403                                           enum amd_clockgating_state state)
1404 {
1405         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1406         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1407
1408         if (enable) {
1409                 /* wait for STATUS to clear */
1410                 if (vcn_v2_0_is_idle(handle))
1411                         return -EBUSY;
1412                 vcn_v2_0_enable_clock_gating(adev);
1413         } else {
1414                 /* disable HW gating and enable Sw gating */
1415                 vcn_v2_0_disable_clock_gating(adev);
1416         }
1417         return 0;
1418 }
1419
1420 /**
1421  * vcn_v2_0_dec_ring_get_rptr - get read pointer
1422  *
1423  * @ring: amdgpu_ring pointer
1424  *
1425  * Returns the current hardware read pointer
1426  */
1427 static uint64_t vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1428 {
1429         struct amdgpu_device *adev = ring->adev;
1430
1431         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1432 }
1433
1434 /**
1435  * vcn_v2_0_dec_ring_get_wptr - get write pointer
1436  *
1437  * @ring: amdgpu_ring pointer
1438  *
1439  * Returns the current hardware write pointer
1440  */
1441 static uint64_t vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1442 {
1443         struct amdgpu_device *adev = ring->adev;
1444
1445         if (ring->use_doorbell)
1446                 return adev->wb.wb[ring->wptr_offs];
1447         else
1448                 return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1449 }
1450
1451 /**
1452  * vcn_v2_0_dec_ring_set_wptr - set write pointer
1453  *
1454  * @ring: amdgpu_ring pointer
1455  *
1456  * Commits the write pointer to the hardware
1457  */
1458 static void vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1459 {
1460         struct amdgpu_device *adev = ring->adev;
1461
1462         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1463                 WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
1464                         lower_32_bits(ring->wptr) | 0x80000000);
1465
1466         if (ring->use_doorbell) {
1467                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1468                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1469         } else {
1470                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1471         }
1472 }
1473
1474 /**
1475  * vcn_v2_0_dec_ring_insert_start - insert a start command
1476  *
1477  * @ring: amdgpu_ring pointer
1478  *
1479  * Write a start command to the ring.
1480  */
1481 void vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1482 {
1483         struct amdgpu_device *adev = ring->adev;
1484
1485         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1486         amdgpu_ring_write(ring, 0);
1487         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1488         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
1489 }
1490
1491 /**
1492  * vcn_v2_0_dec_ring_insert_end - insert a end command
1493  *
1494  * @ring: amdgpu_ring pointer
1495  *
1496  * Write a end command to the ring.
1497  */
1498 void vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1499 {
1500         struct amdgpu_device *adev = ring->adev;
1501
1502         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1503         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_END << 1));
1504 }
1505
1506 /**
1507  * vcn_v2_0_dec_ring_insert_nop - insert a nop command
1508  *
1509  * @ring: amdgpu_ring pointer
1510  *
1511  * Write a nop command to the ring.
1512  */
1513 void vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1514 {
1515         struct amdgpu_device *adev = ring->adev;
1516         int i;
1517
1518         WARN_ON(ring->wptr % 2 || count % 2);
1519
1520         for (i = 0; i < count / 2; i++) {
1521                 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.nop, 0));
1522                 amdgpu_ring_write(ring, 0);
1523         }
1524 }
1525
1526 /**
1527  * vcn_v2_0_dec_ring_emit_fence - emit an fence & trap command
1528  *
1529  * @ring: amdgpu_ring pointer
1530  * @fence: fence to emit
1531  *
1532  * Write a fence and a trap command to the ring.
1533  */
1534 void vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1535                                 unsigned flags)
1536 {
1537         struct amdgpu_device *adev = ring->adev;
1538
1539         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1540         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.context_id, 0));
1541         amdgpu_ring_write(ring, seq);
1542
1543         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1544         amdgpu_ring_write(ring, addr & 0xffffffff);
1545
1546         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1547         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1548
1549         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1550         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_FENCE << 1));
1551
1552         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1553         amdgpu_ring_write(ring, 0);
1554
1555         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1556         amdgpu_ring_write(ring, 0);
1557
1558         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1559
1560         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_TRAP << 1));
1561 }
1562
1563 /**
1564  * vcn_v2_0_dec_ring_emit_ib - execute indirect buffer
1565  *
1566  * @ring: amdgpu_ring pointer
1567  * @ib: indirect buffer to execute
1568  *
1569  * Write ring commands to execute the indirect buffer
1570  */
1571 void vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1572                                struct amdgpu_job *job,
1573                                struct amdgpu_ib *ib,
1574                                uint32_t flags)
1575 {
1576         struct amdgpu_device *adev = ring->adev;
1577         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1578
1579         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_vmid, 0));
1580         amdgpu_ring_write(ring, vmid);
1581
1582         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_bar_low, 0));
1583         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1584         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_bar_high, 0));
1585         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1586         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_size, 0));
1587         amdgpu_ring_write(ring, ib->length_dw);
1588 }
1589
1590 void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1591                                 uint32_t val, uint32_t mask)
1592 {
1593         struct amdgpu_device *adev = ring->adev;
1594
1595         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1596         amdgpu_ring_write(ring, reg << 2);
1597
1598         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1599         amdgpu_ring_write(ring, val);
1600
1601         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.gp_scratch8, 0));
1602         amdgpu_ring_write(ring, mask);
1603
1604         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1605
1606         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_REG_READ_COND_WAIT << 1));
1607 }
1608
1609 void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1610                                 unsigned vmid, uint64_t pd_addr)
1611 {
1612         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1613         uint32_t data0, data1, mask;
1614
1615         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1616
1617         /* wait for register write */
1618         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
1619         data1 = lower_32_bits(pd_addr);
1620         mask = 0xffffffff;
1621         vcn_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1622 }
1623
1624 void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1625                                 uint32_t reg, uint32_t val)
1626 {
1627         struct amdgpu_device *adev = ring->adev;
1628
1629         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1630         amdgpu_ring_write(ring, reg << 2);
1631
1632         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1633         amdgpu_ring_write(ring, val);
1634
1635         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1636
1637         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_WRITE_REG << 1));
1638 }
1639
1640 /**
1641  * vcn_v2_0_enc_ring_get_rptr - get enc read pointer
1642  *
1643  * @ring: amdgpu_ring pointer
1644  *
1645  * Returns the current hardware enc read pointer
1646  */
1647 static uint64_t vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1648 {
1649         struct amdgpu_device *adev = ring->adev;
1650
1651         if (ring == &adev->vcn.inst->ring_enc[0])
1652                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1653         else
1654                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1655 }
1656
1657  /**
1658  * vcn_v2_0_enc_ring_get_wptr - get enc write pointer
1659  *
1660  * @ring: amdgpu_ring pointer
1661  *
1662  * Returns the current hardware enc write pointer
1663  */
1664 static uint64_t vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1665 {
1666         struct amdgpu_device *adev = ring->adev;
1667
1668         if (ring == &adev->vcn.inst->ring_enc[0]) {
1669                 if (ring->use_doorbell)
1670                         return adev->wb.wb[ring->wptr_offs];
1671                 else
1672                         return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1673         } else {
1674                 if (ring->use_doorbell)
1675                         return adev->wb.wb[ring->wptr_offs];
1676                 else
1677                         return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1678         }
1679 }
1680
1681  /**
1682  * vcn_v2_0_enc_ring_set_wptr - set enc write pointer
1683  *
1684  * @ring: amdgpu_ring pointer
1685  *
1686  * Commits the enc write pointer to the hardware
1687  */
1688 static void vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1689 {
1690         struct amdgpu_device *adev = ring->adev;
1691
1692         if (ring == &adev->vcn.inst->ring_enc[0]) {
1693                 if (ring->use_doorbell) {
1694                         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1695                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1696                 } else {
1697                         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1698                 }
1699         } else {
1700                 if (ring->use_doorbell) {
1701                         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1702                         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1703                 } else {
1704                         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1705                 }
1706         }
1707 }
1708
1709 /**
1710  * vcn_v2_0_enc_ring_emit_fence - emit an enc fence & trap command
1711  *
1712  * @ring: amdgpu_ring pointer
1713  * @fence: fence to emit
1714  *
1715  * Write enc a fence and a trap command to the ring.
1716  */
1717 void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1718                                 u64 seq, unsigned flags)
1719 {
1720         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1721
1722         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1723         amdgpu_ring_write(ring, addr);
1724         amdgpu_ring_write(ring, upper_32_bits(addr));
1725         amdgpu_ring_write(ring, seq);
1726         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1727 }
1728
1729 void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1730 {
1731         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1732 }
1733
1734 /**
1735  * vcn_v2_0_enc_ring_emit_ib - enc execute indirect buffer
1736  *
1737  * @ring: amdgpu_ring pointer
1738  * @ib: indirect buffer to execute
1739  *
1740  * Write enc ring commands to execute the indirect buffer
1741  */
1742 void vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1743                                struct amdgpu_job *job,
1744                                struct amdgpu_ib *ib,
1745                                uint32_t flags)
1746 {
1747         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1748
1749         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1750         amdgpu_ring_write(ring, vmid);
1751         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1752         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1753         amdgpu_ring_write(ring, ib->length_dw);
1754 }
1755
1756 void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1757                                 uint32_t val, uint32_t mask)
1758 {
1759         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1760         amdgpu_ring_write(ring, reg << 2);
1761         amdgpu_ring_write(ring, mask);
1762         amdgpu_ring_write(ring, val);
1763 }
1764
1765 void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1766                                 unsigned int vmid, uint64_t pd_addr)
1767 {
1768         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1769
1770         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1771
1772         /* wait for reg writes */
1773         vcn_v2_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
1774                                         lower_32_bits(pd_addr), 0xffffffff);
1775 }
1776
1777 void vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1778 {
1779         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1780         amdgpu_ring_write(ring, reg << 2);
1781         amdgpu_ring_write(ring, val);
1782 }
1783
1784 /**
1785  * vcn_v2_0_jpeg_ring_get_rptr - get read pointer
1786  *
1787  * @ring: amdgpu_ring pointer
1788  *
1789  * Returns the current hardware read pointer
1790  */
1791 static uint64_t vcn_v2_0_jpeg_ring_get_rptr(struct amdgpu_ring *ring)
1792 {
1793         struct amdgpu_device *adev = ring->adev;
1794
1795         return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR);
1796 }
1797
1798 /**
1799  * vcn_v2_0_jpeg_ring_get_wptr - get write pointer
1800  *
1801  * @ring: amdgpu_ring pointer
1802  *
1803  * Returns the current hardware write pointer
1804  */
1805 static uint64_t vcn_v2_0_jpeg_ring_get_wptr(struct amdgpu_ring *ring)
1806 {
1807         struct amdgpu_device *adev = ring->adev;
1808
1809         if (ring->use_doorbell)
1810                 return adev->wb.wb[ring->wptr_offs];
1811         else
1812                 return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1813 }
1814
1815 /**
1816  * vcn_v2_0_jpeg_ring_set_wptr - set write pointer
1817  *
1818  * @ring: amdgpu_ring pointer
1819  *
1820  * Commits the write pointer to the hardware
1821  */
1822 static void vcn_v2_0_jpeg_ring_set_wptr(struct amdgpu_ring *ring)
1823 {
1824         struct amdgpu_device *adev = ring->adev;
1825
1826         if (ring->use_doorbell) {
1827                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1828                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1829         } else {
1830                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
1831         }
1832 }
1833
1834 /**
1835  * vcn_v2_0_jpeg_ring_insert_start - insert a start command
1836  *
1837  * @ring: amdgpu_ring pointer
1838  *
1839  * Write a start command to the ring.
1840  */
1841 void vcn_v2_0_jpeg_ring_insert_start(struct amdgpu_ring *ring)
1842 {
1843         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
1844                 0, 0, PACKETJ_TYPE0));
1845         amdgpu_ring_write(ring, 0x68e04);
1846
1847         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
1848                 0, 0, PACKETJ_TYPE0));
1849         amdgpu_ring_write(ring, 0x80010000);
1850 }
1851
1852 /**
1853  * vcn_v2_0_jpeg_ring_insert_end - insert a end command
1854  *
1855  * @ring: amdgpu_ring pointer
1856  *
1857  * Write a end command to the ring.
1858  */
1859 void vcn_v2_0_jpeg_ring_insert_end(struct amdgpu_ring *ring)
1860 {
1861         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
1862                 0, 0, PACKETJ_TYPE0));
1863         amdgpu_ring_write(ring, 0x68e04);
1864
1865         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
1866                 0, 0, PACKETJ_TYPE0));
1867         amdgpu_ring_write(ring, 0x00010000);
1868 }
1869
1870 /**
1871  * vcn_v2_0_jpeg_ring_emit_fence - emit an fence & trap command
1872  *
1873  * @ring: amdgpu_ring pointer
1874  * @fence: fence to emit
1875  *
1876  * Write a fence and a trap command to the ring.
1877  */
1878 void vcn_v2_0_jpeg_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1879                                 unsigned flags)
1880 {
1881         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1882
1883         amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
1884                 0, 0, PACKETJ_TYPE0));
1885         amdgpu_ring_write(ring, seq);
1886
1887         amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
1888                 0, 0, PACKETJ_TYPE0));
1889         amdgpu_ring_write(ring, seq);
1890
1891         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
1892                 0, 0, PACKETJ_TYPE0));
1893         amdgpu_ring_write(ring, lower_32_bits(addr));
1894
1895         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
1896                 0, 0, PACKETJ_TYPE0));
1897         amdgpu_ring_write(ring, upper_32_bits(addr));
1898
1899         amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
1900                 0, 0, PACKETJ_TYPE0));
1901         amdgpu_ring_write(ring, 0x8);
1902
1903         amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
1904                 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
1905         amdgpu_ring_write(ring, 0);
1906
1907         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
1908                 0, 0, PACKETJ_TYPE0));
1909         amdgpu_ring_write(ring, 0x3fbc);
1910
1911         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
1912                 0, 0, PACKETJ_TYPE0));
1913         amdgpu_ring_write(ring, 0x1);
1914
1915         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
1916         amdgpu_ring_write(ring, 0);
1917 }
1918
1919 /**
1920  * vcn_v2_0_jpeg_ring_emit_ib - execute indirect buffer
1921  *
1922  * @ring: amdgpu_ring pointer
1923  * @ib: indirect buffer to execute
1924  *
1925  * Write ring commands to execute the indirect buffer.
1926  */
1927 void vcn_v2_0_jpeg_ring_emit_ib(struct amdgpu_ring *ring,
1928                                 struct amdgpu_job *job,
1929                                 struct amdgpu_ib *ib,
1930                                 uint32_t flags)
1931 {
1932         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1933
1934         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
1935                 0, 0, PACKETJ_TYPE0));
1936         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1937
1938         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
1939                 0, 0, PACKETJ_TYPE0));
1940         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1941
1942         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
1943                 0, 0, PACKETJ_TYPE0));
1944         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1945
1946         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
1947                 0, 0, PACKETJ_TYPE0));
1948         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1949
1950         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
1951                 0, 0, PACKETJ_TYPE0));
1952         amdgpu_ring_write(ring, ib->length_dw);
1953
1954         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
1955                 0, 0, PACKETJ_TYPE0));
1956         amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
1957
1958         amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
1959                 0, 0, PACKETJ_TYPE0));
1960         amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
1961
1962         amdgpu_ring_write(ring, PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
1963         amdgpu_ring_write(ring, 0);
1964
1965         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
1966                 0, 0, PACKETJ_TYPE0));
1967         amdgpu_ring_write(ring, 0x01400200);
1968
1969         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
1970                 0, 0, PACKETJ_TYPE0));
1971         amdgpu_ring_write(ring, 0x2);
1972
1973         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_STATUS_INTERNAL_OFFSET,
1974                 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
1975         amdgpu_ring_write(ring, 0x2);
1976 }
1977
1978 void vcn_v2_0_jpeg_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1979                                 uint32_t val, uint32_t mask)
1980 {
1981         uint32_t reg_offset = (reg << 2);
1982
1983         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
1984                 0, 0, PACKETJ_TYPE0));
1985         amdgpu_ring_write(ring, 0x01400200);
1986
1987         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
1988                 0, 0, PACKETJ_TYPE0));
1989         amdgpu_ring_write(ring, val);
1990
1991         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
1992                 0, 0, PACKETJ_TYPE0));
1993         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
1994                 amdgpu_ring_write(ring, 0);
1995                 amdgpu_ring_write(ring,
1996                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
1997         } else {
1998                 amdgpu_ring_write(ring, reg_offset);
1999                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
2000                         0, 0, PACKETJ_TYPE3));
2001         }
2002         amdgpu_ring_write(ring, mask);
2003 }
2004
2005 void vcn_v2_0_jpeg_ring_emit_vm_flush(struct amdgpu_ring *ring,
2006                                 unsigned vmid, uint64_t pd_addr)
2007 {
2008         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
2009         uint32_t data0, data1, mask;
2010
2011         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
2012
2013         /* wait for register write */
2014         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
2015         data1 = lower_32_bits(pd_addr);
2016         mask = 0xffffffff;
2017         vcn_v2_0_jpeg_ring_emit_reg_wait(ring, data0, data1, mask);
2018 }
2019
2020 void vcn_v2_0_jpeg_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
2021 {
2022         uint32_t reg_offset = (reg << 2);
2023
2024         amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
2025                 0, 0, PACKETJ_TYPE0));
2026         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
2027                 amdgpu_ring_write(ring, 0);
2028                 amdgpu_ring_write(ring,
2029                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
2030         } else {
2031                 amdgpu_ring_write(ring, reg_offset);
2032                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
2033                         0, 0, PACKETJ_TYPE0));
2034         }
2035         amdgpu_ring_write(ring, val);
2036 }
2037
2038 void vcn_v2_0_jpeg_ring_nop(struct amdgpu_ring *ring, uint32_t count)
2039 {
2040         int i;
2041
2042         WARN_ON(ring->wptr % 2 || count % 2);
2043
2044         for (i = 0; i < count / 2; i++) {
2045                 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
2046                 amdgpu_ring_write(ring, 0);
2047         }
2048 }
2049
2050 static int vcn_v2_0_set_interrupt_state(struct amdgpu_device *adev,
2051                                         struct amdgpu_irq_src *source,
2052                                         unsigned type,
2053                                         enum amdgpu_interrupt_state state)
2054 {
2055         return 0;
2056 }
2057
2058 static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev,
2059                                       struct amdgpu_irq_src *source,
2060                                       struct amdgpu_iv_entry *entry)
2061 {
2062         DRM_DEBUG("IH: VCN TRAP\n");
2063
2064         switch (entry->src_id) {
2065         case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2066                 amdgpu_fence_process(&adev->vcn.inst->ring_dec);
2067                 break;
2068         case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2069                 amdgpu_fence_process(&adev->vcn.inst->ring_enc[0]);
2070                 break;
2071         case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2072                 amdgpu_fence_process(&adev->vcn.inst->ring_enc[1]);
2073                 break;
2074         case VCN_2_0__SRCID__JPEG_DECODE:
2075                 amdgpu_fence_process(&adev->vcn.inst->ring_jpeg);
2076                 break;
2077         default:
2078                 DRM_ERROR("Unhandled interrupt: %d %d\n",
2079                           entry->src_id, entry->src_data[0]);
2080                 break;
2081         }
2082
2083         return 0;
2084 }
2085
2086 static int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring)
2087 {
2088         struct amdgpu_device *adev = ring->adev;
2089         uint32_t tmp = 0;
2090         unsigned i;
2091         int r;
2092
2093         WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD);
2094         r = amdgpu_ring_alloc(ring, 4);
2095         if (r)
2096                 return r;
2097         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
2098         amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
2099         amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
2100         amdgpu_ring_write(ring, 0xDEADBEEF);
2101         amdgpu_ring_commit(ring);
2102         for (i = 0; i < adev->usec_timeout; i++) {
2103                 tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9);
2104                 if (tmp == 0xDEADBEEF)
2105                         break;
2106                 udelay(1);
2107         }
2108
2109         if (i >= adev->usec_timeout)
2110                 r = -ETIMEDOUT;
2111
2112         return r;
2113 }
2114
2115
2116 static int vcn_v2_0_set_powergating_state(void *handle,
2117                                           enum amd_powergating_state state)
2118 {
2119         /* This doesn't actually powergate the VCN block.
2120          * That's done in the dpm code via the SMC.  This
2121          * just re-inits the block as necessary.  The actual
2122          * gating still happens in the dpm code.  We should
2123          * revisit this when there is a cleaner line between
2124          * the smc and the hw blocks
2125          */
2126         int ret;
2127         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2128
2129         if (state == adev->vcn.cur_state)
2130                 return 0;
2131
2132         if (state == AMD_PG_STATE_GATE)
2133                 ret = vcn_v2_0_stop(adev);
2134         else
2135                 ret = vcn_v2_0_start(adev);
2136
2137         if (!ret)
2138                 adev->vcn.cur_state = state;
2139         return ret;
2140 }
2141
2142 static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
2143         .name = "vcn_v2_0",
2144         .early_init = vcn_v2_0_early_init,
2145         .late_init = NULL,
2146         .sw_init = vcn_v2_0_sw_init,
2147         .sw_fini = vcn_v2_0_sw_fini,
2148         .hw_init = vcn_v2_0_hw_init,
2149         .hw_fini = vcn_v2_0_hw_fini,
2150         .suspend = vcn_v2_0_suspend,
2151         .resume = vcn_v2_0_resume,
2152         .is_idle = vcn_v2_0_is_idle,
2153         .wait_for_idle = vcn_v2_0_wait_for_idle,
2154         .check_soft_reset = NULL,
2155         .pre_soft_reset = NULL,
2156         .soft_reset = NULL,
2157         .post_soft_reset = NULL,
2158         .set_clockgating_state = vcn_v2_0_set_clockgating_state,
2159         .set_powergating_state = vcn_v2_0_set_powergating_state,
2160 };
2161
2162 static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
2163         .type = AMDGPU_RING_TYPE_VCN_DEC,
2164         .align_mask = 0xf,
2165         .vmhub = AMDGPU_MMHUB_0,
2166         .get_rptr = vcn_v2_0_dec_ring_get_rptr,
2167         .get_wptr = vcn_v2_0_dec_ring_get_wptr,
2168         .set_wptr = vcn_v2_0_dec_ring_set_wptr,
2169         .emit_frame_size =
2170                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2171                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2172                 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
2173                 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
2174                 6,
2175         .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
2176         .emit_ib = vcn_v2_0_dec_ring_emit_ib,
2177         .emit_fence = vcn_v2_0_dec_ring_emit_fence,
2178         .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
2179         .test_ring = vcn_v2_0_dec_ring_test_ring,
2180         .test_ib = amdgpu_vcn_dec_ring_test_ib,
2181         .insert_nop = vcn_v2_0_dec_ring_insert_nop,
2182         .insert_start = vcn_v2_0_dec_ring_insert_start,
2183         .insert_end = vcn_v2_0_dec_ring_insert_end,
2184         .pad_ib = amdgpu_ring_generic_pad_ib,
2185         .begin_use = amdgpu_vcn_ring_begin_use,
2186         .end_use = amdgpu_vcn_ring_end_use,
2187         .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
2188         .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
2189         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2190 };
2191
2192 static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
2193         .type = AMDGPU_RING_TYPE_VCN_ENC,
2194         .align_mask = 0x3f,
2195         .nop = VCN_ENC_CMD_NO_OP,
2196         .vmhub = AMDGPU_MMHUB_0,
2197         .get_rptr = vcn_v2_0_enc_ring_get_rptr,
2198         .get_wptr = vcn_v2_0_enc_ring_get_wptr,
2199         .set_wptr = vcn_v2_0_enc_ring_set_wptr,
2200         .emit_frame_size =
2201                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2202                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2203                 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2204                 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2205                 1, /* vcn_v2_0_enc_ring_insert_end */
2206         .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2207         .emit_ib = vcn_v2_0_enc_ring_emit_ib,
2208         .emit_fence = vcn_v2_0_enc_ring_emit_fence,
2209         .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2210         .test_ring = amdgpu_vcn_enc_ring_test_ring,
2211         .test_ib = amdgpu_vcn_enc_ring_test_ib,
2212         .insert_nop = amdgpu_ring_insert_nop,
2213         .insert_end = vcn_v2_0_enc_ring_insert_end,
2214         .pad_ib = amdgpu_ring_generic_pad_ib,
2215         .begin_use = amdgpu_vcn_ring_begin_use,
2216         .end_use = amdgpu_vcn_ring_end_use,
2217         .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2218         .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2219         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2220 };
2221
2222 static const struct amdgpu_ring_funcs vcn_v2_0_jpeg_ring_vm_funcs = {
2223         .type = AMDGPU_RING_TYPE_VCN_JPEG,
2224         .align_mask = 0xf,
2225         .vmhub = AMDGPU_MMHUB_0,
2226         .get_rptr = vcn_v2_0_jpeg_ring_get_rptr,
2227         .get_wptr = vcn_v2_0_jpeg_ring_get_wptr,
2228         .set_wptr = vcn_v2_0_jpeg_ring_set_wptr,
2229         .emit_frame_size =
2230                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2231                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2232                 8 + /* vcn_v2_0_jpeg_ring_emit_vm_flush */
2233                 18 + 18 + /* vcn_v2_0_jpeg_ring_emit_fence x2 vm fence */
2234                 8 + 16,
2235         .emit_ib_size = 22, /* vcn_v2_0_jpeg_ring_emit_ib */
2236         .emit_ib = vcn_v2_0_jpeg_ring_emit_ib,
2237         .emit_fence = vcn_v2_0_jpeg_ring_emit_fence,
2238         .emit_vm_flush = vcn_v2_0_jpeg_ring_emit_vm_flush,
2239         .test_ring = amdgpu_vcn_jpeg_ring_test_ring,
2240         .test_ib = amdgpu_vcn_jpeg_ring_test_ib,
2241         .insert_nop = vcn_v2_0_jpeg_ring_nop,
2242         .insert_start = vcn_v2_0_jpeg_ring_insert_start,
2243         .insert_end = vcn_v2_0_jpeg_ring_insert_end,
2244         .pad_ib = amdgpu_ring_generic_pad_ib,
2245         .begin_use = amdgpu_vcn_ring_begin_use,
2246         .end_use = amdgpu_vcn_ring_end_use,
2247         .emit_wreg = vcn_v2_0_jpeg_ring_emit_wreg,
2248         .emit_reg_wait = vcn_v2_0_jpeg_ring_emit_reg_wait,
2249         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2250 };
2251
2252 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2253 {
2254         adev->vcn.inst->ring_dec.funcs = &vcn_v2_0_dec_ring_vm_funcs;
2255         DRM_INFO("VCN decode is enabled in VM mode\n");
2256 }
2257
2258 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2259 {
2260         int i;
2261
2262         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2263                 adev->vcn.inst->ring_enc[i].funcs = &vcn_v2_0_enc_ring_vm_funcs;
2264
2265         DRM_INFO("VCN encode is enabled in VM mode\n");
2266 }
2267
2268 static void vcn_v2_0_set_jpeg_ring_funcs(struct amdgpu_device *adev)
2269 {
2270         adev->vcn.inst->ring_jpeg.funcs = &vcn_v2_0_jpeg_ring_vm_funcs;
2271         DRM_INFO("VCN jpeg decode is enabled in VM mode\n");
2272 }
2273
2274 static const struct amdgpu_irq_src_funcs vcn_v2_0_irq_funcs = {
2275         .set = vcn_v2_0_set_interrupt_state,
2276         .process = vcn_v2_0_process_interrupt,
2277 };
2278
2279 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev)
2280 {
2281         adev->vcn.inst->irq.num_types = adev->vcn.num_enc_rings + 2;
2282         adev->vcn.inst->irq.funcs = &vcn_v2_0_irq_funcs;
2283 }
2284
2285 const struct amdgpu_ip_block_version vcn_v2_0_ip_block =
2286 {
2287                 .type = AMD_IP_BLOCK_TYPE_VCN,
2288                 .major = 2,
2289                 .minor = 0,
2290                 .rev = 0,
2291                 .funcs = &vcn_v2_0_ip_funcs,
2292 };