drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_cs.h"
  28 #include "amdgpu_vcn.h"
  29 #include "amdgpu_pm.h"
  30 #include "soc15.h"
  31 #include "soc15d.h"
  32 #include "soc15_common.h"
  33
  34 #include "vcn/vcn_1_0_offset.h"
  35 #include "vcn/vcn_1_0_sh_mask.h"
  36 #include "mmhub/mmhub_9_1_offset.h"
  37 #include "mmhub/mmhub_9_1_sh_mask.h"
  38
  39 #include "ivsrcid/vcn/irqsrcs_vcn_1_0.h"
  40 #include "jpeg_v1_0.h"
  41 #include "vcn_v1_0.h"
  42
  43 #define mmUVD_RBC_XX_IB_REG_CHECK_1_0           0x05ab
  44 #define mmUVD_RBC_XX_IB_REG_CHECK_1_0_BASE_IDX  1
  45 #define mmUVD_REG_XX_MASK_1_0                   0x05ac
  46 #define mmUVD_REG_XX_MASK_1_0_BASE_IDX          1
  47
  48 static int vcn_v1_0_stop(struct amdgpu_device *adev);
  49 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  50 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  51 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
  52 static int vcn_v1_0_set_powergating_state(void *handle, enum amd_powergating_state state);
  53 static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
  54                                 int inst_idx, struct dpg_pause_state *new_state);
  55
  56 static void vcn_v1_0_idle_work_handler(struct work_struct *work);
  57 static void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring);
  58
  59 /**
  60  * vcn_v1_0_early_init - set function pointers and load microcode
  61  *
  62  * @handle: amdgpu_device pointer
  63  *
  64  * Set ring and irq function pointers
  65  * Load microcode from filesystem
  66  */
  67 static int vcn_v1_0_early_init(void *handle)
  68 {
  69         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  70
  71         adev->vcn.num_enc_rings = 2;
  72
  73         vcn_v1_0_set_dec_ring_funcs(adev);
  74         vcn_v1_0_set_enc_ring_funcs(adev);
  75         vcn_v1_0_set_irq_funcs(adev);
  76
  77         jpeg_v1_0_early_init(handle);
  78
  79         return amdgpu_vcn_early_init(adev);
  80 }
  81
  82 /**
  83  * vcn_v1_0_sw_init - sw init for VCN block
  84  *
  85  * @handle: amdgpu_device pointer
  86  *
  87  * Load firmware and sw initialization
  88  */
  89 static int vcn_v1_0_sw_init(void *handle)
  90 {
  91         struct amdgpu_ring *ring;
  92         int i, r;
  93         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  94
  95         /* VCN DEC TRAP */
  96         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
  97                         VCN_1_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst->irq);
  98         if (r)
  99                 return r;
 100
 101         /* VCN ENC TRAP */
 102         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 103                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, i + VCN_1_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
 104                                         &adev->vcn.inst->irq);
 105                 if (r)
 106                         return r;
 107         }
 108
 109         r = amdgpu_vcn_sw_init(adev);
 110         if (r)
 111                 return r;
 112
 113         /* Override the work func */
 114         adev->vcn.idle_work.work.func = vcn_v1_0_idle_work_handler;
 115
 116         amdgpu_vcn_setup_ucode(adev);
 117
 118         r = amdgpu_vcn_resume(adev);
 119         if (r)
 120                 return r;
 121
 122         ring = &adev->vcn.inst->ring_dec;
 123         ring->vm_hub = AMDGPU_MMHUB0(0);
 124         sprintf(ring->name, "vcn_dec");
 125         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 126                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 127         if (r)
 128                 return r;
 129
 130         adev->vcn.internal.scratch9 = adev->vcn.inst->external.scratch9 =
 131                 SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
 132         adev->vcn.internal.data0 = adev->vcn.inst->external.data0 =
 133                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
 134         adev->vcn.internal.data1 = adev->vcn.inst->external.data1 =
 135                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
 136         adev->vcn.internal.cmd = adev->vcn.inst->external.cmd =
 137                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
 138         adev->vcn.internal.nop = adev->vcn.inst->external.nop =
 139                 SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
 140
 141         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 142                 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
 143
 144                 ring = &adev->vcn.inst->ring_enc[i];
 145                 ring->vm_hub = AMDGPU_MMHUB0(0);
 146                 sprintf(ring->name, "vcn_enc%d", i);
 147                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 148                                      hw_prio, NULL);
 149                 if (r)
 150                         return r;
 151         }
 152
 153         adev->vcn.pause_dpg_mode = vcn_v1_0_pause_dpg_mode;
 154
 155         if (amdgpu_vcnfw_log) {
 156                 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 157
 158                 fw_shared->present_flag_0 = 0;
 159                 amdgpu_vcn_fwlog_init(adev->vcn.inst);
 160         }
 161
 162         r = jpeg_v1_0_sw_init(handle);
 163
 164         return r;
 165 }
 166
 167 /**
 168  * vcn_v1_0_sw_fini - sw fini for VCN block
 169  *
 170  * @handle: amdgpu_device pointer
 171  *
 172  * VCN suspend and free up sw allocation
 173  */
 174 static int vcn_v1_0_sw_fini(void *handle)
 175 {
 176         int r;
 177         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 178
 179         r = amdgpu_vcn_suspend(adev);
 180         if (r)
 181                 return r;
 182
 183         jpeg_v1_0_sw_fini(handle);
 184
 185         r = amdgpu_vcn_sw_fini(adev);
 186
 187         return r;
 188 }
 189
 190 /**
 191  * vcn_v1_0_hw_init - start and test VCN block
 192  *
 193  * @handle: amdgpu_device pointer
 194  *
 195  * Initialize the hardware, boot up the VCPU and do some testing
 196  */
 197 static int vcn_v1_0_hw_init(void *handle)
 198 {
 199         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 200         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 201         int i, r;
 202
 203         r = amdgpu_ring_test_helper(ring);
 204         if (r)
 205                 return r;
 206
 207         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 208                 ring = &adev->vcn.inst->ring_enc[i];
 209                 r = amdgpu_ring_test_helper(ring);
 210                 if (r)
 211                         return r;
 212         }
 213
 214         ring = adev->jpeg.inst->ring_dec;
 215         r = amdgpu_ring_test_helper(ring);
 216
 217         return r;
 218 }
 219
 220 /**
 221  * vcn_v1_0_hw_fini - stop the hardware block
 222  *
 223  * @handle: amdgpu_device pointer
 224  *
 225  * Stop the VCN block, mark ring as not ready any more
 226  */
 227 static int vcn_v1_0_hw_fini(void *handle)
 228 {
 229         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 230
 231         cancel_delayed_work_sync(&adev->vcn.idle_work);
 232
 233         if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 234                 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 235                  RREG32_SOC15(VCN, 0, mmUVD_STATUS))) {
 236                 vcn_v1_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 237         }
 238
 239         return 0;
 240 }
 241
 242 /**
 243  * vcn_v1_0_suspend - suspend VCN block
 244  *
 245  * @handle: amdgpu_device pointer
 246  *
 247  * HW fini and suspend VCN block
 248  */
 249 static int vcn_v1_0_suspend(void *handle)
 250 {
 251         int r;
 252         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 253         bool idle_work_unexecuted;
 254
 255         idle_work_unexecuted = cancel_delayed_work_sync(&adev->vcn.idle_work);
 256         if (idle_work_unexecuted) {
 257                 if (adev->pm.dpm_enabled)
 258                         amdgpu_dpm_enable_uvd(adev, false);
 259         }
 260
 261         r = vcn_v1_0_hw_fini(adev);
 262         if (r)
 263                 return r;
 264
 265         r = amdgpu_vcn_suspend(adev);
 266
 267         return r;
 268 }
 269
 270 /**
 271  * vcn_v1_0_resume - resume VCN block
 272  *
 273  * @handle: amdgpu_device pointer
 274  *
 275  * Resume firmware and hw init VCN block
 276  */
 277 static int vcn_v1_0_resume(void *handle)
 278 {
 279         int r;
 280         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 281
 282         r = amdgpu_vcn_resume(adev);
 283         if (r)
 284                 return r;
 285
 286         r = vcn_v1_0_hw_init(adev);
 287
 288         return r;
 289 }
 290
 291 /**
 292  * vcn_v1_0_mc_resume_spg_mode - memory controller programming
 293  *
 294  * @adev: amdgpu_device pointer
 295  *
 296  * Let the VCN memory controller know it's offsets
 297  */
 298 static void vcn_v1_0_mc_resume_spg_mode(struct amdgpu_device *adev)
 299 {
 300         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
 301         uint32_t offset;
 302
 303         /* cache window 0: fw */
 304         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 305                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 306                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
 307                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 308                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
 309                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
 310                 offset = 0;
 311         } else {
 312                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 313                         lower_32_bits(adev->vcn.inst->gpu_addr));
 314                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 315                         upper_32_bits(adev->vcn.inst->gpu_addr));
 316                 offset = size;
 317                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 318                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 319         }
 320
 321         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 322
 323         /* cache window 1: stack */
 324         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 325                      lower_32_bits(adev->vcn.inst->gpu_addr + offset));
 326         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 327                      upper_32_bits(adev->vcn.inst->gpu_addr + offset));
 328         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 329         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 330
 331         /* cache window 2: context */
 332         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 333                      lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 334         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 335                      upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 336         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 337         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 338
 339         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 340                         adev->gfx.config.gb_addr_config);
 341         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 342                         adev->gfx.config.gb_addr_config);
 343         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 344                         adev->gfx.config.gb_addr_config);
 345         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_UV_ADDR_CONFIG,
 346                         adev->gfx.config.gb_addr_config);
 347         WREG32_SOC15(UVD, 0, mmUVD_MIF_CURR_ADDR_CONFIG,
 348                         adev->gfx.config.gb_addr_config);
 349         WREG32_SOC15(UVD, 0, mmUVD_MIF_CURR_UV_ADDR_CONFIG,
 350                         adev->gfx.config.gb_addr_config);
 351         WREG32_SOC15(UVD, 0, mmUVD_MIF_RECON1_ADDR_CONFIG,
 352                         adev->gfx.config.gb_addr_config);
 353         WREG32_SOC15(UVD, 0, mmUVD_MIF_RECON1_UV_ADDR_CONFIG,
 354                         adev->gfx.config.gb_addr_config);
 355         WREG32_SOC15(UVD, 0, mmUVD_MIF_REF_ADDR_CONFIG,
 356                         adev->gfx.config.gb_addr_config);
 357         WREG32_SOC15(UVD, 0, mmUVD_MIF_REF_UV_ADDR_CONFIG,
 358                         adev->gfx.config.gb_addr_config);
 359         WREG32_SOC15(UVD, 0, mmUVD_JPEG_ADDR_CONFIG,
 360                         adev->gfx.config.gb_addr_config);
 361         WREG32_SOC15(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG,
 362                         adev->gfx.config.gb_addr_config);
 363 }
 364
 365 static void vcn_v1_0_mc_resume_dpg_mode(struct amdgpu_device *adev)
 366 {
 367         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
 368         uint32_t offset;
 369
 370         /* cache window 0: fw */
 371         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 372                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 373                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo),
 374                              0xFFFFFFFF, 0);
 375                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 376                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi),
 377                              0xFFFFFFFF, 0);
 378                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0,
 379                              0xFFFFFFFF, 0);
 380                 offset = 0;
 381         } else {
 382                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 383                         lower_32_bits(adev->vcn.inst->gpu_addr), 0xFFFFFFFF, 0);
 384                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 385                         upper_32_bits(adev->vcn.inst->gpu_addr), 0xFFFFFFFF, 0);
 386                 offset = size;
 387                 WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 388                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0xFFFFFFFF, 0);
 389         }
 390
 391         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size, 0xFFFFFFFF, 0);
 392
 393         /* cache window 1: stack */
 394         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 395                      lower_32_bits(adev->vcn.inst->gpu_addr + offset), 0xFFFFFFFF, 0);
 396         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 397                      upper_32_bits(adev->vcn.inst->gpu_addr + offset), 0xFFFFFFFF, 0);
 398         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0,
 399                              0xFFFFFFFF, 0);
 400         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE,
 401                              0xFFFFFFFF, 0);
 402
 403         /* cache window 2: context */
 404         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 405                      lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 406                              0xFFFFFFFF, 0);
 407         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 408                      upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 409                              0xFFFFFFFF, 0);
 410         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0, 0xFFFFFFFF, 0);
 411         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE,
 412                              0xFFFFFFFF, 0);
 413
 414         /* VCN global tiling registers */
 415         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 416                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 417         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 418                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 419         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 420                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 421         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_UDEC_DBW_UV_ADDR_CONFIG,
 422                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 423         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_CURR_ADDR_CONFIG,
 424                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 425         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_CURR_UV_ADDR_CONFIG,
 426                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 427         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_RECON1_ADDR_CONFIG,
 428                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 429         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_RECON1_UV_ADDR_CONFIG,
 430                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 431         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_REF_ADDR_CONFIG,
 432                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 433         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MIF_REF_UV_ADDR_CONFIG,
 434                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 435 }
 436
 437 /**
 438  * vcn_v1_0_disable_clock_gating - disable VCN clock gating
 439  *
 440  * @adev: amdgpu_device pointer
 441  *
 442  * Disable clock gating for VCN block
 443  */
 444 static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev)
 445 {
 446         uint32_t data;
 447
 448         /* JPEG disable CGC */
 449         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 450
 451         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 452                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 453         else
 454                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 455
 456         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 457         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 458         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 459
 460         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 461         data &= ~(JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 462         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 463
 464         /* UVD disable CGC */
 465         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 466         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 467                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 468         else
 469                 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 470
 471         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 472         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 473         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 474
 475         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 476         data &= ~(UVD_CGC_GATE__SYS_MASK
 477                 | UVD_CGC_GATE__UDEC_MASK
 478                 | UVD_CGC_GATE__MPEG2_MASK
 479                 | UVD_CGC_GATE__REGS_MASK
 480                 | UVD_CGC_GATE__RBC_MASK
 481                 | UVD_CGC_GATE__LMI_MC_MASK
 482                 | UVD_CGC_GATE__LMI_UMC_MASK
 483                 | UVD_CGC_GATE__IDCT_MASK
 484                 | UVD_CGC_GATE__MPRD_MASK
 485                 | UVD_CGC_GATE__MPC_MASK
 486                 | UVD_CGC_GATE__LBSI_MASK
 487                 | UVD_CGC_GATE__LRBBM_MASK
 488                 | UVD_CGC_GATE__UDEC_RE_MASK
 489                 | UVD_CGC_GATE__UDEC_CM_MASK
 490                 | UVD_CGC_GATE__UDEC_IT_MASK
 491                 | UVD_CGC_GATE__UDEC_DB_MASK
 492                 | UVD_CGC_GATE__UDEC_MP_MASK
 493                 | UVD_CGC_GATE__WCB_MASK
 494                 | UVD_CGC_GATE__VCPU_MASK
 495                 | UVD_CGC_GATE__SCPU_MASK);
 496         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 497
 498         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 499         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 500                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 501                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 502                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 503                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 504                 | UVD_CGC_CTRL__SYS_MODE_MASK
 505                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 506                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 507                 | UVD_CGC_CTRL__REGS_MODE_MASK
 508                 | UVD_CGC_CTRL__RBC_MODE_MASK
 509                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 510                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 511                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 512                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 513                 | UVD_CGC_CTRL__MPC_MODE_MASK
 514                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 515                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 516                 | UVD_CGC_CTRL__WCB_MODE_MASK
 517                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 518                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 519         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 520
 521         /* turn on */
 522         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 523         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 524                 | UVD_SUVD_CGC_GATE__SIT_MASK
 525                 | UVD_SUVD_CGC_GATE__SMP_MASK
 526                 | UVD_SUVD_CGC_GATE__SCM_MASK
 527                 | UVD_SUVD_CGC_GATE__SDB_MASK
 528                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 529                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 530                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 531                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 532                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 533                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 534                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 535                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 536                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 537                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 538                 | UVD_SUVD_CGC_GATE__ENT_MASK
 539                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 540                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 541                 | UVD_SUVD_CGC_GATE__SITE_MASK
 542                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 543                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 544                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 545                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 546                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 547         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 548
 549         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 550         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 551                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 552                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 553                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 554                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 555                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 556                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 557                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 558                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 559                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 560         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 561 }
 562
 563 /**
 564  * vcn_v1_0_enable_clock_gating - enable VCN clock gating
 565  *
 566  * @adev: amdgpu_device pointer
 567  *
 568  * Enable clock gating for VCN block
 569  */
 570 static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev)
 571 {
 572         uint32_t data = 0;
 573
 574         /* enable JPEG CGC */
 575         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 576         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 577                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 578         else
 579                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 580         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 581         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 582         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 583
 584         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 585         data |= (JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 586         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 587
 588         /* enable UVD CGC */
 589         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 590         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 591                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 592         else
 593                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 594         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 595         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 596         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 597
 598         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 599         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 600                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 601                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 602                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 603                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 604                 | UVD_CGC_CTRL__SYS_MODE_MASK
 605                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 606                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 607                 | UVD_CGC_CTRL__REGS_MODE_MASK
 608                 | UVD_CGC_CTRL__RBC_MODE_MASK
 609                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 610                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 611                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 612                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 613                 | UVD_CGC_CTRL__MPC_MODE_MASK
 614                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 615                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 616                 | UVD_CGC_CTRL__WCB_MODE_MASK
 617                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 618                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 619         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 620
 621         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 622         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 623                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 624                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 625                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 626                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 627                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 628                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 629                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 630                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 631                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 632         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 633 }
 634
 635 static void vcn_v1_0_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel)
 636 {
 637         uint32_t reg_data = 0;
 638
 639         /* disable JPEG CGC */
 640         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 641                 reg_data = 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 642         else
 643                 reg_data = 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 644         reg_data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 645         reg_data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 646         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmJPEG_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
 647
 648         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmJPEG_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
 649
 650         /* enable sw clock gating control */
 651         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 652                 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 653         else
 654                 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 655         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 656         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 657         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 658                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 659                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 660                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 661                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 662                  UVD_CGC_CTRL__SYS_MODE_MASK |
 663                  UVD_CGC_CTRL__UDEC_MODE_MASK |
 664                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
 665                  UVD_CGC_CTRL__REGS_MODE_MASK |
 666                  UVD_CGC_CTRL__RBC_MODE_MASK |
 667                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 668                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 669                  UVD_CGC_CTRL__IDCT_MODE_MASK |
 670                  UVD_CGC_CTRL__MPRD_MODE_MASK |
 671                  UVD_CGC_CTRL__MPC_MODE_MASK |
 672                  UVD_CGC_CTRL__LBSI_MODE_MASK |
 673                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
 674                  UVD_CGC_CTRL__WCB_MODE_MASK |
 675                  UVD_CGC_CTRL__VCPU_MODE_MASK |
 676                  UVD_CGC_CTRL__SCPU_MODE_MASK);
 677         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
 678
 679         /* turn off clock gating */
 680         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
 681
 682         /* turn on SUVD clock gating */
 683         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_SUVD_CGC_GATE, 1, 0xFFFFFFFF, sram_sel);
 684
 685         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
 686         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_SUVD_CGC_CTRL, 0, 0xFFFFFFFF, sram_sel);
 687 }
 688
 689 static void vcn_1_0_disable_static_power_gating(struct amdgpu_device *adev)
 690 {
 691         uint32_t data = 0;
 692
 693         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 694                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 695                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 696                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 697                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 698                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 699                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 700                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 701                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 702                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 703                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 704                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 705
 706                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 707                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON, 0xFFFFFF);
 708         } else {
 709                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 710                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 711                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 712                         | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 713                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 714                         | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 715                         | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 716                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 717                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 718                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 719                         | 1 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 720                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 721                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFFFFF);
 722         }
 723
 724         /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS , UVDU_PWR_STATUS are 0 (power on) */
 725
 726         data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 727         data &= ~0x103;
 728         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 729                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON | UVD_POWER_STATUS__UVD_PG_EN_MASK;
 730
 731         WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 732 }
 733
 734 static void vcn_1_0_enable_static_power_gating(struct amdgpu_device *adev)
 735 {
 736         uint32_t data = 0;
 737
 738         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 739                 /* Before power off, this indicator has to be turned on */
 740                 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 741                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 742                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 743                 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 744
 745
 746                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 747                         | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 748                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 749                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 750                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 751                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 752                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 753                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 754                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 755                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 756                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 757
 758                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 759
 760                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 761                         | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
 762                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 763                         | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
 764                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 765                         | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
 766                         | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
 767                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 768                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 769                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
 770                         | 2 << UVD_PGFSM_STATUS__UVDW_PWR_STATUS__SHIFT);
 771                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFFFFF);
 772         }
 773 }
 774
 775 /**
 776  * vcn_v1_0_start_spg_mode - start VCN block
 777  *
 778  * @adev: amdgpu_device pointer
 779  *
 780  * Setup and start the VCN block
 781  */
 782 static int vcn_v1_0_start_spg_mode(struct amdgpu_device *adev)
 783 {
 784         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 785         uint32_t rb_bufsz, tmp;
 786         uint32_t lmi_swap_cntl;
 787         int i, j, r;
 788
 789         /* disable byte swapping */
 790         lmi_swap_cntl = 0;
 791
 792         vcn_1_0_disable_static_power_gating(adev);
 793
 794         tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
 795         WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
 796
 797         /* disable clock gating */
 798         vcn_v1_0_disable_clock_gating(adev);
 799
 800         /* disable interupt */
 801         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 802                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 803
 804         /* initialize VCN memory controller */
 805         tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
 806         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp                |
 807                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 808                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK                       |
 809                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK            |
 810                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
 811
 812 #ifdef __BIG_ENDIAN
 813         /* swap (8 in 32) RB and IB */
 814         lmi_swap_cntl = 0xa;
 815 #endif
 816         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 817
 818         tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
 819         tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
 820         tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
 821         WREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL, tmp);
 822
 823         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
 824                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 825                 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 826                 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 827                 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
 828
 829         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
 830                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 831                 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 832                 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 833                 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
 834
 835         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
 836                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 837                 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 838                 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
 839
 840         vcn_v1_0_mc_resume_spg_mode(adev);
 841
 842         WREG32_SOC15(UVD, 0, mmUVD_REG_XX_MASK_1_0, 0x10);
 843         WREG32_SOC15(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK_1_0,
 844                 RREG32_SOC15(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK_1_0) | 0x3);
 845
 846         /* enable VCPU clock */
 847         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
 848
 849         /* boot up the VCPU */
 850         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 851                         ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 852
 853         /* enable UMC */
 854         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 855                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 856
 857         tmp = RREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET);
 858         tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 859         tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 860         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, tmp);
 861
 862         for (i = 0; i < 10; ++i) {
 863                 uint32_t status;
 864
 865                 for (j = 0; j < 100; ++j) {
 866                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
 867                         if (status & UVD_STATUS__IDLE)
 868                                 break;
 869                         mdelay(10);
 870                 }
 871                 r = 0;
 872                 if (status & UVD_STATUS__IDLE)
 873                         break;
 874
 875                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
 876                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 877                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
 878                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 879                 mdelay(10);
 880                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 881                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 882                 mdelay(10);
 883                 r = -1;
 884         }
 885
 886         if (r) {
 887                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
 888                 return r;
 889         }
 890         /* enable master interrupt */
 891         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 892                 UVD_MASTINT_EN__VCPU_EN_MASK, ~UVD_MASTINT_EN__VCPU_EN_MASK);
 893
 894         /* enable system interrupt for JRBC, TODO: move to set interrupt*/
 895         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SYS_INT_EN),
 896                 UVD_SYS_INT_EN__UVD_JRBC_EN_MASK,
 897                 ~UVD_SYS_INT_EN__UVD_JRBC_EN_MASK);
 898
 899         /* clear the busy bit of UVD_STATUS */
 900         tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) & ~UVD_STATUS__UVD_BUSY;
 901         WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
 902
 903         /* force RBC into idle state */
 904         rb_bufsz = order_base_2(ring->ring_size);
 905         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 906         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 907         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 908         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 909         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 910         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
 911
 912         /* set the write pointer delay */
 913         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
 914
 915         /* set the wb address */
 916         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
 917                         (upper_32_bits(ring->gpu_addr) >> 2));
 918
 919         /* program the RB_BASE for ring buffer */
 920         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 921                         lower_32_bits(ring->gpu_addr));
 922         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 923                         upper_32_bits(ring->gpu_addr));
 924
 925         /* Initialize the ring buffer's read and write pointers */
 926         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
 927
 928         WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
 929
 930         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 931         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
 932                         lower_32_bits(ring->wptr));
 933
 934         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
 935                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
 936
 937         ring = &adev->vcn.inst->ring_enc[0];
 938         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 939         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 940         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
 941         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 942         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
 943
 944         ring = &adev->vcn.inst->ring_enc[1];
 945         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 946         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 947         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
 948         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 949         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
 950
 951         jpeg_v1_0_start(adev, 0);
 952
 953         return 0;
 954 }
 955
 956 static int vcn_v1_0_start_dpg_mode(struct amdgpu_device *adev)
 957 {
 958         struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
 959         uint32_t rb_bufsz, tmp;
 960         uint32_t lmi_swap_cntl;
 961
 962         /* disable byte swapping */
 963         lmi_swap_cntl = 0;
 964
 965         vcn_1_0_enable_static_power_gating(adev);
 966
 967         /* enable dynamic power gating mode */
 968         tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
 969         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 970         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 971         WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
 972
 973         /* enable clock gating */
 974         vcn_v1_0_clock_gating_dpg_mode(adev, 0);
 975
 976         /* enable VCPU clock */
 977         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 978         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 979         tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
 980         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_VCPU_CNTL, tmp, 0xFFFFFFFF, 0);
 981
 982         /* disable interupt */
 983         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MASTINT_EN,
 984                         0, UVD_MASTINT_EN__VCPU_EN_MASK, 0);
 985
 986         /* initialize VCN memory controller */
 987         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_CTRL,
 988                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 989                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 990                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 991                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 992                 UVD_LMI_CTRL__REQ_MODE_MASK |
 993                 UVD_LMI_CTRL__CRC_RESET_MASK |
 994                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 995                 0x00100000L, 0xFFFFFFFF, 0);
 996
 997 #ifdef __BIG_ENDIAN
 998         /* swap (8 in 32) RB and IB */
 999         lmi_swap_cntl = 0xa;
1000 #endif
1001         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl, 0xFFFFFFFF, 0);
1002
1003         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MPC_CNTL,
1004                 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0xFFFFFFFF, 0);
1005
1006         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MPC_SET_MUXA0,
1007                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1008                  (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1009                  (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1010                  (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0xFFFFFFFF, 0);
1011
1012         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MPC_SET_MUXB0,
1013                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1014                  (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1015                  (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1016                  (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0xFFFFFFFF, 0);
1017
1018         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MPC_SET_MUX,
1019                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1020                  (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1021                  (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0xFFFFFFFF, 0);
1022
1023         vcn_v1_0_mc_resume_dpg_mode(adev);
1024
1025         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_REG_XX_MASK, 0x10, 0xFFFFFFFF, 0);
1026         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK, 0x3, 0xFFFFFFFF, 0);
1027
1028         /* boot up the VCPU */
1029         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_SOFT_RESET, 0, 0xFFFFFFFF, 0);
1030
1031         /* enable UMC */
1032         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_CTRL2,
1033                 0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT,
1034                 0xFFFFFFFF, 0);
1035
1036         /* enable master interrupt */
1037         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_MASTINT_EN,
1038                         UVD_MASTINT_EN__VCPU_EN_MASK, UVD_MASTINT_EN__VCPU_EN_MASK, 0);
1039
1040         vcn_v1_0_clock_gating_dpg_mode(adev, 1);
1041         /* setup mmUVD_LMI_CTRL */
1042         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_LMI_CTRL,
1043                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1044                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1045                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1046                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1047                 UVD_LMI_CTRL__REQ_MODE_MASK |
1048                 UVD_LMI_CTRL__CRC_RESET_MASK |
1049                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1050                 0x00100000L, 0xFFFFFFFF, 1);
1051
1052         tmp = adev->gfx.config.gb_addr_config;
1053         /* setup VCN global tiling registers */
1054         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_JPEG_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1055         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1056
1057         /* enable System Interrupt for JRBC */
1058         WREG32_SOC15_DPG_MODE_1_0(UVD, 0, mmUVD_SYS_INT_EN,
1059                                                                         UVD_SYS_INT_EN__UVD_JRBC_EN_MASK, 0xFFFFFFFF, 1);
1060
1061         /* force RBC into idle state */
1062         rb_bufsz = order_base_2(ring->ring_size);
1063         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1064         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1065         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1066         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1067         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1068         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1069
1070         /* set the write pointer delay */
1071         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
1072
1073         /* set the wb address */
1074         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
1075                                                                 (upper_32_bits(ring->gpu_addr) >> 2));
1076
1077         /* program the RB_BASE for ring buffer */
1078         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1079                                                                 lower_32_bits(ring->gpu_addr));
1080         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1081                                                                 upper_32_bits(ring->gpu_addr));
1082
1083         /* Initialize the ring buffer's read and write pointers */
1084         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1085
1086         WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
1087
1088         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1089         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1090                                                                 lower_32_bits(ring->wptr));
1091
1092         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
1093                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
1094
1095         jpeg_v1_0_start(adev, 1);
1096
1097         return 0;
1098 }
1099
1100 static int vcn_v1_0_start(struct amdgpu_device *adev)
1101 {
1102         return (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ?
1103                 vcn_v1_0_start_dpg_mode(adev) : vcn_v1_0_start_spg_mode(adev);
1104 }
1105
1106 /**
1107  * vcn_v1_0_stop_spg_mode - stop VCN block
1108  *
1109  * @adev: amdgpu_device pointer
1110  *
1111  * stop the VCN block
1112  */
1113 static int vcn_v1_0_stop_spg_mode(struct amdgpu_device *adev)
1114 {
1115         int tmp;
1116
1117         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1118
1119         tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1120                 UVD_LMI_STATUS__READ_CLEAN_MASK |
1121                 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1122                 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1123         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_LMI_STATUS, tmp, tmp);
1124
1125         /* stall UMC channel */
1126         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
1127                 UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
1128                 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1129
1130         tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1131                 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1132         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_LMI_STATUS, tmp, tmp);
1133
1134         /* disable VCPU clock */
1135         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
1136                 ~UVD_VCPU_CNTL__CLK_EN_MASK);
1137
1138         /* reset LMI UMC/LMI */
1139         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1140                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
1141                 ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
1142
1143         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1144                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
1145                 ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
1146
1147         /* put VCPU into reset */
1148         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1149                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1150                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1151
1152         WREG32_SOC15(UVD, 0, mmUVD_STATUS, 0);
1153
1154         vcn_v1_0_enable_clock_gating(adev);
1155         vcn_1_0_enable_static_power_gating(adev);
1156         return 0;
1157 }
1158
1159 static int vcn_v1_0_stop_dpg_mode(struct amdgpu_device *adev)
1160 {
1161         uint32_t tmp;
1162
1163         /* Wait for power status to be UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF */
1164         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1165                         UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1166                         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1167
1168         /* wait for read ptr to be equal to write ptr */
1169         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1170         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1171
1172         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1173         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1174
1175         tmp = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1176         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_JRBC_RB_RPTR, tmp, 0xFFFFFFFF);
1177
1178         tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1179         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1180
1181         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1182                 UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1183                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1184
1185         /* disable dynamic power gating mode */
1186         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1187                         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1188
1189         return 0;
1190 }
1191
1192 static int vcn_v1_0_stop(struct amdgpu_device *adev)
1193 {
1194         int r;
1195
1196         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1197                 r = vcn_v1_0_stop_dpg_mode(adev);
1198         else
1199                 r = vcn_v1_0_stop_spg_mode(adev);
1200
1201         return r;
1202 }
1203
1204 static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
1205                                 int inst_idx, struct dpg_pause_state *new_state)
1206 {
1207         int ret_code;
1208         uint32_t reg_data = 0;
1209         uint32_t reg_data2 = 0;
1210         struct amdgpu_ring *ring;
1211
1212         /* pause/unpause if state is changed */
1213         if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1214                 DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
1215                         adev->vcn.inst[inst_idx].pause_state.fw_based,
1216                         adev->vcn.inst[inst_idx].pause_state.jpeg,
1217                         new_state->fw_based, new_state->jpeg);
1218
1219                 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1220                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1221
1222                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1223                         ret_code = 0;
1224
1225                         if (!(reg_data & UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK))
1226                                 ret_code = SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1227                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1228                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1229
1230                         if (!ret_code) {
1231                                 /* pause DPG non-jpeg */
1232                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1233                                 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1234                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1235                                                    UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1236                                                    UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1237
1238                                 /* Restore */
1239                                 ring = &adev->vcn.inst->ring_enc[0];
1240                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1241                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1242                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1243                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1244                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1245
1246                                 ring = &adev->vcn.inst->ring_enc[1];
1247                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1248                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1249                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1250                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1251                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1252
1253                                 ring = &adev->vcn.inst->ring_dec;
1254                                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1255                                                    RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1256                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1257                                                    UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1258                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1259                         }
1260                 } else {
1261                         /* unpause dpg non-jpeg, no need to wait */
1262                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1263                         WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1264                 }
1265                 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1266         }
1267
1268         /* pause/unpause if state is changed */
1269         if (adev->vcn.inst[inst_idx].pause_state.jpeg != new_state->jpeg) {
1270                 DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
1271                         adev->vcn.inst[inst_idx].pause_state.fw_based,
1272                         adev->vcn.inst[inst_idx].pause_state.jpeg,
1273                         new_state->fw_based, new_state->jpeg);
1274
1275                 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1276                         (~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK);
1277
1278                 if (new_state->jpeg == VCN_DPG_STATE__PAUSE) {
1279                         ret_code = 0;
1280
1281                         if (!(reg_data & UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK))
1282                                 ret_code = SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1283                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1284                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1285
1286                         if (!ret_code) {
1287                                 /* Make sure JPRG Snoop is disabled before sending the pause */
1288                                 reg_data2 = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
1289                                 reg_data2 |= UVD_POWER_STATUS__JRBC_SNOOP_DIS_MASK;
1290                                 WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, reg_data2);
1291
1292                                 /* pause DPG jpeg */
1293                                 reg_data |= UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
1294                                 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1295                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1296                                                         UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK,
1297                                                         UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK);
1298
1299                                 /* Restore */
1300                                 ring = adev->jpeg.inst->ring_dec;
1301                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
1302                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
1303                                                         UVD_JRBC_RB_CNTL__RB_NO_FETCH_MASK |
1304                                                         UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
1305                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
1306                                                         lower_32_bits(ring->gpu_addr));
1307                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
1308                                                         upper_32_bits(ring->gpu_addr));
1309                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, ring->wptr);
1310                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, ring->wptr);
1311                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
1312                                                         UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
1313
1314                                 ring = &adev->vcn.inst->ring_dec;
1315                                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1316                                                    RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1317                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1318                                                    UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1319                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1320                         }
1321                 } else {
1322                         /* unpause dpg jpeg, no need to wait */
1323                         reg_data &= ~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
1324                         WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1325                 }
1326                 adev->vcn.inst[inst_idx].pause_state.jpeg = new_state->jpeg;
1327         }
1328
1329         return 0;
1330 }
1331
1332 static bool vcn_v1_0_is_idle(void *handle)
1333 {
1334         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1335
1336         return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
1337 }
1338
1339 static int vcn_v1_0_wait_for_idle(void *handle)
1340 {
1341         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1342         int ret;
1343
1344         ret = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
1345                 UVD_STATUS__IDLE);
1346
1347         return ret;
1348 }
1349
1350 static int vcn_v1_0_set_clockgating_state(void *handle,
1351                                           enum amd_clockgating_state state)
1352 {
1353         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1354         bool enable = (state == AMD_CG_STATE_GATE);
1355
1356         if (enable) {
1357                 /* wait for STATUS to clear */
1358                 if (!vcn_v1_0_is_idle(handle))
1359                         return -EBUSY;
1360                 vcn_v1_0_enable_clock_gating(adev);
1361         } else {
1362                 /* disable HW gating and enable Sw gating */
1363                 vcn_v1_0_disable_clock_gating(adev);
1364         }
1365         return 0;
1366 }
1367
1368 /**
1369  * vcn_v1_0_dec_ring_get_rptr - get read pointer
1370  *
1371  * @ring: amdgpu_ring pointer
1372  *
1373  * Returns the current hardware read pointer
1374  */
1375 static uint64_t vcn_v1_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1376 {
1377         struct amdgpu_device *adev = ring->adev;
1378
1379         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1380 }
1381
1382 /**
1383  * vcn_v1_0_dec_ring_get_wptr - get write pointer
1384  *
1385  * @ring: amdgpu_ring pointer
1386  *
1387  * Returns the current hardware write pointer
1388  */
1389 static uint64_t vcn_v1_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1390 {
1391         struct amdgpu_device *adev = ring->adev;
1392
1393         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1394 }
1395
1396 /**
1397  * vcn_v1_0_dec_ring_set_wptr - set write pointer
1398  *
1399  * @ring: amdgpu_ring pointer
1400  *
1401  * Commits the write pointer to the hardware
1402  */
1403 static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1404 {
1405         struct amdgpu_device *adev = ring->adev;
1406
1407         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1408                 WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
1409                         lower_32_bits(ring->wptr) | 0x80000000);
1410
1411         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1412 }
1413
1414 /**
1415  * vcn_v1_0_dec_ring_insert_start - insert a start command
1416  *
1417  * @ring: amdgpu_ring pointer
1418  *
1419  * Write a start command to the ring.
1420  */
1421 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1422 {
1423         struct amdgpu_device *adev = ring->adev;
1424
1425         amdgpu_ring_write(ring,
1426                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1427         amdgpu_ring_write(ring, 0);
1428         amdgpu_ring_write(ring,
1429                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1430         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_START << 1);
1431 }
1432
1433 /**
1434  * vcn_v1_0_dec_ring_insert_end - insert a end command
1435  *
1436  * @ring: amdgpu_ring pointer
1437  *
1438  * Write a end command to the ring.
1439  */
1440 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1441 {
1442         struct amdgpu_device *adev = ring->adev;
1443
1444         amdgpu_ring_write(ring,
1445                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1446         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
1447 }
1448
1449 /**
1450  * vcn_v1_0_dec_ring_emit_fence - emit an fence & trap command
1451  *
1452  * @ring: amdgpu_ring pointer
1453  * @addr: address
1454  * @seq: sequence number
1455  * @flags: fence related flags
1456  *
1457  * Write a fence and a trap command to the ring.
1458  */
1459 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1460                                      unsigned flags)
1461 {
1462         struct amdgpu_device *adev = ring->adev;
1463
1464         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1465
1466         amdgpu_ring_write(ring,
1467                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
1468         amdgpu_ring_write(ring, seq);
1469         amdgpu_ring_write(ring,
1470                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1471         amdgpu_ring_write(ring, addr & 0xffffffff);
1472         amdgpu_ring_write(ring,
1473                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1474         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1475         amdgpu_ring_write(ring,
1476                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1477         amdgpu_ring_write(ring, VCN_DEC_CMD_FENCE << 1);
1478
1479         amdgpu_ring_write(ring,
1480                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1481         amdgpu_ring_write(ring, 0);
1482         amdgpu_ring_write(ring,
1483                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1484         amdgpu_ring_write(ring, 0);
1485         amdgpu_ring_write(ring,
1486                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1487         amdgpu_ring_write(ring, VCN_DEC_CMD_TRAP << 1);
1488 }
1489
1490 /**
1491  * vcn_v1_0_dec_ring_emit_ib - execute indirect buffer
1492  *
1493  * @ring: amdgpu_ring pointer
1494  * @job: job to retrieve vmid from
1495  * @ib: indirect buffer to execute
1496  * @flags: unused
1497  *
1498  * Write ring commands to execute the indirect buffer
1499  */
1500 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1501                                         struct amdgpu_job *job,
1502                                         struct amdgpu_ib *ib,
1503                                         uint32_t flags)
1504 {
1505         struct amdgpu_device *adev = ring->adev;
1506         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1507
1508         amdgpu_ring_write(ring,
1509                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
1510         amdgpu_ring_write(ring, vmid);
1511
1512         amdgpu_ring_write(ring,
1513                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
1514         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1515         amdgpu_ring_write(ring,
1516                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
1517         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1518         amdgpu_ring_write(ring,
1519                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
1520         amdgpu_ring_write(ring, ib->length_dw);
1521 }
1522
1523 static void vcn_v1_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
1524                                             uint32_t reg, uint32_t val,
1525                                             uint32_t mask)
1526 {
1527         struct amdgpu_device *adev = ring->adev;
1528
1529         amdgpu_ring_write(ring,
1530                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1531         amdgpu_ring_write(ring, reg << 2);
1532         amdgpu_ring_write(ring,
1533                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1534         amdgpu_ring_write(ring, val);
1535         amdgpu_ring_write(ring,
1536                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
1537         amdgpu_ring_write(ring, mask);
1538         amdgpu_ring_write(ring,
1539                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1540         amdgpu_ring_write(ring, VCN_DEC_CMD_REG_READ_COND_WAIT << 1);
1541 }
1542
1543 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1544                                             unsigned vmid, uint64_t pd_addr)
1545 {
1546         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1547         uint32_t data0, data1, mask;
1548
1549         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1550
1551         /* wait for register write */
1552         data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1553         data1 = lower_32_bits(pd_addr);
1554         mask = 0xffffffff;
1555         vcn_v1_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1556 }
1557
1558 static void vcn_v1_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1559                                         uint32_t reg, uint32_t val)
1560 {
1561         struct amdgpu_device *adev = ring->adev;
1562
1563         amdgpu_ring_write(ring,
1564                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1565         amdgpu_ring_write(ring, reg << 2);
1566         amdgpu_ring_write(ring,
1567                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1568         amdgpu_ring_write(ring, val);
1569         amdgpu_ring_write(ring,
1570                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1571         amdgpu_ring_write(ring, VCN_DEC_CMD_WRITE_REG << 1);
1572 }
1573
1574 /**
1575  * vcn_v1_0_enc_ring_get_rptr - get enc read pointer
1576  *
1577  * @ring: amdgpu_ring pointer
1578  *
1579  * Returns the current hardware enc read pointer
1580  */
1581 static uint64_t vcn_v1_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1582 {
1583         struct amdgpu_device *adev = ring->adev;
1584
1585         if (ring == &adev->vcn.inst->ring_enc[0])
1586                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1587         else
1588                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1589 }
1590
1591  /**
1592  * vcn_v1_0_enc_ring_get_wptr - get enc write pointer
1593  *
1594  * @ring: amdgpu_ring pointer
1595  *
1596  * Returns the current hardware enc write pointer
1597  */
1598 static uint64_t vcn_v1_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1599 {
1600         struct amdgpu_device *adev = ring->adev;
1601
1602         if (ring == &adev->vcn.inst->ring_enc[0])
1603                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1604         else
1605                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1606 }
1607
1608  /**
1609  * vcn_v1_0_enc_ring_set_wptr - set enc write pointer
1610  *
1611  * @ring: amdgpu_ring pointer
1612  *
1613  * Commits the enc write pointer to the hardware
1614  */
1615 static void vcn_v1_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1616 {
1617         struct amdgpu_device *adev = ring->adev;
1618
1619         if (ring == &adev->vcn.inst->ring_enc[0])
1620                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
1621                         lower_32_bits(ring->wptr));
1622         else
1623                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
1624                         lower_32_bits(ring->wptr));
1625 }
1626
1627 /**
1628  * vcn_v1_0_enc_ring_emit_fence - emit an enc fence & trap command
1629  *
1630  * @ring: amdgpu_ring pointer
1631  * @addr: address
1632  * @seq: sequence number
1633  * @flags: fence related flags
1634  *
1635  * Write enc a fence and a trap command to the ring.
1636  */
1637 static void vcn_v1_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1638                         u64 seq, unsigned flags)
1639 {
1640         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1641
1642         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1643         amdgpu_ring_write(ring, addr);
1644         amdgpu_ring_write(ring, upper_32_bits(addr));
1645         amdgpu_ring_write(ring, seq);
1646         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1647 }
1648
1649 static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1650 {
1651         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1652 }
1653
1654 /**
1655  * vcn_v1_0_enc_ring_emit_ib - enc execute indirect buffer
1656  *
1657  * @ring: amdgpu_ring pointer
1658  * @job: job to retrive vmid from
1659  * @ib: indirect buffer to execute
1660  * @flags: unused
1661  *
1662  * Write enc ring commands to execute the indirect buffer
1663  */
1664 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1665                                         struct amdgpu_job *job,
1666                                         struct amdgpu_ib *ib,
1667                                         uint32_t flags)
1668 {
1669         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1670
1671         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1672         amdgpu_ring_write(ring, vmid);
1673         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1674         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1675         amdgpu_ring_write(ring, ib->length_dw);
1676 }
1677
1678 static void vcn_v1_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
1679                                             uint32_t reg, uint32_t val,
1680                                             uint32_t mask)
1681 {
1682         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1683         amdgpu_ring_write(ring, reg << 2);
1684         amdgpu_ring_write(ring, mask);
1685         amdgpu_ring_write(ring, val);
1686 }
1687
1688 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1689                                             unsigned int vmid, uint64_t pd_addr)
1690 {
1691         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1692
1693         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1694
1695         /* wait for reg writes */
1696         vcn_v1_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +
1697                                         vmid * hub->ctx_addr_distance,
1698                                         lower_32_bits(pd_addr), 0xffffffff);
1699 }
1700
1701 static void vcn_v1_0_enc_ring_emit_wreg(struct amdgpu_ring *ring,
1702                                         uint32_t reg, uint32_t val)
1703 {
1704         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1705         amdgpu_ring_write(ring, reg << 2);
1706         amdgpu_ring_write(ring, val);
1707 }
1708
1709 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
1710                                         struct amdgpu_irq_src *source,
1711                                         unsigned type,
1712                                         enum amdgpu_interrupt_state state)
1713 {
1714         return 0;
1715 }
1716
1717 static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
1718                                       struct amdgpu_irq_src *source,
1719                                       struct amdgpu_iv_entry *entry)
1720 {
1721         DRM_DEBUG("IH: VCN TRAP\n");
1722
1723         switch (entry->src_id) {
1724         case 124:
1725                 amdgpu_fence_process(&adev->vcn.inst->ring_dec);
1726                 break;
1727         case 119:
1728                 amdgpu_fence_process(&adev->vcn.inst->ring_enc[0]);
1729                 break;
1730         case 120:
1731                 amdgpu_fence_process(&adev->vcn.inst->ring_enc[1]);
1732                 break;
1733         default:
1734                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1735                           entry->src_id, entry->src_data[0]);
1736                 break;
1737         }
1738
1739         return 0;
1740 }
1741
1742 static void vcn_v1_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1743 {
1744         struct amdgpu_device *adev = ring->adev;
1745         int i;
1746
1747         WARN_ON(ring->wptr % 2 || count % 2);
1748
1749         for (i = 0; i < count / 2; i++) {
1750                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
1751                 amdgpu_ring_write(ring, 0);
1752         }
1753 }
1754
1755 static int vcn_v1_0_set_powergating_state(void *handle,
1756                                           enum amd_powergating_state state)
1757 {
1758         /* This doesn't actually powergate the VCN block.
1759          * That's done in the dpm code via the SMC.  This
1760          * just re-inits the block as necessary.  The actual
1761          * gating still happens in the dpm code.  We should
1762          * revisit this when there is a cleaner line between
1763          * the smc and the hw blocks
1764          */
1765         int ret;
1766         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1767
1768         if (state == adev->vcn.cur_state)
1769                 return 0;
1770
1771         if (state == AMD_PG_STATE_GATE)
1772                 ret = vcn_v1_0_stop(adev);
1773         else
1774                 ret = vcn_v1_0_start(adev);
1775
1776         if (!ret)
1777                 adev->vcn.cur_state = state;
1778         return ret;
1779 }
1780
1781 static void vcn_v1_0_idle_work_handler(struct work_struct *work)
1782 {
1783         struct amdgpu_device *adev =
1784                 container_of(work, struct amdgpu_device, vcn.idle_work.work);
1785         unsigned int fences = 0, i;
1786
1787         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
1788                 fences += amdgpu_fence_count_emitted(&adev->vcn.inst->ring_enc[i]);
1789
1790         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1791                 struct dpg_pause_state new_state;
1792
1793                 if (fences)
1794                         new_state.fw_based = VCN_DPG_STATE__PAUSE;
1795                 else
1796                         new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
1797
1798                 if (amdgpu_fence_count_emitted(adev->jpeg.inst->ring_dec))
1799                         new_state.jpeg = VCN_DPG_STATE__PAUSE;
1800                 else
1801                         new_state.jpeg = VCN_DPG_STATE__UNPAUSE;
1802
1803                 adev->vcn.pause_dpg_mode(adev, 0, &new_state);
1804         }
1805
1806         fences += amdgpu_fence_count_emitted(adev->jpeg.inst->ring_dec);
1807         fences += amdgpu_fence_count_emitted(&adev->vcn.inst->ring_dec);
1808
1809         if (fences == 0) {
1810                 amdgpu_gfx_off_ctrl(adev, true);
1811                 if (adev->pm.dpm_enabled)
1812                         amdgpu_dpm_enable_uvd(adev, false);
1813                 else
1814                         amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
1815                                AMD_PG_STATE_GATE);
1816         } else {
1817                 schedule_delayed_work(&adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
1818         }
1819 }
1820
1821 static void vcn_v1_0_ring_begin_use(struct amdgpu_ring *ring)
1822 {
1823         struct  amdgpu_device *adev = ring->adev;
1824         bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work);
1825
1826         mutex_lock(&adev->vcn.vcn1_jpeg1_workaround);
1827
1828         if (amdgpu_fence_wait_empty(ring->adev->jpeg.inst->ring_dec))
1829                 DRM_ERROR("VCN dec: jpeg dec ring may not be empty\n");
1830
1831         vcn_v1_0_set_pg_for_begin_use(ring, set_clocks);
1832
1833 }
1834
1835 void vcn_v1_0_set_pg_for_begin_use(struct amdgpu_ring *ring, bool set_clocks)
1836 {
1837         struct amdgpu_device *adev = ring->adev;
1838
1839         if (set_clocks) {
1840                 amdgpu_gfx_off_ctrl(adev, false);
1841                 if (adev->pm.dpm_enabled)
1842                         amdgpu_dpm_enable_uvd(adev, true);
1843                 else
1844                         amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
1845                                AMD_PG_STATE_UNGATE);
1846         }
1847
1848         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1849                 struct dpg_pause_state new_state;
1850                 unsigned int fences = 0, i;
1851
1852                 for (i = 0; i < adev->vcn.num_enc_rings; ++i)
1853                         fences += amdgpu_fence_count_emitted(&adev->vcn.inst->ring_enc[i]);
1854
1855                 if (fences)
1856                         new_state.fw_based = VCN_DPG_STATE__PAUSE;
1857                 else
1858                         new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
1859
1860                 if (amdgpu_fence_count_emitted(adev->jpeg.inst->ring_dec))
1861                         new_state.jpeg = VCN_DPG_STATE__PAUSE;
1862                 else
1863                         new_state.jpeg = VCN_DPG_STATE__UNPAUSE;
1864
1865                 if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC)
1866                         new_state.fw_based = VCN_DPG_STATE__PAUSE;
1867                 else if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
1868                         new_state.jpeg = VCN_DPG_STATE__PAUSE;
1869
1870                 adev->vcn.pause_dpg_mode(adev, 0, &new_state);
1871         }
1872 }
1873
1874 void vcn_v1_0_ring_end_use(struct amdgpu_ring *ring)
1875 {
1876         schedule_delayed_work(&ring->adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
1877         mutex_unlock(&ring->adev->vcn.vcn1_jpeg1_workaround);
1878 }
1879
1880 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
1881         .name = "vcn_v1_0",
1882         .early_init = vcn_v1_0_early_init,
1883         .late_init = NULL,
1884         .sw_init = vcn_v1_0_sw_init,
1885         .sw_fini = vcn_v1_0_sw_fini,
1886         .hw_init = vcn_v1_0_hw_init,
1887         .hw_fini = vcn_v1_0_hw_fini,
1888         .suspend = vcn_v1_0_suspend,
1889         .resume = vcn_v1_0_resume,
1890         .is_idle = vcn_v1_0_is_idle,
1891         .wait_for_idle = vcn_v1_0_wait_for_idle,
1892         .check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
1893         .pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
1894         .soft_reset = NULL /* vcn_v1_0_soft_reset */,
1895         .post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
1896         .set_clockgating_state = vcn_v1_0_set_clockgating_state,
1897         .set_powergating_state = vcn_v1_0_set_powergating_state,
1898         .dump_ip_state = NULL,
1899         .print_ip_state = NULL,
1900 };
1901
1902 /*
1903  * It is a hardware issue that VCN can't handle a GTT TMZ buffer on
1904  * CHIP_RAVEN series ASIC. Move such a GTT TMZ buffer to VRAM domain
1905  * before command submission as a workaround.
1906  */
1907 static int vcn_v1_0_validate_bo(struct amdgpu_cs_parser *parser,
1908                                 struct amdgpu_job *job,
1909                                 uint64_t addr)
1910 {
1911         struct ttm_operation_ctx ctx = { false, false };
1912         struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1913         struct amdgpu_vm *vm = &fpriv->vm;
1914         struct amdgpu_bo_va_mapping *mapping;
1915         struct amdgpu_bo *bo;
1916         int r;
1917
1918         addr &= AMDGPU_GMC_HOLE_MASK;
1919         if (addr & 0x7) {
1920                 DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1921                 return -EINVAL;
1922         }
1923
1924         mapping = amdgpu_vm_bo_lookup_mapping(vm, addr/AMDGPU_GPU_PAGE_SIZE);
1925         if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1926                 return -EINVAL;
1927
1928         bo = mapping->bo_va->base.bo;
1929         if (!(bo->flags & AMDGPU_GEM_CREATE_ENCRYPTED))
1930                 return 0;
1931
1932         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
1933         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1934         if (r) {
1935                 DRM_ERROR("Failed to validate the VCN message BO (%d)!\n", r);
1936                 return r;
1937         }
1938
1939         return r;
1940 }
1941
1942 static int vcn_v1_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1943                                            struct amdgpu_job *job,
1944                                            struct amdgpu_ib *ib)
1945 {
1946         uint32_t msg_lo = 0, msg_hi = 0;
1947         int i, r;
1948
1949         if (!(ib->flags & AMDGPU_IB_FLAGS_SECURE))
1950                 return 0;
1951
1952         for (i = 0; i < ib->length_dw; i += 2) {
1953                 uint32_t reg = amdgpu_ib_get_value(ib, i);
1954                 uint32_t val = amdgpu_ib_get_value(ib, i + 1);
1955
1956                 if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1957                         msg_lo = val;
1958                 } else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1959                         msg_hi = val;
1960                 } else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0)) {
1961                         r = vcn_v1_0_validate_bo(p, job,
1962                                                  ((u64)msg_hi) << 32 | msg_lo);
1963                         if (r)
1964                                 return r;
1965                 }
1966         }
1967
1968         return 0;
1969 }
1970
1971 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
1972         .type = AMDGPU_RING_TYPE_VCN_DEC,
1973         .align_mask = 0xf,
1974         .support_64bit_ptrs = false,
1975         .no_user_fence = true,
1976         .secure_submission_supported = true,
1977         .get_rptr = vcn_v1_0_dec_ring_get_rptr,
1978         .get_wptr = vcn_v1_0_dec_ring_get_wptr,
1979         .set_wptr = vcn_v1_0_dec_ring_set_wptr,
1980         .patch_cs_in_place = vcn_v1_0_ring_patch_cs_in_place,
1981         .emit_frame_size =
1982                 6 + 6 + /* hdp invalidate / flush */
1983                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1984                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1985                 8 + /* vcn_v1_0_dec_ring_emit_vm_flush */
1986                 14 + 14 + /* vcn_v1_0_dec_ring_emit_fence x2 vm fence */
1987                 6,
1988         .emit_ib_size = 8, /* vcn_v1_0_dec_ring_emit_ib */
1989         .emit_ib = vcn_v1_0_dec_ring_emit_ib,
1990         .emit_fence = vcn_v1_0_dec_ring_emit_fence,
1991         .emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
1992         .test_ring = amdgpu_vcn_dec_ring_test_ring,
1993         .test_ib = amdgpu_vcn_dec_ring_test_ib,
1994         .insert_nop = vcn_v1_0_dec_ring_insert_nop,
1995         .insert_start = vcn_v1_0_dec_ring_insert_start,
1996         .insert_end = vcn_v1_0_dec_ring_insert_end,
1997         .pad_ib = amdgpu_ring_generic_pad_ib,
1998         .begin_use = vcn_v1_0_ring_begin_use,
1999         .end_use = vcn_v1_0_ring_end_use,
2000         .emit_wreg = vcn_v1_0_dec_ring_emit_wreg,
2001         .emit_reg_wait = vcn_v1_0_dec_ring_emit_reg_wait,
2002         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2003 };
2004
2005 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
2006         .type = AMDGPU_RING_TYPE_VCN_ENC,
2007         .align_mask = 0x3f,
2008         .nop = VCN_ENC_CMD_NO_OP,
2009         .support_64bit_ptrs = false,
2010         .no_user_fence = true,
2011         .get_rptr = vcn_v1_0_enc_ring_get_rptr,
2012         .get_wptr = vcn_v1_0_enc_ring_get_wptr,
2013         .set_wptr = vcn_v1_0_enc_ring_set_wptr,
2014         .emit_frame_size =
2015                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2016                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2017                 4 + /* vcn_v1_0_enc_ring_emit_vm_flush */
2018                 5 + 5 + /* vcn_v1_0_enc_ring_emit_fence x2 vm fence */
2019                 1, /* vcn_v1_0_enc_ring_insert_end */
2020         .emit_ib_size = 5, /* vcn_v1_0_enc_ring_emit_ib */
2021         .emit_ib = vcn_v1_0_enc_ring_emit_ib,
2022         .emit_fence = vcn_v1_0_enc_ring_emit_fence,
2023         .emit_vm_flush = vcn_v1_0_enc_ring_emit_vm_flush,
2024         .test_ring = amdgpu_vcn_enc_ring_test_ring,
2025         .test_ib = amdgpu_vcn_enc_ring_test_ib,
2026         .insert_nop = amdgpu_ring_insert_nop,
2027         .insert_end = vcn_v1_0_enc_ring_insert_end,
2028         .pad_ib = amdgpu_ring_generic_pad_ib,
2029         .begin_use = vcn_v1_0_ring_begin_use,
2030         .end_use = vcn_v1_0_ring_end_use,
2031         .emit_wreg = vcn_v1_0_enc_ring_emit_wreg,
2032         .emit_reg_wait = vcn_v1_0_enc_ring_emit_reg_wait,
2033         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2034 };
2035
2036 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2037 {
2038         adev->vcn.inst->ring_dec.funcs = &vcn_v1_0_dec_ring_vm_funcs;
2039 }
2040
2041 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2042 {
2043         int i;
2044
2045         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2046                 adev->vcn.inst->ring_enc[i].funcs = &vcn_v1_0_enc_ring_vm_funcs;
2047 }
2048
2049 static const struct amdgpu_irq_src_funcs vcn_v1_0_irq_funcs = {
2050         .set = vcn_v1_0_set_interrupt_state,
2051         .process = vcn_v1_0_process_interrupt,
2052 };
2053
2054 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev)
2055 {
2056         adev->vcn.inst->irq.num_types = adev->vcn.num_enc_rings + 2;
2057         adev->vcn.inst->irq.funcs = &vcn_v1_0_irq_funcs;
2058 }
2059
2060 const struct amdgpu_ip_block_version vcn_v1_0_ip_block = {
2061                 .type = AMD_IP_BLOCK_TYPE_VCN,
2062                 .major = 1,
2063                 .minor = 0,
2064                 .rev = 0,
2065                 .funcs = &vcn_v1_0_ip_funcs,
2066 };