drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_vcn.h"
  27 #include "amdgpu_pm.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "soc15_hw_ip.h"
  31 #include "vcn_v2_0.h"
  32
  33 #include "vcn/vcn_5_0_0_offset.h"
  34 #include "vcn/vcn_5_0_0_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  36 #include "vcn_v5_0_0.h"
  37
  38 #include <drm/drm_drv.h>
  39
  40 static int amdgpu_ih_clientid_vcns[] = {
  41         SOC15_IH_CLIENTID_VCN,
  42         SOC15_IH_CLIENTID_VCN1
  43 };
  44
  45 static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev);
  46 static void vcn_v5_0_0_set_irq_funcs(struct amdgpu_device *adev);
  47 static int vcn_v5_0_0_set_powergating_state(void *handle,
  48                 enum amd_powergating_state state);
  49 static int vcn_v5_0_0_pause_dpg_mode(struct amdgpu_device *adev,
  50                 int inst_idx, struct dpg_pause_state *new_state);
  51 static void vcn_v5_0_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
  52
  53 /**
  54  * vcn_v5_0_0_early_init - set function pointers and load microcode
  55  *
  56  * @handle: amdgpu_device pointer
  57  *
  58  * Set ring and irq function pointers
  59  * Load microcode from filesystem
  60  */
  61 static int vcn_v5_0_0_early_init(void *handle)
  62 {
  63         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  64
  65         /* re-use enc ring as unified ring */
  66         adev->vcn.num_enc_rings = 1;
  67
  68         vcn_v5_0_0_set_unified_ring_funcs(adev);
  69         vcn_v5_0_0_set_irq_funcs(adev);
  70
  71         return amdgpu_vcn_early_init(adev);
  72 }
  73
  74 /**
  75  * vcn_v5_0_0_sw_init - sw init for VCN block
  76  *
  77  * @handle: amdgpu_device pointer
  78  *
  79  * Load firmware and sw initialization
  80  */
  81 static int vcn_v5_0_0_sw_init(void *handle)
  82 {
  83         struct amdgpu_ring *ring;
  84         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  85         int i, r;
  86
  87         r = amdgpu_vcn_sw_init(adev);
  88         if (r)
  89                 return r;
  90
  91         amdgpu_vcn_setup_ucode(adev);
  92
  93         r = amdgpu_vcn_resume(adev);
  94         if (r)
  95                 return r;
  96
  97         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
  98                 volatile struct amdgpu_vcn5_fw_shared *fw_shared;
  99
 100                 if (adev->vcn.harvest_config & (1 << i))
 101                         continue;
 102
 103                 atomic_set(&adev->vcn.inst[i].sched_score, 0);
 104
 105                 /* VCN UNIFIED TRAP */
 106                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 107                                 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
 108                 if (r)
 109                         return r;
 110
 111                 /* VCN POISON TRAP */
 112                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 113                                 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].irq);
 114                 if (r)
 115                         return r;
 116
 117                 ring = &adev->vcn.inst[i].ring_enc[0];
 118                 ring->use_doorbell = true;
 119                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
 120
 121                 ring->vm_hub = AMDGPU_MMHUB0(0);
 122                 sprintf(ring->name, "vcn_unified_%d", i);
 123
 124                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 125                                                 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
 126                 if (r)
 127                         return r;
 128
 129                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 130                 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
 131                 fw_shared->sq.is_enabled = 1;
 132
 133                 if (amdgpu_vcnfw_log)
 134                         amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
 135         }
 136
 137         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 138                 adev->vcn.pause_dpg_mode = vcn_v5_0_0_pause_dpg_mode;
 139
 140         return 0;
 141 }
 142
 143 /**
 144  * vcn_v5_0_0_sw_fini - sw fini for VCN block
 145  *
 146  * @handle: amdgpu_device pointer
 147  *
 148  * VCN suspend and free up sw allocation
 149  */
 150 static int vcn_v5_0_0_sw_fini(void *handle)
 151 {
 152         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 153         int i, r, idx;
 154
 155         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 156                 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 157                         volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 158
 159                         if (adev->vcn.harvest_config & (1 << i))
 160                                 continue;
 161
 162                         fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 163                         fw_shared->present_flag_0 = 0;
 164                         fw_shared->sq.is_enabled = 0;
 165                 }
 166
 167                 drm_dev_exit(idx);
 168         }
 169
 170         r = amdgpu_vcn_suspend(adev);
 171         if (r)
 172                 return r;
 173
 174         r = amdgpu_vcn_sw_fini(adev);
 175
 176         return r;
 177 }
 178
 179 /**
 180  * vcn_v5_0_0_hw_init - start and test VCN block
 181  *
 182  * @handle: amdgpu_device pointer
 183  *
 184  * Initialize the hardware, boot up the VCPU and do some testing
 185  */
 186 static int vcn_v5_0_0_hw_init(void *handle)
 187 {
 188         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 189         struct amdgpu_ring *ring;
 190         int i, r;
 191
 192         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 193                 if (adev->vcn.harvest_config & (1 << i))
 194                         continue;
 195
 196                 ring = &adev->vcn.inst[i].ring_enc[0];
 197
 198                 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 199                         ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
 200
 201                 r = amdgpu_ring_test_helper(ring);
 202                 if (r)
 203                         return r;
 204         }
 205
 206         return 0;
 207 }
 208
 209 /**
 210  * vcn_v5_0_0_hw_fini - stop the hardware block
 211  *
 212  * @handle: amdgpu_device pointer
 213  *
 214  * Stop the VCN block, mark ring as not ready any more
 215  */
 216 static int vcn_v5_0_0_hw_fini(void *handle)
 217 {
 218         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 219         int i;
 220
 221         cancel_delayed_work_sync(&adev->vcn.idle_work);
 222
 223         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 224                 if (adev->vcn.harvest_config & (1 << i))
 225                         continue;
 226                 if (!amdgpu_sriov_vf(adev)) {
 227                         if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 228                                 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
 229                                 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
 230                                 vcn_v5_0_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 231                         }
 232                 }
 233         }
 234
 235         return 0;
 236 }
 237
 238 /**
 239  * vcn_v5_0_0_suspend - suspend VCN block
 240  *
 241  * @handle: amdgpu_device pointer
 242  *
 243  * HW fini and suspend VCN block
 244  */
 245 static int vcn_v5_0_0_suspend(void *handle)
 246 {
 247         int r;
 248         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 249
 250         r = vcn_v5_0_0_hw_fini(adev);
 251         if (r)
 252                 return r;
 253
 254         r = amdgpu_vcn_suspend(adev);
 255
 256         return r;
 257 }
 258
 259 /**
 260  * vcn_v5_0_0_resume - resume VCN block
 261  *
 262  * @handle: amdgpu_device pointer
 263  *
 264  * Resume firmware and hw init VCN block
 265  */
 266 static int vcn_v5_0_0_resume(void *handle)
 267 {
 268         int r;
 269         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 270
 271         r = amdgpu_vcn_resume(adev);
 272         if (r)
 273                 return r;
 274
 275         r = vcn_v5_0_0_hw_init(adev);
 276
 277         return r;
 278 }
 279
 280 /**
 281  * vcn_v5_0_0_mc_resume - memory controller programming
 282  *
 283  * @adev: amdgpu_device pointer
 284  * @inst: instance number
 285  *
 286  * Let the VCN memory controller know it's offsets
 287  */
 288 static void vcn_v5_0_0_mc_resume(struct amdgpu_device *adev, int inst)
 289 {
 290         uint32_t offset, size;
 291         const struct common_firmware_header *hdr;
 292
 293         hdr = (const struct common_firmware_header *)adev->vcn.fw[inst]->data;
 294         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 295
 296         /* cache window 0: fw */
 297         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 298                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 299                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
 300                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 301                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
 302                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
 303                 offset = 0;
 304         } else {
 305                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 306                         lower_32_bits(adev->vcn.inst[inst].gpu_addr));
 307                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 308                         upper_32_bits(adev->vcn.inst[inst].gpu_addr));
 309                 offset = size;
 310                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 311         }
 312         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
 313
 314         /* cache window 1: stack */
 315         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 316                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 317         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 318                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 319         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
 320         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 321
 322         /* cache window 2: context */
 323         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 324                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 325         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 326                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 327         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
 328         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 329
 330         /* non-cache window */
 331         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 332                 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 333         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 334                 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 335         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
 336         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
 337                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)));
 338 }
 339
 340 /**
 341  * vcn_v5_0_0_mc_resume_dpg_mode - memory controller programming for dpg mode
 342  *
 343  * @adev: amdgpu_device pointer
 344  * @inst_idx: instance number index
 345  * @indirect: indirectly write sram
 346  *
 347  * Let the VCN memory controller know it's offsets with dpg mode
 348  */
 349 static void vcn_v5_0_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 350 {
 351         uint32_t offset, size;
 352         const struct common_firmware_header *hdr;
 353
 354         hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data;
 355         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 356
 357         /* cache window 0: fw */
 358         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 359                 if (!indirect) {
 360                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 361                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 362                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
 363                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 364                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 365                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
 366                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 367                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 368                 } else {
 369                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 370                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 371                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 372                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 373                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 374                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 375                 }
 376                 offset = 0;
 377         } else {
 378                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 379                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 380                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 381                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 382                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 383                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 384                 offset = size;
 385                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 386                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
 387                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 388         }
 389
 390         if (!indirect)
 391                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 392                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 393         else
 394                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 395                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 396
 397         /* cache window 1: stack */
 398         if (!indirect) {
 399                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 400                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 401                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 402                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 403                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 404                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 405                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 406                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 407         } else {
 408                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 409                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 410                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 411                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 412                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 413                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 414         }
 415                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 416                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 417
 418         /* cache window 2: context */
 419         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 420                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 421                 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 422         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 423                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 424                 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 425         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 426                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 427         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 428                 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 429
 430         /* non-cache window */
 431         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 432                 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 433                 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 434         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 435                 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 436                 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 437         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 438                 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 439         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 440                 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
 441                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)), 0, indirect);
 442
 443         /* VCN global tiling registers */
 444         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 445                 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 446
 447         return;
 448 }
 449
 450 /**
 451  * vcn_v5_0_0_disable_static_power_gating - disable VCN static power gating
 452  *
 453  * @adev: amdgpu_device pointer
 454  * @inst: instance number
 455  *
 456  * Disable static power gating for VCN block
 457  */
 458 static void vcn_v5_0_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
 459 {
 460         uint32_t data = 0;
 461
 462         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 463                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 464                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 465                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 466                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 467
 468                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 469                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 470                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 471                                 1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 472                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 473
 474                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 475                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 476                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 477                                 1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 478                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 479
 480                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 481                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 482                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 483                                 1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 484                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 485         } else {
 486                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 487                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 488                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 489                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 490
 491                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 492                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 493                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 494                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 495
 496                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 497                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 498                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 499                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 500
 501                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 502                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 503                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 504                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 505         }
 506
 507         data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 508         data &= ~0x103;
 509         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 510                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 511                         UVD_POWER_STATUS__UVD_PG_EN_MASK;
 512
 513         WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 514         return;
 515 }
 516
 517 /**
 518  * vcn_v5_0_0_enable_static_power_gating - enable VCN static power gating
 519  *
 520  * @adev: amdgpu_device pointer
 521  * @inst: instance number
 522  *
 523  * Enable static power gating for VCN block
 524  */
 525 static void vcn_v5_0_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
 526 {
 527         uint32_t data;
 528
 529         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 530                 /* Before power off, this indicator has to be turned on */
 531                 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 532                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 533                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 534                 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 535
 536                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 537                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 538                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 539                                 1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 540                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 541
 542                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 543                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 544                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 545                                 1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 546                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 547
 548                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 549                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 550                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 551                                 1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 552                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 553
 554                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 555                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 556                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 557                                 1 << UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS__SHIFT,
 558                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 559         }
 560         return;
 561 }
 562
 563 /**
 564  * vcn_v5_0_0_disable_clock_gating - disable VCN clock gating
 565  *
 566  * @adev: amdgpu_device pointer
 567  * @inst: instance number
 568  *
 569  * Disable clock gating for VCN block
 570  */
 571 static void vcn_v5_0_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
 572 {
 573         return;
 574 }
 575
 576 #if 0
 577 /**
 578  * vcn_v5_0_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
 579  *
 580  * @adev: amdgpu_device pointer
 581  * @sram_sel: sram select
 582  * @inst_idx: instance number index
 583  * @indirect: indirectly write sram
 584  *
 585  * Disable clock gating for VCN block with dpg mode
 586  */
 587 static void vcn_v5_0_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
 588         int inst_idx, uint8_t indirect)
 589 {
 590         return;
 591 }
 592 #endif
 593
 594 /**
 595  * vcn_v5_0_0_enable_clock_gating - enable VCN clock gating
 596  *
 597  * @adev: amdgpu_device pointer
 598  * @inst: instance number
 599  *
 600  * Enable clock gating for VCN block
 601  */
 602 static void vcn_v5_0_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
 603 {
 604         return;
 605 }
 606
 607 /**
 608  * vcn_v5_0_0_start_dpg_mode - VCN start with dpg mode
 609  *
 610  * @adev: amdgpu_device pointer
 611  * @inst_idx: instance number index
 612  * @indirect: indirectly write sram
 613  *
 614  * Start VCN block with dpg mode
 615  */
 616 static int vcn_v5_0_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 617 {
 618         volatile struct amdgpu_vcn5_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
 619         struct amdgpu_ring *ring;
 620         uint32_t tmp;
 621
 622         /* disable register anti-hang mechanism */
 623         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
 624                 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 625
 626         /* enable dynamic power gating mode */
 627         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
 628         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 629         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 630         WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
 631
 632         if (indirect)
 633                 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 634
 635         /* enable VCPU clock */
 636         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 637         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
 638         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 639                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 640
 641         /* disable master interrupt */
 642         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 643                 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
 644
 645         /* setup regUVD_LMI_CTRL */
 646         tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 647                 UVD_LMI_CTRL__REQ_MODE_MASK |
 648                 UVD_LMI_CTRL__CRC_RESET_MASK |
 649                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 650                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 651                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 652                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 653                 0x00100000L);
 654         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 655                 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
 656
 657         vcn_v5_0_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
 658
 659         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 660         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 661         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 662                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 663
 664         /* enable LMI MC and UMC channels */
 665         tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
 666         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 667                 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
 668
 669         /* enable master interrupt */
 670         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 671                 VCN, inst_idx, regUVD_MASTINT_EN),
 672                 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
 673
 674         if (indirect)
 675                 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
 676
 677         ring = &adev->vcn.inst[inst_idx].ring_enc[0];
 678
 679         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
 680         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 681         WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
 682
 683         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
 684         tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
 685         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
 686         fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
 687         WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
 688         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
 689
 690         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
 691         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
 692         ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
 693
 694         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
 695         tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
 696         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
 697         fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
 698
 699         WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
 700                 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
 701                 VCN_RB1_DB_CTRL__EN_MASK);
 702
 703         return 0;
 704 }
 705
 706 /**
 707  * vcn_v5_0_0_start - VCN start
 708  *
 709  * @adev: amdgpu_device pointer
 710  *
 711  * Start VCN block
 712  */
 713 static int vcn_v5_0_0_start(struct amdgpu_device *adev)
 714 {
 715         volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 716         struct amdgpu_ring *ring;
 717         uint32_t tmp;
 718         int i, j, k, r;
 719
 720         if (adev->pm.dpm_enabled)
 721                 amdgpu_dpm_enable_uvd(adev, true);
 722
 723         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 724                 if (adev->vcn.harvest_config & (1 << i))
 725                         continue;
 726
 727                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 728
 729                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
 730                         r = vcn_v5_0_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
 731                         continue;
 732                 }
 733
 734                 /* disable VCN power gating */
 735                 vcn_v5_0_0_disable_static_power_gating(adev, i);
 736
 737                 /* set VCN status busy */
 738                 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
 739                 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
 740
 741                 /* enable VCPU clock */
 742                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 743                         UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
 744
 745                 /* disable master interrupt */
 746                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
 747                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 748
 749                 /* enable LMI MC and UMC channels */
 750                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
 751                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 752
 753                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 754                 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 755                 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 756                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 757
 758                 /* setup regUVD_LMI_CTRL */
 759                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
 760                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
 761                         UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 762                         UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 763                         UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 764                         UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
 765
 766                 vcn_v5_0_0_mc_resume(adev, i);
 767
 768                 /* VCN global tiling registers */
 769                 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
 770                         adev->gfx.config.gb_addr_config);
 771
 772                 /* unblock VCPU register access */
 773                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
 774                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
 775
 776                 /* release VCPU reset to boot */
 777                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 778                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 779
 780                 for (j = 0; j < 10; ++j) {
 781                         uint32_t status;
 782
 783                         for (k = 0; k < 100; ++k) {
 784                                 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
 785                                 if (status & 2)
 786                                         break;
 787                                 mdelay(10);
 788                                 if (amdgpu_emu_mode == 1)
 789                                         msleep(1);
 790                         }
 791
 792                         if (amdgpu_emu_mode == 1) {
 793                                 r = -1;
 794                                 if (status & 2) {
 795                                         r = 0;
 796                                         break;
 797                                 }
 798                         } else {
 799                                 r = 0;
 800                                 if (status & 2)
 801                                         break;
 802
 803                                 dev_err(adev->dev,
 804                                         "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
 805                                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 806                                                         UVD_VCPU_CNTL__BLK_RST_MASK,
 807                                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 808                                 mdelay(10);
 809                                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 810                                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 811
 812                                 mdelay(10);
 813                                 r = -1;
 814                         }
 815                 }
 816
 817                 if (r) {
 818                         dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
 819                         return r;
 820                 }
 821
 822                 /* enable master interrupt */
 823                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
 824                                 UVD_MASTINT_EN__VCPU_EN_MASK,
 825                                 ~UVD_MASTINT_EN__VCPU_EN_MASK);
 826
 827                 /* clear the busy bit of VCN_STATUS */
 828                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
 829                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 830
 831                 ring = &adev->vcn.inst[i].ring_enc[0];
 832                 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
 833                         ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
 834                         VCN_RB1_DB_CTRL__EN_MASK);
 835
 836                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
 837                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 838                 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
 839
 840                 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
 841                 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
 842                 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
 843                 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
 844                 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
 845                 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
 846
 847                 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
 848                 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
 849                 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
 850
 851                 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
 852                 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
 853                 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
 854                 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
 855         }
 856
 857         return 0;
 858 }
 859
 860 /**
 861  * vcn_v5_0_0_stop_dpg_mode - VCN stop with dpg mode
 862  *
 863  * @adev: amdgpu_device pointer
 864  * @inst_idx: instance number index
 865  *
 866  * Stop VCN block with dpg mode
 867  */
 868 static void vcn_v5_0_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
 869 {
 870         struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
 871         uint32_t tmp;
 872
 873         vcn_v5_0_0_pause_dpg_mode(adev, inst_idx, &state);
 874
 875         /* Wait for power status to be 1 */
 876         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
 877                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 878
 879         /* wait for read ptr to be equal to write ptr */
 880         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
 881         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
 882
 883         /* disable dynamic power gating mode */
 884         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
 885                 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
 886
 887         return;
 888 }
 889
 890 /**
 891  * vcn_v5_0_0_stop - VCN stop
 892  *
 893  * @adev: amdgpu_device pointer
 894  *
 895  * Stop VCN block
 896  */
 897 static int vcn_v5_0_0_stop(struct amdgpu_device *adev)
 898 {
 899         volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 900         uint32_t tmp;
 901         int i, r = 0;
 902
 903         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 904                 if (adev->vcn.harvest_config & (1 << i))
 905                         continue;
 906
 907                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 908                 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
 909
 910                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
 911                         vcn_v5_0_0_stop_dpg_mode(adev, i);
 912                         continue;
 913                 }
 914
 915                 /* wait for vcn idle */
 916                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
 917                 if (r)
 918                         return r;
 919
 920                 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
 921                       UVD_LMI_STATUS__READ_CLEAN_MASK |
 922                       UVD_LMI_STATUS__WRITE_CLEAN_MASK |
 923                       UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
 924                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
 925                 if (r)
 926                         return r;
 927
 928                 /* disable LMI UMC channel */
 929                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
 930                 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
 931                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
 932                 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
 933                       UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
 934                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
 935                 if (r)
 936                         return r;
 937
 938                 /* block VCPU register access */
 939                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
 940                         UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
 941                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
 942
 943                 /* reset VCPU */
 944                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 945                         UVD_VCPU_CNTL__BLK_RST_MASK,
 946                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 947
 948                 /* disable VCPU clock */
 949                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 950                         ~(UVD_VCPU_CNTL__CLK_EN_MASK));
 951
 952                 /* apply soft reset */
 953                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 954                 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 955                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 956                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 957                 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 958                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 959
 960                 /* clear status */
 961                 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
 962
 963                 /* enable VCN power gating */
 964                 vcn_v5_0_0_enable_static_power_gating(adev, i);
 965         }
 966
 967         if (adev->pm.dpm_enabled)
 968                 amdgpu_dpm_enable_uvd(adev, false);
 969
 970         return 0;
 971 }
 972
 973 /**
 974  * vcn_v5_0_0_pause_dpg_mode - VCN pause with dpg mode
 975  *
 976  * @adev: amdgpu_device pointer
 977  * @inst_idx: instance number index
 978  * @new_state: pause state
 979  *
 980  * Pause dpg mode for VCN block
 981  */
 982 static int vcn_v5_0_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
 983         struct dpg_pause_state *new_state)
 984 {
 985         uint32_t reg_data = 0;
 986         int ret_code;
 987
 988         /* pause/unpause if state is changed */
 989         if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
 990                 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
 991                         adev->vcn.inst[inst_idx].pause_state.fw_based,  new_state->fw_based);
 992                 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
 993                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
 994
 995                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
 996                         ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
 997                                         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 998
 999                         if (!ret_code) {
1000                                 /* pause DPG */
1001                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1002                                 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1003
1004                                 /* wait for ACK */
1005                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1006                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1007                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1008                         }
1009                 } else {
1010                         /* unpause dpg, no need to wait */
1011                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1012                         WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1013                 }
1014                 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1015         }
1016
1017         return 0;
1018 }
1019
1020 /**
1021  * vcn_v5_0_0_unified_ring_get_rptr - get unified read pointer
1022  *
1023  * @ring: amdgpu_ring pointer
1024  *
1025  * Returns the current hardware unified read pointer
1026  */
1027 static uint64_t vcn_v5_0_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1028 {
1029         struct amdgpu_device *adev = ring->adev;
1030
1031         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1032                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1033
1034         return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1035 }
1036
1037 /**
1038  * vcn_v5_0_0_unified_ring_get_wptr - get unified write pointer
1039  *
1040  * @ring: amdgpu_ring pointer
1041  *
1042  * Returns the current hardware unified write pointer
1043  */
1044 static uint64_t vcn_v5_0_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1045 {
1046         struct amdgpu_device *adev = ring->adev;
1047
1048         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1049                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1050
1051         if (ring->use_doorbell)
1052                 return *ring->wptr_cpu_addr;
1053         else
1054                 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1055 }
1056
1057 /**
1058  * vcn_v5_0_0_unified_ring_set_wptr - set enc write pointer
1059  *
1060  * @ring: amdgpu_ring pointer
1061  *
1062  * Commits the enc write pointer to the hardware
1063  */
1064 static void vcn_v5_0_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1065 {
1066         struct amdgpu_device *adev = ring->adev;
1067
1068         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1069                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1070
1071         if (ring->use_doorbell) {
1072                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1073                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1074         } else {
1075                 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1076         }
1077 }
1078
1079 static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs = {
1080         .type = AMDGPU_RING_TYPE_VCN_ENC,
1081         .align_mask = 0x3f,
1082         .nop = VCN_ENC_CMD_NO_OP,
1083         .get_rptr = vcn_v5_0_0_unified_ring_get_rptr,
1084         .get_wptr = vcn_v5_0_0_unified_ring_get_wptr,
1085         .set_wptr = vcn_v5_0_0_unified_ring_set_wptr,
1086         .emit_frame_size =
1087                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1088                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1089                 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1090                 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1091                 1, /* vcn_v2_0_enc_ring_insert_end */
1092         .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1093         .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1094         .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1095         .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1096         .test_ring = amdgpu_vcn_enc_ring_test_ring,
1097         .test_ib = amdgpu_vcn_unified_ring_test_ib,
1098         .insert_nop = amdgpu_ring_insert_nop,
1099         .insert_end = vcn_v2_0_enc_ring_insert_end,
1100         .pad_ib = amdgpu_ring_generic_pad_ib,
1101         .begin_use = amdgpu_vcn_ring_begin_use,
1102         .end_use = amdgpu_vcn_ring_end_use,
1103         .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1104         .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1105         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1106 };
1107
1108 /**
1109  * vcn_v5_0_0_set_unified_ring_funcs - set unified ring functions
1110  *
1111  * @adev: amdgpu_device pointer
1112  *
1113  * Set unified ring functions
1114  */
1115 static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev)
1116 {
1117         int i;
1118
1119         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1120                 if (adev->vcn.harvest_config & (1 << i))
1121                         continue;
1122
1123                 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v5_0_0_unified_ring_vm_funcs;
1124                 adev->vcn.inst[i].ring_enc[0].me = i;
1125         }
1126 }
1127
1128 /**
1129  * vcn_v5_0_0_is_idle - check VCN block is idle
1130  *
1131  * @handle: amdgpu_device pointer
1132  *
1133  * Check whether VCN block is idle
1134  */
1135 static bool vcn_v5_0_0_is_idle(void *handle)
1136 {
1137         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1138         int i, ret = 1;
1139
1140         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1141                 if (adev->vcn.harvest_config & (1 << i))
1142                         continue;
1143
1144                 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1145         }
1146
1147         return ret;
1148 }
1149
1150 /**
1151  * vcn_v5_0_0_wait_for_idle - wait for VCN block idle
1152  *
1153  * @handle: amdgpu_device pointer
1154  *
1155  * Wait for VCN block idle
1156  */
1157 static int vcn_v5_0_0_wait_for_idle(void *handle)
1158 {
1159         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1160         int i, ret = 0;
1161
1162         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1163                 if (adev->vcn.harvest_config & (1 << i))
1164                         continue;
1165
1166                 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1167                         UVD_STATUS__IDLE);
1168                 if (ret)
1169                         return ret;
1170         }
1171
1172         return ret;
1173 }
1174
1175 /**
1176  * vcn_v5_0_0_set_clockgating_state - set VCN block clockgating state
1177  *
1178  * @handle: amdgpu_device pointer
1179  * @state: clock gating state
1180  *
1181  * Set VCN block clockgating state
1182  */
1183 static int vcn_v5_0_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1184 {
1185         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1186         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1187         int i;
1188
1189         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1190                 if (adev->vcn.harvest_config & (1 << i))
1191                         continue;
1192
1193                 if (enable) {
1194                         if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1195                                 return -EBUSY;
1196                         vcn_v5_0_0_enable_clock_gating(adev, i);
1197                 } else {
1198                         vcn_v5_0_0_disable_clock_gating(adev, i);
1199                 }
1200         }
1201
1202         return 0;
1203 }
1204
1205 /**
1206  * vcn_v5_0_0_set_powergating_state - set VCN block powergating state
1207  *
1208  * @handle: amdgpu_device pointer
1209  * @state: power gating state
1210  *
1211  * Set VCN block powergating state
1212  */
1213 static int vcn_v5_0_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1214 {
1215         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1216         int ret;
1217
1218         if (state == adev->vcn.cur_state)
1219                 return 0;
1220
1221         if (state == AMD_PG_STATE_GATE)
1222                 ret = vcn_v5_0_0_stop(adev);
1223         else
1224                 ret = vcn_v5_0_0_start(adev);
1225
1226         if (!ret)
1227                 adev->vcn.cur_state = state;
1228
1229         return ret;
1230 }
1231
1232 /**
1233  * vcn_v5_0_0_process_interrupt - process VCN block interrupt
1234  *
1235  * @adev: amdgpu_device pointer
1236  * @source: interrupt sources
1237  * @entry: interrupt entry from clients and sources
1238  *
1239  * Process VCN block interrupt
1240  */
1241 static int vcn_v5_0_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1242         struct amdgpu_iv_entry *entry)
1243 {
1244         uint32_t ip_instance;
1245
1246         switch (entry->client_id) {
1247         case SOC15_IH_CLIENTID_VCN:
1248                 ip_instance = 0;
1249                 break;
1250         case SOC15_IH_CLIENTID_VCN1:
1251                 ip_instance = 1;
1252                 break;
1253         default:
1254                 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1255                 return 0;
1256         }
1257
1258         DRM_DEBUG("IH: VCN TRAP\n");
1259
1260         switch (entry->src_id) {
1261         case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1262                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1263                 break;
1264         case VCN_4_0__SRCID_UVD_POISON:
1265                 amdgpu_vcn_process_poison_irq(adev, source, entry);
1266                 break;
1267         default:
1268                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1269                           entry->src_id, entry->src_data[0]);
1270                 break;
1271         }
1272
1273         return 0;
1274 }
1275
1276 static const struct amdgpu_irq_src_funcs vcn_v5_0_0_irq_funcs = {
1277         .process = vcn_v5_0_0_process_interrupt,
1278 };
1279
1280 /**
1281  * vcn_v5_0_0_set_irq_funcs - set VCN block interrupt irq functions
1282  *
1283  * @adev: amdgpu_device pointer
1284  *
1285  * Set VCN block interrupt irq functions
1286  */
1287 static void vcn_v5_0_0_set_irq_funcs(struct amdgpu_device *adev)
1288 {
1289         int i;
1290
1291         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1292                 if (adev->vcn.harvest_config & (1 << i))
1293                         continue;
1294
1295                 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1296                 adev->vcn.inst[i].irq.funcs = &vcn_v5_0_0_irq_funcs;
1297         }
1298 }
1299
1300 static const struct amd_ip_funcs vcn_v5_0_0_ip_funcs = {
1301         .name = "vcn_v5_0_0",
1302         .early_init = vcn_v5_0_0_early_init,
1303         .late_init = NULL,
1304         .sw_init = vcn_v5_0_0_sw_init,
1305         .sw_fini = vcn_v5_0_0_sw_fini,
1306         .hw_init = vcn_v5_0_0_hw_init,
1307         .hw_fini = vcn_v5_0_0_hw_fini,
1308         .suspend = vcn_v5_0_0_suspend,
1309         .resume = vcn_v5_0_0_resume,
1310         .is_idle = vcn_v5_0_0_is_idle,
1311         .wait_for_idle = vcn_v5_0_0_wait_for_idle,
1312         .check_soft_reset = NULL,
1313         .pre_soft_reset = NULL,
1314         .soft_reset = NULL,
1315         .post_soft_reset = NULL,
1316         .set_clockgating_state = vcn_v5_0_0_set_clockgating_state,
1317         .set_powergating_state = vcn_v5_0_0_set_powergating_state,
1318         .dump_ip_state = NULL,
1319         .print_ip_state = NULL,
1320 };
1321
1322 const struct amdgpu_ip_block_version vcn_v5_0_0_ip_block = {
1323         .type = AMD_IP_BLOCK_TYPE_VCN,
1324         .major = 5,
1325         .minor = 0,
1326         .rev = 0,
1327         .funcs = &vcn_v5_0_0_ip_funcs,
1328 };