drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "amdgpu.h"
  25 #include "amdgpu_jpeg.h"
  26 #include "amdgpu_pm.h"
  27 #include "soc15.h"
  28 #include "soc15d.h"
  29 #include "jpeg_v2_0.h"
  30 #include "jpeg_v4_0_5.h"
  31 #include "mmsch_v4_0.h"
  32
  33 #include "vcn/vcn_4_0_5_offset.h"
  34 #include "vcn/vcn_4_0_5_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  36
  37 #define mmUVD_DPG_LMA_CTL                                               regUVD_DPG_LMA_CTL
  38 #define mmUVD_DPG_LMA_CTL_BASE_IDX                                      regUVD_DPG_LMA_CTL_BASE_IDX
  39 #define mmUVD_DPG_LMA_DATA                                              regUVD_DPG_LMA_DATA
  40 #define mmUVD_DPG_LMA_DATA_BASE_IDX                                     regUVD_DPG_LMA_DATA_BASE_IDX
  41
  42 #define regUVD_JPEG_PITCH_INTERNAL_OFFSET               0x401f
  43 #define regJPEG_DEC_GFX10_ADDR_CONFIG_INTERNAL_OFFSET   0x4026
  44 #define regJPEG_SYS_INT_EN_INTERNAL_OFFSET              0x4141
  45 #define regJPEG_CGC_CTRL_INTERNAL_OFFSET                0x4161
  46 #define regJPEG_CGC_GATE_INTERNAL_OFFSET                0x4160
  47 #define regUVD_NO_OP_INTERNAL_OFFSET                    0x0029
  48
  49 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev);
  50 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev);
  51 static int jpeg_v4_0_5_set_powergating_state(void *handle,
  52                                 enum amd_powergating_state state);
  53
  54 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring);
  55
  56 static int amdgpu_ih_clientid_jpeg[] = {
  57         SOC15_IH_CLIENTID_VCN,
  58         SOC15_IH_CLIENTID_VCN1
  59 };
  60
  61 /**
  62  * jpeg_v4_0_5_early_init - set function pointers
  63  *
  64  * @handle: amdgpu_device pointer
  65  *
  66  * Set ring and irq function pointers
  67  */
  68 static int jpeg_v4_0_5_early_init(void *handle)
  69 {
  70         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  71
  72         switch (amdgpu_ip_version(adev, UVD_HWIP, 0)) {
  73         case IP_VERSION(4, 0, 5):
  74                 adev->jpeg.num_jpeg_inst = 1;
  75                 break;
  76         case IP_VERSION(4, 0, 6):
  77                 adev->jpeg.num_jpeg_inst = 2;
  78                 break;
  79         default:
  80                 DRM_DEV_ERROR(adev->dev,
  81                         "Failed to init vcn ip block(UVD_HWIP:0x%x)\n",
  82                         amdgpu_ip_version(adev, UVD_HWIP, 0));
  83                 return -EINVAL;
  84         }
  85
  86         adev->jpeg.num_jpeg_rings = 1;
  87
  88         jpeg_v4_0_5_set_dec_ring_funcs(adev);
  89         jpeg_v4_0_5_set_irq_funcs(adev);
  90
  91         return 0;
  92 }
  93
  94 /**
  95  * jpeg_v4_0_5_sw_init - sw init for JPEG block
  96  *
  97  * @handle: amdgpu_device pointer
  98  *
  99  * Load firmware and sw initialization
 100  */
 101 static int jpeg_v4_0_5_sw_init(void *handle)
 102 {
 103         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 104         struct amdgpu_ring *ring;
 105         int r, i;
 106
 107         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 108                 if (adev->jpeg.harvest_config & (1 << i))
 109                         continue;
 110
 111                 /* JPEG TRAP */
 112                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
 113                                 VCN_4_0__SRCID__JPEG_DECODE, &adev->jpeg.inst[i].irq);
 114                 if (r)
 115                         return r;
 116
 117                 /* JPEG DJPEG POISON EVENT */
 118                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
 119                         VCN_4_0__SRCID_DJPEG0_POISON, &adev->jpeg.inst[i].irq);
 120                 if (r)
 121                         return r;
 122
 123                 /* JPEG EJPEG POISON EVENT */
 124                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
 125                         VCN_4_0__SRCID_EJPEG0_POISON, &adev->jpeg.inst[i].irq);
 126                 if (r)
 127                         return r;
 128         }
 129
 130         r = amdgpu_jpeg_sw_init(adev);
 131         if (r)
 132                 return r;
 133
 134         r = amdgpu_jpeg_resume(adev);
 135         if (r)
 136                 return r;
 137
 138         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 139                 if (adev->jpeg.harvest_config & (1 << i))
 140                         continue;
 141
 142                 ring = adev->jpeg.inst[i].ring_dec;
 143                 ring->use_doorbell = true;
 144                 ring->vm_hub = AMDGPU_MMHUB0(0);
 145                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i;
 146                 sprintf(ring->name, "jpeg_dec_%d", i);
 147                 r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq,
 148                                      0, AMDGPU_RING_PRIO_DEFAULT, NULL);
 149                 if (r)
 150                         return r;
 151
 152                 adev->jpeg.internal.jpeg_pitch[0] = regUVD_JPEG_PITCH_INTERNAL_OFFSET;
 153                 adev->jpeg.inst[i].external.jpeg_pitch[0] = SOC15_REG_OFFSET(JPEG, i, regUVD_JPEG_PITCH);
 154         }
 155
 156         return 0;
 157 }
 158
 159 /**
 160  * jpeg_v4_0_5_sw_fini - sw fini for JPEG block
 161  *
 162  * @handle: amdgpu_device pointer
 163  *
 164  * JPEG suspend and free up sw allocation
 165  */
 166 static int jpeg_v4_0_5_sw_fini(void *handle)
 167 {
 168         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 169         int r;
 170
 171         r = amdgpu_jpeg_suspend(adev);
 172         if (r)
 173                 return r;
 174
 175         r = amdgpu_jpeg_sw_fini(adev);
 176
 177         return r;
 178 }
 179
 180 /**
 181  * jpeg_v4_0_5_hw_init - start and test JPEG block
 182  *
 183  * @handle: amdgpu_device pointer
 184  *
 185  */
 186 static int jpeg_v4_0_5_hw_init(void *handle)
 187 {
 188         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 189         struct amdgpu_ring *ring;
 190         int i, r = 0;
 191
 192         // TODO: Enable ring test with DPG support
 193         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
 194                 return 0;
 195         }
 196
 197         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 198                 if (adev->jpeg.harvest_config & (1 << i))
 199                         continue;
 200
 201                 ring = adev->jpeg.inst[i].ring_dec;
 202                 r = amdgpu_ring_test_helper(ring);
 203                 if (r)
 204                         return r;
 205         }
 206
 207         return 0;
 208 }
 209
 210 /**
 211  * jpeg_v4_0_5_hw_fini - stop the hardware block
 212  *
 213  * @handle: amdgpu_device pointer
 214  *
 215  * Stop the JPEG block, mark ring as not ready any more
 216  */
 217 static int jpeg_v4_0_5_hw_fini(void *handle)
 218 {
 219         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 220         int i;
 221
 222         cancel_delayed_work_sync(&adev->vcn.idle_work);
 223
 224         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 225                 if (adev->jpeg.harvest_config & (1 << i))
 226                         continue;
 227
 228                 if (!amdgpu_sriov_vf(adev)) {
 229                         if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
 230                             RREG32_SOC15(JPEG, i, regUVD_JRBC_STATUS))
 231                                 jpeg_v4_0_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
 232                 }
 233         }
 234         return 0;
 235 }
 236
 237 /**
 238  * jpeg_v4_0_5_suspend - suspend JPEG block
 239  *
 240  * @handle: amdgpu_device pointer
 241  *
 242  * HW fini and suspend JPEG block
 243  */
 244 static int jpeg_v4_0_5_suspend(void *handle)
 245 {
 246         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 247         int r;
 248
 249         r = jpeg_v4_0_5_hw_fini(adev);
 250         if (r)
 251                 return r;
 252
 253         r = amdgpu_jpeg_suspend(adev);
 254
 255         return r;
 256 }
 257
 258 /**
 259  * jpeg_v4_0_5_resume - resume JPEG block
 260  *
 261  * @handle: amdgpu_device pointer
 262  *
 263  * Resume firmware and hw init JPEG block
 264  */
 265 static int jpeg_v4_0_5_resume(void *handle)
 266 {
 267         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 268         int r;
 269
 270         r = amdgpu_jpeg_resume(adev);
 271         if (r)
 272                 return r;
 273
 274         r = jpeg_v4_0_5_hw_init(adev);
 275
 276         return r;
 277 }
 278
 279 static void jpeg_v4_0_5_disable_clock_gating(struct amdgpu_device *adev, int inst)
 280 {
 281         uint32_t data = 0;
 282
 283         data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL);
 284         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 285                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 286                 data &= (~JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK);
 287         } else {
 288                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 289         }
 290
 291         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 292         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 293         WREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL, data);
 294
 295         data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE);
 296         data &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
 297                 | JPEG_CGC_GATE__JPEG2_DEC_MASK
 298                 | JPEG_CGC_GATE__JMCIF_MASK
 299                 | JPEG_CGC_GATE__JRBBM_MASK);
 300         WREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE, data);
 301 }
 302
 303 static void jpeg_v4_0_5_enable_clock_gating(struct amdgpu_device *adev, int inst)
 304 {
 305         uint32_t data = 0;
 306
 307         data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL);
 308         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 309                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 310                 data |= JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK;
 311         } else {
 312                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 313         }
 314
 315         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 316         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 317         WREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL, data);
 318
 319         data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE);
 320         data |= (JPEG_CGC_GATE__JPEG_DEC_MASK
 321                 |JPEG_CGC_GATE__JPEG2_DEC_MASK
 322                 |JPEG_CGC_GATE__JMCIF_MASK
 323                 |JPEG_CGC_GATE__JRBBM_MASK);
 324         WREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE, data);
 325 }
 326
 327 static void jpeg_engine_4_0_5_dpg_clock_gating_mode(struct amdgpu_device *adev,
 328                         int inst_idx, uint8_t indirect)
 329 {
 330         uint32_t data = 0;
 331
 332         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG)
 333                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 334         else
 335                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 336
 337         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 338         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 339         WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_CGC_CTRL_INTERNAL_OFFSET, data, indirect);
 340
 341         data = 0;
 342         WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_CGC_GATE_INTERNAL_OFFSET,
 343                                 data, indirect);
 344 }
 345
 346 static int jpeg_v4_0_5_disable_static_power_gating(struct amdgpu_device *adev, int inst)
 347 {
 348         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
 349                 WREG32(SOC15_REG_OFFSET(JPEG, inst, regUVD_IPX_DLDO_CONFIG),
 350                         1 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
 351                 SOC15_WAIT_ON_RREG(JPEG, inst, regUVD_IPX_DLDO_STATUS,
 352                         0, UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
 353         }
 354
 355         /* disable anti hang mechanism */
 356         WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS), 0,
 357                 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 358
 359         /* keep the JPEG in static PG mode */
 360         WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS), 0,
 361                 ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK);
 362
 363         return 0;
 364 }
 365
 366 static int jpeg_v4_0_5_enable_static_power_gating(struct amdgpu_device *adev, int inst)
 367 {
 368         /* enable anti hang mechanism */
 369         WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS),
 370                 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
 371                 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 372
 373         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
 374                 WREG32(SOC15_REG_OFFSET(JPEG, inst, regUVD_IPX_DLDO_CONFIG),
 375                         2 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
 376                 SOC15_WAIT_ON_RREG(JPEG, inst, regUVD_IPX_DLDO_STATUS,
 377                         1 << UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS__SHIFT,
 378                         UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
 379         }
 380
 381         return 0;
 382 }
 383
 384 /**
 385  * jpeg_v4_0_5_start_dpg_mode - Jpeg start with dpg mode
 386  *
 387  * @adev: amdgpu_device pointer
 388  * @inst_idx: instance number index
 389  * @indirect: indirectly write sram
 390  *
 391  * Start JPEG block with dpg mode
 392  */
 393 static void jpeg_v4_0_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 394 {
 395         struct amdgpu_ring *ring = adev->jpeg.inst[inst_idx].ring_dec;
 396         uint32_t reg_data = 0;
 397
 398         /* enable anti hang mechanism */
 399         reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
 400         reg_data &= ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK;
 401         reg_data |=  0x1;
 402         WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
 403
 404         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
 405                 WREG32(SOC15_REG_OFFSET(JPEG, inst_idx, regUVD_IPX_DLDO_CONFIG),
 406                         2 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
 407                 SOC15_WAIT_ON_RREG(JPEG, inst_idx, regUVD_IPX_DLDO_STATUS,
 408                         1 << UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS__SHIFT,
 409                         UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
 410         }
 411
 412         reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
 413         reg_data |= UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK;
 414         WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
 415
 416         if (indirect)
 417                 adev->jpeg.inst[inst_idx].dpg_sram_curr_addr =
 418                                         (uint32_t *)adev->jpeg.inst[inst_idx].dpg_sram_cpu_addr;
 419
 420         jpeg_engine_4_0_5_dpg_clock_gating_mode(adev, inst_idx, indirect);
 421
 422         /* MJPEG global tiling registers */
 423         WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_DEC_GFX10_ADDR_CONFIG_INTERNAL_OFFSET,
 424         adev->gfx.config.gb_addr_config, indirect);
 425         /* enable System Interrupt for JRBC */
 426         WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_SYS_INT_EN_INTERNAL_OFFSET,
 427         JPEG_SYS_INT_EN__DJRBC_MASK, indirect);
 428
 429         /* add nop to workaround PSP size check */
 430         WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regUVD_NO_OP_INTERNAL_OFFSET, 0, indirect);
 431
 432         if (indirect)
 433                 amdgpu_jpeg_psp_update_sram(adev, inst_idx, 0);
 434
 435         WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_VMID, 0);
 436         WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
 437         WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 438                 lower_32_bits(ring->gpu_addr));
 439         WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 440                 upper_32_bits(ring->gpu_addr));
 441         WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_RPTR, 0);
 442         WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_WPTR, 0);
 443         WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_CNTL, 0x00000002L);
 444         WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
 445         ring->wptr = RREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_WPTR);
 446 }
 447
 448 /**
 449  * jpeg_v4_0_5_stop_dpg_mode - Jpeg stop with dpg mode
 450  *
 451  * @adev: amdgpu_device pointer
 452  * @inst_idx: instance number index
 453  *
 454  * Stop JPEG block with dpg mode
 455  */
 456 static void jpeg_v4_0_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
 457 {
 458         uint32_t reg_data = 0;
 459
 460         reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
 461         reg_data &= ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK;
 462         WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
 463
 464 }
 465
 466 /**
 467  * jpeg_v4_0_5_start - start JPEG block
 468  *
 469  * @adev: amdgpu_device pointer
 470  *
 471  * Setup and start the JPEG block
 472  */
 473 static int jpeg_v4_0_5_start(struct amdgpu_device *adev)
 474 {
 475         struct amdgpu_ring *ring;
 476         int r, i;
 477
 478         if (adev->pm.dpm_enabled)
 479                 amdgpu_dpm_enable_jpeg(adev, true);
 480
 481         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 482                 if (adev->jpeg.harvest_config & (1 << i))
 483                         continue;
 484
 485                 ring = adev->jpeg.inst[i].ring_dec;
 486                 /* doorbell programming is done for every playback */
 487                 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 488                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i, i);
 489
 490                 WREG32_SOC15(VCN, i, regVCN_JPEG_DB_CTRL,
 491                         ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
 492                         VCN_JPEG_DB_CTRL__EN_MASK);
 493
 494                 if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
 495                         jpeg_v4_0_5_start_dpg_mode(adev, i, adev->jpeg.indirect_sram);
 496                         continue;
 497                 }
 498
 499                 /* disable power gating */
 500                 r = jpeg_v4_0_5_disable_static_power_gating(adev, i);
 501                 if (r)
 502                         return r;
 503
 504                 /* JPEG disable CGC */
 505                 jpeg_v4_0_5_disable_clock_gating(adev, i);
 506
 507                 /* MJPEG global tiling registers */
 508                 WREG32_SOC15(JPEG, i, regJPEG_DEC_GFX10_ADDR_CONFIG,
 509                         adev->gfx.config.gb_addr_config);
 510
 511                 /* enable JMI channel */
 512                 WREG32_P(SOC15_REG_OFFSET(JPEG, i, regUVD_JMI_CNTL), 0,
 513                         ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 514
 515                 /* enable System Interrupt for JRBC */
 516                 WREG32_P(SOC15_REG_OFFSET(JPEG, i, regJPEG_SYS_INT_EN),
 517                         JPEG_SYS_INT_EN__DJRBC_MASK,
 518                         ~JPEG_SYS_INT_EN__DJRBC_MASK);
 519
 520                 WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_VMID, 0);
 521                 WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
 522                 WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 523                         lower_32_bits(ring->gpu_addr));
 524                 WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 525                         upper_32_bits(ring->gpu_addr));
 526                 WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_RPTR, 0);
 527                 WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_WPTR, 0);
 528                 WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_CNTL, 0x00000002L);
 529                 WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
 530                 ring->wptr = RREG32_SOC15(JPEG, i, regUVD_JRBC_RB_WPTR);
 531         }
 532
 533         return 0;
 534 }
 535
 536 /**
 537  * jpeg_v4_0_5_stop - stop JPEG block
 538  *
 539  * @adev: amdgpu_device pointer
 540  *
 541  * stop the JPEG block
 542  */
 543 static int jpeg_v4_0_5_stop(struct amdgpu_device *adev)
 544 {
 545         int r, i;
 546
 547         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 548                 if (adev->jpeg.harvest_config & (1 << i))
 549                         continue;
 550
 551                 if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
 552                         jpeg_v4_0_5_stop_dpg_mode(adev, i);
 553                         continue;
 554                 }
 555
 556                 /* reset JMI */
 557                 WREG32_P(SOC15_REG_OFFSET(JPEG, i, regUVD_JMI_CNTL),
 558                         UVD_JMI_CNTL__SOFT_RESET_MASK,
 559                         ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 560
 561                 jpeg_v4_0_5_enable_clock_gating(adev, i);
 562
 563                 /* enable power gating */
 564                 r = jpeg_v4_0_5_enable_static_power_gating(adev, i);
 565                 if (r)
 566                         return r;
 567         }
 568         if (adev->pm.dpm_enabled)
 569                 amdgpu_dpm_enable_jpeg(adev, false);
 570
 571         return 0;
 572 }
 573
 574 /**
 575  * jpeg_v4_0_5_dec_ring_get_rptr - get read pointer
 576  *
 577  * @ring: amdgpu_ring pointer
 578  *
 579  * Returns the current hardware read pointer
 580  */
 581 static uint64_t jpeg_v4_0_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
 582 {
 583         struct amdgpu_device *adev = ring->adev;
 584
 585         return RREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_RPTR);
 586 }
 587
 588 /**
 589  * jpeg_v4_0_5_dec_ring_get_wptr - get write pointer
 590  *
 591  * @ring: amdgpu_ring pointer
 592  *
 593  * Returns the current hardware write pointer
 594  */
 595 static uint64_t jpeg_v4_0_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
 596 {
 597         struct amdgpu_device *adev = ring->adev;
 598
 599         if (ring->use_doorbell)
 600                 return *ring->wptr_cpu_addr;
 601         else
 602                 return RREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_WPTR);
 603 }
 604
 605 /**
 606  * jpeg_v4_0_5_dec_ring_set_wptr - set write pointer
 607  *
 608  * @ring: amdgpu_ring pointer
 609  *
 610  * Commits the write pointer to the hardware
 611  */
 612 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
 613 {
 614         struct amdgpu_device *adev = ring->adev;
 615
 616         if (ring->use_doorbell) {
 617                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
 618                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 619         } else {
 620                 WREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
 621         }
 622 }
 623
 624 static bool jpeg_v4_0_5_is_idle(void *handle)
 625 {
 626         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 627         int i, ret = 1;
 628
 629         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 630                 if (adev->jpeg.harvest_config & (1 << i))
 631                         continue;
 632
 633                 ret &= (((RREG32_SOC15(JPEG, i, regUVD_JRBC_STATUS) &
 634                         UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
 635                         UVD_JRBC_STATUS__RB_JOB_DONE_MASK));
 636         }
 637         return ret;
 638 }
 639
 640 static int jpeg_v4_0_5_wait_for_idle(void *handle)
 641 {
 642         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 643         int i;
 644
 645         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 646                 if (adev->jpeg.harvest_config & (1 << i))
 647                         continue;
 648
 649                 return SOC15_WAIT_ON_RREG(JPEG, i, regUVD_JRBC_STATUS,
 650                         UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
 651                         UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 652         }
 653
 654         return 0;
 655 }
 656
 657 static int jpeg_v4_0_5_set_clockgating_state(void *handle,
 658                                           enum amd_clockgating_state state)
 659 {
 660         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 661         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
 662         int i;
 663
 664         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 665                 if (adev->jpeg.harvest_config & (1 << i))
 666                         continue;
 667
 668                 if (enable) {
 669                         if (!jpeg_v4_0_5_is_idle(handle))
 670                                 return -EBUSY;
 671
 672                         jpeg_v4_0_5_enable_clock_gating(adev, i);
 673                 } else {
 674                         jpeg_v4_0_5_disable_clock_gating(adev, i);
 675                 }
 676         }
 677
 678         return 0;
 679 }
 680
 681 static int jpeg_v4_0_5_set_powergating_state(void *handle,
 682                                           enum amd_powergating_state state)
 683 {
 684         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 685         int ret;
 686
 687         if (amdgpu_sriov_vf(adev)) {
 688                 adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
 689                 return 0;
 690         }
 691
 692         if (state == adev->jpeg.cur_state)
 693                 return 0;
 694
 695         if (state == AMD_PG_STATE_GATE)
 696                 ret = jpeg_v4_0_5_stop(adev);
 697         else
 698                 ret = jpeg_v4_0_5_start(adev);
 699
 700         if (!ret)
 701                 adev->jpeg.cur_state = state;
 702
 703         return ret;
 704 }
 705
 706 static int jpeg_v4_0_5_process_interrupt(struct amdgpu_device *adev,
 707                                       struct amdgpu_irq_src *source,
 708                                       struct amdgpu_iv_entry *entry)
 709 {
 710         uint32_t ip_instance;
 711
 712         DRM_DEBUG("IH: JPEG TRAP\n");
 713
 714         switch (entry->client_id) {
 715         case SOC15_IH_CLIENTID_VCN:
 716                 ip_instance = 0;
 717                 break;
 718         case SOC15_IH_CLIENTID_VCN1:
 719                 ip_instance = 1;
 720                 break;
 721         default:
 722                 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
 723                 return 0;
 724         }
 725
 726         switch (entry->src_id) {
 727         case VCN_4_0__SRCID__JPEG_DECODE:
 728                 amdgpu_fence_process(adev->jpeg.inst[ip_instance].ring_dec);
 729                 break;
 730         case VCN_4_0__SRCID_DJPEG0_POISON:
 731         case VCN_4_0__SRCID_EJPEG0_POISON:
 732                 amdgpu_jpeg_process_poison_irq(adev, source, entry);
 733                 break;
 734         default:
 735                 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
 736                           entry->src_id, entry->src_data[0]);
 737                 break;
 738         }
 739
 740         return 0;
 741 }
 742
 743 static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
 744         .name = "jpeg_v4_0_5",
 745         .early_init = jpeg_v4_0_5_early_init,
 746         .late_init = NULL,
 747         .sw_init = jpeg_v4_0_5_sw_init,
 748         .sw_fini = jpeg_v4_0_5_sw_fini,
 749         .hw_init = jpeg_v4_0_5_hw_init,
 750         .hw_fini = jpeg_v4_0_5_hw_fini,
 751         .suspend = jpeg_v4_0_5_suspend,
 752         .resume = jpeg_v4_0_5_resume,
 753         .is_idle = jpeg_v4_0_5_is_idle,
 754         .wait_for_idle = jpeg_v4_0_5_wait_for_idle,
 755         .check_soft_reset = NULL,
 756         .pre_soft_reset = NULL,
 757         .soft_reset = NULL,
 758         .post_soft_reset = NULL,
 759         .set_clockgating_state = jpeg_v4_0_5_set_clockgating_state,
 760         .set_powergating_state = jpeg_v4_0_5_set_powergating_state,
 761         .dump_ip_state = NULL,
 762         .print_ip_state = NULL,
 763 };
 764
 765 static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = {
 766         .type = AMDGPU_RING_TYPE_VCN_JPEG,
 767         .align_mask = 0xf,
 768         .get_rptr = jpeg_v4_0_5_dec_ring_get_rptr,
 769         .get_wptr = jpeg_v4_0_5_dec_ring_get_wptr,
 770         .set_wptr = jpeg_v4_0_5_dec_ring_set_wptr,
 771         .emit_frame_size =
 772                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
 773                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
 774                 8 + /* jpeg_v4_0_5_dec_ring_emit_vm_flush */
 775                 18 + 18 + /* jpeg_v4_0_5_dec_ring_emit_fence x2 vm fence */
 776                 8 + 16,
 777         .emit_ib_size = 22, /* jpeg_v4_0_5_dec_ring_emit_ib */
 778         .emit_ib = jpeg_v2_0_dec_ring_emit_ib,
 779         .emit_fence = jpeg_v2_0_dec_ring_emit_fence,
 780         .emit_vm_flush = jpeg_v2_0_dec_ring_emit_vm_flush,
 781         .test_ring = amdgpu_jpeg_dec_ring_test_ring,
 782         .test_ib = amdgpu_jpeg_dec_ring_test_ib,
 783         .insert_nop = jpeg_v2_0_dec_ring_nop,
 784         .insert_start = jpeg_v2_0_dec_ring_insert_start,
 785         .insert_end = jpeg_v2_0_dec_ring_insert_end,
 786         .pad_ib = amdgpu_ring_generic_pad_ib,
 787         .begin_use = amdgpu_jpeg_ring_begin_use,
 788         .end_use = amdgpu_jpeg_ring_end_use,
 789         .emit_wreg = jpeg_v2_0_dec_ring_emit_wreg,
 790         .emit_reg_wait = jpeg_v2_0_dec_ring_emit_reg_wait,
 791         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 792 };
 793
 794 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev)
 795 {
 796         int i;
 797
 798         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 799                 if (adev->jpeg.harvest_config & (1 << i))
 800                         continue;
 801
 802                 adev->jpeg.inst[i].ring_dec->funcs = &jpeg_v4_0_5_dec_ring_vm_funcs;
 803                 adev->jpeg.inst[i].ring_dec->me = i;
 804         }
 805 }
 806
 807 static const struct amdgpu_irq_src_funcs jpeg_v4_0_5_irq_funcs = {
 808         .process = jpeg_v4_0_5_process_interrupt,
 809 };
 810
 811 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev)
 812 {
 813         int i;
 814
 815         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 816                 if (adev->jpeg.harvest_config & (1 << i))
 817                         continue;
 818
 819                 adev->jpeg.inst[i].irq.num_types = 1;
 820                 adev->jpeg.inst[i].irq.funcs = &jpeg_v4_0_5_irq_funcs;
 821         }
 822 }
 823
 824 const struct amdgpu_ip_block_version jpeg_v4_0_5_ip_block = {
 825         .type = AMD_IP_BLOCK_TYPE_JPEG,
 826         .major = 4,
 827         .minor = 0,
 828         .rev = 5,
 829         .funcs = &jpeg_v4_0_5_ip_funcs,
 830 };
 831