drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c

   1 /*
   2  * Copyright 2022 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "amdgpu.h"
  25 #include "amdgpu_jpeg.h"
  26 #include "soc15.h"
  27 #include "soc15d.h"
  28 #include "jpeg_v2_0.h"
  29 #include "jpeg_v4_0_3.h"
  30 #include "mmsch_v4_0_3.h"
  31
  32 #include "vcn/vcn_4_0_3_offset.h"
  33 #include "vcn/vcn_4_0_3_sh_mask.h"
  34 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  35
  36 #define NORMALIZE_JPEG_REG_OFFSET(offset) \
  37                 (offset & 0x1FFFF)
  38
  39 enum jpeg_engin_status {
  40         UVD_PGFSM_STATUS__UVDJ_PWR_ON  = 0,
  41         UVD_PGFSM_STATUS__UVDJ_PWR_OFF = 2,
  42 };
  43
  44 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev);
  45 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev);
  46 static int jpeg_v4_0_3_set_powergating_state(void *handle,
  47                                 enum amd_powergating_state state);
  48 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev);
  49 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring);
  50
  51 static int amdgpu_ih_srcid_jpeg[] = {
  52         VCN_4_0__SRCID__JPEG_DECODE,
  53         VCN_4_0__SRCID__JPEG1_DECODE,
  54         VCN_4_0__SRCID__JPEG2_DECODE,
  55         VCN_4_0__SRCID__JPEG3_DECODE,
  56         VCN_4_0__SRCID__JPEG4_DECODE,
  57         VCN_4_0__SRCID__JPEG5_DECODE,
  58         VCN_4_0__SRCID__JPEG6_DECODE,
  59         VCN_4_0__SRCID__JPEG7_DECODE
  60 };
  61
  62 static inline bool jpeg_v4_0_3_normalizn_reqd(struct amdgpu_device *adev)
  63 {
  64         return amdgpu_sriov_vf(adev) ||
  65                (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4));
  66 }
  67
  68 /**
  69  * jpeg_v4_0_3_early_init - set function pointers
  70  *
  71  * @handle: amdgpu_device pointer
  72  *
  73  * Set ring and irq function pointers
  74  */
  75 static int jpeg_v4_0_3_early_init(void *handle)
  76 {
  77         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  78
  79         adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
  80
  81         jpeg_v4_0_3_set_dec_ring_funcs(adev);
  82         jpeg_v4_0_3_set_irq_funcs(adev);
  83         jpeg_v4_0_3_set_ras_funcs(adev);
  84
  85         return 0;
  86 }
  87
  88 /**
  89  * jpeg_v4_0_3_sw_init - sw init for JPEG block
  90  *
  91  * @handle: amdgpu_device pointer
  92  *
  93  * Load firmware and sw initialization
  94  */
  95 static int jpeg_v4_0_3_sw_init(void *handle)
  96 {
  97         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  98         struct amdgpu_ring *ring;
  99         int i, j, r, jpeg_inst;
 100
 101         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 102                 /* JPEG TRAP */
 103                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 104                                 amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
 105                 if (r)
 106                         return r;
 107         }
 108
 109         r = amdgpu_jpeg_sw_init(adev);
 110         if (r)
 111                 return r;
 112
 113         r = amdgpu_jpeg_resume(adev);
 114         if (r)
 115                 return r;
 116
 117         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 118                 jpeg_inst = GET_INST(JPEG, i);
 119
 120                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 121                         ring = &adev->jpeg.inst[i].ring_dec[j];
 122                         ring->use_doorbell = true;
 123                         ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
 124                         if (!amdgpu_sriov_vf(adev)) {
 125                                 ring->doorbell_index =
 126                                         (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 127                                         1 + j + 9 * jpeg_inst;
 128                         } else {
 129                                 if (j < 4)
 130                                         ring->doorbell_index =
 131                                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 132                                                 4 + j + 32 * jpeg_inst;
 133                                 else
 134                                         ring->doorbell_index =
 135                                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 136                                                 8 + j + 32 * jpeg_inst;
 137                         }
 138                         sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
 139                         r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 140                                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
 141                         if (r)
 142                                 return r;
 143
 144                         adev->jpeg.internal.jpeg_pitch[j] =
 145                                 regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET;
 146                         adev->jpeg.inst[i].external.jpeg_pitch[j] =
 147                                 SOC15_REG_OFFSET1(
 148                                         JPEG, jpeg_inst,
 149                                         regUVD_JRBC0_UVD_JRBC_SCRATCH0,
 150                                         (j ? (0x40 * j - 0xc80) : 0));
 151                 }
 152         }
 153
 154         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
 155                 r = amdgpu_jpeg_ras_sw_init(adev);
 156                 if (r) {
 157                         dev_err(adev->dev, "Failed to initialize jpeg ras block!\n");
 158                         return r;
 159                 }
 160         }
 161
 162         return 0;
 163 }
 164
 165 /**
 166  * jpeg_v4_0_3_sw_fini - sw fini for JPEG block
 167  *
 168  * @handle: amdgpu_device pointer
 169  *
 170  * JPEG suspend and free up sw allocation
 171  */
 172 static int jpeg_v4_0_3_sw_fini(void *handle)
 173 {
 174         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 175         int r;
 176
 177         r = amdgpu_jpeg_suspend(adev);
 178         if (r)
 179                 return r;
 180
 181         r = amdgpu_jpeg_sw_fini(adev);
 182
 183         return r;
 184 }
 185
 186 static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 187 {
 188         struct amdgpu_ring *ring;
 189         uint64_t ctx_addr;
 190         uint32_t param, resp, expected;
 191         uint32_t tmp, timeout;
 192
 193         struct amdgpu_mm_table *table = &adev->virt.mm_table;
 194         uint32_t *table_loc;
 195         uint32_t table_size;
 196         uint32_t size, size_dw, item_offset;
 197         uint32_t init_status;
 198         int i, j, jpeg_inst;
 199
 200         struct mmsch_v4_0_cmd_direct_write
 201                 direct_wt = { {0} };
 202         struct mmsch_v4_0_cmd_end end = { {0} };
 203         struct mmsch_v4_0_3_init_header header;
 204
 205         direct_wt.cmd_header.command_type =
 206                 MMSCH_COMMAND__DIRECT_REG_WRITE;
 207         end.cmd_header.command_type =
 208                 MMSCH_COMMAND__END;
 209
 210         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
 211                 jpeg_inst = GET_INST(JPEG, i);
 212
 213                 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header));
 214                 header.version = MMSCH_VERSION;
 215                 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2;
 216
 217                 table_loc = (uint32_t *)table->cpu_addr;
 218                 table_loc += header.total_size;
 219
 220                 item_offset = header.total_size;
 221
 222                 for (j = 0; j < adev->jpeg.num_jpeg_rings; j++) {
 223                         ring = &adev->jpeg.inst[i].ring_dec[j];
 224                         table_size = 0;
 225
 226                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW);
 227                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, lower_32_bits(ring->gpu_addr));
 228                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH);
 229                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, upper_32_bits(ring->gpu_addr));
 230                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_SIZE);
 231                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
 232
 233                         if (j <= 3) {
 234                                 header.mjpegdec0[j].table_offset = item_offset;
 235                                 header.mjpegdec0[j].init_status = 0;
 236                                 header.mjpegdec0[j].table_size = table_size;
 237                         } else {
 238                                 header.mjpegdec1[j - 4].table_offset = item_offset;
 239                                 header.mjpegdec1[j - 4].init_status = 0;
 240                                 header.mjpegdec1[j - 4].table_size = table_size;
 241                         }
 242                         header.total_size += table_size;
 243                         item_offset += table_size;
 244                 }
 245
 246                 MMSCH_V4_0_INSERT_END();
 247
 248                 /* send init table to MMSCH */
 249                 size = sizeof(struct mmsch_v4_0_3_init_header);
 250                 table_loc = (uint32_t *)table->cpu_addr;
 251                 memcpy((void *)table_loc, &header, size);
 252
 253                 ctx_addr = table->gpu_addr;
 254                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
 255                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
 256
 257                 tmp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID);
 258                 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
 259                 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
 260                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID, tmp);
 261
 262                 size = header.total_size;
 263                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_SIZE, size);
 264
 265                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP, 0);
 266
 267                 param = 0x00000001;
 268                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_HOST, param);
 269                 tmp = 0;
 270                 timeout = 1000;
 271                 resp = 0;
 272                 expected = MMSCH_VF_MAILBOX_RESP__OK;
 273                 init_status =
 274                         ((struct mmsch_v4_0_3_init_header *)(table_loc))->mjpegdec0[i].init_status;
 275                 while (resp != expected) {
 276                         resp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP);
 277
 278                         if (resp != 0)
 279                                 break;
 280                         udelay(10);
 281                         tmp = tmp + 10;
 282                         if (tmp >= timeout) {
 283                                 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
 284                                         " waiting for regMMSCH_VF_MAILBOX_RESP "\
 285                                         "(expected=0x%08x, readback=0x%08x)\n",
 286                                         tmp, expected, resp);
 287                                 return -EBUSY;
 288                         }
 289                 }
 290                 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE &&
 291                                 init_status != MMSCH_VF_ENGINE_STATUS__PASS)
 292                         DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n",
 293                                         resp, init_status);
 294
 295         }
 296         return 0;
 297 }
 298
 299 /**
 300  * jpeg_v4_0_3_hw_init - start and test JPEG block
 301  *
 302  * @handle: amdgpu_device pointer
 303  *
 304  */
 305 static int jpeg_v4_0_3_hw_init(void *handle)
 306 {
 307         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 308         struct amdgpu_ring *ring;
 309         int i, j, r, jpeg_inst;
 310
 311         if (amdgpu_sriov_vf(adev)) {
 312                 r = jpeg_v4_0_3_start_sriov(adev);
 313                 if (r)
 314                         return r;
 315
 316                 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 317                         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 318                                 ring = &adev->jpeg.inst[i].ring_dec[j];
 319                                 ring->wptr = 0;
 320                                 ring->wptr_old = 0;
 321                                 jpeg_v4_0_3_dec_ring_set_wptr(ring);
 322                                 ring->sched.ready = true;
 323                         }
 324                 }
 325         } else {
 326                 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 327                         jpeg_inst = GET_INST(JPEG, i);
 328
 329                         ring = adev->jpeg.inst[i].ring_dec;
 330
 331                         if (ring->use_doorbell)
 332                                 adev->nbio.funcs->vcn_doorbell_range(
 333                                         adev, ring->use_doorbell,
 334                                         (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 335                                                 9 * jpeg_inst,
 336                                         adev->jpeg.inst[i].aid_id);
 337
 338                         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 339                                 ring = &adev->jpeg.inst[i].ring_dec[j];
 340                                 if (ring->use_doorbell)
 341                                         WREG32_SOC15_OFFSET(
 342                                                 VCN, GET_INST(VCN, i),
 343                                                 regVCN_JPEG_DB_CTRL,
 344                                                 (ring->pipe ? (ring->pipe - 0x15) : 0),
 345                                                 ring->doorbell_index
 346                                                         << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
 347                                                         VCN_JPEG_DB_CTRL__EN_MASK);
 348                                 r = amdgpu_ring_test_helper(ring);
 349                                 if (r)
 350                                         return r;
 351                         }
 352                 }
 353         }
 354
 355         return 0;
 356 }
 357
 358 /**
 359  * jpeg_v4_0_3_hw_fini - stop the hardware block
 360  *
 361  * @handle: amdgpu_device pointer
 362  *
 363  * Stop the JPEG block, mark ring as not ready any more
 364  */
 365 static int jpeg_v4_0_3_hw_fini(void *handle)
 366 {
 367         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 368         int ret = 0;
 369
 370         cancel_delayed_work_sync(&adev->jpeg.idle_work);
 371
 372         if (!amdgpu_sriov_vf(adev)) {
 373                 if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
 374                         ret = jpeg_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE);
 375         }
 376
 377         return ret;
 378 }
 379
 380 /**
 381  * jpeg_v4_0_3_suspend - suspend JPEG block
 382  *
 383  * @handle: amdgpu_device pointer
 384  *
 385  * HW fini and suspend JPEG block
 386  */
 387 static int jpeg_v4_0_3_suspend(void *handle)
 388 {
 389         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 390         int r;
 391
 392         r = jpeg_v4_0_3_hw_fini(adev);
 393         if (r)
 394                 return r;
 395
 396         r = amdgpu_jpeg_suspend(adev);
 397
 398         return r;
 399 }
 400
 401 /**
 402  * jpeg_v4_0_3_resume - resume JPEG block
 403  *
 404  * @handle: amdgpu_device pointer
 405  *
 406  * Resume firmware and hw init JPEG block
 407  */
 408 static int jpeg_v4_0_3_resume(void *handle)
 409 {
 410         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 411         int r;
 412
 413         r = amdgpu_jpeg_resume(adev);
 414         if (r)
 415                 return r;
 416
 417         r = jpeg_v4_0_3_hw_init(adev);
 418
 419         return r;
 420 }
 421
 422 static void jpeg_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 423 {
 424         int i, jpeg_inst;
 425         uint32_t data;
 426
 427         jpeg_inst = GET_INST(JPEG, inst_idx);
 428         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 429         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 430                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 431                 data &= (~(JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1));
 432         } else {
 433                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 434         }
 435
 436         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 437         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 438         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 439
 440         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 441         data &= ~(JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 442         for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 443                 data &= ~(JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 444         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 445 }
 446
 447 static void jpeg_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 448 {
 449         int i, jpeg_inst;
 450         uint32_t data;
 451
 452         jpeg_inst = GET_INST(JPEG, inst_idx);
 453         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 454         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 455                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 456                 data |= (JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1);
 457         } else {
 458                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 459         }
 460
 461         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 462         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 463         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 464
 465         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 466         data |= (JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 467         for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 468                 data |= (JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 469         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 470 }
 471
 472 /**
 473  * jpeg_v4_0_3_start - start JPEG block
 474  *
 475  * @adev: amdgpu_device pointer
 476  *
 477  * Setup and start the JPEG block
 478  */
 479 static int jpeg_v4_0_3_start(struct amdgpu_device *adev)
 480 {
 481         struct amdgpu_ring *ring;
 482         int i, j, jpeg_inst;
 483
 484         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 485                 jpeg_inst = GET_INST(JPEG, i);
 486
 487                 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
 488                              1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
 489                 SOC15_WAIT_ON_RREG(
 490                         JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
 491                         UVD_PGFSM_STATUS__UVDJ_PWR_ON
 492                                 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
 493                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 494
 495                 /* disable anti hang mechanism */
 496                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 497                                           regUVD_JPEG_POWER_STATUS),
 498                          0, ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 499
 500                 /* JPEG disable CGC */
 501                 jpeg_v4_0_3_disable_clock_gating(adev, i);
 502
 503                 /* MJPEG global tiling registers */
 504                 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX8_ADDR_CONFIG,
 505                              adev->gfx.config.gb_addr_config);
 506                 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX10_ADDR_CONFIG,
 507                              adev->gfx.config.gb_addr_config);
 508
 509                 /* enable JMI channel */
 510                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL), 0,
 511                          ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 512
 513                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 514                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 515
 516                         ring = &adev->jpeg.inst[i].ring_dec[j];
 517
 518                         /* enable System Interrupt for JRBC */
 519                         WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 520                                                   regJPEG_SYS_INT_EN),
 521                                  JPEG_SYS_INT_EN__DJRBC0_MASK << j,
 522                                  ~(JPEG_SYS_INT_EN__DJRBC0_MASK << j));
 523
 524                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 525                                             regUVD_JMI0_UVD_LMI_JRBC_RB_VMID,
 526                                             reg_offset, 0);
 527                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 528                                             regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 529                                             reg_offset,
 530                                             (0x00000001L | 0x00000002L));
 531                         WREG32_SOC15_OFFSET(
 532                                 JPEG, jpeg_inst,
 533                                 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 534                                 reg_offset, lower_32_bits(ring->gpu_addr));
 535                         WREG32_SOC15_OFFSET(
 536                                 JPEG, jpeg_inst,
 537                                 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 538                                 reg_offset, upper_32_bits(ring->gpu_addr));
 539                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 540                                             regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 541                                             reg_offset, 0);
 542                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 543                                             regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 544                                             reg_offset, 0);
 545                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 546                                             regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 547                                             reg_offset, 0x00000002L);
 548                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 549                                             regUVD_JRBC0_UVD_JRBC_RB_SIZE,
 550                                             reg_offset, ring->ring_size / 4);
 551                         ring->wptr = RREG32_SOC15_OFFSET(
 552                                 JPEG, jpeg_inst, regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 553                                 reg_offset);
 554                 }
 555         }
 556
 557         return 0;
 558 }
 559
 560 /**
 561  * jpeg_v4_0_3_stop - stop JPEG block
 562  *
 563  * @adev: amdgpu_device pointer
 564  *
 565  * stop the JPEG block
 566  */
 567 static int jpeg_v4_0_3_stop(struct amdgpu_device *adev)
 568 {
 569         int i, jpeg_inst;
 570
 571         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 572                 jpeg_inst = GET_INST(JPEG, i);
 573                 /* reset JMI */
 574                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL),
 575                          UVD_JMI_CNTL__SOFT_RESET_MASK,
 576                          ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 577
 578                 jpeg_v4_0_3_enable_clock_gating(adev, i);
 579
 580                 /* enable anti hang mechanism */
 581                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 582                                           regUVD_JPEG_POWER_STATUS),
 583                          UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
 584                          ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 585
 586                 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
 587                              2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
 588                 SOC15_WAIT_ON_RREG(
 589                         JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
 590                         UVD_PGFSM_STATUS__UVDJ_PWR_OFF
 591                                 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
 592                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 593         }
 594
 595         return 0;
 596 }
 597
 598 /**
 599  * jpeg_v4_0_3_dec_ring_get_rptr - get read pointer
 600  *
 601  * @ring: amdgpu_ring pointer
 602  *
 603  * Returns the current hardware read pointer
 604  */
 605 static uint64_t jpeg_v4_0_3_dec_ring_get_rptr(struct amdgpu_ring *ring)
 606 {
 607         struct amdgpu_device *adev = ring->adev;
 608
 609         return RREG32_SOC15_OFFSET(
 610                 JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 611                 ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
 612 }
 613
 614 /**
 615  * jpeg_v4_0_3_dec_ring_get_wptr - get write pointer
 616  *
 617  * @ring: amdgpu_ring pointer
 618  *
 619  * Returns the current hardware write pointer
 620  */
 621 static uint64_t jpeg_v4_0_3_dec_ring_get_wptr(struct amdgpu_ring *ring)
 622 {
 623         struct amdgpu_device *adev = ring->adev;
 624
 625         if (ring->use_doorbell)
 626                 return adev->wb.wb[ring->wptr_offs];
 627         else
 628                 return RREG32_SOC15_OFFSET(
 629                         JPEG, GET_INST(JPEG, ring->me),
 630                         regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 631                         ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
 632 }
 633
 634 static void jpeg_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 635 {
 636         /* JPEG engine access for HDP flush doesn't work when RRMT is enabled.
 637          * This is a workaround to avoid any HDP flush through JPEG ring.
 638          */
 639 }
 640
 641 /**
 642  * jpeg_v4_0_3_dec_ring_set_wptr - set write pointer
 643  *
 644  * @ring: amdgpu_ring pointer
 645  *
 646  * Commits the write pointer to the hardware
 647  */
 648 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring)
 649 {
 650         struct amdgpu_device *adev = ring->adev;
 651
 652         if (ring->use_doorbell) {
 653                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
 654                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 655         } else {
 656                 WREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me),
 657                                     regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 658                                     (ring->pipe ? (0x40 * ring->pipe - 0xc80) :
 659                                                   0),
 660                                     lower_32_bits(ring->wptr));
 661         }
 662 }
 663
 664 /**
 665  * jpeg_v4_0_3_dec_ring_insert_start - insert a start command
 666  *
 667  * @ring: amdgpu_ring pointer
 668  *
 669  * Write a start command to the ring.
 670  */
 671 void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring)
 672 {
 673         if (!amdgpu_sriov_vf(ring->adev)) {
 674                 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 675                         0, 0, PACKETJ_TYPE0));
 676                 amdgpu_ring_write(ring, 0x62a04); /* PCTL0_MMHUB_DEEPSLEEP_IB */
 677         }
 678
 679         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 680                 0, 0, PACKETJ_TYPE0));
 681         amdgpu_ring_write(ring, 0x80004000);
 682 }
 683
 684 /**
 685  * jpeg_v4_0_3_dec_ring_insert_end - insert a end command
 686  *
 687  * @ring: amdgpu_ring pointer
 688  *
 689  * Write a end command to the ring.
 690  */
 691 void jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring *ring)
 692 {
 693         if (!amdgpu_sriov_vf(ring->adev)) {
 694                 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 695                         0, 0, PACKETJ_TYPE0));
 696                 amdgpu_ring_write(ring, 0x62a04);
 697         }
 698
 699         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 700                 0, 0, PACKETJ_TYPE0));
 701         amdgpu_ring_write(ring, 0x00004000);
 702 }
 703
 704 /**
 705  * jpeg_v4_0_3_dec_ring_emit_fence - emit an fence & trap command
 706  *
 707  * @ring: amdgpu_ring pointer
 708  * @addr: address
 709  * @seq: sequence number
 710  * @flags: fence related flags
 711  *
 712  * Write a fence and a trap command to the ring.
 713  */
 714 void jpeg_v4_0_3_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 715                                 unsigned int flags)
 716 {
 717         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 718
 719         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
 720                 0, 0, PACKETJ_TYPE0));
 721         amdgpu_ring_write(ring, seq);
 722
 723         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
 724                 0, 0, PACKETJ_TYPE0));
 725         amdgpu_ring_write(ring, seq);
 726
 727         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
 728                 0, 0, PACKETJ_TYPE0));
 729         amdgpu_ring_write(ring, lower_32_bits(addr));
 730
 731         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 732                 0, 0, PACKETJ_TYPE0));
 733         amdgpu_ring_write(ring, upper_32_bits(addr));
 734
 735         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 736                 0, 0, PACKETJ_TYPE0));
 737         amdgpu_ring_write(ring, 0x8);
 738
 739         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 740                 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
 741         amdgpu_ring_write(ring, 0);
 742
 743         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 744         amdgpu_ring_write(ring, 0);
 745
 746         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 747                 0, 0, PACKETJ_TYPE0));
 748         amdgpu_ring_write(ring, 0x3fbc);
 749
 750         amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 751                 0, 0, PACKETJ_TYPE0));
 752         amdgpu_ring_write(ring, 0x1);
 753
 754         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 755         amdgpu_ring_write(ring, 0);
 756
 757         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
 758         amdgpu_ring_write(ring, 0);
 759 }
 760
 761 /**
 762  * jpeg_v4_0_3_dec_ring_emit_ib - execute indirect buffer
 763  *
 764  * @ring: amdgpu_ring pointer
 765  * @job: job to retrieve vmid from
 766  * @ib: indirect buffer to execute
 767  * @flags: unused
 768  *
 769  * Write ring commands to execute the indirect buffer.
 770  */
 771 void jpeg_v4_0_3_dec_ring_emit_ib(struct amdgpu_ring *ring,
 772                                 struct amdgpu_job *job,
 773                                 struct amdgpu_ib *ib,
 774                                 uint32_t flags)
 775 {
 776         unsigned int vmid = AMDGPU_JOB_GET_VMID(job);
 777
 778         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
 779                 0, 0, PACKETJ_TYPE0));
 780
 781         if (ring->funcs->parse_cs)
 782                 amdgpu_ring_write(ring, 0);
 783         else
 784                 amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 785
 786         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
 787                 0, 0, PACKETJ_TYPE0));
 788         amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 789
 790         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
 791                 0, 0, PACKETJ_TYPE0));
 792         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 793
 794         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 795                 0, 0, PACKETJ_TYPE0));
 796         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 797
 798         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
 799                 0, 0, PACKETJ_TYPE0));
 800         amdgpu_ring_write(ring, ib->length_dw);
 801
 802         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
 803                 0, 0, PACKETJ_TYPE0));
 804         amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
 805
 806         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 807                 0, 0, PACKETJ_TYPE0));
 808         amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
 809
 810         amdgpu_ring_write(ring, PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
 811         amdgpu_ring_write(ring, 0);
 812
 813         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 814                 0, 0, PACKETJ_TYPE0));
 815         amdgpu_ring_write(ring, 0x01400200);
 816
 817         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 818                 0, 0, PACKETJ_TYPE0));
 819         amdgpu_ring_write(ring, 0x2);
 820
 821         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_STATUS_INTERNAL_OFFSET,
 822                 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
 823         amdgpu_ring_write(ring, 0x2);
 824 }
 825
 826 void jpeg_v4_0_3_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 827                                 uint32_t val, uint32_t mask)
 828 {
 829         uint32_t reg_offset;
 830
 831         /* Use normalized offsets if required */
 832         if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 833                 reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 834
 835         reg_offset = (reg << 2);
 836
 837         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 838                 0, 0, PACKETJ_TYPE0));
 839         amdgpu_ring_write(ring, 0x01400200);
 840
 841         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 842                 0, 0, PACKETJ_TYPE0));
 843         amdgpu_ring_write(ring, val);
 844
 845         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 846                 0, 0, PACKETJ_TYPE0));
 847         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 848                 amdgpu_ring_write(ring, 0);
 849                 amdgpu_ring_write(ring,
 850                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
 851         } else {
 852                 amdgpu_ring_write(ring, reg_offset);
 853                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 854                         0, 0, PACKETJ_TYPE3));
 855         }
 856         amdgpu_ring_write(ring, mask);
 857 }
 858
 859 void jpeg_v4_0_3_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 860                                 unsigned int vmid, uint64_t pd_addr)
 861 {
 862         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 863         uint32_t data0, data1, mask;
 864
 865         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 866
 867         /* wait for register write */
 868         data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
 869         data1 = lower_32_bits(pd_addr);
 870         mask = 0xffffffff;
 871         jpeg_v4_0_3_dec_ring_emit_reg_wait(ring, data0, data1, mask);
 872 }
 873
 874 void jpeg_v4_0_3_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
 875 {
 876         uint32_t reg_offset;
 877
 878         /* Use normalized offsets if required */
 879         if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 880                 reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 881
 882         reg_offset = (reg << 2);
 883
 884         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 885                 0, 0, PACKETJ_TYPE0));
 886         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 887                 amdgpu_ring_write(ring, 0);
 888                 amdgpu_ring_write(ring,
 889                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
 890         } else {
 891                 amdgpu_ring_write(ring, reg_offset);
 892                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 893                         0, 0, PACKETJ_TYPE0));
 894         }
 895         amdgpu_ring_write(ring, val);
 896 }
 897
 898 void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count)
 899 {
 900         int i;
 901
 902         WARN_ON(ring->wptr % 2 || count % 2);
 903
 904         for (i = 0; i < count / 2; i++) {
 905                 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 906                 amdgpu_ring_write(ring, 0);
 907         }
 908 }
 909
 910 static bool jpeg_v4_0_3_is_idle(void *handle)
 911 {
 912         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 913         bool ret = false;
 914         int i, j;
 915
 916         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 917                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 918                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 919
 920                         ret &= ((RREG32_SOC15_OFFSET(
 921                                          JPEG, GET_INST(JPEG, i),
 922                                          regUVD_JRBC0_UVD_JRBC_STATUS,
 923                                          reg_offset) &
 924                                  UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
 925                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 926                 }
 927         }
 928
 929         return ret;
 930 }
 931
 932 static int jpeg_v4_0_3_wait_for_idle(void *handle)
 933 {
 934         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 935         int ret = 0;
 936         int i, j;
 937
 938         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 939                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 940                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 941
 942                         ret &= SOC15_WAIT_ON_RREG_OFFSET(
 943                                 JPEG, GET_INST(JPEG, i),
 944                                 regUVD_JRBC0_UVD_JRBC_STATUS, reg_offset,
 945                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
 946                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 947                 }
 948         }
 949         return ret;
 950 }
 951
 952 static int jpeg_v4_0_3_set_clockgating_state(void *handle,
 953                                           enum amd_clockgating_state state)
 954 {
 955         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 956         bool enable = state == AMD_CG_STATE_GATE;
 957         int i;
 958
 959         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 960                 if (enable) {
 961                         if (!jpeg_v4_0_3_is_idle(handle))
 962                                 return -EBUSY;
 963                         jpeg_v4_0_3_enable_clock_gating(adev, i);
 964                 } else {
 965                         jpeg_v4_0_3_disable_clock_gating(adev, i);
 966                 }
 967         }
 968         return 0;
 969 }
 970
 971 static int jpeg_v4_0_3_set_powergating_state(void *handle,
 972                                           enum amd_powergating_state state)
 973 {
 974         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 975         int ret;
 976
 977         if (amdgpu_sriov_vf(adev)) {
 978                 adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
 979                 return 0;
 980         }
 981
 982         if (state == adev->jpeg.cur_state)
 983                 return 0;
 984
 985         if (state == AMD_PG_STATE_GATE)
 986                 ret = jpeg_v4_0_3_stop(adev);
 987         else
 988                 ret = jpeg_v4_0_3_start(adev);
 989
 990         if (!ret)
 991                 adev->jpeg.cur_state = state;
 992
 993         return ret;
 994 }
 995
 996 static int jpeg_v4_0_3_set_interrupt_state(struct amdgpu_device *adev,
 997                                         struct amdgpu_irq_src *source,
 998                                         unsigned int type,
 999                                         enum amdgpu_interrupt_state state)
1000 {
1001         return 0;
1002 }
1003
1004 static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
1005                                       struct amdgpu_irq_src *source,
1006                                       struct amdgpu_iv_entry *entry)
1007 {
1008         uint32_t i, inst;
1009
1010         i = node_id_to_phys_map[entry->node_id];
1011         DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
1012
1013         for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
1014                 if (adev->jpeg.inst[inst].aid_id == i)
1015                         break;
1016
1017         if (inst >= adev->jpeg.num_jpeg_inst) {
1018                 dev_WARN_ONCE(adev->dev, 1,
1019                               "Interrupt received for unknown JPEG instance %d",
1020                               entry->node_id);
1021                 return 0;
1022         }
1023
1024         switch (entry->src_id) {
1025         case VCN_4_0__SRCID__JPEG_DECODE:
1026                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
1027                 break;
1028         case VCN_4_0__SRCID__JPEG1_DECODE:
1029                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
1030                 break;
1031         case VCN_4_0__SRCID__JPEG2_DECODE:
1032                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
1033                 break;
1034         case VCN_4_0__SRCID__JPEG3_DECODE:
1035                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
1036                 break;
1037         case VCN_4_0__SRCID__JPEG4_DECODE:
1038                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
1039                 break;
1040         case VCN_4_0__SRCID__JPEG5_DECODE:
1041                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
1042                 break;
1043         case VCN_4_0__SRCID__JPEG6_DECODE:
1044                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
1045                 break;
1046         case VCN_4_0__SRCID__JPEG7_DECODE:
1047                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
1048                 break;
1049         default:
1050                 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
1051                           entry->src_id, entry->src_data[0]);
1052                 break;
1053         }
1054
1055         return 0;
1056 }
1057
1058 static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
1059         .name = "jpeg_v4_0_3",
1060         .early_init = jpeg_v4_0_3_early_init,
1061         .late_init = NULL,
1062         .sw_init = jpeg_v4_0_3_sw_init,
1063         .sw_fini = jpeg_v4_0_3_sw_fini,
1064         .hw_init = jpeg_v4_0_3_hw_init,
1065         .hw_fini = jpeg_v4_0_3_hw_fini,
1066         .suspend = jpeg_v4_0_3_suspend,
1067         .resume = jpeg_v4_0_3_resume,
1068         .is_idle = jpeg_v4_0_3_is_idle,
1069         .wait_for_idle = jpeg_v4_0_3_wait_for_idle,
1070         .check_soft_reset = NULL,
1071         .pre_soft_reset = NULL,
1072         .soft_reset = NULL,
1073         .post_soft_reset = NULL,
1074         .set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
1075         .set_powergating_state = jpeg_v4_0_3_set_powergating_state,
1076         .dump_ip_state = NULL,
1077         .print_ip_state = NULL,
1078 };
1079
1080 static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
1081         .type = AMDGPU_RING_TYPE_VCN_JPEG,
1082         .align_mask = 0xf,
1083         .get_rptr = jpeg_v4_0_3_dec_ring_get_rptr,
1084         .get_wptr = jpeg_v4_0_3_dec_ring_get_wptr,
1085         .set_wptr = jpeg_v4_0_3_dec_ring_set_wptr,
1086         .parse_cs = jpeg_v2_dec_ring_parse_cs,
1087         .emit_frame_size =
1088                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1089                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1090                 8 + /* jpeg_v4_0_3_dec_ring_emit_vm_flush */
1091                 22 + 22 + /* jpeg_v4_0_3_dec_ring_emit_fence x2 vm fence */
1092                 8 + 16,
1093         .emit_ib_size = 22, /* jpeg_v4_0_3_dec_ring_emit_ib */
1094         .emit_ib = jpeg_v4_0_3_dec_ring_emit_ib,
1095         .emit_fence = jpeg_v4_0_3_dec_ring_emit_fence,
1096         .emit_vm_flush = jpeg_v4_0_3_dec_ring_emit_vm_flush,
1097         .emit_hdp_flush = jpeg_v4_0_3_ring_emit_hdp_flush,
1098         .test_ring = amdgpu_jpeg_dec_ring_test_ring,
1099         .test_ib = amdgpu_jpeg_dec_ring_test_ib,
1100         .insert_nop = jpeg_v4_0_3_dec_ring_nop,
1101         .insert_start = jpeg_v4_0_3_dec_ring_insert_start,
1102         .insert_end = jpeg_v4_0_3_dec_ring_insert_end,
1103         .pad_ib = amdgpu_ring_generic_pad_ib,
1104         .begin_use = amdgpu_jpeg_ring_begin_use,
1105         .end_use = amdgpu_jpeg_ring_end_use,
1106         .emit_wreg = jpeg_v4_0_3_dec_ring_emit_wreg,
1107         .emit_reg_wait = jpeg_v4_0_3_dec_ring_emit_reg_wait,
1108         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1109 };
1110
1111 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev)
1112 {
1113         int i, j, jpeg_inst;
1114
1115         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1116                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
1117                         adev->jpeg.inst[i].ring_dec[j].funcs = &jpeg_v4_0_3_dec_ring_vm_funcs;
1118                         adev->jpeg.inst[i].ring_dec[j].me = i;
1119                         adev->jpeg.inst[i].ring_dec[j].pipe = j;
1120                 }
1121                 jpeg_inst = GET_INST(JPEG, i);
1122                 adev->jpeg.inst[i].aid_id =
1123                         jpeg_inst / adev->jpeg.num_inst_per_aid;
1124         }
1125 }
1126
1127 static const struct amdgpu_irq_src_funcs jpeg_v4_0_3_irq_funcs = {
1128         .set = jpeg_v4_0_3_set_interrupt_state,
1129         .process = jpeg_v4_0_3_process_interrupt,
1130 };
1131
1132 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev)
1133 {
1134         int i;
1135
1136         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1137                 adev->jpeg.inst->irq.num_types += adev->jpeg.num_jpeg_rings;
1138         }
1139         adev->jpeg.inst->irq.funcs = &jpeg_v4_0_3_irq_funcs;
1140 }
1141
1142 const struct amdgpu_ip_block_version jpeg_v4_0_3_ip_block = {
1143         .type = AMD_IP_BLOCK_TYPE_JPEG,
1144         .major = 4,
1145         .minor = 0,
1146         .rev = 3,
1147         .funcs = &jpeg_v4_0_3_ip_funcs,
1148 };
1149
1150 static const struct amdgpu_ras_err_status_reg_entry jpeg_v4_0_3_ue_reg_list[] = {
1151         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0S, regVCN_UE_ERR_STATUS_HI_JPEG0S),
1152         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0S"},
1153         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0D, regVCN_UE_ERR_STATUS_HI_JPEG0D),
1154         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0D"},
1155         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1S, regVCN_UE_ERR_STATUS_HI_JPEG1S),
1156         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1S"},
1157         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1D, regVCN_UE_ERR_STATUS_HI_JPEG1D),
1158         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1D"},
1159         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2S, regVCN_UE_ERR_STATUS_HI_JPEG2S),
1160         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2S"},
1161         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2D, regVCN_UE_ERR_STATUS_HI_JPEG2D),
1162         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2D"},
1163         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3S, regVCN_UE_ERR_STATUS_HI_JPEG3S),
1164         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3S"},
1165         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3D, regVCN_UE_ERR_STATUS_HI_JPEG3D),
1166         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3D"},
1167         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4S, regVCN_UE_ERR_STATUS_HI_JPEG4S),
1168         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4S"},
1169         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4D, regVCN_UE_ERR_STATUS_HI_JPEG4D),
1170         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4D"},
1171         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5S, regVCN_UE_ERR_STATUS_HI_JPEG5S),
1172         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5S"},
1173         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5D, regVCN_UE_ERR_STATUS_HI_JPEG5D),
1174         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5D"},
1175         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6S, regVCN_UE_ERR_STATUS_HI_JPEG6S),
1176         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6S"},
1177         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6D, regVCN_UE_ERR_STATUS_HI_JPEG6D),
1178         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6D"},
1179         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7S, regVCN_UE_ERR_STATUS_HI_JPEG7S),
1180         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7S"},
1181         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7D, regVCN_UE_ERR_STATUS_HI_JPEG7D),
1182         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7D"},
1183 };
1184
1185 static void jpeg_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev,
1186                                                    uint32_t jpeg_inst,
1187                                                    void *ras_err_status)
1188 {
1189         struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
1190
1191         /* jpeg v4_0_3 only support uncorrectable errors */
1192         amdgpu_ras_inst_query_ras_error_count(adev,
1193                         jpeg_v4_0_3_ue_reg_list,
1194                         ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1195                         NULL, 0, GET_INST(VCN, jpeg_inst),
1196                         AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
1197                         &err_data->ue_count);
1198 }
1199
1200 static void jpeg_v4_0_3_query_ras_error_count(struct amdgpu_device *adev,
1201                                               void *ras_err_status)
1202 {
1203         uint32_t i;
1204
1205         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1206                 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1207                 return;
1208         }
1209
1210         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1211                 jpeg_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status);
1212 }
1213
1214 static void jpeg_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev,
1215                                                    uint32_t jpeg_inst)
1216 {
1217         amdgpu_ras_inst_reset_ras_error_count(adev,
1218                         jpeg_v4_0_3_ue_reg_list,
1219                         ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1220                         GET_INST(VCN, jpeg_inst));
1221 }
1222
1223 static void jpeg_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev)
1224 {
1225         uint32_t i;
1226
1227         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1228                 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1229                 return;
1230         }
1231
1232         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1233                 jpeg_v4_0_3_inst_reset_ras_error_count(adev, i);
1234 }
1235
1236 static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = {
1237         .query_ras_error_count = jpeg_v4_0_3_query_ras_error_count,
1238         .reset_ras_error_count = jpeg_v4_0_3_reset_ras_error_count,
1239 };
1240
1241 static struct amdgpu_jpeg_ras jpeg_v4_0_3_ras = {
1242         .ras_block = {
1243                 .hw_ops = &jpeg_v4_0_3_ras_hw_ops,
1244         },
1245 };
1246
1247 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev)
1248 {
1249         adev->jpeg.ras = &jpeg_v4_0_3_ras;
1250 }