drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c

   1 /*
   2  * Copyright 2022 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "amdgpu.h"
  25 #include "amdgpu_jpeg.h"
  26 #include "soc15.h"
  27 #include "soc15d.h"
  28 #include "jpeg_v2_0.h"
  29 #include "jpeg_v4_0_3.h"
  30 #include "mmsch_v4_0_3.h"
  31
  32 #include "vcn/vcn_4_0_3_offset.h"
  33 #include "vcn/vcn_4_0_3_sh_mask.h"
  34 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  35
  36 #define NORMALIZE_JPEG_REG_OFFSET(offset) \
  37                 (offset & 0x1FFFF)
  38
  39 enum jpeg_engin_status {
  40         UVD_PGFSM_STATUS__UVDJ_PWR_ON  = 0,
  41         UVD_PGFSM_STATUS__UVDJ_PWR_OFF = 2,
  42 };
  43
  44 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev);
  45 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev);
  46 static int jpeg_v4_0_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
  47                                 enum amd_powergating_state state);
  48 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev);
  49 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring);
  50
  51 static int amdgpu_ih_srcid_jpeg[] = {
  52         VCN_4_0__SRCID__JPEG_DECODE,
  53         VCN_4_0__SRCID__JPEG1_DECODE,
  54         VCN_4_0__SRCID__JPEG2_DECODE,
  55         VCN_4_0__SRCID__JPEG3_DECODE,
  56         VCN_4_0__SRCID__JPEG4_DECODE,
  57         VCN_4_0__SRCID__JPEG5_DECODE,
  58         VCN_4_0__SRCID__JPEG6_DECODE,
  59         VCN_4_0__SRCID__JPEG7_DECODE
  60 };
  61
  62 static inline bool jpeg_v4_0_3_normalizn_reqd(struct amdgpu_device *adev)
  63 {
  64         return amdgpu_sriov_vf(adev) ||
  65                (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4));
  66 }
  67
  68 /**
  69  * jpeg_v4_0_3_early_init - set function pointers
  70  *
  71  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
  72  *
  73  * Set ring and irq function pointers
  74  */
  75 static int jpeg_v4_0_3_early_init(struct amdgpu_ip_block *ip_block)
  76 {
  77         struct amdgpu_device *adev = ip_block->adev;
  78
  79         adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS_4_0_3;
  80
  81         jpeg_v4_0_3_set_dec_ring_funcs(adev);
  82         jpeg_v4_0_3_set_irq_funcs(adev);
  83         jpeg_v4_0_3_set_ras_funcs(adev);
  84
  85         return 0;
  86 }
  87
  88 /**
  89  * jpeg_v4_0_3_sw_init - sw init for JPEG block
  90  *
  91  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
  92  *
  93  * Load firmware and sw initialization
  94  */
  95 static int jpeg_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block)
  96 {
  97         struct amdgpu_device *adev = ip_block->adev;
  98         struct amdgpu_ring *ring;
  99         int i, j, r, jpeg_inst;
 100
 101         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 102                 /* JPEG TRAP */
 103                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 104                                 amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
 105                 if (r)
 106                         return r;
 107         }
 108
 109         r = amdgpu_jpeg_sw_init(adev);
 110         if (r)
 111                 return r;
 112
 113         r = amdgpu_jpeg_resume(adev);
 114         if (r)
 115                 return r;
 116
 117         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 118                 jpeg_inst = GET_INST(JPEG, i);
 119
 120                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 121                         ring = &adev->jpeg.inst[i].ring_dec[j];
 122                         ring->use_doorbell = true;
 123                         ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
 124                         if (!amdgpu_sriov_vf(adev)) {
 125                                 ring->doorbell_index =
 126                                         (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 127                                         1 + j + 9 * jpeg_inst;
 128                         } else {
 129                                 if (j < 4)
 130                                         ring->doorbell_index =
 131                                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 132                                                 4 + j + 32 * jpeg_inst;
 133                                 else
 134                                         ring->doorbell_index =
 135                                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 136                                                 8 + j + 32 * jpeg_inst;
 137                         }
 138                         sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
 139                         r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 140                                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
 141                         if (r)
 142                                 return r;
 143
 144                         adev->jpeg.internal.jpeg_pitch[j] =
 145                                 regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET;
 146                         adev->jpeg.inst[i].external.jpeg_pitch[j] =
 147                                 SOC15_REG_OFFSET1(
 148                                         JPEG, jpeg_inst,
 149                                         regUVD_JRBC0_UVD_JRBC_SCRATCH0,
 150                                         (j ? (0x40 * j - 0xc80) : 0));
 151                 }
 152         }
 153
 154         if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
 155                 r = amdgpu_jpeg_ras_sw_init(adev);
 156                 if (r) {
 157                         dev_err(adev->dev, "Failed to initialize jpeg ras block!\n");
 158                         return r;
 159                 }
 160         }
 161
 162         /* TODO: Add queue reset mask when FW fully supports it */
 163         adev->jpeg.supported_reset =
 164                 amdgpu_get_soft_full_reset_mask(&adev->jpeg.inst[0].ring_dec[0]);
 165         r = amdgpu_jpeg_sysfs_reset_mask_init(adev);
 166         if (r)
 167                 return r;
 168
 169         return 0;
 170 }
 171
 172 /**
 173  * jpeg_v4_0_3_sw_fini - sw fini for JPEG block
 174  *
 175  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 176  *
 177  * JPEG suspend and free up sw allocation
 178  */
 179 static int jpeg_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
 180 {
 181         struct amdgpu_device *adev = ip_block->adev;
 182         int r;
 183
 184         r = amdgpu_jpeg_suspend(adev);
 185         if (r)
 186                 return r;
 187
 188         amdgpu_jpeg_sysfs_reset_mask_fini(adev);
 189         r = amdgpu_jpeg_sw_fini(adev);
 190
 191         return r;
 192 }
 193
 194 static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 195 {
 196         struct amdgpu_ring *ring;
 197         uint64_t ctx_addr;
 198         uint32_t param, resp, expected;
 199         uint32_t tmp, timeout;
 200
 201         struct amdgpu_mm_table *table = &adev->virt.mm_table;
 202         uint32_t *table_loc;
 203         uint32_t table_size;
 204         uint32_t size, size_dw, item_offset;
 205         uint32_t init_status;
 206         int i, j, jpeg_inst;
 207
 208         struct mmsch_v4_0_cmd_direct_write
 209                 direct_wt = { {0} };
 210         struct mmsch_v4_0_cmd_end end = { {0} };
 211         struct mmsch_v4_0_3_init_header header;
 212
 213         direct_wt.cmd_header.command_type =
 214                 MMSCH_COMMAND__DIRECT_REG_WRITE;
 215         end.cmd_header.command_type =
 216                 MMSCH_COMMAND__END;
 217
 218         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
 219                 jpeg_inst = GET_INST(JPEG, i);
 220
 221                 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header));
 222                 header.version = MMSCH_VERSION;
 223                 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2;
 224
 225                 table_loc = (uint32_t *)table->cpu_addr;
 226                 table_loc += header.total_size;
 227
 228                 item_offset = header.total_size;
 229
 230                 for (j = 0; j < adev->jpeg.num_jpeg_rings; j++) {
 231                         ring = &adev->jpeg.inst[i].ring_dec[j];
 232                         table_size = 0;
 233
 234                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW);
 235                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, lower_32_bits(ring->gpu_addr));
 236                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH);
 237                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, upper_32_bits(ring->gpu_addr));
 238                         tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_SIZE);
 239                         MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
 240
 241                         if (j <= 3) {
 242                                 header.mjpegdec0[j].table_offset = item_offset;
 243                                 header.mjpegdec0[j].init_status = 0;
 244                                 header.mjpegdec0[j].table_size = table_size;
 245                         } else {
 246                                 header.mjpegdec1[j - 4].table_offset = item_offset;
 247                                 header.mjpegdec1[j - 4].init_status = 0;
 248                                 header.mjpegdec1[j - 4].table_size = table_size;
 249                         }
 250                         header.total_size += table_size;
 251                         item_offset += table_size;
 252                 }
 253
 254                 MMSCH_V4_0_INSERT_END();
 255
 256                 /* send init table to MMSCH */
 257                 size = sizeof(struct mmsch_v4_0_3_init_header);
 258                 table_loc = (uint32_t *)table->cpu_addr;
 259                 memcpy((void *)table_loc, &header, size);
 260
 261                 ctx_addr = table->gpu_addr;
 262                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
 263                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
 264
 265                 tmp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID);
 266                 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
 267                 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
 268                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID, tmp);
 269
 270                 size = header.total_size;
 271                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_SIZE, size);
 272
 273                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP, 0);
 274
 275                 param = 0x00000001;
 276                 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_HOST, param);
 277                 tmp = 0;
 278                 timeout = 1000;
 279                 resp = 0;
 280                 expected = MMSCH_VF_MAILBOX_RESP__OK;
 281                 init_status =
 282                         ((struct mmsch_v4_0_3_init_header *)(table_loc))->mjpegdec0[i].init_status;
 283                 while (resp != expected) {
 284                         resp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP);
 285
 286                         if (resp != 0)
 287                                 break;
 288                         udelay(10);
 289                         tmp = tmp + 10;
 290                         if (tmp >= timeout) {
 291                                 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
 292                                         " waiting for regMMSCH_VF_MAILBOX_RESP "\
 293                                         "(expected=0x%08x, readback=0x%08x)\n",
 294                                         tmp, expected, resp);
 295                                 return -EBUSY;
 296                         }
 297                 }
 298                 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE &&
 299                                 init_status != MMSCH_VF_ENGINE_STATUS__PASS)
 300                         DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n",
 301                                         resp, init_status);
 302
 303         }
 304         return 0;
 305 }
 306
 307 /**
 308  * jpeg_v4_0_3_hw_init - start and test JPEG block
 309  *
 310  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 311  *
 312  */
 313 static int jpeg_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block)
 314 {
 315         struct amdgpu_device *adev = ip_block->adev;
 316         struct amdgpu_ring *ring;
 317         int i, j, r, jpeg_inst;
 318
 319         if (amdgpu_sriov_vf(adev)) {
 320                 r = jpeg_v4_0_3_start_sriov(adev);
 321                 if (r)
 322                         return r;
 323
 324                 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 325                         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 326                                 ring = &adev->jpeg.inst[i].ring_dec[j];
 327                                 ring->wptr = 0;
 328                                 ring->wptr_old = 0;
 329                                 jpeg_v4_0_3_dec_ring_set_wptr(ring);
 330                                 ring->sched.ready = true;
 331                         }
 332                 }
 333         } else {
 334                 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 335                         jpeg_inst = GET_INST(JPEG, i);
 336
 337                         ring = adev->jpeg.inst[i].ring_dec;
 338
 339                         if (ring->use_doorbell)
 340                                 adev->nbio.funcs->vcn_doorbell_range(
 341                                         adev, ring->use_doorbell,
 342                                         (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 343                                                 9 * jpeg_inst,
 344                                         adev->jpeg.inst[i].aid_id);
 345
 346                         for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 347                                 ring = &adev->jpeg.inst[i].ring_dec[j];
 348                                 if (ring->use_doorbell)
 349                                         WREG32_SOC15_OFFSET(
 350                                                 VCN, GET_INST(VCN, i),
 351                                                 regVCN_JPEG_DB_CTRL,
 352                                                 (ring->pipe ? (ring->pipe - 0x15) : 0),
 353                                                 ring->doorbell_index
 354                                                         << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
 355                                                         VCN_JPEG_DB_CTRL__EN_MASK);
 356                                 r = amdgpu_ring_test_helper(ring);
 357                                 if (r)
 358                                         return r;
 359                         }
 360                 }
 361         }
 362
 363         return 0;
 364 }
 365
 366 /**
 367  * jpeg_v4_0_3_hw_fini - stop the hardware block
 368  *
 369  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 370  *
 371  * Stop the JPEG block, mark ring as not ready any more
 372  */
 373 static int jpeg_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block)
 374 {
 375         struct amdgpu_device *adev = ip_block->adev;
 376         int ret = 0;
 377
 378         cancel_delayed_work_sync(&adev->jpeg.idle_work);
 379
 380         if (!amdgpu_sriov_vf(adev)) {
 381                 if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
 382                         ret = jpeg_v4_0_3_set_powergating_state(ip_block, AMD_PG_STATE_GATE);
 383         }
 384
 385         return ret;
 386 }
 387
 388 /**
 389  * jpeg_v4_0_3_suspend - suspend JPEG block
 390  *
 391  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 392  *
 393  * HW fini and suspend JPEG block
 394  */
 395 static int jpeg_v4_0_3_suspend(struct amdgpu_ip_block *ip_block)
 396 {
 397         int r;
 398
 399         r = jpeg_v4_0_3_hw_fini(ip_block);
 400         if (r)
 401                 return r;
 402
 403         r = amdgpu_jpeg_suspend(ip_block->adev);
 404
 405         return r;
 406 }
 407
 408 /**
 409  * jpeg_v4_0_3_resume - resume JPEG block
 410  *
 411  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 412  *
 413  * Resume firmware and hw init JPEG block
 414  */
 415 static int jpeg_v4_0_3_resume(struct amdgpu_ip_block *ip_block)
 416 {
 417         int r;
 418
 419         r = amdgpu_jpeg_resume(ip_block->adev);
 420         if (r)
 421                 return r;
 422
 423         r = jpeg_v4_0_3_hw_init(ip_block);
 424
 425         return r;
 426 }
 427
 428 static void jpeg_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 429 {
 430         int i, jpeg_inst;
 431         uint32_t data;
 432
 433         jpeg_inst = GET_INST(JPEG, inst_idx);
 434         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 435         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 436                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 437                 data &= (~(JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1));
 438         } else {
 439                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 440         }
 441
 442         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 443         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 444         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 445
 446         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 447         data &= ~(JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 448         for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 449                 data &= ~(JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 450         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 451 }
 452
 453 static void jpeg_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 454 {
 455         int i, jpeg_inst;
 456         uint32_t data;
 457
 458         jpeg_inst = GET_INST(JPEG, inst_idx);
 459         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 460         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 461                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 462                 data |= (JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1);
 463         } else {
 464                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 465         }
 466
 467         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 468         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 469         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 470
 471         data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 472         data |= (JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 473         for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 474                 data |= (JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 475         WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 476 }
 477
 478 /**
 479  * jpeg_v4_0_3_start - start JPEG block
 480  *
 481  * @adev: amdgpu_device pointer
 482  *
 483  * Setup and start the JPEG block
 484  */
 485 static int jpeg_v4_0_3_start(struct amdgpu_device *adev)
 486 {
 487         struct amdgpu_ring *ring;
 488         int i, j, jpeg_inst;
 489
 490         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 491                 jpeg_inst = GET_INST(JPEG, i);
 492
 493                 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
 494                              1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
 495                 SOC15_WAIT_ON_RREG(
 496                         JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
 497                         UVD_PGFSM_STATUS__UVDJ_PWR_ON
 498                                 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
 499                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 500
 501                 /* disable anti hang mechanism */
 502                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 503                                           regUVD_JPEG_POWER_STATUS),
 504                          0, ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 505
 506                 /* JPEG disable CGC */
 507                 jpeg_v4_0_3_disable_clock_gating(adev, i);
 508
 509                 /* MJPEG global tiling registers */
 510                 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX8_ADDR_CONFIG,
 511                              adev->gfx.config.gb_addr_config);
 512                 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX10_ADDR_CONFIG,
 513                              adev->gfx.config.gb_addr_config);
 514
 515                 /* enable JMI channel */
 516                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL), 0,
 517                          ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 518
 519                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 520                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 521
 522                         ring = &adev->jpeg.inst[i].ring_dec[j];
 523
 524                         /* enable System Interrupt for JRBC */
 525                         WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 526                                                   regJPEG_SYS_INT_EN),
 527                                  JPEG_SYS_INT_EN__DJRBC0_MASK << j,
 528                                  ~(JPEG_SYS_INT_EN__DJRBC0_MASK << j));
 529
 530                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 531                                             regUVD_JMI0_UVD_LMI_JRBC_RB_VMID,
 532                                             reg_offset, 0);
 533                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 534                                             regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 535                                             reg_offset,
 536                                             (0x00000001L | 0x00000002L));
 537                         WREG32_SOC15_OFFSET(
 538                                 JPEG, jpeg_inst,
 539                                 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 540                                 reg_offset, lower_32_bits(ring->gpu_addr));
 541                         WREG32_SOC15_OFFSET(
 542                                 JPEG, jpeg_inst,
 543                                 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 544                                 reg_offset, upper_32_bits(ring->gpu_addr));
 545                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 546                                             regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 547                                             reg_offset, 0);
 548                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 549                                             regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 550                                             reg_offset, 0);
 551                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 552                                             regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 553                                             reg_offset, 0x00000002L);
 554                         WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 555                                             regUVD_JRBC0_UVD_JRBC_RB_SIZE,
 556                                             reg_offset, ring->ring_size / 4);
 557                         ring->wptr = RREG32_SOC15_OFFSET(
 558                                 JPEG, jpeg_inst, regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 559                                 reg_offset);
 560                 }
 561         }
 562
 563         return 0;
 564 }
 565
 566 /**
 567  * jpeg_v4_0_3_stop - stop JPEG block
 568  *
 569  * @adev: amdgpu_device pointer
 570  *
 571  * stop the JPEG block
 572  */
 573 static int jpeg_v4_0_3_stop(struct amdgpu_device *adev)
 574 {
 575         int i, jpeg_inst;
 576
 577         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 578                 jpeg_inst = GET_INST(JPEG, i);
 579                 /* reset JMI */
 580                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL),
 581                          UVD_JMI_CNTL__SOFT_RESET_MASK,
 582                          ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 583
 584                 jpeg_v4_0_3_enable_clock_gating(adev, i);
 585
 586                 /* enable anti hang mechanism */
 587                 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
 588                                           regUVD_JPEG_POWER_STATUS),
 589                          UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
 590                          ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 591
 592                 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
 593                              2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
 594                 SOC15_WAIT_ON_RREG(
 595                         JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
 596                         UVD_PGFSM_STATUS__UVDJ_PWR_OFF
 597                                 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
 598                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 599         }
 600
 601         return 0;
 602 }
 603
 604 /**
 605  * jpeg_v4_0_3_dec_ring_get_rptr - get read pointer
 606  *
 607  * @ring: amdgpu_ring pointer
 608  *
 609  * Returns the current hardware read pointer
 610  */
 611 static uint64_t jpeg_v4_0_3_dec_ring_get_rptr(struct amdgpu_ring *ring)
 612 {
 613         struct amdgpu_device *adev = ring->adev;
 614
 615         return RREG32_SOC15_OFFSET(
 616                 JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 617                 ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
 618 }
 619
 620 /**
 621  * jpeg_v4_0_3_dec_ring_get_wptr - get write pointer
 622  *
 623  * @ring: amdgpu_ring pointer
 624  *
 625  * Returns the current hardware write pointer
 626  */
 627 static uint64_t jpeg_v4_0_3_dec_ring_get_wptr(struct amdgpu_ring *ring)
 628 {
 629         struct amdgpu_device *adev = ring->adev;
 630
 631         if (ring->use_doorbell)
 632                 return adev->wb.wb[ring->wptr_offs];
 633         else
 634                 return RREG32_SOC15_OFFSET(
 635                         JPEG, GET_INST(JPEG, ring->me),
 636                         regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 637                         ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
 638 }
 639
 640 static void jpeg_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 641 {
 642         /* JPEG engine access for HDP flush doesn't work when RRMT is enabled.
 643          * This is a workaround to avoid any HDP flush through JPEG ring.
 644          */
 645 }
 646
 647 /**
 648  * jpeg_v4_0_3_dec_ring_set_wptr - set write pointer
 649  *
 650  * @ring: amdgpu_ring pointer
 651  *
 652  * Commits the write pointer to the hardware
 653  */
 654 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring)
 655 {
 656         struct amdgpu_device *adev = ring->adev;
 657
 658         if (ring->use_doorbell) {
 659                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
 660                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 661         } else {
 662                 WREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me),
 663                                     regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 664                                     (ring->pipe ? (0x40 * ring->pipe - 0xc80) :
 665                                                   0),
 666                                     lower_32_bits(ring->wptr));
 667         }
 668 }
 669
 670 /**
 671  * jpeg_v4_0_3_dec_ring_insert_start - insert a start command
 672  *
 673  * @ring: amdgpu_ring pointer
 674  *
 675  * Write a start command to the ring.
 676  */
 677 void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring)
 678 {
 679         if (!amdgpu_sriov_vf(ring->adev)) {
 680                 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 681                         0, 0, PACKETJ_TYPE0));
 682                 amdgpu_ring_write(ring, 0x62a04); /* PCTL0_MMHUB_DEEPSLEEP_IB */
 683
 684                 amdgpu_ring_write(ring,
 685                                   PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0,
 686                                           0, PACKETJ_TYPE0));
 687                 amdgpu_ring_write(ring, 0x80004000);
 688         }
 689 }
 690
 691 /**
 692  * jpeg_v4_0_3_dec_ring_insert_end - insert a end command
 693  *
 694  * @ring: amdgpu_ring pointer
 695  *
 696  * Write a end command to the ring.
 697  */
 698 void jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring *ring)
 699 {
 700         if (!amdgpu_sriov_vf(ring->adev)) {
 701                 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 702                         0, 0, PACKETJ_TYPE0));
 703                 amdgpu_ring_write(ring, 0x62a04);
 704
 705                 amdgpu_ring_write(ring,
 706                                   PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0,
 707                                           0, PACKETJ_TYPE0));
 708                 amdgpu_ring_write(ring, 0x00004000);
 709         }
 710 }
 711
 712 /**
 713  * jpeg_v4_0_3_dec_ring_emit_fence - emit an fence & trap command
 714  *
 715  * @ring: amdgpu_ring pointer
 716  * @addr: address
 717  * @seq: sequence number
 718  * @flags: fence related flags
 719  *
 720  * Write a fence and a trap command to the ring.
 721  */
 722 void jpeg_v4_0_3_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 723                                 unsigned int flags)
 724 {
 725         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 726
 727         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
 728                 0, 0, PACKETJ_TYPE0));
 729         amdgpu_ring_write(ring, seq);
 730
 731         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
 732                 0, 0, PACKETJ_TYPE0));
 733         amdgpu_ring_write(ring, seq);
 734
 735         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
 736                 0, 0, PACKETJ_TYPE0));
 737         amdgpu_ring_write(ring, lower_32_bits(addr));
 738
 739         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 740                 0, 0, PACKETJ_TYPE0));
 741         amdgpu_ring_write(ring, upper_32_bits(addr));
 742
 743         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 744                 0, 0, PACKETJ_TYPE0));
 745         amdgpu_ring_write(ring, 0x8);
 746
 747         amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 748                 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
 749         amdgpu_ring_write(ring, 0);
 750
 751         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 752         amdgpu_ring_write(ring, 0);
 753
 754         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 755         amdgpu_ring_write(ring, 0);
 756
 757         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
 758         amdgpu_ring_write(ring, 0);
 759 }
 760
 761 /**
 762  * jpeg_v4_0_3_dec_ring_emit_ib - execute indirect buffer
 763  *
 764  * @ring: amdgpu_ring pointer
 765  * @job: job to retrieve vmid from
 766  * @ib: indirect buffer to execute
 767  * @flags: unused
 768  *
 769  * Write ring commands to execute the indirect buffer.
 770  */
 771 void jpeg_v4_0_3_dec_ring_emit_ib(struct amdgpu_ring *ring,
 772                                 struct amdgpu_job *job,
 773                                 struct amdgpu_ib *ib,
 774                                 uint32_t flags)
 775 {
 776         unsigned int vmid = AMDGPU_JOB_GET_VMID(job);
 777
 778         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
 779                 0, 0, PACKETJ_TYPE0));
 780
 781         if (ring->funcs->parse_cs)
 782                 amdgpu_ring_write(ring, 0);
 783         else
 784                 amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 785
 786         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
 787                 0, 0, PACKETJ_TYPE0));
 788         amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 789
 790         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
 791                 0, 0, PACKETJ_TYPE0));
 792         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 793
 794         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 795                 0, 0, PACKETJ_TYPE0));
 796         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 797
 798         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
 799                 0, 0, PACKETJ_TYPE0));
 800         amdgpu_ring_write(ring, ib->length_dw);
 801
 802         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
 803                 0, 0, PACKETJ_TYPE0));
 804         amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
 805
 806         amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 807                 0, 0, PACKETJ_TYPE0));
 808         amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
 809
 810         amdgpu_ring_write(ring, PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
 811         amdgpu_ring_write(ring, 0);
 812
 813         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 814                 0, 0, PACKETJ_TYPE0));
 815         amdgpu_ring_write(ring, 0x01400200);
 816
 817         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 818                 0, 0, PACKETJ_TYPE0));
 819         amdgpu_ring_write(ring, 0x2);
 820
 821         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_STATUS_INTERNAL_OFFSET,
 822                 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
 823         amdgpu_ring_write(ring, 0x2);
 824 }
 825
 826 void jpeg_v4_0_3_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 827                                 uint32_t val, uint32_t mask)
 828 {
 829         uint32_t reg_offset;
 830
 831         /* Use normalized offsets if required */
 832         if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 833                 reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 834
 835         reg_offset = (reg << 2);
 836
 837         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 838                 0, 0, PACKETJ_TYPE0));
 839         amdgpu_ring_write(ring, 0x01400200);
 840
 841         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 842                 0, 0, PACKETJ_TYPE0));
 843         amdgpu_ring_write(ring, val);
 844
 845         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 846                 0, 0, PACKETJ_TYPE0));
 847         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 848                 amdgpu_ring_write(ring, 0);
 849                 amdgpu_ring_write(ring,
 850                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
 851         } else {
 852                 amdgpu_ring_write(ring, reg_offset);
 853                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 854                         0, 0, PACKETJ_TYPE3));
 855         }
 856         amdgpu_ring_write(ring, mask);
 857 }
 858
 859 void jpeg_v4_0_3_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 860                                 unsigned int vmid, uint64_t pd_addr)
 861 {
 862         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 863         uint32_t data0, data1, mask;
 864
 865         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 866
 867         /* wait for register write */
 868         data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
 869         data1 = lower_32_bits(pd_addr);
 870         mask = 0xffffffff;
 871         jpeg_v4_0_3_dec_ring_emit_reg_wait(ring, data0, data1, mask);
 872 }
 873
 874 void jpeg_v4_0_3_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
 875 {
 876         uint32_t reg_offset;
 877
 878         /* Use normalized offsets if required */
 879         if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 880                 reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 881
 882         reg_offset = (reg << 2);
 883
 884         amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 885                 0, 0, PACKETJ_TYPE0));
 886         if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 887                 amdgpu_ring_write(ring, 0);
 888                 amdgpu_ring_write(ring,
 889                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
 890         } else {
 891                 amdgpu_ring_write(ring, reg_offset);
 892                 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 893                         0, 0, PACKETJ_TYPE0));
 894         }
 895         amdgpu_ring_write(ring, val);
 896 }
 897
 898 void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count)
 899 {
 900         int i;
 901
 902         WARN_ON(ring->wptr % 2 || count % 2);
 903
 904         for (i = 0; i < count / 2; i++) {
 905                 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 906                 amdgpu_ring_write(ring, 0);
 907         }
 908 }
 909
 910 static bool jpeg_v4_0_3_is_idle(void *handle)
 911 {
 912         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 913         bool ret = false;
 914         int i, j;
 915
 916         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 917                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 918                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 919
 920                         ret &= ((RREG32_SOC15_OFFSET(
 921                                          JPEG, GET_INST(JPEG, i),
 922                                          regUVD_JRBC0_UVD_JRBC_STATUS,
 923                                          reg_offset) &
 924                                  UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
 925                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 926                 }
 927         }
 928
 929         return ret;
 930 }
 931
 932 static int jpeg_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block)
 933 {
 934         struct amdgpu_device *adev = ip_block->adev;
 935         int ret = 0;
 936         int i, j;
 937
 938         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 939                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 940                         unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
 941
 942                         ret &= SOC15_WAIT_ON_RREG_OFFSET(
 943                                 JPEG, GET_INST(JPEG, i),
 944                                 regUVD_JRBC0_UVD_JRBC_STATUS, reg_offset,
 945                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
 946                                 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 947                 }
 948         }
 949         return ret;
 950 }
 951
 952 static int jpeg_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 953                                           enum amd_clockgating_state state)
 954 {
 955         struct amdgpu_device *adev = ip_block->adev;
 956         bool enable = state == AMD_CG_STATE_GATE;
 957         int i;
 958
 959         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 960                 if (enable) {
 961                         if (!jpeg_v4_0_3_is_idle(adev))
 962                                 return -EBUSY;
 963                         jpeg_v4_0_3_enable_clock_gating(adev, i);
 964                 } else {
 965                         jpeg_v4_0_3_disable_clock_gating(adev, i);
 966                 }
 967         }
 968         return 0;
 969 }
 970
 971 static int jpeg_v4_0_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
 972                                           enum amd_powergating_state state)
 973 {
 974         struct amdgpu_device *adev = ip_block->adev;
 975         int ret;
 976
 977         if (amdgpu_sriov_vf(adev)) {
 978                 adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
 979                 return 0;
 980         }
 981
 982         if (state == adev->jpeg.cur_state)
 983                 return 0;
 984
 985         if (state == AMD_PG_STATE_GATE)
 986                 ret = jpeg_v4_0_3_stop(adev);
 987         else
 988                 ret = jpeg_v4_0_3_start(adev);
 989
 990         if (!ret)
 991                 adev->jpeg.cur_state = state;
 992
 993         return ret;
 994 }
 995
 996 static int jpeg_v4_0_3_set_interrupt_state(struct amdgpu_device *adev,
 997                                         struct amdgpu_irq_src *source,
 998                                         unsigned int type,
 999                                         enum amdgpu_interrupt_state state)
1000 {
1001         return 0;
1002 }
1003
1004 static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
1005                                       struct amdgpu_irq_src *source,
1006                                       struct amdgpu_iv_entry *entry)
1007 {
1008         uint32_t i, inst;
1009
1010         i = node_id_to_phys_map[entry->node_id];
1011         DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
1012
1013         for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
1014                 if (adev->jpeg.inst[inst].aid_id == i)
1015                         break;
1016
1017         if (inst >= adev->jpeg.num_jpeg_inst) {
1018                 dev_WARN_ONCE(adev->dev, 1,
1019                               "Interrupt received for unknown JPEG instance %d",
1020                               entry->node_id);
1021                 return 0;
1022         }
1023
1024         switch (entry->src_id) {
1025         case VCN_4_0__SRCID__JPEG_DECODE:
1026                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
1027                 break;
1028         case VCN_4_0__SRCID__JPEG1_DECODE:
1029                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
1030                 break;
1031         case VCN_4_0__SRCID__JPEG2_DECODE:
1032                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
1033                 break;
1034         case VCN_4_0__SRCID__JPEG3_DECODE:
1035                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
1036                 break;
1037         case VCN_4_0__SRCID__JPEG4_DECODE:
1038                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
1039                 break;
1040         case VCN_4_0__SRCID__JPEG5_DECODE:
1041                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
1042                 break;
1043         case VCN_4_0__SRCID__JPEG6_DECODE:
1044                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
1045                 break;
1046         case VCN_4_0__SRCID__JPEG7_DECODE:
1047                 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
1048                 break;
1049         default:
1050                 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
1051                           entry->src_id, entry->src_data[0]);
1052                 break;
1053         }
1054
1055         return 0;
1056 }
1057
1058 static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
1059         .name = "jpeg_v4_0_3",
1060         .early_init = jpeg_v4_0_3_early_init,
1061         .sw_init = jpeg_v4_0_3_sw_init,
1062         .sw_fini = jpeg_v4_0_3_sw_fini,
1063         .hw_init = jpeg_v4_0_3_hw_init,
1064         .hw_fini = jpeg_v4_0_3_hw_fini,
1065         .suspend = jpeg_v4_0_3_suspend,
1066         .resume = jpeg_v4_0_3_resume,
1067         .is_idle = jpeg_v4_0_3_is_idle,
1068         .wait_for_idle = jpeg_v4_0_3_wait_for_idle,
1069         .set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
1070         .set_powergating_state = jpeg_v4_0_3_set_powergating_state,
1071 };
1072
1073 static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
1074         .type = AMDGPU_RING_TYPE_VCN_JPEG,
1075         .align_mask = 0xf,
1076         .get_rptr = jpeg_v4_0_3_dec_ring_get_rptr,
1077         .get_wptr = jpeg_v4_0_3_dec_ring_get_wptr,
1078         .set_wptr = jpeg_v4_0_3_dec_ring_set_wptr,
1079         .parse_cs = jpeg_v2_dec_ring_parse_cs,
1080         .emit_frame_size =
1081                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1082                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1083                 8 + /* jpeg_v4_0_3_dec_ring_emit_vm_flush */
1084                 18 + 18 + /* jpeg_v4_0_3_dec_ring_emit_fence x2 vm fence */
1085                 8 + 16,
1086         .emit_ib_size = 22, /* jpeg_v4_0_3_dec_ring_emit_ib */
1087         .emit_ib = jpeg_v4_0_3_dec_ring_emit_ib,
1088         .emit_fence = jpeg_v4_0_3_dec_ring_emit_fence,
1089         .emit_vm_flush = jpeg_v4_0_3_dec_ring_emit_vm_flush,
1090         .emit_hdp_flush = jpeg_v4_0_3_ring_emit_hdp_flush,
1091         .test_ring = amdgpu_jpeg_dec_ring_test_ring,
1092         .test_ib = amdgpu_jpeg_dec_ring_test_ib,
1093         .insert_nop = jpeg_v4_0_3_dec_ring_nop,
1094         .insert_start = jpeg_v4_0_3_dec_ring_insert_start,
1095         .insert_end = jpeg_v4_0_3_dec_ring_insert_end,
1096         .pad_ib = amdgpu_ring_generic_pad_ib,
1097         .begin_use = amdgpu_jpeg_ring_begin_use,
1098         .end_use = amdgpu_jpeg_ring_end_use,
1099         .emit_wreg = jpeg_v4_0_3_dec_ring_emit_wreg,
1100         .emit_reg_wait = jpeg_v4_0_3_dec_ring_emit_reg_wait,
1101         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1102 };
1103
1104 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev)
1105 {
1106         int i, j, jpeg_inst;
1107
1108         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1109                 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
1110                         adev->jpeg.inst[i].ring_dec[j].funcs = &jpeg_v4_0_3_dec_ring_vm_funcs;
1111                         adev->jpeg.inst[i].ring_dec[j].me = i;
1112                         adev->jpeg.inst[i].ring_dec[j].pipe = j;
1113                 }
1114                 jpeg_inst = GET_INST(JPEG, i);
1115                 adev->jpeg.inst[i].aid_id =
1116                         jpeg_inst / adev->jpeg.num_inst_per_aid;
1117         }
1118 }
1119
1120 static const struct amdgpu_irq_src_funcs jpeg_v4_0_3_irq_funcs = {
1121         .set = jpeg_v4_0_3_set_interrupt_state,
1122         .process = jpeg_v4_0_3_process_interrupt,
1123 };
1124
1125 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev)
1126 {
1127         int i;
1128
1129         for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1130                 adev->jpeg.inst->irq.num_types += adev->jpeg.num_jpeg_rings;
1131         }
1132         adev->jpeg.inst->irq.funcs = &jpeg_v4_0_3_irq_funcs;
1133 }
1134
1135 const struct amdgpu_ip_block_version jpeg_v4_0_3_ip_block = {
1136         .type = AMD_IP_BLOCK_TYPE_JPEG,
1137         .major = 4,
1138         .minor = 0,
1139         .rev = 3,
1140         .funcs = &jpeg_v4_0_3_ip_funcs,
1141 };
1142
1143 static const struct amdgpu_ras_err_status_reg_entry jpeg_v4_0_3_ue_reg_list[] = {
1144         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0S, regVCN_UE_ERR_STATUS_HI_JPEG0S),
1145         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0S"},
1146         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0D, regVCN_UE_ERR_STATUS_HI_JPEG0D),
1147         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0D"},
1148         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1S, regVCN_UE_ERR_STATUS_HI_JPEG1S),
1149         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1S"},
1150         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1D, regVCN_UE_ERR_STATUS_HI_JPEG1D),
1151         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1D"},
1152         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2S, regVCN_UE_ERR_STATUS_HI_JPEG2S),
1153         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2S"},
1154         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2D, regVCN_UE_ERR_STATUS_HI_JPEG2D),
1155         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2D"},
1156         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3S, regVCN_UE_ERR_STATUS_HI_JPEG3S),
1157         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3S"},
1158         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3D, regVCN_UE_ERR_STATUS_HI_JPEG3D),
1159         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3D"},
1160         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4S, regVCN_UE_ERR_STATUS_HI_JPEG4S),
1161         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4S"},
1162         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4D, regVCN_UE_ERR_STATUS_HI_JPEG4D),
1163         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4D"},
1164         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5S, regVCN_UE_ERR_STATUS_HI_JPEG5S),
1165         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5S"},
1166         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5D, regVCN_UE_ERR_STATUS_HI_JPEG5D),
1167         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5D"},
1168         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6S, regVCN_UE_ERR_STATUS_HI_JPEG6S),
1169         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6S"},
1170         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6D, regVCN_UE_ERR_STATUS_HI_JPEG6D),
1171         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6D"},
1172         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7S, regVCN_UE_ERR_STATUS_HI_JPEG7S),
1173         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7S"},
1174         {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7D, regVCN_UE_ERR_STATUS_HI_JPEG7D),
1175         1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7D"},
1176 };
1177
1178 static void jpeg_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev,
1179                                                    uint32_t jpeg_inst,
1180                                                    void *ras_err_status)
1181 {
1182         struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
1183
1184         /* jpeg v4_0_3 only support uncorrectable errors */
1185         amdgpu_ras_inst_query_ras_error_count(adev,
1186                         jpeg_v4_0_3_ue_reg_list,
1187                         ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1188                         NULL, 0, GET_INST(VCN, jpeg_inst),
1189                         AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
1190                         &err_data->ue_count);
1191 }
1192
1193 static void jpeg_v4_0_3_query_ras_error_count(struct amdgpu_device *adev,
1194                                               void *ras_err_status)
1195 {
1196         uint32_t i;
1197
1198         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1199                 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1200                 return;
1201         }
1202
1203         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1204                 jpeg_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status);
1205 }
1206
1207 static void jpeg_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev,
1208                                                    uint32_t jpeg_inst)
1209 {
1210         amdgpu_ras_inst_reset_ras_error_count(adev,
1211                         jpeg_v4_0_3_ue_reg_list,
1212                         ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1213                         GET_INST(VCN, jpeg_inst));
1214 }
1215
1216 static void jpeg_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev)
1217 {
1218         uint32_t i;
1219
1220         if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1221                 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1222                 return;
1223         }
1224
1225         for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1226                 jpeg_v4_0_3_inst_reset_ras_error_count(adev, i);
1227 }
1228
1229 static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = {
1230         .query_ras_error_count = jpeg_v4_0_3_query_ras_error_count,
1231         .reset_ras_error_count = jpeg_v4_0_3_reset_ras_error_count,
1232 };
1233
1234 static int jpeg_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1235                                       enum aca_smu_type type, void *data)
1236 {
1237         struct aca_bank_info info;
1238         u64 misc0;
1239         int ret;
1240
1241         ret = aca_bank_info_decode(bank, &info);
1242         if (ret)
1243                 return ret;
1244
1245         misc0 = bank->regs[ACA_REG_IDX_MISC0];
1246         switch (type) {
1247         case ACA_SMU_TYPE_UE:
1248                 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
1249                                                      1ULL);
1250                 break;
1251         case ACA_SMU_TYPE_CE:
1252                 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE,
1253                                                      ACA_REG__MISC0__ERRCNT(misc0));
1254                 break;
1255         default:
1256                 return -EINVAL;
1257         }
1258
1259         return ret;
1260 }
1261
1262 /* reference to smu driver if header file */
1263 static int jpeg_v4_0_3_err_codes[] = {
1264         16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-7][S|D] */
1265         24, 25, 26, 27, 28, 29, 30, 31
1266 };
1267
1268 static bool jpeg_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
1269                                          enum aca_smu_type type, void *data)
1270 {
1271         u32 instlo;
1272
1273         instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
1274         instlo &= GENMASK(31, 1);
1275
1276         if (instlo != mmSMNAID_AID0_MCA_SMU)
1277                 return false;
1278
1279         if (aca_bank_check_error_codes(handle->adev, bank,
1280                                        jpeg_v4_0_3_err_codes,
1281                                        ARRAY_SIZE(jpeg_v4_0_3_err_codes)))
1282                 return false;
1283
1284         return true;
1285 }
1286
1287 static const struct aca_bank_ops jpeg_v4_0_3_aca_bank_ops = {
1288         .aca_bank_parser = jpeg_v4_0_3_aca_bank_parser,
1289         .aca_bank_is_valid = jpeg_v4_0_3_aca_bank_is_valid,
1290 };
1291
1292 static const struct aca_info jpeg_v4_0_3_aca_info = {
1293         .hwip = ACA_HWIP_TYPE_SMU,
1294         .mask = ACA_ERROR_UE_MASK,
1295         .bank_ops = &jpeg_v4_0_3_aca_bank_ops,
1296 };
1297
1298 static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1299 {
1300         int r;
1301
1302         r = amdgpu_ras_block_late_init(adev, ras_block);
1303         if (r)
1304                 return r;
1305
1306         r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG,
1307                                 &jpeg_v4_0_3_aca_info, NULL);
1308         if (r)
1309                 goto late_fini;
1310
1311         return 0;
1312
1313 late_fini:
1314         amdgpu_ras_block_late_fini(adev, ras_block);
1315
1316         return r;
1317 }
1318
1319 static struct amdgpu_jpeg_ras jpeg_v4_0_3_ras = {
1320         .ras_block = {
1321                 .hw_ops = &jpeg_v4_0_3_ras_hw_ops,
1322                 .ras_late_init = jpeg_v4_0_3_ras_late_init,
1323         },
1324 };
1325
1326 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev)
1327 {
1328         adev->jpeg.ras = &jpeg_v4_0_3_ras;
1329 }