drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <drm/drmP.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_uvd.h"
  28 #include "soc15d.h"
  29 #include "soc15_common.h"
  30 #include "mmsch_v1_0.h"
  31
  32 #include "uvd/uvd_7_0_offset.h"
  33 #include "uvd/uvd_7_0_sh_mask.h"
  34 #include "vce/vce_4_0_offset.h"
  35 #include "vce/vce_4_0_default.h"
  36 #include "vce/vce_4_0_sh_mask.h"
  37 #include "nbif/nbif_6_1_offset.h"
  38 #include "hdp/hdp_4_0_offset.h"
  39 #include "mmhub/mmhub_1_0_offset.h"
  40 #include "mmhub/mmhub_1_0_sh_mask.h"
  41
  42 static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev);
  43 static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  44 static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev);
  45 static int uvd_v7_0_start(struct amdgpu_device *adev);
  46 static void uvd_v7_0_stop(struct amdgpu_device *adev);
  47 static int uvd_v7_0_sriov_start(struct amdgpu_device *adev);
  48
  49 /**
  50  * uvd_v7_0_ring_get_rptr - get read pointer
  51  *
  52  * @ring: amdgpu_ring pointer
  53  *
  54  * Returns the current hardware read pointer
  55  */
  56 static uint64_t uvd_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
  57 {
  58         struct amdgpu_device *adev = ring->adev;
  59
  60         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
  61 }
  62
  63 /**
  64  * uvd_v7_0_enc_ring_get_rptr - get enc read pointer
  65  *
  66  * @ring: amdgpu_ring pointer
  67  *
  68  * Returns the current hardware enc read pointer
  69  */
  70 static uint64_t uvd_v7_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
  71 {
  72         struct amdgpu_device *adev = ring->adev;
  73
  74         if (ring == &adev->uvd.ring_enc[0])
  75                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
  76         else
  77                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
  78 }
  79
  80 /**
  81  * uvd_v7_0_ring_get_wptr - get write pointer
  82  *
  83  * @ring: amdgpu_ring pointer
  84  *
  85  * Returns the current hardware write pointer
  86  */
  87 static uint64_t uvd_v7_0_ring_get_wptr(struct amdgpu_ring *ring)
  88 {
  89         struct amdgpu_device *adev = ring->adev;
  90
  91         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
  92 }
  93
  94 /**
  95  * uvd_v7_0_enc_ring_get_wptr - get enc write pointer
  96  *
  97  * @ring: amdgpu_ring pointer
  98  *
  99  * Returns the current hardware enc write pointer
 100  */
 101 static uint64_t uvd_v7_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 102 {
 103         struct amdgpu_device *adev = ring->adev;
 104
 105         if (ring->use_doorbell)
 106                 return adev->wb.wb[ring->wptr_offs];
 107
 108         if (ring == &adev->uvd.ring_enc[0])
 109                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
 110         else
 111                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
 112 }
 113
 114 /**
 115  * uvd_v7_0_ring_set_wptr - set write pointer
 116  *
 117  * @ring: amdgpu_ring pointer
 118  *
 119  * Commits the write pointer to the hardware
 120  */
 121 static void uvd_v7_0_ring_set_wptr(struct amdgpu_ring *ring)
 122 {
 123         struct amdgpu_device *adev = ring->adev;
 124
 125         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 126 }
 127
 128 /**
 129  * uvd_v7_0_enc_ring_set_wptr - set enc write pointer
 130  *
 131  * @ring: amdgpu_ring pointer
 132  *
 133  * Commits the enc write pointer to the hardware
 134  */
 135 static void uvd_v7_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 136 {
 137         struct amdgpu_device *adev = ring->adev;
 138
 139         if (ring->use_doorbell) {
 140                 /* XXX check if swapping is necessary on BE */
 141                 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
 142                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 143                 return;
 144         }
 145
 146         if (ring == &adev->uvd.ring_enc[0])
 147                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
 148                         lower_32_bits(ring->wptr));
 149         else
 150                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
 151                         lower_32_bits(ring->wptr));
 152 }
 153
 154 /**
 155  * uvd_v7_0_enc_ring_test_ring - test if UVD ENC ring is working
 156  *
 157  * @ring: the engine to test on
 158  *
 159  */
 160 static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 161 {
 162         struct amdgpu_device *adev = ring->adev;
 163         uint32_t rptr = amdgpu_ring_get_rptr(ring);
 164         unsigned i;
 165         int r;
 166
 167         if (amdgpu_sriov_vf(adev))
 168                 return 0;
 169
 170         r = amdgpu_ring_alloc(ring, 16);
 171         if (r) {
 172                 DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
 173                           ring->idx, r);
 174                 return r;
 175         }
 176         amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
 177         amdgpu_ring_commit(ring);
 178
 179         for (i = 0; i < adev->usec_timeout; i++) {
 180                 if (amdgpu_ring_get_rptr(ring) != rptr)
 181                         break;
 182                 DRM_UDELAY(1);
 183         }
 184
 185         if (i < adev->usec_timeout) {
 186                 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 187                          ring->idx, i);
 188         } else {
 189                 DRM_ERROR("amdgpu: ring %d test failed\n",
 190                           ring->idx);
 191                 r = -ETIMEDOUT;
 192         }
 193
 194         return r;
 195 }
 196
 197 /**
 198  * uvd_v7_0_enc_get_create_msg - generate a UVD ENC create msg
 199  *
 200  * @adev: amdgpu_device pointer
 201  * @ring: ring we should submit the msg to
 202  * @handle: session handle to use
 203  * @fence: optional fence to return
 204  *
 205  * Open up a stream for HW test
 206  */
 207 static int uvd_v7_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 208                                        struct dma_fence **fence)
 209 {
 210         const unsigned ib_size_dw = 16;
 211         struct amdgpu_job *job;
 212         struct amdgpu_ib *ib;
 213         struct dma_fence *f = NULL;
 214         uint64_t dummy;
 215         int i, r;
 216
 217         r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
 218         if (r)
 219                 return r;
 220
 221         ib = &job->ibs[0];
 222         dummy = ib->gpu_addr + 1024;
 223
 224         ib->length_dw = 0;
 225         ib->ptr[ib->length_dw++] = 0x00000018;
 226         ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
 227         ib->ptr[ib->length_dw++] = handle;
 228         ib->ptr[ib->length_dw++] = 0x00000000;
 229         ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
 230         ib->ptr[ib->length_dw++] = dummy;
 231
 232         ib->ptr[ib->length_dw++] = 0x00000014;
 233         ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
 234         ib->ptr[ib->length_dw++] = 0x0000001c;
 235         ib->ptr[ib->length_dw++] = 0x00000000;
 236         ib->ptr[ib->length_dw++] = 0x00000000;
 237
 238         ib->ptr[ib->length_dw++] = 0x00000008;
 239         ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
 240
 241         for (i = ib->length_dw; i < ib_size_dw; ++i)
 242                 ib->ptr[i] = 0x0;
 243
 244         r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
 245         job->fence = dma_fence_get(f);
 246         if (r)
 247                 goto err;
 248
 249         amdgpu_job_free(job);
 250         if (fence)
 251                 *fence = dma_fence_get(f);
 252         dma_fence_put(f);
 253         return 0;
 254
 255 err:
 256         amdgpu_job_free(job);
 257         return r;
 258 }
 259
 260 /**
 261  * uvd_v7_0_enc_get_destroy_msg - generate a UVD ENC destroy msg
 262  *
 263  * @adev: amdgpu_device pointer
 264  * @ring: ring we should submit the msg to
 265  * @handle: session handle to use
 266  * @fence: optional fence to return
 267  *
 268  * Close up a stream for HW test or if userspace failed to do so
 269  */
 270 int uvd_v7_0_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 271                                  bool direct, struct dma_fence **fence)
 272 {
 273         const unsigned ib_size_dw = 16;
 274         struct amdgpu_job *job;
 275         struct amdgpu_ib *ib;
 276         struct dma_fence *f = NULL;
 277         uint64_t dummy;
 278         int i, r;
 279
 280         r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
 281         if (r)
 282                 return r;
 283
 284         ib = &job->ibs[0];
 285         dummy = ib->gpu_addr + 1024;
 286
 287         ib->length_dw = 0;
 288         ib->ptr[ib->length_dw++] = 0x00000018;
 289         ib->ptr[ib->length_dw++] = 0x00000001;
 290         ib->ptr[ib->length_dw++] = handle;
 291         ib->ptr[ib->length_dw++] = 0x00000000;
 292         ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
 293         ib->ptr[ib->length_dw++] = dummy;
 294
 295         ib->ptr[ib->length_dw++] = 0x00000014;
 296         ib->ptr[ib->length_dw++] = 0x00000002;
 297         ib->ptr[ib->length_dw++] = 0x0000001c;
 298         ib->ptr[ib->length_dw++] = 0x00000000;
 299         ib->ptr[ib->length_dw++] = 0x00000000;
 300
 301         ib->ptr[ib->length_dw++] = 0x00000008;
 302         ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
 303
 304         for (i = ib->length_dw; i < ib_size_dw; ++i)
 305                 ib->ptr[i] = 0x0;
 306
 307         if (direct) {
 308                 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
 309                 job->fence = dma_fence_get(f);
 310                 if (r)
 311                         goto err;
 312
 313                 amdgpu_job_free(job);
 314         } else {
 315                 r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
 316                                       AMDGPU_FENCE_OWNER_UNDEFINED, &f);
 317                 if (r)
 318                         goto err;
 319         }
 320
 321         if (fence)
 322                 *fence = dma_fence_get(f);
 323         dma_fence_put(f);
 324         return 0;
 325
 326 err:
 327         amdgpu_job_free(job);
 328         return r;
 329 }
 330
 331 /**
 332  * uvd_v7_0_enc_ring_test_ib - test if UVD ENC IBs are working
 333  *
 334  * @ring: the engine to test on
 335  *
 336  */
 337 static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 338 {
 339         struct dma_fence *fence = NULL;
 340         long r;
 341
 342         r = uvd_v7_0_enc_get_create_msg(ring, 1, NULL);
 343         if (r) {
 344                 DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
 345                 goto error;
 346         }
 347
 348         r = uvd_v7_0_enc_get_destroy_msg(ring, 1, true, &fence);
 349         if (r) {
 350                 DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
 351                 goto error;
 352         }
 353
 354         r = dma_fence_wait_timeout(fence, false, timeout);
 355         if (r == 0) {
 356                 DRM_ERROR("amdgpu: IB test timed out.\n");
 357                 r = -ETIMEDOUT;
 358         } else if (r < 0) {
 359                 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 360         } else {
 361                 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 362                 r = 0;
 363         }
 364 error:
 365         dma_fence_put(fence);
 366         return r;
 367 }
 368
 369 static int uvd_v7_0_early_init(void *handle)
 370 {
 371         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 372
 373         if (amdgpu_sriov_vf(adev))
 374                 adev->uvd.num_enc_rings = 1;
 375         else
 376                 adev->uvd.num_enc_rings = 2;
 377         uvd_v7_0_set_ring_funcs(adev);
 378         uvd_v7_0_set_enc_ring_funcs(adev);
 379         uvd_v7_0_set_irq_funcs(adev);
 380
 381         return 0;
 382 }
 383
 384 static int uvd_v7_0_sw_init(void *handle)
 385 {
 386         struct amdgpu_ring *ring;
 387         struct drm_sched_rq *rq;
 388         int i, r;
 389         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 390
 391         /* UVD TRAP */
 392         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_UVD, 124, &adev->uvd.irq);
 393         if (r)
 394                 return r;
 395
 396         /* UVD ENC TRAP */
 397         for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 398                 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_UVD, i + 119, &adev->uvd.irq);
 399                 if (r)
 400                         return r;
 401         }
 402
 403         r = amdgpu_uvd_sw_init(adev);
 404         if (r)
 405                 return r;
 406
 407         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 408                 const struct common_firmware_header *hdr;
 409                 hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
 410                 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].ucode_id = AMDGPU_UCODE_ID_UVD;
 411                 adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].fw = adev->uvd.fw;
 412                 adev->firmware.fw_size +=
 413                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
 414                 DRM_INFO("PSP loading UVD firmware\n");
 415         }
 416
 417         ring = &adev->uvd.ring_enc[0];
 418         rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
 419         r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
 420                                   rq, amdgpu_sched_jobs, NULL);
 421         if (r) {
 422                 DRM_ERROR("Failed setting up UVD ENC run queue.\n");
 423                 return r;
 424         }
 425
 426         r = amdgpu_uvd_resume(adev);
 427         if (r)
 428                 return r;
 429         if (!amdgpu_sriov_vf(adev)) {
 430                 ring = &adev->uvd.ring;
 431                 sprintf(ring->name, "uvd");
 432                 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 433                 if (r)
 434                         return r;
 435         }
 436
 437         for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 438                 ring = &adev->uvd.ring_enc[i];
 439                 sprintf(ring->name, "uvd_enc%d", i);
 440                 if (amdgpu_sriov_vf(adev)) {
 441                         ring->use_doorbell = true;
 442
 443                         /* currently only use the first enconding ring for
 444                          * sriov, so set unused location for other unused rings.
 445                          */
 446                         if (i == 0)
 447                                 ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
 448                         else
 449                                 ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING2_3 * 2 + 1;
 450                 }
 451                 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 452                 if (r)
 453                         return r;
 454         }
 455
 456         r = amdgpu_virt_alloc_mm_table(adev);
 457         if (r)
 458                 return r;
 459
 460         return r;
 461 }
 462
 463 static int uvd_v7_0_sw_fini(void *handle)
 464 {
 465         int i, r;
 466         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 467
 468         amdgpu_virt_free_mm_table(adev);
 469
 470         r = amdgpu_uvd_suspend(adev);
 471         if (r)
 472                 return r;
 473
 474         drm_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
 475
 476         for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 477                 amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
 478
 479         return amdgpu_uvd_sw_fini(adev);
 480 }
 481
 482 /**
 483  * uvd_v7_0_hw_init - start and test UVD block
 484  *
 485  * @adev: amdgpu_device pointer
 486  *
 487  * Initialize the hardware, boot up the VCPU and do some testing
 488  */
 489 static int uvd_v7_0_hw_init(void *handle)
 490 {
 491         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 492         struct amdgpu_ring *ring = &adev->uvd.ring;
 493         uint32_t tmp;
 494         int i, r;
 495
 496         if (amdgpu_sriov_vf(adev))
 497                 r = uvd_v7_0_sriov_start(adev);
 498         else
 499                 r = uvd_v7_0_start(adev);
 500         if (r)
 501                 goto done;
 502
 503         if (!amdgpu_sriov_vf(adev)) {
 504                 ring->ready = true;
 505                 r = amdgpu_ring_test_ring(ring);
 506                 if (r) {
 507                         ring->ready = false;
 508                         goto done;
 509                 }
 510
 511                 r = amdgpu_ring_alloc(ring, 10);
 512                 if (r) {
 513                         DRM_ERROR("amdgpu: ring failed to lock UVD ring (%d).\n", r);
 514                         goto done;
 515                 }
 516
 517                 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
 518                         mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0);
 519                 amdgpu_ring_write(ring, tmp);
 520                 amdgpu_ring_write(ring, 0xFFFFF);
 521
 522                 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
 523                         mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0);
 524                 amdgpu_ring_write(ring, tmp);
 525                 amdgpu_ring_write(ring, 0xFFFFF);
 526
 527                 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
 528                         mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0);
 529                 amdgpu_ring_write(ring, tmp);
 530                 amdgpu_ring_write(ring, 0xFFFFF);
 531
 532                 /* Clear timeout status bits */
 533                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
 534                         mmUVD_SEMA_TIMEOUT_STATUS), 0));
 535                 amdgpu_ring_write(ring, 0x8);
 536
 537                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
 538                         mmUVD_SEMA_CNTL), 0));
 539                 amdgpu_ring_write(ring, 3);
 540
 541                 amdgpu_ring_commit(ring);
 542         }
 543
 544         for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 545                 ring = &adev->uvd.ring_enc[i];
 546                 ring->ready = true;
 547                 r = amdgpu_ring_test_ring(ring);
 548                 if (r) {
 549                         ring->ready = false;
 550                         goto done;
 551                 }
 552         }
 553
 554 done:
 555         if (!r)
 556                 DRM_INFO("UVD and UVD ENC initialized successfully.\n");
 557
 558         return r;
 559 }
 560
 561 /**
 562  * uvd_v7_0_hw_fini - stop the hardware block
 563  *
 564  * @adev: amdgpu_device pointer
 565  *
 566  * Stop the UVD block, mark ring as not ready any more
 567  */
 568 static int uvd_v7_0_hw_fini(void *handle)
 569 {
 570         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 571         struct amdgpu_ring *ring = &adev->uvd.ring;
 572
 573         if (!amdgpu_sriov_vf(adev))
 574                 uvd_v7_0_stop(adev);
 575         else {
 576                 /* full access mode, so don't touch any UVD register */
 577                 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
 578         }
 579
 580         ring->ready = false;
 581
 582         return 0;
 583 }
 584
 585 static int uvd_v7_0_suspend(void *handle)
 586 {
 587         int r;
 588         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 589
 590         r = uvd_v7_0_hw_fini(adev);
 591         if (r)
 592                 return r;
 593
 594         return amdgpu_uvd_suspend(adev);
 595 }
 596
 597 static int uvd_v7_0_resume(void *handle)
 598 {
 599         int r;
 600         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 601
 602         r = amdgpu_uvd_resume(adev);
 603         if (r)
 604                 return r;
 605
 606         return uvd_v7_0_hw_init(adev);
 607 }
 608
 609 /**
 610  * uvd_v7_0_mc_resume - memory controller programming
 611  *
 612  * @adev: amdgpu_device pointer
 613  *
 614  * Let the UVD memory controller know it's offsets
 615  */
 616 static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
 617 {
 618         uint32_t size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 619         uint32_t offset;
 620
 621         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 622                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 623                         lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
 624                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 625                         upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
 626                 offset = 0;
 627         } else {
 628                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 629                         lower_32_bits(adev->uvd.gpu_addr));
 630                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 631                         upper_32_bits(adev->uvd.gpu_addr));
 632                 offset = size;
 633         }
 634
 635         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 636                                 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 637         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 638
 639         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 640                         lower_32_bits(adev->uvd.gpu_addr + offset));
 641         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 642                         upper_32_bits(adev->uvd.gpu_addr + offset));
 643         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, (1 << 21));
 644         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_UVD_HEAP_SIZE);
 645
 646         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 647                         lower_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
 648         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 649                         upper_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
 650         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, (2 << 21));
 651         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
 652                         AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
 653
 654         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 655                         adev->gfx.config.gb_addr_config);
 656         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 657                         adev->gfx.config.gb_addr_config);
 658         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 659                         adev->gfx.config.gb_addr_config);
 660
 661         WREG32_SOC15(UVD, 0, mmUVD_GP_SCRATCH4, adev->uvd.max_handles);
 662 }
 663
 664 static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 665                                 struct amdgpu_mm_table *table)
 666 {
 667         uint32_t data = 0, loop;
 668         uint64_t addr = table->gpu_addr;
 669         struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;
 670         uint32_t size;
 671
 672         size = header->header_size + header->vce_table_size + header->uvd_table_size;
 673
 674         /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */
 675         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
 676         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
 677
 678         /* 2, update vmid of descriptor */
 679         data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_VMID);
 680         data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;
 681         data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */
 682         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_VMID, data);
 683
 684         /* 3, notify mmsch about the size of this descriptor */
 685         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE, size);
 686
 687         /* 4, set resp to zero */
 688         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP, 0);
 689
 690         WDOORBELL32(adev->uvd.ring_enc[0].doorbell_index, 0);
 691         adev->wb.wb[adev->uvd.ring_enc[0].wptr_offs] = 0;
 692         adev->uvd.ring_enc[0].wptr = 0;
 693         adev->uvd.ring_enc[0].wptr_old = 0;
 694
 695         /* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
 696         WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST, 0x10000001);
 697
 698         data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP);
 699         loop = 1000;
 700         while ((data & 0x10000002) != 0x10000002) {
 701                 udelay(10);
 702                 data = RREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP);
 703                 loop--;
 704                 if (!loop)
 705                         break;
 706         }
 707
 708         if (!loop) {
 709                 dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
 710                 return -EBUSY;
 711         }
 712
 713         return 0;
 714 }
 715
 716 static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 717 {
 718         struct amdgpu_ring *ring;
 719         uint32_t offset, size, tmp;
 720         uint32_t table_size = 0;
 721         struct mmsch_v1_0_cmd_direct_write direct_wt = { {0} };
 722         struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} };
 723         struct mmsch_v1_0_cmd_direct_polling direct_poll = { {0} };
 724         struct mmsch_v1_0_cmd_end end = { {0} };
 725         uint32_t *init_table = adev->virt.mm_table.cpu_addr;
 726         struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;
 727
 728         direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
 729         direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
 730         direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;
 731         end.cmd_header.command_type = MMSCH_COMMAND__END;
 732
 733         if (header->uvd_table_offset == 0 && header->uvd_table_size == 0) {
 734                 header->version = MMSCH_VERSION;
 735                 header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;
 736
 737                 if (header->vce_table_offset == 0 && header->vce_table_size == 0)
 738                         header->uvd_table_offset = header->header_size;
 739                 else
 740                         header->uvd_table_offset = header->vce_table_size + header->vce_table_offset;
 741
 742                 init_table += header->uvd_table_offset;
 743
 744                 ring = &adev->uvd.ring;
 745                 ring->wptr = 0;
 746                 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
 747
 748                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
 749                                                    0xFFFFFFFF, 0x00000004);
 750                 /* mc resume*/
 751                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 752                         MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 753                                                     lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
 754                         MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 755                                                     upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
 756                         offset = 0;
 757                 } else {
 758                         MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 759                                                     lower_32_bits(adev->uvd.gpu_addr));
 760                         MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 761                                                     upper_32_bits(adev->uvd.gpu_addr));
 762                         offset = size;
 763                 }
 764
 765                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
 766                                             AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 767                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size);
 768
 769                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 770                                             lower_32_bits(adev->uvd.gpu_addr + offset));
 771                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 772                                             upper_32_bits(adev->uvd.gpu_addr + offset));
 773                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), (1 << 21));
 774                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_UVD_HEAP_SIZE);
 775
 776                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 777                                             lower_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
 778                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 779                                             upper_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
 780                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), (2 << 21));
 781                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2),
 782                                             AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
 783
 784                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
 785                 /* mc resume end*/
 786
 787                 /* disable clock gating */
 788                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_CGC_CTRL),
 789                                                    ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK, 0);
 790
 791                 /* disable interupt */
 792                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 793                                                    ~UVD_MASTINT_EN__VCPU_EN_MASK, 0);
 794
 795                 /* stall UMC and register bus before resetting VCPU */
 796                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 797                                                    ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 798                                                    UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 799
 800                 /* put LMI, VCPU, RBC etc... into reset */
 801                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 802                                             (uint32_t)(UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
 803                                                        UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
 804                                                        UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
 805                                                        UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
 806                                                        UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
 807                                                        UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
 808                                                        UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
 809                                                        UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK));
 810
 811                 /* initialize UVD memory controller */
 812                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL),
 813                                             (uint32_t)((0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 814                                                        UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 815                                                        UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 816                                                        UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 817                                                        UVD_LMI_CTRL__REQ_MODE_MASK |
 818                                                        0x00100000L));
 819
 820                 /* take all subblocks out of reset, except VCPU */
 821                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 822                                             UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 823
 824                 /* enable VCPU clock */
 825                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
 826                                             UVD_VCPU_CNTL__CLK_EN_MASK);
 827
 828                 /* enable master interrupt */
 829                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 830                                                    ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
 831                                                    (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 832
 833                 /* clear the bit 4 of UVD_STATUS */
 834                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
 835                                                    ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT), 0);
 836
 837                 /* force RBC into idle state */
 838                 size = order_base_2(ring->ring_size);
 839                 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);
 840                 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 841                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp);
 842
 843                 ring = &adev->uvd.ring_enc[0];
 844                 ring->wptr = 0;
 845                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), ring->gpu_addr);
 846                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
 847                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4);
 848
 849                 /* boot up the VCPU */
 850                 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0);
 851
 852                 /* enable UMC */
 853                 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 854                                                                                    ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
 855
 856                 MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0x02, 0x02);
 857
 858                 /* add end packet */
 859                 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
 860                 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
 861                 header->uvd_table_size = table_size;
 862
 863         }
 864         return uvd_v7_0_mmsch_start(adev, &adev->virt.mm_table);
 865 }
 866
 867 /**
 868  * uvd_v7_0_start - start UVD block
 869  *
 870  * @adev: amdgpu_device pointer
 871  *
 872  * Setup and start the UVD block
 873  */
 874 static int uvd_v7_0_start(struct amdgpu_device *adev)
 875 {
 876         struct amdgpu_ring *ring = &adev->uvd.ring;
 877         uint32_t rb_bufsz, tmp;
 878         uint32_t lmi_swap_cntl;
 879         uint32_t mp_swap_cntl;
 880         int i, j, r;
 881
 882         /* disable DPG */
 883         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
 884                         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
 885
 886         /* disable byte swapping */
 887         lmi_swap_cntl = 0;
 888         mp_swap_cntl = 0;
 889
 890         uvd_v7_0_mc_resume(adev);
 891
 892         /* disable clock gating */
 893         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_CGC_CTRL), 0,
 894                         ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK);
 895
 896         /* disable interupt */
 897         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 898                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 899
 900         /* stall UMC and register bus before resetting VCPU */
 901         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
 902                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
 903                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 904         mdelay(1);
 905
 906         /* put LMI, VCPU, RBC etc... into reset */
 907         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 908                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
 909                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
 910                 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
 911                 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
 912                 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
 913                 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
 914                 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
 915                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
 916         mdelay(5);
 917
 918         /* initialize UVD memory controller */
 919         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL,
 920                 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 921                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 922                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 923                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 924                 UVD_LMI_CTRL__REQ_MODE_MASK |
 925                 0x00100000L);
 926
 927 #ifdef __BIG_ENDIAN
 928         /* swap (8 in 32) RB and IB */
 929         lmi_swap_cntl = 0xa;
 930         mp_swap_cntl = 0;
 931 #endif
 932         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 933         WREG32_SOC15(UVD, 0, mmUVD_MP_SWAP_CNTL, mp_swap_cntl);
 934
 935         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040);
 936         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0);
 937         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040);
 938         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0);
 939         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_ALU, 0);
 940         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 0x88);
 941
 942         /* take all subblocks out of reset, except VCPU */
 943         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
 944                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 945         mdelay(5);
 946
 947         /* enable VCPU clock */
 948         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL,
 949                         UVD_VCPU_CNTL__CLK_EN_MASK);
 950
 951         /* enable UMC */
 952         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 953                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 954
 955         /* boot up the VCPU */
 956         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, 0);
 957         mdelay(10);
 958
 959         for (i = 0; i < 10; ++i) {
 960                 uint32_t status;
 961
 962                 for (j = 0; j < 100; ++j) {
 963                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
 964                         if (status & 2)
 965                                 break;
 966                         mdelay(10);
 967                 }
 968                 r = 0;
 969                 if (status & 2)
 970                         break;
 971
 972                 DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
 973                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 974                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
 975                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 976                 mdelay(10);
 977                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 978                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 979                 mdelay(10);
 980                 r = -1;
 981         }
 982
 983         if (r) {
 984                 DRM_ERROR("UVD not responding, giving up!!!\n");
 985                 return r;
 986         }
 987         /* enable master interrupt */
 988         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 989                 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
 990                 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 991
 992         /* clear the bit 4 of UVD_STATUS */
 993         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
 994                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 995
 996         /* force RBC into idle state */
 997         rb_bufsz = order_base_2(ring->ring_size);
 998         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 999         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1000         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1001         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
1002         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1003         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1004         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1005
1006         /* set the write pointer delay */
1007         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
1008
1009         /* set the wb address */
1010         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
1011                         (upper_32_bits(ring->gpu_addr) >> 2));
1012
1013         /* programm the RB_BASE for ring buffer */
1014         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1015                         lower_32_bits(ring->gpu_addr));
1016         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1017                         upper_32_bits(ring->gpu_addr));
1018
1019         /* Initialize the ring buffer's read and write pointers */
1020         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1021
1022         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1023         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1024                         lower_32_bits(ring->wptr));
1025
1026         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
1027                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
1028
1029         ring = &adev->uvd.ring_enc[0];
1030         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1031         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1032         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1033         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1034         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1035
1036         ring = &adev->uvd.ring_enc[1];
1037         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1038         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1039         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1040         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1041         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1042
1043         return 0;
1044 }
1045
1046 /**
1047  * uvd_v7_0_stop - stop UVD block
1048  *
1049  * @adev: amdgpu_device pointer
1050  *
1051  * stop the UVD block
1052  */
1053 static void uvd_v7_0_stop(struct amdgpu_device *adev)
1054 {
1055         /* force RBC into idle state */
1056         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, 0x11010101);
1057
1058         /* Stall UMC and register bus before resetting VCPU */
1059         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
1060                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
1061                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1062         mdelay(1);
1063
1064         /* put VCPU into reset */
1065         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
1066                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1067         mdelay(5);
1068
1069         /* disable VCPU clock */
1070         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, 0x0);
1071
1072         /* Unstall UMC and register bus */
1073         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
1074                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1075 }
1076
1077 /**
1078  * uvd_v7_0_ring_emit_fence - emit an fence & trap command
1079  *
1080  * @ring: amdgpu_ring pointer
1081  * @fence: fence to emit
1082  *
1083  * Write a fence and a trap command to the ring.
1084  */
1085 static void uvd_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1086                                      unsigned flags)
1087 {
1088         struct amdgpu_device *adev = ring->adev;
1089
1090         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1091
1092         amdgpu_ring_write(ring,
1093                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
1094         amdgpu_ring_write(ring, seq);
1095         amdgpu_ring_write(ring,
1096                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1097         amdgpu_ring_write(ring, addr & 0xffffffff);
1098         amdgpu_ring_write(ring,
1099                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1100         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1101         amdgpu_ring_write(ring,
1102                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1103         amdgpu_ring_write(ring, 0);
1104
1105         amdgpu_ring_write(ring,
1106                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1107         amdgpu_ring_write(ring, 0);
1108         amdgpu_ring_write(ring,
1109                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1110         amdgpu_ring_write(ring, 0);
1111         amdgpu_ring_write(ring,
1112                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1113         amdgpu_ring_write(ring, 2);
1114 }
1115
1116 /**
1117  * uvd_v7_0_enc_ring_emit_fence - emit an enc fence & trap command
1118  *
1119  * @ring: amdgpu_ring pointer
1120  * @fence: fence to emit
1121  *
1122  * Write enc a fence and a trap command to the ring.
1123  */
1124 static void uvd_v7_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1125                         u64 seq, unsigned flags)
1126 {
1127
1128         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1129
1130         amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
1131         amdgpu_ring_write(ring, addr);
1132         amdgpu_ring_write(ring, upper_32_bits(addr));
1133         amdgpu_ring_write(ring, seq);
1134         amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
1135 }
1136
1137 /**
1138  * uvd_v7_0_ring_emit_hdp_flush - emit an hdp flush
1139  *
1140  * @ring: amdgpu_ring pointer
1141  *
1142  * Emits an hdp flush.
1143  */
1144 static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
1145 {
1146         struct amdgpu_device *adev = ring->adev;
1147
1148         amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(NBIF, 0,
1149                 mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0));
1150         amdgpu_ring_write(ring, 0);
1151 }
1152
1153 /**
1154  * uvd_v7_0_ring_hdp_invalidate - emit an hdp invalidate
1155  *
1156  * @ring: amdgpu_ring pointer
1157  *
1158  * Emits an hdp invalidate.
1159  */
1160 static void uvd_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
1161 {
1162         struct amdgpu_device *adev = ring->adev;
1163
1164         amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
1165         amdgpu_ring_write(ring, 1);
1166 }
1167
1168 /**
1169  * uvd_v7_0_ring_test_ring - register write test
1170  *
1171  * @ring: amdgpu_ring pointer
1172  *
1173  * Test if we can successfully write to the context register
1174  */
1175 static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
1176 {
1177         struct amdgpu_device *adev = ring->adev;
1178         uint32_t tmp = 0;
1179         unsigned i;
1180         int r;
1181
1182         WREG32_SOC15(UVD, 0, mmUVD_CONTEXT_ID, 0xCAFEDEAD);
1183         r = amdgpu_ring_alloc(ring, 3);
1184         if (r) {
1185                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
1186                           ring->idx, r);
1187                 return r;
1188         }
1189         amdgpu_ring_write(ring,
1190                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
1191         amdgpu_ring_write(ring, 0xDEADBEEF);
1192         amdgpu_ring_commit(ring);
1193         for (i = 0; i < adev->usec_timeout; i++) {
1194                 tmp = RREG32_SOC15(UVD, 0, mmUVD_CONTEXT_ID);
1195                 if (tmp == 0xDEADBEEF)
1196                         break;
1197                 DRM_UDELAY(1);
1198         }
1199
1200         if (i < adev->usec_timeout) {
1201                 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
1202                          ring->idx, i);
1203         } else {
1204                 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
1205                           ring->idx, tmp);
1206                 r = -EINVAL;
1207         }
1208         return r;
1209 }
1210
1211 /**
1212  * uvd_v7_0_ring_emit_ib - execute indirect buffer
1213  *
1214  * @ring: amdgpu_ring pointer
1215  * @ib: indirect buffer to execute
1216  *
1217  * Write ring commands to execute the indirect buffer
1218  */
1219 static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
1220                                   struct amdgpu_ib *ib,
1221                                   unsigned vmid, bool ctx_switch)
1222 {
1223         struct amdgpu_device *adev = ring->adev;
1224
1225         amdgpu_ring_write(ring,
1226                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
1227         amdgpu_ring_write(ring, vmid);
1228
1229         amdgpu_ring_write(ring,
1230                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
1231         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1232         amdgpu_ring_write(ring,
1233                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
1234         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1235         amdgpu_ring_write(ring,
1236                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
1237         amdgpu_ring_write(ring, ib->length_dw);
1238 }
1239
1240 /**
1241  * uvd_v7_0_enc_ring_emit_ib - enc execute indirect buffer
1242  *
1243  * @ring: amdgpu_ring pointer
1244  * @ib: indirect buffer to execute
1245  *
1246  * Write enc ring commands to execute the indirect buffer
1247  */
1248 static void uvd_v7_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1249                 struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
1250 {
1251         amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
1252         amdgpu_ring_write(ring, vmid);
1253         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1254         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1255         amdgpu_ring_write(ring, ib->length_dw);
1256 }
1257
1258 static void uvd_v7_0_vm_reg_write(struct amdgpu_ring *ring,
1259                                 uint32_t data0, uint32_t data1)
1260 {
1261         struct amdgpu_device *adev = ring->adev;
1262
1263         amdgpu_ring_write(ring,
1264                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1265         amdgpu_ring_write(ring, data0);
1266         amdgpu_ring_write(ring,
1267                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1268         amdgpu_ring_write(ring, data1);
1269         amdgpu_ring_write(ring,
1270                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1271         amdgpu_ring_write(ring, 8);
1272 }
1273
1274 static void uvd_v7_0_vm_reg_wait(struct amdgpu_ring *ring,
1275                                 uint32_t data0, uint32_t data1, uint32_t mask)
1276 {
1277         struct amdgpu_device *adev = ring->adev;
1278
1279         amdgpu_ring_write(ring,
1280                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1281         amdgpu_ring_write(ring, data0);
1282         amdgpu_ring_write(ring,
1283                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1284         amdgpu_ring_write(ring, data1);
1285         amdgpu_ring_write(ring,
1286                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
1287         amdgpu_ring_write(ring, mask);
1288         amdgpu_ring_write(ring,
1289                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1290         amdgpu_ring_write(ring, 12);
1291 }
1292
1293 static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1294                                         unsigned vmid, uint64_t pd_addr)
1295 {
1296         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1297         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
1298         uint64_t flags = AMDGPU_PTE_VALID;
1299         unsigned eng = ring->vm_inv_eng;
1300         uint32_t data0, data1, mask;
1301
1302         amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
1303         pd_addr |= flags;
1304
1305         data0 = (hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2;
1306         data1 = upper_32_bits(pd_addr);
1307         uvd_v7_0_vm_reg_write(ring, data0, data1);
1308
1309         data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
1310         data1 = lower_32_bits(pd_addr);
1311         uvd_v7_0_vm_reg_write(ring, data0, data1);
1312
1313         data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
1314         data1 = lower_32_bits(pd_addr);
1315         mask = 0xffffffff;
1316         uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
1317
1318         /* flush TLB */
1319         data0 = (hub->vm_inv_eng0_req + eng) << 2;
1320         data1 = req;
1321         uvd_v7_0_vm_reg_write(ring, data0, data1);
1322
1323         /* wait for flush */
1324         data0 = (hub->vm_inv_eng0_ack + eng) << 2;
1325         data1 = 1 << vmid;
1326         mask =  1 << vmid;
1327         uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
1328 }
1329
1330 static void uvd_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1331 {
1332         int i;
1333         struct amdgpu_device *adev = ring->adev;
1334
1335         for (i = 0; i < count; i++)
1336                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
1337
1338 }
1339
1340 static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1341 {
1342         amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
1343 }
1344
1345 static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1346                          unsigned int vmid, uint64_t pd_addr)
1347 {
1348         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1349         uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
1350         uint64_t flags = AMDGPU_PTE_VALID;
1351         unsigned eng = ring->vm_inv_eng;
1352
1353         amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
1354         pd_addr |= flags;
1355
1356         amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1357         amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2);
1358         amdgpu_ring_write(ring, upper_32_bits(pd_addr));
1359
1360         amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1361         amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
1362         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1363
1364         amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
1365         amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
1366         amdgpu_ring_write(ring, 0xffffffff);
1367         amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1368
1369         /* flush TLB */
1370         amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1371         amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2);
1372         amdgpu_ring_write(ring, req);
1373
1374         /* wait for flush */
1375         amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
1376         amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1377         amdgpu_ring_write(ring, 1 << vmid);
1378         amdgpu_ring_write(ring, 1 << vmid);
1379 }
1380
1381 #if 0
1382 static bool uvd_v7_0_is_idle(void *handle)
1383 {
1384         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1385
1386         return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);
1387 }
1388
1389 static int uvd_v7_0_wait_for_idle(void *handle)
1390 {
1391         unsigned i;
1392         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1393
1394         for (i = 0; i < adev->usec_timeout; i++) {
1395                 if (uvd_v7_0_is_idle(handle))
1396                         return 0;
1397         }
1398         return -ETIMEDOUT;
1399 }
1400
1401 #define AMDGPU_UVD_STATUS_BUSY_MASK    0xfd
1402 static bool uvd_v7_0_check_soft_reset(void *handle)
1403 {
1404         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1405         u32 srbm_soft_reset = 0;
1406         u32 tmp = RREG32(mmSRBM_STATUS);
1407
1408         if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
1409             REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
1410             (RREG32_SOC15(UVD, 0, mmUVD_STATUS) &
1411                     AMDGPU_UVD_STATUS_BUSY_MASK))
1412                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1413                                 SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
1414
1415         if (srbm_soft_reset) {
1416                 adev->uvd.srbm_soft_reset = srbm_soft_reset;
1417                 return true;
1418         } else {
1419                 adev->uvd.srbm_soft_reset = 0;
1420                 return false;
1421         }
1422 }
1423
1424 static int uvd_v7_0_pre_soft_reset(void *handle)
1425 {
1426         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1427
1428         if (!adev->uvd.srbm_soft_reset)
1429                 return 0;
1430
1431         uvd_v7_0_stop(adev);
1432         return 0;
1433 }
1434
1435 static int uvd_v7_0_soft_reset(void *handle)
1436 {
1437         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1438         u32 srbm_soft_reset;
1439
1440         if (!adev->uvd.srbm_soft_reset)
1441                 return 0;
1442         srbm_soft_reset = adev->uvd.srbm_soft_reset;
1443
1444         if (srbm_soft_reset) {
1445                 u32 tmp;
1446
1447                 tmp = RREG32(mmSRBM_SOFT_RESET);
1448                 tmp |= srbm_soft_reset;
1449                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1450                 WREG32(mmSRBM_SOFT_RESET, tmp);
1451                 tmp = RREG32(mmSRBM_SOFT_RESET);
1452
1453                 udelay(50);
1454
1455                 tmp &= ~srbm_soft_reset;
1456                 WREG32(mmSRBM_SOFT_RESET, tmp);
1457                 tmp = RREG32(mmSRBM_SOFT_RESET);
1458
1459                 /* Wait a little for things to settle down */
1460                 udelay(50);
1461         }
1462
1463         return 0;
1464 }
1465
1466 static int uvd_v7_0_post_soft_reset(void *handle)
1467 {
1468         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1469
1470         if (!adev->uvd.srbm_soft_reset)
1471                 return 0;
1472
1473         mdelay(5);
1474
1475         return uvd_v7_0_start(adev);
1476 }
1477 #endif
1478
1479 static int uvd_v7_0_set_interrupt_state(struct amdgpu_device *adev,
1480                                         struct amdgpu_irq_src *source,
1481                                         unsigned type,
1482                                         enum amdgpu_interrupt_state state)
1483 {
1484         // TODO
1485         return 0;
1486 }
1487
1488 static int uvd_v7_0_process_interrupt(struct amdgpu_device *adev,
1489                                       struct amdgpu_irq_src *source,
1490                                       struct amdgpu_iv_entry *entry)
1491 {
1492         DRM_DEBUG("IH: UVD TRAP\n");
1493         switch (entry->src_id) {
1494         case 124:
1495                 amdgpu_fence_process(&adev->uvd.ring);
1496                 break;
1497         case 119:
1498                 amdgpu_fence_process(&adev->uvd.ring_enc[0]);
1499                 break;
1500         case 120:
1501                 if (!amdgpu_sriov_vf(adev))
1502                         amdgpu_fence_process(&adev->uvd.ring_enc[1]);
1503                 break;
1504         default:
1505                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1506                           entry->src_id, entry->src_data[0]);
1507                 break;
1508         }
1509
1510         return 0;
1511 }
1512
1513 #if 0
1514 static void uvd_v7_0_set_sw_clock_gating(struct amdgpu_device *adev)
1515 {
1516         uint32_t data, data1, data2, suvd_flags;
1517
1518         data = RREG32_SOC15(UVD, 0, mmUVD_CGC_CTRL);
1519         data1 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE);
1520         data2 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_CTRL);
1521
1522         data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
1523                   UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
1524
1525         suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |
1526                      UVD_SUVD_CGC_GATE__SIT_MASK |
1527                      UVD_SUVD_CGC_GATE__SMP_MASK |
1528                      UVD_SUVD_CGC_GATE__SCM_MASK |
1529                      UVD_SUVD_CGC_GATE__SDB_MASK;
1530
1531         data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
1532                 (1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER)) |
1533                 (4 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_OFF_DELAY));
1534
1535         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
1536                         UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
1537                         UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
1538                         UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
1539                         UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
1540                         UVD_CGC_CTRL__SYS_MODE_MASK |
1541                         UVD_CGC_CTRL__UDEC_MODE_MASK |
1542                         UVD_CGC_CTRL__MPEG2_MODE_MASK |
1543                         UVD_CGC_CTRL__REGS_MODE_MASK |
1544                         UVD_CGC_CTRL__RBC_MODE_MASK |
1545                         UVD_CGC_CTRL__LMI_MC_MODE_MASK |
1546                         UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
1547                         UVD_CGC_CTRL__IDCT_MODE_MASK |
1548                         UVD_CGC_CTRL__MPRD_MODE_MASK |
1549                         UVD_CGC_CTRL__MPC_MODE_MASK |
1550                         UVD_CGC_CTRL__LBSI_MODE_MASK |
1551                         UVD_CGC_CTRL__LRBBM_MODE_MASK |
1552                         UVD_CGC_CTRL__WCB_MODE_MASK |
1553                         UVD_CGC_CTRL__VCPU_MODE_MASK |
1554                         UVD_CGC_CTRL__JPEG_MODE_MASK |
1555                         UVD_CGC_CTRL__JPEG2_MODE_MASK |
1556                         UVD_CGC_CTRL__SCPU_MODE_MASK);
1557         data2 &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK |
1558                         UVD_SUVD_CGC_CTRL__SIT_MODE_MASK |
1559                         UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
1560                         UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
1561                         UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
1562         data1 |= suvd_flags;
1563
1564         WREG32_SOC15(UVD, 0, mmUVD_CGC_CTRL, data);
1565         WREG32_SOC15(UVD, 0, mmUVD_CGC_GATE, 0);
1566         WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE, data1);
1567         WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_CTRL, data2);
1568 }
1569
1570 static void uvd_v7_0_set_hw_clock_gating(struct amdgpu_device *adev)
1571 {
1572         uint32_t data, data1, cgc_flags, suvd_flags;
1573
1574         data = RREG32_SOC15(UVD, 0, mmUVD_CGC_GATE);
1575         data1 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE);
1576
1577         cgc_flags = UVD_CGC_GATE__SYS_MASK |
1578                 UVD_CGC_GATE__UDEC_MASK |
1579                 UVD_CGC_GATE__MPEG2_MASK |
1580                 UVD_CGC_GATE__RBC_MASK |
1581                 UVD_CGC_GATE__LMI_MC_MASK |
1582                 UVD_CGC_GATE__IDCT_MASK |
1583                 UVD_CGC_GATE__MPRD_MASK |
1584                 UVD_CGC_GATE__MPC_MASK |
1585                 UVD_CGC_GATE__LBSI_MASK |
1586                 UVD_CGC_GATE__LRBBM_MASK |
1587                 UVD_CGC_GATE__UDEC_RE_MASK |
1588                 UVD_CGC_GATE__UDEC_CM_MASK |
1589                 UVD_CGC_GATE__UDEC_IT_MASK |
1590                 UVD_CGC_GATE__UDEC_DB_MASK |
1591                 UVD_CGC_GATE__UDEC_MP_MASK |
1592                 UVD_CGC_GATE__WCB_MASK |
1593                 UVD_CGC_GATE__VCPU_MASK |
1594                 UVD_CGC_GATE__SCPU_MASK |
1595                 UVD_CGC_GATE__JPEG_MASK |
1596                 UVD_CGC_GATE__JPEG2_MASK;
1597
1598         suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |
1599                                 UVD_SUVD_CGC_GATE__SIT_MASK |
1600                                 UVD_SUVD_CGC_GATE__SMP_MASK |
1601                                 UVD_SUVD_CGC_GATE__SCM_MASK |
1602                                 UVD_SUVD_CGC_GATE__SDB_MASK;
1603
1604         data |= cgc_flags;
1605         data1 |= suvd_flags;
1606
1607         WREG32_SOC15(UVD, 0, mmUVD_CGC_GATE, data);
1608         WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE, data1);
1609 }
1610
1611 static void uvd_v7_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
1612 {
1613         u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
1614
1615         if (enable)
1616                 tmp |= (GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |
1617                         GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);
1618         else
1619                 tmp &= ~(GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |
1620                          GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);
1621
1622         WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
1623 }
1624
1625
1626 static int uvd_v7_0_set_clockgating_state(void *handle,
1627                                           enum amd_clockgating_state state)
1628 {
1629         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1630         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1631
1632         uvd_v7_0_set_bypass_mode(adev, enable);
1633
1634         if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG))
1635                 return 0;
1636
1637         if (enable) {
1638                 /* disable HW gating and enable Sw gating */
1639                 uvd_v7_0_set_sw_clock_gating(adev);
1640         } else {
1641                 /* wait for STATUS to clear */
1642                 if (uvd_v7_0_wait_for_idle(handle))
1643                         return -EBUSY;
1644
1645                 /* enable HW gates because UVD is idle */
1646                 /* uvd_v7_0_set_hw_clock_gating(adev); */
1647         }
1648
1649         return 0;
1650 }
1651
1652 static int uvd_v7_0_set_powergating_state(void *handle,
1653                                           enum amd_powergating_state state)
1654 {
1655         /* This doesn't actually powergate the UVD block.
1656          * That's done in the dpm code via the SMC.  This
1657          * just re-inits the block as necessary.  The actual
1658          * gating still happens in the dpm code.  We should
1659          * revisit this when there is a cleaner line between
1660          * the smc and the hw blocks
1661          */
1662         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1663
1664         if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))
1665                 return 0;
1666
1667         WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);
1668
1669         if (state == AMD_PG_STATE_GATE) {
1670                 uvd_v7_0_stop(adev);
1671                 return 0;
1672         } else {
1673                 return uvd_v7_0_start(adev);
1674         }
1675 }
1676 #endif
1677
1678 static int uvd_v7_0_set_clockgating_state(void *handle,
1679                                           enum amd_clockgating_state state)
1680 {
1681         /* needed for driver unload*/
1682         return 0;
1683 }
1684
1685 const struct amd_ip_funcs uvd_v7_0_ip_funcs = {
1686         .name = "uvd_v7_0",
1687         .early_init = uvd_v7_0_early_init,
1688         .late_init = NULL,
1689         .sw_init = uvd_v7_0_sw_init,
1690         .sw_fini = uvd_v7_0_sw_fini,
1691         .hw_init = uvd_v7_0_hw_init,
1692         .hw_fini = uvd_v7_0_hw_fini,
1693         .suspend = uvd_v7_0_suspend,
1694         .resume = uvd_v7_0_resume,
1695         .is_idle = NULL /* uvd_v7_0_is_idle */,
1696         .wait_for_idle = NULL /* uvd_v7_0_wait_for_idle */,
1697         .check_soft_reset = NULL /* uvd_v7_0_check_soft_reset */,
1698         .pre_soft_reset = NULL /* uvd_v7_0_pre_soft_reset */,
1699         .soft_reset = NULL /* uvd_v7_0_soft_reset */,
1700         .post_soft_reset = NULL /* uvd_v7_0_post_soft_reset */,
1701         .set_clockgating_state = uvd_v7_0_set_clockgating_state,
1702         .set_powergating_state = NULL /* uvd_v7_0_set_powergating_state */,
1703 };
1704
1705 static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
1706         .type = AMDGPU_RING_TYPE_UVD,
1707         .align_mask = 0xf,
1708         .nop = PACKET0(0x81ff, 0),
1709         .support_64bit_ptrs = false,
1710         .vmhub = AMDGPU_MMHUB,
1711         .get_rptr = uvd_v7_0_ring_get_rptr,
1712         .get_wptr = uvd_v7_0_ring_get_wptr,
1713         .set_wptr = uvd_v7_0_ring_set_wptr,
1714         .emit_frame_size =
1715                 2 + /* uvd_v7_0_ring_emit_hdp_flush */
1716                 2 + /* uvd_v7_0_ring_emit_hdp_invalidate */
1717                 34 + /* uvd_v7_0_ring_emit_vm_flush */
1718                 14 + 14, /* uvd_v7_0_ring_emit_fence x2 vm fence */
1719         .emit_ib_size = 8, /* uvd_v7_0_ring_emit_ib */
1720         .emit_ib = uvd_v7_0_ring_emit_ib,
1721         .emit_fence = uvd_v7_0_ring_emit_fence,
1722         .emit_vm_flush = uvd_v7_0_ring_emit_vm_flush,
1723         .emit_hdp_flush = uvd_v7_0_ring_emit_hdp_flush,
1724         .emit_hdp_invalidate = uvd_v7_0_ring_emit_hdp_invalidate,
1725         .test_ring = uvd_v7_0_ring_test_ring,
1726         .test_ib = amdgpu_uvd_ring_test_ib,
1727         .insert_nop = uvd_v7_0_ring_insert_nop,
1728         .pad_ib = amdgpu_ring_generic_pad_ib,
1729         .begin_use = amdgpu_uvd_ring_begin_use,
1730         .end_use = amdgpu_uvd_ring_end_use,
1731 };
1732
1733 static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
1734         .type = AMDGPU_RING_TYPE_UVD_ENC,
1735         .align_mask = 0x3f,
1736         .nop = HEVC_ENC_CMD_NO_OP,
1737         .support_64bit_ptrs = false,
1738         .vmhub = AMDGPU_MMHUB,
1739         .get_rptr = uvd_v7_0_enc_ring_get_rptr,
1740         .get_wptr = uvd_v7_0_enc_ring_get_wptr,
1741         .set_wptr = uvd_v7_0_enc_ring_set_wptr,
1742         .emit_frame_size =
1743                 17 + /* uvd_v7_0_enc_ring_emit_vm_flush */
1744                 5 + 5 + /* uvd_v7_0_enc_ring_emit_fence x2 vm fence */
1745                 1, /* uvd_v7_0_enc_ring_insert_end */
1746         .emit_ib_size = 5, /* uvd_v7_0_enc_ring_emit_ib */
1747         .emit_ib = uvd_v7_0_enc_ring_emit_ib,
1748         .emit_fence = uvd_v7_0_enc_ring_emit_fence,
1749         .emit_vm_flush = uvd_v7_0_enc_ring_emit_vm_flush,
1750         .test_ring = uvd_v7_0_enc_ring_test_ring,
1751         .test_ib = uvd_v7_0_enc_ring_test_ib,
1752         .insert_nop = amdgpu_ring_insert_nop,
1753         .insert_end = uvd_v7_0_enc_ring_insert_end,
1754         .pad_ib = amdgpu_ring_generic_pad_ib,
1755         .begin_use = amdgpu_uvd_ring_begin_use,
1756         .end_use = amdgpu_uvd_ring_end_use,
1757 };
1758
1759 static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev)
1760 {
1761         adev->uvd.ring.funcs = &uvd_v7_0_ring_vm_funcs;
1762         DRM_INFO("UVD is enabled in VM mode\n");
1763 }
1764
1765 static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1766 {
1767         int i;
1768
1769         for (i = 0; i < adev->uvd.num_enc_rings; ++i)
1770                 adev->uvd.ring_enc[i].funcs = &uvd_v7_0_enc_ring_vm_funcs;
1771
1772         DRM_INFO("UVD ENC is enabled in VM mode\n");
1773 }
1774
1775 static const struct amdgpu_irq_src_funcs uvd_v7_0_irq_funcs = {
1776         .set = uvd_v7_0_set_interrupt_state,
1777         .process = uvd_v7_0_process_interrupt,
1778 };
1779
1780 static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1781 {
1782         adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
1783         adev->uvd.irq.funcs = &uvd_v7_0_irq_funcs;
1784 }
1785
1786 const struct amdgpu_ip_block_version uvd_v7_0_ip_block =
1787 {
1788                 .type = AMD_IP_BLOCK_TYPE_UVD,
1789                 .major = 7,
1790                 .minor = 0,
1791                 .rev = 0,
1792                 .funcs = &uvd_v7_0_ip_funcs,
1793 };