drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/module.h>
  25
  26 #ifdef CONFIG_X86
  27 #include <asm/hypervisor.h>
  28 #endif
  29
  30 #include <drm/drm_drv.h>
  31 #include <xen/xen.h>
  32
  33 #include "amdgpu.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_reset.h"
  36 #include "vi.h"
  37 #include "soc15.h"
  38 #include "nv.h"
  39
  40 #define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \
  41         do { \
  42                 vf2pf_info->ucode_info[ucode].id = ucode; \
  43                 vf2pf_info->ucode_info[ucode].version = ver; \
  44         } while (0)
  45
  46 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev)
  47 {
  48         /* By now all MMIO pages except mailbox are blocked */
  49         /* if blocking is enabled in hypervisor. Choose the */
  50         /* SCRATCH_REG0 to test. */
  51         return RREG32_NO_KIQ(0xc040) == 0xffffffff;
  52 }
  53
  54 void amdgpu_virt_init_setting(struct amdgpu_device *adev)
  55 {
  56         struct drm_device *ddev = adev_to_drm(adev);
  57
  58         /* enable virtual display */
  59         if (adev->asic_type != CHIP_ALDEBARAN &&
  60             adev->asic_type != CHIP_ARCTURUS &&
  61             ((adev->pdev->class >> 8) != PCI_CLASS_ACCELERATOR_PROCESSING)) {
  62                 if (adev->mode_info.num_crtc == 0)
  63                         adev->mode_info.num_crtc = 1;
  64                 adev->enable_virtual_display = true;
  65         }
  66         ddev->driver_features &= ~DRIVER_ATOMIC;
  67         adev->cg_flags = 0;
  68         adev->pg_flags = 0;
  69
  70         /* Reduce kcq number to 2 to reduce latency */
  71         if (amdgpu_num_kcq == -1)
  72                 amdgpu_num_kcq = 2;
  73 }
  74
  75 /**
  76  * amdgpu_virt_request_full_gpu() - request full gpu access
  77  * @adev:       amdgpu device.
  78  * @init:       is driver init time.
  79  * When start to init/fini driver, first need to request full gpu access.
  80  * Return: Zero if request success, otherwise will return error.
  81  */
  82 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
  83 {
  84         struct amdgpu_virt *virt = &adev->virt;
  85         int r;
  86
  87         if (virt->ops && virt->ops->req_full_gpu) {
  88                 r = virt->ops->req_full_gpu(adev, init);
  89                 if (r)
  90                         return r;
  91
  92                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
  93         }
  94
  95         return 0;
  96 }
  97
  98 /**
  99  * amdgpu_virt_release_full_gpu() - release full gpu access
 100  * @adev:       amdgpu device.
 101  * @init:       is driver init time.
 102  * When finishing driver init/fini, need to release full gpu access.
 103  * Return: Zero if release success, otherwise will returen error.
 104  */
 105 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
 106 {
 107         struct amdgpu_virt *virt = &adev->virt;
 108         int r;
 109
 110         if (virt->ops && virt->ops->rel_full_gpu) {
 111                 r = virt->ops->rel_full_gpu(adev, init);
 112                 if (r)
 113                         return r;
 114
 115                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
 116         }
 117         return 0;
 118 }
 119
 120 /**
 121  * amdgpu_virt_reset_gpu() - reset gpu
 122  * @adev:       amdgpu device.
 123  * Send reset command to GPU hypervisor to reset GPU that VM is using
 124  * Return: Zero if reset success, otherwise will return error.
 125  */
 126 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
 127 {
 128         struct amdgpu_virt *virt = &adev->virt;
 129         int r;
 130
 131         if (virt->ops && virt->ops->reset_gpu) {
 132                 r = virt->ops->reset_gpu(adev);
 133                 if (r)
 134                         return r;
 135
 136                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
 137         }
 138
 139         return 0;
 140 }
 141
 142 void amdgpu_virt_request_init_data(struct amdgpu_device *adev)
 143 {
 144         struct amdgpu_virt *virt = &adev->virt;
 145
 146         if (virt->ops && virt->ops->req_init_data)
 147                 virt->ops->req_init_data(adev);
 148
 149         if (adev->virt.req_init_data_ver > 0)
 150                 DRM_INFO("host supports REQ_INIT_DATA handshake\n");
 151         else
 152                 DRM_WARN("host doesn't support REQ_INIT_DATA handshake\n");
 153 }
 154
 155 /**
 156  * amdgpu_virt_ready_to_reset() - send ready to reset to host
 157  * @adev:       amdgpu device.
 158  * Send ready to reset message to GPU hypervisor to signal we have stopped GPU
 159  * activity and is ready for host FLR
 160  */
 161 void amdgpu_virt_ready_to_reset(struct amdgpu_device *adev)
 162 {
 163         struct amdgpu_virt *virt = &adev->virt;
 164
 165         if (virt->ops && virt->ops->reset_gpu)
 166                 virt->ops->ready_to_reset(adev);
 167 }
 168
 169 /**
 170  * amdgpu_virt_wait_reset() - wait for reset gpu completed
 171  * @adev:       amdgpu device.
 172  * Wait for GPU reset completed.
 173  * Return: Zero if reset success, otherwise will return error.
 174  */
 175 int amdgpu_virt_wait_reset(struct amdgpu_device *adev)
 176 {
 177         struct amdgpu_virt *virt = &adev->virt;
 178
 179         if (!virt->ops || !virt->ops->wait_reset)
 180                 return -EINVAL;
 181
 182         return virt->ops->wait_reset(adev);
 183 }
 184
 185 /**
 186  * amdgpu_virt_alloc_mm_table() - alloc memory for mm table
 187  * @adev:       amdgpu device.
 188  * MM table is used by UVD and VCE for its initialization
 189  * Return: Zero if allocate success.
 190  */
 191 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev)
 192 {
 193         int r;
 194
 195         if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr)
 196                 return 0;
 197
 198         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 199                                     AMDGPU_GEM_DOMAIN_VRAM |
 200                                     AMDGPU_GEM_DOMAIN_GTT,
 201                                     &adev->virt.mm_table.bo,
 202                                     &adev->virt.mm_table.gpu_addr,
 203                                     (void *)&adev->virt.mm_table.cpu_addr);
 204         if (r) {
 205                 DRM_ERROR("failed to alloc mm table and error = %d.\n", r);
 206                 return r;
 207         }
 208
 209         memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE);
 210         DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n",
 211                  adev->virt.mm_table.gpu_addr,
 212                  adev->virt.mm_table.cpu_addr);
 213         return 0;
 214 }
 215
 216 /**
 217  * amdgpu_virt_free_mm_table() - free mm table memory
 218  * @adev:       amdgpu device.
 219  * Free MM table memory
 220  */
 221 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev)
 222 {
 223         if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr)
 224                 return;
 225
 226         amdgpu_bo_free_kernel(&adev->virt.mm_table.bo,
 227                               &adev->virt.mm_table.gpu_addr,
 228                               (void *)&adev->virt.mm_table.cpu_addr);
 229         adev->virt.mm_table.gpu_addr = 0;
 230 }
 231
 232 /**
 233  * amdgpu_virt_rcvd_ras_interrupt() - receive ras interrupt
 234  * @adev:       amdgpu device.
 235  * Check whether host sent RAS error message
 236  * Return: true if found, otherwise false
 237  */
 238 bool amdgpu_virt_rcvd_ras_interrupt(struct amdgpu_device *adev)
 239 {
 240         struct amdgpu_virt *virt = &adev->virt;
 241
 242         if (!virt->ops || !virt->ops->rcvd_ras_intr)
 243                 return false;
 244
 245         return virt->ops->rcvd_ras_intr(adev);
 246 }
 247
 248
 249 unsigned int amd_sriov_msg_checksum(void *obj,
 250                                 unsigned long obj_size,
 251                                 unsigned int key,
 252                                 unsigned int checksum)
 253 {
 254         unsigned int ret = key;
 255         unsigned long i = 0;
 256         unsigned char *pos;
 257
 258         pos = (char *)obj;
 259         /* calculate checksum */
 260         for (i = 0; i < obj_size; ++i)
 261                 ret += *(pos + i);
 262         /* minus the checksum itself */
 263         pos = (char *)&checksum;
 264         for (i = 0; i < sizeof(checksum); ++i)
 265                 ret -= *(pos + i);
 266         return ret;
 267 }
 268
 269 static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev)
 270 {
 271         struct amdgpu_virt *virt = &adev->virt;
 272         struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data;
 273         /* GPU will be marked bad on host if bp count more then 10,
 274          * so alloc 512 is enough.
 275          */
 276         unsigned int align_space = 512;
 277         void *bps = NULL;
 278         struct amdgpu_bo **bps_bo = NULL;
 279
 280         *data = kmalloc(sizeof(struct amdgpu_virt_ras_err_handler_data), GFP_KERNEL);
 281         if (!*data)
 282                 goto data_failure;
 283
 284         bps = kmalloc_array(align_space, sizeof(*(*data)->bps), GFP_KERNEL);
 285         if (!bps)
 286                 goto bps_failure;
 287
 288         bps_bo = kmalloc_array(align_space, sizeof(*(*data)->bps_bo), GFP_KERNEL);
 289         if (!bps_bo)
 290                 goto bps_bo_failure;
 291
 292         (*data)->bps = bps;
 293         (*data)->bps_bo = bps_bo;
 294         (*data)->count = 0;
 295         (*data)->last_reserved = 0;
 296
 297         virt->ras_init_done = true;
 298
 299         return 0;
 300
 301 bps_bo_failure:
 302         kfree(bps);
 303 bps_failure:
 304         kfree(*data);
 305 data_failure:
 306         return -ENOMEM;
 307 }
 308
 309 static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev)
 310 {
 311         struct amdgpu_virt *virt = &adev->virt;
 312         struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
 313         struct amdgpu_bo *bo;
 314         int i;
 315
 316         if (!data)
 317                 return;
 318
 319         for (i = data->last_reserved - 1; i >= 0; i--) {
 320                 bo = data->bps_bo[i];
 321                 if (bo) {
 322                         amdgpu_bo_free_kernel(&bo, NULL, NULL);
 323                         data->bps_bo[i] = bo;
 324                 }
 325                 data->last_reserved = i;
 326         }
 327 }
 328
 329 void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev)
 330 {
 331         struct amdgpu_virt *virt = &adev->virt;
 332         struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
 333
 334         virt->ras_init_done = false;
 335
 336         if (!data)
 337                 return;
 338
 339         amdgpu_virt_ras_release_bp(adev);
 340
 341         kfree(data->bps);
 342         kfree(data->bps_bo);
 343         kfree(data);
 344         virt->virt_eh_data = NULL;
 345 }
 346
 347 static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev,
 348                 struct eeprom_table_record *bps, int pages)
 349 {
 350         struct amdgpu_virt *virt = &adev->virt;
 351         struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
 352
 353         if (!data)
 354                 return;
 355
 356         memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps));
 357         data->count += pages;
 358 }
 359
 360 static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev)
 361 {
 362         struct amdgpu_virt *virt = &adev->virt;
 363         struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
 364         struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
 365         struct ttm_resource_manager *man = &mgr->manager;
 366         struct amdgpu_bo *bo = NULL;
 367         uint64_t bp;
 368         int i;
 369
 370         if (!data)
 371                 return;
 372
 373         for (i = data->last_reserved; i < data->count; i++) {
 374                 bp = data->bps[i].retired_page;
 375
 376                 /* There are two cases of reserve error should be ignored:
 377                  * 1) a ras bad page has been allocated (used by someone);
 378                  * 2) a ras bad page has been reserved (duplicate error injection
 379                  *    for one page);
 380                  */
 381                 if  (ttm_resource_manager_used(man)) {
 382                         amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
 383                                 bp << AMDGPU_GPU_PAGE_SHIFT,
 384                                 AMDGPU_GPU_PAGE_SIZE);
 385                         data->bps_bo[i] = NULL;
 386                 } else {
 387                         if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT,
 388                                                         AMDGPU_GPU_PAGE_SIZE,
 389                                                         &bo, NULL))
 390                                 DRM_DEBUG("RAS WARN: reserve vram for retired page %llx fail\n", bp);
 391                         data->bps_bo[i] = bo;
 392                 }
 393                 data->last_reserved = i + 1;
 394                 bo = NULL;
 395         }
 396 }
 397
 398 static bool amdgpu_virt_ras_check_bad_page(struct amdgpu_device *adev,
 399                 uint64_t retired_page)
 400 {
 401         struct amdgpu_virt *virt = &adev->virt;
 402         struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
 403         int i;
 404
 405         if (!data)
 406                 return true;
 407
 408         for (i = 0; i < data->count; i++)
 409                 if (retired_page == data->bps[i].retired_page)
 410                         return true;
 411
 412         return false;
 413 }
 414
 415 static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev,
 416                 uint64_t bp_block_offset, uint32_t bp_block_size)
 417 {
 418         struct eeprom_table_record bp;
 419         uint64_t retired_page;
 420         uint32_t bp_idx, bp_cnt;
 421         void *vram_usage_va = NULL;
 422
 423         if (adev->mman.fw_vram_usage_va)
 424                 vram_usage_va = adev->mman.fw_vram_usage_va;
 425         else
 426                 vram_usage_va = adev->mman.drv_vram_usage_va;
 427
 428         memset(&bp, 0, sizeof(bp));
 429
 430         if (bp_block_size) {
 431                 bp_cnt = bp_block_size / sizeof(uint64_t);
 432                 for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) {
 433                         retired_page = *(uint64_t *)(vram_usage_va +
 434                                         bp_block_offset + bp_idx * sizeof(uint64_t));
 435                         bp.retired_page = retired_page;
 436
 437                         if (amdgpu_virt_ras_check_bad_page(adev, retired_page))
 438                                 continue;
 439
 440                         amdgpu_virt_ras_add_bps(adev, &bp, 1);
 441
 442                         amdgpu_virt_ras_reserve_bps(adev);
 443                 }
 444         }
 445 }
 446
 447 static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 448 {
 449         struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf;
 450         uint32_t checksum;
 451         uint32_t checkval;
 452
 453         uint32_t i;
 454         uint32_t tmp;
 455
 456         if (adev->virt.fw_reserve.p_pf2vf == NULL)
 457                 return -EINVAL;
 458
 459         if (pf2vf_info->size > 1024) {
 460                 dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size);
 461                 return -EINVAL;
 462         }
 463
 464         switch (pf2vf_info->version) {
 465         case 1:
 466                 checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum;
 467                 checkval = amd_sriov_msg_checksum(
 468                         adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 469                         adev->virt.fw_reserve.checksum_key, checksum);
 470                 if (checksum != checkval) {
 471                         dev_err(adev->dev,
 472                                 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
 473                                 checksum, checkval);
 474                         return -EINVAL;
 475                 }
 476
 477                 adev->virt.gim_feature =
 478                         ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags;
 479                 break;
 480         case 2:
 481                 /* TODO: missing key, need to add it later */
 482                 checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum;
 483                 checkval = amd_sriov_msg_checksum(
 484                         adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 485                         0, checksum);
 486                 if (checksum != checkval) {
 487                         dev_err(adev->dev,
 488                                 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
 489                                 checksum, checkval);
 490                         return -EINVAL;
 491                 }
 492
 493                 adev->virt.vf2pf_update_interval_ms =
 494                         ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms;
 495                 adev->virt.gim_feature =
 496                         ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all;
 497                 adev->virt.reg_access =
 498                         ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all;
 499
 500                 adev->virt.decode_max_dimension_pixels = 0;
 501                 adev->virt.decode_max_frame_pixels = 0;
 502                 adev->virt.encode_max_dimension_pixels = 0;
 503                 adev->virt.encode_max_frame_pixels = 0;
 504                 adev->virt.is_mm_bw_enabled = false;
 505                 for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) {
 506                         tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels;
 507                         adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels);
 508
 509                         tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels;
 510                         adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels);
 511
 512                         tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels;
 513                         adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels);
 514
 515                         tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels;
 516                         adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels);
 517                 }
 518                 if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0))
 519                         adev->virt.is_mm_bw_enabled = true;
 520
 521                 adev->unique_id =
 522                         ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid;
 523                 break;
 524         default:
 525                 dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
 526                 return -EINVAL;
 527         }
 528
 529         /* correct too large or too little interval value */
 530         if (adev->virt.vf2pf_update_interval_ms < 200 || adev->virt.vf2pf_update_interval_ms > 10000)
 531                 adev->virt.vf2pf_update_interval_ms = 2000;
 532
 533         return 0;
 534 }
 535
 536 static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev)
 537 {
 538         struct amd_sriov_msg_vf2pf_info *vf2pf_info;
 539         vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf;
 540
 541         if (adev->virt.fw_reserve.p_vf2pf == NULL)
 542                 return;
 543
 544         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCE,      adev->vce.fw_version);
 545         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_UVD,      adev->uvd.fw_version);
 546         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MC,       adev->gmc.fw_version);
 547         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ME,       adev->gfx.me_fw_version);
 548         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_PFP,      adev->gfx.pfp_fw_version);
 549         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_CE,       adev->gfx.ce_fw_version);
 550         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC,      adev->gfx.rlc_fw_version);
 551         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLC, adev->gfx.rlc_srlc_fw_version);
 552         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLG, adev->gfx.rlc_srlg_fw_version);
 553         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version);
 554         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC,      adev->gfx.mec_fw_version);
 555         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2,     adev->gfx.mec2_fw_version);
 556         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS,      adev->psp.sos.fw_version);
 557         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD,
 558                             adev->psp.asd_context.bin_desc.fw_version);
 559         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS,
 560                             adev->psp.ras_context.context.bin_desc.fw_version);
 561         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI,
 562                             adev->psp.xgmi_context.context.bin_desc.fw_version);
 563         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC,      adev->pm.fw_version);
 564         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA,     adev->sdma.instance[0].fw_version);
 565         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2,    adev->sdma.instance[1].fw_version);
 566         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCN,      adev->vcn.fw_version);
 567         POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_DMCU,     adev->dm.dmcu_fw_version);
 568 }
 569
 570 static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev)
 571 {
 572         struct amd_sriov_msg_vf2pf_info *vf2pf_info;
 573
 574         vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf;
 575
 576         if (adev->virt.fw_reserve.p_vf2pf == NULL)
 577                 return -EINVAL;
 578
 579         memset(vf2pf_info, 0, sizeof(struct amd_sriov_msg_vf2pf_info));
 580
 581         vf2pf_info->header.size = sizeof(struct amd_sriov_msg_vf2pf_info);
 582         vf2pf_info->header.version = AMD_SRIOV_MSG_FW_VRAM_VF2PF_VER;
 583
 584 #ifdef MODULE
 585         if (THIS_MODULE->version != NULL)
 586                 strcpy(vf2pf_info->driver_version, THIS_MODULE->version);
 587         else
 588 #endif
 589                 strcpy(vf2pf_info->driver_version, "N/A");
 590
 591         vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all
 592         vf2pf_info->driver_cert = 0;
 593         vf2pf_info->os_info.all = 0;
 594
 595         vf2pf_info->fb_usage =
 596                 ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) >> 20;
 597         vf2pf_info->fb_vis_usage =
 598                 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr) >> 20;
 599         vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20;
 600         vf2pf_info->fb_vis_size = adev->gmc.visible_vram_size >> 20;
 601
 602         amdgpu_virt_populate_vf2pf_ucode_info(adev);
 603
 604         /* TODO: read dynamic info */
 605         vf2pf_info->gfx_usage = 0;
 606         vf2pf_info->compute_usage = 0;
 607         vf2pf_info->encode_usage = 0;
 608         vf2pf_info->decode_usage = 0;
 609
 610         vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr;
 611         vf2pf_info->mes_info_addr = (uint64_t)adev->mes.resource_1_gpu_addr;
 612
 613         if (adev->mes.resource_1) {
 614                 vf2pf_info->mes_info_size = adev->mes.resource_1->tbo.base.size;
 615         }
 616         vf2pf_info->checksum =
 617                 amd_sriov_msg_checksum(
 618                 vf2pf_info, sizeof(*vf2pf_info), 0, 0);
 619
 620         return 0;
 621 }
 622
 623 static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work)
 624 {
 625         struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work);
 626         int ret;
 627
 628         ret = amdgpu_virt_read_pf2vf_data(adev);
 629         if (ret) {
 630                 adev->virt.vf2pf_update_retry_cnt++;
 631
 632                 if ((amdgpu_virt_rcvd_ras_interrupt(adev) ||
 633                         adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) &&
 634                         amdgpu_sriov_runtime(adev)) {
 635
 636                         amdgpu_ras_set_fed(adev, true);
 637                         if (amdgpu_reset_domain_schedule(adev->reset_domain,
 638                                                         &adev->kfd.reset_work))
 639                                 return;
 640                         else
 641                                 dev_err(adev->dev, "Failed to queue work! at %s", __func__);
 642                 }
 643
 644                 goto out;
 645         }
 646
 647         adev->virt.vf2pf_update_retry_cnt = 0;
 648         amdgpu_virt_write_vf2pf_data(adev);
 649
 650 out:
 651         schedule_delayed_work(&(adev->virt.vf2pf_work), adev->virt.vf2pf_update_interval_ms);
 652 }
 653
 654 void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev)
 655 {
 656         if (adev->virt.vf2pf_update_interval_ms != 0) {
 657                 DRM_INFO("clean up the vf2pf work item\n");
 658                 cancel_delayed_work_sync(&adev->virt.vf2pf_work);
 659                 adev->virt.vf2pf_update_interval_ms = 0;
 660         }
 661 }
 662
 663 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 664 {
 665         adev->virt.fw_reserve.p_pf2vf = NULL;
 666         adev->virt.fw_reserve.p_vf2pf = NULL;
 667         adev->virt.vf2pf_update_interval_ms = 0;
 668         adev->virt.vf2pf_update_retry_cnt = 0;
 669
 670         if (adev->mman.fw_vram_usage_va && adev->mman.drv_vram_usage_va) {
 671                 DRM_WARN("Currently fw_vram and drv_vram should not have values at the same time!");
 672         } else if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) {
 673                 /* go through this logic in ip_init and reset to init workqueue*/
 674                 amdgpu_virt_exchange_data(adev);
 675
 676                 INIT_DELAYED_WORK(&adev->virt.vf2pf_work, amdgpu_virt_update_vf2pf_work_item);
 677                 schedule_delayed_work(&(adev->virt.vf2pf_work), msecs_to_jiffies(adev->virt.vf2pf_update_interval_ms));
 678         } else if (adev->bios != NULL) {
 679                 /* got through this logic in early init stage to get necessary flags, e.g. rlcg_acc related*/
 680                 adev->virt.fw_reserve.p_pf2vf =
 681                         (struct amd_sriov_msg_pf2vf_info_header *)
 682                         (adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
 683
 684                 amdgpu_virt_read_pf2vf_data(adev);
 685         }
 686 }
 687
 688
 689 void amdgpu_virt_exchange_data(struct amdgpu_device *adev)
 690 {
 691         uint64_t bp_block_offset = 0;
 692         uint32_t bp_block_size = 0;
 693         struct amd_sriov_msg_pf2vf_info *pf2vf_v2 = NULL;
 694
 695         if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) {
 696                 if (adev->mman.fw_vram_usage_va) {
 697                         adev->virt.fw_reserve.p_pf2vf =
 698                                 (struct amd_sriov_msg_pf2vf_info_header *)
 699                                 (adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
 700                         adev->virt.fw_reserve.p_vf2pf =
 701                                 (struct amd_sriov_msg_vf2pf_info_header *)
 702                                 (adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10));
 703                 } else if (adev->mman.drv_vram_usage_va) {
 704                         adev->virt.fw_reserve.p_pf2vf =
 705                                 (struct amd_sriov_msg_pf2vf_info_header *)
 706                                 (adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
 707                         adev->virt.fw_reserve.p_vf2pf =
 708                                 (struct amd_sriov_msg_vf2pf_info_header *)
 709                                 (adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10));
 710                 }
 711
 712                 amdgpu_virt_read_pf2vf_data(adev);
 713                 amdgpu_virt_write_vf2pf_data(adev);
 714
 715                 /* bad page handling for version 2 */
 716                 if (adev->virt.fw_reserve.p_pf2vf->version == 2) {
 717                         pf2vf_v2 = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf;
 718
 719                         bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_low & 0xFFFFFFFF) |
 720                                 ((((uint64_t)pf2vf_v2->bp_block_offset_high) << 32) & 0xFFFFFFFF00000000);
 721                         bp_block_size = pf2vf_v2->bp_block_size;
 722
 723                         if (bp_block_size && !adev->virt.ras_init_done)
 724                                 amdgpu_virt_init_ras_err_handler_data(adev);
 725
 726                         if (adev->virt.ras_init_done)
 727                                 amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size);
 728                 }
 729         }
 730 }
 731
 732 void amdgpu_detect_virtualization(struct amdgpu_device *adev)
 733 {
 734         uint32_t reg;
 735
 736         switch (adev->asic_type) {
 737         case CHIP_TONGA:
 738         case CHIP_FIJI:
 739                 reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
 740                 break;
 741         case CHIP_VEGA10:
 742         case CHIP_VEGA20:
 743         case CHIP_NAVI10:
 744         case CHIP_NAVI12:
 745         case CHIP_SIENNA_CICHLID:
 746         case CHIP_ARCTURUS:
 747         case CHIP_ALDEBARAN:
 748         case CHIP_IP_DISCOVERY:
 749                 reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER);
 750                 break;
 751         default: /* other chip doesn't support SRIOV */
 752                 reg = 0;
 753                 break;
 754         }
 755
 756         if (reg & 1)
 757                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
 758
 759         if (reg & 0x80000000)
 760                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
 761
 762         if (!reg) {
 763                 /* passthrough mode exclus sriov mod */
 764                 if (is_virtual_machine() && !xen_initial_domain())
 765                         adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
 766         }
 767
 768         /* we have the ability to check now */
 769         if (amdgpu_sriov_vf(adev)) {
 770                 switch (adev->asic_type) {
 771                 case CHIP_TONGA:
 772                 case CHIP_FIJI:
 773                         vi_set_virt_ops(adev);
 774                         break;
 775                 case CHIP_VEGA10:
 776                         soc15_set_virt_ops(adev);
 777 #ifdef CONFIG_X86
 778                         /* not send GPU_INIT_DATA with MS_HYPERV*/
 779                         if (!hypervisor_is_type(X86_HYPER_MS_HYPERV))
 780 #endif
 781                                 /* send a dummy GPU_INIT_DATA request to host on vega10 */
 782                                 amdgpu_virt_request_init_data(adev);
 783                         break;
 784                 case CHIP_VEGA20:
 785                 case CHIP_ARCTURUS:
 786                 case CHIP_ALDEBARAN:
 787                         soc15_set_virt_ops(adev);
 788                         break;
 789                 case CHIP_NAVI10:
 790                 case CHIP_NAVI12:
 791                 case CHIP_SIENNA_CICHLID:
 792                 case CHIP_IP_DISCOVERY:
 793                         nv_set_virt_ops(adev);
 794                         /* try send GPU_INIT_DATA request to host */
 795                         amdgpu_virt_request_init_data(adev);
 796                         break;
 797                 default: /* other chip doesn't support SRIOV */
 798                         DRM_ERROR("Unknown asic type: %d!\n", adev->asic_type);
 799                         break;
 800                 }
 801         }
 802 }
 803
 804 static bool amdgpu_virt_access_debugfs_is_mmio(struct amdgpu_device *adev)
 805 {
 806         return amdgpu_sriov_is_debug(adev) ? true : false;
 807 }
 808
 809 static bool amdgpu_virt_access_debugfs_is_kiq(struct amdgpu_device *adev)
 810 {
 811         return amdgpu_sriov_is_normal(adev) ? true : false;
 812 }
 813
 814 int amdgpu_virt_enable_access_debugfs(struct amdgpu_device *adev)
 815 {
 816         if (!amdgpu_sriov_vf(adev) ||
 817             amdgpu_virt_access_debugfs_is_kiq(adev))
 818                 return 0;
 819
 820         if (amdgpu_virt_access_debugfs_is_mmio(adev))
 821                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
 822         else
 823                 return -EPERM;
 824
 825         return 0;
 826 }
 827
 828 void amdgpu_virt_disable_access_debugfs(struct amdgpu_device *adev)
 829 {
 830         if (amdgpu_sriov_vf(adev))
 831                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
 832 }
 833
 834 enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *adev)
 835 {
 836         enum amdgpu_sriov_vf_mode mode;
 837
 838         if (amdgpu_sriov_vf(adev)) {
 839                 if (amdgpu_sriov_is_pp_one_vf(adev))
 840                         mode = SRIOV_VF_MODE_ONE_VF;
 841                 else
 842                         mode = SRIOV_VF_MODE_MULTI_VF;
 843         } else {
 844                 mode = SRIOV_VF_MODE_BARE_METAL;
 845         }
 846
 847         return mode;
 848 }
 849
 850 void amdgpu_virt_post_reset(struct amdgpu_device *adev)
 851 {
 852         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) {
 853                 /* force set to GFXOFF state after reset,
 854                  * to avoid some invalid operation before GC enable
 855                  */
 856                 adev->gfx.is_poweron = false;
 857         }
 858
 859         adev->mes.ring.sched.ready = false;
 860 }
 861
 862 bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id)
 863 {
 864         switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
 865         case IP_VERSION(13, 0, 0):
 866                 /* no vf autoload, white list */
 867                 if (ucode_id == AMDGPU_UCODE_ID_VCN1 ||
 868                     ucode_id == AMDGPU_UCODE_ID_VCN)
 869                         return false;
 870                 else
 871                         return true;
 872         case IP_VERSION(11, 0, 9):
 873         case IP_VERSION(11, 0, 7):
 874                 /* black list for CHIP_NAVI12 and CHIP_SIENNA_CICHLID */
 875                 if (ucode_id == AMDGPU_UCODE_ID_RLC_G
 876                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
 877                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
 878                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
 879                     || ucode_id == AMDGPU_UCODE_ID_SMC)
 880                         return true;
 881                 else
 882                         return false;
 883         case IP_VERSION(13, 0, 10):
 884                 /* white list */
 885                 if (ucode_id == AMDGPU_UCODE_ID_CAP
 886                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP
 887                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME
 888                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC
 889                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK
 890                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK
 891                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK
 892                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK
 893                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK
 894                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK
 895                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK
 896                 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK
 897                 || ucode_id == AMDGPU_UCODE_ID_CP_MES
 898                 || ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA
 899                 || ucode_id == AMDGPU_UCODE_ID_CP_MES1
 900                 || ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA
 901                 || ucode_id == AMDGPU_UCODE_ID_VCN1
 902                 || ucode_id == AMDGPU_UCODE_ID_VCN)
 903                         return false;
 904                 else
 905                         return true;
 906         default:
 907                 /* lagacy black list */
 908                 if (ucode_id == AMDGPU_UCODE_ID_SDMA0
 909                     || ucode_id == AMDGPU_UCODE_ID_SDMA1
 910                     || ucode_id == AMDGPU_UCODE_ID_SDMA2
 911                     || ucode_id == AMDGPU_UCODE_ID_SDMA3
 912                     || ucode_id == AMDGPU_UCODE_ID_SDMA4
 913                     || ucode_id == AMDGPU_UCODE_ID_SDMA5
 914                     || ucode_id == AMDGPU_UCODE_ID_SDMA6
 915                     || ucode_id == AMDGPU_UCODE_ID_SDMA7
 916                     || ucode_id == AMDGPU_UCODE_ID_RLC_G
 917                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
 918                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
 919                     || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
 920                     || ucode_id == AMDGPU_UCODE_ID_SMC)
 921                         return true;
 922                 else
 923                         return false;
 924         }
 925 }
 926
 927 void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev,
 928                         struct amdgpu_video_codec_info *encode, uint32_t encode_array_size,
 929                         struct amdgpu_video_codec_info *decode, uint32_t decode_array_size)
 930 {
 931         uint32_t i;
 932
 933         if (!adev->virt.is_mm_bw_enabled)
 934                 return;
 935
 936         if (encode) {
 937                 for (i = 0; i < encode_array_size; i++) {
 938                         encode[i].max_width = adev->virt.encode_max_dimension_pixels;
 939                         encode[i].max_pixels_per_frame = adev->virt.encode_max_frame_pixels;
 940                         if (encode[i].max_width > 0)
 941                                 encode[i].max_height = encode[i].max_pixels_per_frame / encode[i].max_width;
 942                         else
 943                                 encode[i].max_height = 0;
 944                 }
 945         }
 946
 947         if (decode) {
 948                 for (i = 0; i < decode_array_size; i++) {
 949                         decode[i].max_width = adev->virt.decode_max_dimension_pixels;
 950                         decode[i].max_pixels_per_frame = adev->virt.decode_max_frame_pixels;
 951                         if (decode[i].max_width > 0)
 952                                 decode[i].max_height = decode[i].max_pixels_per_frame / decode[i].max_width;
 953                         else
 954                                 decode[i].max_height = 0;
 955                 }
 956         }
 957 }
 958
 959 bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev,
 960                                                  u32 acc_flags, u32 hwip,
 961                                                  bool write, u32 *rlcg_flag)
 962 {
 963         bool ret = false;
 964
 965         switch (hwip) {
 966         case GC_HWIP:
 967                 if (amdgpu_sriov_reg_indirect_gc(adev)) {
 968                         *rlcg_flag =
 969                                 write ? AMDGPU_RLCG_GC_WRITE : AMDGPU_RLCG_GC_READ;
 970                         ret = true;
 971                 /* only in new version, AMDGPU_REGS_NO_KIQ and
 972                  * AMDGPU_REGS_RLC are enabled simultaneously */
 973                 } else if ((acc_flags & AMDGPU_REGS_RLC) &&
 974                                 !(acc_flags & AMDGPU_REGS_NO_KIQ) && write) {
 975                         *rlcg_flag = AMDGPU_RLCG_GC_WRITE_LEGACY;
 976                         ret = true;
 977                 }
 978                 break;
 979         case MMHUB_HWIP:
 980                 if (amdgpu_sriov_reg_indirect_mmhub(adev) &&
 981                     (acc_flags & AMDGPU_REGS_RLC) && write) {
 982                         *rlcg_flag = AMDGPU_RLCG_MMHUB_WRITE;
 983                         ret = true;
 984                 }
 985                 break;
 986         default:
 987                 break;
 988         }
 989         return ret;
 990 }
 991
 992 u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id)
 993 {
 994         struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
 995         uint32_t timeout = 50000;
 996         uint32_t i, tmp;
 997         uint32_t ret = 0;
 998         void *scratch_reg0;
 999         void *scratch_reg1;
1000         void *scratch_reg2;
1001         void *scratch_reg3;
1002         void *spare_int;
1003
1004         if (!adev->gfx.rlc.rlcg_reg_access_supported) {
1005                 dev_err(adev->dev,
1006                         "indirect registers access through rlcg is not available\n");
1007                 return 0;
1008         }
1009
1010         if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) {
1011                 dev_err(adev->dev, "invalid xcc\n");
1012                 return 0;
1013         }
1014
1015         if (amdgpu_device_skip_hw_access(adev))
1016                 return 0;
1017
1018         reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id];
1019         scratch_reg0 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg0;
1020         scratch_reg1 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg1;
1021         scratch_reg2 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg2;
1022         scratch_reg3 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg3;
1023
1024         mutex_lock(&adev->virt.rlcg_reg_lock);
1025
1026         if (reg_access_ctrl->spare_int)
1027                 spare_int = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->spare_int;
1028
1029         if (offset == reg_access_ctrl->grbm_cntl) {
1030                 /* if the target reg offset is grbm_cntl, write to scratch_reg2 */
1031                 writel(v, scratch_reg2);
1032                 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY)
1033                         writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
1034         } else if (offset == reg_access_ctrl->grbm_idx) {
1035                 /* if the target reg offset is grbm_idx, write to scratch_reg3 */
1036                 writel(v, scratch_reg3);
1037                 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY)
1038                         writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
1039         } else {
1040                 /*
1041                  * SCRATCH_REG0         = read/write value
1042                  * SCRATCH_REG1[30:28]  = command
1043                  * SCRATCH_REG1[19:0]   = address in dword
1044                  * SCRATCH_REG1[27:24]  = Error reporting
1045                  */
1046                 writel(v, scratch_reg0);
1047                 writel((offset | flag), scratch_reg1);
1048                 if (reg_access_ctrl->spare_int)
1049                         writel(1, spare_int);
1050
1051                 for (i = 0; i < timeout; i++) {
1052                         tmp = readl(scratch_reg1);
1053                         if (!(tmp & AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK))
1054                                 break;
1055                         udelay(10);
1056                 }
1057
1058                 tmp = readl(scratch_reg1);
1059                 if (i >= timeout || (tmp & AMDGPU_RLCG_SCRATCH1_ERROR_MASK) != 0) {
1060                         if (amdgpu_sriov_rlcg_error_report_enabled(adev)) {
1061                                 if (tmp & AMDGPU_RLCG_VFGATE_DISABLED) {
1062                                         dev_err(adev->dev,
1063                                                 "vfgate is disabled, rlcg failed to program reg: 0x%05x\n", offset);
1064                                 } else if (tmp & AMDGPU_RLCG_WRONG_OPERATION_TYPE) {
1065                                         dev_err(adev->dev,
1066                                                 "wrong operation type, rlcg failed to program reg: 0x%05x\n", offset);
1067                                 } else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) {
1068                                         dev_err(adev->dev,
1069                                                 "register is not in range, rlcg failed to program reg: 0x%05x\n", offset);
1070                                 } else {
1071                                         dev_err(adev->dev,
1072                                                 "unknown error type, rlcg failed to program reg: 0x%05x\n", offset);
1073                                 }
1074                         } else {
1075                                 dev_err(adev->dev,
1076                                         "timeout: rlcg faled to program reg: 0x%05x\n", offset);
1077                         }
1078                 }
1079         }
1080
1081         ret = readl(scratch_reg0);
1082
1083         mutex_unlock(&adev->virt.rlcg_reg_lock);
1084
1085         return ret;
1086 }
1087
1088 void amdgpu_sriov_wreg(struct amdgpu_device *adev,
1089                        u32 offset, u32 value,
1090                        u32 acc_flags, u32 hwip, u32 xcc_id)
1091 {
1092         u32 rlcg_flag;
1093
1094         if (amdgpu_device_skip_hw_access(adev))
1095                 return;
1096
1097         if (!amdgpu_sriov_runtime(adev) &&
1098                 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, true, &rlcg_flag)) {
1099                 amdgpu_virt_rlcg_reg_rw(adev, offset, value, rlcg_flag, xcc_id);
1100                 return;
1101         }
1102
1103         if (acc_flags & AMDGPU_REGS_NO_KIQ)
1104                 WREG32_NO_KIQ(offset, value);
1105         else
1106                 WREG32(offset, value);
1107 }
1108
1109 u32 amdgpu_sriov_rreg(struct amdgpu_device *adev,
1110                       u32 offset, u32 acc_flags, u32 hwip, u32 xcc_id)
1111 {
1112         u32 rlcg_flag;
1113
1114         if (amdgpu_device_skip_hw_access(adev))
1115                 return 0;
1116
1117         if (!amdgpu_sriov_runtime(adev) &&
1118                 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, false, &rlcg_flag))
1119                 return amdgpu_virt_rlcg_reg_rw(adev, offset, 0, rlcg_flag, xcc_id);
1120
1121         if (acc_flags & AMDGPU_REGS_NO_KIQ)
1122                 return RREG32_NO_KIQ(offset);
1123         else
1124                 return RREG32(offset);
1125 }
1126
1127 bool amdgpu_sriov_xnack_support(struct amdgpu_device *adev)
1128 {
1129         bool xnack_mode = true;
1130
1131         if (amdgpu_sriov_vf(adev) &&
1132             amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
1133                 xnack_mode = false;
1134
1135         return xnack_mode;
1136 }