drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_sdma.h"
  27 #include "amdgpu_ras.h"
  28
  29 #define AMDGPU_CSA_SDMA_SIZE 64
  30 /* SDMA CSA reside in the 3rd page of CSA */
  31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
  32
  33 /*
  34  * GPU SDMA IP block helpers function.
  35  */
  36
  37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
  38 {
  39         struct amdgpu_device *adev = ring->adev;
  40         int i;
  41
  42         for (i = 0; i < adev->sdma.num_instances; i++)
  43                 if (ring == &adev->sdma.instance[i].ring ||
  44                     ring == &adev->sdma.instance[i].page)
  45                         return &adev->sdma.instance[i];
  46
  47         return NULL;
  48 }
  49
  50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
  51 {
  52         struct amdgpu_device *adev = ring->adev;
  53         int i;
  54
  55         for (i = 0; i < adev->sdma.num_instances; i++) {
  56                 if (ring == &adev->sdma.instance[i].ring ||
  57                         ring == &adev->sdma.instance[i].page) {
  58                         *index = i;
  59                         return 0;
  60                 }
  61         }
  62
  63         return -EINVAL;
  64 }
  65
  66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
  67                                      unsigned int vmid)
  68 {
  69         struct amdgpu_device *adev = ring->adev;
  70         uint64_t csa_mc_addr;
  71         uint32_t index = 0;
  72         int r;
  73
  74         /* don't enable OS preemption on SDMA under SRIOV */
  75         if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp)
  76                 return 0;
  77
  78         if (ring->is_mes_queue) {
  79                 uint32_t offset = 0;
  80
  81                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
  82                                   sdma[ring->idx].sdma_meta_data);
  83                 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
  84         } else {
  85                 r = amdgpu_sdma_get_index_from_ring(ring, &index);
  86
  87                 if (r || index > 31)
  88                         csa_mc_addr = 0;
  89                 else
  90                         csa_mc_addr = amdgpu_csa_vaddr(adev) +
  91                                 AMDGPU_CSA_SDMA_OFFSET +
  92                                 index * AMDGPU_CSA_SDMA_SIZE;
  93         }
  94
  95         return csa_mc_addr;
  96 }
  97
  98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
  99                               struct ras_common_if *ras_block)
 100 {
 101         int r, i;
 102
 103         r = amdgpu_ras_block_late_init(adev, ras_block);
 104         if (r)
 105                 return r;
 106
 107         if (amdgpu_ras_is_supported(adev, ras_block->block)) {
 108                 for (i = 0; i < adev->sdma.num_instances; i++) {
 109                         r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
 110                                 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
 111                         if (r)
 112                                 goto late_fini;
 113                 }
 114         }
 115
 116         return 0;
 117
 118 late_fini:
 119         amdgpu_ras_block_late_fini(adev, ras_block);
 120         return r;
 121 }
 122
 123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
 124                 void *err_data,
 125                 struct amdgpu_iv_entry *entry)
 126 {
 127         kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 128
 129         if (amdgpu_sriov_vf(adev))
 130                 return AMDGPU_RAS_SUCCESS;
 131
 132         amdgpu_ras_reset_gpu(adev);
 133
 134         return AMDGPU_RAS_SUCCESS;
 135 }
 136
 137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
 138                                       struct amdgpu_irq_src *source,
 139                                       struct amdgpu_iv_entry *entry)
 140 {
 141         struct ras_common_if *ras_if = adev->sdma.ras_if;
 142         struct ras_dispatch_if ih_data = {
 143                 .entry = entry,
 144         };
 145
 146         if (!ras_if)
 147                 return 0;
 148
 149         ih_data.head = *ras_if;
 150
 151         amdgpu_ras_interrupt_dispatch(adev, &ih_data);
 152         return 0;
 153 }
 154
 155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
 156 {
 157         uint16_t version_major;
 158         const struct common_firmware_header *header = NULL;
 159         const struct sdma_firmware_header_v1_0 *hdr;
 160         const struct sdma_firmware_header_v2_0 *hdr_v2;
 161         const struct sdma_firmware_header_v3_0 *hdr_v3;
 162
 163         header = (const struct common_firmware_header *)
 164                 sdma_inst->fw->data;
 165         version_major = le16_to_cpu(header->header_version_major);
 166
 167         switch (version_major) {
 168         case 1:
 169                 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
 170                 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
 171                 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
 172                 break;
 173         case 2:
 174                 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
 175                 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
 176                 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
 177                 break;
 178         case 3:
 179                 hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data;
 180                 sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version);
 181                 sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version);
 182                 break;
 183         default:
 184                 return -EINVAL;
 185         }
 186
 187         if (sdma_inst->feature_version >= 20)
 188                 sdma_inst->burst_nop = true;
 189
 190         return 0;
 191 }
 192
 193 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
 194                                   bool duplicate)
 195 {
 196         int i;
 197
 198         for (i = 0; i < adev->sdma.num_instances; i++) {
 199                 amdgpu_ucode_release(&adev->sdma.instance[i].fw);
 200                 if (duplicate)
 201                         break;
 202         }
 203
 204         memset((void *)adev->sdma.instance, 0,
 205                sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
 206 }
 207
 208 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
 209                                u32 instance, bool duplicate)
 210 {
 211         struct amdgpu_firmware_info *info = NULL;
 212         const struct common_firmware_header *header = NULL;
 213         int err, i;
 214         const struct sdma_firmware_header_v2_0 *sdma_hdr;
 215         const struct sdma_firmware_header_v3_0 *sdma_hv3;
 216         uint16_t version_major;
 217         char ucode_prefix[30];
 218
 219         amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
 220         if (instance == 0)
 221                 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
 222                                            "amdgpu/%s.bin", ucode_prefix);
 223         else
 224                 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
 225                                            "amdgpu/%s%d.bin", ucode_prefix, instance);
 226         if (err)
 227                 goto out;
 228
 229         header = (const struct common_firmware_header *)
 230                 adev->sdma.instance[instance].fw->data;
 231         version_major = le16_to_cpu(header->header_version_major);
 232
 233         if ((duplicate && instance) || (!duplicate && version_major > 1)) {
 234                 err = -EINVAL;
 235                 goto out;
 236         }
 237
 238         err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
 239         if (err)
 240                 goto out;
 241
 242         if (duplicate) {
 243                 for (i = 1; i < adev->sdma.num_instances; i++)
 244                         memcpy((void *)&adev->sdma.instance[i],
 245                                (void *)&adev->sdma.instance[0],
 246                                sizeof(struct amdgpu_sdma_instance));
 247         }
 248
 249         DRM_DEBUG("psp_load == '%s'\n",
 250                   adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
 251
 252         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 253                 switch (version_major) {
 254                 case 1:
 255                         for (i = 0; i < adev->sdma.num_instances; i++) {
 256                                 if (!duplicate && (instance != i))
 257                                         continue;
 258                                 else {
 259                                         /* Use a single copy per SDMA firmware type. PSP uses the same instance for all
 260                                          * groups of SDMAs */
 261                                         if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
 262                                                 IP_VERSION(4, 4, 2) ||
 263                                              amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
 264                                                 IP_VERSION(4, 4, 5)) &&
 265                                             adev->firmware.load_type ==
 266                                                 AMDGPU_FW_LOAD_PSP &&
 267                                             adev->sdma.num_inst_per_aid == i) {
 268                                                 break;
 269                                         }
 270                                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
 271                                         info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
 272                                         info->fw = adev->sdma.instance[i].fw;
 273                                         adev->firmware.fw_size +=
 274                                                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 275                                 }
 276                         }
 277                         break;
 278                 case 2:
 279                         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
 280                                 adev->sdma.instance[0].fw->data;
 281                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
 282                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
 283                         info->fw = adev->sdma.instance[0].fw;
 284                         adev->firmware.fw_size +=
 285                                 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
 286                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
 287                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
 288                         info->fw = adev->sdma.instance[0].fw;
 289                         adev->firmware.fw_size +=
 290                                 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
 291                         break;
 292                 case 3:
 293                         sdma_hv3 = (const struct sdma_firmware_header_v3_0 *)
 294                                 adev->sdma.instance[0].fw->data;
 295                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64];
 296                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64;
 297                         info->fw = adev->sdma.instance[0].fw;
 298                         adev->firmware.fw_size +=
 299                                 ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE);
 300                         break;
 301                 default:
 302                         err = -EINVAL;
 303                 }
 304         }
 305
 306 out:
 307         if (err)
 308                 amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
 309         return err;
 310 }
 311
 312 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)
 313 {
 314         int err = 0;
 315         struct amdgpu_sdma_ras *ras = NULL;
 316
 317         /* adev->sdma.ras is NULL, which means sdma does not
 318          * support ras function, then do nothing here.
 319          */
 320         if (!adev->sdma.ras)
 321                 return 0;
 322
 323         ras = adev->sdma.ras;
 324
 325         err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
 326         if (err) {
 327                 dev_err(adev->dev, "Failed to register sdma ras block!\n");
 328                 return err;
 329         }
 330
 331         strcpy(ras->ras_block.ras_comm.name, "sdma");
 332         ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;
 333         ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 334         adev->sdma.ras_if = &ras->ras_block.ras_comm;
 335
 336         /* If not define special ras_late_init function, use default ras_late_init */
 337         if (!ras->ras_block.ras_late_init)
 338                 ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;
 339
 340         /* If not defined special ras_cb function, use default ras_cb */
 341         if (!ras->ras_block.ras_cb)
 342                 ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;
 343
 344         return 0;
 345 }
 346
 347 /*
 348  * debugfs for to enable/disable sdma job submission to specific core.
 349  */
 350 #if defined(CONFIG_DEBUG_FS)
 351 static int amdgpu_debugfs_sdma_sched_mask_set(void *data, u64 val)
 352 {
 353         struct amdgpu_device *adev = (struct amdgpu_device *)data;
 354         u32 i;
 355         u64 mask = 0;
 356         struct amdgpu_ring *ring;
 357
 358         if (!adev)
 359                 return -ENODEV;
 360
 361         mask = (1 << adev->sdma.num_instances) - 1;
 362         if ((val & mask) == 0)
 363                 return -EINVAL;
 364
 365         for (i = 0; i < adev->sdma.num_instances; ++i) {
 366                 ring = &adev->sdma.instance[i].ring;
 367                 if (val & (1 << i))
 368                         ring->sched.ready = true;
 369                 else
 370                         ring->sched.ready = false;
 371         }
 372         /* publish sched.ready flag update effective immediately across smp */
 373         smp_rmb();
 374         return 0;
 375 }
 376
 377 static int amdgpu_debugfs_sdma_sched_mask_get(void *data, u64 *val)
 378 {
 379         struct amdgpu_device *adev = (struct amdgpu_device *)data;
 380         u32 i;
 381         u64 mask = 0;
 382         struct amdgpu_ring *ring;
 383
 384         if (!adev)
 385                 return -ENODEV;
 386         for (i = 0; i < adev->sdma.num_instances; ++i) {
 387                 ring = &adev->sdma.instance[i].ring;
 388                 if (ring->sched.ready)
 389                         mask |= 1 << i;
 390         }
 391
 392         *val = mask;
 393         return 0;
 394 }
 395
 396 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_sdma_sched_mask_fops,
 397                          amdgpu_debugfs_sdma_sched_mask_get,
 398                          amdgpu_debugfs_sdma_sched_mask_set, "%llx\n");
 399
 400 #endif
 401
 402 void amdgpu_debugfs_sdma_sched_mask_init(struct amdgpu_device *adev)
 403 {
 404 #if defined(CONFIG_DEBUG_FS)
 405         struct drm_minor *minor = adev_to_drm(adev)->primary;
 406         struct dentry *root = minor->debugfs_root;
 407         char name[32];
 408
 409         if (!(adev->sdma.num_instances > 1))
 410                 return;
 411         sprintf(name, "amdgpu_sdma_sched_mask");
 412         debugfs_create_file(name, 0600, root, adev,
 413                             &amdgpu_debugfs_sdma_sched_mask_fops);
 414 #endif
 415 }
 416
 417 static ssize_t amdgpu_get_sdma_reset_mask(struct device *dev,
 418                                                 struct device_attribute *attr,
 419                                                 char *buf)
 420 {
 421         struct drm_device *ddev = dev_get_drvdata(dev);
 422         struct amdgpu_device *adev = drm_to_adev(ddev);
 423
 424         if (!adev)
 425                 return -ENODEV;
 426
 427         return amdgpu_show_reset_mask(buf, adev->sdma.supported_reset);
 428 }
 429
 430 static DEVICE_ATTR(sdma_reset_mask, 0444,
 431                    amdgpu_get_sdma_reset_mask, NULL);
 432
 433 int amdgpu_sdma_sysfs_reset_mask_init(struct amdgpu_device *adev)
 434 {
 435         int r = 0;
 436
 437         if (!amdgpu_gpu_recovery)
 438                 return r;
 439
 440         if (adev->sdma.num_instances) {
 441                 r = device_create_file(adev->dev, &dev_attr_sdma_reset_mask);
 442                 if (r)
 443                         return r;
 444         }
 445
 446         return r;
 447 }
 448
 449 void amdgpu_sdma_sysfs_reset_mask_fini(struct amdgpu_device *adev)
 450 {
 451         if (!amdgpu_gpu_recovery)
 452                 return;
 453
 454         if (adev->dev->kobj.sd) {
 455                 if (adev->sdma.num_instances)
 456                         device_remove_file(adev->dev, &dev_attr_sdma_reset_mask);
 457         }
 458 }