drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_sdma.h"
  27 #include "amdgpu_ras.h"
  28
  29 #define AMDGPU_CSA_SDMA_SIZE 64
  30 /* SDMA CSA reside in the 3rd page of CSA */
  31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
  32
  33 /*
  34  * GPU SDMA IP block helpers function.
  35  */
  36
  37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
  38 {
  39         struct amdgpu_device *adev = ring->adev;
  40         int i;
  41
  42         for (i = 0; i < adev->sdma.num_instances; i++)
  43                 if (ring == &adev->sdma.instance[i].ring ||
  44                     ring == &adev->sdma.instance[i].page)
  45                         return &adev->sdma.instance[i];
  46
  47         return NULL;
  48 }
  49
  50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
  51 {
  52         struct amdgpu_device *adev = ring->adev;
  53         int i;
  54
  55         for (i = 0; i < adev->sdma.num_instances; i++) {
  56                 if (ring == &adev->sdma.instance[i].ring ||
  57                         ring == &adev->sdma.instance[i].page) {
  58                         *index = i;
  59                         return 0;
  60                 }
  61         }
  62
  63         return -EINVAL;
  64 }
  65
  66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
  67                                      unsigned int vmid)
  68 {
  69         struct amdgpu_device *adev = ring->adev;
  70         uint64_t csa_mc_addr;
  71         uint32_t index = 0;
  72         int r;
  73
  74         /* don't enable OS preemption on SDMA under SRIOV */
  75         if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp)
  76                 return 0;
  77
  78         if (ring->is_mes_queue) {
  79                 uint32_t offset = 0;
  80
  81                 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
  82                                   sdma[ring->idx].sdma_meta_data);
  83                 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
  84         } else {
  85                 r = amdgpu_sdma_get_index_from_ring(ring, &index);
  86
  87                 if (r || index > 31)
  88                         csa_mc_addr = 0;
  89                 else
  90                         csa_mc_addr = amdgpu_csa_vaddr(adev) +
  91                                 AMDGPU_CSA_SDMA_OFFSET +
  92                                 index * AMDGPU_CSA_SDMA_SIZE;
  93         }
  94
  95         return csa_mc_addr;
  96 }
  97
  98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
  99                               struct ras_common_if *ras_block)
 100 {
 101         int r, i;
 102
 103         r = amdgpu_ras_block_late_init(adev, ras_block);
 104         if (r)
 105                 return r;
 106
 107         if (amdgpu_ras_is_supported(adev, ras_block->block)) {
 108                 for (i = 0; i < adev->sdma.num_instances; i++) {
 109                         r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
 110                                 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
 111                         if (r)
 112                                 goto late_fini;
 113                 }
 114         }
 115
 116         return 0;
 117
 118 late_fini:
 119         amdgpu_ras_block_late_fini(adev, ras_block);
 120         return r;
 121 }
 122
 123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
 124                 void *err_data,
 125                 struct amdgpu_iv_entry *entry)
 126 {
 127         kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 128
 129         if (amdgpu_sriov_vf(adev))
 130                 return AMDGPU_RAS_SUCCESS;
 131
 132         amdgpu_ras_reset_gpu(adev);
 133
 134         return AMDGPU_RAS_SUCCESS;
 135 }
 136
 137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
 138                                       struct amdgpu_irq_src *source,
 139                                       struct amdgpu_iv_entry *entry)
 140 {
 141         struct ras_common_if *ras_if = adev->sdma.ras_if;
 142         struct ras_dispatch_if ih_data = {
 143                 .entry = entry,
 144         };
 145
 146         if (!ras_if)
 147                 return 0;
 148
 149         ih_data.head = *ras_if;
 150
 151         amdgpu_ras_interrupt_dispatch(adev, &ih_data);
 152         return 0;
 153 }
 154
 155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
 156 {
 157         uint16_t version_major;
 158         const struct common_firmware_header *header = NULL;
 159         const struct sdma_firmware_header_v1_0 *hdr;
 160         const struct sdma_firmware_header_v2_0 *hdr_v2;
 161         const struct sdma_firmware_header_v3_0 *hdr_v3;
 162
 163         header = (const struct common_firmware_header *)
 164                 sdma_inst->fw->data;
 165         version_major = le16_to_cpu(header->header_version_major);
 166
 167         switch (version_major) {
 168         case 1:
 169                 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
 170                 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
 171                 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
 172                 break;
 173         case 2:
 174                 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
 175                 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
 176                 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
 177                 break;
 178         case 3:
 179                 hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data;
 180                 sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version);
 181                 sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version);
 182                 break;
 183         default:
 184                 return -EINVAL;
 185         }
 186
 187         if (sdma_inst->feature_version >= 20)
 188                 sdma_inst->burst_nop = true;
 189
 190         return 0;
 191 }
 192
 193 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
 194                                   bool duplicate)
 195 {
 196         int i;
 197
 198         for (i = 0; i < adev->sdma.num_instances; i++) {
 199                 amdgpu_ucode_release(&adev->sdma.instance[i].fw);
 200                 if (duplicate)
 201                         break;
 202         }
 203
 204         memset((void *)adev->sdma.instance, 0,
 205                sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
 206 }
 207
 208 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
 209                                u32 instance, bool duplicate)
 210 {
 211         struct amdgpu_firmware_info *info = NULL;
 212         const struct common_firmware_header *header = NULL;
 213         int err, i;
 214         const struct sdma_firmware_header_v2_0 *sdma_hdr;
 215         const struct sdma_firmware_header_v3_0 *sdma_hv3;
 216         uint16_t version_major;
 217         char ucode_prefix[30];
 218
 219         amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
 220         if (instance == 0)
 221                 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
 222                                            AMDGPU_UCODE_REQUIRED,
 223                                            "amdgpu/%s.bin", ucode_prefix);
 224         else
 225                 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
 226                                            AMDGPU_UCODE_REQUIRED,
 227                                            "amdgpu/%s%d.bin", ucode_prefix, instance);
 228         if (err)
 229                 goto out;
 230
 231         header = (const struct common_firmware_header *)
 232                 adev->sdma.instance[instance].fw->data;
 233         version_major = le16_to_cpu(header->header_version_major);
 234
 235         if ((duplicate && instance) || (!duplicate && version_major > 1)) {
 236                 err = -EINVAL;
 237                 goto out;
 238         }
 239
 240         err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
 241         if (err)
 242                 goto out;
 243
 244         if (duplicate) {
 245                 for (i = 1; i < adev->sdma.num_instances; i++)
 246                         memcpy((void *)&adev->sdma.instance[i],
 247                                (void *)&adev->sdma.instance[0],
 248                                sizeof(struct amdgpu_sdma_instance));
 249         }
 250
 251         DRM_DEBUG("psp_load == '%s'\n",
 252                   adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
 253
 254         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 255                 switch (version_major) {
 256                 case 1:
 257                         for (i = 0; i < adev->sdma.num_instances; i++) {
 258                                 if (!duplicate && (instance != i))
 259                                         continue;
 260                                 else {
 261                                         /* Use a single copy per SDMA firmware type. PSP uses the same instance for all
 262                                          * groups of SDMAs */
 263                                         if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
 264                                                 IP_VERSION(4, 4, 2) ||
 265                                              amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
 266                                                 IP_VERSION(4, 4, 4) ||
 267                                              amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
 268                                                 IP_VERSION(4, 4, 5)) &&
 269                                             adev->firmware.load_type ==
 270                                                 AMDGPU_FW_LOAD_PSP &&
 271                                             adev->sdma.num_inst_per_aid == i) {
 272                                                 break;
 273                                         }
 274                                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
 275                                         info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
 276                                         info->fw = adev->sdma.instance[i].fw;
 277                                         adev->firmware.fw_size +=
 278                                                 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 279                                 }
 280                         }
 281                         break;
 282                 case 2:
 283                         sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
 284                                 adev->sdma.instance[0].fw->data;
 285                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
 286                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
 287                         info->fw = adev->sdma.instance[0].fw;
 288                         adev->firmware.fw_size +=
 289                                 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
 290                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
 291                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
 292                         info->fw = adev->sdma.instance[0].fw;
 293                         adev->firmware.fw_size +=
 294                                 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
 295                         break;
 296                 case 3:
 297                         sdma_hv3 = (const struct sdma_firmware_header_v3_0 *)
 298                                 adev->sdma.instance[0].fw->data;
 299                         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64];
 300                         info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64;
 301                         info->fw = adev->sdma.instance[0].fw;
 302                         adev->firmware.fw_size +=
 303                                 ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE);
 304                         break;
 305                 default:
 306                         err = -EINVAL;
 307                 }
 308         }
 309
 310 out:
 311         if (err)
 312                 amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
 313         return err;
 314 }
 315
 316 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)
 317 {
 318         int err = 0;
 319         struct amdgpu_sdma_ras *ras = NULL;
 320
 321         /* adev->sdma.ras is NULL, which means sdma does not
 322          * support ras function, then do nothing here.
 323          */
 324         if (!adev->sdma.ras)
 325                 return 0;
 326
 327         ras = adev->sdma.ras;
 328
 329         err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
 330         if (err) {
 331                 dev_err(adev->dev, "Failed to register sdma ras block!\n");
 332                 return err;
 333         }
 334
 335         strcpy(ras->ras_block.ras_comm.name, "sdma");
 336         ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;
 337         ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 338         adev->sdma.ras_if = &ras->ras_block.ras_comm;
 339
 340         /* If not define special ras_late_init function, use default ras_late_init */
 341         if (!ras->ras_block.ras_late_init)
 342                 ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;
 343
 344         /* If not defined special ras_cb function, use default ras_cb */
 345         if (!ras->ras_block.ras_cb)
 346                 ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;
 347
 348         return 0;
 349 }
 350
 351 /*
 352  * debugfs for to enable/disable sdma job submission to specific core.
 353  */
 354 #if defined(CONFIG_DEBUG_FS)
 355 static int amdgpu_debugfs_sdma_sched_mask_set(void *data, u64 val)
 356 {
 357         struct amdgpu_device *adev = (struct amdgpu_device *)data;
 358         u32 i;
 359         u64 mask = 0;
 360         struct amdgpu_ring *ring;
 361
 362         if (!adev)
 363                 return -ENODEV;
 364
 365         mask = BIT_ULL(adev->sdma.num_instances) - 1;
 366         if ((val & mask) == 0)
 367                 return -EINVAL;
 368
 369         for (i = 0; i < adev->sdma.num_instances; ++i) {
 370                 ring = &adev->sdma.instance[i].ring;
 371                 if (val & BIT_ULL(i))
 372                         ring->sched.ready = true;
 373                 else
 374                         ring->sched.ready = false;
 375         }
 376         /* publish sched.ready flag update effective immediately across smp */
 377         smp_rmb();
 378         return 0;
 379 }
 380
 381 static int amdgpu_debugfs_sdma_sched_mask_get(void *data, u64 *val)
 382 {
 383         struct amdgpu_device *adev = (struct amdgpu_device *)data;
 384         u32 i;
 385         u64 mask = 0;
 386         struct amdgpu_ring *ring;
 387
 388         if (!adev)
 389                 return -ENODEV;
 390         for (i = 0; i < adev->sdma.num_instances; ++i) {
 391                 ring = &adev->sdma.instance[i].ring;
 392                 if (ring->sched.ready)
 393                         mask |= 1 << i;
 394         }
 395
 396         *val = mask;
 397         return 0;
 398 }
 399
 400 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_sdma_sched_mask_fops,
 401                          amdgpu_debugfs_sdma_sched_mask_get,
 402                          amdgpu_debugfs_sdma_sched_mask_set, "%llx\n");
 403
 404 #endif
 405
 406 void amdgpu_debugfs_sdma_sched_mask_init(struct amdgpu_device *adev)
 407 {
 408 #if defined(CONFIG_DEBUG_FS)
 409         struct drm_minor *minor = adev_to_drm(adev)->primary;
 410         struct dentry *root = minor->debugfs_root;
 411         char name[32];
 412
 413         if (!(adev->sdma.num_instances > 1))
 414                 return;
 415         sprintf(name, "amdgpu_sdma_sched_mask");
 416         debugfs_create_file(name, 0600, root, adev,
 417                             &amdgpu_debugfs_sdma_sched_mask_fops);
 418 #endif
 419 }
 420
 421 static ssize_t amdgpu_get_sdma_reset_mask(struct device *dev,
 422                                                 struct device_attribute *attr,
 423                                                 char *buf)
 424 {
 425         struct drm_device *ddev = dev_get_drvdata(dev);
 426         struct amdgpu_device *adev = drm_to_adev(ddev);
 427
 428         if (!adev)
 429                 return -ENODEV;
 430
 431         return amdgpu_show_reset_mask(buf, adev->sdma.supported_reset);
 432 }
 433
 434 static DEVICE_ATTR(sdma_reset_mask, 0444,
 435                    amdgpu_get_sdma_reset_mask, NULL);
 436
 437 int amdgpu_sdma_sysfs_reset_mask_init(struct amdgpu_device *adev)
 438 {
 439         int r = 0;
 440
 441         if (!amdgpu_gpu_recovery)
 442                 return r;
 443
 444         if (adev->sdma.num_instances) {
 445                 r = device_create_file(adev->dev, &dev_attr_sdma_reset_mask);
 446                 if (r)
 447                         return r;
 448         }
 449
 450         return r;
 451 }
 452
 453 void amdgpu_sdma_sysfs_reset_mask_fini(struct amdgpu_device *adev)
 454 {
 455         if (!amdgpu_gpu_recovery)
 456                 return;
 457
 458         if (adev->dev->kobj.sd) {
 459                 if (adev->sdma.num_instances)
 460                         device_remove_file(adev->dev, &dev_attr_sdma_reset_mask);
 461         }
 462 }