drivers/gpu/drm/amd/amdkfd/kfd_debug.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "kfd_debug.h"
  24 #include "kfd_device_queue_manager.h"
  25 #include "kfd_topology.h"
  26 #include <linux/file.h>
  27 #include <uapi/linux/kfd_ioctl.h>
  28
  29 #define MAX_WATCH_ADDRESSES     4
  30
  31 int kfd_dbg_ev_query_debug_event(struct kfd_process *process,
  32                       unsigned int *queue_id,
  33                       unsigned int *gpu_id,
  34                       uint64_t exception_clear_mask,
  35                       uint64_t *event_status)
  36 {
  37         struct process_queue_manager *pqm;
  38         struct process_queue_node *pqn;
  39         int i;
  40
  41         if (!(process && process->debug_trap_enabled))
  42                 return -ENODATA;
  43
  44         mutex_lock(&process->event_mutex);
  45         *event_status = 0;
  46         *queue_id = 0;
  47         *gpu_id = 0;
  48
  49         /* find and report queue events */
  50         pqm = &process->pqm;
  51         list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
  52                 uint64_t tmp = process->exception_enable_mask;
  53
  54                 if (!pqn->q)
  55                         continue;
  56
  57                 tmp &= pqn->q->properties.exception_status;
  58
  59                 if (!tmp)
  60                         continue;
  61
  62                 *event_status = pqn->q->properties.exception_status;
  63                 *queue_id = pqn->q->properties.queue_id;
  64                 *gpu_id = pqn->q->device->id;
  65                 pqn->q->properties.exception_status &= ~exception_clear_mask;
  66                 goto out;
  67         }
  68
  69         /* find and report device events */
  70         for (i = 0; i < process->n_pdds; i++) {
  71                 struct kfd_process_device *pdd = process->pdds[i];
  72                 uint64_t tmp = process->exception_enable_mask
  73                                                 & pdd->exception_status;
  74
  75                 if (!tmp)
  76                         continue;
  77
  78                 *event_status = pdd->exception_status;
  79                 *gpu_id = pdd->dev->id;
  80                 pdd->exception_status &= ~exception_clear_mask;
  81                 goto out;
  82         }
  83
  84         /* report process events */
  85         if (process->exception_enable_mask & process->exception_status) {
  86                 *event_status = process->exception_status;
  87                 process->exception_status &= ~exception_clear_mask;
  88         }
  89
  90 out:
  91         mutex_unlock(&process->event_mutex);
  92         return *event_status ? 0 : -EAGAIN;
  93 }
  94
  95 void debug_event_write_work_handler(struct work_struct *work)
  96 {
  97         struct kfd_process *process;
  98
  99         static const char write_data = '.';
 100         loff_t pos = 0;
 101
 102         process = container_of(work,
 103                         struct kfd_process,
 104                         debug_event_workarea);
 105
 106         kernel_write(process->dbg_ev_file, &write_data, 1, &pos);
 107 }
 108
 109 /* update process/device/queue exception status, write to descriptor
 110  * only if exception_status is enabled.
 111  */
 112 bool kfd_dbg_ev_raise(uint64_t event_mask,
 113                         struct kfd_process *process, struct kfd_node *dev,
 114                         unsigned int source_id, bool use_worker,
 115                         void *exception_data, size_t exception_data_size)
 116 {
 117         struct process_queue_manager *pqm;
 118         struct process_queue_node *pqn;
 119         int i;
 120         static const char write_data = '.';
 121         loff_t pos = 0;
 122         bool is_subscribed = true;
 123
 124         if (!(process && process->debug_trap_enabled))
 125                 return false;
 126
 127         mutex_lock(&process->event_mutex);
 128
 129         if (event_mask & KFD_EC_MASK_DEVICE) {
 130                 for (i = 0; i < process->n_pdds; i++) {
 131                         struct kfd_process_device *pdd = process->pdds[i];
 132
 133                         if (pdd->dev != dev)
 134                                 continue;
 135
 136                         pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE;
 137
 138                         if (event_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 139                                 if (!pdd->vm_fault_exc_data) {
 140                                         pdd->vm_fault_exc_data = kmemdup(
 141                                                         exception_data,
 142                                                         exception_data_size,
 143                                                         GFP_KERNEL);
 144                                         if (!pdd->vm_fault_exc_data)
 145                                                 pr_debug("Failed to allocate exception data memory");
 146                                 } else {
 147                                         pr_debug("Debugger exception data not saved\n");
 148                                         print_hex_dump_bytes("exception data: ",
 149                                                         DUMP_PREFIX_OFFSET,
 150                                                         exception_data,
 151                                                         exception_data_size);
 152                                 }
 153                         }
 154                         break;
 155                 }
 156         } else if (event_mask & KFD_EC_MASK_PROCESS) {
 157                 process->exception_status |= event_mask & KFD_EC_MASK_PROCESS;
 158         } else {
 159                 pqm = &process->pqm;
 160                 list_for_each_entry(pqn, &pqm->queues,
 161                                 process_queue_list) {
 162                         int target_id;
 163
 164                         if (!pqn->q)
 165                                 continue;
 166
 167                         target_id = event_mask & KFD_EC_MASK(EC_QUEUE_NEW) ?
 168                                         pqn->q->properties.queue_id :
 169                                                         pqn->q->doorbell_id;
 170
 171                         if (pqn->q->device != dev || target_id != source_id)
 172                                 continue;
 173
 174                         pqn->q->properties.exception_status |= event_mask;
 175                         break;
 176                 }
 177         }
 178
 179         if (process->exception_enable_mask & event_mask) {
 180                 if (use_worker)
 181                         schedule_work(&process->debug_event_workarea);
 182                 else
 183                         kernel_write(process->dbg_ev_file,
 184                                         &write_data,
 185                                         1,
 186                                         &pos);
 187         } else {
 188                 is_subscribed = false;
 189         }
 190
 191         mutex_unlock(&process->event_mutex);
 192
 193         return is_subscribed;
 194 }
 195
 196 /* set pending event queue entry from ring entry  */
 197 bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
 198                                    unsigned int pasid,
 199                                    uint32_t doorbell_id,
 200                                    uint64_t trap_mask,
 201                                    void *exception_data,
 202                                    size_t exception_data_size)
 203 {
 204         struct kfd_process *p;
 205         bool signaled_to_debugger_or_runtime = false;
 206
 207         p = kfd_lookup_process_by_pasid(pasid);
 208
 209         if (!p)
 210                 return false;
 211
 212         if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true,
 213                               exception_data, exception_data_size)) {
 214                 struct process_queue_manager *pqm;
 215                 struct process_queue_node *pqn;
 216
 217                 if (!!(trap_mask & KFD_EC_MASK_QUEUE) &&
 218                        p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) {
 219                         mutex_lock(&p->mutex);
 220
 221                         pqm = &p->pqm;
 222                         list_for_each_entry(pqn, &pqm->queues,
 223                                                         process_queue_list) {
 224
 225                                 if (!(pqn->q && pqn->q->device == dev &&
 226                                       pqn->q->doorbell_id == doorbell_id))
 227                                         continue;
 228
 229                                 kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id,
 230                                                               trap_mask);
 231
 232                                 signaled_to_debugger_or_runtime = true;
 233
 234                                 break;
 235                         }
 236
 237                         mutex_unlock(&p->mutex);
 238                 } else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 239                         kfd_dqm_evict_pasid(dev->dqm, p->pasid);
 240                         kfd_signal_vm_fault_event(dev, p->pasid, NULL,
 241                                                         exception_data);
 242
 243                         signaled_to_debugger_or_runtime = true;
 244                 }
 245         } else {
 246                 signaled_to_debugger_or_runtime = true;
 247         }
 248
 249         kfd_unref_process(p);
 250
 251         return signaled_to_debugger_or_runtime;
 252 }
 253
 254 int kfd_dbg_send_exception_to_runtime(struct kfd_process *p,
 255                                         unsigned int dev_id,
 256                                         unsigned int queue_id,
 257                                         uint64_t error_reason)
 258 {
 259         if (error_reason & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 260                 struct kfd_process_device *pdd = NULL;
 261                 struct kfd_hsa_memory_exception_data *data;
 262                 int i;
 263
 264                 for (i = 0; i < p->n_pdds; i++) {
 265                         if (p->pdds[i]->dev->id == dev_id) {
 266                                 pdd = p->pdds[i];
 267                                 break;
 268                         }
 269                 }
 270
 271                 if (!pdd)
 272                         return -ENODEV;
 273
 274                 data = (struct kfd_hsa_memory_exception_data *)
 275                                                 pdd->vm_fault_exc_data;
 276
 277                 kfd_dqm_evict_pasid(pdd->dev->dqm, p->pasid);
 278                 kfd_signal_vm_fault_event(pdd->dev, p->pasid, NULL, data);
 279                 error_reason &= ~KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION);
 280         }
 281
 282         if (error_reason & (KFD_EC_MASK(EC_PROCESS_RUNTIME))) {
 283                 /*
 284                  * block should only happen after the debugger receives runtime
 285                  * enable notice.
 286                  */
 287                 up(&p->runtime_enable_sema);
 288                 error_reason &= ~KFD_EC_MASK(EC_PROCESS_RUNTIME);
 289         }
 290
 291         if (error_reason)
 292                 return kfd_send_exception_to_runtime(p, queue_id, error_reason);
 293
 294         return 0;
 295 }
 296
 297 static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable)
 298 {
 299         struct mqd_update_info minfo = {0};
 300         int err;
 301
 302         if (!q)
 303                 return 0;
 304
 305         if (!kfd_dbg_has_cwsr_workaround(q->device))
 306                 return 0;
 307
 308         if (enable && q->properties.is_user_cu_masked)
 309                 return -EBUSY;
 310
 311         minfo.update_flag = enable ? UPDATE_FLAG_DBG_WA_ENABLE : UPDATE_FLAG_DBG_WA_DISABLE;
 312
 313         q->properties.is_dbg_wa = enable;
 314         err = q->device->dqm->ops.update_queue(q->device->dqm, q, &minfo);
 315         if (err)
 316                 q->properties.is_dbg_wa = false;
 317
 318         return err;
 319 }
 320
 321 static int kfd_dbg_set_workaround(struct kfd_process *target, bool enable)
 322 {
 323         struct process_queue_manager *pqm = &target->pqm;
 324         struct process_queue_node *pqn;
 325         int r = 0;
 326
 327         list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
 328                 r = kfd_dbg_set_queue_workaround(pqn->q, enable);
 329                 if (enable && r)
 330                         goto unwind;
 331         }
 332
 333         return 0;
 334
 335 unwind:
 336         list_for_each_entry(pqn, &pqm->queues, process_queue_list)
 337                 kfd_dbg_set_queue_workaround(pqn->q, false);
 338
 339         if (enable)
 340                 target->runtime_info.runtime_state = r == -EBUSY ?
 341                                 DEBUG_RUNTIME_STATE_ENABLED_BUSY :
 342                                 DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 343
 344         return r;
 345 }
 346
 347 int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en)
 348 {
 349         uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode;
 350         uint32_t flags = pdd->process->dbg_flags;
 351
 352         if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 353                 return 0;
 354
 355         return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl,
 356                                                 pdd->watch_points, flags, sq_trap_en);
 357 }
 358
 359 #define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1
 360 static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id)
 361 {
 362         int i;
 363
 364         *watch_id = KFD_DEBUGGER_INVALID_WATCH_POINT_ID;
 365
 366         spin_lock(&pdd->dev->kfd->watch_points_lock);
 367
 368         for (i = 0; i < MAX_WATCH_ADDRESSES; i++) {
 369                 /* device watchpoint in use so skip */
 370                 if ((pdd->dev->kfd->alloc_watch_ids >> i) & 0x1)
 371                         continue;
 372
 373                 pdd->alloc_watch_ids |= 0x1 << i;
 374                 pdd->dev->kfd->alloc_watch_ids |= 0x1 << i;
 375                 *watch_id = i;
 376                 spin_unlock(&pdd->dev->kfd->watch_points_lock);
 377                 return 0;
 378         }
 379
 380         spin_unlock(&pdd->dev->kfd->watch_points_lock);
 381
 382         return -ENOMEM;
 383 }
 384
 385 static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id)
 386 {
 387         spin_lock(&pdd->dev->kfd->watch_points_lock);
 388
 389         /* process owns device watch point so safe to clear */
 390         if ((pdd->alloc_watch_ids >> watch_id) & 0x1) {
 391                 pdd->alloc_watch_ids &= ~(0x1 << watch_id);
 392                 pdd->dev->kfd->alloc_watch_ids &= ~(0x1 << watch_id);
 393         }
 394
 395         spin_unlock(&pdd->dev->kfd->watch_points_lock);
 396 }
 397
 398 static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id)
 399 {
 400         bool owns_watch_id = false;
 401
 402         spin_lock(&pdd->dev->kfd->watch_points_lock);
 403         owns_watch_id = watch_id < MAX_WATCH_ADDRESSES &&
 404                         ((pdd->alloc_watch_ids >> watch_id) & 0x1);
 405
 406         spin_unlock(&pdd->dev->kfd->watch_points_lock);
 407
 408         return owns_watch_id;
 409 }
 410
 411 int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd,
 412                                         uint32_t watch_id)
 413 {
 414         int r;
 415
 416         if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id))
 417                 return -EINVAL;
 418
 419         if (!pdd->dev->kfd->shared_resources.enable_mes) {
 420                 r = debug_lock_and_unmap(pdd->dev->dqm);
 421                 if (r)
 422                         return r;
 423         }
 424
 425         amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 426         pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch(
 427                                                         pdd->dev->adev,
 428                                                         watch_id);
 429         amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 430
 431         if (!pdd->dev->kfd->shared_resources.enable_mes)
 432                 r = debug_map_and_unlock(pdd->dev->dqm);
 433         else
 434                 r = kfd_dbg_set_mes_debug_mode(pdd, true);
 435
 436         kfd_dbg_clear_dev_watch_id(pdd, watch_id);
 437
 438         return r;
 439 }
 440
 441 int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd,
 442                                         uint64_t watch_address,
 443                                         uint32_t watch_address_mask,
 444                                         uint32_t *watch_id,
 445                                         uint32_t watch_mode)
 446 {
 447         int xcc_id, r = kfd_dbg_get_dev_watch_id(pdd, watch_id);
 448         uint32_t xcc_mask = pdd->dev->xcc_mask;
 449
 450         if (r)
 451                 return r;
 452
 453         if (!pdd->dev->kfd->shared_resources.enable_mes) {
 454                 r = debug_lock_and_unmap(pdd->dev->dqm);
 455                 if (r) {
 456                         kfd_dbg_clear_dev_watch_id(pdd, *watch_id);
 457                         return r;
 458                 }
 459         }
 460
 461         amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 462         for_each_inst(xcc_id, xcc_mask)
 463                 pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch(
 464                                 pdd->dev->adev,
 465                                 watch_address,
 466                                 watch_address_mask,
 467                                 *watch_id,
 468                                 watch_mode,
 469                                 pdd->dev->vm_info.last_vmid_kfd,
 470                                 xcc_id);
 471         amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 472
 473         if (!pdd->dev->kfd->shared_resources.enable_mes)
 474                 r = debug_map_and_unlock(pdd->dev->dqm);
 475         else
 476                 r = kfd_dbg_set_mes_debug_mode(pdd, true);
 477
 478         /* HWS is broken so no point in HW rollback but release the watchpoint anyways */
 479         if (r)
 480                 kfd_dbg_clear_dev_watch_id(pdd, *watch_id);
 481
 482         return 0;
 483 }
 484
 485 static void kfd_dbg_clear_process_address_watch(struct kfd_process *target)
 486 {
 487         int i, j;
 488
 489         for (i = 0; i < target->n_pdds; i++)
 490                 for (j = 0; j < MAX_WATCH_ADDRESSES; j++)
 491                         kfd_dbg_trap_clear_dev_address_watch(target->pdds[i], j);
 492 }
 493
 494 int kfd_dbg_trap_set_flags(struct kfd_process *target, uint32_t *flags)
 495 {
 496         uint32_t prev_flags = target->dbg_flags;
 497         int i, r = 0, rewind_count = 0;
 498
 499         for (i = 0; i < target->n_pdds; i++) {
 500                 if (!kfd_dbg_is_per_vmid_supported(target->pdds[i]->dev) &&
 501                         (*flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP)) {
 502                         *flags = prev_flags;
 503                         return -EACCES;
 504                 }
 505         }
 506
 507         target->dbg_flags = *flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP;
 508         *flags = prev_flags;
 509         for (i = 0; i < target->n_pdds; i++) {
 510                 struct kfd_process_device *pdd = target->pdds[i];
 511
 512                 if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 513                         continue;
 514
 515                 if (!pdd->dev->kfd->shared_resources.enable_mes)
 516                         r = debug_refresh_runlist(pdd->dev->dqm);
 517                 else
 518                         r = kfd_dbg_set_mes_debug_mode(pdd, true);
 519
 520                 if (r) {
 521                         target->dbg_flags = prev_flags;
 522                         break;
 523                 }
 524
 525                 rewind_count++;
 526         }
 527
 528         /* Rewind flags */
 529         if (r) {
 530                 target->dbg_flags = prev_flags;
 531
 532                 for (i = 0; i < rewind_count; i++) {
 533                         struct kfd_process_device *pdd = target->pdds[i];
 534
 535                         if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 536                                 continue;
 537
 538                         if (!pdd->dev->kfd->shared_resources.enable_mes)
 539                                 debug_refresh_runlist(pdd->dev->dqm);
 540                         else
 541                                 kfd_dbg_set_mes_debug_mode(pdd, true);
 542                 }
 543         }
 544
 545         return r;
 546 }
 547
 548 /* kfd_dbg_trap_deactivate:
 549  *      target: target process
 550  *      unwind: If this is unwinding a failed kfd_dbg_trap_enable()
 551  *      unwind_count:
 552  *              If unwind == true, how far down the pdd list we need
 553  *                              to unwind
 554  *              else: ignored
 555  */
 556 void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count)
 557 {
 558         int i;
 559
 560         if (!unwind) {
 561                 uint32_t flags = 0;
 562                 int resume_count = resume_queues(target, 0, NULL);
 563
 564                 if (resume_count)
 565                         pr_debug("Resumed %d queues\n", resume_count);
 566
 567                 cancel_work_sync(&target->debug_event_workarea);
 568                 kfd_dbg_clear_process_address_watch(target);
 569                 kfd_dbg_trap_set_wave_launch_mode(target, 0);
 570
 571                 kfd_dbg_trap_set_flags(target, &flags);
 572         }
 573
 574         for (i = 0; i < target->n_pdds; i++) {
 575                 struct kfd_process_device *pdd = target->pdds[i];
 576
 577                 /* If this is an unwind, and we have unwound the required
 578                  * enable calls on the pdd list, we need to stop now
 579                  * otherwise we may mess up another debugger session.
 580                  */
 581                 if (unwind && i == unwind_count)
 582                         break;
 583
 584                 kfd_process_set_trap_debug_flag(&pdd->qpd, false);
 585
 586                 /* GFX off is already disabled by debug activate if not RLC restore supported. */
 587                 if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
 588                         amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 589                 pdd->spi_dbg_override =
 590                                 pdd->dev->kfd2kgd->disable_debug_trap(
 591                                 pdd->dev->adev,
 592                                 target->runtime_info.ttmp_setup,
 593                                 pdd->dev->vm_info.last_vmid_kfd);
 594                 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 595
 596                 if (!kfd_dbg_is_per_vmid_supported(pdd->dev) &&
 597                                 release_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd))
 598                         pr_err("Failed to release debug vmid on [%i]\n", pdd->dev->id);
 599
 600                 if (!pdd->dev->kfd->shared_resources.enable_mes)
 601                         debug_refresh_runlist(pdd->dev->dqm);
 602                 else
 603                         kfd_dbg_set_mes_debug_mode(pdd, !kfd_dbg_has_cwsr_workaround(pdd->dev));
 604         }
 605
 606         kfd_dbg_set_workaround(target, false);
 607 }
 608
 609 static void kfd_dbg_clean_exception_status(struct kfd_process *target)
 610 {
 611         struct process_queue_manager *pqm;
 612         struct process_queue_node *pqn;
 613         int i;
 614
 615         for (i = 0; i < target->n_pdds; i++) {
 616                 struct kfd_process_device *pdd = target->pdds[i];
 617
 618                 kfd_process_drain_interrupts(pdd);
 619
 620                 pdd->exception_status = 0;
 621         }
 622
 623         pqm = &target->pqm;
 624         list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
 625                 if (!pqn->q)
 626                         continue;
 627
 628                 pqn->q->properties.exception_status = 0;
 629         }
 630
 631         target->exception_status = 0;
 632 }
 633
 634 int kfd_dbg_trap_disable(struct kfd_process *target)
 635 {
 636         if (!target->debug_trap_enabled)
 637                 return 0;
 638
 639         /*
 640          * Defer deactivation to runtime if runtime not enabled otherwise reset
 641          * attached running target runtime state to enable for re-attach.
 642          */
 643         if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED)
 644                 kfd_dbg_trap_deactivate(target, false, 0);
 645         else if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
 646                 target->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
 647
 648         fput(target->dbg_ev_file);
 649         target->dbg_ev_file = NULL;
 650
 651         if (target->debugger_process) {
 652                 atomic_dec(&target->debugger_process->debugged_process_count);
 653                 target->debugger_process = NULL;
 654         }
 655
 656         target->debug_trap_enabled = false;
 657         kfd_dbg_clean_exception_status(target);
 658         kfd_unref_process(target);
 659
 660         return 0;
 661 }
 662
 663 int kfd_dbg_trap_activate(struct kfd_process *target)
 664 {
 665         int i, r = 0;
 666
 667         r = kfd_dbg_set_workaround(target, true);
 668         if (r)
 669                 return r;
 670
 671         for (i = 0; i < target->n_pdds; i++) {
 672                 struct kfd_process_device *pdd = target->pdds[i];
 673
 674                 if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) {
 675                         r = reserve_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd);
 676
 677                         if (r) {
 678                                 target->runtime_info.runtime_state = (r == -EBUSY) ?
 679                                                         DEBUG_RUNTIME_STATE_ENABLED_BUSY :
 680                                                         DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 681
 682                                 goto unwind_err;
 683                         }
 684                 }
 685
 686                 /* Disable GFX OFF to prevent garbage read/writes to debug registers.
 687                  * If RLC restore of debug registers is not supported and runtime enable
 688                  * hasn't done so already on ttmp setup request, restore the trap config registers.
 689                  *
 690                  * If RLC restore of debug registers is not supported, keep gfx off disabled for
 691                  * the debug session.
 692                  */
 693                 amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 694                 if (!(kfd_dbg_is_rlc_restore_supported(pdd->dev) ||
 695                                                 target->runtime_info.ttmp_setup))
 696                         pdd->dev->kfd2kgd->enable_debug_trap(pdd->dev->adev, true,
 697                                                                 pdd->dev->vm_info.last_vmid_kfd);
 698
 699                 pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
 700                                         pdd->dev->adev,
 701                                         false,
 702                                         pdd->dev->vm_info.last_vmid_kfd);
 703
 704                 if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
 705                         amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 706
 707                 /*
 708                  * Setting the debug flag in the trap handler requires that the TMA has been
 709                  * allocated, which occurs during CWSR initialization.
 710                  * In the event that CWSR has not been initialized at this point, setting the
 711                  * flag will be called again during CWSR initialization if the target process
 712                  * is still debug enabled.
 713                  */
 714                 kfd_process_set_trap_debug_flag(&pdd->qpd, true);
 715
 716                 if (!pdd->dev->kfd->shared_resources.enable_mes)
 717                         r = debug_refresh_runlist(pdd->dev->dqm);
 718                 else
 719                         r = kfd_dbg_set_mes_debug_mode(pdd, true);
 720
 721                 if (r) {
 722                         target->runtime_info.runtime_state =
 723                                         DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 724                         goto unwind_err;
 725                 }
 726         }
 727
 728         return 0;
 729
 730 unwind_err:
 731         /* Enabling debug failed, we need to disable on
 732          * all GPUs so the enable is all or nothing.
 733          */
 734         kfd_dbg_trap_deactivate(target, true, i);
 735         return r;
 736 }
 737
 738 int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd,
 739                         void __user *runtime_info, uint32_t *runtime_size)
 740 {
 741         struct file *f;
 742         uint32_t copy_size;
 743         int i, r = 0;
 744
 745         if (target->debug_trap_enabled)
 746                 return -EALREADY;
 747
 748         /* Enable pre-checks */
 749         for (i = 0; i < target->n_pdds; i++) {
 750                 struct kfd_process_device *pdd = target->pdds[i];
 751
 752                 if (!KFD_IS_SOC15(pdd->dev))
 753                         return -ENODEV;
 754
 755                 if (pdd->qpd.num_gws && (!kfd_dbg_has_gws_support(pdd->dev) ||
 756                                          kfd_dbg_has_cwsr_workaround(pdd->dev)))
 757                         return -EBUSY;
 758         }
 759
 760         copy_size = min((size_t)(*runtime_size), sizeof(target->runtime_info));
 761
 762         f = fget(fd);
 763         if (!f) {
 764                 pr_err("Failed to get file for (%i)\n", fd);
 765                 return -EBADF;
 766         }
 767
 768         target->dbg_ev_file = f;
 769
 770         /* defer activation to runtime if not runtime enabled */
 771         if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED)
 772                 kfd_dbg_trap_activate(target);
 773
 774         /* We already hold the process reference but hold another one for the
 775          * debug session.
 776          */
 777         kref_get(&target->ref);
 778         target->debug_trap_enabled = true;
 779
 780         if (target->debugger_process)
 781                 atomic_inc(&target->debugger_process->debugged_process_count);
 782
 783         if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) {
 784                 kfd_dbg_trap_deactivate(target, false, 0);
 785                 r = -EFAULT;
 786         }
 787
 788         *runtime_size = sizeof(target->runtime_info);
 789
 790         return r;
 791 }
 792
 793 static int kfd_dbg_validate_trap_override_request(struct kfd_process *p,
 794                                                 uint32_t trap_override,
 795                                                 uint32_t trap_mask_request,
 796                                                 uint32_t *trap_mask_supported)
 797 {
 798         int i = 0;
 799
 800         *trap_mask_supported = 0xffffffff;
 801
 802         for (i = 0; i < p->n_pdds; i++) {
 803                 struct kfd_process_device *pdd = p->pdds[i];
 804                 int err = pdd->dev->kfd2kgd->validate_trap_override_request(
 805                                                                 pdd->dev->adev,
 806                                                                 trap_override,
 807                                                                 trap_mask_supported);
 808
 809                 if (err)
 810                         return err;
 811         }
 812
 813         if (trap_mask_request & ~*trap_mask_supported)
 814                 return -EACCES;
 815
 816         return 0;
 817 }
 818
 819 int kfd_dbg_trap_set_wave_launch_override(struct kfd_process *target,
 820                                         uint32_t trap_override,
 821                                         uint32_t trap_mask_bits,
 822                                         uint32_t trap_mask_request,
 823                                         uint32_t *trap_mask_prev,
 824                                         uint32_t *trap_mask_supported)
 825 {
 826         int r = 0, i;
 827
 828         r = kfd_dbg_validate_trap_override_request(target,
 829                                                 trap_override,
 830                                                 trap_mask_request,
 831                                                 trap_mask_supported);
 832
 833         if (r)
 834                 return r;
 835
 836         for (i = 0; i < target->n_pdds; i++) {
 837                 struct kfd_process_device *pdd = target->pdds[i];
 838
 839                 amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 840                 pdd->spi_dbg_override = pdd->dev->kfd2kgd->set_wave_launch_trap_override(
 841                                 pdd->dev->adev,
 842                                 pdd->dev->vm_info.last_vmid_kfd,
 843                                 trap_override,
 844                                 trap_mask_bits,
 845                                 trap_mask_request,
 846                                 trap_mask_prev,
 847                                 pdd->spi_dbg_override);
 848                 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 849
 850                 if (!pdd->dev->kfd->shared_resources.enable_mes)
 851                         r = debug_refresh_runlist(pdd->dev->dqm);
 852                 else
 853                         r = kfd_dbg_set_mes_debug_mode(pdd, true);
 854
 855                 if (r)
 856                         break;
 857         }
 858
 859         return r;
 860 }
 861
 862 int kfd_dbg_trap_set_wave_launch_mode(struct kfd_process *target,
 863                                         uint8_t wave_launch_mode)
 864 {
 865         int r = 0, i;
 866
 867         if (wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_NORMAL &&
 868                         wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_HALT &&
 869                         wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_DEBUG)
 870                 return -EINVAL;
 871
 872         for (i = 0; i < target->n_pdds; i++) {
 873                 struct kfd_process_device *pdd = target->pdds[i];
 874
 875                 amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 876                 pdd->spi_dbg_launch_mode = pdd->dev->kfd2kgd->set_wave_launch_mode(
 877                                 pdd->dev->adev,
 878                                 wave_launch_mode,
 879                                 pdd->dev->vm_info.last_vmid_kfd);
 880                 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 881
 882                 if (!pdd->dev->kfd->shared_resources.enable_mes)
 883                         r = debug_refresh_runlist(pdd->dev->dqm);
 884                 else
 885                         r = kfd_dbg_set_mes_debug_mode(pdd, true);
 886
 887                 if (r)
 888                         break;
 889         }
 890
 891         return r;
 892 }
 893
 894 int kfd_dbg_trap_query_exception_info(struct kfd_process *target,
 895                 uint32_t source_id,
 896                 uint32_t exception_code,
 897                 bool clear_exception,
 898                 void __user *info,
 899                 uint32_t *info_size)
 900 {
 901         bool found = false;
 902         int r = 0;
 903         uint32_t copy_size, actual_info_size = 0;
 904         uint64_t *exception_status_ptr = NULL;
 905
 906         if (!target)
 907                 return -EINVAL;
 908
 909         if (!info || !info_size)
 910                 return -EINVAL;
 911
 912         mutex_lock(&target->event_mutex);
 913
 914         if (KFD_DBG_EC_TYPE_IS_QUEUE(exception_code)) {
 915                 /* Per queue exceptions */
 916                 struct queue *queue = NULL;
 917                 int i;
 918
 919                 for (i = 0; i < target->n_pdds; i++) {
 920                         struct kfd_process_device *pdd = target->pdds[i];
 921                         struct qcm_process_device *qpd = &pdd->qpd;
 922
 923                         list_for_each_entry(queue, &qpd->queues_list, list) {
 924                                 if (!found && queue->properties.queue_id == source_id) {
 925                                         found = true;
 926                                         break;
 927                                 }
 928                         }
 929                         if (found)
 930                                 break;
 931                 }
 932
 933                 if (!found) {
 934                         r = -EINVAL;
 935                         goto out;
 936                 }
 937
 938                 if (!(queue->properties.exception_status & KFD_EC_MASK(exception_code))) {
 939                         r = -ENODATA;
 940                         goto out;
 941                 }
 942                 exception_status_ptr = &queue->properties.exception_status;
 943         } else if (KFD_DBG_EC_TYPE_IS_DEVICE(exception_code)) {
 944                 /* Per device exceptions */
 945                 struct kfd_process_device *pdd = NULL;
 946                 int i;
 947
 948                 for (i = 0; i < target->n_pdds; i++) {
 949                         pdd = target->pdds[i];
 950                         if (pdd->dev->id == source_id) {
 951                                 found = true;
 952                                 break;
 953                         }
 954                 }
 955
 956                 if (!found) {
 957                         r = -EINVAL;
 958                         goto out;
 959                 }
 960
 961                 if (!(pdd->exception_status & KFD_EC_MASK(exception_code))) {
 962                         r = -ENODATA;
 963                         goto out;
 964                 }
 965
 966                 if (exception_code == EC_DEVICE_MEMORY_VIOLATION) {
 967                         copy_size = min((size_t)(*info_size), pdd->vm_fault_exc_data_size);
 968
 969                         if (copy_to_user(info, pdd->vm_fault_exc_data, copy_size)) {
 970                                 r = -EFAULT;
 971                                 goto out;
 972                         }
 973                         actual_info_size = pdd->vm_fault_exc_data_size;
 974                         if (clear_exception) {
 975                                 kfree(pdd->vm_fault_exc_data);
 976                                 pdd->vm_fault_exc_data = NULL;
 977                                 pdd->vm_fault_exc_data_size = 0;
 978                         }
 979                 }
 980                 exception_status_ptr = &pdd->exception_status;
 981         } else if (KFD_DBG_EC_TYPE_IS_PROCESS(exception_code)) {
 982                 /* Per process exceptions */
 983                 if (!(target->exception_status & KFD_EC_MASK(exception_code))) {
 984                         r = -ENODATA;
 985                         goto out;
 986                 }
 987
 988                 if (exception_code == EC_PROCESS_RUNTIME) {
 989                         copy_size = min((size_t)(*info_size), sizeof(target->runtime_info));
 990
 991                         if (copy_to_user(info, (void *)&target->runtime_info, copy_size)) {
 992                                 r = -EFAULT;
 993                                 goto out;
 994                         }
 995
 996                         actual_info_size = sizeof(target->runtime_info);
 997                 }
 998
 999                 exception_status_ptr = &target->exception_status;
1000         } else {
1001                 pr_debug("Bad exception type [%i]\n", exception_code);
1002                 r = -EINVAL;
1003                 goto out;
1004         }
1005
1006         *info_size = actual_info_size;
1007         if (clear_exception)
1008                 *exception_status_ptr &= ~KFD_EC_MASK(exception_code);
1009 out:
1010         mutex_unlock(&target->event_mutex);
1011         return r;
1012 }
1013
1014 int kfd_dbg_trap_device_snapshot(struct kfd_process *target,
1015                 uint64_t exception_clear_mask,
1016                 void __user *user_info,
1017                 uint32_t *number_of_device_infos,
1018                 uint32_t *entry_size)
1019 {
1020         struct kfd_dbg_device_info_entry device_info;
1021         uint32_t tmp_entry_size = *entry_size, tmp_num_devices;
1022         int i, r = 0;
1023
1024         if (!(target && user_info && number_of_device_infos && entry_size))
1025                 return -EINVAL;
1026
1027         tmp_num_devices = min_t(size_t, *number_of_device_infos, target->n_pdds);
1028         *number_of_device_infos = target->n_pdds;
1029         *entry_size = min_t(size_t, *entry_size, sizeof(device_info));
1030
1031         if (!tmp_num_devices)
1032                 return 0;
1033
1034         memset(&device_info, 0, sizeof(device_info));
1035
1036         mutex_lock(&target->event_mutex);
1037
1038         /* Run over all pdd of the process */
1039         for (i = 0; i < tmp_num_devices; i++) {
1040                 struct kfd_process_device *pdd = target->pdds[i];
1041                 struct kfd_topology_device *topo_dev = kfd_topology_device_by_id(pdd->dev->id);
1042
1043                 device_info.gpu_id = pdd->dev->id;
1044                 device_info.exception_status = pdd->exception_status;
1045                 device_info.lds_base = pdd->lds_base;
1046                 device_info.lds_limit = pdd->lds_limit;
1047                 device_info.scratch_base = pdd->scratch_base;
1048                 device_info.scratch_limit = pdd->scratch_limit;
1049                 device_info.gpuvm_base = pdd->gpuvm_base;
1050                 device_info.gpuvm_limit = pdd->gpuvm_limit;
1051                 device_info.location_id = topo_dev->node_props.location_id;
1052                 device_info.vendor_id = topo_dev->node_props.vendor_id;
1053                 device_info.device_id = topo_dev->node_props.device_id;
1054                 device_info.revision_id = pdd->dev->adev->pdev->revision;
1055                 device_info.subsystem_vendor_id = pdd->dev->adev->pdev->subsystem_vendor;
1056                 device_info.subsystem_device_id = pdd->dev->adev->pdev->subsystem_device;
1057                 device_info.fw_version = pdd->dev->kfd->mec_fw_version;
1058                 device_info.gfx_target_version =
1059                         topo_dev->node_props.gfx_target_version;
1060                 device_info.simd_count = topo_dev->node_props.simd_count;
1061                 device_info.max_waves_per_simd =
1062                         topo_dev->node_props.max_waves_per_simd;
1063                 device_info.array_count = topo_dev->node_props.array_count;
1064                 device_info.simd_arrays_per_engine =
1065                         topo_dev->node_props.simd_arrays_per_engine;
1066                 device_info.num_xcc = NUM_XCC(pdd->dev->xcc_mask);
1067                 device_info.capability = topo_dev->node_props.capability;
1068                 device_info.debug_prop = topo_dev->node_props.debug_prop;
1069
1070                 if (exception_clear_mask)
1071                         pdd->exception_status &= ~exception_clear_mask;
1072
1073                 if (copy_to_user(user_info, &device_info, *entry_size)) {
1074                         r = -EFAULT;
1075                         break;
1076                 }
1077
1078                 user_info += tmp_entry_size;
1079         }
1080
1081         mutex_unlock(&target->event_mutex);
1082
1083         return r;
1084 }
1085
1086 void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target,
1087                                         uint64_t exception_set_mask)
1088 {
1089         uint64_t found_mask = 0;
1090         struct process_queue_manager *pqm;
1091         struct process_queue_node *pqn;
1092         static const char write_data = '.';
1093         loff_t pos = 0;
1094         int i;
1095
1096         mutex_lock(&target->event_mutex);
1097
1098         found_mask |= target->exception_status;
1099
1100         pqm = &target->pqm;
1101         list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
1102                 if (!pqn->q)
1103                         continue;
1104
1105                 found_mask |= pqn->q->properties.exception_status;
1106         }
1107
1108         for (i = 0; i < target->n_pdds; i++) {
1109                 struct kfd_process_device *pdd = target->pdds[i];
1110
1111                 found_mask |= pdd->exception_status;
1112         }
1113
1114         if (exception_set_mask & found_mask)
1115                 kernel_write(target->dbg_ev_file, &write_data, 1, &pos);
1116
1117         target->exception_enable_mask = exception_set_mask;
1118
1119         mutex_unlock(&target->event_mutex);
1120 }