drivers/gpu/drm/amd/amdkfd/kfd_process.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/mutex.h>
  25 #include <linux/log2.h>
  26 #include <linux/sched.h>
  27 #include <linux/sched/mm.h>
  28 #include <linux/sched/task.h>
  29 #include <linux/mmu_context.h>
  30 #include <linux/slab.h>
  31 #include <linux/notifier.h>
  32 #include <linux/compat.h>
  33 #include <linux/mman.h>
  34 #include <linux/file.h>
  35 #include <linux/pm_runtime.h>
  36 #include "amdgpu_amdkfd.h"
  37 #include "amdgpu.h"
  38
  39 struct mm_struct;
  40
  41 #include "kfd_priv.h"
  42 #include "kfd_device_queue_manager.h"
  43 #include "kfd_svm.h"
  44 #include "kfd_smi_events.h"
  45 #include "kfd_debug.h"
  46
  47 /*
  48  * List of struct kfd_process (field kfd_process).
  49  * Unique/indexed by mm_struct*
  50  */
  51 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
  52 DEFINE_MUTEX(kfd_processes_mutex);
  53
  54 DEFINE_SRCU(kfd_processes_srcu);
  55
  56 /* For process termination handling */
  57 static struct workqueue_struct *kfd_process_wq;
  58
  59 /* Ordered, single-threaded workqueue for restoring evicted
  60  * processes. Restoring multiple processes concurrently under memory
  61  * pressure can lead to processes blocking each other from validating
  62  * their BOs and result in a live-lock situation where processes
  63  * remain evicted indefinitely.
  64  */
  65 static struct workqueue_struct *kfd_restore_wq;
  66
  67 static struct kfd_process *find_process(const struct task_struct *thread,
  68                                         bool ref);
  69 static void kfd_process_ref_release(struct kref *ref);
  70 static struct kfd_process *create_process(const struct task_struct *thread);
  71
  72 static void evict_process_worker(struct work_struct *work);
  73 static void restore_process_worker(struct work_struct *work);
  74
  75 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
  76
  77 struct kfd_procfs_tree {
  78         struct kobject *kobj;
  79 };
  80
  81 static struct kfd_procfs_tree procfs;
  82
  83 /*
  84  * Structure for SDMA activity tracking
  85  */
  86 struct kfd_sdma_activity_handler_workarea {
  87         struct work_struct sdma_activity_work;
  88         struct kfd_process_device *pdd;
  89         uint64_t sdma_activity_counter;
  90 };
  91
  92 struct temp_sdma_queue_list {
  93         uint64_t __user *rptr;
  94         uint64_t sdma_val;
  95         unsigned int queue_id;
  96         struct list_head list;
  97 };
  98
  99 static void kfd_sdma_activity_worker(struct work_struct *work)
 100 {
 101         struct kfd_sdma_activity_handler_workarea *workarea;
 102         struct kfd_process_device *pdd;
 103         uint64_t val;
 104         struct mm_struct *mm;
 105         struct queue *q;
 106         struct qcm_process_device *qpd;
 107         struct device_queue_manager *dqm;
 108         int ret = 0;
 109         struct temp_sdma_queue_list sdma_q_list;
 110         struct temp_sdma_queue_list *sdma_q, *next;
 111
 112         workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
 113                                 sdma_activity_work);
 114
 115         pdd = workarea->pdd;
 116         if (!pdd)
 117                 return;
 118         dqm = pdd->dev->dqm;
 119         qpd = &pdd->qpd;
 120         if (!dqm || !qpd)
 121                 return;
 122         /*
 123          * Total SDMA activity is current SDMA activity + past SDMA activity
 124          * Past SDMA count is stored in pdd.
 125          * To get the current activity counters for all active SDMA queues,
 126          * we loop over all SDMA queues and get their counts from user-space.
 127          *
 128          * We cannot call get_user() with dqm_lock held as it can cause
 129          * a circular lock dependency situation. To read the SDMA stats,
 130          * we need to do the following:
 131          *
 132          * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
 133          *    with dqm_lock/dqm_unlock().
 134          * 2. Call get_user() for each node in temporary list without dqm_lock.
 135          *    Save the SDMA count for each node and also add the count to the total
 136          *    SDMA count counter.
 137          *    Its possible, during this step, a few SDMA queue nodes got deleted
 138          *    from the qpd->queues_list.
 139          * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
 140          *    If any node got deleted, its SDMA count would be captured in the sdma
 141          *    past activity counter. So subtract the SDMA counter stored in step 2
 142          *    for this node from the total SDMA count.
 143          */
 144         INIT_LIST_HEAD(&sdma_q_list.list);
 145
 146         /*
 147          * Create the temp list of all SDMA queues
 148          */
 149         dqm_lock(dqm);
 150
 151         list_for_each_entry(q, &qpd->queues_list, list) {
 152                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 153                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 154                         continue;
 155
 156                 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
 157                 if (!sdma_q) {
 158                         dqm_unlock(dqm);
 159                         goto cleanup;
 160                 }
 161
 162                 INIT_LIST_HEAD(&sdma_q->list);
 163                 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
 164                 sdma_q->queue_id = q->properties.queue_id;
 165                 list_add_tail(&sdma_q->list, &sdma_q_list.list);
 166         }
 167
 168         /*
 169          * If the temp list is empty, then no SDMA queues nodes were found in
 170          * qpd->queues_list. Return the past activity count as the total sdma
 171          * count
 172          */
 173         if (list_empty(&sdma_q_list.list)) {
 174                 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
 175                 dqm_unlock(dqm);
 176                 return;
 177         }
 178
 179         dqm_unlock(dqm);
 180
 181         /*
 182          * Get the usage count for each SDMA queue in temp_list.
 183          */
 184         mm = get_task_mm(pdd->process->lead_thread);
 185         if (!mm)
 186                 goto cleanup;
 187
 188         kthread_use_mm(mm);
 189
 190         list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
 191                 val = 0;
 192                 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
 193                 if (ret) {
 194                         pr_debug("Failed to read SDMA queue active counter for queue id: %d",
 195                                  sdma_q->queue_id);
 196                 } else {
 197                         sdma_q->sdma_val = val;
 198                         workarea->sdma_activity_counter += val;
 199                 }
 200         }
 201
 202         kthread_unuse_mm(mm);
 203         mmput(mm);
 204
 205         /*
 206          * Do a second iteration over qpd_queues_list to check if any SDMA
 207          * nodes got deleted while fetching SDMA counter.
 208          */
 209         dqm_lock(dqm);
 210
 211         workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
 212
 213         list_for_each_entry(q, &qpd->queues_list, list) {
 214                 if (list_empty(&sdma_q_list.list))
 215                         break;
 216
 217                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 218                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 219                         continue;
 220
 221                 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 222                         if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
 223                              (sdma_q->queue_id == q->properties.queue_id)) {
 224                                 list_del(&sdma_q->list);
 225                                 kfree(sdma_q);
 226                                 break;
 227                         }
 228                 }
 229         }
 230
 231         dqm_unlock(dqm);
 232
 233         /*
 234          * If temp list is not empty, it implies some queues got deleted
 235          * from qpd->queues_list during SDMA usage read. Subtract the SDMA
 236          * count for each node from the total SDMA count.
 237          */
 238         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 239                 workarea->sdma_activity_counter -= sdma_q->sdma_val;
 240                 list_del(&sdma_q->list);
 241                 kfree(sdma_q);
 242         }
 243
 244         return;
 245
 246 cleanup:
 247         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 248                 list_del(&sdma_q->list);
 249                 kfree(sdma_q);
 250         }
 251 }
 252
 253 /**
 254  * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
 255  * by current process. Translates acquired wave count into number of compute units
 256  * that are occupied.
 257  *
 258  * @attr: Handle of attribute that allows reporting of wave count. The attribute
 259  * handle encapsulates GPU device it is associated with, thereby allowing collection
 260  * of waves in flight, etc
 261  * @buffer: Handle of user provided buffer updated with wave count
 262  *
 263  * Return: Number of bytes written to user buffer or an error value
 264  */
 265 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
 266 {
 267         int cu_cnt;
 268         int wave_cnt;
 269         int max_waves_per_cu;
 270         struct kfd_node *dev = NULL;
 271         struct kfd_process *proc = NULL;
 272         struct kfd_process_device *pdd = NULL;
 273
 274         pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
 275         dev = pdd->dev;
 276         if (dev->kfd2kgd->get_cu_occupancy == NULL)
 277                 return -EINVAL;
 278
 279         cu_cnt = 0;
 280         proc = pdd->process;
 281         if (pdd->qpd.queue_count == 0) {
 282                 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
 283                          dev->id, proc->pasid);
 284                 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 285         }
 286
 287         /* Collect wave count from device if it supports */
 288         wave_cnt = 0;
 289         max_waves_per_cu = 0;
 290         dev->kfd2kgd->get_cu_occupancy(dev->adev, proc->pasid, &wave_cnt,
 291                         &max_waves_per_cu, 0);
 292
 293         /* Translate wave count to number of compute units */
 294         cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
 295         return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 296 }
 297
 298 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 299                                char *buffer)
 300 {
 301         if (strcmp(attr->name, "pasid") == 0) {
 302                 struct kfd_process *p = container_of(attr, struct kfd_process,
 303                                                      attr_pasid);
 304
 305                 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
 306         } else if (strncmp(attr->name, "vram_", 5) == 0) {
 307                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 308                                                               attr_vram);
 309                 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
 310         } else if (strncmp(attr->name, "sdma_", 5) == 0) {
 311                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 312                                                               attr_sdma);
 313                 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
 314
 315                 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
 316                                         kfd_sdma_activity_worker);
 317
 318                 sdma_activity_work_handler.pdd = pdd;
 319                 sdma_activity_work_handler.sdma_activity_counter = 0;
 320
 321                 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
 322
 323                 flush_work(&sdma_activity_work_handler.sdma_activity_work);
 324
 325                 return snprintf(buffer, PAGE_SIZE, "%llu\n",
 326                                 (sdma_activity_work_handler.sdma_activity_counter)/
 327                                  SDMA_ACTIVITY_DIVISOR);
 328         } else {
 329                 pr_err("Invalid attribute");
 330                 return -EINVAL;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static void kfd_procfs_kobj_release(struct kobject *kobj)
 337 {
 338         kfree(kobj);
 339 }
 340
 341 static const struct sysfs_ops kfd_procfs_ops = {
 342         .show = kfd_procfs_show,
 343 };
 344
 345 static const struct kobj_type procfs_type = {
 346         .release = kfd_procfs_kobj_release,
 347         .sysfs_ops = &kfd_procfs_ops,
 348 };
 349
 350 void kfd_procfs_init(void)
 351 {
 352         int ret = 0;
 353
 354         procfs.kobj = kfd_alloc_struct(procfs.kobj);
 355         if (!procfs.kobj)
 356                 return;
 357
 358         ret = kobject_init_and_add(procfs.kobj, &procfs_type,
 359                                    &kfd_device->kobj, "proc");
 360         if (ret) {
 361                 pr_warn("Could not create procfs proc folder");
 362                 /* If we fail to create the procfs, clean up */
 363                 kfd_procfs_shutdown();
 364         }
 365 }
 366
 367 void kfd_procfs_shutdown(void)
 368 {
 369         if (procfs.kobj) {
 370                 kobject_del(procfs.kobj);
 371                 kobject_put(procfs.kobj);
 372                 procfs.kobj = NULL;
 373         }
 374 }
 375
 376 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
 377                                      struct attribute *attr, char *buffer)
 378 {
 379         struct queue *q = container_of(kobj, struct queue, kobj);
 380
 381         if (!strcmp(attr->name, "size"))
 382                 return snprintf(buffer, PAGE_SIZE, "%llu",
 383                                 q->properties.queue_size);
 384         else if (!strcmp(attr->name, "type"))
 385                 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
 386         else if (!strcmp(attr->name, "gpuid"))
 387                 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
 388         else
 389                 pr_err("Invalid attribute");
 390
 391         return 0;
 392 }
 393
 394 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
 395                                      struct attribute *attr, char *buffer)
 396 {
 397         if (strcmp(attr->name, "evicted_ms") == 0) {
 398                 struct kfd_process_device *pdd = container_of(attr,
 399                                 struct kfd_process_device,
 400                                 attr_evict);
 401                 uint64_t evict_jiffies;
 402
 403                 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
 404
 405                 return snprintf(buffer,
 406                                 PAGE_SIZE,
 407                                 "%llu\n",
 408                                 jiffies64_to_msecs(evict_jiffies));
 409
 410         /* Sysfs handle that gets CU occupancy is per device */
 411         } else if (strcmp(attr->name, "cu_occupancy") == 0) {
 412                 return kfd_get_cu_occupancy(attr, buffer);
 413         } else {
 414                 pr_err("Invalid attribute");
 415         }
 416
 417         return 0;
 418 }
 419
 420 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
 421                                        struct attribute *attr, char *buf)
 422 {
 423         struct kfd_process_device *pdd;
 424
 425         if (!strcmp(attr->name, "faults")) {
 426                 pdd = container_of(attr, struct kfd_process_device,
 427                                    attr_faults);
 428                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
 429         }
 430         if (!strcmp(attr->name, "page_in")) {
 431                 pdd = container_of(attr, struct kfd_process_device,
 432                                    attr_page_in);
 433                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
 434         }
 435         if (!strcmp(attr->name, "page_out")) {
 436                 pdd = container_of(attr, struct kfd_process_device,
 437                                    attr_page_out);
 438                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
 439         }
 440         return 0;
 441 }
 442
 443 static struct attribute attr_queue_size = {
 444         .name = "size",
 445         .mode = KFD_SYSFS_FILE_MODE
 446 };
 447
 448 static struct attribute attr_queue_type = {
 449         .name = "type",
 450         .mode = KFD_SYSFS_FILE_MODE
 451 };
 452
 453 static struct attribute attr_queue_gpuid = {
 454         .name = "gpuid",
 455         .mode = KFD_SYSFS_FILE_MODE
 456 };
 457
 458 static struct attribute *procfs_queue_attrs[] = {
 459         &attr_queue_size,
 460         &attr_queue_type,
 461         &attr_queue_gpuid,
 462         NULL
 463 };
 464 ATTRIBUTE_GROUPS(procfs_queue);
 465
 466 static const struct sysfs_ops procfs_queue_ops = {
 467         .show = kfd_procfs_queue_show,
 468 };
 469
 470 static const struct kobj_type procfs_queue_type = {
 471         .sysfs_ops = &procfs_queue_ops,
 472         .default_groups = procfs_queue_groups,
 473 };
 474
 475 static const struct sysfs_ops procfs_stats_ops = {
 476         .show = kfd_procfs_stats_show,
 477 };
 478
 479 static const struct kobj_type procfs_stats_type = {
 480         .sysfs_ops = &procfs_stats_ops,
 481         .release = kfd_procfs_kobj_release,
 482 };
 483
 484 static const struct sysfs_ops sysfs_counters_ops = {
 485         .show = kfd_sysfs_counters_show,
 486 };
 487
 488 static const struct kobj_type sysfs_counters_type = {
 489         .sysfs_ops = &sysfs_counters_ops,
 490         .release = kfd_procfs_kobj_release,
 491 };
 492
 493 int kfd_procfs_add_queue(struct queue *q)
 494 {
 495         struct kfd_process *proc;
 496         int ret;
 497
 498         if (!q || !q->process)
 499                 return -EINVAL;
 500         proc = q->process;
 501
 502         /* Create proc/<pid>/queues/<queue id> folder */
 503         if (!proc->kobj_queues)
 504                 return -EFAULT;
 505         ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
 506                         proc->kobj_queues, "%u", q->properties.queue_id);
 507         if (ret < 0) {
 508                 pr_warn("Creating proc/<pid>/queues/%u failed",
 509                         q->properties.queue_id);
 510                 kobject_put(&q->kobj);
 511                 return ret;
 512         }
 513
 514         return 0;
 515 }
 516
 517 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
 518                                  char *name)
 519 {
 520         int ret;
 521
 522         if (!kobj || !attr || !name)
 523                 return;
 524
 525         attr->name = name;
 526         attr->mode = KFD_SYSFS_FILE_MODE;
 527         sysfs_attr_init(attr);
 528
 529         ret = sysfs_create_file(kobj, attr);
 530         if (ret)
 531                 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
 532 }
 533
 534 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 535 {
 536         int ret;
 537         int i;
 538         char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
 539
 540         if (!p || !p->kobj)
 541                 return;
 542
 543         /*
 544          * Create sysfs files for each GPU:
 545          * - proc/<pid>/stats_<gpuid>/
 546          * - proc/<pid>/stats_<gpuid>/evicted_ms
 547          * - proc/<pid>/stats_<gpuid>/cu_occupancy
 548          */
 549         for (i = 0; i < p->n_pdds; i++) {
 550                 struct kfd_process_device *pdd = p->pdds[i];
 551
 552                 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
 553                                 "stats_%u", pdd->dev->id);
 554                 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
 555                 if (!pdd->kobj_stats)
 556                         return;
 557
 558                 ret = kobject_init_and_add(pdd->kobj_stats,
 559                                            &procfs_stats_type,
 560                                            p->kobj,
 561                                            stats_dir_filename);
 562
 563                 if (ret) {
 564                         pr_warn("Creating KFD proc/stats_%s folder failed",
 565                                 stats_dir_filename);
 566                         kobject_put(pdd->kobj_stats);
 567                         pdd->kobj_stats = NULL;
 568                         return;
 569                 }
 570
 571                 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
 572                                       "evicted_ms");
 573                 /* Add sysfs file to report compute unit occupancy */
 574                 if (pdd->dev->kfd2kgd->get_cu_occupancy)
 575                         kfd_sysfs_create_file(pdd->kobj_stats,
 576                                               &pdd->attr_cu_occupancy,
 577                                               "cu_occupancy");
 578         }
 579 }
 580
 581 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
 582 {
 583         int ret = 0;
 584         int i;
 585         char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
 586
 587         if (!p || !p->kobj)
 588                 return;
 589
 590         /*
 591          * Create sysfs files for each GPU which supports SVM
 592          * - proc/<pid>/counters_<gpuid>/
 593          * - proc/<pid>/counters_<gpuid>/faults
 594          * - proc/<pid>/counters_<gpuid>/page_in
 595          * - proc/<pid>/counters_<gpuid>/page_out
 596          */
 597         for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
 598                 struct kfd_process_device *pdd = p->pdds[i];
 599                 struct kobject *kobj_counters;
 600
 601                 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
 602                         "counters_%u", pdd->dev->id);
 603                 kobj_counters = kfd_alloc_struct(kobj_counters);
 604                 if (!kobj_counters)
 605                         return;
 606
 607                 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
 608                                            p->kobj, counters_dir_filename);
 609                 if (ret) {
 610                         pr_warn("Creating KFD proc/%s folder failed",
 611                                 counters_dir_filename);
 612                         kobject_put(kobj_counters);
 613                         return;
 614                 }
 615
 616                 pdd->kobj_counters = kobj_counters;
 617                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
 618                                       "faults");
 619                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
 620                                       "page_in");
 621                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
 622                                       "page_out");
 623         }
 624 }
 625
 626 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
 627 {
 628         int i;
 629
 630         if (!p || !p->kobj)
 631                 return;
 632
 633         /*
 634          * Create sysfs files for each GPU:
 635          * - proc/<pid>/vram_<gpuid>
 636          * - proc/<pid>/sdma_<gpuid>
 637          */
 638         for (i = 0; i < p->n_pdds; i++) {
 639                 struct kfd_process_device *pdd = p->pdds[i];
 640
 641                 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
 642                          pdd->dev->id);
 643                 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
 644                                       pdd->vram_filename);
 645
 646                 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
 647                          pdd->dev->id);
 648                 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
 649                                             pdd->sdma_filename);
 650         }
 651 }
 652
 653 void kfd_procfs_del_queue(struct queue *q)
 654 {
 655         if (!q)
 656                 return;
 657
 658         kobject_del(&q->kobj);
 659         kobject_put(&q->kobj);
 660 }
 661
 662 int kfd_process_create_wq(void)
 663 {
 664         if (!kfd_process_wq)
 665                 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
 666         if (!kfd_restore_wq)
 667                 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
 668
 669         if (!kfd_process_wq || !kfd_restore_wq) {
 670                 kfd_process_destroy_wq();
 671                 return -ENOMEM;
 672         }
 673
 674         return 0;
 675 }
 676
 677 void kfd_process_destroy_wq(void)
 678 {
 679         if (kfd_process_wq) {
 680                 destroy_workqueue(kfd_process_wq);
 681                 kfd_process_wq = NULL;
 682         }
 683         if (kfd_restore_wq) {
 684                 destroy_workqueue(kfd_restore_wq);
 685                 kfd_restore_wq = NULL;
 686         }
 687 }
 688
 689 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 690                         struct kfd_process_device *pdd, void **kptr)
 691 {
 692         struct kfd_node *dev = pdd->dev;
 693
 694         if (kptr && *kptr) {
 695                 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
 696                 *kptr = NULL;
 697         }
 698
 699         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
 700         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
 701                                                NULL);
 702 }
 703
 704 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
 705  *      This function should be only called right after the process
 706  *      is created and when kfd_processes_mutex is still being held
 707  *      to avoid concurrency. Because of that exclusiveness, we do
 708  *      not need to take p->mutex.
 709  */
 710 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
 711                                    uint64_t gpu_va, uint32_t size,
 712                                    uint32_t flags, struct kgd_mem **mem, void **kptr)
 713 {
 714         struct kfd_node *kdev = pdd->dev;
 715         int err;
 716
 717         err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
 718                                                  pdd->drm_priv, mem, NULL,
 719                                                  flags, false);
 720         if (err)
 721                 goto err_alloc_mem;
 722
 723         err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
 724                         pdd->drm_priv);
 725         if (err)
 726                 goto err_map_mem;
 727
 728         err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
 729         if (err) {
 730                 pr_debug("Sync memory failed, wait interrupted by user signal\n");
 731                 goto sync_memory_failed;
 732         }
 733
 734         if (kptr) {
 735                 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
 736                                 (struct kgd_mem *)*mem, kptr, NULL);
 737                 if (err) {
 738                         pr_debug("Map GTT BO to kernel failed\n");
 739                         goto sync_memory_failed;
 740                 }
 741         }
 742
 743         return err;
 744
 745 sync_memory_failed:
 746         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
 747
 748 err_map_mem:
 749         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
 750                                                NULL);
 751 err_alloc_mem:
 752         *mem = NULL;
 753         *kptr = NULL;
 754         return err;
 755 }
 756
 757 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
 758  *      process for IB usage The memory reserved is for KFD to submit
 759  *      IB to AMDGPU from kernel.  If the memory is reserved
 760  *      successfully, ib_kaddr will have the CPU/kernel
 761  *      address. Check ib_kaddr before accessing the memory.
 762  */
 763 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
 764 {
 765         struct qcm_process_device *qpd = &pdd->qpd;
 766         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
 767                         KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
 768                         KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
 769                         KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
 770         struct kgd_mem *mem;
 771         void *kaddr;
 772         int ret;
 773
 774         if (qpd->ib_kaddr || !qpd->ib_base)
 775                 return 0;
 776
 777         /* ib_base is only set for dGPU */
 778         ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
 779                                       &mem, &kaddr);
 780         if (ret)
 781                 return ret;
 782
 783         qpd->ib_mem = mem;
 784         qpd->ib_kaddr = kaddr;
 785
 786         return 0;
 787 }
 788
 789 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
 790 {
 791         struct qcm_process_device *qpd = &pdd->qpd;
 792
 793         if (!qpd->ib_kaddr || !qpd->ib_base)
 794                 return;
 795
 796         kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
 797 }
 798
 799 struct kfd_process *kfd_create_process(struct task_struct *thread)
 800 {
 801         struct kfd_process *process;
 802         int ret;
 803
 804         if (!(thread->mm && mmget_not_zero(thread->mm)))
 805                 return ERR_PTR(-EINVAL);
 806
 807         /* Only the pthreads threading model is supported. */
 808         if (thread->group_leader->mm != thread->mm) {
 809                 mmput(thread->mm);
 810                 return ERR_PTR(-EINVAL);
 811         }
 812
 813         /*
 814          * take kfd processes mutex before starting of process creation
 815          * so there won't be a case where two threads of the same process
 816          * create two kfd_process structures
 817          */
 818         mutex_lock(&kfd_processes_mutex);
 819
 820         if (kfd_is_locked()) {
 821                 mutex_unlock(&kfd_processes_mutex);
 822                 pr_debug("KFD is locked! Cannot create process");
 823                 return ERR_PTR(-EINVAL);
 824         }
 825
 826         /* A prior open of /dev/kfd could have already created the process. */
 827         process = find_process(thread, false);
 828         if (process) {
 829                 pr_debug("Process already found\n");
 830         } else {
 831                 process = create_process(thread);
 832                 if (IS_ERR(process))
 833                         goto out;
 834
 835                 if (!procfs.kobj)
 836                         goto out;
 837
 838                 process->kobj = kfd_alloc_struct(process->kobj);
 839                 if (!process->kobj) {
 840                         pr_warn("Creating procfs kobject failed");
 841                         goto out;
 842                 }
 843                 ret = kobject_init_and_add(process->kobj, &procfs_type,
 844                                            procfs.kobj, "%d",
 845                                            (int)process->lead_thread->pid);
 846                 if (ret) {
 847                         pr_warn("Creating procfs pid directory failed");
 848                         kobject_put(process->kobj);
 849                         goto out;
 850                 }
 851
 852                 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
 853                                       "pasid");
 854
 855                 process->kobj_queues = kobject_create_and_add("queues",
 856                                                         process->kobj);
 857                 if (!process->kobj_queues)
 858                         pr_warn("Creating KFD proc/queues folder failed");
 859
 860                 kfd_procfs_add_sysfs_stats(process);
 861                 kfd_procfs_add_sysfs_files(process);
 862                 kfd_procfs_add_sysfs_counters(process);
 863
 864                 init_waitqueue_head(&process->wait_irq_drain);
 865         }
 866 out:
 867         if (!IS_ERR(process))
 868                 kref_get(&process->ref);
 869         mutex_unlock(&kfd_processes_mutex);
 870         mmput(thread->mm);
 871
 872         return process;
 873 }
 874
 875 struct kfd_process *kfd_get_process(const struct task_struct *thread)
 876 {
 877         struct kfd_process *process;
 878
 879         if (!thread->mm)
 880                 return ERR_PTR(-EINVAL);
 881
 882         /* Only the pthreads threading model is supported. */
 883         if (thread->group_leader->mm != thread->mm)
 884                 return ERR_PTR(-EINVAL);
 885
 886         process = find_process(thread, false);
 887         if (!process)
 888                 return ERR_PTR(-EINVAL);
 889
 890         return process;
 891 }
 892
 893 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 894 {
 895         struct kfd_process *process;
 896
 897         hash_for_each_possible_rcu(kfd_processes_table, process,
 898                                         kfd_processes, (uintptr_t)mm)
 899                 if (process->mm == mm)
 900                         return process;
 901
 902         return NULL;
 903 }
 904
 905 static struct kfd_process *find_process(const struct task_struct *thread,
 906                                         bool ref)
 907 {
 908         struct kfd_process *p;
 909         int idx;
 910
 911         idx = srcu_read_lock(&kfd_processes_srcu);
 912         p = find_process_by_mm(thread->mm);
 913         if (p && ref)
 914                 kref_get(&p->ref);
 915         srcu_read_unlock(&kfd_processes_srcu, idx);
 916
 917         return p;
 918 }
 919
 920 void kfd_unref_process(struct kfd_process *p)
 921 {
 922         kref_put(&p->ref, kfd_process_ref_release);
 923 }
 924
 925 /* This increments the process->ref counter. */
 926 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
 927 {
 928         struct task_struct *task = NULL;
 929         struct kfd_process *p    = NULL;
 930
 931         if (!pid) {
 932                 task = current;
 933                 get_task_struct(task);
 934         } else {
 935                 task = get_pid_task(pid, PIDTYPE_PID);
 936         }
 937
 938         if (task) {
 939                 p = find_process(task, true);
 940                 put_task_struct(task);
 941         }
 942
 943         return p;
 944 }
 945
 946 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
 947 {
 948         struct kfd_process *p = pdd->process;
 949         void *mem;
 950         int id;
 951         int i;
 952
 953         /*
 954          * Remove all handles from idr and release appropriate
 955          * local memory object
 956          */
 957         idr_for_each_entry(&pdd->alloc_idr, mem, id) {
 958
 959                 for (i = 0; i < p->n_pdds; i++) {
 960                         struct kfd_process_device *peer_pdd = p->pdds[i];
 961
 962                         if (!peer_pdd->drm_priv)
 963                                 continue;
 964                         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
 965                                 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
 966                 }
 967
 968                 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
 969                                                        pdd->drm_priv, NULL);
 970                 kfd_process_device_remove_obj_handle(pdd, id);
 971         }
 972 }
 973
 974 /*
 975  * Just kunmap and unpin signal BO here. It will be freed in
 976  * kfd_process_free_outstanding_kfd_bos()
 977  */
 978 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
 979 {
 980         struct kfd_process_device *pdd;
 981         struct kfd_node *kdev;
 982         void *mem;
 983
 984         kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
 985         if (!kdev)
 986                 return;
 987
 988         mutex_lock(&p->mutex);
 989
 990         pdd = kfd_get_process_device_data(kdev, p);
 991         if (!pdd)
 992                 goto out;
 993
 994         mem = kfd_process_device_translate_handle(
 995                 pdd, GET_IDR_HANDLE(p->signal_handle));
 996         if (!mem)
 997                 goto out;
 998
 999         amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1000
1001 out:
1002         mutex_unlock(&p->mutex);
1003 }
1004
1005 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1006 {
1007         int i;
1008
1009         for (i = 0; i < p->n_pdds; i++)
1010                 kfd_process_device_free_bos(p->pdds[i]);
1011 }
1012
1013 static void kfd_process_destroy_pdds(struct kfd_process *p)
1014 {
1015         int i;
1016
1017         for (i = 0; i < p->n_pdds; i++) {
1018                 struct kfd_process_device *pdd = p->pdds[i];
1019
1020                 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
1021                                 pdd->dev->id, p->pasid);
1022
1023                 kfd_process_device_destroy_cwsr_dgpu(pdd);
1024                 kfd_process_device_destroy_ib_mem(pdd);
1025
1026                 if (pdd->drm_file) {
1027                         amdgpu_amdkfd_gpuvm_release_process_vm(
1028                                         pdd->dev->adev, pdd->drm_priv);
1029                         fput(pdd->drm_file);
1030                 }
1031
1032                 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1033                         free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1034                                 get_order(KFD_CWSR_TBA_TMA_SIZE));
1035
1036                 idr_destroy(&pdd->alloc_idr);
1037
1038                 kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1039
1040                 if (pdd->dev->kfd->shared_resources.enable_mes)
1041                         amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
1042                                                    pdd->proc_ctx_bo);
1043                 /*
1044                  * before destroying pdd, make sure to report availability
1045                  * for auto suspend
1046                  */
1047                 if (pdd->runtime_inuse) {
1048                         pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);
1049                         pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1050                         pdd->runtime_inuse = false;
1051                 }
1052
1053                 kfree(pdd);
1054                 p->pdds[i] = NULL;
1055         }
1056         p->n_pdds = 0;
1057 }
1058
1059 static void kfd_process_remove_sysfs(struct kfd_process *p)
1060 {
1061         struct kfd_process_device *pdd;
1062         int i;
1063
1064         if (!p->kobj)
1065                 return;
1066
1067         sysfs_remove_file(p->kobj, &p->attr_pasid);
1068         kobject_del(p->kobj_queues);
1069         kobject_put(p->kobj_queues);
1070         p->kobj_queues = NULL;
1071
1072         for (i = 0; i < p->n_pdds; i++) {
1073                 pdd = p->pdds[i];
1074
1075                 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1076                 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1077
1078                 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1079                 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1080                         sysfs_remove_file(pdd->kobj_stats,
1081                                           &pdd->attr_cu_occupancy);
1082                 kobject_del(pdd->kobj_stats);
1083                 kobject_put(pdd->kobj_stats);
1084                 pdd->kobj_stats = NULL;
1085         }
1086
1087         for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1088                 pdd = p->pdds[i];
1089
1090                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1091                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1092                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1093                 kobject_del(pdd->kobj_counters);
1094                 kobject_put(pdd->kobj_counters);
1095                 pdd->kobj_counters = NULL;
1096         }
1097
1098         kobject_del(p->kobj);
1099         kobject_put(p->kobj);
1100         p->kobj = NULL;
1101 }
1102
1103 /* No process locking is needed in this function, because the process
1104  * is not findable any more. We must assume that no other thread is
1105  * using it any more, otherwise we couldn't safely free the process
1106  * structure in the end.
1107  */
1108 static void kfd_process_wq_release(struct work_struct *work)
1109 {
1110         struct kfd_process *p = container_of(work, struct kfd_process,
1111                                              release_work);
1112
1113         kfd_process_dequeue_from_all_devices(p);
1114         pqm_uninit(&p->pqm);
1115
1116         /* Signal the eviction fence after user mode queues are
1117          * destroyed. This allows any BOs to be freed without
1118          * triggering pointless evictions or waiting for fences.
1119          */
1120         dma_fence_signal(p->ef);
1121
1122         kfd_process_remove_sysfs(p);
1123
1124         kfd_process_kunmap_signal_bo(p);
1125         kfd_process_free_outstanding_kfd_bos(p);
1126         svm_range_list_fini(p);
1127
1128         kfd_process_destroy_pdds(p);
1129         dma_fence_put(p->ef);
1130
1131         kfd_event_free_process(p);
1132
1133         kfd_pasid_free(p->pasid);
1134         mutex_destroy(&p->mutex);
1135
1136         put_task_struct(p->lead_thread);
1137
1138         kfree(p);
1139 }
1140
1141 static void kfd_process_ref_release(struct kref *ref)
1142 {
1143         struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1144
1145         INIT_WORK(&p->release_work, kfd_process_wq_release);
1146         queue_work(kfd_process_wq, &p->release_work);
1147 }
1148
1149 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1150 {
1151         int idx = srcu_read_lock(&kfd_processes_srcu);
1152         struct kfd_process *p = find_process_by_mm(mm);
1153
1154         srcu_read_unlock(&kfd_processes_srcu, idx);
1155
1156         return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1157 }
1158
1159 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1160 {
1161         kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1162 }
1163
1164 static void kfd_process_notifier_release_internal(struct kfd_process *p)
1165 {
1166         int i;
1167
1168         cancel_delayed_work_sync(&p->eviction_work);
1169         cancel_delayed_work_sync(&p->restore_work);
1170
1171         for (i = 0; i < p->n_pdds; i++) {
1172                 struct kfd_process_device *pdd = p->pdds[i];
1173
1174                 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1175                 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1176                         amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1177         }
1178
1179         /* Indicate to other users that MM is no longer valid */
1180         p->mm = NULL;
1181         kfd_dbg_trap_disable(p);
1182
1183         if (atomic_read(&p->debugged_process_count) > 0) {
1184                 struct kfd_process *target;
1185                 unsigned int temp;
1186                 int idx = srcu_read_lock(&kfd_processes_srcu);
1187
1188                 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1189                         if (target->debugger_process && target->debugger_process == p) {
1190                                 mutex_lock_nested(&target->mutex, 1);
1191                                 kfd_dbg_trap_disable(target);
1192                                 mutex_unlock(&target->mutex);
1193                                 if (atomic_read(&p->debugged_process_count) == 0)
1194                                         break;
1195                         }
1196                 }
1197
1198                 srcu_read_unlock(&kfd_processes_srcu, idx);
1199         }
1200
1201         mmu_notifier_put(&p->mmu_notifier);
1202 }
1203
1204 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1205                                         struct mm_struct *mm)
1206 {
1207         struct kfd_process *p;
1208
1209         /*
1210          * The kfd_process structure can not be free because the
1211          * mmu_notifier srcu is read locked
1212          */
1213         p = container_of(mn, struct kfd_process, mmu_notifier);
1214         if (WARN_ON(p->mm != mm))
1215                 return;
1216
1217         mutex_lock(&kfd_processes_mutex);
1218         /*
1219          * Do early return if table is empty.
1220          *
1221          * This could potentially happen if this function is called concurrently
1222          * by mmu_notifier and by kfd_cleanup_pocesses.
1223          *
1224          */
1225         if (hash_empty(kfd_processes_table)) {
1226                 mutex_unlock(&kfd_processes_mutex);
1227                 return;
1228         }
1229         hash_del_rcu(&p->kfd_processes);
1230         mutex_unlock(&kfd_processes_mutex);
1231         synchronize_srcu(&kfd_processes_srcu);
1232
1233         kfd_process_notifier_release_internal(p);
1234 }
1235
1236 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1237         .release = kfd_process_notifier_release,
1238         .alloc_notifier = kfd_process_alloc_notifier,
1239         .free_notifier = kfd_process_free_notifier,
1240 };
1241
1242 /*
1243  * This code handles the case when driver is being unloaded before all
1244  * mm_struct are released.  We need to safely free the kfd_process and
1245  * avoid race conditions with mmu_notifier that might try to free them.
1246  *
1247  */
1248 void kfd_cleanup_processes(void)
1249 {
1250         struct kfd_process *p;
1251         struct hlist_node *p_temp;
1252         unsigned int temp;
1253         HLIST_HEAD(cleanup_list);
1254
1255         /*
1256          * Move all remaining kfd_process from the process table to a
1257          * temp list for processing.   Once done, callback from mmu_notifier
1258          * release will not see the kfd_process in the table and do early return,
1259          * avoiding double free issues.
1260          */
1261         mutex_lock(&kfd_processes_mutex);
1262         hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1263                 hash_del_rcu(&p->kfd_processes);
1264                 synchronize_srcu(&kfd_processes_srcu);
1265                 hlist_add_head(&p->kfd_processes, &cleanup_list);
1266         }
1267         mutex_unlock(&kfd_processes_mutex);
1268
1269         hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1270                 kfd_process_notifier_release_internal(p);
1271
1272         /*
1273          * Ensures that all outstanding free_notifier get called, triggering
1274          * the release of the kfd_process struct.
1275          */
1276         mmu_notifier_synchronize();
1277 }
1278
1279 int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1280 {
1281         unsigned long  offset;
1282         int i;
1283
1284         if (p->has_cwsr)
1285                 return 0;
1286
1287         for (i = 0; i < p->n_pdds; i++) {
1288                 struct kfd_node *dev = p->pdds[i]->dev;
1289                 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1290
1291                 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1292                         continue;
1293
1294                 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1295                 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1296                         KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1297                         MAP_SHARED, offset);
1298
1299                 if (IS_ERR_VALUE(qpd->tba_addr)) {
1300                         int err = qpd->tba_addr;
1301
1302                         pr_err("Failure to set tba address. error %d.\n", err);
1303                         qpd->tba_addr = 0;
1304                         qpd->cwsr_kaddr = NULL;
1305                         return err;
1306                 }
1307
1308                 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1309
1310                 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
1311
1312                 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1313                 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1314                         qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1315         }
1316
1317         p->has_cwsr = true;
1318
1319         return 0;
1320 }
1321
1322 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1323 {
1324         struct kfd_node *dev = pdd->dev;
1325         struct qcm_process_device *qpd = &pdd->qpd;
1326         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1327                         | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1328                         | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1329         struct kgd_mem *mem;
1330         void *kaddr;
1331         int ret;
1332
1333         if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1334                 return 0;
1335
1336         /* cwsr_base is only set for dGPU */
1337         ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1338                                       KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1339         if (ret)
1340                 return ret;
1341
1342         qpd->cwsr_mem = mem;
1343         qpd->cwsr_kaddr = kaddr;
1344         qpd->tba_addr = qpd->cwsr_base;
1345
1346         memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1347
1348         kfd_process_set_trap_debug_flag(&pdd->qpd,
1349                                         pdd->process->debug_trap_enabled);
1350
1351         qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1352         pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1353                  qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1354
1355         return 0;
1356 }
1357
1358 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1359 {
1360         struct kfd_node *dev = pdd->dev;
1361         struct qcm_process_device *qpd = &pdd->qpd;
1362
1363         if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1364                 return;
1365
1366         kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
1367 }
1368
1369 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1370                                   uint64_t tba_addr,
1371                                   uint64_t tma_addr)
1372 {
1373         if (qpd->cwsr_kaddr) {
1374                 /* KFD trap handler is bound, record as second-level TBA/TMA
1375                  * in first-level TMA. First-level trap will jump to second.
1376                  */
1377                 uint64_t *tma =
1378                         (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1379                 tma[0] = tba_addr;
1380                 tma[1] = tma_addr;
1381         } else {
1382                 /* No trap handler bound, bind as first-level TBA/TMA. */
1383                 qpd->tba_addr = tba_addr;
1384                 qpd->tma_addr = tma_addr;
1385         }
1386 }
1387
1388 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1389 {
1390         int i;
1391
1392         /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1393          * boot time retry setting. Mixing processes with different
1394          * XNACK/retry settings can hang the GPU.
1395          *
1396          * Different GPUs can have different noretry settings depending
1397          * on HW bugs or limitations. We need to find at least one
1398          * XNACK mode for this process that's compatible with all GPUs.
1399          * Fortunately GPUs with retry enabled (noretry=0) can run code
1400          * built for XNACK-off. On GFXv9 it may perform slower.
1401          *
1402          * Therefore applications built for XNACK-off can always be
1403          * supported and will be our fallback if any GPU does not
1404          * support retry.
1405          */
1406         for (i = 0; i < p->n_pdds; i++) {
1407                 struct kfd_node *dev = p->pdds[i]->dev;
1408
1409                 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1410                  * support the SVM APIs and don't need to be considered
1411                  * for the XNACK mode selection.
1412                  */
1413                 if (!KFD_IS_SOC15(dev))
1414                         continue;
1415                 /* Aldebaran can always support XNACK because it can support
1416                  * per-process XNACK mode selection. But let the dev->noretry
1417                  * setting still influence the default XNACK mode.
1418                  */
1419                 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1420                         if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1421                                 pr_debug("SRIOV platform xnack not supported\n");
1422                                 return false;
1423                         }
1424                         continue;
1425                 }
1426
1427                 /* GFXv10 and later GPUs do not support shader preemption
1428                  * during page faults. This can lead to poor QoS for queue
1429                  * management and memory-manager-related preemptions or
1430                  * even deadlocks.
1431                  */
1432                 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1433                         return false;
1434
1435                 if (dev->kfd->noretry)
1436                         return false;
1437         }
1438
1439         return true;
1440 }
1441
1442 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1443                                      bool enabled)
1444 {
1445         if (qpd->cwsr_kaddr) {
1446                 uint64_t *tma =
1447                         (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1448                 tma[2] = enabled;
1449         }
1450 }
1451
1452 /*
1453  * On return the kfd_process is fully operational and will be freed when the
1454  * mm is released
1455  */
1456 static struct kfd_process *create_process(const struct task_struct *thread)
1457 {
1458         struct kfd_process *process;
1459         struct mmu_notifier *mn;
1460         int err = -ENOMEM;
1461
1462         process = kzalloc(sizeof(*process), GFP_KERNEL);
1463         if (!process)
1464                 goto err_alloc_process;
1465
1466         kref_init(&process->ref);
1467         mutex_init(&process->mutex);
1468         process->mm = thread->mm;
1469         process->lead_thread = thread->group_leader;
1470         process->n_pdds = 0;
1471         process->queues_paused = false;
1472         INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1473         INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1474         process->last_restore_timestamp = get_jiffies_64();
1475         err = kfd_event_init_process(process);
1476         if (err)
1477                 goto err_event_init;
1478         process->is_32bit_user_mode = in_compat_syscall();
1479         process->debug_trap_enabled = false;
1480         process->debugger_process = NULL;
1481         process->exception_enable_mask = 0;
1482         atomic_set(&process->debugged_process_count, 0);
1483         sema_init(&process->runtime_enable_sema, 0);
1484
1485         process->pasid = kfd_pasid_alloc();
1486         if (process->pasid == 0) {
1487                 err = -ENOSPC;
1488                 goto err_alloc_pasid;
1489         }
1490
1491         err = pqm_init(&process->pqm, process);
1492         if (err != 0)
1493                 goto err_process_pqm_init;
1494
1495         /* init process apertures*/
1496         err = kfd_init_apertures(process);
1497         if (err != 0)
1498                 goto err_init_apertures;
1499
1500         /* Check XNACK support after PDDs are created in kfd_init_apertures */
1501         process->xnack_enabled = kfd_process_xnack_mode(process, false);
1502
1503         err = svm_range_list_init(process);
1504         if (err)
1505                 goto err_init_svm_range_list;
1506
1507         /* alloc_notifier needs to find the process in the hash table */
1508         hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1509                         (uintptr_t)process->mm);
1510
1511         /* Avoid free_notifier to start kfd_process_wq_release if
1512          * mmu_notifier_get failed because of pending signal.
1513          */
1514         kref_get(&process->ref);
1515
1516         /* MMU notifier registration must be the last call that can fail
1517          * because after this point we cannot unwind the process creation.
1518          * After this point, mmu_notifier_put will trigger the cleanup by
1519          * dropping the last process reference in the free_notifier.
1520          */
1521         mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1522         if (IS_ERR(mn)) {
1523                 err = PTR_ERR(mn);
1524                 goto err_register_notifier;
1525         }
1526         BUG_ON(mn != &process->mmu_notifier);
1527
1528         kfd_unref_process(process);
1529         get_task_struct(process->lead_thread);
1530
1531         INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1532
1533         return process;
1534
1535 err_register_notifier:
1536         hash_del_rcu(&process->kfd_processes);
1537         svm_range_list_fini(process);
1538 err_init_svm_range_list:
1539         kfd_process_free_outstanding_kfd_bos(process);
1540         kfd_process_destroy_pdds(process);
1541 err_init_apertures:
1542         pqm_uninit(&process->pqm);
1543 err_process_pqm_init:
1544         kfd_pasid_free(process->pasid);
1545 err_alloc_pasid:
1546         kfd_event_free_process(process);
1547 err_event_init:
1548         mutex_destroy(&process->mutex);
1549         kfree(process);
1550 err_alloc_process:
1551         return ERR_PTR(err);
1552 }
1553
1554 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1555                                                         struct kfd_process *p)
1556 {
1557         int i;
1558
1559         for (i = 0; i < p->n_pdds; i++)
1560                 if (p->pdds[i]->dev == dev)
1561                         return p->pdds[i];
1562
1563         return NULL;
1564 }
1565
1566 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1567                                                         struct kfd_process *p)
1568 {
1569         struct kfd_process_device *pdd = NULL;
1570         int retval = 0;
1571
1572         if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1573                 return NULL;
1574         pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1575         if (!pdd)
1576                 return NULL;
1577
1578         pdd->dev = dev;
1579         INIT_LIST_HEAD(&pdd->qpd.queues_list);
1580         INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1581         pdd->qpd.dqm = dev->dqm;
1582         pdd->qpd.pqm = &p->pqm;
1583         pdd->qpd.evicted = 0;
1584         pdd->qpd.mapped_gws_queue = false;
1585         pdd->process = p;
1586         pdd->bound = PDD_UNBOUND;
1587         pdd->already_dequeued = false;
1588         pdd->runtime_inuse = false;
1589         pdd->vram_usage = 0;
1590         pdd->sdma_past_activity_counter = 0;
1591         pdd->user_gpu_id = dev->id;
1592         atomic64_set(&pdd->evict_duration_counter, 0);
1593
1594         if (dev->kfd->shared_resources.enable_mes) {
1595                 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,
1596                                                 AMDGPU_MES_PROC_CTX_SIZE,
1597                                                 &pdd->proc_ctx_bo,
1598                                                 &pdd->proc_ctx_gpu_addr,
1599                                                 &pdd->proc_ctx_cpu_ptr,
1600                                                 false);
1601                 if (retval) {
1602                         pr_err("failed to allocate process context bo\n");
1603                         goto err_free_pdd;
1604                 }
1605                 memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
1606         }
1607
1608         p->pdds[p->n_pdds++] = pdd;
1609         if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1610                 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1611                                                         pdd->dev->adev,
1612                                                         false,
1613                                                         0);
1614
1615         /* Init idr used for memory handle translation */
1616         idr_init(&pdd->alloc_idr);
1617
1618         return pdd;
1619
1620 err_free_pdd:
1621         kfree(pdd);
1622         return NULL;
1623 }
1624
1625 /**
1626  * kfd_process_device_init_vm - Initialize a VM for a process-device
1627  *
1628  * @pdd: The process-device
1629  * @drm_file: Optional pointer to a DRM file descriptor
1630  *
1631  * If @drm_file is specified, it will be used to acquire the VM from
1632  * that file descriptor. If successful, the @pdd takes ownership of
1633  * the file descriptor.
1634  *
1635  * If @drm_file is NULL, a new VM is created.
1636  *
1637  * Returns 0 on success, -errno on failure.
1638  */
1639 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1640                                struct file *drm_file)
1641 {
1642         struct amdgpu_fpriv *drv_priv;
1643         struct amdgpu_vm *avm;
1644         struct kfd_process *p;
1645         struct kfd_node *dev;
1646         int ret;
1647
1648         if (!drm_file)
1649                 return -EINVAL;
1650
1651         if (pdd->drm_priv)
1652                 return -EBUSY;
1653
1654         ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1655         if (ret)
1656                 return ret;
1657         avm = &drv_priv->vm;
1658
1659         p = pdd->process;
1660         dev = pdd->dev;
1661
1662         ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1663                                                      &p->kgd_process_info,
1664                                                      &p->ef);
1665         if (ret) {
1666                 pr_err("Failed to create process VM object\n");
1667                 return ret;
1668         }
1669         pdd->drm_priv = drm_file->private_data;
1670         atomic64_set(&pdd->tlb_seq, 0);
1671
1672         ret = kfd_process_device_reserve_ib_mem(pdd);
1673         if (ret)
1674                 goto err_reserve_ib_mem;
1675         ret = kfd_process_device_init_cwsr_dgpu(pdd);
1676         if (ret)
1677                 goto err_init_cwsr;
1678
1679         ret = amdgpu_amdkfd_gpuvm_set_vm_pasid(dev->adev, avm, p->pasid);
1680         if (ret)
1681                 goto err_set_pasid;
1682
1683         pdd->drm_file = drm_file;
1684
1685         return 0;
1686
1687 err_set_pasid:
1688         kfd_process_device_destroy_cwsr_dgpu(pdd);
1689 err_init_cwsr:
1690         kfd_process_device_destroy_ib_mem(pdd);
1691 err_reserve_ib_mem:
1692         pdd->drm_priv = NULL;
1693         amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1694
1695         return ret;
1696 }
1697
1698 /*
1699  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1700  * to the device.
1701  * Unbinding occurs when the process dies or the device is removed.
1702  *
1703  * Assumes that the process lock is held.
1704  */
1705 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1706                                                         struct kfd_process *p)
1707 {
1708         struct kfd_process_device *pdd;
1709         int err;
1710
1711         pdd = kfd_get_process_device_data(dev, p);
1712         if (!pdd) {
1713                 pr_err("Process device data doesn't exist\n");
1714                 return ERR_PTR(-ENOMEM);
1715         }
1716
1717         if (!pdd->drm_priv)
1718                 return ERR_PTR(-ENODEV);
1719
1720         /*
1721          * signal runtime-pm system to auto resume and prevent
1722          * further runtime suspend once device pdd is created until
1723          * pdd is destroyed.
1724          */
1725         if (!pdd->runtime_inuse) {
1726                 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1727                 if (err < 0) {
1728                         pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1729                         return ERR_PTR(err);
1730                 }
1731         }
1732
1733         /*
1734          * make sure that runtime_usage counter is incremented just once
1735          * per pdd
1736          */
1737         pdd->runtime_inuse = true;
1738
1739         return pdd;
1740 }
1741
1742 /* Create specific handle mapped to mem from process local memory idr
1743  * Assumes that the process lock is held.
1744  */
1745 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1746                                         void *mem)
1747 {
1748         return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1749 }
1750
1751 /* Translate specific handle from process local memory idr
1752  * Assumes that the process lock is held.
1753  */
1754 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1755                                         int handle)
1756 {
1757         if (handle < 0)
1758                 return NULL;
1759
1760         return idr_find(&pdd->alloc_idr, handle);
1761 }
1762
1763 /* Remove specific handle from process local memory idr
1764  * Assumes that the process lock is held.
1765  */
1766 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1767                                         int handle)
1768 {
1769         if (handle >= 0)
1770                 idr_remove(&pdd->alloc_idr, handle);
1771 }
1772
1773 /* This increments the process->ref counter. */
1774 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1775 {
1776         struct kfd_process *p, *ret_p = NULL;
1777         unsigned int temp;
1778
1779         int idx = srcu_read_lock(&kfd_processes_srcu);
1780
1781         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1782                 if (p->pasid == pasid) {
1783                         kref_get(&p->ref);
1784                         ret_p = p;
1785                         break;
1786                 }
1787         }
1788
1789         srcu_read_unlock(&kfd_processes_srcu, idx);
1790
1791         return ret_p;
1792 }
1793
1794 /* This increments the process->ref counter. */
1795 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1796 {
1797         struct kfd_process *p;
1798
1799         int idx = srcu_read_lock(&kfd_processes_srcu);
1800
1801         p = find_process_by_mm(mm);
1802         if (p)
1803                 kref_get(&p->ref);
1804
1805         srcu_read_unlock(&kfd_processes_srcu, idx);
1806
1807         return p;
1808 }
1809
1810 /* kfd_process_evict_queues - Evict all user queues of a process
1811  *
1812  * Eviction is reference-counted per process-device. This means multiple
1813  * evictions from different sources can be nested safely.
1814  */
1815 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1816 {
1817         int r = 0;
1818         int i;
1819         unsigned int n_evicted = 0;
1820
1821         for (i = 0; i < p->n_pdds; i++) {
1822                 struct kfd_process_device *pdd = p->pdds[i];
1823
1824                 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
1825                                              trigger);
1826
1827                 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1828                                                             &pdd->qpd);
1829                 /* evict return -EIO if HWS is hang or asic is resetting, in this case
1830                  * we would like to set all the queues to be in evicted state to prevent
1831                  * them been add back since they actually not be saved right now.
1832                  */
1833                 if (r && r != -EIO) {
1834                         pr_err("Failed to evict process queues\n");
1835                         goto fail;
1836                 }
1837                 n_evicted++;
1838         }
1839
1840         return r;
1841
1842 fail:
1843         /* To keep state consistent, roll back partial eviction by
1844          * restoring queues
1845          */
1846         for (i = 0; i < p->n_pdds; i++) {
1847                 struct kfd_process_device *pdd = p->pdds[i];
1848
1849                 if (n_evicted == 0)
1850                         break;
1851
1852                 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1853
1854                 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1855                                                               &pdd->qpd))
1856                         pr_err("Failed to restore queues\n");
1857
1858                 n_evicted--;
1859         }
1860
1861         return r;
1862 }
1863
1864 /* kfd_process_restore_queues - Restore all user queues of a process */
1865 int kfd_process_restore_queues(struct kfd_process *p)
1866 {
1867         int r, ret = 0;
1868         int i;
1869
1870         for (i = 0; i < p->n_pdds; i++) {
1871                 struct kfd_process_device *pdd = p->pdds[i];
1872
1873                 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1874
1875                 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1876                                                               &pdd->qpd);
1877                 if (r) {
1878                         pr_err("Failed to restore process queues\n");
1879                         if (!ret)
1880                                 ret = r;
1881                 }
1882         }
1883
1884         return ret;
1885 }
1886
1887 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1888 {
1889         int i;
1890
1891         for (i = 0; i < p->n_pdds; i++)
1892                 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1893                         return i;
1894         return -EINVAL;
1895 }
1896
1897 int
1898 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1899                             uint32_t *gpuid, uint32_t *gpuidx)
1900 {
1901         int i;
1902
1903         for (i = 0; i < p->n_pdds; i++)
1904                 if (p->pdds[i] && p->pdds[i]->dev == node) {
1905                         *gpuid = p->pdds[i]->user_gpu_id;
1906                         *gpuidx = i;
1907                         return 0;
1908                 }
1909         return -EINVAL;
1910 }
1911
1912 static void evict_process_worker(struct work_struct *work)
1913 {
1914         int ret;
1915         struct kfd_process *p;
1916         struct delayed_work *dwork;
1917
1918         dwork = to_delayed_work(work);
1919
1920         /* Process termination destroys this worker thread. So during the
1921          * lifetime of this thread, kfd_process p will be valid
1922          */
1923         p = container_of(dwork, struct kfd_process, eviction_work);
1924         WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1925                   "Eviction fence mismatch\n");
1926
1927         /* Narrow window of overlap between restore and evict work
1928          * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1929          * unreserves KFD BOs, it is possible to evicted again. But
1930          * restore has few more steps of finish. So lets wait for any
1931          * previous restore work to complete
1932          */
1933         flush_delayed_work(&p->restore_work);
1934
1935         pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1936         ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
1937         if (!ret) {
1938                 dma_fence_signal(p->ef);
1939                 dma_fence_put(p->ef);
1940                 p->ef = NULL;
1941                 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1942                                 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1943
1944                 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1945         } else
1946                 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1947 }
1948
1949 static void restore_process_worker(struct work_struct *work)
1950 {
1951         struct delayed_work *dwork;
1952         struct kfd_process *p;
1953         int ret = 0;
1954
1955         dwork = to_delayed_work(work);
1956
1957         /* Process termination destroys this worker thread. So during the
1958          * lifetime of this thread, kfd_process p will be valid
1959          */
1960         p = container_of(dwork, struct kfd_process, restore_work);
1961         pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1962
1963         /* Setting last_restore_timestamp before successful restoration.
1964          * Otherwise this would have to be set by KGD (restore_process_bos)
1965          * before KFD BOs are unreserved. If not, the process can be evicted
1966          * again before the timestamp is set.
1967          * If restore fails, the timestamp will be set again in the next
1968          * attempt. This would mean that the minimum GPU quanta would be
1969          * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1970          * functions)
1971          */
1972
1973         p->last_restore_timestamp = get_jiffies_64();
1974         /* VMs may not have been acquired yet during debugging. */
1975         if (p->kgd_process_info)
1976                 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1977                                                              &p->ef);
1978         if (ret) {
1979                 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1980                          p->pasid, PROCESS_BACK_OFF_TIME_MS);
1981                 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1982                                 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1983                 WARN(!ret, "reschedule restore work failed\n");
1984                 return;
1985         }
1986
1987         ret = kfd_process_restore_queues(p);
1988         if (!ret)
1989                 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1990         else
1991                 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1992 }
1993
1994 void kfd_suspend_all_processes(void)
1995 {
1996         struct kfd_process *p;
1997         unsigned int temp;
1998         int idx = srcu_read_lock(&kfd_processes_srcu);
1999
2000         WARN(debug_evictions, "Evicting all processes");
2001         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2002                 cancel_delayed_work_sync(&p->eviction_work);
2003                 flush_delayed_work(&p->restore_work);
2004
2005                 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2006                         pr_err("Failed to suspend process 0x%x\n", p->pasid);
2007                 dma_fence_signal(p->ef);
2008                 dma_fence_put(p->ef);
2009                 p->ef = NULL;
2010         }
2011         srcu_read_unlock(&kfd_processes_srcu, idx);
2012 }
2013
2014 int kfd_resume_all_processes(void)
2015 {
2016         struct kfd_process *p;
2017         unsigned int temp;
2018         int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2019
2020         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2021                 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
2022                         pr_err("Restore process %d failed during resume\n",
2023                                p->pasid);
2024                         ret = -EFAULT;
2025                 }
2026         }
2027         srcu_read_unlock(&kfd_processes_srcu, idx);
2028         return ret;
2029 }
2030
2031 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2032                           struct vm_area_struct *vma)
2033 {
2034         struct kfd_process_device *pdd;
2035         struct qcm_process_device *qpd;
2036
2037         if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2038                 pr_err("Incorrect CWSR mapping size.\n");
2039                 return -EINVAL;
2040         }
2041
2042         pdd = kfd_get_process_device_data(dev, process);
2043         if (!pdd)
2044                 return -EINVAL;
2045         qpd = &pdd->qpd;
2046
2047         qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2048                                         get_order(KFD_CWSR_TBA_TMA_SIZE));
2049         if (!qpd->cwsr_kaddr) {
2050                 pr_err("Error allocating per process CWSR buffer.\n");
2051                 return -ENOMEM;
2052         }
2053
2054         vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2055                 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2056         /* Mapping pages to user process */
2057         return remap_pfn_range(vma, vma->vm_start,
2058                                PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2059                                KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2060 }
2061
2062 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
2063 {
2064         struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
2065         uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
2066         struct kfd_node *dev = pdd->dev;
2067         uint32_t xcc_mask = dev->xcc_mask;
2068         int xcc = 0;
2069
2070         /*
2071          * It can be that we race and lose here, but that is extremely unlikely
2072          * and the worst thing which could happen is that we flush the changes
2073          * into the TLB once more which is harmless.
2074          */
2075         if (atomic64_xchg(&pdd->tlb_seq, tlb_seq) == tlb_seq)
2076                 return;
2077
2078         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2079                 /* Nothing to flush until a VMID is assigned, which
2080                  * only happens when the first queue is created.
2081                  */
2082                 if (pdd->qpd.vmid)
2083                         amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->adev,
2084                                                         pdd->qpd.vmid);
2085         } else {
2086                 for_each_inst(xcc, xcc_mask)
2087                         amdgpu_amdkfd_flush_gpu_tlb_pasid(
2088                                 dev->adev, pdd->process->pasid, type, xcc);
2089         }
2090 }
2091
2092 /* assumes caller holds process lock. */
2093 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2094 {
2095         uint32_t irq_drain_fence[8];
2096         uint8_t node_id = 0;
2097         int r = 0;
2098
2099         if (!KFD_IS_SOC15(pdd->dev))
2100                 return 0;
2101
2102         pdd->process->irq_drain_is_open = true;
2103
2104         memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2105         irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2106                                                         KFD_IRQ_FENCE_CLIENTID;
2107         irq_drain_fence[3] = pdd->process->pasid;
2108
2109         /*
2110          * For GFX 9.4.3, send the NodeId also in IH cookie DW[3]
2111          */
2112         if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3)) {
2113                 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2114                 irq_drain_fence[3] |= node_id << 16;
2115         }
2116
2117         /* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2118         if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2119                                                      irq_drain_fence)) {
2120                 pdd->process->irq_drain_is_open = false;
2121                 return 0;
2122         }
2123
2124         r = wait_event_interruptible(pdd->process->wait_irq_drain,
2125                                      !READ_ONCE(pdd->process->irq_drain_is_open));
2126         if (r)
2127                 pdd->process->irq_drain_is_open = false;
2128
2129         return r;
2130 }
2131
2132 void kfd_process_close_interrupt_drain(unsigned int pasid)
2133 {
2134         struct kfd_process *p;
2135
2136         p = kfd_lookup_process_by_pasid(pasid);
2137
2138         if (!p)
2139                 return;
2140
2141         WRITE_ONCE(p->irq_drain_is_open, false);
2142         wake_up_all(&p->wait_irq_drain);
2143         kfd_unref_process(p);
2144 }
2145
2146 struct send_exception_work_handler_workarea {
2147         struct work_struct work;
2148         struct kfd_process *p;
2149         unsigned int queue_id;
2150         uint64_t error_reason;
2151 };
2152
2153 static void send_exception_work_handler(struct work_struct *work)
2154 {
2155         struct send_exception_work_handler_workarea *workarea;
2156         struct kfd_process *p;
2157         struct queue *q;
2158         struct mm_struct *mm;
2159         struct kfd_context_save_area_header __user *csa_header;
2160         uint64_t __user *err_payload_ptr;
2161         uint64_t cur_err;
2162         uint32_t ev_id;
2163
2164         workarea = container_of(work,
2165                                 struct send_exception_work_handler_workarea,
2166                                 work);
2167         p = workarea->p;
2168
2169         mm = get_task_mm(p->lead_thread);
2170
2171         if (!mm)
2172                 return;
2173
2174         kthread_use_mm(mm);
2175
2176         q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2177
2178         if (!q)
2179                 goto out;
2180
2181         csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2182
2183         get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2184         get_user(cur_err, err_payload_ptr);
2185         cur_err |= workarea->error_reason;
2186         put_user(cur_err, err_payload_ptr);
2187         get_user(ev_id, &csa_header->err_event_id);
2188
2189         kfd_set_event(p, ev_id);
2190
2191 out:
2192         kthread_unuse_mm(mm);
2193         mmput(mm);
2194 }
2195
2196 int kfd_send_exception_to_runtime(struct kfd_process *p,
2197                         unsigned int queue_id,
2198                         uint64_t error_reason)
2199 {
2200         struct send_exception_work_handler_workarea worker;
2201
2202         INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2203
2204         worker.p = p;
2205         worker.queue_id = queue_id;
2206         worker.error_reason = error_reason;
2207
2208         schedule_work(&worker.work);
2209         flush_work(&worker.work);
2210         destroy_work_on_stack(&worker.work);
2211
2212         return 0;
2213 }
2214
2215 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2216 {
2217         int i;
2218
2219         if (gpu_id) {
2220                 for (i = 0; i < p->n_pdds; i++) {
2221                         struct kfd_process_device *pdd = p->pdds[i];
2222
2223                         if (pdd->user_gpu_id == gpu_id)
2224                                 return pdd;
2225                 }
2226         }
2227         return NULL;
2228 }
2229
2230 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2231 {
2232         int i;
2233
2234         if (!actual_gpu_id)
2235                 return 0;
2236
2237         for (i = 0; i < p->n_pdds; i++) {
2238                 struct kfd_process_device *pdd = p->pdds[i];
2239
2240                 if (pdd->dev->id == actual_gpu_id)
2241                         return pdd->user_gpu_id;
2242         }
2243         return -EINVAL;
2244 }
2245
2246 #if defined(CONFIG_DEBUG_FS)
2247
2248 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2249 {
2250         struct kfd_process *p;
2251         unsigned int temp;
2252         int r = 0;
2253
2254         int idx = srcu_read_lock(&kfd_processes_srcu);
2255
2256         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2257                 seq_printf(m, "Process %d PASID 0x%x:\n",
2258                            p->lead_thread->tgid, p->pasid);
2259
2260                 mutex_lock(&p->mutex);
2261                 r = pqm_debugfs_mqds(m, &p->pqm);
2262                 mutex_unlock(&p->mutex);
2263
2264                 if (r)
2265                         break;
2266         }
2267
2268         srcu_read_unlock(&kfd_processes_srcu, idx);
2269
2270         return r;
2271 }
2272
2273 #endif