drivers/gpu/drm/amd/amdkfd/kfd_process.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/mutex.h>
  24 #include <linux/log2.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/mm.h>
  27 #include <linux/sched/task.h>
  28 #include <linux/mmu_context.h>
  29 #include <linux/slab.h>
  30 #include <linux/amd-iommu.h>
  31 #include <linux/notifier.h>
  32 #include <linux/compat.h>
  33 #include <linux/mman.h>
  34 #include <linux/file.h>
  35 #include <linux/pm_runtime.h>
  36 #include "amdgpu_amdkfd.h"
  37 #include "amdgpu.h"
  38
  39 struct mm_struct;
  40
  41 #include "kfd_priv.h"
  42 #include "kfd_device_queue_manager.h"
  43 #include "kfd_dbgmgr.h"
  44 #include "kfd_iommu.h"
  45
  46 /*
  47  * List of struct kfd_process (field kfd_process).
  48  * Unique/indexed by mm_struct*
  49  */
  50 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
  51 static DEFINE_MUTEX(kfd_processes_mutex);
  52
  53 DEFINE_SRCU(kfd_processes_srcu);
  54
  55 /* For process termination handling */
  56 static struct workqueue_struct *kfd_process_wq;
  57
  58 /* Ordered, single-threaded workqueue for restoring evicted
  59  * processes. Restoring multiple processes concurrently under memory
  60  * pressure can lead to processes blocking each other from validating
  61  * their BOs and result in a live-lock situation where processes
  62  * remain evicted indefinitely.
  63  */
  64 static struct workqueue_struct *kfd_restore_wq;
  65
  66 static struct kfd_process *find_process(const struct task_struct *thread);
  67 static void kfd_process_ref_release(struct kref *ref);
  68 static struct kfd_process *create_process(const struct task_struct *thread);
  69 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
  70
  71 static void evict_process_worker(struct work_struct *work);
  72 static void restore_process_worker(struct work_struct *work);
  73
  74 struct kfd_procfs_tree {
  75         struct kobject *kobj;
  76 };
  77
  78 static struct kfd_procfs_tree procfs;
  79
  80 /*
  81  * Structure for SDMA activity tracking
  82  */
  83 struct kfd_sdma_activity_handler_workarea {
  84         struct work_struct sdma_activity_work;
  85         struct kfd_process_device *pdd;
  86         uint64_t sdma_activity_counter;
  87 };
  88
  89 struct temp_sdma_queue_list {
  90         uint64_t __user *rptr;
  91         uint64_t sdma_val;
  92         unsigned int queue_id;
  93         struct list_head list;
  94 };
  95
  96 static void kfd_sdma_activity_worker(struct work_struct *work)
  97 {
  98         struct kfd_sdma_activity_handler_workarea *workarea;
  99         struct kfd_process_device *pdd;
 100         uint64_t val;
 101         struct mm_struct *mm;
 102         struct queue *q;
 103         struct qcm_process_device *qpd;
 104         struct device_queue_manager *dqm;
 105         int ret = 0;
 106         struct temp_sdma_queue_list sdma_q_list;
 107         struct temp_sdma_queue_list *sdma_q, *next;
 108
 109         workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
 110                                 sdma_activity_work);
 111         if (!workarea)
 112                 return;
 113
 114         pdd = workarea->pdd;
 115         if (!pdd)
 116                 return;
 117         dqm = pdd->dev->dqm;
 118         qpd = &pdd->qpd;
 119         if (!dqm || !qpd)
 120                 return;
 121         /*
 122          * Total SDMA activity is current SDMA activity + past SDMA activity
 123          * Past SDMA count is stored in pdd.
 124          * To get the current activity counters for all active SDMA queues,
 125          * we loop over all SDMA queues and get their counts from user-space.
 126          *
 127          * We cannot call get_user() with dqm_lock held as it can cause
 128          * a circular lock dependency situation. To read the SDMA stats,
 129          * we need to do the following:
 130          *
 131          * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
 132          *    with dqm_lock/dqm_unlock().
 133          * 2. Call get_user() for each node in temporary list without dqm_lock.
 134          *    Save the SDMA count for each node and also add the count to the total
 135          *    SDMA count counter.
 136          *    Its possible, during this step, a few SDMA queue nodes got deleted
 137          *    from the qpd->queues_list.
 138          * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
 139          *    If any node got deleted, its SDMA count would be captured in the sdma
 140          *    past activity counter. So subtract the SDMA counter stored in step 2
 141          *    for this node from the total SDMA count.
 142          */
 143         INIT_LIST_HEAD(&sdma_q_list.list);
 144
 145         /*
 146          * Create the temp list of all SDMA queues
 147          */
 148         dqm_lock(dqm);
 149
 150         list_for_each_entry(q, &qpd->queues_list, list) {
 151                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 152                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 153                         continue;
 154
 155                 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
 156                 if (!sdma_q) {
 157                         dqm_unlock(dqm);
 158                         goto cleanup;
 159                 }
 160
 161                 INIT_LIST_HEAD(&sdma_q->list);
 162                 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
 163                 sdma_q->queue_id = q->properties.queue_id;
 164                 list_add_tail(&sdma_q->list, &sdma_q_list.list);
 165         }
 166
 167         /*
 168          * If the temp list is empty, then no SDMA queues nodes were found in
 169          * qpd->queues_list. Return the past activity count as the total sdma
 170          * count
 171          */
 172         if (list_empty(&sdma_q_list.list)) {
 173                 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
 174                 dqm_unlock(dqm);
 175                 return;
 176         }
 177
 178         dqm_unlock(dqm);
 179
 180         /*
 181          * Get the usage count for each SDMA queue in temp_list.
 182          */
 183         mm = get_task_mm(pdd->process->lead_thread);
 184         if (!mm)
 185                 goto cleanup;
 186
 187         kthread_use_mm(mm);
 188
 189         list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
 190                 val = 0;
 191                 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
 192                 if (ret) {
 193                         pr_debug("Failed to read SDMA queue active counter for queue id: %d",
 194                                  sdma_q->queue_id);
 195                 } else {
 196                         sdma_q->sdma_val = val;
 197                         workarea->sdma_activity_counter += val;
 198                 }
 199         }
 200
 201         kthread_unuse_mm(mm);
 202         mmput(mm);
 203
 204         /*
 205          * Do a second iteration over qpd_queues_list to check if any SDMA
 206          * nodes got deleted while fetching SDMA counter.
 207          */
 208         dqm_lock(dqm);
 209
 210         workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
 211
 212         list_for_each_entry(q, &qpd->queues_list, list) {
 213                 if (list_empty(&sdma_q_list.list))
 214                         break;
 215
 216                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 217                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 218                         continue;
 219
 220                 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 221                         if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
 222                              (sdma_q->queue_id == q->properties.queue_id)) {
 223                                 list_del(&sdma_q->list);
 224                                 kfree(sdma_q);
 225                                 break;
 226                         }
 227                 }
 228         }
 229
 230         dqm_unlock(dqm);
 231
 232         /*
 233          * If temp list is not empty, it implies some queues got deleted
 234          * from qpd->queues_list during SDMA usage read. Subtract the SDMA
 235          * count for each node from the total SDMA count.
 236          */
 237         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 238                 workarea->sdma_activity_counter -= sdma_q->sdma_val;
 239                 list_del(&sdma_q->list);
 240                 kfree(sdma_q);
 241         }
 242
 243         return;
 244
 245 cleanup:
 246         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 247                 list_del(&sdma_q->list);
 248                 kfree(sdma_q);
 249         }
 250 }
 251
 252 /**
 253  * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
 254  * by current process. Translates acquired wave count into number of compute units
 255  * that are occupied.
 256  *
 257  * @atr: Handle of attribute that allows reporting of wave count. The attribute
 258  * handle encapsulates GPU device it is associated with, thereby allowing collection
 259  * of waves in flight, etc
 260  *
 261  * @buffer: Handle of user provided buffer updated with wave count
 262  *
 263  * Return: Number of bytes written to user buffer or an error value
 264  */
 265 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
 266 {
 267         int cu_cnt;
 268         int wave_cnt;
 269         int max_waves_per_cu;
 270         struct kfd_dev *dev = NULL;
 271         struct kfd_process *proc = NULL;
 272         struct kfd_process_device *pdd = NULL;
 273
 274         pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
 275         dev = pdd->dev;
 276         if (dev->kfd2kgd->get_cu_occupancy == NULL)
 277                 return -EINVAL;
 278
 279         cu_cnt = 0;
 280         proc = pdd->process;
 281         if (pdd->qpd.queue_count == 0) {
 282                 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
 283                          dev->id, proc->pasid);
 284                 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 285         }
 286
 287         /* Collect wave count from device if it supports */
 288         wave_cnt = 0;
 289         max_waves_per_cu = 0;
 290         dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
 291                         &max_waves_per_cu);
 292
 293         /* Translate wave count to number of compute units */
 294         cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
 295         return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 296 }
 297
 298 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 299                                char *buffer)
 300 {
 301         if (strcmp(attr->name, "pasid") == 0) {
 302                 struct kfd_process *p = container_of(attr, struct kfd_process,
 303                                                      attr_pasid);
 304
 305                 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
 306         } else if (strncmp(attr->name, "vram_", 5) == 0) {
 307                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 308                                                               attr_vram);
 309                 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
 310         } else if (strncmp(attr->name, "sdma_", 5) == 0) {
 311                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 312                                                               attr_sdma);
 313                 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
 314
 315                 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
 316                                         kfd_sdma_activity_worker);
 317
 318                 sdma_activity_work_handler.pdd = pdd;
 319                 sdma_activity_work_handler.sdma_activity_counter = 0;
 320
 321                 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
 322
 323                 flush_work(&sdma_activity_work_handler.sdma_activity_work);
 324
 325                 return snprintf(buffer, PAGE_SIZE, "%llu\n",
 326                                 (sdma_activity_work_handler.sdma_activity_counter)/
 327                                  SDMA_ACTIVITY_DIVISOR);
 328         } else {
 329                 pr_err("Invalid attribute");
 330                 return -EINVAL;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static void kfd_procfs_kobj_release(struct kobject *kobj)
 337 {
 338         kfree(kobj);
 339 }
 340
 341 static const struct sysfs_ops kfd_procfs_ops = {
 342         .show = kfd_procfs_show,
 343 };
 344
 345 static struct kobj_type procfs_type = {
 346         .release = kfd_procfs_kobj_release,
 347         .sysfs_ops = &kfd_procfs_ops,
 348 };
 349
 350 void kfd_procfs_init(void)
 351 {
 352         int ret = 0;
 353
 354         procfs.kobj = kfd_alloc_struct(procfs.kobj);
 355         if (!procfs.kobj)
 356                 return;
 357
 358         ret = kobject_init_and_add(procfs.kobj, &procfs_type,
 359                                    &kfd_device->kobj, "proc");
 360         if (ret) {
 361                 pr_warn("Could not create procfs proc folder");
 362                 /* If we fail to create the procfs, clean up */
 363                 kfd_procfs_shutdown();
 364         }
 365 }
 366
 367 void kfd_procfs_shutdown(void)
 368 {
 369         if (procfs.kobj) {
 370                 kobject_del(procfs.kobj);
 371                 kobject_put(procfs.kobj);
 372                 procfs.kobj = NULL;
 373         }
 374 }
 375
 376 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
 377                                      struct attribute *attr, char *buffer)
 378 {
 379         struct queue *q = container_of(kobj, struct queue, kobj);
 380
 381         if (!strcmp(attr->name, "size"))
 382                 return snprintf(buffer, PAGE_SIZE, "%llu",
 383                                 q->properties.queue_size);
 384         else if (!strcmp(attr->name, "type"))
 385                 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
 386         else if (!strcmp(attr->name, "gpuid"))
 387                 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
 388         else
 389                 pr_err("Invalid attribute");
 390
 391         return 0;
 392 }
 393
 394 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
 395                                      struct attribute *attr, char *buffer)
 396 {
 397         if (strcmp(attr->name, "evicted_ms") == 0) {
 398                 struct kfd_process_device *pdd = container_of(attr,
 399                                 struct kfd_process_device,
 400                                 attr_evict);
 401                 uint64_t evict_jiffies;
 402
 403                 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
 404
 405                 return snprintf(buffer,
 406                                 PAGE_SIZE,
 407                                 "%llu\n",
 408                                 jiffies64_to_msecs(evict_jiffies));
 409
 410         /* Sysfs handle that gets CU occupancy is per device */
 411         } else if (strcmp(attr->name, "cu_occupancy") == 0) {
 412                 return kfd_get_cu_occupancy(attr, buffer);
 413         } else {
 414                 pr_err("Invalid attribute");
 415         }
 416
 417         return 0;
 418 }
 419
 420 static struct attribute attr_queue_size = {
 421         .name = "size",
 422         .mode = KFD_SYSFS_FILE_MODE
 423 };
 424
 425 static struct attribute attr_queue_type = {
 426         .name = "type",
 427         .mode = KFD_SYSFS_FILE_MODE
 428 };
 429
 430 static struct attribute attr_queue_gpuid = {
 431         .name = "gpuid",
 432         .mode = KFD_SYSFS_FILE_MODE
 433 };
 434
 435 static struct attribute *procfs_queue_attrs[] = {
 436         &attr_queue_size,
 437         &attr_queue_type,
 438         &attr_queue_gpuid,
 439         NULL
 440 };
 441
 442 static const struct sysfs_ops procfs_queue_ops = {
 443         .show = kfd_procfs_queue_show,
 444 };
 445
 446 static struct kobj_type procfs_queue_type = {
 447         .sysfs_ops = &procfs_queue_ops,
 448         .default_attrs = procfs_queue_attrs,
 449 };
 450
 451 static const struct sysfs_ops procfs_stats_ops = {
 452         .show = kfd_procfs_stats_show,
 453 };
 454
 455 static struct attribute *procfs_stats_attrs[] = {
 456         NULL
 457 };
 458
 459 static struct kobj_type procfs_stats_type = {
 460         .sysfs_ops = &procfs_stats_ops,
 461         .default_attrs = procfs_stats_attrs,
 462 };
 463
 464 int kfd_procfs_add_queue(struct queue *q)
 465 {
 466         struct kfd_process *proc;
 467         int ret;
 468
 469         if (!q || !q->process)
 470                 return -EINVAL;
 471         proc = q->process;
 472
 473         /* Create proc/<pid>/queues/<queue id> folder */
 474         if (!proc->kobj_queues)
 475                 return -EFAULT;
 476         ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
 477                         proc->kobj_queues, "%u", q->properties.queue_id);
 478         if (ret < 0) {
 479                 pr_warn("Creating proc/<pid>/queues/%u failed",
 480                         q->properties.queue_id);
 481                 kobject_put(&q->kobj);
 482                 return ret;
 483         }
 484
 485         return 0;
 486 }
 487
 488 static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr,
 489                                  char *name)
 490 {
 491         int ret = 0;
 492
 493         if (!p || !attr || !name)
 494                 return -EINVAL;
 495
 496         attr->name = name;
 497         attr->mode = KFD_SYSFS_FILE_MODE;
 498         sysfs_attr_init(attr);
 499
 500         ret = sysfs_create_file(p->kobj, attr);
 501
 502         return ret;
 503 }
 504
 505 static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 506 {
 507         int ret = 0;
 508         struct kfd_process_device *pdd;
 509         char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
 510
 511         if (!p)
 512                 return -EINVAL;
 513
 514         if (!p->kobj)
 515                 return -EFAULT;
 516
 517         /*
 518          * Create sysfs files for each GPU:
 519          * - proc/<pid>/stats_<gpuid>/
 520          * - proc/<pid>/stats_<gpuid>/evicted_ms
 521          * - proc/<pid>/stats_<gpuid>/cu_occupancy
 522          */
 523         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 524                 struct kobject *kobj_stats;
 525
 526                 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
 527                                 "stats_%u", pdd->dev->id);
 528                 kobj_stats = kfd_alloc_struct(kobj_stats);
 529                 if (!kobj_stats)
 530                         return -ENOMEM;
 531
 532                 ret = kobject_init_and_add(kobj_stats,
 533                                                 &procfs_stats_type,
 534                                                 p->kobj,
 535                                                 stats_dir_filename);
 536
 537                 if (ret) {
 538                         pr_warn("Creating KFD proc/stats_%s folder failed",
 539                                         stats_dir_filename);
 540                         kobject_put(kobj_stats);
 541                         goto err;
 542                 }
 543
 544                 pdd->kobj_stats = kobj_stats;
 545                 pdd->attr_evict.name = "evicted_ms";
 546                 pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE;
 547                 sysfs_attr_init(&pdd->attr_evict);
 548                 ret = sysfs_create_file(kobj_stats, &pdd->attr_evict);
 549                 if (ret)
 550                         pr_warn("Creating eviction stats for gpuid %d failed",
 551                                         (int)pdd->dev->id);
 552
 553                 /* Add sysfs file to report compute unit occupancy */
 554                 if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
 555                         pdd->attr_cu_occupancy.name = "cu_occupancy";
 556                         pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
 557                         sysfs_attr_init(&pdd->attr_cu_occupancy);
 558                         ret = sysfs_create_file(kobj_stats,
 559                                                 &pdd->attr_cu_occupancy);
 560                         if (ret)
 561                                 pr_warn("Creating %s failed for gpuid: %d",
 562                                         pdd->attr_cu_occupancy.name,
 563                                         (int)pdd->dev->id);
 564                 }
 565         }
 566 err:
 567         return ret;
 568 }
 569
 570
 571 static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
 572 {
 573         int ret = 0;
 574         struct kfd_process_device *pdd;
 575
 576         if (!p)
 577                 return -EINVAL;
 578
 579         if (!p->kobj)
 580                 return -EFAULT;
 581
 582         /*
 583          * Create sysfs files for each GPU:
 584          * - proc/<pid>/vram_<gpuid>
 585          * - proc/<pid>/sdma_<gpuid>
 586          */
 587         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 588                 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
 589                          pdd->dev->id);
 590                 ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename);
 591                 if (ret)
 592                         pr_warn("Creating vram usage for gpu id %d failed",
 593                                 (int)pdd->dev->id);
 594
 595                 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
 596                          pdd->dev->id);
 597                 ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename);
 598                 if (ret)
 599                         pr_warn("Creating sdma usage for gpu id %d failed",
 600                                 (int)pdd->dev->id);
 601         }
 602
 603         return ret;
 604 }
 605
 606 void kfd_procfs_del_queue(struct queue *q)
 607 {
 608         if (!q)
 609                 return;
 610
 611         kobject_del(&q->kobj);
 612         kobject_put(&q->kobj);
 613 }
 614
 615 int kfd_process_create_wq(void)
 616 {
 617         if (!kfd_process_wq)
 618                 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
 619         if (!kfd_restore_wq)
 620                 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
 621
 622         if (!kfd_process_wq || !kfd_restore_wq) {
 623                 kfd_process_destroy_wq();
 624                 return -ENOMEM;
 625         }
 626
 627         return 0;
 628 }
 629
 630 void kfd_process_destroy_wq(void)
 631 {
 632         if (kfd_process_wq) {
 633                 destroy_workqueue(kfd_process_wq);
 634                 kfd_process_wq = NULL;
 635         }
 636         if (kfd_restore_wq) {
 637                 destroy_workqueue(kfd_restore_wq);
 638                 kfd_restore_wq = NULL;
 639         }
 640 }
 641
 642 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 643                         struct kfd_process_device *pdd)
 644 {
 645         struct kfd_dev *dev = pdd->dev;
 646
 647         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);
 648         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);
 649 }
 650
 651 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
 652  *      This function should be only called right after the process
 653  *      is created and when kfd_processes_mutex is still being held
 654  *      to avoid concurrency. Because of that exclusiveness, we do
 655  *      not need to take p->mutex.
 656  */
 657 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
 658                                    uint64_t gpu_va, uint32_t size,
 659                                    uint32_t flags, void **kptr)
 660 {
 661         struct kfd_dev *kdev = pdd->dev;
 662         struct kgd_mem *mem = NULL;
 663         int handle;
 664         int err;
 665
 666         err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
 667                                                  pdd->vm, &mem, NULL, flags);
 668         if (err)
 669                 goto err_alloc_mem;
 670
 671         err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
 672         if (err)
 673                 goto err_map_mem;
 674
 675         err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
 676         if (err) {
 677                 pr_debug("Sync memory failed, wait interrupted by user signal\n");
 678                 goto sync_memory_failed;
 679         }
 680
 681         /* Create an obj handle so kfd_process_device_remove_obj_handle
 682          * will take care of the bo removal when the process finishes.
 683          * We do not need to take p->mutex, because the process is just
 684          * created and the ioctls have not had the chance to run.
 685          */
 686         handle = kfd_process_device_create_obj_handle(pdd, mem);
 687
 688         if (handle < 0) {
 689                 err = handle;
 690                 goto free_gpuvm;
 691         }
 692
 693         if (kptr) {
 694                 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
 695                                 (struct kgd_mem *)mem, kptr, NULL);
 696                 if (err) {
 697                         pr_debug("Map GTT BO to kernel failed\n");
 698                         goto free_obj_handle;
 699                 }
 700         }
 701
 702         return err;
 703
 704 free_obj_handle:
 705         kfd_process_device_remove_obj_handle(pdd, handle);
 706 free_gpuvm:
 707 sync_memory_failed:
 708         kfd_process_free_gpuvm(mem, pdd);
 709         return err;
 710
 711 err_map_mem:
 712         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);
 713 err_alloc_mem:
 714         *kptr = NULL;
 715         return err;
 716 }
 717
 718 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
 719  *      process for IB usage The memory reserved is for KFD to submit
 720  *      IB to AMDGPU from kernel.  If the memory is reserved
 721  *      successfully, ib_kaddr will have the CPU/kernel
 722  *      address. Check ib_kaddr before accessing the memory.
 723  */
 724 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
 725 {
 726         struct qcm_process_device *qpd = &pdd->qpd;
 727         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
 728                         KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
 729                         KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
 730                         KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
 731         void *kaddr;
 732         int ret;
 733
 734         if (qpd->ib_kaddr || !qpd->ib_base)
 735                 return 0;
 736
 737         /* ib_base is only set for dGPU */
 738         ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
 739                                       &kaddr);
 740         if (ret)
 741                 return ret;
 742
 743         qpd->ib_kaddr = kaddr;
 744
 745         return 0;
 746 }
 747
 748 struct kfd_process *kfd_create_process(struct file *filep)
 749 {
 750         struct kfd_process *process;
 751         struct task_struct *thread = current;
 752         int ret;
 753
 754         if (!thread->mm)
 755                 return ERR_PTR(-EINVAL);
 756
 757         /* Only the pthreads threading model is supported. */
 758         if (thread->group_leader->mm != thread->mm)
 759                 return ERR_PTR(-EINVAL);
 760
 761         /*
 762          * take kfd processes mutex before starting of process creation
 763          * so there won't be a case where two threads of the same process
 764          * create two kfd_process structures
 765          */
 766         mutex_lock(&kfd_processes_mutex);
 767
 768         /* A prior open of /dev/kfd could have already created the process. */
 769         process = find_process(thread);
 770         if (process) {
 771                 pr_debug("Process already found\n");
 772         } else {
 773                 process = create_process(thread);
 774                 if (IS_ERR(process))
 775                         goto out;
 776
 777                 ret = kfd_process_init_cwsr_apu(process, filep);
 778                 if (ret) {
 779                         process = ERR_PTR(ret);
 780                         goto out;
 781                 }
 782
 783                 if (!procfs.kobj)
 784                         goto out;
 785
 786                 process->kobj = kfd_alloc_struct(process->kobj);
 787                 if (!process->kobj) {
 788                         pr_warn("Creating procfs kobject failed");
 789                         goto out;
 790                 }
 791                 ret = kobject_init_and_add(process->kobj, &procfs_type,
 792                                            procfs.kobj, "%d",
 793                                            (int)process->lead_thread->pid);
 794                 if (ret) {
 795                         pr_warn("Creating procfs pid directory failed");
 796                         kobject_put(process->kobj);
 797                         goto out;
 798                 }
 799
 800                 process->attr_pasid.name = "pasid";
 801                 process->attr_pasid.mode = KFD_SYSFS_FILE_MODE;
 802                 sysfs_attr_init(&process->attr_pasid);
 803                 ret = sysfs_create_file(process->kobj, &process->attr_pasid);
 804                 if (ret)
 805                         pr_warn("Creating pasid for pid %d failed",
 806                                         (int)process->lead_thread->pid);
 807
 808                 process->kobj_queues = kobject_create_and_add("queues",
 809                                                         process->kobj);
 810                 if (!process->kobj_queues)
 811                         pr_warn("Creating KFD proc/queues folder failed");
 812
 813                 ret = kfd_procfs_add_sysfs_stats(process);
 814                 if (ret)
 815                         pr_warn("Creating sysfs stats dir for pid %d failed",
 816                                 (int)process->lead_thread->pid);
 817
 818                 ret = kfd_procfs_add_sysfs_files(process);
 819                 if (ret)
 820                         pr_warn("Creating sysfs usage file for pid %d failed",
 821                                 (int)process->lead_thread->pid);
 822         }
 823 out:
 824         if (!IS_ERR(process))
 825                 kref_get(&process->ref);
 826         mutex_unlock(&kfd_processes_mutex);
 827
 828         return process;
 829 }
 830
 831 struct kfd_process *kfd_get_process(const struct task_struct *thread)
 832 {
 833         struct kfd_process *process;
 834
 835         if (!thread->mm)
 836                 return ERR_PTR(-EINVAL);
 837
 838         /* Only the pthreads threading model is supported. */
 839         if (thread->group_leader->mm != thread->mm)
 840                 return ERR_PTR(-EINVAL);
 841
 842         process = find_process(thread);
 843         if (!process)
 844                 return ERR_PTR(-EINVAL);
 845
 846         return process;
 847 }
 848
 849 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 850 {
 851         struct kfd_process *process;
 852
 853         hash_for_each_possible_rcu(kfd_processes_table, process,
 854                                         kfd_processes, (uintptr_t)mm)
 855                 if (process->mm == mm)
 856                         return process;
 857
 858         return NULL;
 859 }
 860
 861 static struct kfd_process *find_process(const struct task_struct *thread)
 862 {
 863         struct kfd_process *p;
 864         int idx;
 865
 866         idx = srcu_read_lock(&kfd_processes_srcu);
 867         p = find_process_by_mm(thread->mm);
 868         srcu_read_unlock(&kfd_processes_srcu, idx);
 869
 870         return p;
 871 }
 872
 873 void kfd_unref_process(struct kfd_process *p)
 874 {
 875         kref_put(&p->ref, kfd_process_ref_release);
 876 }
 877
 878 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
 879 {
 880         struct kfd_process *p = pdd->process;
 881         void *mem;
 882         int id;
 883
 884         /*
 885          * Remove all handles from idr and release appropriate
 886          * local memory object
 887          */
 888         idr_for_each_entry(&pdd->alloc_idr, mem, id) {
 889                 struct kfd_process_device *peer_pdd;
 890
 891                 list_for_each_entry(peer_pdd, &p->per_device_data,
 892                                     per_device_list) {
 893                         if (!peer_pdd->vm)
 894                                 continue;
 895                         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
 896                                 peer_pdd->dev->kgd, mem, peer_pdd->vm);
 897                 }
 898
 899                 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);
 900                 kfd_process_device_remove_obj_handle(pdd, id);
 901         }
 902 }
 903
 904 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
 905 {
 906         struct kfd_process_device *pdd;
 907
 908         list_for_each_entry(pdd, &p->per_device_data, per_device_list)
 909                 kfd_process_device_free_bos(pdd);
 910 }
 911
 912 static void kfd_process_destroy_pdds(struct kfd_process *p)
 913 {
 914         struct kfd_process_device *pdd, *temp;
 915
 916         list_for_each_entry_safe(pdd, temp, &p->per_device_data,
 917                                  per_device_list) {
 918                 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
 919                                 pdd->dev->id, p->pasid);
 920
 921                 if (pdd->drm_file) {
 922                         amdgpu_amdkfd_gpuvm_release_process_vm(
 923                                         pdd->dev->kgd, pdd->vm);
 924                         fput(pdd->drm_file);
 925                 }
 926                 else if (pdd->vm)
 927                         amdgpu_amdkfd_gpuvm_destroy_process_vm(
 928                                 pdd->dev->kgd, pdd->vm);
 929
 930                 list_del(&pdd->per_device_list);
 931
 932                 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
 933                         free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
 934                                 get_order(KFD_CWSR_TBA_TMA_SIZE));
 935
 936                 kfree(pdd->qpd.doorbell_bitmap);
 937                 idr_destroy(&pdd->alloc_idr);
 938
 939                 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
 940
 941                 /*
 942                  * before destroying pdd, make sure to report availability
 943                  * for auto suspend
 944                  */
 945                 if (pdd->runtime_inuse) {
 946                         pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
 947                         pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
 948                         pdd->runtime_inuse = false;
 949                 }
 950
 951                 kfree(pdd);
 952         }
 953 }
 954
 955 /* No process locking is needed in this function, because the process
 956  * is not findable any more. We must assume that no other thread is
 957  * using it any more, otherwise we couldn't safely free the process
 958  * structure in the end.
 959  */
 960 static void kfd_process_wq_release(struct work_struct *work)
 961 {
 962         struct kfd_process *p = container_of(work, struct kfd_process,
 963                                              release_work);
 964         struct kfd_process_device *pdd;
 965
 966         /* Remove the procfs files */
 967         if (p->kobj) {
 968                 sysfs_remove_file(p->kobj, &p->attr_pasid);
 969                 kobject_del(p->kobj_queues);
 970                 kobject_put(p->kobj_queues);
 971                 p->kobj_queues = NULL;
 972
 973                 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 974                         sysfs_remove_file(p->kobj, &pdd->attr_vram);
 975                         sysfs_remove_file(p->kobj, &pdd->attr_sdma);
 976                         sysfs_remove_file(p->kobj, &pdd->attr_evict);
 977                         if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
 978                                 sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
 979                         kobject_del(pdd->kobj_stats);
 980                         kobject_put(pdd->kobj_stats);
 981                         pdd->kobj_stats = NULL;
 982                 }
 983
 984                 kobject_del(p->kobj);
 985                 kobject_put(p->kobj);
 986                 p->kobj = NULL;
 987         }
 988
 989         kfd_iommu_unbind_process(p);
 990
 991         kfd_process_free_outstanding_kfd_bos(p);
 992
 993         kfd_process_destroy_pdds(p);
 994         dma_fence_put(p->ef);
 995
 996         kfd_event_free_process(p);
 997
 998         kfd_pasid_free(p->pasid);
 999         mutex_destroy(&p->mutex);
1000
1001         put_task_struct(p->lead_thread);
1002
1003         kfree(p);
1004 }
1005
1006 static void kfd_process_ref_release(struct kref *ref)
1007 {
1008         struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1009
1010         INIT_WORK(&p->release_work, kfd_process_wq_release);
1011         queue_work(kfd_process_wq, &p->release_work);
1012 }
1013
1014 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1015 {
1016         kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1017 }
1018
1019 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1020                                         struct mm_struct *mm)
1021 {
1022         struct kfd_process *p;
1023         struct kfd_process_device *pdd = NULL;
1024
1025         /*
1026          * The kfd_process structure can not be free because the
1027          * mmu_notifier srcu is read locked
1028          */
1029         p = container_of(mn, struct kfd_process, mmu_notifier);
1030         if (WARN_ON(p->mm != mm))
1031                 return;
1032
1033         mutex_lock(&kfd_processes_mutex);
1034         hash_del_rcu(&p->kfd_processes);
1035         mutex_unlock(&kfd_processes_mutex);
1036         synchronize_srcu(&kfd_processes_srcu);
1037
1038         cancel_delayed_work_sync(&p->eviction_work);
1039         cancel_delayed_work_sync(&p->restore_work);
1040
1041         mutex_lock(&p->mutex);
1042
1043         /* Iterate over all process device data structures and if the
1044          * pdd is in debug mode, we should first force unregistration,
1045          * then we will be able to destroy the queues
1046          */
1047         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1048                 struct kfd_dev *dev = pdd->dev;
1049
1050                 mutex_lock(kfd_get_dbgmgr_mutex());
1051                 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1052                         if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1053                                 kfd_dbgmgr_destroy(dev->dbgmgr);
1054                                 dev->dbgmgr = NULL;
1055                         }
1056                 }
1057                 mutex_unlock(kfd_get_dbgmgr_mutex());
1058         }
1059
1060         kfd_process_dequeue_from_all_devices(p);
1061         pqm_uninit(&p->pqm);
1062
1063         /* Indicate to other users that MM is no longer valid */
1064         p->mm = NULL;
1065         /* Signal the eviction fence after user mode queues are
1066          * destroyed. This allows any BOs to be freed without
1067          * triggering pointless evictions or waiting for fences.
1068          */
1069         dma_fence_signal(p->ef);
1070
1071         mutex_unlock(&p->mutex);
1072
1073         mmu_notifier_put(&p->mmu_notifier);
1074 }
1075
1076 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1077         .release = kfd_process_notifier_release,
1078         .free_notifier = kfd_process_free_notifier,
1079 };
1080
1081 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1082 {
1083         unsigned long  offset;
1084         struct kfd_process_device *pdd;
1085
1086         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1087                 struct kfd_dev *dev = pdd->dev;
1088                 struct qcm_process_device *qpd = &pdd->qpd;
1089
1090                 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1091                         continue;
1092
1093                 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1094                 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1095                         KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1096                         MAP_SHARED, offset);
1097
1098                 if (IS_ERR_VALUE(qpd->tba_addr)) {
1099                         int err = qpd->tba_addr;
1100
1101                         pr_err("Failure to set tba address. error %d.\n", err);
1102                         qpd->tba_addr = 0;
1103                         qpd->cwsr_kaddr = NULL;
1104                         return err;
1105                 }
1106
1107                 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1108
1109                 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1110                 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1111                         qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1112         }
1113
1114         return 0;
1115 }
1116
1117 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1118 {
1119         struct kfd_dev *dev = pdd->dev;
1120         struct qcm_process_device *qpd = &pdd->qpd;
1121         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1122                         | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1123                         | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1124         void *kaddr;
1125         int ret;
1126
1127         if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1128                 return 0;
1129
1130         /* cwsr_base is only set for dGPU */
1131         ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1132                                       KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1133         if (ret)
1134                 return ret;
1135
1136         qpd->cwsr_kaddr = kaddr;
1137         qpd->tba_addr = qpd->cwsr_base;
1138
1139         memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1140
1141         qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1142         pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1143                  qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1144
1145         return 0;
1146 }
1147
1148 /*
1149  * On return the kfd_process is fully operational and will be freed when the
1150  * mm is released
1151  */
1152 static struct kfd_process *create_process(const struct task_struct *thread)
1153 {
1154         struct kfd_process *process;
1155         int err = -ENOMEM;
1156
1157         process = kzalloc(sizeof(*process), GFP_KERNEL);
1158         if (!process)
1159                 goto err_alloc_process;
1160
1161         kref_init(&process->ref);
1162         mutex_init(&process->mutex);
1163         process->mm = thread->mm;
1164         process->lead_thread = thread->group_leader;
1165         INIT_LIST_HEAD(&process->per_device_data);
1166         INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1167         INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1168         process->last_restore_timestamp = get_jiffies_64();
1169         kfd_event_init_process(process);
1170         process->is_32bit_user_mode = in_compat_syscall();
1171
1172         process->pasid = kfd_pasid_alloc();
1173         if (process->pasid == 0)
1174                 goto err_alloc_pasid;
1175
1176         err = pqm_init(&process->pqm, process);
1177         if (err != 0)
1178                 goto err_process_pqm_init;
1179
1180         /* init process apertures*/
1181         err = kfd_init_apertures(process);
1182         if (err != 0)
1183                 goto err_init_apertures;
1184
1185         /* Must be last, have to use release destruction after this */
1186         process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
1187         err = mmu_notifier_register(&process->mmu_notifier, process->mm);
1188         if (err)
1189                 goto err_register_notifier;
1190
1191         get_task_struct(process->lead_thread);
1192         hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1193                         (uintptr_t)process->mm);
1194
1195         return process;
1196
1197 err_register_notifier:
1198         kfd_process_free_outstanding_kfd_bos(process);
1199         kfd_process_destroy_pdds(process);
1200 err_init_apertures:
1201         pqm_uninit(&process->pqm);
1202 err_process_pqm_init:
1203         kfd_pasid_free(process->pasid);
1204 err_alloc_pasid:
1205         mutex_destroy(&process->mutex);
1206         kfree(process);
1207 err_alloc_process:
1208         return ERR_PTR(err);
1209 }
1210
1211 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1212                         struct kfd_dev *dev)
1213 {
1214         unsigned int i;
1215         int range_start = dev->shared_resources.non_cp_doorbells_start;
1216         int range_end = dev->shared_resources.non_cp_doorbells_end;
1217
1218         if (!KFD_IS_SOC15(dev->device_info->asic_family))
1219                 return 0;
1220
1221         qpd->doorbell_bitmap =
1222                 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1223                                      BITS_PER_BYTE), GFP_KERNEL);
1224         if (!qpd->doorbell_bitmap)
1225                 return -ENOMEM;
1226
1227         /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1228         pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1229         pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1230                         range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1231                         range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1232
1233         for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1234                 if (i >= range_start && i <= range_end) {
1235                         set_bit(i, qpd->doorbell_bitmap);
1236                         set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1237                                 qpd->doorbell_bitmap);
1238                 }
1239         }
1240
1241         return 0;
1242 }
1243
1244 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1245                                                         struct kfd_process *p)
1246 {
1247         struct kfd_process_device *pdd = NULL;
1248
1249         list_for_each_entry(pdd, &p->per_device_data, per_device_list)
1250                 if (pdd->dev == dev)
1251                         return pdd;
1252
1253         return NULL;
1254 }
1255
1256 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1257                                                         struct kfd_process *p)
1258 {
1259         struct kfd_process_device *pdd = NULL;
1260
1261         pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1262         if (!pdd)
1263                 return NULL;
1264
1265         if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1266                 pr_err("Failed to alloc doorbell for pdd\n");
1267                 goto err_free_pdd;
1268         }
1269
1270         if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1271                 pr_err("Failed to init doorbell for process\n");
1272                 goto err_free_pdd;
1273         }
1274
1275         pdd->dev = dev;
1276         INIT_LIST_HEAD(&pdd->qpd.queues_list);
1277         INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1278         pdd->qpd.dqm = dev->dqm;
1279         pdd->qpd.pqm = &p->pqm;
1280         pdd->qpd.evicted = 0;
1281         pdd->qpd.mapped_gws_queue = false;
1282         pdd->process = p;
1283         pdd->bound = PDD_UNBOUND;
1284         pdd->already_dequeued = false;
1285         pdd->runtime_inuse = false;
1286         pdd->vram_usage = 0;
1287         pdd->sdma_past_activity_counter = 0;
1288         atomic64_set(&pdd->evict_duration_counter, 0);
1289         list_add(&pdd->per_device_list, &p->per_device_data);
1290
1291         /* Init idr used for memory handle translation */
1292         idr_init(&pdd->alloc_idr);
1293
1294         return pdd;
1295
1296 err_free_pdd:
1297         kfree(pdd);
1298         return NULL;
1299 }
1300
1301 /**
1302  * kfd_process_device_init_vm - Initialize a VM for a process-device
1303  *
1304  * @pdd: The process-device
1305  * @drm_file: Optional pointer to a DRM file descriptor
1306  *
1307  * If @drm_file is specified, it will be used to acquire the VM from
1308  * that file descriptor. If successful, the @pdd takes ownership of
1309  * the file descriptor.
1310  *
1311  * If @drm_file is NULL, a new VM is created.
1312  *
1313  * Returns 0 on success, -errno on failure.
1314  */
1315 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1316                                struct file *drm_file)
1317 {
1318         struct kfd_process *p;
1319         struct kfd_dev *dev;
1320         int ret;
1321
1322         if (pdd->vm)
1323                 return drm_file ? -EBUSY : 0;
1324
1325         p = pdd->process;
1326         dev = pdd->dev;
1327
1328         if (drm_file)
1329                 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1330                         dev->kgd, drm_file, p->pasid,
1331                         &pdd->vm, &p->kgd_process_info, &p->ef);
1332         else
1333                 ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid,
1334                         &pdd->vm, &p->kgd_process_info, &p->ef);
1335         if (ret) {
1336                 pr_err("Failed to create process VM object\n");
1337                 return ret;
1338         }
1339
1340         amdgpu_vm_set_task_info(pdd->vm);
1341
1342         ret = kfd_process_device_reserve_ib_mem(pdd);
1343         if (ret)
1344                 goto err_reserve_ib_mem;
1345         ret = kfd_process_device_init_cwsr_dgpu(pdd);
1346         if (ret)
1347                 goto err_init_cwsr;
1348
1349         pdd->drm_file = drm_file;
1350
1351         return 0;
1352
1353 err_init_cwsr:
1354 err_reserve_ib_mem:
1355         kfd_process_device_free_bos(pdd);
1356         if (!drm_file)
1357                 amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm);
1358         pdd->vm = NULL;
1359
1360         return ret;
1361 }
1362
1363 /*
1364  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1365  * to the device.
1366  * Unbinding occurs when the process dies or the device is removed.
1367  *
1368  * Assumes that the process lock is held.
1369  */
1370 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1371                                                         struct kfd_process *p)
1372 {
1373         struct kfd_process_device *pdd;
1374         int err;
1375
1376         pdd = kfd_get_process_device_data(dev, p);
1377         if (!pdd) {
1378                 pr_err("Process device data doesn't exist\n");
1379                 return ERR_PTR(-ENOMEM);
1380         }
1381
1382         /*
1383          * signal runtime-pm system to auto resume and prevent
1384          * further runtime suspend once device pdd is created until
1385          * pdd is destroyed.
1386          */
1387         if (!pdd->runtime_inuse) {
1388                 err = pm_runtime_get_sync(dev->ddev->dev);
1389                 if (err < 0) {
1390                         pm_runtime_put_autosuspend(dev->ddev->dev);
1391                         return ERR_PTR(err);
1392                 }
1393         }
1394
1395         err = kfd_iommu_bind_process_to_device(pdd);
1396         if (err)
1397                 goto out;
1398
1399         err = kfd_process_device_init_vm(pdd, NULL);
1400         if (err)
1401                 goto out;
1402
1403         /*
1404          * make sure that runtime_usage counter is incremented just once
1405          * per pdd
1406          */
1407         pdd->runtime_inuse = true;
1408
1409         return pdd;
1410
1411 out:
1412         /* balance runpm reference count and exit with error */
1413         if (!pdd->runtime_inuse) {
1414                 pm_runtime_mark_last_busy(dev->ddev->dev);
1415                 pm_runtime_put_autosuspend(dev->ddev->dev);
1416         }
1417
1418         return ERR_PTR(err);
1419 }
1420
1421 struct kfd_process_device *kfd_get_first_process_device_data(
1422                                                 struct kfd_process *p)
1423 {
1424         return list_first_entry(&p->per_device_data,
1425                                 struct kfd_process_device,
1426                                 per_device_list);
1427 }
1428
1429 struct kfd_process_device *kfd_get_next_process_device_data(
1430                                                 struct kfd_process *p,
1431                                                 struct kfd_process_device *pdd)
1432 {
1433         if (list_is_last(&pdd->per_device_list, &p->per_device_data))
1434                 return NULL;
1435         return list_next_entry(pdd, per_device_list);
1436 }
1437
1438 bool kfd_has_process_device_data(struct kfd_process *p)
1439 {
1440         return !(list_empty(&p->per_device_data));
1441 }
1442
1443 /* Create specific handle mapped to mem from process local memory idr
1444  * Assumes that the process lock is held.
1445  */
1446 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1447                                         void *mem)
1448 {
1449         return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1450 }
1451
1452 /* Translate specific handle from process local memory idr
1453  * Assumes that the process lock is held.
1454  */
1455 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1456                                         int handle)
1457 {
1458         if (handle < 0)
1459                 return NULL;
1460
1461         return idr_find(&pdd->alloc_idr, handle);
1462 }
1463
1464 /* Remove specific handle from process local memory idr
1465  * Assumes that the process lock is held.
1466  */
1467 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1468                                         int handle)
1469 {
1470         if (handle >= 0)
1471                 idr_remove(&pdd->alloc_idr, handle);
1472 }
1473
1474 /* This increments the process->ref counter. */
1475 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1476 {
1477         struct kfd_process *p, *ret_p = NULL;
1478         unsigned int temp;
1479
1480         int idx = srcu_read_lock(&kfd_processes_srcu);
1481
1482         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1483                 if (p->pasid == pasid) {
1484                         kref_get(&p->ref);
1485                         ret_p = p;
1486                         break;
1487                 }
1488         }
1489
1490         srcu_read_unlock(&kfd_processes_srcu, idx);
1491
1492         return ret_p;
1493 }
1494
1495 /* This increments the process->ref counter. */
1496 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1497 {
1498         struct kfd_process *p;
1499
1500         int idx = srcu_read_lock(&kfd_processes_srcu);
1501
1502         p = find_process_by_mm(mm);
1503         if (p)
1504                 kref_get(&p->ref);
1505
1506         srcu_read_unlock(&kfd_processes_srcu, idx);
1507
1508         return p;
1509 }
1510
1511 /* kfd_process_evict_queues - Evict all user queues of a process
1512  *
1513  * Eviction is reference-counted per process-device. This means multiple
1514  * evictions from different sources can be nested safely.
1515  */
1516 int kfd_process_evict_queues(struct kfd_process *p)
1517 {
1518         struct kfd_process_device *pdd;
1519         int r = 0;
1520         unsigned int n_evicted = 0;
1521
1522         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1523                 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1524                                                             &pdd->qpd);
1525                 if (r) {
1526                         pr_err("Failed to evict process queues\n");
1527                         goto fail;
1528                 }
1529                 n_evicted++;
1530         }
1531
1532         return r;
1533
1534 fail:
1535         /* To keep state consistent, roll back partial eviction by
1536          * restoring queues
1537          */
1538         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1539                 if (n_evicted == 0)
1540                         break;
1541                 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1542                                                               &pdd->qpd))
1543                         pr_err("Failed to restore queues\n");
1544
1545                 n_evicted--;
1546         }
1547
1548         return r;
1549 }
1550
1551 /* kfd_process_restore_queues - Restore all user queues of a process */
1552 int kfd_process_restore_queues(struct kfd_process *p)
1553 {
1554         struct kfd_process_device *pdd;
1555         int r, ret = 0;
1556
1557         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1558                 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1559                                                               &pdd->qpd);
1560                 if (r) {
1561                         pr_err("Failed to restore process queues\n");
1562                         if (!ret)
1563                                 ret = r;
1564                 }
1565         }
1566
1567         return ret;
1568 }
1569
1570 static void evict_process_worker(struct work_struct *work)
1571 {
1572         int ret;
1573         struct kfd_process *p;
1574         struct delayed_work *dwork;
1575
1576         dwork = to_delayed_work(work);
1577
1578         /* Process termination destroys this worker thread. So during the
1579          * lifetime of this thread, kfd_process p will be valid
1580          */
1581         p = container_of(dwork, struct kfd_process, eviction_work);
1582         WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1583                   "Eviction fence mismatch\n");
1584
1585         /* Narrow window of overlap between restore and evict work
1586          * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1587          * unreserves KFD BOs, it is possible to evicted again. But
1588          * restore has few more steps of finish. So lets wait for any
1589          * previous restore work to complete
1590          */
1591         flush_delayed_work(&p->restore_work);
1592
1593         pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1594         ret = kfd_process_evict_queues(p);
1595         if (!ret) {
1596                 dma_fence_signal(p->ef);
1597                 dma_fence_put(p->ef);
1598                 p->ef = NULL;
1599                 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1600                                 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1601
1602                 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1603         } else
1604                 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1605 }
1606
1607 static void restore_process_worker(struct work_struct *work)
1608 {
1609         struct delayed_work *dwork;
1610         struct kfd_process *p;
1611         int ret = 0;
1612
1613         dwork = to_delayed_work(work);
1614
1615         /* Process termination destroys this worker thread. So during the
1616          * lifetime of this thread, kfd_process p will be valid
1617          */
1618         p = container_of(dwork, struct kfd_process, restore_work);
1619         pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1620
1621         /* Setting last_restore_timestamp before successful restoration.
1622          * Otherwise this would have to be set by KGD (restore_process_bos)
1623          * before KFD BOs are unreserved. If not, the process can be evicted
1624          * again before the timestamp is set.
1625          * If restore fails, the timestamp will be set again in the next
1626          * attempt. This would mean that the minimum GPU quanta would be
1627          * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1628          * functions)
1629          */
1630
1631         p->last_restore_timestamp = get_jiffies_64();
1632         ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1633                                                      &p->ef);
1634         if (ret) {
1635                 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1636                          p->pasid, PROCESS_BACK_OFF_TIME_MS);
1637                 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1638                                 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1639                 WARN(!ret, "reschedule restore work failed\n");
1640                 return;
1641         }
1642
1643         ret = kfd_process_restore_queues(p);
1644         if (!ret)
1645                 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1646         else
1647                 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1648 }
1649
1650 void kfd_suspend_all_processes(void)
1651 {
1652         struct kfd_process *p;
1653         unsigned int temp;
1654         int idx = srcu_read_lock(&kfd_processes_srcu);
1655
1656         WARN(debug_evictions, "Evicting all processes");
1657         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1658                 cancel_delayed_work_sync(&p->eviction_work);
1659                 cancel_delayed_work_sync(&p->restore_work);
1660
1661                 if (kfd_process_evict_queues(p))
1662                         pr_err("Failed to suspend process 0x%x\n", p->pasid);
1663                 dma_fence_signal(p->ef);
1664                 dma_fence_put(p->ef);
1665                 p->ef = NULL;
1666         }
1667         srcu_read_unlock(&kfd_processes_srcu, idx);
1668 }
1669
1670 int kfd_resume_all_processes(void)
1671 {
1672         struct kfd_process *p;
1673         unsigned int temp;
1674         int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1675
1676         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1677                 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1678                         pr_err("Restore process %d failed during resume\n",
1679                                p->pasid);
1680                         ret = -EFAULT;
1681                 }
1682         }
1683         srcu_read_unlock(&kfd_processes_srcu, idx);
1684         return ret;
1685 }
1686
1687 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1688                           struct vm_area_struct *vma)
1689 {
1690         struct kfd_process_device *pdd;
1691         struct qcm_process_device *qpd;
1692
1693         if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1694                 pr_err("Incorrect CWSR mapping size.\n");
1695                 return -EINVAL;
1696         }
1697
1698         pdd = kfd_get_process_device_data(dev, process);
1699         if (!pdd)
1700                 return -EINVAL;
1701         qpd = &pdd->qpd;
1702
1703         qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1704                                         get_order(KFD_CWSR_TBA_TMA_SIZE));
1705         if (!qpd->cwsr_kaddr) {
1706                 pr_err("Error allocating per process CWSR buffer.\n");
1707                 return -ENOMEM;
1708         }
1709
1710         vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1711                 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1712         /* Mapping pages to user process */
1713         return remap_pfn_range(vma, vma->vm_start,
1714                                PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1715                                KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1716 }
1717
1718 void kfd_flush_tlb(struct kfd_process_device *pdd)
1719 {
1720         struct kfd_dev *dev = pdd->dev;
1721
1722         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1723                 /* Nothing to flush until a VMID is assigned, which
1724                  * only happens when the first queue is created.
1725                  */
1726                 if (pdd->qpd.vmid)
1727                         amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1728                                                         pdd->qpd.vmid);
1729         } else {
1730                 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1731                                                 pdd->process->pasid);
1732         }
1733 }
1734
1735 #if defined(CONFIG_DEBUG_FS)
1736
1737 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1738 {
1739         struct kfd_process *p;
1740         unsigned int temp;
1741         int r = 0;
1742
1743         int idx = srcu_read_lock(&kfd_processes_srcu);
1744
1745         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1746                 seq_printf(m, "Process %d PASID 0x%x:\n",
1747                            p->lead_thread->tgid, p->pasid);
1748
1749                 mutex_lock(&p->mutex);
1750                 r = pqm_debugfs_mqds(m, &p->pqm);
1751                 mutex_unlock(&p->mutex);
1752
1753                 if (r)
1754                         break;
1755         }
1756
1757         srcu_read_unlock(&kfd_processes_srcu, idx);
1758
1759         return r;
1760 }
1761
1762 #endif
1763