drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/device.h>
  25 #include <linux/export.h>
  26 #include <linux/err.h>
  27 #include <linux/fs.h>
  28 #include <linux/file.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31 #include <linux/uaccess.h>
  32 #include <linux/compat.h>
  33 #include <uapi/linux/kfd_ioctl.h>
  34 #include <linux/time.h>
  35 #include <linux/mm.h>
  36 #include <linux/mman.h>
  37 #include <linux/ptrace.h>
  38 #include <linux/dma-buf.h>
  39 #include <linux/fdtable.h>
  40 #include <linux/processor.h>
  41 #include "kfd_priv.h"
  42 #include "kfd_device_queue_manager.h"
  43 #include "kfd_svm.h"
  44 #include "amdgpu_amdkfd.h"
  45 #include "kfd_smi_events.h"
  46 #include "amdgpu_dma_buf.h"
  47 #include "kfd_debug.h"
  48
  49 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
  50 static int kfd_open(struct inode *, struct file *);
  51 static int kfd_release(struct inode *, struct file *);
  52 static int kfd_mmap(struct file *, struct vm_area_struct *);
  53
  54 static const char kfd_dev_name[] = "kfd";
  55
  56 static const struct file_operations kfd_fops = {
  57         .owner = THIS_MODULE,
  58         .unlocked_ioctl = kfd_ioctl,
  59         .compat_ioctl = compat_ptr_ioctl,
  60         .open = kfd_open,
  61         .release = kfd_release,
  62         .mmap = kfd_mmap,
  63 };
  64
  65 static int kfd_char_dev_major = -1;
  66 static struct class *kfd_class;
  67 struct device *kfd_device;
  68
  69 static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
  70 {
  71         struct kfd_process_device *pdd;
  72
  73         mutex_lock(&p->mutex);
  74         pdd = kfd_process_device_data_by_id(p, gpu_id);
  75
  76         if (pdd)
  77                 return pdd;
  78
  79         mutex_unlock(&p->mutex);
  80         return NULL;
  81 }
  82
  83 static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
  84 {
  85         mutex_unlock(&pdd->process->mutex);
  86 }
  87
  88 int kfd_chardev_init(void)
  89 {
  90         int err = 0;
  91
  92         kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
  93         err = kfd_char_dev_major;
  94         if (err < 0)
  95                 goto err_register_chrdev;
  96
  97         kfd_class = class_create(kfd_dev_name);
  98         err = PTR_ERR(kfd_class);
  99         if (IS_ERR(kfd_class))
 100                 goto err_class_create;
 101
 102         kfd_device = device_create(kfd_class, NULL,
 103                                         MKDEV(kfd_char_dev_major, 0),
 104                                         NULL, kfd_dev_name);
 105         err = PTR_ERR(kfd_device);
 106         if (IS_ERR(kfd_device))
 107                 goto err_device_create;
 108
 109         return 0;
 110
 111 err_device_create:
 112         class_destroy(kfd_class);
 113 err_class_create:
 114         unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 115 err_register_chrdev:
 116         return err;
 117 }
 118
 119 void kfd_chardev_exit(void)
 120 {
 121         device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
 122         class_destroy(kfd_class);
 123         unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 124         kfd_device = NULL;
 125 }
 126
 127
 128 static int kfd_open(struct inode *inode, struct file *filep)
 129 {
 130         struct kfd_process *process;
 131         bool is_32bit_user_mode;
 132
 133         if (iminor(inode) != 0)
 134                 return -ENODEV;
 135
 136         is_32bit_user_mode = in_compat_syscall();
 137
 138         if (is_32bit_user_mode) {
 139                 dev_warn(kfd_device,
 140                         "Process %d (32-bit) failed to open /dev/kfd\n"
 141                         "32-bit processes are not supported by amdkfd\n",
 142                         current->pid);
 143                 return -EPERM;
 144         }
 145
 146         process = kfd_create_process(current);
 147         if (IS_ERR(process))
 148                 return PTR_ERR(process);
 149
 150         if (kfd_process_init_cwsr_apu(process, filep)) {
 151                 kfd_unref_process(process);
 152                 return -EFAULT;
 153         }
 154
 155         /* filep now owns the reference returned by kfd_create_process */
 156         filep->private_data = process;
 157
 158         dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
 159                 process->pasid, process->is_32bit_user_mode);
 160
 161         return 0;
 162 }
 163
 164 static int kfd_release(struct inode *inode, struct file *filep)
 165 {
 166         struct kfd_process *process = filep->private_data;
 167
 168         if (process)
 169                 kfd_unref_process(process);
 170
 171         return 0;
 172 }
 173
 174 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
 175                                         void *data)
 176 {
 177         struct kfd_ioctl_get_version_args *args = data;
 178
 179         args->major_version = KFD_IOCTL_MAJOR_VERSION;
 180         args->minor_version = KFD_IOCTL_MINOR_VERSION;
 181
 182         return 0;
 183 }
 184
 185 static int set_queue_properties_from_user(struct queue_properties *q_properties,
 186                                 struct kfd_ioctl_create_queue_args *args)
 187 {
 188         /*
 189          * Repurpose queue percentage to accommodate new features:
 190          * bit 0-7: queue percentage
 191          * bit 8-15: pm4_target_xcc
 192          */
 193         if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 194                 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 195                 return -EINVAL;
 196         }
 197
 198         if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 199                 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 200                 return -EINVAL;
 201         }
 202
 203         if ((args->ring_base_address) &&
 204                 (!access_ok((const void __user *) args->ring_base_address,
 205                         sizeof(uint64_t)))) {
 206                 pr_err("Can't access ring base address\n");
 207                 return -EFAULT;
 208         }
 209
 210         if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 211                 pr_err("Ring size must be a power of 2 or 0\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (!access_ok((const void __user *) args->read_pointer_address,
 216                         sizeof(uint32_t))) {
 217                 pr_err("Can't access read pointer\n");
 218                 return -EFAULT;
 219         }
 220
 221         if (!access_ok((const void __user *) args->write_pointer_address,
 222                         sizeof(uint32_t))) {
 223                 pr_err("Can't access write pointer\n");
 224                 return -EFAULT;
 225         }
 226
 227         if (args->eop_buffer_address &&
 228                 !access_ok((const void __user *) args->eop_buffer_address,
 229                         sizeof(uint32_t))) {
 230                 pr_debug("Can't access eop buffer");
 231                 return -EFAULT;
 232         }
 233
 234         if (args->ctx_save_restore_address &&
 235                 !access_ok((const void __user *) args->ctx_save_restore_address,
 236                         sizeof(uint32_t))) {
 237                 pr_debug("Can't access ctx save restore buffer");
 238                 return -EFAULT;
 239         }
 240
 241         q_properties->is_interop = false;
 242         q_properties->is_gws = false;
 243         q_properties->queue_percent = args->queue_percentage & 0xFF;
 244         /* bit 8-15 are repurposed to be PM4 target XCC */
 245         q_properties->pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 246         q_properties->priority = args->queue_priority;
 247         q_properties->queue_address = args->ring_base_address;
 248         q_properties->queue_size = args->ring_size;
 249         q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
 250         q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
 251         q_properties->eop_ring_buffer_address = args->eop_buffer_address;
 252         q_properties->eop_ring_buffer_size = args->eop_buffer_size;
 253         q_properties->ctx_save_restore_area_address =
 254                         args->ctx_save_restore_address;
 255         q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
 256         q_properties->ctl_stack_size = args->ctl_stack_size;
 257         if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 258                 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 259                 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
 260         else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
 261                 q_properties->type = KFD_QUEUE_TYPE_SDMA;
 262         else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
 263                 q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
 264         else
 265                 return -ENOTSUPP;
 266
 267         if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 268                 q_properties->format = KFD_QUEUE_FORMAT_AQL;
 269         else
 270                 q_properties->format = KFD_QUEUE_FORMAT_PM4;
 271
 272         pr_debug("Queue Percentage: %d, %d\n",
 273                         q_properties->queue_percent, args->queue_percentage);
 274
 275         pr_debug("Queue Priority: %d, %d\n",
 276                         q_properties->priority, args->queue_priority);
 277
 278         pr_debug("Queue Address: 0x%llX, 0x%llX\n",
 279                         q_properties->queue_address, args->ring_base_address);
 280
 281         pr_debug("Queue Size: 0x%llX, %u\n",
 282                         q_properties->queue_size, args->ring_size);
 283
 284         pr_debug("Queue r/w Pointers: %px, %px\n",
 285                         q_properties->read_ptr,
 286                         q_properties->write_ptr);
 287
 288         pr_debug("Queue Format: %d\n", q_properties->format);
 289
 290         pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
 291
 292         pr_debug("Queue CTX save area: 0x%llX\n",
 293                         q_properties->ctx_save_restore_area_address);
 294
 295         return 0;
 296 }
 297
 298 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 299                                         void *data)
 300 {
 301         struct kfd_ioctl_create_queue_args *args = data;
 302         struct kfd_node *dev;
 303         int err = 0;
 304         unsigned int queue_id;
 305         struct kfd_process_device *pdd;
 306         struct queue_properties q_properties;
 307         uint32_t doorbell_offset_in_process = 0;
 308         struct amdgpu_bo *wptr_bo = NULL;
 309
 310         memset(&q_properties, 0, sizeof(struct queue_properties));
 311
 312         pr_debug("Creating queue ioctl\n");
 313
 314         err = set_queue_properties_from_user(&q_properties, args);
 315         if (err)
 316                 return err;
 317
 318         pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
 319
 320         mutex_lock(&p->mutex);
 321
 322         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 323         if (!pdd) {
 324                 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 325                 err = -EINVAL;
 326                 goto err_pdd;
 327         }
 328         dev = pdd->dev;
 329
 330         pdd = kfd_bind_process_to_device(dev, p);
 331         if (IS_ERR(pdd)) {
 332                 err = -ESRCH;
 333                 goto err_bind_process;
 334         }
 335
 336         if (!pdd->doorbell_index &&
 337             kfd_alloc_process_doorbells(dev->kfd, &pdd->doorbell_index) < 0) {
 338                 err = -ENOMEM;
 339                 goto err_alloc_doorbells;
 340         }
 341
 342         /* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
 343          * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
 344          */
 345         if (dev->kfd->shared_resources.enable_mes &&
 346                         ((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)
 347                         >> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {
 348                 struct amdgpu_bo_va_mapping *wptr_mapping;
 349                 struct amdgpu_vm *wptr_vm;
 350
 351                 wptr_vm = drm_priv_to_vm(pdd->drm_priv);
 352                 err = amdgpu_bo_reserve(wptr_vm->root.bo, false);
 353                 if (err)
 354                         goto err_wptr_map_gart;
 355
 356                 wptr_mapping = amdgpu_vm_bo_lookup_mapping(
 357                                 wptr_vm, args->write_pointer_address >> PAGE_SHIFT);
 358                 amdgpu_bo_unreserve(wptr_vm->root.bo);
 359                 if (!wptr_mapping) {
 360                         pr_err("Failed to lookup wptr bo\n");
 361                         err = -EINVAL;
 362                         goto err_wptr_map_gart;
 363                 }
 364
 365                 wptr_bo = wptr_mapping->bo_va->base.bo;
 366                 if (wptr_bo->tbo.base.size > PAGE_SIZE) {
 367                         pr_err("Requested GART mapping for wptr bo larger than one page\n");
 368                         err = -EINVAL;
 369                         goto err_wptr_map_gart;
 370                 }
 371
 372                 err = amdgpu_amdkfd_map_gtt_bo_to_gart(dev->adev, wptr_bo);
 373                 if (err) {
 374                         pr_err("Failed to map wptr bo to GART\n");
 375                         goto err_wptr_map_gart;
 376                 }
 377         }
 378
 379         pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
 380                         p->pasid,
 381                         dev->id);
 382
 383         err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, wptr_bo,
 384                         NULL, NULL, NULL, &doorbell_offset_in_process);
 385         if (err != 0)
 386                 goto err_create_queue;
 387
 388         args->queue_id = queue_id;
 389
 390
 391         /* Return gpu_id as doorbell offset for mmap usage */
 392         args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
 393         args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
 394         if (KFD_IS_SOC15(dev))
 395                 /* On SOC15 ASICs, include the doorbell offset within the
 396                  * process doorbell frame, which is 2 pages.
 397                  */
 398                 args->doorbell_offset |= doorbell_offset_in_process;
 399
 400         mutex_unlock(&p->mutex);
 401
 402         pr_debug("Queue id %d was created successfully\n", args->queue_id);
 403
 404         pr_debug("Ring buffer address == 0x%016llX\n",
 405                         args->ring_base_address);
 406
 407         pr_debug("Read ptr address    == 0x%016llX\n",
 408                         args->read_pointer_address);
 409
 410         pr_debug("Write ptr address   == 0x%016llX\n",
 411                         args->write_pointer_address);
 412
 413         kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), p, dev, queue_id, false, NULL, 0);
 414         return 0;
 415
 416 err_create_queue:
 417         if (wptr_bo)
 418                 amdgpu_amdkfd_free_gtt_mem(dev->adev, wptr_bo);
 419 err_wptr_map_gart:
 420 err_alloc_doorbells:
 421 err_bind_process:
 422 err_pdd:
 423         mutex_unlock(&p->mutex);
 424         return err;
 425 }
 426
 427 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 428                                         void *data)
 429 {
 430         int retval;
 431         struct kfd_ioctl_destroy_queue_args *args = data;
 432
 433         pr_debug("Destroying queue id %d for pasid 0x%x\n",
 434                                 args->queue_id,
 435                                 p->pasid);
 436
 437         mutex_lock(&p->mutex);
 438
 439         retval = pqm_destroy_queue(&p->pqm, args->queue_id);
 440
 441         mutex_unlock(&p->mutex);
 442         return retval;
 443 }
 444
 445 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 446                                         void *data)
 447 {
 448         int retval;
 449         struct kfd_ioctl_update_queue_args *args = data;
 450         struct queue_properties properties;
 451
 452         /*
 453          * Repurpose queue percentage to accommodate new features:
 454          * bit 0-7: queue percentage
 455          * bit 8-15: pm4_target_xcc
 456          */
 457         if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 458                 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 459                 return -EINVAL;
 460         }
 461
 462         if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 463                 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 464                 return -EINVAL;
 465         }
 466
 467         if ((args->ring_base_address) &&
 468                 (!access_ok((const void __user *) args->ring_base_address,
 469                         sizeof(uint64_t)))) {
 470                 pr_err("Can't access ring base address\n");
 471                 return -EFAULT;
 472         }
 473
 474         if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 475                 pr_err("Ring size must be a power of 2 or 0\n");
 476                 return -EINVAL;
 477         }
 478
 479         properties.queue_address = args->ring_base_address;
 480         properties.queue_size = args->ring_size;
 481         properties.queue_percent = args->queue_percentage & 0xFF;
 482         /* bit 8-15 are repurposed to be PM4 target XCC */
 483         properties.pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 484         properties.priority = args->queue_priority;
 485
 486         pr_debug("Updating queue id %d for pasid 0x%x\n",
 487                         args->queue_id, p->pasid);
 488
 489         mutex_lock(&p->mutex);
 490
 491         retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
 492
 493         mutex_unlock(&p->mutex);
 494
 495         return retval;
 496 }
 497
 498 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
 499                                         void *data)
 500 {
 501         int retval;
 502         const int max_num_cus = 1024;
 503         struct kfd_ioctl_set_cu_mask_args *args = data;
 504         struct mqd_update_info minfo = {0};
 505         uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
 506         size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
 507
 508         if ((args->num_cu_mask % 32) != 0) {
 509                 pr_debug("num_cu_mask 0x%x must be a multiple of 32",
 510                                 args->num_cu_mask);
 511                 return -EINVAL;
 512         }
 513
 514         minfo.cu_mask.count = args->num_cu_mask;
 515         if (minfo.cu_mask.count == 0) {
 516                 pr_debug("CU mask cannot be 0");
 517                 return -EINVAL;
 518         }
 519
 520         /* To prevent an unreasonably large CU mask size, set an arbitrary
 521          * limit of max_num_cus bits.  We can then just drop any CU mask bits
 522          * past max_num_cus bits and just use the first max_num_cus bits.
 523          */
 524         if (minfo.cu_mask.count > max_num_cus) {
 525                 pr_debug("CU mask cannot be greater than 1024 bits");
 526                 minfo.cu_mask.count = max_num_cus;
 527                 cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
 528         }
 529
 530         minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
 531         if (!minfo.cu_mask.ptr)
 532                 return -ENOMEM;
 533
 534         retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
 535         if (retval) {
 536                 pr_debug("Could not copy CU mask from userspace");
 537                 retval = -EFAULT;
 538                 goto out;
 539         }
 540
 541         mutex_lock(&p->mutex);
 542
 543         retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
 544
 545         mutex_unlock(&p->mutex);
 546
 547 out:
 548         kfree(minfo.cu_mask.ptr);
 549         return retval;
 550 }
 551
 552 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
 553                                           struct kfd_process *p, void *data)
 554 {
 555         struct kfd_ioctl_get_queue_wave_state_args *args = data;
 556         int r;
 557
 558         mutex_lock(&p->mutex);
 559
 560         r = pqm_get_wave_state(&p->pqm, args->queue_id,
 561                                (void __user *)args->ctl_stack_address,
 562                                &args->ctl_stack_used_size,
 563                                &args->save_area_used_size);
 564
 565         mutex_unlock(&p->mutex);
 566
 567         return r;
 568 }
 569
 570 static int kfd_ioctl_set_memory_policy(struct file *filep,
 571                                         struct kfd_process *p, void *data)
 572 {
 573         struct kfd_ioctl_set_memory_policy_args *args = data;
 574         int err = 0;
 575         struct kfd_process_device *pdd;
 576         enum cache_policy default_policy, alternate_policy;
 577
 578         if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
 579             && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 580                 return -EINVAL;
 581         }
 582
 583         if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
 584             && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 585                 return -EINVAL;
 586         }
 587
 588         mutex_lock(&p->mutex);
 589         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 590         if (!pdd) {
 591                 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 592                 err = -EINVAL;
 593                 goto err_pdd;
 594         }
 595
 596         pdd = kfd_bind_process_to_device(pdd->dev, p);
 597         if (IS_ERR(pdd)) {
 598                 err = -ESRCH;
 599                 goto out;
 600         }
 601
 602         default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 603                          ? cache_policy_coherent : cache_policy_noncoherent;
 604
 605         alternate_policy =
 606                 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 607                    ? cache_policy_coherent : cache_policy_noncoherent;
 608
 609         if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
 610                                 &pdd->qpd,
 611                                 default_policy,
 612                                 alternate_policy,
 613                                 (void __user *)args->alternate_aperture_base,
 614                                 args->alternate_aperture_size))
 615                 err = -EINVAL;
 616
 617 out:
 618 err_pdd:
 619         mutex_unlock(&p->mutex);
 620
 621         return err;
 622 }
 623
 624 static int kfd_ioctl_set_trap_handler(struct file *filep,
 625                                         struct kfd_process *p, void *data)
 626 {
 627         struct kfd_ioctl_set_trap_handler_args *args = data;
 628         int err = 0;
 629         struct kfd_process_device *pdd;
 630
 631         mutex_lock(&p->mutex);
 632
 633         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 634         if (!pdd) {
 635                 err = -EINVAL;
 636                 goto err_pdd;
 637         }
 638
 639         pdd = kfd_bind_process_to_device(pdd->dev, p);
 640         if (IS_ERR(pdd)) {
 641                 err = -ESRCH;
 642                 goto out;
 643         }
 644
 645         kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
 646
 647 out:
 648 err_pdd:
 649         mutex_unlock(&p->mutex);
 650
 651         return err;
 652 }
 653
 654 static int kfd_ioctl_dbg_register(struct file *filep,
 655                                 struct kfd_process *p, void *data)
 656 {
 657         return -EPERM;
 658 }
 659
 660 static int kfd_ioctl_dbg_unregister(struct file *filep,
 661                                 struct kfd_process *p, void *data)
 662 {
 663         return -EPERM;
 664 }
 665
 666 static int kfd_ioctl_dbg_address_watch(struct file *filep,
 667                                         struct kfd_process *p, void *data)
 668 {
 669         return -EPERM;
 670 }
 671
 672 /* Parse and generate fixed size data structure for wave control */
 673 static int kfd_ioctl_dbg_wave_control(struct file *filep,
 674                                         struct kfd_process *p, void *data)
 675 {
 676         return -EPERM;
 677 }
 678
 679 static int kfd_ioctl_get_clock_counters(struct file *filep,
 680                                 struct kfd_process *p, void *data)
 681 {
 682         struct kfd_ioctl_get_clock_counters_args *args = data;
 683         struct kfd_process_device *pdd;
 684
 685         mutex_lock(&p->mutex);
 686         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 687         mutex_unlock(&p->mutex);
 688         if (pdd)
 689                 /* Reading GPU clock counter from KGD */
 690                 args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(pdd->dev->adev);
 691         else
 692                 /* Node without GPU resource */
 693                 args->gpu_clock_counter = 0;
 694
 695         /* No access to rdtsc. Using raw monotonic time */
 696         args->cpu_clock_counter = ktime_get_raw_ns();
 697         args->system_clock_counter = ktime_get_boottime_ns();
 698
 699         /* Since the counter is in nano-seconds we use 1GHz frequency */
 700         args->system_clock_freq = 1000000000;
 701
 702         return 0;
 703 }
 704
 705
 706 static int kfd_ioctl_get_process_apertures(struct file *filp,
 707                                 struct kfd_process *p, void *data)
 708 {
 709         struct kfd_ioctl_get_process_apertures_args *args = data;
 710         struct kfd_process_device_apertures *pAperture;
 711         int i;
 712
 713         dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 714
 715         args->num_of_nodes = 0;
 716
 717         mutex_lock(&p->mutex);
 718         /* Run over all pdd of the process */
 719         for (i = 0; i < p->n_pdds; i++) {
 720                 struct kfd_process_device *pdd = p->pdds[i];
 721
 722                 pAperture =
 723                         &args->process_apertures[args->num_of_nodes];
 724                 pAperture->gpu_id = pdd->dev->id;
 725                 pAperture->lds_base = pdd->lds_base;
 726                 pAperture->lds_limit = pdd->lds_limit;
 727                 pAperture->gpuvm_base = pdd->gpuvm_base;
 728                 pAperture->gpuvm_limit = pdd->gpuvm_limit;
 729                 pAperture->scratch_base = pdd->scratch_base;
 730                 pAperture->scratch_limit = pdd->scratch_limit;
 731
 732                 dev_dbg(kfd_device,
 733                         "node id %u\n", args->num_of_nodes);
 734                 dev_dbg(kfd_device,
 735                         "gpu id %u\n", pdd->dev->id);
 736                 dev_dbg(kfd_device,
 737                         "lds_base %llX\n", pdd->lds_base);
 738                 dev_dbg(kfd_device,
 739                         "lds_limit %llX\n", pdd->lds_limit);
 740                 dev_dbg(kfd_device,
 741                         "gpuvm_base %llX\n", pdd->gpuvm_base);
 742                 dev_dbg(kfd_device,
 743                         "gpuvm_limit %llX\n", pdd->gpuvm_limit);
 744                 dev_dbg(kfd_device,
 745                         "scratch_base %llX\n", pdd->scratch_base);
 746                 dev_dbg(kfd_device,
 747                         "scratch_limit %llX\n", pdd->scratch_limit);
 748
 749                 if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
 750                         break;
 751         }
 752         mutex_unlock(&p->mutex);
 753
 754         return 0;
 755 }
 756
 757 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
 758                                 struct kfd_process *p, void *data)
 759 {
 760         struct kfd_ioctl_get_process_apertures_new_args *args = data;
 761         struct kfd_process_device_apertures *pa;
 762         int ret;
 763         int i;
 764
 765         dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 766
 767         if (args->num_of_nodes == 0) {
 768                 /* Return number of nodes, so that user space can alloacate
 769                  * sufficient memory
 770                  */
 771                 mutex_lock(&p->mutex);
 772                 args->num_of_nodes = p->n_pdds;
 773                 goto out_unlock;
 774         }
 775
 776         /* Fill in process-aperture information for all available
 777          * nodes, but not more than args->num_of_nodes as that is
 778          * the amount of memory allocated by user
 779          */
 780         pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
 781                                 args->num_of_nodes), GFP_KERNEL);
 782         if (!pa)
 783                 return -ENOMEM;
 784
 785         mutex_lock(&p->mutex);
 786
 787         if (!p->n_pdds) {
 788                 args->num_of_nodes = 0;
 789                 kfree(pa);
 790                 goto out_unlock;
 791         }
 792
 793         /* Run over all pdd of the process */
 794         for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
 795                 struct kfd_process_device *pdd = p->pdds[i];
 796
 797                 pa[i].gpu_id = pdd->dev->id;
 798                 pa[i].lds_base = pdd->lds_base;
 799                 pa[i].lds_limit = pdd->lds_limit;
 800                 pa[i].gpuvm_base = pdd->gpuvm_base;
 801                 pa[i].gpuvm_limit = pdd->gpuvm_limit;
 802                 pa[i].scratch_base = pdd->scratch_base;
 803                 pa[i].scratch_limit = pdd->scratch_limit;
 804
 805                 dev_dbg(kfd_device,
 806                         "gpu id %u\n", pdd->dev->id);
 807                 dev_dbg(kfd_device,
 808                         "lds_base %llX\n", pdd->lds_base);
 809                 dev_dbg(kfd_device,
 810                         "lds_limit %llX\n", pdd->lds_limit);
 811                 dev_dbg(kfd_device,
 812                         "gpuvm_base %llX\n", pdd->gpuvm_base);
 813                 dev_dbg(kfd_device,
 814                         "gpuvm_limit %llX\n", pdd->gpuvm_limit);
 815                 dev_dbg(kfd_device,
 816                         "scratch_base %llX\n", pdd->scratch_base);
 817                 dev_dbg(kfd_device,
 818                         "scratch_limit %llX\n", pdd->scratch_limit);
 819         }
 820         mutex_unlock(&p->mutex);
 821
 822         args->num_of_nodes = i;
 823         ret = copy_to_user(
 824                         (void __user *)args->kfd_process_device_apertures_ptr,
 825                         pa,
 826                         (i * sizeof(struct kfd_process_device_apertures)));
 827         kfree(pa);
 828         return ret ? -EFAULT : 0;
 829
 830 out_unlock:
 831         mutex_unlock(&p->mutex);
 832         return 0;
 833 }
 834
 835 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 836                                         void *data)
 837 {
 838         struct kfd_ioctl_create_event_args *args = data;
 839         int err;
 840
 841         /* For dGPUs the event page is allocated in user mode. The
 842          * handle is passed to KFD with the first call to this IOCTL
 843          * through the event_page_offset field.
 844          */
 845         if (args->event_page_offset) {
 846                 mutex_lock(&p->mutex);
 847                 err = kfd_kmap_event_page(p, args->event_page_offset);
 848                 mutex_unlock(&p->mutex);
 849                 if (err)
 850                         return err;
 851         }
 852
 853         err = kfd_event_create(filp, p, args->event_type,
 854                                 args->auto_reset != 0, args->node_id,
 855                                 &args->event_id, &args->event_trigger_data,
 856                                 &args->event_page_offset,
 857                                 &args->event_slot_index);
 858
 859         pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
 860         return err;
 861 }
 862
 863 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
 864                                         void *data)
 865 {
 866         struct kfd_ioctl_destroy_event_args *args = data;
 867
 868         return kfd_event_destroy(p, args->event_id);
 869 }
 870
 871 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
 872                                 void *data)
 873 {
 874         struct kfd_ioctl_set_event_args *args = data;
 875
 876         return kfd_set_event(p, args->event_id);
 877 }
 878
 879 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
 880                                 void *data)
 881 {
 882         struct kfd_ioctl_reset_event_args *args = data;
 883
 884         return kfd_reset_event(p, args->event_id);
 885 }
 886
 887 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 888                                 void *data)
 889 {
 890         struct kfd_ioctl_wait_events_args *args = data;
 891
 892         return kfd_wait_on_events(p, args->num_events,
 893                         (void __user *)args->events_ptr,
 894                         (args->wait_for_all != 0),
 895                         &args->timeout, &args->wait_result);
 896 }
 897 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
 898                                         struct kfd_process *p, void *data)
 899 {
 900         struct kfd_ioctl_set_scratch_backing_va_args *args = data;
 901         struct kfd_process_device *pdd;
 902         struct kfd_node *dev;
 903         long err;
 904
 905         mutex_lock(&p->mutex);
 906         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 907         if (!pdd) {
 908                 err = -EINVAL;
 909                 goto err_pdd;
 910         }
 911         dev = pdd->dev;
 912
 913         pdd = kfd_bind_process_to_device(dev, p);
 914         if (IS_ERR(pdd)) {
 915                 err = PTR_ERR(pdd);
 916                 goto bind_process_to_device_fail;
 917         }
 918
 919         pdd->qpd.sh_hidden_private_base = args->va_addr;
 920
 921         mutex_unlock(&p->mutex);
 922
 923         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
 924             pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
 925                 dev->kfd2kgd->set_scratch_backing_va(
 926                         dev->adev, args->va_addr, pdd->qpd.vmid);
 927
 928         return 0;
 929
 930 bind_process_to_device_fail:
 931 err_pdd:
 932         mutex_unlock(&p->mutex);
 933         return err;
 934 }
 935
 936 static int kfd_ioctl_get_tile_config(struct file *filep,
 937                 struct kfd_process *p, void *data)
 938 {
 939         struct kfd_ioctl_get_tile_config_args *args = data;
 940         struct kfd_process_device *pdd;
 941         struct tile_config config;
 942         int err = 0;
 943
 944         mutex_lock(&p->mutex);
 945         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 946         mutex_unlock(&p->mutex);
 947         if (!pdd)
 948                 return -EINVAL;
 949
 950         amdgpu_amdkfd_get_tile_config(pdd->dev->adev, &config);
 951
 952         args->gb_addr_config = config.gb_addr_config;
 953         args->num_banks = config.num_banks;
 954         args->num_ranks = config.num_ranks;
 955
 956         if (args->num_tile_configs > config.num_tile_configs)
 957                 args->num_tile_configs = config.num_tile_configs;
 958         err = copy_to_user((void __user *)args->tile_config_ptr,
 959                         config.tile_config_ptr,
 960                         args->num_tile_configs * sizeof(uint32_t));
 961         if (err) {
 962                 args->num_tile_configs = 0;
 963                 return -EFAULT;
 964         }
 965
 966         if (args->num_macro_tile_configs > config.num_macro_tile_configs)
 967                 args->num_macro_tile_configs =
 968                                 config.num_macro_tile_configs;
 969         err = copy_to_user((void __user *)args->macro_tile_config_ptr,
 970                         config.macro_tile_config_ptr,
 971                         args->num_macro_tile_configs * sizeof(uint32_t));
 972         if (err) {
 973                 args->num_macro_tile_configs = 0;
 974                 return -EFAULT;
 975         }
 976
 977         return 0;
 978 }
 979
 980 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
 981                                 void *data)
 982 {
 983         struct kfd_ioctl_acquire_vm_args *args = data;
 984         struct kfd_process_device *pdd;
 985         struct file *drm_file;
 986         int ret;
 987
 988         drm_file = fget(args->drm_fd);
 989         if (!drm_file)
 990                 return -EINVAL;
 991
 992         mutex_lock(&p->mutex);
 993         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 994         if (!pdd) {
 995                 ret = -EINVAL;
 996                 goto err_pdd;
 997         }
 998
 999         if (pdd->drm_file) {
1000                 ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1001                 goto err_drm_file;
1002         }
1003
1004         ret = kfd_process_device_init_vm(pdd, drm_file);
1005         if (ret)
1006                 goto err_unlock;
1007
1008         /* On success, the PDD keeps the drm_file reference */
1009         mutex_unlock(&p->mutex);
1010
1011         return 0;
1012
1013 err_unlock:
1014 err_pdd:
1015 err_drm_file:
1016         mutex_unlock(&p->mutex);
1017         fput(drm_file);
1018         return ret;
1019 }
1020
1021 bool kfd_dev_is_large_bar(struct kfd_node *dev)
1022 {
1023         if (debug_largebar) {
1024                 pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1025                 return true;
1026         }
1027
1028         if (dev->kfd->use_iommu_v2)
1029                 return false;
1030
1031         if (dev->local_mem_info.local_mem_size_private == 0 &&
1032             dev->local_mem_info.local_mem_size_public > 0)
1033                 return true;
1034
1035         if (dev->local_mem_info.local_mem_size_public == 0 &&
1036             dev->kfd->adev->gmc.is_app_apu) {
1037                 pr_debug("APP APU, Consider like a large bar system\n");
1038                 return true;
1039         }
1040
1041         return false;
1042 }
1043
1044 static int kfd_ioctl_get_available_memory(struct file *filep,
1045                                           struct kfd_process *p, void *data)
1046 {
1047         struct kfd_ioctl_get_available_memory_args *args = data;
1048         struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
1049
1050         if (!pdd)
1051                 return -EINVAL;
1052         args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
1053                                                         pdd->dev->node_id);
1054         kfd_unlock_pdd(pdd);
1055         return 0;
1056 }
1057
1058 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1059                                         struct kfd_process *p, void *data)
1060 {
1061         struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1062         struct kfd_process_device *pdd;
1063         void *mem;
1064         struct kfd_node *dev;
1065         int idr_handle;
1066         long err;
1067         uint64_t offset = args->mmap_offset;
1068         uint32_t flags = args->flags;
1069
1070         if (args->size == 0)
1071                 return -EINVAL;
1072
1073 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1074         /* Flush pending deferred work to avoid racing with deferred actions
1075          * from previous memory map changes (e.g. munmap).
1076          */
1077         svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1078         mutex_lock(&p->svms.lock);
1079         mmap_write_unlock(current->mm);
1080         if (interval_tree_iter_first(&p->svms.objects,
1081                                      args->va_addr >> PAGE_SHIFT,
1082                                      (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1083                 pr_err("Address: 0x%llx already allocated by SVM\n",
1084                         args->va_addr);
1085                 mutex_unlock(&p->svms.lock);
1086                 return -EADDRINUSE;
1087         }
1088
1089         /* When register user buffer check if it has been registered by svm by
1090          * buffer cpu virtual address.
1091          */
1092         if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) &&
1093             interval_tree_iter_first(&p->svms.objects,
1094                                      args->mmap_offset >> PAGE_SHIFT,
1095                                      (args->mmap_offset  + args->size - 1) >> PAGE_SHIFT)) {
1096                 pr_err("User Buffer Address: 0x%llx already allocated by SVM\n",
1097                         args->mmap_offset);
1098                 mutex_unlock(&p->svms.lock);
1099                 return -EADDRINUSE;
1100         }
1101
1102         mutex_unlock(&p->svms.lock);
1103 #endif
1104         mutex_lock(&p->mutex);
1105         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1106         if (!pdd) {
1107                 err = -EINVAL;
1108                 goto err_pdd;
1109         }
1110
1111         dev = pdd->dev;
1112
1113         if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1114                 (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1115                 !kfd_dev_is_large_bar(dev)) {
1116                 pr_err("Alloc host visible vram on small bar is not allowed\n");
1117                 err = -EINVAL;
1118                 goto err_large_bar;
1119         }
1120
1121         pdd = kfd_bind_process_to_device(dev, p);
1122         if (IS_ERR(pdd)) {
1123                 err = PTR_ERR(pdd);
1124                 goto err_unlock;
1125         }
1126
1127         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1128                 if (args->size != kfd_doorbell_process_slice(dev->kfd)) {
1129                         err = -EINVAL;
1130                         goto err_unlock;
1131                 }
1132                 offset = kfd_get_process_doorbells(pdd);
1133                 if (!offset) {
1134                         err = -ENOMEM;
1135                         goto err_unlock;
1136                 }
1137         } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1138                 if (args->size != PAGE_SIZE) {
1139                         err = -EINVAL;
1140                         goto err_unlock;
1141                 }
1142                 offset = dev->adev->rmmio_remap.bus_addr;
1143                 if (!offset) {
1144                         err = -ENOMEM;
1145                         goto err_unlock;
1146                 }
1147         }
1148
1149         err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1150                 dev->adev, args->va_addr, args->size,
1151                 pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1152                 flags, false);
1153
1154         if (err)
1155                 goto err_unlock;
1156
1157         idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1158         if (idr_handle < 0) {
1159                 err = -EFAULT;
1160                 goto err_free;
1161         }
1162
1163         /* Update the VRAM usage count */
1164         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1165                 uint64_t size = args->size;
1166
1167                 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
1168                         size >>= 1;
1169                 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + PAGE_ALIGN(size));
1170         }
1171
1172         mutex_unlock(&p->mutex);
1173
1174         args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1175         args->mmap_offset = offset;
1176
1177         /* MMIO is mapped through kfd device
1178          * Generate a kfd mmap offset
1179          */
1180         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1181                 args->mmap_offset = KFD_MMAP_TYPE_MMIO
1182                                         | KFD_MMAP_GPU_ID(args->gpu_id);
1183
1184         return 0;
1185
1186 err_free:
1187         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
1188                                                pdd->drm_priv, NULL);
1189 err_unlock:
1190 err_pdd:
1191 err_large_bar:
1192         mutex_unlock(&p->mutex);
1193         return err;
1194 }
1195
1196 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1197                                         struct kfd_process *p, void *data)
1198 {
1199         struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1200         struct kfd_process_device *pdd;
1201         void *mem;
1202         int ret;
1203         uint64_t size = 0;
1204
1205         mutex_lock(&p->mutex);
1206         /*
1207          * Safeguard to prevent user space from freeing signal BO.
1208          * It will be freed at process termination.
1209          */
1210         if (p->signal_handle && (p->signal_handle == args->handle)) {
1211                 pr_err("Free signal BO is not allowed\n");
1212                 ret = -EPERM;
1213                 goto err_unlock;
1214         }
1215
1216         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1217         if (!pdd) {
1218                 pr_err("Process device data doesn't exist\n");
1219                 ret = -EINVAL;
1220                 goto err_pdd;
1221         }
1222
1223         mem = kfd_process_device_translate_handle(
1224                 pdd, GET_IDR_HANDLE(args->handle));
1225         if (!mem) {
1226                 ret = -EINVAL;
1227                 goto err_unlock;
1228         }
1229
1230         ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
1231                                 (struct kgd_mem *)mem, pdd->drm_priv, &size);
1232
1233         /* If freeing the buffer failed, leave the handle in place for
1234          * clean-up during process tear-down.
1235          */
1236         if (!ret)
1237                 kfd_process_device_remove_obj_handle(
1238                         pdd, GET_IDR_HANDLE(args->handle));
1239
1240         WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1241
1242 err_unlock:
1243 err_pdd:
1244         mutex_unlock(&p->mutex);
1245         return ret;
1246 }
1247
1248 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1249                                         struct kfd_process *p, void *data)
1250 {
1251         struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1252         struct kfd_process_device *pdd, *peer_pdd;
1253         void *mem;
1254         struct kfd_node *dev;
1255         long err = 0;
1256         int i;
1257         uint32_t *devices_arr = NULL;
1258
1259         if (!args->n_devices) {
1260                 pr_debug("Device IDs array empty\n");
1261                 return -EINVAL;
1262         }
1263         if (args->n_success > args->n_devices) {
1264                 pr_debug("n_success exceeds n_devices\n");
1265                 return -EINVAL;
1266         }
1267
1268         devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1269                                     GFP_KERNEL);
1270         if (!devices_arr)
1271                 return -ENOMEM;
1272
1273         err = copy_from_user(devices_arr,
1274                              (void __user *)args->device_ids_array_ptr,
1275                              args->n_devices * sizeof(*devices_arr));
1276         if (err != 0) {
1277                 err = -EFAULT;
1278                 goto copy_from_user_failed;
1279         }
1280
1281         mutex_lock(&p->mutex);
1282         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1283         if (!pdd) {
1284                 err = -EINVAL;
1285                 goto get_process_device_data_failed;
1286         }
1287         dev = pdd->dev;
1288
1289         pdd = kfd_bind_process_to_device(dev, p);
1290         if (IS_ERR(pdd)) {
1291                 err = PTR_ERR(pdd);
1292                 goto bind_process_to_device_failed;
1293         }
1294
1295         mem = kfd_process_device_translate_handle(pdd,
1296                                                 GET_IDR_HANDLE(args->handle));
1297         if (!mem) {
1298                 err = -ENOMEM;
1299                 goto get_mem_obj_from_handle_failed;
1300         }
1301
1302         for (i = args->n_success; i < args->n_devices; i++) {
1303                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1304                 if (!peer_pdd) {
1305                         pr_debug("Getting device by id failed for 0x%x\n",
1306                                  devices_arr[i]);
1307                         err = -EINVAL;
1308                         goto get_mem_obj_from_handle_failed;
1309                 }
1310
1311                 peer_pdd = kfd_bind_process_to_device(peer_pdd->dev, p);
1312                 if (IS_ERR(peer_pdd)) {
1313                         err = PTR_ERR(peer_pdd);
1314                         goto get_mem_obj_from_handle_failed;
1315                 }
1316
1317                 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1318                         peer_pdd->dev->adev, (struct kgd_mem *)mem,
1319                         peer_pdd->drm_priv);
1320                 if (err) {
1321                         struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1322
1323                         dev_err(dev->adev->dev,
1324                                "Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1325                                pci_domain_nr(pdev->bus),
1326                                pdev->bus->number,
1327                                PCI_SLOT(pdev->devfn),
1328                                PCI_FUNC(pdev->devfn),
1329                                ((struct kgd_mem *)mem)->domain);
1330                         goto map_memory_to_gpu_failed;
1331                 }
1332                 args->n_success = i+1;
1333         }
1334
1335         err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
1336         if (err) {
1337                 pr_debug("Sync memory failed, wait interrupted by user signal\n");
1338                 goto sync_memory_failed;
1339         }
1340
1341         mutex_unlock(&p->mutex);
1342
1343         /* Flush TLBs after waiting for the page table updates to complete */
1344         for (i = 0; i < args->n_devices; i++) {
1345                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1346                 if (WARN_ON_ONCE(!peer_pdd))
1347                         continue;
1348                 kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1349         }
1350         kfree(devices_arr);
1351
1352         return err;
1353
1354 get_process_device_data_failed:
1355 bind_process_to_device_failed:
1356 get_mem_obj_from_handle_failed:
1357 map_memory_to_gpu_failed:
1358 sync_memory_failed:
1359         mutex_unlock(&p->mutex);
1360 copy_from_user_failed:
1361         kfree(devices_arr);
1362
1363         return err;
1364 }
1365
1366 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1367                                         struct kfd_process *p, void *data)
1368 {
1369         struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1370         struct kfd_process_device *pdd, *peer_pdd;
1371         void *mem;
1372         long err = 0;
1373         uint32_t *devices_arr = NULL, i;
1374         bool flush_tlb;
1375
1376         if (!args->n_devices) {
1377                 pr_debug("Device IDs array empty\n");
1378                 return -EINVAL;
1379         }
1380         if (args->n_success > args->n_devices) {
1381                 pr_debug("n_success exceeds n_devices\n");
1382                 return -EINVAL;
1383         }
1384
1385         devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1386                                     GFP_KERNEL);
1387         if (!devices_arr)
1388                 return -ENOMEM;
1389
1390         err = copy_from_user(devices_arr,
1391                              (void __user *)args->device_ids_array_ptr,
1392                              args->n_devices * sizeof(*devices_arr));
1393         if (err != 0) {
1394                 err = -EFAULT;
1395                 goto copy_from_user_failed;
1396         }
1397
1398         mutex_lock(&p->mutex);
1399         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1400         if (!pdd) {
1401                 err = -EINVAL;
1402                 goto bind_process_to_device_failed;
1403         }
1404
1405         mem = kfd_process_device_translate_handle(pdd,
1406                                                 GET_IDR_HANDLE(args->handle));
1407         if (!mem) {
1408                 err = -ENOMEM;
1409                 goto get_mem_obj_from_handle_failed;
1410         }
1411
1412         for (i = args->n_success; i < args->n_devices; i++) {
1413                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1414                 if (!peer_pdd) {
1415                         err = -EINVAL;
1416                         goto get_mem_obj_from_handle_failed;
1417                 }
1418                 err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1419                         peer_pdd->dev->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1420                 if (err) {
1421                         pr_err("Failed to unmap from gpu %d/%d\n",
1422                                i, args->n_devices);
1423                         goto unmap_memory_from_gpu_failed;
1424                 }
1425                 args->n_success = i+1;
1426         }
1427
1428         flush_tlb = kfd_flush_tlb_after_unmap(pdd->dev->kfd);
1429         if (flush_tlb) {
1430                 err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
1431                                 (struct kgd_mem *) mem, true);
1432                 if (err) {
1433                         pr_debug("Sync memory failed, wait interrupted by user signal\n");
1434                         goto sync_memory_failed;
1435                 }
1436         }
1437         mutex_unlock(&p->mutex);
1438
1439         if (flush_tlb) {
1440                 /* Flush TLBs after waiting for the page table updates to complete */
1441                 for (i = 0; i < args->n_devices; i++) {
1442                         peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1443                         if (WARN_ON_ONCE(!peer_pdd))
1444                                 continue;
1445                         kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1446                 }
1447         }
1448         kfree(devices_arr);
1449
1450         return 0;
1451
1452 bind_process_to_device_failed:
1453 get_mem_obj_from_handle_failed:
1454 unmap_memory_from_gpu_failed:
1455 sync_memory_failed:
1456         mutex_unlock(&p->mutex);
1457 copy_from_user_failed:
1458         kfree(devices_arr);
1459         return err;
1460 }
1461
1462 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1463                 struct kfd_process *p, void *data)
1464 {
1465         int retval;
1466         struct kfd_ioctl_alloc_queue_gws_args *args = data;
1467         struct queue *q;
1468         struct kfd_node *dev;
1469
1470         mutex_lock(&p->mutex);
1471         q = pqm_get_user_queue(&p->pqm, args->queue_id);
1472
1473         if (q) {
1474                 dev = q->device;
1475         } else {
1476                 retval = -EINVAL;
1477                 goto out_unlock;
1478         }
1479
1480         if (!dev->gws) {
1481                 retval = -ENODEV;
1482                 goto out_unlock;
1483         }
1484
1485         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1486                 retval = -ENODEV;
1487                 goto out_unlock;
1488         }
1489
1490         if (!kfd_dbg_has_gws_support(dev) && p->debug_trap_enabled) {
1491                 retval = -EBUSY;
1492                 goto out_unlock;
1493         }
1494
1495         retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1496         mutex_unlock(&p->mutex);
1497
1498         args->first_gws = 0;
1499         return retval;
1500
1501 out_unlock:
1502         mutex_unlock(&p->mutex);
1503         return retval;
1504 }
1505
1506 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1507                 struct kfd_process *p, void *data)
1508 {
1509         struct kfd_ioctl_get_dmabuf_info_args *args = data;
1510         struct kfd_node *dev = NULL;
1511         struct amdgpu_device *dmabuf_adev;
1512         void *metadata_buffer = NULL;
1513         uint32_t flags;
1514         int8_t xcp_id;
1515         unsigned int i;
1516         int r;
1517
1518         /* Find a KFD GPU device that supports the get_dmabuf_info query */
1519         for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1520                 if (dev)
1521                         break;
1522         if (!dev)
1523                 return -EINVAL;
1524
1525         if (args->metadata_ptr) {
1526                 metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1527                 if (!metadata_buffer)
1528                         return -ENOMEM;
1529         }
1530
1531         /* Get dmabuf info from KGD */
1532         r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
1533                                           &dmabuf_adev, &args->size,
1534                                           metadata_buffer, args->metadata_size,
1535                                           &args->metadata_size, &flags, &xcp_id);
1536         if (r)
1537                 goto exit;
1538
1539         if (xcp_id >= 0)
1540                 args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
1541         else
1542                 args->gpu_id = dmabuf_adev->kfd.dev->nodes[0]->id;
1543         args->flags = flags;
1544
1545         /* Copy metadata buffer to user mode */
1546         if (metadata_buffer) {
1547                 r = copy_to_user((void __user *)args->metadata_ptr,
1548                                  metadata_buffer, args->metadata_size);
1549                 if (r != 0)
1550                         r = -EFAULT;
1551         }
1552
1553 exit:
1554         kfree(metadata_buffer);
1555
1556         return r;
1557 }
1558
1559 static int kfd_ioctl_import_dmabuf(struct file *filep,
1560                                    struct kfd_process *p, void *data)
1561 {
1562         struct kfd_ioctl_import_dmabuf_args *args = data;
1563         struct kfd_process_device *pdd;
1564         struct dma_buf *dmabuf;
1565         int idr_handle;
1566         uint64_t size;
1567         void *mem;
1568         int r;
1569
1570         dmabuf = dma_buf_get(args->dmabuf_fd);
1571         if (IS_ERR(dmabuf))
1572                 return PTR_ERR(dmabuf);
1573
1574         mutex_lock(&p->mutex);
1575         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1576         if (!pdd) {
1577                 r = -EINVAL;
1578                 goto err_unlock;
1579         }
1580
1581         pdd = kfd_bind_process_to_device(pdd->dev, p);
1582         if (IS_ERR(pdd)) {
1583                 r = PTR_ERR(pdd);
1584                 goto err_unlock;
1585         }
1586
1587         r = amdgpu_amdkfd_gpuvm_import_dmabuf(pdd->dev->adev, dmabuf,
1588                                               args->va_addr, pdd->drm_priv,
1589                                               (struct kgd_mem **)&mem, &size,
1590                                               NULL);
1591         if (r)
1592                 goto err_unlock;
1593
1594         idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1595         if (idr_handle < 0) {
1596                 r = -EFAULT;
1597                 goto err_free;
1598         }
1599
1600         mutex_unlock(&p->mutex);
1601         dma_buf_put(dmabuf);
1602
1603         args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1604
1605         return 0;
1606
1607 err_free:
1608         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem,
1609                                                pdd->drm_priv, NULL);
1610 err_unlock:
1611         mutex_unlock(&p->mutex);
1612         dma_buf_put(dmabuf);
1613         return r;
1614 }
1615
1616 static int kfd_ioctl_export_dmabuf(struct file *filep,
1617                                    struct kfd_process *p, void *data)
1618 {
1619         struct kfd_ioctl_export_dmabuf_args *args = data;
1620         struct kfd_process_device *pdd;
1621         struct dma_buf *dmabuf;
1622         struct kfd_node *dev;
1623         void *mem;
1624         int ret = 0;
1625
1626         dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1627         if (!dev)
1628                 return -EINVAL;
1629
1630         mutex_lock(&p->mutex);
1631
1632         pdd = kfd_get_process_device_data(dev, p);
1633         if (!pdd) {
1634                 ret = -EINVAL;
1635                 goto err_unlock;
1636         }
1637
1638         mem = kfd_process_device_translate_handle(pdd,
1639                                                 GET_IDR_HANDLE(args->handle));
1640         if (!mem) {
1641                 ret = -EINVAL;
1642                 goto err_unlock;
1643         }
1644
1645         ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
1646         mutex_unlock(&p->mutex);
1647         if (ret)
1648                 goto err_out;
1649
1650         ret = dma_buf_fd(dmabuf, args->flags);
1651         if (ret < 0) {
1652                 dma_buf_put(dmabuf);
1653                 goto err_out;
1654         }
1655         /* dma_buf_fd assigns the reference count to the fd, no need to
1656          * put the reference here.
1657          */
1658         args->dmabuf_fd = ret;
1659
1660         return 0;
1661
1662 err_unlock:
1663         mutex_unlock(&p->mutex);
1664 err_out:
1665         return ret;
1666 }
1667
1668 /* Handle requests for watching SMI events */
1669 static int kfd_ioctl_smi_events(struct file *filep,
1670                                 struct kfd_process *p, void *data)
1671 {
1672         struct kfd_ioctl_smi_events_args *args = data;
1673         struct kfd_process_device *pdd;
1674
1675         mutex_lock(&p->mutex);
1676
1677         pdd = kfd_process_device_data_by_id(p, args->gpuid);
1678         mutex_unlock(&p->mutex);
1679         if (!pdd)
1680                 return -EINVAL;
1681
1682         return kfd_smi_event_open(pdd->dev, &args->anon_fd);
1683 }
1684
1685 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1686
1687 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1688                                     struct kfd_process *p, void *data)
1689 {
1690         struct kfd_ioctl_set_xnack_mode_args *args = data;
1691         int r = 0;
1692
1693         mutex_lock(&p->mutex);
1694         if (args->xnack_enabled >= 0) {
1695                 if (!list_empty(&p->pqm.queues)) {
1696                         pr_debug("Process has user queues running\n");
1697                         r = -EBUSY;
1698                         goto out_unlock;
1699                 }
1700
1701                 if (p->xnack_enabled == args->xnack_enabled)
1702                         goto out_unlock;
1703
1704                 if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
1705                         r = -EPERM;
1706                         goto out_unlock;
1707                 }
1708
1709                 r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
1710         } else {
1711                 args->xnack_enabled = p->xnack_enabled;
1712         }
1713
1714 out_unlock:
1715         mutex_unlock(&p->mutex);
1716
1717         return r;
1718 }
1719
1720 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1721 {
1722         struct kfd_ioctl_svm_args *args = data;
1723         int r = 0;
1724
1725         pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1726                  args->start_addr, args->size, args->op, args->nattr);
1727
1728         if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1729                 return -EINVAL;
1730         if (!args->start_addr || !args->size)
1731                 return -EINVAL;
1732
1733         r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1734                       args->attrs);
1735
1736         return r;
1737 }
1738 #else
1739 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1740                                     struct kfd_process *p, void *data)
1741 {
1742         return -EPERM;
1743 }
1744 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1745 {
1746         return -EPERM;
1747 }
1748 #endif
1749
1750 static int criu_checkpoint_process(struct kfd_process *p,
1751                              uint8_t __user *user_priv_data,
1752                              uint64_t *priv_offset)
1753 {
1754         struct kfd_criu_process_priv_data process_priv;
1755         int ret;
1756
1757         memset(&process_priv, 0, sizeof(process_priv));
1758
1759         process_priv.version = KFD_CRIU_PRIV_VERSION;
1760         /* For CR, we don't consider negative xnack mode which is used for
1761          * querying without changing it, here 0 simply means disabled and 1
1762          * means enabled so retry for finding a valid PTE.
1763          */
1764         process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1765
1766         ret = copy_to_user(user_priv_data + *priv_offset,
1767                                 &process_priv, sizeof(process_priv));
1768
1769         if (ret) {
1770                 pr_err("Failed to copy process information to user\n");
1771                 ret = -EFAULT;
1772         }
1773
1774         *priv_offset += sizeof(process_priv);
1775         return ret;
1776 }
1777
1778 static int criu_checkpoint_devices(struct kfd_process *p,
1779                              uint32_t num_devices,
1780                              uint8_t __user *user_addr,
1781                              uint8_t __user *user_priv_data,
1782                              uint64_t *priv_offset)
1783 {
1784         struct kfd_criu_device_priv_data *device_priv = NULL;
1785         struct kfd_criu_device_bucket *device_buckets = NULL;
1786         int ret = 0, i;
1787
1788         device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1789         if (!device_buckets) {
1790                 ret = -ENOMEM;
1791                 goto exit;
1792         }
1793
1794         device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1795         if (!device_priv) {
1796                 ret = -ENOMEM;
1797                 goto exit;
1798         }
1799
1800         for (i = 0; i < num_devices; i++) {
1801                 struct kfd_process_device *pdd = p->pdds[i];
1802
1803                 device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1804                 device_buckets[i].actual_gpu_id = pdd->dev->id;
1805
1806                 /*
1807                  * priv_data does not contain useful information for now and is reserved for
1808                  * future use, so we do not set its contents.
1809                  */
1810         }
1811
1812         ret = copy_to_user(user_addr, device_buckets, num_devices * sizeof(*device_buckets));
1813         if (ret) {
1814                 pr_err("Failed to copy device information to user\n");
1815                 ret = -EFAULT;
1816                 goto exit;
1817         }
1818
1819         ret = copy_to_user(user_priv_data + *priv_offset,
1820                            device_priv,
1821                            num_devices * sizeof(*device_priv));
1822         if (ret) {
1823                 pr_err("Failed to copy device information to user\n");
1824                 ret = -EFAULT;
1825         }
1826         *priv_offset += num_devices * sizeof(*device_priv);
1827
1828 exit:
1829         kvfree(device_buckets);
1830         kvfree(device_priv);
1831         return ret;
1832 }
1833
1834 static uint32_t get_process_num_bos(struct kfd_process *p)
1835 {
1836         uint32_t num_of_bos = 0;
1837         int i;
1838
1839         /* Run over all PDDs of the process */
1840         for (i = 0; i < p->n_pdds; i++) {
1841                 struct kfd_process_device *pdd = p->pdds[i];
1842                 void *mem;
1843                 int id;
1844
1845                 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1846                         struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1847
1848                         if ((uint64_t)kgd_mem->va > pdd->gpuvm_base)
1849                                 num_of_bos++;
1850                 }
1851         }
1852         return num_of_bos;
1853 }
1854
1855 static int criu_get_prime_handle(struct drm_gem_object *gobj, int flags,
1856                                       u32 *shared_fd)
1857 {
1858         struct dma_buf *dmabuf;
1859         int ret;
1860
1861         dmabuf = amdgpu_gem_prime_export(gobj, flags);
1862         if (IS_ERR(dmabuf)) {
1863                 ret = PTR_ERR(dmabuf);
1864                 pr_err("dmabuf export failed for the BO\n");
1865                 return ret;
1866         }
1867
1868         ret = dma_buf_fd(dmabuf, flags);
1869         if (ret < 0) {
1870                 pr_err("dmabuf create fd failed, ret:%d\n", ret);
1871                 goto out_free_dmabuf;
1872         }
1873
1874         *shared_fd = ret;
1875         return 0;
1876
1877 out_free_dmabuf:
1878         dma_buf_put(dmabuf);
1879         return ret;
1880 }
1881
1882 static int criu_checkpoint_bos(struct kfd_process *p,
1883                                uint32_t num_bos,
1884                                uint8_t __user *user_bos,
1885                                uint8_t __user *user_priv_data,
1886                                uint64_t *priv_offset)
1887 {
1888         struct kfd_criu_bo_bucket *bo_buckets;
1889         struct kfd_criu_bo_priv_data *bo_privs;
1890         int ret = 0, pdd_index, bo_index = 0, id;
1891         void *mem;
1892
1893         bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1894         if (!bo_buckets)
1895                 return -ENOMEM;
1896
1897         bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1898         if (!bo_privs) {
1899                 ret = -ENOMEM;
1900                 goto exit;
1901         }
1902
1903         for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1904                 struct kfd_process_device *pdd = p->pdds[pdd_index];
1905                 struct amdgpu_bo *dumper_bo;
1906                 struct kgd_mem *kgd_mem;
1907
1908                 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1909                         struct kfd_criu_bo_bucket *bo_bucket;
1910                         struct kfd_criu_bo_priv_data *bo_priv;
1911                         int i, dev_idx = 0;
1912
1913                         if (!mem) {
1914                                 ret = -ENOMEM;
1915                                 goto exit;
1916                         }
1917
1918                         kgd_mem = (struct kgd_mem *)mem;
1919                         dumper_bo = kgd_mem->bo;
1920
1921                         if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
1922                                 continue;
1923
1924                         bo_bucket = &bo_buckets[bo_index];
1925                         bo_priv = &bo_privs[bo_index];
1926
1927                         bo_bucket->gpu_id = pdd->user_gpu_id;
1928                         bo_bucket->addr = (uint64_t)kgd_mem->va;
1929                         bo_bucket->size = amdgpu_bo_size(dumper_bo);
1930                         bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1931                         bo_priv->idr_handle = id;
1932
1933                         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1934                                 ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
1935                                                                 &bo_priv->user_addr);
1936                                 if (ret) {
1937                                         pr_err("Failed to obtain user address for user-pointer bo\n");
1938                                         goto exit;
1939                                 }
1940                         }
1941                         if (bo_bucket->alloc_flags
1942                             & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1943                                 ret = criu_get_prime_handle(&dumper_bo->tbo.base,
1944                                                 bo_bucket->alloc_flags &
1945                                                 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1946                                                 &bo_bucket->dmabuf_fd);
1947                                 if (ret)
1948                                         goto exit;
1949                         } else {
1950                                 bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1951                         }
1952
1953                         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1954                                 bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1955                                         KFD_MMAP_GPU_ID(pdd->dev->id);
1956                         else if (bo_bucket->alloc_flags &
1957                                 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1958                                 bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1959                                         KFD_MMAP_GPU_ID(pdd->dev->id);
1960                         else
1961                                 bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
1962
1963                         for (i = 0; i < p->n_pdds; i++) {
1964                                 if (amdgpu_amdkfd_bo_mapped_to_dev(p->pdds[i]->dev->adev, kgd_mem))
1965                                         bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1966                         }
1967
1968                         pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1969                                         "gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1970                                         bo_bucket->size,
1971                                         bo_bucket->addr,
1972                                         bo_bucket->offset,
1973                                         bo_bucket->gpu_id,
1974                                         bo_bucket->alloc_flags,
1975                                         bo_priv->idr_handle);
1976                         bo_index++;
1977                 }
1978         }
1979
1980         ret = copy_to_user(user_bos, bo_buckets, num_bos * sizeof(*bo_buckets));
1981         if (ret) {
1982                 pr_err("Failed to copy BO information to user\n");
1983                 ret = -EFAULT;
1984                 goto exit;
1985         }
1986
1987         ret = copy_to_user(user_priv_data + *priv_offset, bo_privs, num_bos * sizeof(*bo_privs));
1988         if (ret) {
1989                 pr_err("Failed to copy BO priv information to user\n");
1990                 ret = -EFAULT;
1991                 goto exit;
1992         }
1993
1994         *priv_offset += num_bos * sizeof(*bo_privs);
1995
1996 exit:
1997         while (ret && bo_index--) {
1998                 if (bo_buckets[bo_index].alloc_flags
1999                     & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2000                         close_fd(bo_buckets[bo_index].dmabuf_fd);
2001         }
2002
2003         kvfree(bo_buckets);
2004         kvfree(bo_privs);
2005         return ret;
2006 }
2007
2008 static int criu_get_process_object_info(struct kfd_process *p,
2009                                         uint32_t *num_devices,
2010                                         uint32_t *num_bos,
2011                                         uint32_t *num_objects,
2012                                         uint64_t *objs_priv_size)
2013 {
2014         uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
2015         uint32_t num_queues, num_events, num_svm_ranges;
2016         int ret;
2017
2018         *num_devices = p->n_pdds;
2019         *num_bos = get_process_num_bos(p);
2020
2021         ret = kfd_process_get_queue_info(p, &num_queues, &queues_priv_data_size);
2022         if (ret)
2023                 return ret;
2024
2025         num_events = kfd_get_num_events(p);
2026
2027         ret = svm_range_get_info(p, &num_svm_ranges, &svm_priv_data_size);
2028         if (ret)
2029                 return ret;
2030
2031         *num_objects = num_queues + num_events + num_svm_ranges;
2032
2033         if (objs_priv_size) {
2034                 priv_size = sizeof(struct kfd_criu_process_priv_data);
2035                 priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
2036                 priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
2037                 priv_size += queues_priv_data_size;
2038                 priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
2039                 priv_size += svm_priv_data_size;
2040                 *objs_priv_size = priv_size;
2041         }
2042         return 0;
2043 }
2044
2045 static int criu_checkpoint(struct file *filep,
2046                            struct kfd_process *p,
2047                            struct kfd_ioctl_criu_args *args)
2048 {
2049         int ret;
2050         uint32_t num_devices, num_bos, num_objects;
2051         uint64_t priv_size, priv_offset = 0, bo_priv_offset;
2052
2053         if (!args->devices || !args->bos || !args->priv_data)
2054                 return -EINVAL;
2055
2056         mutex_lock(&p->mutex);
2057
2058         if (!p->n_pdds) {
2059                 pr_err("No pdd for given process\n");
2060                 ret = -ENODEV;
2061                 goto exit_unlock;
2062         }
2063
2064         /* Confirm all process queues are evicted */
2065         if (!p->queues_paused) {
2066                 pr_err("Cannot dump process when queues are not in evicted state\n");
2067                 /* CRIU plugin did not call op PROCESS_INFO before checkpointing */
2068                 ret = -EINVAL;
2069                 goto exit_unlock;
2070         }
2071
2072         ret = criu_get_process_object_info(p, &num_devices, &num_bos, &num_objects, &priv_size);
2073         if (ret)
2074                 goto exit_unlock;
2075
2076         if (num_devices != args->num_devices ||
2077             num_bos != args->num_bos ||
2078             num_objects != args->num_objects ||
2079             priv_size != args->priv_data_size) {
2080
2081                 ret = -EINVAL;
2082                 goto exit_unlock;
2083         }
2084
2085         /* each function will store private data inside priv_data and adjust priv_offset */
2086         ret = criu_checkpoint_process(p, (uint8_t __user *)args->priv_data, &priv_offset);
2087         if (ret)
2088                 goto exit_unlock;
2089
2090         ret = criu_checkpoint_devices(p, num_devices, (uint8_t __user *)args->devices,
2091                                 (uint8_t __user *)args->priv_data, &priv_offset);
2092         if (ret)
2093                 goto exit_unlock;
2094
2095         /* Leave room for BOs in the private data. They need to be restored
2096          * before events, but we checkpoint them last to simplify the error
2097          * handling.
2098          */
2099         bo_priv_offset = priv_offset;
2100         priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2101
2102         if (num_objects) {
2103                 ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data,
2104                                                  &priv_offset);
2105                 if (ret)
2106                         goto exit_unlock;
2107
2108                 ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
2109                                                  &priv_offset);
2110                 if (ret)
2111                         goto exit_unlock;
2112
2113                 ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset);
2114                 if (ret)
2115                         goto exit_unlock;
2116         }
2117
2118         /* This must be the last thing in this function that can fail.
2119          * Otherwise we leak dmabuf file descriptors.
2120          */
2121         ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos,
2122                            (uint8_t __user *)args->priv_data, &bo_priv_offset);
2123
2124 exit_unlock:
2125         mutex_unlock(&p->mutex);
2126         if (ret)
2127                 pr_err("Failed to dump CRIU ret:%d\n", ret);
2128         else
2129                 pr_debug("CRIU dump ret:%d\n", ret);
2130
2131         return ret;
2132 }
2133
2134 static int criu_restore_process(struct kfd_process *p,
2135                                 struct kfd_ioctl_criu_args *args,
2136                                 uint64_t *priv_offset,
2137                                 uint64_t max_priv_data_size)
2138 {
2139         int ret = 0;
2140         struct kfd_criu_process_priv_data process_priv;
2141
2142         if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2143                 return -EINVAL;
2144
2145         ret = copy_from_user(&process_priv,
2146                                 (void __user *)(args->priv_data + *priv_offset),
2147                                 sizeof(process_priv));
2148         if (ret) {
2149                 pr_err("Failed to copy process private information from user\n");
2150                 ret = -EFAULT;
2151                 goto exit;
2152         }
2153         *priv_offset += sizeof(process_priv);
2154
2155         if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2156                 pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2157                         process_priv.version, KFD_CRIU_PRIV_VERSION);
2158                 return -EINVAL;
2159         }
2160
2161         pr_debug("Setting XNACK mode\n");
2162         if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, true)) {
2163                 pr_err("xnack mode cannot be set\n");
2164                 ret = -EPERM;
2165                 goto exit;
2166         } else {
2167                 pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2168                 p->xnack_enabled = process_priv.xnack_mode;
2169         }
2170
2171 exit:
2172         return ret;
2173 }
2174
2175 static int criu_restore_devices(struct kfd_process *p,
2176                                 struct kfd_ioctl_criu_args *args,
2177                                 uint64_t *priv_offset,
2178                                 uint64_t max_priv_data_size)
2179 {
2180         struct kfd_criu_device_bucket *device_buckets;
2181         struct kfd_criu_device_priv_data *device_privs;
2182         int ret = 0;
2183         uint32_t i;
2184
2185         if (args->num_devices != p->n_pdds)
2186                 return -EINVAL;
2187
2188         if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2189                 return -EINVAL;
2190
2191         device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2192         if (!device_buckets)
2193                 return -ENOMEM;
2194
2195         ret = copy_from_user(device_buckets, (void __user *)args->devices,
2196                                 args->num_devices * sizeof(*device_buckets));
2197         if (ret) {
2198                 pr_err("Failed to copy devices buckets from user\n");
2199                 ret = -EFAULT;
2200                 goto exit;
2201         }
2202
2203         for (i = 0; i < args->num_devices; i++) {
2204                 struct kfd_node *dev;
2205                 struct kfd_process_device *pdd;
2206                 struct file *drm_file;
2207
2208                 /* device private data is not currently used */
2209
2210                 if (!device_buckets[i].user_gpu_id) {
2211                         pr_err("Invalid user gpu_id\n");
2212                         ret = -EINVAL;
2213                         goto exit;
2214                 }
2215
2216                 dev = kfd_device_by_id(device_buckets[i].actual_gpu_id);
2217                 if (!dev) {
2218                         pr_err("Failed to find device with gpu_id = %x\n",
2219                                 device_buckets[i].actual_gpu_id);
2220                         ret = -EINVAL;
2221                         goto exit;
2222                 }
2223
2224                 pdd = kfd_get_process_device_data(dev, p);
2225                 if (!pdd) {
2226                         pr_err("Failed to get pdd for gpu_id = %x\n",
2227                                         device_buckets[i].actual_gpu_id);
2228                         ret = -EINVAL;
2229                         goto exit;
2230                 }
2231                 pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2232
2233                 drm_file = fget(device_buckets[i].drm_fd);
2234                 if (!drm_file) {
2235                         pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2236                                 device_buckets[i].drm_fd);
2237                         ret = -EINVAL;
2238                         goto exit;
2239                 }
2240
2241                 if (pdd->drm_file) {
2242                         ret = -EINVAL;
2243                         goto exit;
2244                 }
2245
2246                 /* create the vm using render nodes for kfd pdd */
2247                 if (kfd_process_device_init_vm(pdd, drm_file)) {
2248                         pr_err("could not init vm for given pdd\n");
2249                         /* On success, the PDD keeps the drm_file reference */
2250                         fput(drm_file);
2251                         ret = -EINVAL;
2252                         goto exit;
2253                 }
2254                 /*
2255                  * pdd now already has the vm bound to render node so below api won't create a new
2256                  * exclusive kfd mapping but use existing one with renderDXXX but is still needed
2257                  * for iommu v2 binding  and runtime pm.
2258                  */
2259                 pdd = kfd_bind_process_to_device(dev, p);
2260                 if (IS_ERR(pdd)) {
2261                         ret = PTR_ERR(pdd);
2262                         goto exit;
2263                 }
2264
2265                 if (!pdd->doorbell_index &&
2266                     kfd_alloc_process_doorbells(pdd->dev->kfd, &pdd->doorbell_index) < 0) {
2267                         ret = -ENOMEM;
2268                         goto exit;
2269                 }
2270         }
2271
2272         /*
2273          * We are not copying device private data from user as we are not using the data for now,
2274          * but we still adjust for its private data.
2275          */
2276         *priv_offset += args->num_devices * sizeof(*device_privs);
2277
2278 exit:
2279         kfree(device_buckets);
2280         return ret;
2281 }
2282
2283 static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2284                                       struct kfd_criu_bo_bucket *bo_bucket,
2285                                       struct kfd_criu_bo_priv_data *bo_priv,
2286                                       struct kgd_mem **kgd_mem)
2287 {
2288         int idr_handle;
2289         int ret;
2290         const bool criu_resume = true;
2291         u64 offset;
2292
2293         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2294                 if (bo_bucket->size !=
2295                                 kfd_doorbell_process_slice(pdd->dev->kfd))
2296                         return -EINVAL;
2297
2298                 offset = kfd_get_process_doorbells(pdd);
2299                 if (!offset)
2300                         return -ENOMEM;
2301         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2302                 /* MMIO BOs need remapped bus address */
2303                 if (bo_bucket->size != PAGE_SIZE) {
2304                         pr_err("Invalid page size\n");
2305                         return -EINVAL;
2306                 }
2307                 offset = pdd->dev->adev->rmmio_remap.bus_addr;
2308                 if (!offset) {
2309                         pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2310                         return -ENOMEM;
2311                 }
2312         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2313                 offset = bo_priv->user_addr;
2314         }
2315         /* Create the BO */
2316         ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(pdd->dev->adev, bo_bucket->addr,
2317                                                       bo_bucket->size, pdd->drm_priv, kgd_mem,
2318                                                       &offset, bo_bucket->alloc_flags, criu_resume);
2319         if (ret) {
2320                 pr_err("Could not create the BO\n");
2321                 return ret;
2322         }
2323         pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2324                  bo_bucket->size, bo_bucket->addr, offset);
2325
2326         /* Restore previous IDR handle */
2327         pr_debug("Restoring old IDR handle for the BO");
2328         idr_handle = idr_alloc(&pdd->alloc_idr, *kgd_mem, bo_priv->idr_handle,
2329                                bo_priv->idr_handle + 1, GFP_KERNEL);
2330
2331         if (idr_handle < 0) {
2332                 pr_err("Could not allocate idr\n");
2333                 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, *kgd_mem, pdd->drm_priv,
2334                                                        NULL);
2335                 return -ENOMEM;
2336         }
2337
2338         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2339                 bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2340         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2341                 bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2342         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2343                 bo_bucket->restored_offset = offset;
2344         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2345                 bo_bucket->restored_offset = offset;
2346                 /* Update the VRAM usage count */
2347                 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + bo_bucket->size);
2348         }
2349         return 0;
2350 }
2351
2352 static int criu_restore_bo(struct kfd_process *p,
2353                            struct kfd_criu_bo_bucket *bo_bucket,
2354                            struct kfd_criu_bo_priv_data *bo_priv)
2355 {
2356         struct kfd_process_device *pdd;
2357         struct kgd_mem *kgd_mem;
2358         int ret;
2359         int j;
2360
2361         pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2362                  bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2363                  bo_priv->idr_handle);
2364
2365         pdd = kfd_process_device_data_by_id(p, bo_bucket->gpu_id);
2366         if (!pdd) {
2367                 pr_err("Failed to get pdd\n");
2368                 return -ENODEV;
2369         }
2370
2371         ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, &kgd_mem);
2372         if (ret)
2373                 return ret;
2374
2375         /* now map these BOs to GPU/s */
2376         for (j = 0; j < p->n_pdds; j++) {
2377                 struct kfd_node *peer;
2378                 struct kfd_process_device *peer_pdd;
2379
2380                 if (!bo_priv->mapped_gpuids[j])
2381                         break;
2382
2383                 peer_pdd = kfd_process_device_data_by_id(p, bo_priv->mapped_gpuids[j]);
2384                 if (!peer_pdd)
2385                         return -EINVAL;
2386
2387                 peer = peer_pdd->dev;
2388
2389                 peer_pdd = kfd_bind_process_to_device(peer, p);
2390                 if (IS_ERR(peer_pdd))
2391                         return PTR_ERR(peer_pdd);
2392
2393                 ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
2394                                                             peer_pdd->drm_priv);
2395                 if (ret) {
2396                         pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2397                         return ret;
2398                 }
2399         }
2400
2401         pr_debug("map memory was successful for the BO\n");
2402         /* create the dmabuf object and export the bo */
2403         if (bo_bucket->alloc_flags
2404             & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2405                 ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
2406                                             &bo_bucket->dmabuf_fd);
2407                 if (ret)
2408                         return ret;
2409         } else {
2410                 bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2411         }
2412
2413         return 0;
2414 }
2415
2416 static int criu_restore_bos(struct kfd_process *p,
2417                             struct kfd_ioctl_criu_args *args,
2418                             uint64_t *priv_offset,
2419                             uint64_t max_priv_data_size)
2420 {
2421         struct kfd_criu_bo_bucket *bo_buckets = NULL;
2422         struct kfd_criu_bo_priv_data *bo_privs = NULL;
2423         int ret = 0;
2424         uint32_t i = 0;
2425
2426         if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2427                 return -EINVAL;
2428
2429         /* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2430         amdgpu_amdkfd_block_mmu_notifications(p->kgd_process_info);
2431
2432         bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2433         if (!bo_buckets)
2434                 return -ENOMEM;
2435
2436         ret = copy_from_user(bo_buckets, (void __user *)args->bos,
2437                              args->num_bos * sizeof(*bo_buckets));
2438         if (ret) {
2439                 pr_err("Failed to copy BOs information from user\n");
2440                 ret = -EFAULT;
2441                 goto exit;
2442         }
2443
2444         bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2445         if (!bo_privs) {
2446                 ret = -ENOMEM;
2447                 goto exit;
2448         }
2449
2450         ret = copy_from_user(bo_privs, (void __user *)args->priv_data + *priv_offset,
2451                              args->num_bos * sizeof(*bo_privs));
2452         if (ret) {
2453                 pr_err("Failed to copy BOs information from user\n");
2454                 ret = -EFAULT;
2455                 goto exit;
2456         }
2457         *priv_offset += args->num_bos * sizeof(*bo_privs);
2458
2459         /* Create and map new BOs */
2460         for (; i < args->num_bos; i++) {
2461                 ret = criu_restore_bo(p, &bo_buckets[i], &bo_privs[i]);
2462                 if (ret) {
2463                         pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2464                         goto exit;
2465                 }
2466         } /* done */
2467
2468         /* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2469         ret = copy_to_user((void __user *)args->bos,
2470                                 bo_buckets,
2471                                 (args->num_bos * sizeof(*bo_buckets)));
2472         if (ret)
2473                 ret = -EFAULT;
2474
2475 exit:
2476         while (ret && i--) {
2477                 if (bo_buckets[i].alloc_flags
2478                    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2479                         close_fd(bo_buckets[i].dmabuf_fd);
2480         }
2481         kvfree(bo_buckets);
2482         kvfree(bo_privs);
2483         return ret;
2484 }
2485
2486 static int criu_restore_objects(struct file *filep,
2487                                 struct kfd_process *p,
2488                                 struct kfd_ioctl_criu_args *args,
2489                                 uint64_t *priv_offset,
2490                                 uint64_t max_priv_data_size)
2491 {
2492         int ret = 0;
2493         uint32_t i;
2494
2495         BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2496         BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2497         BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2498
2499         for (i = 0; i < args->num_objects; i++) {
2500                 uint32_t object_type;
2501
2502                 if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2503                         pr_err("Invalid private data size\n");
2504                         return -EINVAL;
2505                 }
2506
2507                 ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2508                 if (ret) {
2509                         pr_err("Failed to copy private information from user\n");
2510                         goto exit;
2511                 }
2512
2513                 switch (object_type) {
2514                 case KFD_CRIU_OBJECT_TYPE_QUEUE:
2515                         ret = kfd_criu_restore_queue(p, (uint8_t __user *)args->priv_data,
2516                                                      priv_offset, max_priv_data_size);
2517                         if (ret)
2518                                 goto exit;
2519                         break;
2520                 case KFD_CRIU_OBJECT_TYPE_EVENT:
2521                         ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
2522                                                      priv_offset, max_priv_data_size);
2523                         if (ret)
2524                                 goto exit;
2525                         break;
2526                 case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2527                         ret = kfd_criu_restore_svm(p, (uint8_t __user *)args->priv_data,
2528                                                      priv_offset, max_priv_data_size);
2529                         if (ret)
2530                                 goto exit;
2531                         break;
2532                 default:
2533                         pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2534                         ret = -EINVAL;
2535                         goto exit;
2536                 }
2537         }
2538 exit:
2539         return ret;
2540 }
2541
2542 static int criu_restore(struct file *filep,
2543                         struct kfd_process *p,
2544                         struct kfd_ioctl_criu_args *args)
2545 {
2546         uint64_t priv_offset = 0;
2547         int ret = 0;
2548
2549         pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2550                  args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2551
2552         if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size ||
2553             !args->num_devices || !args->num_bos)
2554                 return -EINVAL;
2555
2556         mutex_lock(&p->mutex);
2557
2558         /*
2559          * Set the process to evicted state to avoid running any new queues before all the memory
2560          * mappings are ready.
2561          */
2562         ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_RESTORE);
2563         if (ret)
2564                 goto exit_unlock;
2565
2566         /* Each function will adjust priv_offset based on how many bytes they consumed */
2567         ret = criu_restore_process(p, args, &priv_offset, args->priv_data_size);
2568         if (ret)
2569                 goto exit_unlock;
2570
2571         ret = criu_restore_devices(p, args, &priv_offset, args->priv_data_size);
2572         if (ret)
2573                 goto exit_unlock;
2574
2575         ret = criu_restore_bos(p, args, &priv_offset, args->priv_data_size);
2576         if (ret)
2577                 goto exit_unlock;
2578
2579         ret = criu_restore_objects(filep, p, args, &priv_offset, args->priv_data_size);
2580         if (ret)
2581                 goto exit_unlock;
2582
2583         if (priv_offset != args->priv_data_size) {
2584                 pr_err("Invalid private data size\n");
2585                 ret = -EINVAL;
2586         }
2587
2588 exit_unlock:
2589         mutex_unlock(&p->mutex);
2590         if (ret)
2591                 pr_err("Failed to restore CRIU ret:%d\n", ret);
2592         else
2593                 pr_debug("CRIU restore successful\n");
2594
2595         return ret;
2596 }
2597
2598 static int criu_unpause(struct file *filep,
2599                         struct kfd_process *p,
2600                         struct kfd_ioctl_criu_args *args)
2601 {
2602         int ret;
2603
2604         mutex_lock(&p->mutex);
2605
2606         if (!p->queues_paused) {
2607                 mutex_unlock(&p->mutex);
2608                 return -EINVAL;
2609         }
2610
2611         ret = kfd_process_restore_queues(p);
2612         if (ret)
2613                 pr_err("Failed to unpause queues ret:%d\n", ret);
2614         else
2615                 p->queues_paused = false;
2616
2617         mutex_unlock(&p->mutex);
2618
2619         return ret;
2620 }
2621
2622 static int criu_resume(struct file *filep,
2623                         struct kfd_process *p,
2624                         struct kfd_ioctl_criu_args *args)
2625 {
2626         struct kfd_process *target = NULL;
2627         struct pid *pid = NULL;
2628         int ret = 0;
2629
2630         pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2631                  args->pid);
2632
2633         pid = find_get_pid(args->pid);
2634         if (!pid) {
2635                 pr_err("Cannot find pid info for %i\n", args->pid);
2636                 return -ESRCH;
2637         }
2638
2639         pr_debug("calling kfd_lookup_process_by_pid\n");
2640         target = kfd_lookup_process_by_pid(pid);
2641
2642         put_pid(pid);
2643
2644         if (!target) {
2645                 pr_debug("Cannot find process info for %i\n", args->pid);
2646                 return -ESRCH;
2647         }
2648
2649         mutex_lock(&target->mutex);
2650         ret = kfd_criu_resume_svm(target);
2651         if (ret) {
2652                 pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2653                 goto exit;
2654         }
2655
2656         ret =  amdgpu_amdkfd_criu_resume(target->kgd_process_info);
2657         if (ret)
2658                 pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2659
2660 exit:
2661         mutex_unlock(&target->mutex);
2662
2663         kfd_unref_process(target);
2664         return ret;
2665 }
2666
2667 static int criu_process_info(struct file *filep,
2668                                 struct kfd_process *p,
2669                                 struct kfd_ioctl_criu_args *args)
2670 {
2671         int ret = 0;
2672
2673         mutex_lock(&p->mutex);
2674
2675         if (!p->n_pdds) {
2676                 pr_err("No pdd for given process\n");
2677                 ret = -ENODEV;
2678                 goto err_unlock;
2679         }
2680
2681         ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2682         if (ret)
2683                 goto err_unlock;
2684
2685         p->queues_paused = true;
2686
2687         args->pid = task_pid_nr_ns(p->lead_thread,
2688                                         task_active_pid_ns(p->lead_thread));
2689
2690         ret = criu_get_process_object_info(p, &args->num_devices, &args->num_bos,
2691                                            &args->num_objects, &args->priv_data_size);
2692         if (ret)
2693                 goto err_unlock;
2694
2695         dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2696                                 args->num_devices, args->num_bos, args->num_objects,
2697                                 args->priv_data_size);
2698
2699 err_unlock:
2700         if (ret) {
2701                 kfd_process_restore_queues(p);
2702                 p->queues_paused = false;
2703         }
2704         mutex_unlock(&p->mutex);
2705         return ret;
2706 }
2707
2708 static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2709 {
2710         struct kfd_ioctl_criu_args *args = data;
2711         int ret;
2712
2713         dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2714         switch (args->op) {
2715         case KFD_CRIU_OP_PROCESS_INFO:
2716                 ret = criu_process_info(filep, p, args);
2717                 break;
2718         case KFD_CRIU_OP_CHECKPOINT:
2719                 ret = criu_checkpoint(filep, p, args);
2720                 break;
2721         case KFD_CRIU_OP_UNPAUSE:
2722                 ret = criu_unpause(filep, p, args);
2723                 break;
2724         case KFD_CRIU_OP_RESTORE:
2725                 ret = criu_restore(filep, p, args);
2726                 break;
2727         case KFD_CRIU_OP_RESUME:
2728                 ret = criu_resume(filep, p, args);
2729                 break;
2730         default:
2731                 dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2732                 ret = -EINVAL;
2733                 break;
2734         }
2735
2736         if (ret)
2737                 dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2738
2739         return ret;
2740 }
2741
2742 static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
2743                         bool enable_ttmp_setup)
2744 {
2745         int i = 0, ret = 0;
2746
2747         if (p->is_runtime_retry)
2748                 goto retry;
2749
2750         if (p->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
2751                 return -EBUSY;
2752
2753         for (i = 0; i < p->n_pdds; i++) {
2754                 struct kfd_process_device *pdd = p->pdds[i];
2755
2756                 if (pdd->qpd.queue_count)
2757                         return -EEXIST;
2758         }
2759
2760         p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
2761         p->runtime_info.r_debug = r_debug;
2762         p->runtime_info.ttmp_setup = enable_ttmp_setup;
2763
2764         if (p->runtime_info.ttmp_setup) {
2765                 for (i = 0; i < p->n_pdds; i++) {
2766                         struct kfd_process_device *pdd = p->pdds[i];
2767
2768                         if (!kfd_dbg_is_rlc_restore_supported(pdd->dev)) {
2769                                 amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
2770                                 pdd->dev->kfd2kgd->enable_debug_trap(
2771                                                 pdd->dev->adev,
2772                                                 true,
2773                                                 pdd->dev->vm_info.last_vmid_kfd);
2774                         } else if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2775                                 pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
2776                                                 pdd->dev->adev,
2777                                                 false,
2778                                                 0);
2779                         }
2780                 }
2781         }
2782
2783 retry:
2784         if (p->debug_trap_enabled) {
2785                 if (!p->is_runtime_retry) {
2786                         kfd_dbg_trap_activate(p);
2787                         kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2788                                         p, NULL, 0, false, NULL, 0);
2789                 }
2790
2791                 mutex_unlock(&p->mutex);
2792                 ret = down_interruptible(&p->runtime_enable_sema);
2793                 mutex_lock(&p->mutex);
2794
2795                 p->is_runtime_retry = !!ret;
2796         }
2797
2798         return ret;
2799 }
2800
2801 static int runtime_disable(struct kfd_process *p)
2802 {
2803         int i = 0, ret;
2804         bool was_enabled = p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED;
2805
2806         p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_DISABLED;
2807         p->runtime_info.r_debug = 0;
2808
2809         if (p->debug_trap_enabled) {
2810                 if (was_enabled)
2811                         kfd_dbg_trap_deactivate(p, false, 0);
2812
2813                 if (!p->is_runtime_retry)
2814                         kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2815                                         p, NULL, 0, false, NULL, 0);
2816
2817                 mutex_unlock(&p->mutex);
2818                 ret = down_interruptible(&p->runtime_enable_sema);
2819                 mutex_lock(&p->mutex);
2820
2821                 p->is_runtime_retry = !!ret;
2822                 if (ret)
2823                         return ret;
2824         }
2825
2826         if (was_enabled && p->runtime_info.ttmp_setup) {
2827                 for (i = 0; i < p->n_pdds; i++) {
2828                         struct kfd_process_device *pdd = p->pdds[i];
2829
2830                         if (!kfd_dbg_is_rlc_restore_supported(pdd->dev))
2831                                 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
2832                 }
2833         }
2834
2835         p->runtime_info.ttmp_setup = false;
2836
2837         /* disable ttmp setup */
2838         for (i = 0; i < p->n_pdds; i++) {
2839                 struct kfd_process_device *pdd = p->pdds[i];
2840
2841                 if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2842                         pdd->spi_dbg_override =
2843                                         pdd->dev->kfd2kgd->disable_debug_trap(
2844                                         pdd->dev->adev,
2845                                         false,
2846                                         pdd->dev->vm_info.last_vmid_kfd);
2847
2848                         if (!pdd->dev->kfd->shared_resources.enable_mes)
2849                                 debug_refresh_runlist(pdd->dev->dqm);
2850                         else
2851                                 kfd_dbg_set_mes_debug_mode(pdd);
2852                 }
2853         }
2854
2855         return 0;
2856 }
2857
2858 static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
2859 {
2860         struct kfd_ioctl_runtime_enable_args *args = data;
2861         int r;
2862
2863         mutex_lock(&p->mutex);
2864
2865         if (args->mode_mask & KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK)
2866                 r = runtime_enable(p, args->r_debug,
2867                                 !!(args->mode_mask & KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK));
2868         else
2869                 r = runtime_disable(p);
2870
2871         mutex_unlock(&p->mutex);
2872
2873         return r;
2874 }
2875
2876 static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
2877 {
2878         struct kfd_ioctl_dbg_trap_args *args = data;
2879         struct task_struct *thread = NULL;
2880         struct mm_struct *mm = NULL;
2881         struct pid *pid = NULL;
2882         struct kfd_process *target = NULL;
2883         struct kfd_process_device *pdd = NULL;
2884         int r = 0;
2885
2886         if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2887                 pr_err("Debugging does not support sched_policy %i", sched_policy);
2888                 return -EINVAL;
2889         }
2890
2891         pid = find_get_pid(args->pid);
2892         if (!pid) {
2893                 pr_debug("Cannot find pid info for %i\n", args->pid);
2894                 r = -ESRCH;
2895                 goto out;
2896         }
2897
2898         thread = get_pid_task(pid, PIDTYPE_PID);
2899         if (!thread) {
2900                 r = -ESRCH;
2901                 goto out;
2902         }
2903
2904         mm = get_task_mm(thread);
2905         if (!mm) {
2906                 r = -ESRCH;
2907                 goto out;
2908         }
2909
2910         if (args->op == KFD_IOC_DBG_TRAP_ENABLE) {
2911                 bool create_process;
2912
2913                 rcu_read_lock();
2914                 create_process = thread && thread != current && ptrace_parent(thread) == current;
2915                 rcu_read_unlock();
2916
2917                 target = create_process ? kfd_create_process(thread) :
2918                                         kfd_lookup_process_by_pid(pid);
2919         } else {
2920                 target = kfd_lookup_process_by_pid(pid);
2921         }
2922
2923         if (IS_ERR_OR_NULL(target)) {
2924                 pr_debug("Cannot find process PID %i to debug\n", args->pid);
2925                 r = target ? PTR_ERR(target) : -ESRCH;
2926                 goto out;
2927         }
2928
2929         /* Check if target is still PTRACED. */
2930         rcu_read_lock();
2931         if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
2932                                 && ptrace_parent(target->lead_thread) != current) {
2933                 pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid);
2934                 r = -EPERM;
2935         }
2936         rcu_read_unlock();
2937
2938         if (r)
2939                 goto out;
2940
2941         mutex_lock(&target->mutex);
2942
2943         if (args->op != KFD_IOC_DBG_TRAP_ENABLE && !target->debug_trap_enabled) {
2944                 pr_err("PID %i not debug enabled for op %i\n", args->pid, args->op);
2945                 r = -EINVAL;
2946                 goto unlock_out;
2947         }
2948
2949         if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_ENABLED &&
2950                         (args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE ||
2951                          args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE ||
2952                          args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ||
2953                          args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES ||
2954                          args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2955                          args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH ||
2956                          args->op == KFD_IOC_DBG_TRAP_SET_FLAGS)) {
2957                 r = -EPERM;
2958                 goto unlock_out;
2959         }
2960
2961         if (args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2962             args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH) {
2963                 int user_gpu_id = kfd_process_get_user_gpu_id(target,
2964                                 args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ?
2965                                         args->set_node_address_watch.gpu_id :
2966                                         args->clear_node_address_watch.gpu_id);
2967
2968                 pdd = kfd_process_device_data_by_id(target, user_gpu_id);
2969                 if (user_gpu_id == -EINVAL || !pdd) {
2970                         r = -ENODEV;
2971                         goto unlock_out;
2972                 }
2973         }
2974
2975         switch (args->op) {
2976         case KFD_IOC_DBG_TRAP_ENABLE:
2977                 if (target != p)
2978                         target->debugger_process = p;
2979
2980                 r = kfd_dbg_trap_enable(target,
2981                                         args->enable.dbg_fd,
2982                                         (void __user *)args->enable.rinfo_ptr,
2983                                         &args->enable.rinfo_size);
2984                 if (!r)
2985                         target->exception_enable_mask = args->enable.exception_mask;
2986
2987                 break;
2988         case KFD_IOC_DBG_TRAP_DISABLE:
2989                 r = kfd_dbg_trap_disable(target);
2990                 break;
2991         case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
2992                 r = kfd_dbg_send_exception_to_runtime(target,
2993                                 args->send_runtime_event.gpu_id,
2994                                 args->send_runtime_event.queue_id,
2995                                 args->send_runtime_event.exception_mask);
2996                 break;
2997         case KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED:
2998                 kfd_dbg_set_enabled_debug_exception_mask(target,
2999                                 args->set_exceptions_enabled.exception_mask);
3000                 break;
3001         case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
3002                 r = kfd_dbg_trap_set_wave_launch_override(target,
3003                                 args->launch_override.override_mode,
3004                                 args->launch_override.enable_mask,
3005                                 args->launch_override.support_request_mask,
3006                                 &args->launch_override.enable_mask,
3007                                 &args->launch_override.support_request_mask);
3008                 break;
3009         case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
3010                 r = kfd_dbg_trap_set_wave_launch_mode(target,
3011                                 args->launch_mode.launch_mode);
3012                 break;
3013         case KFD_IOC_DBG_TRAP_SUSPEND_QUEUES:
3014                 r = suspend_queues(target,
3015                                 args->suspend_queues.num_queues,
3016                                 args->suspend_queues.grace_period,
3017                                 args->suspend_queues.exception_mask,
3018                                 (uint32_t *)args->suspend_queues.queue_array_ptr);
3019
3020                 break;
3021         case KFD_IOC_DBG_TRAP_RESUME_QUEUES:
3022                 r = resume_queues(target, args->resume_queues.num_queues,
3023                                 (uint32_t *)args->resume_queues.queue_array_ptr);
3024                 break;
3025         case KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH:
3026                 r = kfd_dbg_trap_set_dev_address_watch(pdd,
3027                                 args->set_node_address_watch.address,
3028                                 args->set_node_address_watch.mask,
3029                                 &args->set_node_address_watch.id,
3030                                 args->set_node_address_watch.mode);
3031                 break;
3032         case KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH:
3033                 r = kfd_dbg_trap_clear_dev_address_watch(pdd,
3034                                 args->clear_node_address_watch.id);
3035                 break;
3036         case KFD_IOC_DBG_TRAP_SET_FLAGS:
3037                 r = kfd_dbg_trap_set_flags(target, &args->set_flags.flags);
3038                 break;
3039         case KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
3040                 r = kfd_dbg_ev_query_debug_event(target,
3041                                 &args->query_debug_event.queue_id,
3042                                 &args->query_debug_event.gpu_id,
3043                                 args->query_debug_event.exception_mask,
3044                                 &args->query_debug_event.exception_mask);
3045                 break;
3046         case KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO:
3047                 r = kfd_dbg_trap_query_exception_info(target,
3048                                 args->query_exception_info.source_id,
3049                                 args->query_exception_info.exception_code,
3050                                 args->query_exception_info.clear_exception,
3051                                 (void __user *)args->query_exception_info.info_ptr,
3052                                 &args->query_exception_info.info_size);
3053                 break;
3054         case KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT:
3055                 r = pqm_get_queue_snapshot(&target->pqm,
3056                                 args->queue_snapshot.exception_mask,
3057                                 (void __user *)args->queue_snapshot.snapshot_buf_ptr,
3058                                 &args->queue_snapshot.num_queues,
3059                                 &args->queue_snapshot.entry_size);
3060                 break;
3061         case KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT:
3062                 r = kfd_dbg_trap_device_snapshot(target,
3063                                 args->device_snapshot.exception_mask,
3064                                 (void __user *)args->device_snapshot.snapshot_buf_ptr,
3065                                 &args->device_snapshot.num_devices,
3066                                 &args->device_snapshot.entry_size);
3067                 break;
3068         default:
3069                 pr_err("Invalid option: %i\n", args->op);
3070                 r = -EINVAL;
3071         }
3072
3073 unlock_out:
3074         mutex_unlock(&target->mutex);
3075
3076 out:
3077         if (thread)
3078                 put_task_struct(thread);
3079
3080         if (mm)
3081                 mmput(mm);
3082
3083         if (pid)
3084                 put_pid(pid);
3085
3086         if (target)
3087                 kfd_unref_process(target);
3088
3089         return r;
3090 }
3091
3092 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
3093         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
3094                             .cmd_drv = 0, .name = #ioctl}
3095
3096 /** Ioctl table */
3097 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
3098         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
3099                         kfd_ioctl_get_version, 0),
3100
3101         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
3102                         kfd_ioctl_create_queue, 0),
3103
3104         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
3105                         kfd_ioctl_destroy_queue, 0),
3106
3107         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
3108                         kfd_ioctl_set_memory_policy, 0),
3109
3110         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
3111                         kfd_ioctl_get_clock_counters, 0),
3112
3113         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
3114                         kfd_ioctl_get_process_apertures, 0),
3115
3116         AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
3117                         kfd_ioctl_update_queue, 0),
3118
3119         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
3120                         kfd_ioctl_create_event, 0),
3121
3122         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
3123                         kfd_ioctl_destroy_event, 0),
3124
3125         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
3126                         kfd_ioctl_set_event, 0),
3127
3128         AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
3129                         kfd_ioctl_reset_event, 0),
3130
3131         AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
3132                         kfd_ioctl_wait_events, 0),
3133
3134         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
3135                         kfd_ioctl_dbg_register, 0),
3136
3137         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
3138                         kfd_ioctl_dbg_unregister, 0),
3139
3140         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
3141                         kfd_ioctl_dbg_address_watch, 0),
3142
3143         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
3144                         kfd_ioctl_dbg_wave_control, 0),
3145
3146         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
3147                         kfd_ioctl_set_scratch_backing_va, 0),
3148
3149         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
3150                         kfd_ioctl_get_tile_config, 0),
3151
3152         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
3153                         kfd_ioctl_set_trap_handler, 0),
3154
3155         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
3156                         kfd_ioctl_get_process_apertures_new, 0),
3157
3158         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
3159                         kfd_ioctl_acquire_vm, 0),
3160
3161         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
3162                         kfd_ioctl_alloc_memory_of_gpu, 0),
3163
3164         AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
3165                         kfd_ioctl_free_memory_of_gpu, 0),
3166
3167         AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
3168                         kfd_ioctl_map_memory_to_gpu, 0),
3169
3170         AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
3171                         kfd_ioctl_unmap_memory_from_gpu, 0),
3172
3173         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
3174                         kfd_ioctl_set_cu_mask, 0),
3175
3176         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
3177                         kfd_ioctl_get_queue_wave_state, 0),
3178
3179         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
3180                                 kfd_ioctl_get_dmabuf_info, 0),
3181
3182         AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
3183                                 kfd_ioctl_import_dmabuf, 0),
3184
3185         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
3186                         kfd_ioctl_alloc_queue_gws, 0),
3187
3188         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
3189                         kfd_ioctl_smi_events, 0),
3190
3191         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
3192
3193         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
3194                         kfd_ioctl_set_xnack_mode, 0),
3195
3196         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
3197                         kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
3198
3199         AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
3200                         kfd_ioctl_get_available_memory, 0),
3201
3202         AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
3203                                 kfd_ioctl_export_dmabuf, 0),
3204
3205         AMDKFD_IOCTL_DEF(AMDKFD_IOC_RUNTIME_ENABLE,
3206                         kfd_ioctl_runtime_enable, 0),
3207
3208         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
3209                         kfd_ioctl_set_debug_trap, 0),
3210 };
3211
3212 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
3213
3214 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
3215 {
3216         struct kfd_process *process;
3217         amdkfd_ioctl_t *func;
3218         const struct amdkfd_ioctl_desc *ioctl = NULL;
3219         unsigned int nr = _IOC_NR(cmd);
3220         char stack_kdata[128];
3221         char *kdata = NULL;
3222         unsigned int usize, asize;
3223         int retcode = -EINVAL;
3224         bool ptrace_attached = false;
3225
3226         if (nr >= AMDKFD_CORE_IOCTL_COUNT)
3227                 goto err_i1;
3228
3229         if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
3230                 u32 amdkfd_size;
3231
3232                 ioctl = &amdkfd_ioctls[nr];
3233
3234                 amdkfd_size = _IOC_SIZE(ioctl->cmd);
3235                 usize = asize = _IOC_SIZE(cmd);
3236                 if (amdkfd_size > asize)
3237                         asize = amdkfd_size;
3238
3239                 cmd = ioctl->cmd;
3240         } else
3241                 goto err_i1;
3242
3243         dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
3244
3245         /* Get the process struct from the filep. Only the process
3246          * that opened /dev/kfd can use the file descriptor. Child
3247          * processes need to create their own KFD device context.
3248          */
3249         process = filep->private_data;
3250
3251         rcu_read_lock();
3252         if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
3253             ptrace_parent(process->lead_thread) == current)
3254                 ptrace_attached = true;
3255         rcu_read_unlock();
3256
3257         if (process->lead_thread != current->group_leader
3258             && !ptrace_attached) {
3259                 dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
3260                 retcode = -EBADF;
3261                 goto err_i1;
3262         }
3263
3264         /* Do not trust userspace, use our own definition */
3265         func = ioctl->func;
3266
3267         if (unlikely(!func)) {
3268                 dev_dbg(kfd_device, "no function\n");
3269                 retcode = -EINVAL;
3270                 goto err_i1;
3271         }
3272
3273         /*
3274          * Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
3275          * CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
3276          * more priviledged access.
3277          */
3278         if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
3279                 if (!capable(CAP_CHECKPOINT_RESTORE) &&
3280                                                 !capable(CAP_SYS_ADMIN)) {
3281                         retcode = -EACCES;
3282                         goto err_i1;
3283                 }
3284         }
3285
3286         if (cmd & (IOC_IN | IOC_OUT)) {
3287                 if (asize <= sizeof(stack_kdata)) {
3288                         kdata = stack_kdata;
3289                 } else {
3290                         kdata = kmalloc(asize, GFP_KERNEL);
3291                         if (!kdata) {
3292                                 retcode = -ENOMEM;
3293                                 goto err_i1;
3294                         }
3295                 }
3296                 if (asize > usize)
3297                         memset(kdata + usize, 0, asize - usize);
3298         }
3299
3300         if (cmd & IOC_IN) {
3301                 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
3302                         retcode = -EFAULT;
3303                         goto err_i1;
3304                 }
3305         } else if (cmd & IOC_OUT) {
3306                 memset(kdata, 0, usize);
3307         }
3308
3309         retcode = func(filep, process, kdata);
3310
3311         if (cmd & IOC_OUT)
3312                 if (copy_to_user((void __user *)arg, kdata, usize) != 0)
3313                         retcode = -EFAULT;
3314
3315 err_i1:
3316         if (!ioctl)
3317                 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
3318                           task_pid_nr(current), cmd, nr);
3319
3320         if (kdata != stack_kdata)
3321                 kfree(kdata);
3322
3323         if (retcode)
3324                 dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
3325                                 nr, arg, retcode);
3326
3327         return retcode;
3328 }
3329
3330 static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
3331                       struct vm_area_struct *vma)
3332 {
3333         phys_addr_t address;
3334
3335         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3336                 return -EINVAL;
3337
3338         address = dev->adev->rmmio_remap.bus_addr;
3339
3340         vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
3341                                 VM_DONTDUMP | VM_PFNMAP);
3342
3343         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3344
3345         pr_debug("pasid 0x%x mapping mmio page\n"
3346                  "     target user address == 0x%08llX\n"
3347                  "     physical address    == 0x%08llX\n"
3348                  "     vm_flags            == 0x%04lX\n"
3349                  "     size                == 0x%04lX\n",
3350                  process->pasid, (unsigned long long) vma->vm_start,
3351                  address, vma->vm_flags, PAGE_SIZE);
3352
3353         return io_remap_pfn_range(vma,
3354                                 vma->vm_start,
3355                                 address >> PAGE_SHIFT,
3356                                 PAGE_SIZE,
3357                                 vma->vm_page_prot);
3358 }
3359
3360
3361 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
3362 {
3363         struct kfd_process *process;
3364         struct kfd_node *dev = NULL;
3365         unsigned long mmap_offset;
3366         unsigned int gpu_id;
3367
3368         process = kfd_get_process(current);
3369         if (IS_ERR(process))
3370                 return PTR_ERR(process);
3371
3372         mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
3373         gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
3374         if (gpu_id)
3375                 dev = kfd_device_by_id(gpu_id);
3376
3377         switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
3378         case KFD_MMAP_TYPE_DOORBELL:
3379                 if (!dev)
3380                         return -ENODEV;
3381                 return kfd_doorbell_mmap(dev, process, vma);
3382
3383         case KFD_MMAP_TYPE_EVENTS:
3384                 return kfd_event_mmap(process, vma);
3385
3386         case KFD_MMAP_TYPE_RESERVED_MEM:
3387                 if (!dev)
3388                         return -ENODEV;
3389                 return kfd_reserved_mem_mmap(dev, process, vma);
3390         case KFD_MMAP_TYPE_MMIO:
3391                 if (!dev)
3392                         return -ENODEV;
3393                 return kfd_mmio_mmap(dev, process, vma);
3394         }
3395
3396         return -EFAULT;
3397 }