drivers/gpu/drm/amd/amdkfd/kfd_chardev.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/device.h>
  25 #include <linux/export.h>
  26 #include <linux/err.h>
  27 #include <linux/fs.h>
  28 #include <linux/file.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31 #include <linux/uaccess.h>
  32 #include <linux/compat.h>
  33 #include <uapi/linux/kfd_ioctl.h>
  34 #include <linux/time.h>
  35 #include <linux/mm.h>
  36 #include <linux/mman.h>
  37 #include <linux/ptrace.h>
  38 #include <linux/dma-buf.h>
  39 #include <linux/processor.h>
  40 #include "kfd_priv.h"
  41 #include "kfd_device_queue_manager.h"
  42 #include "kfd_svm.h"
  43 #include "amdgpu_amdkfd.h"
  44 #include "kfd_smi_events.h"
  45 #include "amdgpu_dma_buf.h"
  46 #include "kfd_debug.h"
  47
  48 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
  49 static int kfd_open(struct inode *, struct file *);
  50 static int kfd_release(struct inode *, struct file *);
  51 static int kfd_mmap(struct file *, struct vm_area_struct *);
  52
  53 static const char kfd_dev_name[] = "kfd";
  54
  55 static const struct file_operations kfd_fops = {
  56         .owner = THIS_MODULE,
  57         .unlocked_ioctl = kfd_ioctl,
  58         .compat_ioctl = compat_ptr_ioctl,
  59         .open = kfd_open,
  60         .release = kfd_release,
  61         .mmap = kfd_mmap,
  62 };
  63
  64 static int kfd_char_dev_major = -1;
  65 struct device *kfd_device;
  66 static const struct class kfd_class = {
  67         .name = kfd_dev_name,
  68 };
  69
  70 static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
  71 {
  72         struct kfd_process_device *pdd;
  73
  74         mutex_lock(&p->mutex);
  75         pdd = kfd_process_device_data_by_id(p, gpu_id);
  76
  77         if (pdd)
  78                 return pdd;
  79
  80         mutex_unlock(&p->mutex);
  81         return NULL;
  82 }
  83
  84 static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
  85 {
  86         mutex_unlock(&pdd->process->mutex);
  87 }
  88
  89 int kfd_chardev_init(void)
  90 {
  91         int err = 0;
  92
  93         kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
  94         err = kfd_char_dev_major;
  95         if (err < 0)
  96                 goto err_register_chrdev;
  97
  98         err = class_register(&kfd_class);
  99         if (err)
 100                 goto err_class_create;
 101
 102         kfd_device = device_create(&kfd_class, NULL,
 103                                    MKDEV(kfd_char_dev_major, 0),
 104                                    NULL, kfd_dev_name);
 105         err = PTR_ERR(kfd_device);
 106         if (IS_ERR(kfd_device))
 107                 goto err_device_create;
 108
 109         return 0;
 110
 111 err_device_create:
 112         class_unregister(&kfd_class);
 113 err_class_create:
 114         unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 115 err_register_chrdev:
 116         return err;
 117 }
 118
 119 void kfd_chardev_exit(void)
 120 {
 121         device_destroy(&kfd_class, MKDEV(kfd_char_dev_major, 0));
 122         class_unregister(&kfd_class);
 123         unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 124         kfd_device = NULL;
 125 }
 126
 127
 128 static int kfd_open(struct inode *inode, struct file *filep)
 129 {
 130         struct kfd_process *process;
 131         bool is_32bit_user_mode;
 132
 133         if (iminor(inode) != 0)
 134                 return -ENODEV;
 135
 136         is_32bit_user_mode = in_compat_syscall();
 137
 138         if (is_32bit_user_mode) {
 139                 dev_warn(kfd_device,
 140                         "Process %d (32-bit) failed to open /dev/kfd\n"
 141                         "32-bit processes are not supported by amdkfd\n",
 142                         current->pid);
 143                 return -EPERM;
 144         }
 145
 146         process = kfd_create_process(current);
 147         if (IS_ERR(process))
 148                 return PTR_ERR(process);
 149
 150         if (kfd_process_init_cwsr_apu(process, filep)) {
 151                 kfd_unref_process(process);
 152                 return -EFAULT;
 153         }
 154
 155         /* filep now owns the reference returned by kfd_create_process */
 156         filep->private_data = process;
 157
 158         dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
 159                 process->pasid, process->is_32bit_user_mode);
 160
 161         return 0;
 162 }
 163
 164 static int kfd_release(struct inode *inode, struct file *filep)
 165 {
 166         struct kfd_process *process = filep->private_data;
 167
 168         if (process)
 169                 kfd_unref_process(process);
 170
 171         return 0;
 172 }
 173
 174 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
 175                                         void *data)
 176 {
 177         struct kfd_ioctl_get_version_args *args = data;
 178
 179         args->major_version = KFD_IOCTL_MAJOR_VERSION;
 180         args->minor_version = KFD_IOCTL_MINOR_VERSION;
 181
 182         return 0;
 183 }
 184
 185 static int set_queue_properties_from_user(struct queue_properties *q_properties,
 186                                 struct kfd_ioctl_create_queue_args *args)
 187 {
 188         /*
 189          * Repurpose queue percentage to accommodate new features:
 190          * bit 0-7: queue percentage
 191          * bit 8-15: pm4_target_xcc
 192          */
 193         if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 194                 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 195                 return -EINVAL;
 196         }
 197
 198         if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 199                 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 200                 return -EINVAL;
 201         }
 202
 203         if ((args->ring_base_address) &&
 204                 (!access_ok((const void __user *) args->ring_base_address,
 205                         sizeof(uint64_t)))) {
 206                 pr_err("Can't access ring base address\n");
 207                 return -EFAULT;
 208         }
 209
 210         if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 211                 pr_err("Ring size must be a power of 2 or 0\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (!access_ok((const void __user *) args->read_pointer_address,
 216                         sizeof(uint32_t))) {
 217                 pr_err("Can't access read pointer\n");
 218                 return -EFAULT;
 219         }
 220
 221         if (!access_ok((const void __user *) args->write_pointer_address,
 222                         sizeof(uint32_t))) {
 223                 pr_err("Can't access write pointer\n");
 224                 return -EFAULT;
 225         }
 226
 227         if (args->eop_buffer_address &&
 228                 !access_ok((const void __user *) args->eop_buffer_address,
 229                         sizeof(uint32_t))) {
 230                 pr_debug("Can't access eop buffer");
 231                 return -EFAULT;
 232         }
 233
 234         if (args->ctx_save_restore_address &&
 235                 !access_ok((const void __user *) args->ctx_save_restore_address,
 236                         sizeof(uint32_t))) {
 237                 pr_debug("Can't access ctx save restore buffer");
 238                 return -EFAULT;
 239         }
 240
 241         q_properties->is_interop = false;
 242         q_properties->is_gws = false;
 243         q_properties->queue_percent = args->queue_percentage & 0xFF;
 244         /* bit 8-15 are repurposed to be PM4 target XCC */
 245         q_properties->pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 246         q_properties->priority = args->queue_priority;
 247         q_properties->queue_address = args->ring_base_address;
 248         q_properties->queue_size = args->ring_size;
 249         q_properties->read_ptr = (void __user *)args->read_pointer_address;
 250         q_properties->write_ptr = (void __user *)args->write_pointer_address;
 251         q_properties->eop_ring_buffer_address = args->eop_buffer_address;
 252         q_properties->eop_ring_buffer_size = args->eop_buffer_size;
 253         q_properties->ctx_save_restore_area_address =
 254                         args->ctx_save_restore_address;
 255         q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
 256         q_properties->ctl_stack_size = args->ctl_stack_size;
 257         q_properties->sdma_engine_id = args->sdma_engine_id;
 258         if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 259                 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 260                 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
 261         else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
 262                 q_properties->type = KFD_QUEUE_TYPE_SDMA;
 263         else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
 264                 q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
 265         else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID)
 266                 q_properties->type = KFD_QUEUE_TYPE_SDMA_BY_ENG_ID;
 267         else
 268                 return -ENOTSUPP;
 269
 270         if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 271                 q_properties->format = KFD_QUEUE_FORMAT_AQL;
 272         else
 273                 q_properties->format = KFD_QUEUE_FORMAT_PM4;
 274
 275         pr_debug("Queue Percentage: %d, %d\n",
 276                         q_properties->queue_percent, args->queue_percentage);
 277
 278         pr_debug("Queue Priority: %d, %d\n",
 279                         q_properties->priority, args->queue_priority);
 280
 281         pr_debug("Queue Address: 0x%llX, 0x%llX\n",
 282                         q_properties->queue_address, args->ring_base_address);
 283
 284         pr_debug("Queue Size: 0x%llX, %u\n",
 285                         q_properties->queue_size, args->ring_size);
 286
 287         pr_debug("Queue r/w Pointers: %px, %px\n",
 288                         q_properties->read_ptr,
 289                         q_properties->write_ptr);
 290
 291         pr_debug("Queue Format: %d\n", q_properties->format);
 292
 293         pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
 294
 295         pr_debug("Queue CTX save area: 0x%llX\n",
 296                         q_properties->ctx_save_restore_area_address);
 297
 298         return 0;
 299 }
 300
 301 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 302                                         void *data)
 303 {
 304         struct kfd_ioctl_create_queue_args *args = data;
 305         struct kfd_node *dev;
 306         int err = 0;
 307         unsigned int queue_id;
 308         struct kfd_process_device *pdd;
 309         struct queue_properties q_properties;
 310         uint32_t doorbell_offset_in_process = 0;
 311
 312         memset(&q_properties, 0, sizeof(struct queue_properties));
 313
 314         pr_debug("Creating queue ioctl\n");
 315
 316         err = set_queue_properties_from_user(&q_properties, args);
 317         if (err)
 318                 return err;
 319
 320         pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
 321
 322         mutex_lock(&p->mutex);
 323
 324         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 325         if (!pdd) {
 326                 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 327                 err = -EINVAL;
 328                 goto err_pdd;
 329         }
 330         dev = pdd->dev;
 331
 332         pdd = kfd_bind_process_to_device(dev, p);
 333         if (IS_ERR(pdd)) {
 334                 err = -ESRCH;
 335                 goto err_bind_process;
 336         }
 337
 338         if (q_properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
 339                 int max_sdma_eng_id = kfd_get_num_sdma_engines(dev) +
 340                                       kfd_get_num_xgmi_sdma_engines(dev) - 1;
 341
 342                 if (q_properties.sdma_engine_id > max_sdma_eng_id) {
 343                         err = -EINVAL;
 344                         pr_err("sdma_engine_id %i exceeds maximum id of %i\n",
 345                                q_properties.sdma_engine_id, max_sdma_eng_id);
 346                         goto err_sdma_engine_id;
 347                 }
 348         }
 349
 350         if (!pdd->qpd.proc_doorbells) {
 351                 err = kfd_alloc_process_doorbells(dev->kfd, pdd);
 352                 if (err) {
 353                         pr_debug("failed to allocate process doorbells\n");
 354                         goto err_bind_process;
 355                 }
 356         }
 357
 358         err = kfd_queue_acquire_buffers(pdd, &q_properties);
 359         if (err) {
 360                 pr_debug("failed to acquire user queue buffers\n");
 361                 goto err_acquire_queue_buf;
 362         }
 363
 364         pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
 365                         p->pasid,
 366                         dev->id);
 367
 368         err = pqm_create_queue(&p->pqm, dev, &q_properties, &queue_id,
 369                         NULL, NULL, NULL, &doorbell_offset_in_process);
 370         if (err != 0)
 371                 goto err_create_queue;
 372
 373         args->queue_id = queue_id;
 374
 375
 376         /* Return gpu_id as doorbell offset for mmap usage */
 377         args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
 378         args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
 379         if (KFD_IS_SOC15(dev))
 380                 /* On SOC15 ASICs, include the doorbell offset within the
 381                  * process doorbell frame, which is 2 pages.
 382                  */
 383                 args->doorbell_offset |= doorbell_offset_in_process;
 384
 385         mutex_unlock(&p->mutex);
 386
 387         pr_debug("Queue id %d was created successfully\n", args->queue_id);
 388
 389         pr_debug("Ring buffer address == 0x%016llX\n",
 390                         args->ring_base_address);
 391
 392         pr_debug("Read ptr address    == 0x%016llX\n",
 393                         args->read_pointer_address);
 394
 395         pr_debug("Write ptr address   == 0x%016llX\n",
 396                         args->write_pointer_address);
 397
 398         kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), p, dev, queue_id, false, NULL, 0);
 399         return 0;
 400
 401 err_create_queue:
 402         kfd_queue_unref_bo_vas(pdd, &q_properties);
 403         kfd_queue_release_buffers(pdd, &q_properties);
 404 err_acquire_queue_buf:
 405 err_sdma_engine_id:
 406 err_bind_process:
 407 err_pdd:
 408         mutex_unlock(&p->mutex);
 409         return err;
 410 }
 411
 412 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 413                                         void *data)
 414 {
 415         int retval;
 416         struct kfd_ioctl_destroy_queue_args *args = data;
 417
 418         pr_debug("Destroying queue id %d for pasid 0x%x\n",
 419                                 args->queue_id,
 420                                 p->pasid);
 421
 422         mutex_lock(&p->mutex);
 423
 424         retval = pqm_destroy_queue(&p->pqm, args->queue_id);
 425
 426         mutex_unlock(&p->mutex);
 427         return retval;
 428 }
 429
 430 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 431                                         void *data)
 432 {
 433         int retval;
 434         struct kfd_ioctl_update_queue_args *args = data;
 435         struct queue_properties properties;
 436
 437         /*
 438          * Repurpose queue percentage to accommodate new features:
 439          * bit 0-7: queue percentage
 440          * bit 8-15: pm4_target_xcc
 441          */
 442         if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 443                 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 444                 return -EINVAL;
 445         }
 446
 447         if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 448                 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 449                 return -EINVAL;
 450         }
 451
 452         if ((args->ring_base_address) &&
 453                 (!access_ok((const void __user *) args->ring_base_address,
 454                         sizeof(uint64_t)))) {
 455                 pr_err("Can't access ring base address\n");
 456                 return -EFAULT;
 457         }
 458
 459         if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 460                 pr_err("Ring size must be a power of 2 or 0\n");
 461                 return -EINVAL;
 462         }
 463
 464         properties.queue_address = args->ring_base_address;
 465         properties.queue_size = args->ring_size;
 466         properties.queue_percent = args->queue_percentage & 0xFF;
 467         /* bit 8-15 are repurposed to be PM4 target XCC */
 468         properties.pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 469         properties.priority = args->queue_priority;
 470
 471         pr_debug("Updating queue id %d for pasid 0x%x\n",
 472                         args->queue_id, p->pasid);
 473
 474         mutex_lock(&p->mutex);
 475
 476         retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
 477
 478         mutex_unlock(&p->mutex);
 479
 480         return retval;
 481 }
 482
 483 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
 484                                         void *data)
 485 {
 486         int retval;
 487         const int max_num_cus = 1024;
 488         struct kfd_ioctl_set_cu_mask_args *args = data;
 489         struct mqd_update_info minfo = {0};
 490         uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
 491         size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
 492
 493         if ((args->num_cu_mask % 32) != 0) {
 494                 pr_debug("num_cu_mask 0x%x must be a multiple of 32",
 495                                 args->num_cu_mask);
 496                 return -EINVAL;
 497         }
 498
 499         minfo.cu_mask.count = args->num_cu_mask;
 500         if (minfo.cu_mask.count == 0) {
 501                 pr_debug("CU mask cannot be 0");
 502                 return -EINVAL;
 503         }
 504
 505         /* To prevent an unreasonably large CU mask size, set an arbitrary
 506          * limit of max_num_cus bits.  We can then just drop any CU mask bits
 507          * past max_num_cus bits and just use the first max_num_cus bits.
 508          */
 509         if (minfo.cu_mask.count > max_num_cus) {
 510                 pr_debug("CU mask cannot be greater than 1024 bits");
 511                 minfo.cu_mask.count = max_num_cus;
 512                 cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
 513         }
 514
 515         minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
 516         if (!minfo.cu_mask.ptr)
 517                 return -ENOMEM;
 518
 519         retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
 520         if (retval) {
 521                 pr_debug("Could not copy CU mask from userspace");
 522                 retval = -EFAULT;
 523                 goto out;
 524         }
 525
 526         mutex_lock(&p->mutex);
 527
 528         retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
 529
 530         mutex_unlock(&p->mutex);
 531
 532 out:
 533         kfree(minfo.cu_mask.ptr);
 534         return retval;
 535 }
 536
 537 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
 538                                           struct kfd_process *p, void *data)
 539 {
 540         struct kfd_ioctl_get_queue_wave_state_args *args = data;
 541         int r;
 542
 543         mutex_lock(&p->mutex);
 544
 545         r = pqm_get_wave_state(&p->pqm, args->queue_id,
 546                                (void __user *)args->ctl_stack_address,
 547                                &args->ctl_stack_used_size,
 548                                &args->save_area_used_size);
 549
 550         mutex_unlock(&p->mutex);
 551
 552         return r;
 553 }
 554
 555 static int kfd_ioctl_set_memory_policy(struct file *filep,
 556                                         struct kfd_process *p, void *data)
 557 {
 558         struct kfd_ioctl_set_memory_policy_args *args = data;
 559         int err = 0;
 560         struct kfd_process_device *pdd;
 561         enum cache_policy default_policy, alternate_policy;
 562
 563         if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
 564             && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 565                 return -EINVAL;
 566         }
 567
 568         if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
 569             && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 570                 return -EINVAL;
 571         }
 572
 573         mutex_lock(&p->mutex);
 574         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 575         if (!pdd) {
 576                 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 577                 err = -EINVAL;
 578                 goto err_pdd;
 579         }
 580
 581         pdd = kfd_bind_process_to_device(pdd->dev, p);
 582         if (IS_ERR(pdd)) {
 583                 err = -ESRCH;
 584                 goto out;
 585         }
 586
 587         default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 588                          ? cache_policy_coherent : cache_policy_noncoherent;
 589
 590         alternate_policy =
 591                 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 592                    ? cache_policy_coherent : cache_policy_noncoherent;
 593
 594         if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
 595                                 &pdd->qpd,
 596                                 default_policy,
 597                                 alternate_policy,
 598                                 (void __user *)args->alternate_aperture_base,
 599                                 args->alternate_aperture_size))
 600                 err = -EINVAL;
 601
 602 out:
 603 err_pdd:
 604         mutex_unlock(&p->mutex);
 605
 606         return err;
 607 }
 608
 609 static int kfd_ioctl_set_trap_handler(struct file *filep,
 610                                         struct kfd_process *p, void *data)
 611 {
 612         struct kfd_ioctl_set_trap_handler_args *args = data;
 613         int err = 0;
 614         struct kfd_process_device *pdd;
 615
 616         mutex_lock(&p->mutex);
 617
 618         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 619         if (!pdd) {
 620                 err = -EINVAL;
 621                 goto err_pdd;
 622         }
 623
 624         pdd = kfd_bind_process_to_device(pdd->dev, p);
 625         if (IS_ERR(pdd)) {
 626                 err = -ESRCH;
 627                 goto out;
 628         }
 629
 630         kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
 631
 632 out:
 633 err_pdd:
 634         mutex_unlock(&p->mutex);
 635
 636         return err;
 637 }
 638
 639 static int kfd_ioctl_dbg_register(struct file *filep,
 640                                 struct kfd_process *p, void *data)
 641 {
 642         return -EPERM;
 643 }
 644
 645 static int kfd_ioctl_dbg_unregister(struct file *filep,
 646                                 struct kfd_process *p, void *data)
 647 {
 648         return -EPERM;
 649 }
 650
 651 static int kfd_ioctl_dbg_address_watch(struct file *filep,
 652                                         struct kfd_process *p, void *data)
 653 {
 654         return -EPERM;
 655 }
 656
 657 /* Parse and generate fixed size data structure for wave control */
 658 static int kfd_ioctl_dbg_wave_control(struct file *filep,
 659                                         struct kfd_process *p, void *data)
 660 {
 661         return -EPERM;
 662 }
 663
 664 static int kfd_ioctl_get_clock_counters(struct file *filep,
 665                                 struct kfd_process *p, void *data)
 666 {
 667         struct kfd_ioctl_get_clock_counters_args *args = data;
 668         struct kfd_process_device *pdd;
 669
 670         mutex_lock(&p->mutex);
 671         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 672         mutex_unlock(&p->mutex);
 673         if (pdd)
 674                 /* Reading GPU clock counter from KGD */
 675                 args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(pdd->dev->adev);
 676         else
 677                 /* Node without GPU resource */
 678                 args->gpu_clock_counter = 0;
 679
 680         /* No access to rdtsc. Using raw monotonic time */
 681         args->cpu_clock_counter = ktime_get_raw_ns();
 682         args->system_clock_counter = ktime_get_boottime_ns();
 683
 684         /* Since the counter is in nano-seconds we use 1GHz frequency */
 685         args->system_clock_freq = 1000000000;
 686
 687         return 0;
 688 }
 689
 690
 691 static int kfd_ioctl_get_process_apertures(struct file *filp,
 692                                 struct kfd_process *p, void *data)
 693 {
 694         struct kfd_ioctl_get_process_apertures_args *args = data;
 695         struct kfd_process_device_apertures *pAperture;
 696         int i;
 697
 698         dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 699
 700         args->num_of_nodes = 0;
 701
 702         mutex_lock(&p->mutex);
 703         /* Run over all pdd of the process */
 704         for (i = 0; i < p->n_pdds; i++) {
 705                 struct kfd_process_device *pdd = p->pdds[i];
 706
 707                 pAperture =
 708                         &args->process_apertures[args->num_of_nodes];
 709                 pAperture->gpu_id = pdd->dev->id;
 710                 pAperture->lds_base = pdd->lds_base;
 711                 pAperture->lds_limit = pdd->lds_limit;
 712                 pAperture->gpuvm_base = pdd->gpuvm_base;
 713                 pAperture->gpuvm_limit = pdd->gpuvm_limit;
 714                 pAperture->scratch_base = pdd->scratch_base;
 715                 pAperture->scratch_limit = pdd->scratch_limit;
 716
 717                 dev_dbg(kfd_device,
 718                         "node id %u\n", args->num_of_nodes);
 719                 dev_dbg(kfd_device,
 720                         "gpu id %u\n", pdd->dev->id);
 721                 dev_dbg(kfd_device,
 722                         "lds_base %llX\n", pdd->lds_base);
 723                 dev_dbg(kfd_device,
 724                         "lds_limit %llX\n", pdd->lds_limit);
 725                 dev_dbg(kfd_device,
 726                         "gpuvm_base %llX\n", pdd->gpuvm_base);
 727                 dev_dbg(kfd_device,
 728                         "gpuvm_limit %llX\n", pdd->gpuvm_limit);
 729                 dev_dbg(kfd_device,
 730                         "scratch_base %llX\n", pdd->scratch_base);
 731                 dev_dbg(kfd_device,
 732                         "scratch_limit %llX\n", pdd->scratch_limit);
 733
 734                 if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
 735                         break;
 736         }
 737         mutex_unlock(&p->mutex);
 738
 739         return 0;
 740 }
 741
 742 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
 743                                 struct kfd_process *p, void *data)
 744 {
 745         struct kfd_ioctl_get_process_apertures_new_args *args = data;
 746         struct kfd_process_device_apertures *pa;
 747         int ret;
 748         int i;
 749
 750         dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
 751
 752         if (args->num_of_nodes == 0) {
 753                 /* Return number of nodes, so that user space can alloacate
 754                  * sufficient memory
 755                  */
 756                 mutex_lock(&p->mutex);
 757                 args->num_of_nodes = p->n_pdds;
 758                 goto out_unlock;
 759         }
 760
 761         /* Fill in process-aperture information for all available
 762          * nodes, but not more than args->num_of_nodes as that is
 763          * the amount of memory allocated by user
 764          */
 765         pa = kcalloc(args->num_of_nodes, sizeof(struct kfd_process_device_apertures),
 766                      GFP_KERNEL);
 767         if (!pa)
 768                 return -ENOMEM;
 769
 770         mutex_lock(&p->mutex);
 771
 772         if (!p->n_pdds) {
 773                 args->num_of_nodes = 0;
 774                 kfree(pa);
 775                 goto out_unlock;
 776         }
 777
 778         /* Run over all pdd of the process */
 779         for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
 780                 struct kfd_process_device *pdd = p->pdds[i];
 781
 782                 pa[i].gpu_id = pdd->dev->id;
 783                 pa[i].lds_base = pdd->lds_base;
 784                 pa[i].lds_limit = pdd->lds_limit;
 785                 pa[i].gpuvm_base = pdd->gpuvm_base;
 786                 pa[i].gpuvm_limit = pdd->gpuvm_limit;
 787                 pa[i].scratch_base = pdd->scratch_base;
 788                 pa[i].scratch_limit = pdd->scratch_limit;
 789
 790                 dev_dbg(kfd_device,
 791                         "gpu id %u\n", pdd->dev->id);
 792                 dev_dbg(kfd_device,
 793                         "lds_base %llX\n", pdd->lds_base);
 794                 dev_dbg(kfd_device,
 795                         "lds_limit %llX\n", pdd->lds_limit);
 796                 dev_dbg(kfd_device,
 797                         "gpuvm_base %llX\n", pdd->gpuvm_base);
 798                 dev_dbg(kfd_device,
 799                         "gpuvm_limit %llX\n", pdd->gpuvm_limit);
 800                 dev_dbg(kfd_device,
 801                         "scratch_base %llX\n", pdd->scratch_base);
 802                 dev_dbg(kfd_device,
 803                         "scratch_limit %llX\n", pdd->scratch_limit);
 804         }
 805         mutex_unlock(&p->mutex);
 806
 807         args->num_of_nodes = i;
 808         ret = copy_to_user(
 809                         (void __user *)args->kfd_process_device_apertures_ptr,
 810                         pa,
 811                         (i * sizeof(struct kfd_process_device_apertures)));
 812         kfree(pa);
 813         return ret ? -EFAULT : 0;
 814
 815 out_unlock:
 816         mutex_unlock(&p->mutex);
 817         return 0;
 818 }
 819
 820 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 821                                         void *data)
 822 {
 823         struct kfd_ioctl_create_event_args *args = data;
 824         int err;
 825
 826         /* For dGPUs the event page is allocated in user mode. The
 827          * handle is passed to KFD with the first call to this IOCTL
 828          * through the event_page_offset field.
 829          */
 830         if (args->event_page_offset) {
 831                 mutex_lock(&p->mutex);
 832                 err = kfd_kmap_event_page(p, args->event_page_offset);
 833                 mutex_unlock(&p->mutex);
 834                 if (err)
 835                         return err;
 836         }
 837
 838         err = kfd_event_create(filp, p, args->event_type,
 839                                 args->auto_reset != 0, args->node_id,
 840                                 &args->event_id, &args->event_trigger_data,
 841                                 &args->event_page_offset,
 842                                 &args->event_slot_index);
 843
 844         pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
 845         return err;
 846 }
 847
 848 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
 849                                         void *data)
 850 {
 851         struct kfd_ioctl_destroy_event_args *args = data;
 852
 853         return kfd_event_destroy(p, args->event_id);
 854 }
 855
 856 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
 857                                 void *data)
 858 {
 859         struct kfd_ioctl_set_event_args *args = data;
 860
 861         return kfd_set_event(p, args->event_id);
 862 }
 863
 864 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
 865                                 void *data)
 866 {
 867         struct kfd_ioctl_reset_event_args *args = data;
 868
 869         return kfd_reset_event(p, args->event_id);
 870 }
 871
 872 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 873                                 void *data)
 874 {
 875         struct kfd_ioctl_wait_events_args *args = data;
 876
 877         return kfd_wait_on_events(p, args->num_events,
 878                         (void __user *)args->events_ptr,
 879                         (args->wait_for_all != 0),
 880                         &args->timeout, &args->wait_result);
 881 }
 882 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
 883                                         struct kfd_process *p, void *data)
 884 {
 885         struct kfd_ioctl_set_scratch_backing_va_args *args = data;
 886         struct kfd_process_device *pdd;
 887         struct kfd_node *dev;
 888         long err;
 889
 890         mutex_lock(&p->mutex);
 891         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 892         if (!pdd) {
 893                 err = -EINVAL;
 894                 goto err_pdd;
 895         }
 896         dev = pdd->dev;
 897
 898         pdd = kfd_bind_process_to_device(dev, p);
 899         if (IS_ERR(pdd)) {
 900                 err = PTR_ERR(pdd);
 901                 goto bind_process_to_device_fail;
 902         }
 903
 904         pdd->qpd.sh_hidden_private_base = args->va_addr;
 905
 906         mutex_unlock(&p->mutex);
 907
 908         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
 909             pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
 910                 dev->kfd2kgd->set_scratch_backing_va(
 911                         dev->adev, args->va_addr, pdd->qpd.vmid);
 912
 913         return 0;
 914
 915 bind_process_to_device_fail:
 916 err_pdd:
 917         mutex_unlock(&p->mutex);
 918         return err;
 919 }
 920
 921 static int kfd_ioctl_get_tile_config(struct file *filep,
 922                 struct kfd_process *p, void *data)
 923 {
 924         struct kfd_ioctl_get_tile_config_args *args = data;
 925         struct kfd_process_device *pdd;
 926         struct tile_config config;
 927         int err = 0;
 928
 929         mutex_lock(&p->mutex);
 930         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 931         mutex_unlock(&p->mutex);
 932         if (!pdd)
 933                 return -EINVAL;
 934
 935         amdgpu_amdkfd_get_tile_config(pdd->dev->adev, &config);
 936
 937         args->gb_addr_config = config.gb_addr_config;
 938         args->num_banks = config.num_banks;
 939         args->num_ranks = config.num_ranks;
 940
 941         if (args->num_tile_configs > config.num_tile_configs)
 942                 args->num_tile_configs = config.num_tile_configs;
 943         err = copy_to_user((void __user *)args->tile_config_ptr,
 944                         config.tile_config_ptr,
 945                         args->num_tile_configs * sizeof(uint32_t));
 946         if (err) {
 947                 args->num_tile_configs = 0;
 948                 return -EFAULT;
 949         }
 950
 951         if (args->num_macro_tile_configs > config.num_macro_tile_configs)
 952                 args->num_macro_tile_configs =
 953                                 config.num_macro_tile_configs;
 954         err = copy_to_user((void __user *)args->macro_tile_config_ptr,
 955                         config.macro_tile_config_ptr,
 956                         args->num_macro_tile_configs * sizeof(uint32_t));
 957         if (err) {
 958                 args->num_macro_tile_configs = 0;
 959                 return -EFAULT;
 960         }
 961
 962         return 0;
 963 }
 964
 965 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
 966                                 void *data)
 967 {
 968         struct kfd_ioctl_acquire_vm_args *args = data;
 969         struct kfd_process_device *pdd;
 970         struct file *drm_file;
 971         int ret;
 972
 973         drm_file = fget(args->drm_fd);
 974         if (!drm_file)
 975                 return -EINVAL;
 976
 977         mutex_lock(&p->mutex);
 978         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 979         if (!pdd) {
 980                 ret = -EINVAL;
 981                 goto err_pdd;
 982         }
 983
 984         if (pdd->drm_file) {
 985                 ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
 986                 goto err_drm_file;
 987         }
 988
 989         ret = kfd_process_device_init_vm(pdd, drm_file);
 990         if (ret)
 991                 goto err_unlock;
 992
 993         /* On success, the PDD keeps the drm_file reference */
 994         mutex_unlock(&p->mutex);
 995
 996         return 0;
 997
 998 err_unlock:
 999 err_pdd:
1000 err_drm_file:
1001         mutex_unlock(&p->mutex);
1002         fput(drm_file);
1003         return ret;
1004 }
1005
1006 bool kfd_dev_is_large_bar(struct kfd_node *dev)
1007 {
1008         if (dev->kfd->adev->debug_largebar) {
1009                 pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1010                 return true;
1011         }
1012
1013         if (dev->local_mem_info.local_mem_size_private == 0 &&
1014             dev->local_mem_info.local_mem_size_public > 0)
1015                 return true;
1016
1017         if (dev->local_mem_info.local_mem_size_public == 0 &&
1018             dev->kfd->adev->gmc.is_app_apu) {
1019                 pr_debug("APP APU, Consider like a large bar system\n");
1020                 return true;
1021         }
1022
1023         return false;
1024 }
1025
1026 static int kfd_ioctl_get_available_memory(struct file *filep,
1027                                           struct kfd_process *p, void *data)
1028 {
1029         struct kfd_ioctl_get_available_memory_args *args = data;
1030         struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
1031
1032         if (!pdd)
1033                 return -EINVAL;
1034         args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
1035                                                         pdd->dev->node_id);
1036         kfd_unlock_pdd(pdd);
1037         return 0;
1038 }
1039
1040 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1041                                         struct kfd_process *p, void *data)
1042 {
1043         struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1044         struct kfd_process_device *pdd;
1045         void *mem;
1046         struct kfd_node *dev;
1047         int idr_handle;
1048         long err;
1049         uint64_t offset = args->mmap_offset;
1050         uint32_t flags = args->flags;
1051
1052         if (args->size == 0)
1053                 return -EINVAL;
1054
1055 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1056         /* Flush pending deferred work to avoid racing with deferred actions
1057          * from previous memory map changes (e.g. munmap).
1058          */
1059         svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1060         mutex_lock(&p->svms.lock);
1061         mmap_write_unlock(current->mm);
1062         if (interval_tree_iter_first(&p->svms.objects,
1063                                      args->va_addr >> PAGE_SHIFT,
1064                                      (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1065                 pr_err("Address: 0x%llx already allocated by SVM\n",
1066                         args->va_addr);
1067                 mutex_unlock(&p->svms.lock);
1068                 return -EADDRINUSE;
1069         }
1070
1071         /* When register user buffer check if it has been registered by svm by
1072          * buffer cpu virtual address.
1073          */
1074         if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) &&
1075             interval_tree_iter_first(&p->svms.objects,
1076                                      args->mmap_offset >> PAGE_SHIFT,
1077                                      (args->mmap_offset  + args->size - 1) >> PAGE_SHIFT)) {
1078                 pr_err("User Buffer Address: 0x%llx already allocated by SVM\n",
1079                         args->mmap_offset);
1080                 mutex_unlock(&p->svms.lock);
1081                 return -EADDRINUSE;
1082         }
1083
1084         mutex_unlock(&p->svms.lock);
1085 #endif
1086         mutex_lock(&p->mutex);
1087         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1088         if (!pdd) {
1089                 err = -EINVAL;
1090                 goto err_pdd;
1091         }
1092
1093         dev = pdd->dev;
1094
1095         if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1096                 (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1097                 !kfd_dev_is_large_bar(dev)) {
1098                 pr_err("Alloc host visible vram on small bar is not allowed\n");
1099                 err = -EINVAL;
1100                 goto err_large_bar;
1101         }
1102
1103         pdd = kfd_bind_process_to_device(dev, p);
1104         if (IS_ERR(pdd)) {
1105                 err = PTR_ERR(pdd);
1106                 goto err_unlock;
1107         }
1108
1109         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1110                 if (args->size != kfd_doorbell_process_slice(dev->kfd)) {
1111                         err = -EINVAL;
1112                         goto err_unlock;
1113                 }
1114                 offset = kfd_get_process_doorbells(pdd);
1115                 if (!offset) {
1116                         err = -ENOMEM;
1117                         goto err_unlock;
1118                 }
1119         } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1120                 if (args->size != PAGE_SIZE) {
1121                         err = -EINVAL;
1122                         goto err_unlock;
1123                 }
1124                 offset = dev->adev->rmmio_remap.bus_addr;
1125                 if (!offset || (PAGE_SIZE > 4096)) {
1126                         err = -ENOMEM;
1127                         goto err_unlock;
1128                 }
1129         }
1130
1131         err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1132                 dev->adev, args->va_addr, args->size,
1133                 pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1134                 flags, false);
1135
1136         if (err)
1137                 goto err_unlock;
1138
1139         idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1140         if (idr_handle < 0) {
1141                 err = -EFAULT;
1142                 goto err_free;
1143         }
1144
1145         /* Update the VRAM usage count */
1146         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1147                 uint64_t size = args->size;
1148
1149                 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
1150                         size >>= 1;
1151                 atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage);
1152         }
1153
1154         mutex_unlock(&p->mutex);
1155
1156         args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1157         args->mmap_offset = offset;
1158
1159         /* MMIO is mapped through kfd device
1160          * Generate a kfd mmap offset
1161          */
1162         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1163                 args->mmap_offset = KFD_MMAP_TYPE_MMIO
1164                                         | KFD_MMAP_GPU_ID(args->gpu_id);
1165
1166         return 0;
1167
1168 err_free:
1169         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
1170                                                pdd->drm_priv, NULL);
1171 err_unlock:
1172 err_pdd:
1173 err_large_bar:
1174         mutex_unlock(&p->mutex);
1175         return err;
1176 }
1177
1178 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1179                                         struct kfd_process *p, void *data)
1180 {
1181         struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1182         struct kfd_process_device *pdd;
1183         void *mem;
1184         int ret;
1185         uint64_t size = 0;
1186
1187         mutex_lock(&p->mutex);
1188         /*
1189          * Safeguard to prevent user space from freeing signal BO.
1190          * It will be freed at process termination.
1191          */
1192         if (p->signal_handle && (p->signal_handle == args->handle)) {
1193                 pr_err("Free signal BO is not allowed\n");
1194                 ret = -EPERM;
1195                 goto err_unlock;
1196         }
1197
1198         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1199         if (!pdd) {
1200                 pr_err("Process device data doesn't exist\n");
1201                 ret = -EINVAL;
1202                 goto err_pdd;
1203         }
1204
1205         mem = kfd_process_device_translate_handle(
1206                 pdd, GET_IDR_HANDLE(args->handle));
1207         if (!mem) {
1208                 ret = -EINVAL;
1209                 goto err_unlock;
1210         }
1211
1212         ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
1213                                 (struct kgd_mem *)mem, pdd->drm_priv, &size);
1214
1215         /* If freeing the buffer failed, leave the handle in place for
1216          * clean-up during process tear-down.
1217          */
1218         if (!ret)
1219                 kfd_process_device_remove_obj_handle(
1220                         pdd, GET_IDR_HANDLE(args->handle));
1221
1222         atomic64_sub(size, &pdd->vram_usage);
1223
1224 err_unlock:
1225 err_pdd:
1226         mutex_unlock(&p->mutex);
1227         return ret;
1228 }
1229
1230 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1231                                         struct kfd_process *p, void *data)
1232 {
1233         struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1234         struct kfd_process_device *pdd, *peer_pdd;
1235         void *mem;
1236         struct kfd_node *dev;
1237         long err = 0;
1238         int i;
1239         uint32_t *devices_arr = NULL;
1240
1241         if (!args->n_devices) {
1242                 pr_debug("Device IDs array empty\n");
1243                 return -EINVAL;
1244         }
1245         if (args->n_success > args->n_devices) {
1246                 pr_debug("n_success exceeds n_devices\n");
1247                 return -EINVAL;
1248         }
1249
1250         devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1251                                     GFP_KERNEL);
1252         if (!devices_arr)
1253                 return -ENOMEM;
1254
1255         err = copy_from_user(devices_arr,
1256                              (void __user *)args->device_ids_array_ptr,
1257                              args->n_devices * sizeof(*devices_arr));
1258         if (err != 0) {
1259                 err = -EFAULT;
1260                 goto copy_from_user_failed;
1261         }
1262
1263         mutex_lock(&p->mutex);
1264         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1265         if (!pdd) {
1266                 err = -EINVAL;
1267                 goto get_process_device_data_failed;
1268         }
1269         dev = pdd->dev;
1270
1271         pdd = kfd_bind_process_to_device(dev, p);
1272         if (IS_ERR(pdd)) {
1273                 err = PTR_ERR(pdd);
1274                 goto bind_process_to_device_failed;
1275         }
1276
1277         mem = kfd_process_device_translate_handle(pdd,
1278                                                 GET_IDR_HANDLE(args->handle));
1279         if (!mem) {
1280                 err = -ENOMEM;
1281                 goto get_mem_obj_from_handle_failed;
1282         }
1283
1284         for (i = args->n_success; i < args->n_devices; i++) {
1285                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1286                 if (!peer_pdd) {
1287                         pr_debug("Getting device by id failed for 0x%x\n",
1288                                  devices_arr[i]);
1289                         err = -EINVAL;
1290                         goto get_mem_obj_from_handle_failed;
1291                 }
1292
1293                 peer_pdd = kfd_bind_process_to_device(peer_pdd->dev, p);
1294                 if (IS_ERR(peer_pdd)) {
1295                         err = PTR_ERR(peer_pdd);
1296                         goto get_mem_obj_from_handle_failed;
1297                 }
1298
1299                 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1300                         peer_pdd->dev->adev, (struct kgd_mem *)mem,
1301                         peer_pdd->drm_priv);
1302                 if (err) {
1303                         struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1304
1305                         dev_err(dev->adev->dev,
1306                                "Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1307                                pci_domain_nr(pdev->bus),
1308                                pdev->bus->number,
1309                                PCI_SLOT(pdev->devfn),
1310                                PCI_FUNC(pdev->devfn),
1311                                ((struct kgd_mem *)mem)->domain);
1312                         goto map_memory_to_gpu_failed;
1313                 }
1314                 args->n_success = i+1;
1315         }
1316
1317         err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
1318         if (err) {
1319                 pr_debug("Sync memory failed, wait interrupted by user signal\n");
1320                 goto sync_memory_failed;
1321         }
1322
1323         mutex_unlock(&p->mutex);
1324
1325         /* Flush TLBs after waiting for the page table updates to complete */
1326         for (i = 0; i < args->n_devices; i++) {
1327                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1328                 if (WARN_ON_ONCE(!peer_pdd))
1329                         continue;
1330                 kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1331         }
1332         kfree(devices_arr);
1333
1334         return err;
1335
1336 get_process_device_data_failed:
1337 bind_process_to_device_failed:
1338 get_mem_obj_from_handle_failed:
1339 map_memory_to_gpu_failed:
1340 sync_memory_failed:
1341         mutex_unlock(&p->mutex);
1342 copy_from_user_failed:
1343         kfree(devices_arr);
1344
1345         return err;
1346 }
1347
1348 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1349                                         struct kfd_process *p, void *data)
1350 {
1351         struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1352         struct kfd_process_device *pdd, *peer_pdd;
1353         void *mem;
1354         long err = 0;
1355         uint32_t *devices_arr = NULL, i;
1356         bool flush_tlb;
1357
1358         if (!args->n_devices) {
1359                 pr_debug("Device IDs array empty\n");
1360                 return -EINVAL;
1361         }
1362         if (args->n_success > args->n_devices) {
1363                 pr_debug("n_success exceeds n_devices\n");
1364                 return -EINVAL;
1365         }
1366
1367         devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1368                                     GFP_KERNEL);
1369         if (!devices_arr)
1370                 return -ENOMEM;
1371
1372         err = copy_from_user(devices_arr,
1373                              (void __user *)args->device_ids_array_ptr,
1374                              args->n_devices * sizeof(*devices_arr));
1375         if (err != 0) {
1376                 err = -EFAULT;
1377                 goto copy_from_user_failed;
1378         }
1379
1380         mutex_lock(&p->mutex);
1381         pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1382         if (!pdd) {
1383                 err = -EINVAL;
1384                 goto bind_process_to_device_failed;
1385         }
1386
1387         mem = kfd_process_device_translate_handle(pdd,
1388                                                 GET_IDR_HANDLE(args->handle));
1389         if (!mem) {
1390                 err = -ENOMEM;
1391                 goto get_mem_obj_from_handle_failed;
1392         }
1393
1394         for (i = args->n_success; i < args->n_devices; i++) {
1395                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1396                 if (!peer_pdd) {
1397                         err = -EINVAL;
1398                         goto get_mem_obj_from_handle_failed;
1399                 }
1400                 err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1401                         peer_pdd->dev->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1402                 if (err) {
1403                         pr_debug("Failed to unmap from gpu %d/%d\n", i, args->n_devices);
1404                         goto unmap_memory_from_gpu_failed;
1405                 }
1406                 args->n_success = i+1;
1407         }
1408
1409         flush_tlb = kfd_flush_tlb_after_unmap(pdd->dev->kfd);
1410         if (flush_tlb) {
1411                 err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
1412                                 (struct kgd_mem *) mem, true);
1413                 if (err) {
1414                         pr_debug("Sync memory failed, wait interrupted by user signal\n");
1415                         goto sync_memory_failed;
1416                 }
1417         }
1418
1419         /* Flush TLBs after waiting for the page table updates to complete */
1420         for (i = 0; i < args->n_devices; i++) {
1421                 peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1422                 if (WARN_ON_ONCE(!peer_pdd))
1423                         continue;
1424                 if (flush_tlb)
1425                         kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1426
1427                 /* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
1428                 err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv);
1429                 if (err)
1430                         goto sync_memory_failed;
1431         }
1432
1433         mutex_unlock(&p->mutex);
1434
1435         kfree(devices_arr);
1436
1437         return 0;
1438
1439 bind_process_to_device_failed:
1440 get_mem_obj_from_handle_failed:
1441 unmap_memory_from_gpu_failed:
1442 sync_memory_failed:
1443         mutex_unlock(&p->mutex);
1444 copy_from_user_failed:
1445         kfree(devices_arr);
1446         return err;
1447 }
1448
1449 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1450                 struct kfd_process *p, void *data)
1451 {
1452         int retval;
1453         struct kfd_ioctl_alloc_queue_gws_args *args = data;
1454         struct queue *q;
1455         struct kfd_node *dev;
1456
1457         mutex_lock(&p->mutex);
1458         q = pqm_get_user_queue(&p->pqm, args->queue_id);
1459
1460         if (q) {
1461                 dev = q->device;
1462         } else {
1463                 retval = -EINVAL;
1464                 goto out_unlock;
1465         }
1466
1467         if (!dev->gws) {
1468                 retval = -ENODEV;
1469                 goto out_unlock;
1470         }
1471
1472         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1473                 retval = -ENODEV;
1474                 goto out_unlock;
1475         }
1476
1477         if (p->debug_trap_enabled && (!kfd_dbg_has_gws_support(dev) ||
1478                                       kfd_dbg_has_cwsr_workaround(dev))) {
1479                 retval = -EBUSY;
1480                 goto out_unlock;
1481         }
1482
1483         retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1484         mutex_unlock(&p->mutex);
1485
1486         args->first_gws = 0;
1487         return retval;
1488
1489 out_unlock:
1490         mutex_unlock(&p->mutex);
1491         return retval;
1492 }
1493
1494 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1495                 struct kfd_process *p, void *data)
1496 {
1497         struct kfd_ioctl_get_dmabuf_info_args *args = data;
1498         struct kfd_node *dev = NULL;
1499         struct amdgpu_device *dmabuf_adev;
1500         void *metadata_buffer = NULL;
1501         uint32_t flags;
1502         int8_t xcp_id;
1503         unsigned int i;
1504         int r;
1505
1506         /* Find a KFD GPU device that supports the get_dmabuf_info query */
1507         for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1508                 if (dev && !kfd_devcgroup_check_permission(dev))
1509                         break;
1510         if (!dev)
1511                 return -EINVAL;
1512
1513         if (args->metadata_ptr) {
1514                 metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1515                 if (!metadata_buffer)
1516                         return -ENOMEM;
1517         }
1518
1519         /* Get dmabuf info from KGD */
1520         r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
1521                                           &dmabuf_adev, &args->size,
1522                                           metadata_buffer, args->metadata_size,
1523                                           &args->metadata_size, &flags, &xcp_id);
1524         if (r)
1525                 goto exit;
1526
1527         if (xcp_id >= 0)
1528                 args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
1529         else
1530                 args->gpu_id = dev->id;
1531         args->flags = flags;
1532
1533         /* Copy metadata buffer to user mode */
1534         if (metadata_buffer) {
1535                 r = copy_to_user((void __user *)args->metadata_ptr,
1536                                  metadata_buffer, args->metadata_size);
1537                 if (r != 0)
1538                         r = -EFAULT;
1539         }
1540
1541 exit:
1542         kfree(metadata_buffer);
1543
1544         return r;
1545 }
1546
1547 static int kfd_ioctl_import_dmabuf(struct file *filep,
1548                                    struct kfd_process *p, void *data)
1549 {
1550         struct kfd_ioctl_import_dmabuf_args *args = data;
1551         struct kfd_process_device *pdd;
1552         int idr_handle;
1553         uint64_t size;
1554         void *mem;
1555         int r;
1556
1557         mutex_lock(&p->mutex);
1558         pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1559         if (!pdd) {
1560                 r = -EINVAL;
1561                 goto err_unlock;
1562         }
1563
1564         pdd = kfd_bind_process_to_device(pdd->dev, p);
1565         if (IS_ERR(pdd)) {
1566                 r = PTR_ERR(pdd);
1567                 goto err_unlock;
1568         }
1569
1570         r = amdgpu_amdkfd_gpuvm_import_dmabuf_fd(pdd->dev->adev, args->dmabuf_fd,
1571                                                  args->va_addr, pdd->drm_priv,
1572                                                  (struct kgd_mem **)&mem, &size,
1573                                                  NULL);
1574         if (r)
1575                 goto err_unlock;
1576
1577         idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1578         if (idr_handle < 0) {
1579                 r = -EFAULT;
1580                 goto err_free;
1581         }
1582
1583         mutex_unlock(&p->mutex);
1584
1585         args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1586
1587         return 0;
1588
1589 err_free:
1590         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem,
1591                                                pdd->drm_priv, NULL);
1592 err_unlock:
1593         mutex_unlock(&p->mutex);
1594         return r;
1595 }
1596
1597 static int kfd_ioctl_export_dmabuf(struct file *filep,
1598                                    struct kfd_process *p, void *data)
1599 {
1600         struct kfd_ioctl_export_dmabuf_args *args = data;
1601         struct kfd_process_device *pdd;
1602         struct dma_buf *dmabuf;
1603         struct kfd_node *dev;
1604         void *mem;
1605         int ret = 0;
1606
1607         dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1608         if (!dev)
1609                 return -EINVAL;
1610
1611         mutex_lock(&p->mutex);
1612
1613         pdd = kfd_get_process_device_data(dev, p);
1614         if (!pdd) {
1615                 ret = -EINVAL;
1616                 goto err_unlock;
1617         }
1618
1619         mem = kfd_process_device_translate_handle(pdd,
1620                                                 GET_IDR_HANDLE(args->handle));
1621         if (!mem) {
1622                 ret = -EINVAL;
1623                 goto err_unlock;
1624         }
1625
1626         ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
1627         mutex_unlock(&p->mutex);
1628         if (ret)
1629                 goto err_out;
1630
1631         ret = dma_buf_fd(dmabuf, args->flags);
1632         if (ret < 0) {
1633                 dma_buf_put(dmabuf);
1634                 goto err_out;
1635         }
1636         /* dma_buf_fd assigns the reference count to the fd, no need to
1637          * put the reference here.
1638          */
1639         args->dmabuf_fd = ret;
1640
1641         return 0;
1642
1643 err_unlock:
1644         mutex_unlock(&p->mutex);
1645 err_out:
1646         return ret;
1647 }
1648
1649 /* Handle requests for watching SMI events */
1650 static int kfd_ioctl_smi_events(struct file *filep,
1651                                 struct kfd_process *p, void *data)
1652 {
1653         struct kfd_ioctl_smi_events_args *args = data;
1654         struct kfd_process_device *pdd;
1655
1656         mutex_lock(&p->mutex);
1657
1658         pdd = kfd_process_device_data_by_id(p, args->gpuid);
1659         mutex_unlock(&p->mutex);
1660         if (!pdd)
1661                 return -EINVAL;
1662
1663         return kfd_smi_event_open(pdd->dev, &args->anon_fd);
1664 }
1665
1666 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1667
1668 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1669                                     struct kfd_process *p, void *data)
1670 {
1671         struct kfd_ioctl_set_xnack_mode_args *args = data;
1672         int r = 0;
1673
1674         mutex_lock(&p->mutex);
1675         if (args->xnack_enabled >= 0) {
1676                 if (!list_empty(&p->pqm.queues)) {
1677                         pr_debug("Process has user queues running\n");
1678                         r = -EBUSY;
1679                         goto out_unlock;
1680                 }
1681
1682                 if (p->xnack_enabled == args->xnack_enabled)
1683                         goto out_unlock;
1684
1685                 if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
1686                         r = -EPERM;
1687                         goto out_unlock;
1688                 }
1689
1690                 r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
1691         } else {
1692                 args->xnack_enabled = p->xnack_enabled;
1693         }
1694
1695 out_unlock:
1696         mutex_unlock(&p->mutex);
1697
1698         return r;
1699 }
1700
1701 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1702 {
1703         struct kfd_ioctl_svm_args *args = data;
1704         int r = 0;
1705
1706         pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1707                  args->start_addr, args->size, args->op, args->nattr);
1708
1709         if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1710                 return -EINVAL;
1711         if (!args->start_addr || !args->size)
1712                 return -EINVAL;
1713
1714         r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1715                       args->attrs);
1716
1717         return r;
1718 }
1719 #else
1720 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1721                                     struct kfd_process *p, void *data)
1722 {
1723         return -EPERM;
1724 }
1725 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1726 {
1727         return -EPERM;
1728 }
1729 #endif
1730
1731 static int criu_checkpoint_process(struct kfd_process *p,
1732                              uint8_t __user *user_priv_data,
1733                              uint64_t *priv_offset)
1734 {
1735         struct kfd_criu_process_priv_data process_priv;
1736         int ret;
1737
1738         memset(&process_priv, 0, sizeof(process_priv));
1739
1740         process_priv.version = KFD_CRIU_PRIV_VERSION;
1741         /* For CR, we don't consider negative xnack mode which is used for
1742          * querying without changing it, here 0 simply means disabled and 1
1743          * means enabled so retry for finding a valid PTE.
1744          */
1745         process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1746
1747         ret = copy_to_user(user_priv_data + *priv_offset,
1748                                 &process_priv, sizeof(process_priv));
1749
1750         if (ret) {
1751                 pr_err("Failed to copy process information to user\n");
1752                 ret = -EFAULT;
1753         }
1754
1755         *priv_offset += sizeof(process_priv);
1756         return ret;
1757 }
1758
1759 static int criu_checkpoint_devices(struct kfd_process *p,
1760                              uint32_t num_devices,
1761                              uint8_t __user *user_addr,
1762                              uint8_t __user *user_priv_data,
1763                              uint64_t *priv_offset)
1764 {
1765         struct kfd_criu_device_priv_data *device_priv = NULL;
1766         struct kfd_criu_device_bucket *device_buckets = NULL;
1767         int ret = 0, i;
1768
1769         device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1770         if (!device_buckets) {
1771                 ret = -ENOMEM;
1772                 goto exit;
1773         }
1774
1775         device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1776         if (!device_priv) {
1777                 ret = -ENOMEM;
1778                 goto exit;
1779         }
1780
1781         for (i = 0; i < num_devices; i++) {
1782                 struct kfd_process_device *pdd = p->pdds[i];
1783
1784                 device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1785                 device_buckets[i].actual_gpu_id = pdd->dev->id;
1786
1787                 /*
1788                  * priv_data does not contain useful information for now and is reserved for
1789                  * future use, so we do not set its contents.
1790                  */
1791         }
1792
1793         ret = copy_to_user(user_addr, device_buckets, num_devices * sizeof(*device_buckets));
1794         if (ret) {
1795                 pr_err("Failed to copy device information to user\n");
1796                 ret = -EFAULT;
1797                 goto exit;
1798         }
1799
1800         ret = copy_to_user(user_priv_data + *priv_offset,
1801                            device_priv,
1802                            num_devices * sizeof(*device_priv));
1803         if (ret) {
1804                 pr_err("Failed to copy device information to user\n");
1805                 ret = -EFAULT;
1806         }
1807         *priv_offset += num_devices * sizeof(*device_priv);
1808
1809 exit:
1810         kvfree(device_buckets);
1811         kvfree(device_priv);
1812         return ret;
1813 }
1814
1815 static uint32_t get_process_num_bos(struct kfd_process *p)
1816 {
1817         uint32_t num_of_bos = 0;
1818         int i;
1819
1820         /* Run over all PDDs of the process */
1821         for (i = 0; i < p->n_pdds; i++) {
1822                 struct kfd_process_device *pdd = p->pdds[i];
1823                 void *mem;
1824                 int id;
1825
1826                 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1827                         struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1828
1829                         if (!kgd_mem->va || kgd_mem->va > pdd->gpuvm_base)
1830                                 num_of_bos++;
1831                 }
1832         }
1833         return num_of_bos;
1834 }
1835
1836 static int criu_get_prime_handle(struct kgd_mem *mem,
1837                                  int flags, u32 *shared_fd,
1838                                  struct file **file)
1839 {
1840         struct dma_buf *dmabuf;
1841         int ret;
1842
1843         ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
1844         if (ret) {
1845                 pr_err("dmabuf export failed for the BO\n");
1846                 return ret;
1847         }
1848
1849         ret = get_unused_fd_flags(flags);
1850         if (ret < 0) {
1851                 pr_err("dmabuf create fd failed, ret:%d\n", ret);
1852                 goto out_free_dmabuf;
1853         }
1854
1855         *shared_fd = ret;
1856         *file = dmabuf->file;
1857         return 0;
1858
1859 out_free_dmabuf:
1860         dma_buf_put(dmabuf);
1861         return ret;
1862 }
1863
1864 static void commit_files(struct file **files,
1865                          struct kfd_criu_bo_bucket *bo_buckets,
1866                          unsigned int count,
1867                          int err)
1868 {
1869         while (count--) {
1870                 struct file *file = files[count];
1871
1872                 if (!file)
1873                         continue;
1874                 if (err) {
1875                         fput(file);
1876                         put_unused_fd(bo_buckets[count].dmabuf_fd);
1877                 } else {
1878                         fd_install(bo_buckets[count].dmabuf_fd, file);
1879                 }
1880         }
1881 }
1882
1883 static int criu_checkpoint_bos(struct kfd_process *p,
1884                                uint32_t num_bos,
1885                                uint8_t __user *user_bos,
1886                                uint8_t __user *user_priv_data,
1887                                uint64_t *priv_offset)
1888 {
1889         struct kfd_criu_bo_bucket *bo_buckets;
1890         struct kfd_criu_bo_priv_data *bo_privs;
1891         struct file **files = NULL;
1892         int ret = 0, pdd_index, bo_index = 0, id;
1893         void *mem;
1894
1895         bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1896         if (!bo_buckets)
1897                 return -ENOMEM;
1898
1899         bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1900         if (!bo_privs) {
1901                 ret = -ENOMEM;
1902                 goto exit;
1903         }
1904
1905         files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL);
1906         if (!files) {
1907                 ret = -ENOMEM;
1908                 goto exit;
1909         }
1910
1911         for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1912                 struct kfd_process_device *pdd = p->pdds[pdd_index];
1913                 struct amdgpu_bo *dumper_bo;
1914                 struct kgd_mem *kgd_mem;
1915
1916                 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1917                         struct kfd_criu_bo_bucket *bo_bucket;
1918                         struct kfd_criu_bo_priv_data *bo_priv;
1919                         int i, dev_idx = 0;
1920
1921                         kgd_mem = (struct kgd_mem *)mem;
1922                         dumper_bo = kgd_mem->bo;
1923
1924                         /* Skip checkpointing BOs that are used for Trap handler
1925                          * code and state. Currently, these BOs have a VA that
1926                          * is less GPUVM Base
1927                          */
1928                         if (kgd_mem->va && kgd_mem->va <= pdd->gpuvm_base)
1929                                 continue;
1930
1931                         bo_bucket = &bo_buckets[bo_index];
1932                         bo_priv = &bo_privs[bo_index];
1933
1934                         bo_bucket->gpu_id = pdd->user_gpu_id;
1935                         bo_bucket->addr = (uint64_t)kgd_mem->va;
1936                         bo_bucket->size = amdgpu_bo_size(dumper_bo);
1937                         bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1938                         bo_priv->idr_handle = id;
1939
1940                         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1941                                 ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
1942                                                                 &bo_priv->user_addr);
1943                                 if (ret) {
1944                                         pr_err("Failed to obtain user address for user-pointer bo\n");
1945                                         goto exit;
1946                                 }
1947                         }
1948                         if (bo_bucket->alloc_flags
1949                             & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1950                                 ret = criu_get_prime_handle(kgd_mem,
1951                                                 bo_bucket->alloc_flags &
1952                                                 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1953                                                 &bo_bucket->dmabuf_fd, &files[bo_index]);
1954                                 if (ret)
1955                                         goto exit;
1956                         } else {
1957                                 bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1958                         }
1959
1960                         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1961                                 bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1962                                         KFD_MMAP_GPU_ID(pdd->dev->id);
1963                         else if (bo_bucket->alloc_flags &
1964                                 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1965                                 bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1966                                         KFD_MMAP_GPU_ID(pdd->dev->id);
1967                         else
1968                                 bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
1969
1970                         for (i = 0; i < p->n_pdds; i++) {
1971                                 if (amdgpu_amdkfd_bo_mapped_to_dev(p->pdds[i]->drm_priv, kgd_mem))
1972                                         bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1973                         }
1974
1975                         pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1976                                         "gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1977                                         bo_bucket->size,
1978                                         bo_bucket->addr,
1979                                         bo_bucket->offset,
1980                                         bo_bucket->gpu_id,
1981                                         bo_bucket->alloc_flags,
1982                                         bo_priv->idr_handle);
1983                         bo_index++;
1984                 }
1985         }
1986
1987         ret = copy_to_user(user_bos, bo_buckets, num_bos * sizeof(*bo_buckets));
1988         if (ret) {
1989                 pr_err("Failed to copy BO information to user\n");
1990                 ret = -EFAULT;
1991                 goto exit;
1992         }
1993
1994         ret = copy_to_user(user_priv_data + *priv_offset, bo_privs, num_bos * sizeof(*bo_privs));
1995         if (ret) {
1996                 pr_err("Failed to copy BO priv information to user\n");
1997                 ret = -EFAULT;
1998                 goto exit;
1999         }
2000
2001         *priv_offset += num_bos * sizeof(*bo_privs);
2002
2003 exit:
2004         commit_files(files, bo_buckets, bo_index, ret);
2005         kvfree(files);
2006         kvfree(bo_buckets);
2007         kvfree(bo_privs);
2008         return ret;
2009 }
2010
2011 static int criu_get_process_object_info(struct kfd_process *p,
2012                                         uint32_t *num_devices,
2013                                         uint32_t *num_bos,
2014                                         uint32_t *num_objects,
2015                                         uint64_t *objs_priv_size)
2016 {
2017         uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
2018         uint32_t num_queues, num_events, num_svm_ranges;
2019         int ret;
2020
2021         *num_devices = p->n_pdds;
2022         *num_bos = get_process_num_bos(p);
2023
2024         ret = kfd_process_get_queue_info(p, &num_queues, &queues_priv_data_size);
2025         if (ret)
2026                 return ret;
2027
2028         num_events = kfd_get_num_events(p);
2029
2030         ret = svm_range_get_info(p, &num_svm_ranges, &svm_priv_data_size);
2031         if (ret)
2032                 return ret;
2033
2034         *num_objects = num_queues + num_events + num_svm_ranges;
2035
2036         if (objs_priv_size) {
2037                 priv_size = sizeof(struct kfd_criu_process_priv_data);
2038                 priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
2039                 priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
2040                 priv_size += queues_priv_data_size;
2041                 priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
2042                 priv_size += svm_priv_data_size;
2043                 *objs_priv_size = priv_size;
2044         }
2045         return 0;
2046 }
2047
2048 static int criu_checkpoint(struct file *filep,
2049                            struct kfd_process *p,
2050                            struct kfd_ioctl_criu_args *args)
2051 {
2052         int ret;
2053         uint32_t num_devices, num_bos, num_objects;
2054         uint64_t priv_size, priv_offset = 0, bo_priv_offset;
2055
2056         if (!args->devices || !args->bos || !args->priv_data)
2057                 return -EINVAL;
2058
2059         mutex_lock(&p->mutex);
2060
2061         if (!p->n_pdds) {
2062                 pr_err("No pdd for given process\n");
2063                 ret = -ENODEV;
2064                 goto exit_unlock;
2065         }
2066
2067         /* Confirm all process queues are evicted */
2068         if (!p->queues_paused) {
2069                 pr_err("Cannot dump process when queues are not in evicted state\n");
2070                 /* CRIU plugin did not call op PROCESS_INFO before checkpointing */
2071                 ret = -EINVAL;
2072                 goto exit_unlock;
2073         }
2074
2075         ret = criu_get_process_object_info(p, &num_devices, &num_bos, &num_objects, &priv_size);
2076         if (ret)
2077                 goto exit_unlock;
2078
2079         if (num_devices != args->num_devices ||
2080             num_bos != args->num_bos ||
2081             num_objects != args->num_objects ||
2082             priv_size != args->priv_data_size) {
2083
2084                 ret = -EINVAL;
2085                 goto exit_unlock;
2086         }
2087
2088         /* each function will store private data inside priv_data and adjust priv_offset */
2089         ret = criu_checkpoint_process(p, (uint8_t __user *)args->priv_data, &priv_offset);
2090         if (ret)
2091                 goto exit_unlock;
2092
2093         ret = criu_checkpoint_devices(p, num_devices, (uint8_t __user *)args->devices,
2094                                 (uint8_t __user *)args->priv_data, &priv_offset);
2095         if (ret)
2096                 goto exit_unlock;
2097
2098         /* Leave room for BOs in the private data. They need to be restored
2099          * before events, but we checkpoint them last to simplify the error
2100          * handling.
2101          */
2102         bo_priv_offset = priv_offset;
2103         priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2104
2105         if (num_objects) {
2106                 ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data,
2107                                                  &priv_offset);
2108                 if (ret)
2109                         goto exit_unlock;
2110
2111                 ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
2112                                                  &priv_offset);
2113                 if (ret)
2114                         goto exit_unlock;
2115
2116                 ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset);
2117                 if (ret)
2118                         goto exit_unlock;
2119         }
2120
2121         /* This must be the last thing in this function that can fail.
2122          * Otherwise we leak dmabuf file descriptors.
2123          */
2124         ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos,
2125                            (uint8_t __user *)args->priv_data, &bo_priv_offset);
2126
2127 exit_unlock:
2128         mutex_unlock(&p->mutex);
2129         if (ret)
2130                 pr_err("Failed to dump CRIU ret:%d\n", ret);
2131         else
2132                 pr_debug("CRIU dump ret:%d\n", ret);
2133
2134         return ret;
2135 }
2136
2137 static int criu_restore_process(struct kfd_process *p,
2138                                 struct kfd_ioctl_criu_args *args,
2139                                 uint64_t *priv_offset,
2140                                 uint64_t max_priv_data_size)
2141 {
2142         int ret = 0;
2143         struct kfd_criu_process_priv_data process_priv;
2144
2145         if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2146                 return -EINVAL;
2147
2148         ret = copy_from_user(&process_priv,
2149                                 (void __user *)(args->priv_data + *priv_offset),
2150                                 sizeof(process_priv));
2151         if (ret) {
2152                 pr_err("Failed to copy process private information from user\n");
2153                 ret = -EFAULT;
2154                 goto exit;
2155         }
2156         *priv_offset += sizeof(process_priv);
2157
2158         if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2159                 pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2160                         process_priv.version, KFD_CRIU_PRIV_VERSION);
2161                 return -EINVAL;
2162         }
2163
2164         pr_debug("Setting XNACK mode\n");
2165         if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, true)) {
2166                 pr_err("xnack mode cannot be set\n");
2167                 ret = -EPERM;
2168                 goto exit;
2169         } else {
2170                 pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2171                 p->xnack_enabled = process_priv.xnack_mode;
2172         }
2173
2174 exit:
2175         return ret;
2176 }
2177
2178 static int criu_restore_devices(struct kfd_process *p,
2179                                 struct kfd_ioctl_criu_args *args,
2180                                 uint64_t *priv_offset,
2181                                 uint64_t max_priv_data_size)
2182 {
2183         struct kfd_criu_device_bucket *device_buckets;
2184         struct kfd_criu_device_priv_data *device_privs;
2185         int ret = 0;
2186         uint32_t i;
2187
2188         if (args->num_devices != p->n_pdds)
2189                 return -EINVAL;
2190
2191         if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2192                 return -EINVAL;
2193
2194         device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2195         if (!device_buckets)
2196                 return -ENOMEM;
2197
2198         ret = copy_from_user(device_buckets, (void __user *)args->devices,
2199                                 args->num_devices * sizeof(*device_buckets));
2200         if (ret) {
2201                 pr_err("Failed to copy devices buckets from user\n");
2202                 ret = -EFAULT;
2203                 goto exit;
2204         }
2205
2206         for (i = 0; i < args->num_devices; i++) {
2207                 struct kfd_node *dev;
2208                 struct kfd_process_device *pdd;
2209                 struct file *drm_file;
2210
2211                 /* device private data is not currently used */
2212
2213                 if (!device_buckets[i].user_gpu_id) {
2214                         pr_err("Invalid user gpu_id\n");
2215                         ret = -EINVAL;
2216                         goto exit;
2217                 }
2218
2219                 dev = kfd_device_by_id(device_buckets[i].actual_gpu_id);
2220                 if (!dev) {
2221                         pr_err("Failed to find device with gpu_id = %x\n",
2222                                 device_buckets[i].actual_gpu_id);
2223                         ret = -EINVAL;
2224                         goto exit;
2225                 }
2226
2227                 pdd = kfd_get_process_device_data(dev, p);
2228                 if (!pdd) {
2229                         pr_err("Failed to get pdd for gpu_id = %x\n",
2230                                         device_buckets[i].actual_gpu_id);
2231                         ret = -EINVAL;
2232                         goto exit;
2233                 }
2234                 pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2235
2236                 drm_file = fget(device_buckets[i].drm_fd);
2237                 if (!drm_file) {
2238                         pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2239                                 device_buckets[i].drm_fd);
2240                         ret = -EINVAL;
2241                         goto exit;
2242                 }
2243
2244                 if (pdd->drm_file) {
2245                         ret = -EINVAL;
2246                         goto exit;
2247                 }
2248
2249                 /* create the vm using render nodes for kfd pdd */
2250                 if (kfd_process_device_init_vm(pdd, drm_file)) {
2251                         pr_err("could not init vm for given pdd\n");
2252                         /* On success, the PDD keeps the drm_file reference */
2253                         fput(drm_file);
2254                         ret = -EINVAL;
2255                         goto exit;
2256                 }
2257                 /*
2258                  * pdd now already has the vm bound to render node so below api won't create a new
2259                  * exclusive kfd mapping but use existing one with renderDXXX but is still needed
2260                  * for iommu v2 binding  and runtime pm.
2261                  */
2262                 pdd = kfd_bind_process_to_device(dev, p);
2263                 if (IS_ERR(pdd)) {
2264                         ret = PTR_ERR(pdd);
2265                         goto exit;
2266                 }
2267
2268                 if (!pdd->qpd.proc_doorbells) {
2269                         ret = kfd_alloc_process_doorbells(dev->kfd, pdd);
2270                         if (ret)
2271                                 goto exit;
2272                 }
2273         }
2274
2275         /*
2276          * We are not copying device private data from user as we are not using the data for now,
2277          * but we still adjust for its private data.
2278          */
2279         *priv_offset += args->num_devices * sizeof(*device_privs);
2280
2281 exit:
2282         kfree(device_buckets);
2283         return ret;
2284 }
2285
2286 static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2287                                       struct kfd_criu_bo_bucket *bo_bucket,
2288                                       struct kfd_criu_bo_priv_data *bo_priv,
2289                                       struct kgd_mem **kgd_mem)
2290 {
2291         int idr_handle;
2292         int ret;
2293         const bool criu_resume = true;
2294         u64 offset;
2295
2296         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2297                 if (bo_bucket->size !=
2298                                 kfd_doorbell_process_slice(pdd->dev->kfd))
2299                         return -EINVAL;
2300
2301                 offset = kfd_get_process_doorbells(pdd);
2302                 if (!offset)
2303                         return -ENOMEM;
2304         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2305                 /* MMIO BOs need remapped bus address */
2306                 if (bo_bucket->size != PAGE_SIZE) {
2307                         pr_err("Invalid page size\n");
2308                         return -EINVAL;
2309                 }
2310                 offset = pdd->dev->adev->rmmio_remap.bus_addr;
2311                 if (!offset || (PAGE_SIZE > 4096)) {
2312                         pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2313                         return -ENOMEM;
2314                 }
2315         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2316                 offset = bo_priv->user_addr;
2317         }
2318         /* Create the BO */
2319         ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(pdd->dev->adev, bo_bucket->addr,
2320                                                       bo_bucket->size, pdd->drm_priv, kgd_mem,
2321                                                       &offset, bo_bucket->alloc_flags, criu_resume);
2322         if (ret) {
2323                 pr_err("Could not create the BO\n");
2324                 return ret;
2325         }
2326         pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2327                  bo_bucket->size, bo_bucket->addr, offset);
2328
2329         /* Restore previous IDR handle */
2330         pr_debug("Restoring old IDR handle for the BO");
2331         idr_handle = idr_alloc(&pdd->alloc_idr, *kgd_mem, bo_priv->idr_handle,
2332                                bo_priv->idr_handle + 1, GFP_KERNEL);
2333
2334         if (idr_handle < 0) {
2335                 pr_err("Could not allocate idr\n");
2336                 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, *kgd_mem, pdd->drm_priv,
2337                                                        NULL);
2338                 return -ENOMEM;
2339         }
2340
2341         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2342                 bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2343         if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2344                 bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2345         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2346                 bo_bucket->restored_offset = offset;
2347         } else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2348                 bo_bucket->restored_offset = offset;
2349                 /* Update the VRAM usage count */
2350                 atomic64_add(bo_bucket->size, &pdd->vram_usage);
2351         }
2352         return 0;
2353 }
2354
2355 static int criu_restore_bo(struct kfd_process *p,
2356                            struct kfd_criu_bo_bucket *bo_bucket,
2357                            struct kfd_criu_bo_priv_data *bo_priv,
2358                            struct file **file)
2359 {
2360         struct kfd_process_device *pdd;
2361         struct kgd_mem *kgd_mem;
2362         int ret;
2363         int j;
2364
2365         pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2366                  bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2367                  bo_priv->idr_handle);
2368
2369         pdd = kfd_process_device_data_by_id(p, bo_bucket->gpu_id);
2370         if (!pdd) {
2371                 pr_err("Failed to get pdd\n");
2372                 return -ENODEV;
2373         }
2374
2375         ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, &kgd_mem);
2376         if (ret)
2377                 return ret;
2378
2379         /* now map these BOs to GPU/s */
2380         for (j = 0; j < p->n_pdds; j++) {
2381                 struct kfd_node *peer;
2382                 struct kfd_process_device *peer_pdd;
2383
2384                 if (!bo_priv->mapped_gpuids[j])
2385                         break;
2386
2387                 peer_pdd = kfd_process_device_data_by_id(p, bo_priv->mapped_gpuids[j]);
2388                 if (!peer_pdd)
2389                         return -EINVAL;
2390
2391                 peer = peer_pdd->dev;
2392
2393                 peer_pdd = kfd_bind_process_to_device(peer, p);
2394                 if (IS_ERR(peer_pdd))
2395                         return PTR_ERR(peer_pdd);
2396
2397                 ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
2398                                                             peer_pdd->drm_priv);
2399                 if (ret) {
2400                         pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2401                         return ret;
2402                 }
2403         }
2404
2405         pr_debug("map memory was successful for the BO\n");
2406         /* create the dmabuf object and export the bo */
2407         if (bo_bucket->alloc_flags
2408             & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2409                 ret = criu_get_prime_handle(kgd_mem, DRM_RDWR,
2410                                             &bo_bucket->dmabuf_fd, file);
2411                 if (ret)
2412                         return ret;
2413         } else {
2414                 bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2415         }
2416
2417         return 0;
2418 }
2419
2420 static int criu_restore_bos(struct kfd_process *p,
2421                             struct kfd_ioctl_criu_args *args,
2422                             uint64_t *priv_offset,
2423                             uint64_t max_priv_data_size)
2424 {
2425         struct kfd_criu_bo_bucket *bo_buckets = NULL;
2426         struct kfd_criu_bo_priv_data *bo_privs = NULL;
2427         struct file **files = NULL;
2428         int ret = 0;
2429         uint32_t i = 0;
2430
2431         if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2432                 return -EINVAL;
2433
2434         /* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2435         amdgpu_amdkfd_block_mmu_notifications(p->kgd_process_info);
2436
2437         bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2438         if (!bo_buckets)
2439                 return -ENOMEM;
2440
2441         files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL);
2442         if (!files) {
2443                 ret = -ENOMEM;
2444                 goto exit;
2445         }
2446
2447         ret = copy_from_user(bo_buckets, (void __user *)args->bos,
2448                              args->num_bos * sizeof(*bo_buckets));
2449         if (ret) {
2450                 pr_err("Failed to copy BOs information from user\n");
2451                 ret = -EFAULT;
2452                 goto exit;
2453         }
2454
2455         bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2456         if (!bo_privs) {
2457                 ret = -ENOMEM;
2458                 goto exit;
2459         }
2460
2461         ret = copy_from_user(bo_privs, (void __user *)args->priv_data + *priv_offset,
2462                              args->num_bos * sizeof(*bo_privs));
2463         if (ret) {
2464                 pr_err("Failed to copy BOs information from user\n");
2465                 ret = -EFAULT;
2466                 goto exit;
2467         }
2468         *priv_offset += args->num_bos * sizeof(*bo_privs);
2469
2470         /* Create and map new BOs */
2471         for (; i < args->num_bos; i++) {
2472                 ret = criu_restore_bo(p, &bo_buckets[i], &bo_privs[i], &files[i]);
2473                 if (ret) {
2474                         pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2475                         goto exit;
2476                 }
2477         } /* done */
2478
2479         /* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2480         ret = copy_to_user((void __user *)args->bos,
2481                                 bo_buckets,
2482                                 (args->num_bos * sizeof(*bo_buckets)));
2483         if (ret)
2484                 ret = -EFAULT;
2485
2486 exit:
2487         commit_files(files, bo_buckets, i, ret);
2488         kvfree(files);
2489         kvfree(bo_buckets);
2490         kvfree(bo_privs);
2491         return ret;
2492 }
2493
2494 static int criu_restore_objects(struct file *filep,
2495                                 struct kfd_process *p,
2496                                 struct kfd_ioctl_criu_args *args,
2497                                 uint64_t *priv_offset,
2498                                 uint64_t max_priv_data_size)
2499 {
2500         int ret = 0;
2501         uint32_t i;
2502
2503         BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2504         BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2505         BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2506
2507         for (i = 0; i < args->num_objects; i++) {
2508                 uint32_t object_type;
2509
2510                 if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2511                         pr_err("Invalid private data size\n");
2512                         return -EINVAL;
2513                 }
2514
2515                 ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2516                 if (ret) {
2517                         pr_err("Failed to copy private information from user\n");
2518                         goto exit;
2519                 }
2520
2521                 switch (object_type) {
2522                 case KFD_CRIU_OBJECT_TYPE_QUEUE:
2523                         ret = kfd_criu_restore_queue(p, (uint8_t __user *)args->priv_data,
2524                                                      priv_offset, max_priv_data_size);
2525                         if (ret)
2526                                 goto exit;
2527                         break;
2528                 case KFD_CRIU_OBJECT_TYPE_EVENT:
2529                         ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
2530                                                      priv_offset, max_priv_data_size);
2531                         if (ret)
2532                                 goto exit;
2533                         break;
2534                 case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2535                         ret = kfd_criu_restore_svm(p, (uint8_t __user *)args->priv_data,
2536                                                      priv_offset, max_priv_data_size);
2537                         if (ret)
2538                                 goto exit;
2539                         break;
2540                 default:
2541                         pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2542                         ret = -EINVAL;
2543                         goto exit;
2544                 }
2545         }
2546 exit:
2547         return ret;
2548 }
2549
2550 static int criu_restore(struct file *filep,
2551                         struct kfd_process *p,
2552                         struct kfd_ioctl_criu_args *args)
2553 {
2554         uint64_t priv_offset = 0;
2555         int ret = 0;
2556
2557         pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2558                  args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2559
2560         if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size ||
2561             !args->num_devices || !args->num_bos)
2562                 return -EINVAL;
2563
2564         mutex_lock(&p->mutex);
2565
2566         /*
2567          * Set the process to evicted state to avoid running any new queues before all the memory
2568          * mappings are ready.
2569          */
2570         ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_RESTORE);
2571         if (ret)
2572                 goto exit_unlock;
2573
2574         /* Each function will adjust priv_offset based on how many bytes they consumed */
2575         ret = criu_restore_process(p, args, &priv_offset, args->priv_data_size);
2576         if (ret)
2577                 goto exit_unlock;
2578
2579         ret = criu_restore_devices(p, args, &priv_offset, args->priv_data_size);
2580         if (ret)
2581                 goto exit_unlock;
2582
2583         ret = criu_restore_bos(p, args, &priv_offset, args->priv_data_size);
2584         if (ret)
2585                 goto exit_unlock;
2586
2587         ret = criu_restore_objects(filep, p, args, &priv_offset, args->priv_data_size);
2588         if (ret)
2589                 goto exit_unlock;
2590
2591         if (priv_offset != args->priv_data_size) {
2592                 pr_err("Invalid private data size\n");
2593                 ret = -EINVAL;
2594         }
2595
2596 exit_unlock:
2597         mutex_unlock(&p->mutex);
2598         if (ret)
2599                 pr_err("Failed to restore CRIU ret:%d\n", ret);
2600         else
2601                 pr_debug("CRIU restore successful\n");
2602
2603         return ret;
2604 }
2605
2606 static int criu_unpause(struct file *filep,
2607                         struct kfd_process *p,
2608                         struct kfd_ioctl_criu_args *args)
2609 {
2610         int ret;
2611
2612         mutex_lock(&p->mutex);
2613
2614         if (!p->queues_paused) {
2615                 mutex_unlock(&p->mutex);
2616                 return -EINVAL;
2617         }
2618
2619         ret = kfd_process_restore_queues(p);
2620         if (ret)
2621                 pr_err("Failed to unpause queues ret:%d\n", ret);
2622         else
2623                 p->queues_paused = false;
2624
2625         mutex_unlock(&p->mutex);
2626
2627         return ret;
2628 }
2629
2630 static int criu_resume(struct file *filep,
2631                         struct kfd_process *p,
2632                         struct kfd_ioctl_criu_args *args)
2633 {
2634         struct kfd_process *target = NULL;
2635         struct pid *pid = NULL;
2636         int ret = 0;
2637
2638         pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2639                  args->pid);
2640
2641         pid = find_get_pid(args->pid);
2642         if (!pid) {
2643                 pr_err("Cannot find pid info for %i\n", args->pid);
2644                 return -ESRCH;
2645         }
2646
2647         pr_debug("calling kfd_lookup_process_by_pid\n");
2648         target = kfd_lookup_process_by_pid(pid);
2649
2650         put_pid(pid);
2651
2652         if (!target) {
2653                 pr_debug("Cannot find process info for %i\n", args->pid);
2654                 return -ESRCH;
2655         }
2656
2657         mutex_lock(&target->mutex);
2658         ret = kfd_criu_resume_svm(target);
2659         if (ret) {
2660                 pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2661                 goto exit;
2662         }
2663
2664         ret =  amdgpu_amdkfd_criu_resume(target->kgd_process_info);
2665         if (ret)
2666                 pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2667
2668 exit:
2669         mutex_unlock(&target->mutex);
2670
2671         kfd_unref_process(target);
2672         return ret;
2673 }
2674
2675 static int criu_process_info(struct file *filep,
2676                                 struct kfd_process *p,
2677                                 struct kfd_ioctl_criu_args *args)
2678 {
2679         int ret = 0;
2680
2681         mutex_lock(&p->mutex);
2682
2683         if (!p->n_pdds) {
2684                 pr_err("No pdd for given process\n");
2685                 ret = -ENODEV;
2686                 goto err_unlock;
2687         }
2688
2689         ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2690         if (ret)
2691                 goto err_unlock;
2692
2693         p->queues_paused = true;
2694
2695         args->pid = task_pid_nr_ns(p->lead_thread,
2696                                         task_active_pid_ns(p->lead_thread));
2697
2698         ret = criu_get_process_object_info(p, &args->num_devices, &args->num_bos,
2699                                            &args->num_objects, &args->priv_data_size);
2700         if (ret)
2701                 goto err_unlock;
2702
2703         dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2704                                 args->num_devices, args->num_bos, args->num_objects,
2705                                 args->priv_data_size);
2706
2707 err_unlock:
2708         if (ret) {
2709                 kfd_process_restore_queues(p);
2710                 p->queues_paused = false;
2711         }
2712         mutex_unlock(&p->mutex);
2713         return ret;
2714 }
2715
2716 static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2717 {
2718         struct kfd_ioctl_criu_args *args = data;
2719         int ret;
2720
2721         dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2722         switch (args->op) {
2723         case KFD_CRIU_OP_PROCESS_INFO:
2724                 ret = criu_process_info(filep, p, args);
2725                 break;
2726         case KFD_CRIU_OP_CHECKPOINT:
2727                 ret = criu_checkpoint(filep, p, args);
2728                 break;
2729         case KFD_CRIU_OP_UNPAUSE:
2730                 ret = criu_unpause(filep, p, args);
2731                 break;
2732         case KFD_CRIU_OP_RESTORE:
2733                 ret = criu_restore(filep, p, args);
2734                 break;
2735         case KFD_CRIU_OP_RESUME:
2736                 ret = criu_resume(filep, p, args);
2737                 break;
2738         default:
2739                 dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2740                 ret = -EINVAL;
2741                 break;
2742         }
2743
2744         if (ret)
2745                 dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2746
2747         return ret;
2748 }
2749
2750 static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
2751                         bool enable_ttmp_setup)
2752 {
2753         int i = 0, ret = 0;
2754
2755         if (p->is_runtime_retry)
2756                 goto retry;
2757
2758         if (p->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
2759                 return -EBUSY;
2760
2761         for (i = 0; i < p->n_pdds; i++) {
2762                 struct kfd_process_device *pdd = p->pdds[i];
2763
2764                 if (pdd->qpd.queue_count)
2765                         return -EEXIST;
2766
2767                 /*
2768                  * Setup TTMPs by default.
2769                  * Note that this call must remain here for MES ADD QUEUE to
2770                  * skip_process_ctx_clear unconditionally as the first call to
2771                  * SET_SHADER_DEBUGGER clears any stale process context data
2772                  * saved in MES.
2773                  */
2774                 if (pdd->dev->kfd->shared_resources.enable_mes)
2775                         kfd_dbg_set_mes_debug_mode(pdd, !kfd_dbg_has_cwsr_workaround(pdd->dev));
2776         }
2777
2778         p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
2779         p->runtime_info.r_debug = r_debug;
2780         p->runtime_info.ttmp_setup = enable_ttmp_setup;
2781
2782         if (p->runtime_info.ttmp_setup) {
2783                 for (i = 0; i < p->n_pdds; i++) {
2784                         struct kfd_process_device *pdd = p->pdds[i];
2785
2786                         if (!kfd_dbg_is_rlc_restore_supported(pdd->dev)) {
2787                                 amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
2788                                 pdd->dev->kfd2kgd->enable_debug_trap(
2789                                                 pdd->dev->adev,
2790                                                 true,
2791                                                 pdd->dev->vm_info.last_vmid_kfd);
2792                         } else if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2793                                 pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
2794                                                 pdd->dev->adev,
2795                                                 false,
2796                                                 0);
2797                         }
2798                 }
2799         }
2800
2801 retry:
2802         if (p->debug_trap_enabled) {
2803                 if (!p->is_runtime_retry) {
2804                         kfd_dbg_trap_activate(p);
2805                         kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2806                                         p, NULL, 0, false, NULL, 0);
2807                 }
2808
2809                 mutex_unlock(&p->mutex);
2810                 ret = down_interruptible(&p->runtime_enable_sema);
2811                 mutex_lock(&p->mutex);
2812
2813                 p->is_runtime_retry = !!ret;
2814         }
2815
2816         return ret;
2817 }
2818
2819 static int runtime_disable(struct kfd_process *p)
2820 {
2821         int i = 0, ret;
2822         bool was_enabled = p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED;
2823
2824         p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_DISABLED;
2825         p->runtime_info.r_debug = 0;
2826
2827         if (p->debug_trap_enabled) {
2828                 if (was_enabled)
2829                         kfd_dbg_trap_deactivate(p, false, 0);
2830
2831                 if (!p->is_runtime_retry)
2832                         kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2833                                         p, NULL, 0, false, NULL, 0);
2834
2835                 mutex_unlock(&p->mutex);
2836                 ret = down_interruptible(&p->runtime_enable_sema);
2837                 mutex_lock(&p->mutex);
2838
2839                 p->is_runtime_retry = !!ret;
2840                 if (ret)
2841                         return ret;
2842         }
2843
2844         if (was_enabled && p->runtime_info.ttmp_setup) {
2845                 for (i = 0; i < p->n_pdds; i++) {
2846                         struct kfd_process_device *pdd = p->pdds[i];
2847
2848                         if (!kfd_dbg_is_rlc_restore_supported(pdd->dev))
2849                                 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
2850                 }
2851         }
2852
2853         p->runtime_info.ttmp_setup = false;
2854
2855         /* disable ttmp setup */
2856         for (i = 0; i < p->n_pdds; i++) {
2857                 struct kfd_process_device *pdd = p->pdds[i];
2858
2859                 if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2860                         pdd->spi_dbg_override =
2861                                         pdd->dev->kfd2kgd->disable_debug_trap(
2862                                         pdd->dev->adev,
2863                                         false,
2864                                         pdd->dev->vm_info.last_vmid_kfd);
2865
2866                         if (!pdd->dev->kfd->shared_resources.enable_mes)
2867                                 debug_refresh_runlist(pdd->dev->dqm);
2868                         else
2869                                 kfd_dbg_set_mes_debug_mode(pdd,
2870                                                            !kfd_dbg_has_cwsr_workaround(pdd->dev));
2871                 }
2872         }
2873
2874         return 0;
2875 }
2876
2877 static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
2878 {
2879         struct kfd_ioctl_runtime_enable_args *args = data;
2880         int r;
2881
2882         mutex_lock(&p->mutex);
2883
2884         if (args->mode_mask & KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK)
2885                 r = runtime_enable(p, args->r_debug,
2886                                 !!(args->mode_mask & KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK));
2887         else
2888                 r = runtime_disable(p);
2889
2890         mutex_unlock(&p->mutex);
2891
2892         return r;
2893 }
2894
2895 static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
2896 {
2897         struct kfd_ioctl_dbg_trap_args *args = data;
2898         struct task_struct *thread = NULL;
2899         struct mm_struct *mm = NULL;
2900         struct pid *pid = NULL;
2901         struct kfd_process *target = NULL;
2902         struct kfd_process_device *pdd = NULL;
2903         int r = 0;
2904
2905         if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2906                 pr_err("Debugging does not support sched_policy %i", sched_policy);
2907                 return -EINVAL;
2908         }
2909
2910         pid = find_get_pid(args->pid);
2911         if (!pid) {
2912                 pr_debug("Cannot find pid info for %i\n", args->pid);
2913                 r = -ESRCH;
2914                 goto out;
2915         }
2916
2917         thread = get_pid_task(pid, PIDTYPE_PID);
2918         if (!thread) {
2919                 r = -ESRCH;
2920                 goto out;
2921         }
2922
2923         mm = get_task_mm(thread);
2924         if (!mm) {
2925                 r = -ESRCH;
2926                 goto out;
2927         }
2928
2929         if (args->op == KFD_IOC_DBG_TRAP_ENABLE) {
2930                 bool create_process;
2931
2932                 rcu_read_lock();
2933                 create_process = thread && thread != current && ptrace_parent(thread) == current;
2934                 rcu_read_unlock();
2935
2936                 target = create_process ? kfd_create_process(thread) :
2937                                         kfd_lookup_process_by_pid(pid);
2938         } else {
2939                 target = kfd_lookup_process_by_pid(pid);
2940         }
2941
2942         if (IS_ERR_OR_NULL(target)) {
2943                 pr_debug("Cannot find process PID %i to debug\n", args->pid);
2944                 r = target ? PTR_ERR(target) : -ESRCH;
2945                 target = NULL;
2946                 goto out;
2947         }
2948
2949         /* Check if target is still PTRACED. */
2950         rcu_read_lock();
2951         if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
2952                                 && ptrace_parent(target->lead_thread) != current) {
2953                 pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid);
2954                 r = -EPERM;
2955         }
2956         rcu_read_unlock();
2957
2958         if (r)
2959                 goto out;
2960
2961         mutex_lock(&target->mutex);
2962
2963         if (args->op != KFD_IOC_DBG_TRAP_ENABLE && !target->debug_trap_enabled) {
2964                 pr_err("PID %i not debug enabled for op %i\n", args->pid, args->op);
2965                 r = -EINVAL;
2966                 goto unlock_out;
2967         }
2968
2969         if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_ENABLED &&
2970                         (args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE ||
2971                          args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE ||
2972                          args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ||
2973                          args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES ||
2974                          args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2975                          args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH ||
2976                          args->op == KFD_IOC_DBG_TRAP_SET_FLAGS)) {
2977                 r = -EPERM;
2978                 goto unlock_out;
2979         }
2980
2981         if (args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2982             args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH) {
2983                 int user_gpu_id = kfd_process_get_user_gpu_id(target,
2984                                 args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ?
2985                                         args->set_node_address_watch.gpu_id :
2986                                         args->clear_node_address_watch.gpu_id);
2987
2988                 pdd = kfd_process_device_data_by_id(target, user_gpu_id);
2989                 if (user_gpu_id == -EINVAL || !pdd) {
2990                         r = -ENODEV;
2991                         goto unlock_out;
2992                 }
2993         }
2994
2995         switch (args->op) {
2996         case KFD_IOC_DBG_TRAP_ENABLE:
2997                 if (target != p)
2998                         target->debugger_process = p;
2999
3000                 r = kfd_dbg_trap_enable(target,
3001                                         args->enable.dbg_fd,
3002                                         (void __user *)args->enable.rinfo_ptr,
3003                                         &args->enable.rinfo_size);
3004                 if (!r)
3005                         target->exception_enable_mask = args->enable.exception_mask;
3006
3007                 break;
3008         case KFD_IOC_DBG_TRAP_DISABLE:
3009                 r = kfd_dbg_trap_disable(target);
3010                 break;
3011         case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
3012                 r = kfd_dbg_send_exception_to_runtime(target,
3013                                 args->send_runtime_event.gpu_id,
3014                                 args->send_runtime_event.queue_id,
3015                                 args->send_runtime_event.exception_mask);
3016                 break;
3017         case KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED:
3018                 kfd_dbg_set_enabled_debug_exception_mask(target,
3019                                 args->set_exceptions_enabled.exception_mask);
3020                 break;
3021         case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
3022                 r = kfd_dbg_trap_set_wave_launch_override(target,
3023                                 args->launch_override.override_mode,
3024                                 args->launch_override.enable_mask,
3025                                 args->launch_override.support_request_mask,
3026                                 &args->launch_override.enable_mask,
3027                                 &args->launch_override.support_request_mask);
3028                 break;
3029         case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
3030                 r = kfd_dbg_trap_set_wave_launch_mode(target,
3031                                 args->launch_mode.launch_mode);
3032                 break;
3033         case KFD_IOC_DBG_TRAP_SUSPEND_QUEUES:
3034                 r = suspend_queues(target,
3035                                 args->suspend_queues.num_queues,
3036                                 args->suspend_queues.grace_period,
3037                                 args->suspend_queues.exception_mask,
3038                                 (uint32_t *)args->suspend_queues.queue_array_ptr);
3039
3040                 break;
3041         case KFD_IOC_DBG_TRAP_RESUME_QUEUES:
3042                 r = resume_queues(target, args->resume_queues.num_queues,
3043                                 (uint32_t *)args->resume_queues.queue_array_ptr);
3044                 break;
3045         case KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH:
3046                 r = kfd_dbg_trap_set_dev_address_watch(pdd,
3047                                 args->set_node_address_watch.address,
3048                                 args->set_node_address_watch.mask,
3049                                 &args->set_node_address_watch.id,
3050                                 args->set_node_address_watch.mode);
3051                 break;
3052         case KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH:
3053                 r = kfd_dbg_trap_clear_dev_address_watch(pdd,
3054                                 args->clear_node_address_watch.id);
3055                 break;
3056         case KFD_IOC_DBG_TRAP_SET_FLAGS:
3057                 r = kfd_dbg_trap_set_flags(target, &args->set_flags.flags);
3058                 break;
3059         case KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
3060                 r = kfd_dbg_ev_query_debug_event(target,
3061                                 &args->query_debug_event.queue_id,
3062                                 &args->query_debug_event.gpu_id,
3063                                 args->query_debug_event.exception_mask,
3064                                 &args->query_debug_event.exception_mask);
3065                 break;
3066         case KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO:
3067                 r = kfd_dbg_trap_query_exception_info(target,
3068                                 args->query_exception_info.source_id,
3069                                 args->query_exception_info.exception_code,
3070                                 args->query_exception_info.clear_exception,
3071                                 (void __user *)args->query_exception_info.info_ptr,
3072                                 &args->query_exception_info.info_size);
3073                 break;
3074         case KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT:
3075                 r = pqm_get_queue_snapshot(&target->pqm,
3076                                 args->queue_snapshot.exception_mask,
3077                                 (void __user *)args->queue_snapshot.snapshot_buf_ptr,
3078                                 &args->queue_snapshot.num_queues,
3079                                 &args->queue_snapshot.entry_size);
3080                 break;
3081         case KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT:
3082                 r = kfd_dbg_trap_device_snapshot(target,
3083                                 args->device_snapshot.exception_mask,
3084                                 (void __user *)args->device_snapshot.snapshot_buf_ptr,
3085                                 &args->device_snapshot.num_devices,
3086                                 &args->device_snapshot.entry_size);
3087                 break;
3088         default:
3089                 pr_err("Invalid option: %i\n", args->op);
3090                 r = -EINVAL;
3091         }
3092
3093 unlock_out:
3094         mutex_unlock(&target->mutex);
3095
3096 out:
3097         if (thread)
3098                 put_task_struct(thread);
3099
3100         if (mm)
3101                 mmput(mm);
3102
3103         if (pid)
3104                 put_pid(pid);
3105
3106         if (target)
3107                 kfd_unref_process(target);
3108
3109         return r;
3110 }
3111
3112 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
3113         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
3114                             .cmd_drv = 0, .name = #ioctl}
3115
3116 /** Ioctl table */
3117 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
3118         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
3119                         kfd_ioctl_get_version, 0),
3120
3121         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
3122                         kfd_ioctl_create_queue, 0),
3123
3124         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
3125                         kfd_ioctl_destroy_queue, 0),
3126
3127         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
3128                         kfd_ioctl_set_memory_policy, 0),
3129
3130         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
3131                         kfd_ioctl_get_clock_counters, 0),
3132
3133         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
3134                         kfd_ioctl_get_process_apertures, 0),
3135
3136         AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
3137                         kfd_ioctl_update_queue, 0),
3138
3139         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
3140                         kfd_ioctl_create_event, 0),
3141
3142         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
3143                         kfd_ioctl_destroy_event, 0),
3144
3145         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
3146                         kfd_ioctl_set_event, 0),
3147
3148         AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
3149                         kfd_ioctl_reset_event, 0),
3150
3151         AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
3152                         kfd_ioctl_wait_events, 0),
3153
3154         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
3155                         kfd_ioctl_dbg_register, 0),
3156
3157         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
3158                         kfd_ioctl_dbg_unregister, 0),
3159
3160         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
3161                         kfd_ioctl_dbg_address_watch, 0),
3162
3163         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
3164                         kfd_ioctl_dbg_wave_control, 0),
3165
3166         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
3167                         kfd_ioctl_set_scratch_backing_va, 0),
3168
3169         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
3170                         kfd_ioctl_get_tile_config, 0),
3171
3172         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
3173                         kfd_ioctl_set_trap_handler, 0),
3174
3175         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
3176                         kfd_ioctl_get_process_apertures_new, 0),
3177
3178         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
3179                         kfd_ioctl_acquire_vm, 0),
3180
3181         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
3182                         kfd_ioctl_alloc_memory_of_gpu, 0),
3183
3184         AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
3185                         kfd_ioctl_free_memory_of_gpu, 0),
3186
3187         AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
3188                         kfd_ioctl_map_memory_to_gpu, 0),
3189
3190         AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
3191                         kfd_ioctl_unmap_memory_from_gpu, 0),
3192
3193         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
3194                         kfd_ioctl_set_cu_mask, 0),
3195
3196         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
3197                         kfd_ioctl_get_queue_wave_state, 0),
3198
3199         AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
3200                                 kfd_ioctl_get_dmabuf_info, 0),
3201
3202         AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
3203                                 kfd_ioctl_import_dmabuf, 0),
3204
3205         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
3206                         kfd_ioctl_alloc_queue_gws, 0),
3207
3208         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
3209                         kfd_ioctl_smi_events, 0),
3210
3211         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
3212
3213         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
3214                         kfd_ioctl_set_xnack_mode, 0),
3215
3216         AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
3217                         kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
3218
3219         AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
3220                         kfd_ioctl_get_available_memory, 0),
3221
3222         AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
3223                                 kfd_ioctl_export_dmabuf, 0),
3224
3225         AMDKFD_IOCTL_DEF(AMDKFD_IOC_RUNTIME_ENABLE,
3226                         kfd_ioctl_runtime_enable, 0),
3227
3228         AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
3229                         kfd_ioctl_set_debug_trap, 0),
3230 };
3231
3232 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
3233
3234 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
3235 {
3236         struct kfd_process *process;
3237         amdkfd_ioctl_t *func;
3238         const struct amdkfd_ioctl_desc *ioctl = NULL;
3239         unsigned int nr = _IOC_NR(cmd);
3240         char stack_kdata[128];
3241         char *kdata = NULL;
3242         unsigned int usize, asize;
3243         int retcode = -EINVAL;
3244         bool ptrace_attached = false;
3245
3246         if (nr >= AMDKFD_CORE_IOCTL_COUNT)
3247                 goto err_i1;
3248
3249         if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
3250                 u32 amdkfd_size;
3251
3252                 ioctl = &amdkfd_ioctls[nr];
3253
3254                 amdkfd_size = _IOC_SIZE(ioctl->cmd);
3255                 usize = asize = _IOC_SIZE(cmd);
3256                 if (amdkfd_size > asize)
3257                         asize = amdkfd_size;
3258
3259                 cmd = ioctl->cmd;
3260         } else
3261                 goto err_i1;
3262
3263         dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
3264
3265         /* Get the process struct from the filep. Only the process
3266          * that opened /dev/kfd can use the file descriptor. Child
3267          * processes need to create their own KFD device context.
3268          */
3269         process = filep->private_data;
3270
3271         rcu_read_lock();
3272         if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
3273             ptrace_parent(process->lead_thread) == current)
3274                 ptrace_attached = true;
3275         rcu_read_unlock();
3276
3277         if (process->lead_thread != current->group_leader
3278             && !ptrace_attached) {
3279                 dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
3280                 retcode = -EBADF;
3281                 goto err_i1;
3282         }
3283
3284         /* Do not trust userspace, use our own definition */
3285         func = ioctl->func;
3286
3287         if (unlikely(!func)) {
3288                 dev_dbg(kfd_device, "no function\n");
3289                 retcode = -EINVAL;
3290                 goto err_i1;
3291         }
3292
3293         /*
3294          * Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
3295          * CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
3296          * more priviledged access.
3297          */
3298         if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
3299                 if (!capable(CAP_CHECKPOINT_RESTORE) &&
3300                                                 !capable(CAP_SYS_ADMIN)) {
3301                         retcode = -EACCES;
3302                         goto err_i1;
3303                 }
3304         }
3305
3306         if (cmd & (IOC_IN | IOC_OUT)) {
3307                 if (asize <= sizeof(stack_kdata)) {
3308                         kdata = stack_kdata;
3309                 } else {
3310                         kdata = kmalloc(asize, GFP_KERNEL);
3311                         if (!kdata) {
3312                                 retcode = -ENOMEM;
3313                                 goto err_i1;
3314                         }
3315                 }
3316                 if (asize > usize)
3317                         memset(kdata + usize, 0, asize - usize);
3318         }
3319
3320         if (cmd & IOC_IN) {
3321                 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
3322                         retcode = -EFAULT;
3323                         goto err_i1;
3324                 }
3325         } else if (cmd & IOC_OUT) {
3326                 memset(kdata, 0, usize);
3327         }
3328
3329         retcode = func(filep, process, kdata);
3330
3331         if (cmd & IOC_OUT)
3332                 if (copy_to_user((void __user *)arg, kdata, usize) != 0)
3333                         retcode = -EFAULT;
3334
3335 err_i1:
3336         if (!ioctl)
3337                 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
3338                           task_pid_nr(current), cmd, nr);
3339
3340         if (kdata != stack_kdata)
3341                 kfree(kdata);
3342
3343         if (retcode)
3344                 dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
3345                                 nr, arg, retcode);
3346
3347         return retcode;
3348 }
3349
3350 static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
3351                       struct vm_area_struct *vma)
3352 {
3353         phys_addr_t address;
3354
3355         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3356                 return -EINVAL;
3357
3358         if (PAGE_SIZE > 4096)
3359                 return -EINVAL;
3360
3361         address = dev->adev->rmmio_remap.bus_addr;
3362
3363         vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
3364                                 VM_DONTDUMP | VM_PFNMAP);
3365
3366         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3367
3368         pr_debug("pasid 0x%x mapping mmio page\n"
3369                  "     target user address == 0x%08llX\n"
3370                  "     physical address    == 0x%08llX\n"
3371                  "     vm_flags            == 0x%04lX\n"
3372                  "     size                == 0x%04lX\n",
3373                  process->pasid, (unsigned long long) vma->vm_start,
3374                  address, vma->vm_flags, PAGE_SIZE);
3375
3376         return io_remap_pfn_range(vma,
3377                                 vma->vm_start,
3378                                 address >> PAGE_SHIFT,
3379                                 PAGE_SIZE,
3380                                 vma->vm_page_prot);
3381 }
3382
3383
3384 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
3385 {
3386         struct kfd_process *process;
3387         struct kfd_node *dev = NULL;
3388         unsigned long mmap_offset;
3389         unsigned int gpu_id;
3390
3391         process = kfd_get_process(current);
3392         if (IS_ERR(process))
3393                 return PTR_ERR(process);
3394
3395         mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
3396         gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
3397         if (gpu_id)
3398                 dev = kfd_device_by_id(gpu_id);
3399
3400         switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
3401         case KFD_MMAP_TYPE_DOORBELL:
3402                 if (!dev)
3403                         return -ENODEV;
3404                 return kfd_doorbell_mmap(dev, process, vma);
3405
3406         case KFD_MMAP_TYPE_EVENTS:
3407                 return kfd_event_mmap(process, vma);
3408
3409         case KFD_MMAP_TYPE_RESERVED_MEM:
3410                 if (!dev)
3411                         return -ENODEV;
3412                 return kfd_reserved_mem_mmap(dev, process, vma);
3413         case KFD_MMAP_TYPE_MMIO:
3414                 if (!dev)
3415                         return -ENODEV;
3416                 return kfd_mmio_mmap(dev, process, vma);
3417         }
3418
3419         return -EFAULT;
3420 }