drivers/accel/qaic/qaic_drv.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
   4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
   5
   6 #include <linux/delay.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/idr.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/list.h>
  11 #include <linux/kref.h>
  12 #include <linux/mhi.h>
  13 #include <linux/module.h>
  14 #include <linux/msi.h>
  15 #include <linux/mutex.h>
  16 #include <linux/pci.h>
  17 #include <linux/spinlock.h>
  18 #include <linux/workqueue.h>
  19 #include <linux/wait.h>
  20 #include <drm/drm_accel.h>
  21 #include <drm/drm_drv.h>
  22 #include <drm/drm_file.h>
  23 #include <drm/drm_gem.h>
  24 #include <drm/drm_ioctl.h>
  25 #include <uapi/drm/qaic_accel.h>
  26
  27 #include "mhi_controller.h"
  28 #include "qaic.h"
  29
  30 MODULE_IMPORT_NS(DMA_BUF);
  31
  32 #define PCI_DEV_AIC100                  0xa100
  33 #define QAIC_NAME                       "qaic"
  34 #define QAIC_DESC                       "Qualcomm Cloud AI Accelerators"
  35 #define CNTL_MAJOR                      5
  36 #define CNTL_MINOR                      0
  37
  38 bool datapath_polling;
  39 module_param(datapath_polling, bool, 0400);
  40 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
  41 static bool link_up;
  42 static DEFINE_IDA(qaic_usrs);
  43
  44 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id);
  45 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id);
  46
  47 static void free_usr(struct kref *kref)
  48 {
  49         struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
  50
  51         cleanup_srcu_struct(&usr->qddev_lock);
  52         ida_free(&qaic_usrs, usr->handle);
  53         kfree(usr);
  54 }
  55
  56 static int qaic_open(struct drm_device *dev, struct drm_file *file)
  57 {
  58         struct qaic_drm_device *qddev = dev->dev_private;
  59         struct qaic_device *qdev = qddev->qdev;
  60         struct qaic_user *usr;
  61         int rcu_id;
  62         int ret;
  63
  64         rcu_id = srcu_read_lock(&qdev->dev_lock);
  65         if (qdev->in_reset) {
  66                 ret = -ENODEV;
  67                 goto dev_unlock;
  68         }
  69
  70         usr = kmalloc(sizeof(*usr), GFP_KERNEL);
  71         if (!usr) {
  72                 ret = -ENOMEM;
  73                 goto dev_unlock;
  74         }
  75
  76         usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
  77         if (usr->handle < 0) {
  78                 ret = usr->handle;
  79                 goto free_usr;
  80         }
  81         usr->qddev = qddev;
  82         atomic_set(&usr->chunk_id, 0);
  83         init_srcu_struct(&usr->qddev_lock);
  84         kref_init(&usr->ref_count);
  85
  86         ret = mutex_lock_interruptible(&qddev->users_mutex);
  87         if (ret)
  88                 goto cleanup_usr;
  89
  90         list_add(&usr->node, &qddev->users);
  91         mutex_unlock(&qddev->users_mutex);
  92
  93         file->driver_priv = usr;
  94
  95         srcu_read_unlock(&qdev->dev_lock, rcu_id);
  96         return 0;
  97
  98 cleanup_usr:
  99         cleanup_srcu_struct(&usr->qddev_lock);
 100 free_usr:
 101         kfree(usr);
 102 dev_unlock:
 103         srcu_read_unlock(&qdev->dev_lock, rcu_id);
 104         return ret;
 105 }
 106
 107 static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
 108 {
 109         struct qaic_user *usr = file->driver_priv;
 110         struct qaic_drm_device *qddev;
 111         struct qaic_device *qdev;
 112         int qdev_rcu_id;
 113         int usr_rcu_id;
 114         int i;
 115
 116         qddev = usr->qddev;
 117         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
 118         if (qddev) {
 119                 qdev = qddev->qdev;
 120                 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
 121                 if (!qdev->in_reset) {
 122                         qaic_release_usr(qdev, usr);
 123                         for (i = 0; i < qdev->num_dbc; ++i)
 124                                 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
 125                                         release_dbc(qdev, i);
 126                 }
 127                 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
 128
 129                 mutex_lock(&qddev->users_mutex);
 130                 if (!list_empty(&usr->node))
 131                         list_del_init(&usr->node);
 132                 mutex_unlock(&qddev->users_mutex);
 133         }
 134
 135         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
 136         kref_put(&usr->ref_count, free_usr);
 137
 138         file->driver_priv = NULL;
 139 }
 140
 141 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
 142
 143 static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
 144         DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
 145         DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
 146         DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
 147         DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
 148         DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
 149         DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
 150         DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
 151         DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
 152 };
 153
 154 static const struct drm_driver qaic_accel_driver = {
 155         .driver_features        = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
 156
 157         .name                   = QAIC_NAME,
 158         .desc                   = QAIC_DESC,
 159         .date                   = "20190618",
 160
 161         .fops                   = &qaic_accel_fops,
 162         .open                   = qaic_open,
 163         .postclose              = qaic_postclose,
 164
 165         .ioctls                 = qaic_drm_ioctls,
 166         .num_ioctls             = ARRAY_SIZE(qaic_drm_ioctls),
 167         .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
 168         .gem_prime_import       = qaic_gem_prime_import,
 169 };
 170
 171 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
 172 {
 173         struct qaic_drm_device *qddev;
 174         struct drm_device *ddev;
 175         struct device *pdev;
 176         int ret;
 177
 178         /* Hold off implementing partitions until the uapi is determined */
 179         if (partition_id != QAIC_NO_PARTITION)
 180                 return -EINVAL;
 181
 182         pdev = &qdev->pdev->dev;
 183
 184         qddev = kzalloc(sizeof(*qddev), GFP_KERNEL);
 185         if (!qddev)
 186                 return -ENOMEM;
 187
 188         ddev = drm_dev_alloc(&qaic_accel_driver, pdev);
 189         if (IS_ERR(ddev)) {
 190                 ret = PTR_ERR(ddev);
 191                 goto ddev_fail;
 192         }
 193
 194         ddev->dev_private = qddev;
 195         qddev->ddev = ddev;
 196
 197         qddev->qdev = qdev;
 198         qddev->partition_id = partition_id;
 199         INIT_LIST_HEAD(&qddev->users);
 200         mutex_init(&qddev->users_mutex);
 201
 202         qdev->qddev = qddev;
 203
 204         ret = drm_dev_register(ddev, 0);
 205         if (ret) {
 206                 pci_dbg(qdev->pdev, "%s: drm_dev_register failed %d\n", __func__, ret);
 207                 goto drm_reg_fail;
 208         }
 209
 210         return 0;
 211
 212 drm_reg_fail:
 213         mutex_destroy(&qddev->users_mutex);
 214         qdev->qddev = NULL;
 215         drm_dev_put(ddev);
 216 ddev_fail:
 217         kfree(qddev);
 218         return ret;
 219 }
 220
 221 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
 222 {
 223         struct qaic_drm_device *qddev;
 224         struct qaic_user *usr;
 225
 226         qddev = qdev->qddev;
 227
 228         /*
 229          * Existing users get unresolvable errors till they close FDs.
 230          * Need to sync carefully with users calling close(). The
 231          * list of users can be modified elsewhere when the lock isn't
 232          * held here, but the sync'ing the srcu with the mutex held
 233          * could deadlock. Grab the mutex so that the list will be
 234          * unmodified. The user we get will exist as long as the
 235          * lock is held. Signal that the qcdev is going away, and
 236          * grab a reference to the user so they don't go away for
 237          * synchronize_srcu(). Then release the mutex to avoid
 238          * deadlock and make sure the user has observed the signal.
 239          * With the lock released, we cannot maintain any state of the
 240          * user list.
 241          */
 242         mutex_lock(&qddev->users_mutex);
 243         while (!list_empty(&qddev->users)) {
 244                 usr = list_first_entry(&qddev->users, struct qaic_user, node);
 245                 list_del_init(&usr->node);
 246                 kref_get(&usr->ref_count);
 247                 usr->qddev = NULL;
 248                 mutex_unlock(&qddev->users_mutex);
 249                 synchronize_srcu(&usr->qddev_lock);
 250                 kref_put(&usr->ref_count, free_usr);
 251                 mutex_lock(&qddev->users_mutex);
 252         }
 253         mutex_unlock(&qddev->users_mutex);
 254
 255         if (qddev->ddev) {
 256                 drm_dev_unregister(qddev->ddev);
 257                 drm_dev_put(qddev->ddev);
 258         }
 259
 260         kfree(qddev);
 261 }
 262
 263 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
 264 {
 265         struct qaic_device *qdev;
 266         u16 major, minor;
 267         int ret;
 268
 269         /*
 270          * Invoking this function indicates that the control channel to the
 271          * device is available. We use that as a signal to indicate that
 272          * the device side firmware has booted. The device side firmware
 273          * manages the device resources, so we need to communicate with it
 274          * via the control channel in order to utilize the device. Therefore
 275          * we wait until this signal to create the drm dev that userspace will
 276          * use to control the device, because without the device side firmware,
 277          * userspace can't do anything useful.
 278          */
 279
 280         qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
 281
 282         qdev->in_reset = false;
 283
 284         dev_set_drvdata(&mhi_dev->dev, qdev);
 285         qdev->cntl_ch = mhi_dev;
 286
 287         ret = qaic_control_open(qdev);
 288         if (ret) {
 289                 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
 290                 return ret;
 291         }
 292
 293         ret = get_cntl_version(qdev, NULL, &major, &minor);
 294         if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
 295                 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
 296                         __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
 297                 ret = -EINVAL;
 298                 goto close_control;
 299         }
 300
 301         ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
 302
 303         return ret;
 304
 305 close_control:
 306         qaic_control_close(qdev);
 307         return ret;
 308 }
 309
 310 static void qaic_mhi_remove(struct mhi_device *mhi_dev)
 311 {
 312 /* This is redundant since we have already observed the device crash */
 313 }
 314
 315 static void qaic_notify_reset(struct qaic_device *qdev)
 316 {
 317         int i;
 318
 319         qdev->in_reset = true;
 320         /* wake up any waiters to avoid waiting for timeouts at sync */
 321         wake_all_cntl(qdev);
 322         for (i = 0; i < qdev->num_dbc; ++i)
 323                 wakeup_dbc(qdev, i);
 324         synchronize_srcu(&qdev->dev_lock);
 325 }
 326
 327 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev, bool exit_reset)
 328 {
 329         int i;
 330
 331         qaic_notify_reset(qdev);
 332
 333         /* remove drmdevs to prevent new users from coming in */
 334         qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
 335
 336         /* start tearing things down */
 337         for (i = 0; i < qdev->num_dbc; ++i)
 338                 release_dbc(qdev, i);
 339
 340         if (exit_reset)
 341                 qdev->in_reset = false;
 342 }
 343
 344 static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
 345 {
 346         struct qaic_device *qdev;
 347         int i;
 348
 349         qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
 350         if (!qdev)
 351                 return NULL;
 352
 353         if (id->device == PCI_DEV_AIC100) {
 354                 qdev->num_dbc = 16;
 355                 qdev->dbc = devm_kcalloc(&pdev->dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
 356                 if (!qdev->dbc)
 357                         return NULL;
 358         }
 359
 360         qdev->cntl_wq = alloc_workqueue("qaic_cntl", WQ_UNBOUND, 0);
 361         if (!qdev->cntl_wq)
 362                 return NULL;
 363
 364         pci_set_drvdata(pdev, qdev);
 365         qdev->pdev = pdev;
 366
 367         mutex_init(&qdev->cntl_mutex);
 368         INIT_LIST_HEAD(&qdev->cntl_xfer_list);
 369         init_srcu_struct(&qdev->dev_lock);
 370
 371         for (i = 0; i < qdev->num_dbc; ++i) {
 372                 spin_lock_init(&qdev->dbc[i].xfer_lock);
 373                 qdev->dbc[i].qdev = qdev;
 374                 qdev->dbc[i].id = i;
 375                 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
 376                 init_srcu_struct(&qdev->dbc[i].ch_lock);
 377                 init_waitqueue_head(&qdev->dbc[i].dbc_release);
 378                 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
 379         }
 380
 381         return qdev;
 382 }
 383
 384 static void cleanup_qdev(struct qaic_device *qdev)
 385 {
 386         int i;
 387
 388         for (i = 0; i < qdev->num_dbc; ++i)
 389                 cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
 390         cleanup_srcu_struct(&qdev->dev_lock);
 391         pci_set_drvdata(qdev->pdev, NULL);
 392         destroy_workqueue(qdev->cntl_wq);
 393 }
 394
 395 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev)
 396 {
 397         int bars;
 398         int ret;
 399
 400         bars = pci_select_bars(pdev, IORESOURCE_MEM);
 401
 402         /* make sure the device has the expected BARs */
 403         if (bars != (BIT(0) | BIT(2) | BIT(4))) {
 404                 pci_dbg(pdev, "%s: expected BARs 0, 2, and 4 not found in device. Found 0x%x\n",
 405                         __func__, bars);
 406                 return -EINVAL;
 407         }
 408
 409         ret = pcim_enable_device(pdev);
 410         if (ret)
 411                 return ret;
 412
 413         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 414         if (ret)
 415                 return ret;
 416         ret = dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 417         if (ret)
 418                 return ret;
 419
 420         qdev->bar_0 = devm_ioremap_resource(&pdev->dev, &pdev->resource[0]);
 421         if (IS_ERR(qdev->bar_0))
 422                 return PTR_ERR(qdev->bar_0);
 423
 424         qdev->bar_2 = devm_ioremap_resource(&pdev->dev, &pdev->resource[2]);
 425         if (IS_ERR(qdev->bar_2))
 426                 return PTR_ERR(qdev->bar_2);
 427
 428         /* Managed release since we use pcim_enable_device above */
 429         pci_set_master(pdev);
 430
 431         return 0;
 432 }
 433
 434 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
 435 {
 436         int mhi_irq;
 437         int ret;
 438         int i;
 439
 440         /* Managed release since we use pcim_enable_device */
 441         ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
 442         if (ret < 0)
 443                 return ret;
 444
 445         if (ret < 32) {
 446                 pci_err(pdev, "%s: Requested 32 MSIs. Obtained %d MSIs which is less than the 32 required.\n",
 447                         __func__, ret);
 448                 return -ENODEV;
 449         }
 450
 451         mhi_irq = pci_irq_vector(pdev, 0);
 452         if (mhi_irq < 0)
 453                 return mhi_irq;
 454
 455         for (i = 0; i < qdev->num_dbc; ++i) {
 456                 ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, i + 1),
 457                                                 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
 458                                                 "qaic_dbc", &qdev->dbc[i]);
 459                 if (ret)
 460                         return ret;
 461
 462                 if (datapath_polling) {
 463                         qdev->dbc[i].irq = pci_irq_vector(pdev, i + 1);
 464                         disable_irq_nosync(qdev->dbc[i].irq);
 465                         INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
 466                 }
 467         }
 468
 469         return mhi_irq;
 470 }
 471
 472 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 473 {
 474         struct qaic_device *qdev;
 475         int mhi_irq;
 476         int ret;
 477         int i;
 478
 479         qdev = create_qdev(pdev, id);
 480         if (!qdev)
 481                 return -ENOMEM;
 482
 483         ret = init_pci(qdev, pdev);
 484         if (ret)
 485                 goto cleanup_qdev;
 486
 487         for (i = 0; i < qdev->num_dbc; ++i)
 488                 qdev->dbc[i].dbc_base = qdev->bar_2 + QAIC_DBC_OFF(i);
 489
 490         mhi_irq = init_msi(qdev, pdev);
 491         if (mhi_irq < 0) {
 492                 ret = mhi_irq;
 493                 goto cleanup_qdev;
 494         }
 495
 496         qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq);
 497         if (IS_ERR(qdev->mhi_cntrl)) {
 498                 ret = PTR_ERR(qdev->mhi_cntrl);
 499                 goto cleanup_qdev;
 500         }
 501
 502         return 0;
 503
 504 cleanup_qdev:
 505         cleanup_qdev(qdev);
 506         return ret;
 507 }
 508
 509 static void qaic_pci_remove(struct pci_dev *pdev)
 510 {
 511         struct qaic_device *qdev = pci_get_drvdata(pdev);
 512
 513         if (!qdev)
 514                 return;
 515
 516         qaic_dev_reset_clean_local_state(qdev, false);
 517         qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
 518         cleanup_qdev(qdev);
 519 }
 520
 521 static void qaic_pci_shutdown(struct pci_dev *pdev)
 522 {
 523         /* see qaic_exit for what link_up is doing */
 524         link_up = true;
 525         qaic_pci_remove(pdev);
 526 }
 527
 528 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
 529 {
 530         return PCI_ERS_RESULT_NEED_RESET;
 531 }
 532
 533 static void qaic_pci_reset_prepare(struct pci_dev *pdev)
 534 {
 535         struct qaic_device *qdev = pci_get_drvdata(pdev);
 536
 537         qaic_notify_reset(qdev);
 538         qaic_mhi_start_reset(qdev->mhi_cntrl);
 539         qaic_dev_reset_clean_local_state(qdev, false);
 540 }
 541
 542 static void qaic_pci_reset_done(struct pci_dev *pdev)
 543 {
 544         struct qaic_device *qdev = pci_get_drvdata(pdev);
 545
 546         qdev->in_reset = false;
 547         qaic_mhi_reset_done(qdev->mhi_cntrl);
 548 }
 549
 550 static const struct mhi_device_id qaic_mhi_match_table[] = {
 551         { .chan = "QAIC_CONTROL", },
 552         {},
 553 };
 554
 555 static struct mhi_driver qaic_mhi_driver = {
 556         .id_table = qaic_mhi_match_table,
 557         .remove = qaic_mhi_remove,
 558         .probe = qaic_mhi_probe,
 559         .ul_xfer_cb = qaic_mhi_ul_xfer_cb,
 560         .dl_xfer_cb = qaic_mhi_dl_xfer_cb,
 561         .driver = {
 562                 .name = "qaic_mhi",
 563         },
 564 };
 565
 566 static const struct pci_device_id qaic_ids[] = {
 567         { PCI_DEVICE(PCI_VENDOR_ID_QCOM, PCI_DEV_AIC100), },
 568         { }
 569 };
 570 MODULE_DEVICE_TABLE(pci, qaic_ids);
 571
 572 static const struct pci_error_handlers qaic_pci_err_handler = {
 573         .error_detected = qaic_pci_error_detected,
 574         .reset_prepare = qaic_pci_reset_prepare,
 575         .reset_done = qaic_pci_reset_done,
 576 };
 577
 578 static struct pci_driver qaic_pci_driver = {
 579         .name = QAIC_NAME,
 580         .id_table = qaic_ids,
 581         .probe = qaic_pci_probe,
 582         .remove = qaic_pci_remove,
 583         .shutdown = qaic_pci_shutdown,
 584         .err_handler = &qaic_pci_err_handler,
 585 };
 586
 587 static int __init qaic_init(void)
 588 {
 589         int ret;
 590
 591         ret = mhi_driver_register(&qaic_mhi_driver);
 592         if (ret) {
 593                 pr_debug("qaic: mhi_driver_register failed %d\n", ret);
 594                 return ret;
 595         }
 596
 597         ret = pci_register_driver(&qaic_pci_driver);
 598         if (ret) {
 599                 pr_debug("qaic: pci_register_driver failed %d\n", ret);
 600                 goto free_mhi;
 601         }
 602
 603         return 0;
 604
 605 free_mhi:
 606         mhi_driver_unregister(&qaic_mhi_driver);
 607         return ret;
 608 }
 609
 610 static void __exit qaic_exit(void)
 611 {
 612         /*
 613          * We assume that qaic_pci_remove() is called due to a hotplug event
 614          * which would mean that the link is down, and thus
 615          * qaic_mhi_free_controller() should not try to access the device during
 616          * cleanup.
 617          * We call pci_unregister_driver() below, which also triggers
 618          * qaic_pci_remove(), but since this is module exit, we expect the link
 619          * to the device to be up, in which case qaic_mhi_free_controller()
 620          * should try to access the device during cleanup to put the device in
 621          * a sane state.
 622          * For that reason, we set link_up here to let qaic_mhi_free_controller
 623          * know the expected link state. Since the module is going to be
 624          * removed at the end of this, we don't need to worry about
 625          * reinitializing the link_up state after the cleanup is done.
 626          */
 627         link_up = true;
 628         pci_unregister_driver(&qaic_pci_driver);
 629         mhi_driver_unregister(&qaic_mhi_driver);
 630 }
 631
 632 module_init(qaic_init);
 633 module_exit(qaic_exit);
 634
 635 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
 636 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
 637 MODULE_LICENSE("GPL");