drivers/nvme/host/apple.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Apple ANS NVM Express device driver
   4  * Copyright The Asahi Linux Contributors
   5  *
   6  * Based on the pci.c NVM Express device driver
   7  * Copyright (c) 2011-2014, Intel Corporation.
   8  * and on the rdma.c NVMe over Fabrics RDMA host code.
   9  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
  10  */
  11
  12 #include <linux/async.h>
  13 #include <linux/blkdev.h>
  14 #include <linux/blk-mq.h>
  15 #include <linux/device.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/dmapool.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/io-64-nonatomic-lo-hi.h>
  20 #include <linux/io.h>
  21 #include <linux/iopoll.h>
  22 #include <linux/jiffies.h>
  23 #include <linux/mempool.h>
  24 #include <linux/module.h>
  25 #include <linux/of.h>
  26 #include <linux/of_platform.h>
  27 #include <linux/once.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/pm_domain.h>
  30 #include <linux/soc/apple/rtkit.h>
  31 #include <linux/soc/apple/sart.h>
  32 #include <linux/reset.h>
  33 #include <linux/time64.h>
  34
  35 #include "nvme.h"
  36
  37 #define APPLE_ANS_BOOT_TIMEOUT    USEC_PER_SEC
  38 #define APPLE_ANS_MAX_QUEUE_DEPTH 64
  39
  40 #define APPLE_ANS_COPROC_CPU_CONTROL     0x44
  41 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4)
  42
  43 #define APPLE_ANS_ACQ_DB  0x1004
  44 #define APPLE_ANS_IOCQ_DB 0x100c
  45
  46 #define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210
  47
  48 #define APPLE_ANS_BOOT_STATUS    0x1300
  49 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55
  50
  51 #define APPLE_ANS_UNKNOWN_CTRL   0x24008
  52 #define APPLE_ANS_PRP_NULL_CHECK BIT(11)
  53
  54 #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908
  55 #define APPLE_ANS_LINEAR_SQ_EN   BIT(0)
  56
  57 #define APPLE_ANS_LINEAR_ASQ_DB  0x2490c
  58 #define APPLE_ANS_LINEAR_IOSQ_DB 0x24910
  59
  60 #define APPLE_NVMMU_NUM_TCBS      0x28100
  61 #define APPLE_NVMMU_ASQ_TCB_BASE  0x28108
  62 #define APPLE_NVMMU_IOSQ_TCB_BASE 0x28110
  63 #define APPLE_NVMMU_TCB_INVAL     0x28118
  64 #define APPLE_NVMMU_TCB_STAT      0x28120
  65
  66 /*
  67  * This controller is a bit weird in the way command tags works: Both the
  68  * admin and the IO queue share the same tag space. Additionally, tags
  69  * cannot be higher than 0x40 which effectively limits the combined
  70  * queue depth to 0x40. Instead of wasting half of that on the admin queue
  71  * which gets much less traffic we instead reduce its size here.
  72  * The controller also doesn't support async event such that no space must
  73  * be reserved for NVME_NR_AEN_COMMANDS.
  74  */
  75 #define APPLE_NVME_AQ_DEPTH        2
  76 #define APPLE_NVME_AQ_MQ_TAG_DEPTH (APPLE_NVME_AQ_DEPTH - 1)
  77
  78 /*
  79  * These can be higher, but we need to ensure that any command doesn't
  80  * require an sg allocation that needs more than a page of data.
  81  */
  82 #define NVME_MAX_KB_SZ 4096
  83 #define NVME_MAX_SEGS  127
  84
  85 /*
  86  * This controller comes with an embedded IOMMU known as NVMMU.
  87  * The NVMMU is pointed to an array of TCBs indexed by the command tag.
  88  * Each command must be configured inside this structure before it's allowed
  89  * to execute, including commands that don't require DMA transfers.
  90  *
  91  * An exception to this are Apple's vendor-specific commands (opcode 0xD8 on the
  92  * admin queue): Those commands must still be added to the NVMMU but the DMA
  93  * buffers cannot be represented as PRPs and must instead be allowed using SART.
  94  *
  95  * Programming the PRPs to the same values as those in the submission queue
  96  * looks rather silly at first. This hardware is however designed for a kernel
  97  * that runs the NVMMU code in a higher exception level than the NVMe driver.
  98  * In that setting the NVMe driver first programs the submission queue entry
  99  * and then executes a hypercall to the code that is allowed to program the
 100  * NVMMU. The NVMMU driver then creates a shadow copy of the PRPs while
 101  * verifying that they don't point to kernel text, data, pagetables, or similar
 102  * protected areas before programming the TCB to point to this shadow copy.
 103  * Since Linux doesn't do any of that we may as well just point both the queue
 104  * and the TCB PRP pointer to the same memory.
 105  */
 106 struct apple_nvmmu_tcb {
 107         u8 opcode;
 108
 109 #define APPLE_ANS_TCB_DMA_FROM_DEVICE BIT(0)
 110 #define APPLE_ANS_TCB_DMA_TO_DEVICE   BIT(1)
 111         u8 dma_flags;
 112
 113         u8 command_id;
 114         u8 _unk0;
 115         __le16 length;
 116         u8 _unk1[18];
 117         __le64 prp1;
 118         __le64 prp2;
 119         u8 _unk2[16];
 120         u8 aes_iv[8];
 121         u8 _aes_unk[64];
 122 };
 123
 124 /*
 125  * The Apple NVMe controller only supports a single admin and a single IO queue
 126  * which are both limited to 64 entries and share a single interrupt.
 127  *
 128  * The completion queue works as usual. The submission "queue" instead is
 129  * an array indexed by the command tag on this hardware. Commands must also be
 130  * present in the NVMMU's tcb array. They are triggered by writing their tag to
 131  * a MMIO register.
 132  */
 133 struct apple_nvme_queue {
 134         struct nvme_command *sqes;
 135         struct nvme_completion *cqes;
 136         struct apple_nvmmu_tcb *tcbs;
 137
 138         dma_addr_t sq_dma_addr;
 139         dma_addr_t cq_dma_addr;
 140         dma_addr_t tcb_dma_addr;
 141
 142         u32 __iomem *sq_db;
 143         u32 __iomem *cq_db;
 144
 145         u16 cq_head;
 146         u8 cq_phase;
 147
 148         bool is_adminq;
 149         bool enabled;
 150 };
 151
 152 /*
 153  * The apple_nvme_iod describes the data in an I/O.
 154  *
 155  * The sg pointer contains the list of PRP chunk allocations in addition
 156  * to the actual struct scatterlist.
 157  */
 158 struct apple_nvme_iod {
 159         struct nvme_request req;
 160         struct nvme_command cmd;
 161         struct apple_nvme_queue *q;
 162         int npages; /* In the PRP list. 0 means small pool in use */
 163         int nents; /* Used in scatterlist */
 164         dma_addr_t first_dma;
 165         unsigned int dma_len; /* length of single DMA segment mapping */
 166         struct scatterlist *sg;
 167 };
 168
 169 struct apple_nvme {
 170         struct device *dev;
 171
 172         void __iomem *mmio_coproc;
 173         void __iomem *mmio_nvme;
 174
 175         struct device **pd_dev;
 176         struct device_link **pd_link;
 177         int pd_count;
 178
 179         struct apple_sart *sart;
 180         struct apple_rtkit *rtk;
 181         struct reset_control *reset;
 182
 183         struct dma_pool *prp_page_pool;
 184         struct dma_pool *prp_small_pool;
 185         mempool_t *iod_mempool;
 186
 187         struct nvme_ctrl ctrl;
 188         struct work_struct remove_work;
 189
 190         struct apple_nvme_queue adminq;
 191         struct apple_nvme_queue ioq;
 192
 193         struct blk_mq_tag_set admin_tagset;
 194         struct blk_mq_tag_set tagset;
 195
 196         int irq;
 197         spinlock_t lock;
 198 };
 199
 200 static_assert(sizeof(struct nvme_command) == 64);
 201 static_assert(sizeof(struct apple_nvmmu_tcb) == 128);
 202
 203 static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl)
 204 {
 205         return container_of(ctrl, struct apple_nvme, ctrl);
 206 }
 207
 208 static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q)
 209 {
 210         if (q->is_adminq)
 211                 return container_of(q, struct apple_nvme, adminq);
 212
 213         return container_of(q, struct apple_nvme, ioq);
 214 }
 215
 216 static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q)
 217 {
 218         if (q->is_adminq)
 219                 return APPLE_NVME_AQ_DEPTH;
 220
 221         return APPLE_ANS_MAX_QUEUE_DEPTH;
 222 }
 223
 224 static void apple_nvme_rtkit_crashed(void *cookie)
 225 {
 226         struct apple_nvme *anv = cookie;
 227
 228         dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot");
 229         nvme_reset_ctrl(&anv->ctrl);
 230 }
 231
 232 static int apple_nvme_sart_dma_setup(void *cookie,
 233                                      struct apple_rtkit_shmem *bfr)
 234 {
 235         struct apple_nvme *anv = cookie;
 236         int ret;
 237
 238         if (bfr->iova)
 239                 return -EINVAL;
 240         if (!bfr->size)
 241                 return -EINVAL;
 242
 243         bfr->buffer =
 244                 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL);
 245         if (!bfr->buffer)
 246                 return -ENOMEM;
 247
 248         ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size);
 249         if (ret) {
 250                 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
 251                 bfr->buffer = NULL;
 252                 return -ENOMEM;
 253         }
 254
 255         return 0;
 256 }
 257
 258 static void apple_nvme_sart_dma_destroy(void *cookie,
 259                                         struct apple_rtkit_shmem *bfr)
 260 {
 261         struct apple_nvme *anv = cookie;
 262
 263         apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size);
 264         dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
 265 }
 266
 267 static const struct apple_rtkit_ops apple_nvme_rtkit_ops = {
 268         .crashed = apple_nvme_rtkit_crashed,
 269         .shmem_setup = apple_nvme_sart_dma_setup,
 270         .shmem_destroy = apple_nvme_sart_dma_destroy,
 271 };
 272
 273 static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag)
 274 {
 275         struct apple_nvme *anv = queue_to_apple_nvme(q);
 276
 277         writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL);
 278         if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT))
 279                 dev_warn_ratelimited(anv->dev,
 280                                      "NVMMU TCB invalidation failed\n");
 281 }
 282
 283 static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,
 284                                   struct nvme_command *cmd)
 285 {
 286         struct apple_nvme *anv = queue_to_apple_nvme(q);
 287         u32 tag = nvme_tag_from_cid(cmd->common.command_id);
 288         struct apple_nvmmu_tcb *tcb = &q->tcbs[tag];
 289
 290         tcb->opcode = cmd->common.opcode;
 291         tcb->prp1 = cmd->common.dptr.prp1;
 292         tcb->prp2 = cmd->common.dptr.prp2;
 293         tcb->length = cmd->rw.length;
 294         tcb->command_id = tag;
 295
 296         if (nvme_is_write(cmd))
 297                 tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE;
 298         else
 299                 tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE;
 300
 301         memcpy(&q->sqes[tag], cmd, sizeof(*cmd));
 302
 303         /*
 304          * This lock here doesn't make much sense at a first glace but
 305          * removing it will result in occasional missed completetion
 306          * interrupts even though the commands still appear on the CQ.
 307          * It's unclear why this happens but our best guess is that
 308          * there is a bug in the firmware triggered when a new command
 309          * is issued while we're inside the irq handler between the
 310          * NVMMU invalidation (and making the tag available again)
 311          * and the final CQ update.
 312          */
 313         spin_lock_irq(&anv->lock);
 314         writel(tag, q->sq_db);
 315         spin_unlock_irq(&anv->lock);
 316 }
 317
 318 /*
 319  * From pci.c:
 320  * Will slightly overestimate the number of pages needed.  This is OK
 321  * as it only leads to a small amount of wasted memory for the lifetime of
 322  * the I/O.
 323  */
 324 static inline size_t apple_nvme_iod_alloc_size(void)
 325 {
 326         const unsigned int nprps = DIV_ROUND_UP(
 327                 NVME_MAX_KB_SZ + NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE);
 328         const int npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 329         const size_t alloc_size = sizeof(__le64 *) * npages +
 330                                   sizeof(struct scatterlist) * NVME_MAX_SEGS;
 331
 332         return alloc_size;
 333 }
 334
 335 static void **apple_nvme_iod_list(struct request *req)
 336 {
 337         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 338
 339         return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
 340 }
 341
 342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req)
 343 {
 344         const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
 345         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 346         dma_addr_t dma_addr = iod->first_dma;
 347         int i;
 348
 349         for (i = 0; i < iod->npages; i++) {
 350                 __le64 *prp_list = apple_nvme_iod_list(req)[i];
 351                 dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
 352
 353                 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr);
 354                 dma_addr = next_dma_addr;
 355         }
 356 }
 357
 358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req)
 359 {
 360         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 361
 362         if (iod->dma_len) {
 363                 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len,
 364                                rq_dma_dir(req));
 365                 return;
 366         }
 367
 368         WARN_ON_ONCE(!iod->nents);
 369
 370         dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
 371         if (iod->npages == 0)
 372                 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0],
 373                               iod->first_dma);
 374         else
 375                 apple_nvme_free_prps(anv, req);
 376         mempool_free(iod->sg, anv->iod_mempool);
 377 }
 378
 379 static void apple_nvme_print_sgl(struct scatterlist *sgl, int nents)
 380 {
 381         int i;
 382         struct scatterlist *sg;
 383
 384         for_each_sg(sgl, sg, nents, i) {
 385                 dma_addr_t phys = sg_phys(sg);
 386
 387                 pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d dma_address:%pad dma_length:%d\n",
 388                         i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
 389                         sg_dma_len(sg));
 390         }
 391 }
 392
 393 static blk_status_t apple_nvme_setup_prps(struct apple_nvme *anv,
 394                                           struct request *req,
 395                                           struct nvme_rw_command *cmnd)
 396 {
 397         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 398         struct dma_pool *pool;
 399         int length = blk_rq_payload_bytes(req);
 400         struct scatterlist *sg = iod->sg;
 401         int dma_len = sg_dma_len(sg);
 402         u64 dma_addr = sg_dma_address(sg);
 403         int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
 404         __le64 *prp_list;
 405         void **list = apple_nvme_iod_list(req);
 406         dma_addr_t prp_dma;
 407         int nprps, i;
 408
 409         length -= (NVME_CTRL_PAGE_SIZE - offset);
 410         if (length <= 0) {
 411                 iod->first_dma = 0;
 412                 goto done;
 413         }
 414
 415         dma_len -= (NVME_CTRL_PAGE_SIZE - offset);
 416         if (dma_len) {
 417                 dma_addr += (NVME_CTRL_PAGE_SIZE - offset);
 418         } else {
 419                 sg = sg_next(sg);
 420                 dma_addr = sg_dma_address(sg);
 421                 dma_len = sg_dma_len(sg);
 422         }
 423
 424         if (length <= NVME_CTRL_PAGE_SIZE) {
 425                 iod->first_dma = dma_addr;
 426                 goto done;
 427         }
 428
 429         nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
 430         if (nprps <= (256 / 8)) {
 431                 pool = anv->prp_small_pool;
 432                 iod->npages = 0;
 433         } else {
 434                 pool = anv->prp_page_pool;
 435                 iod->npages = 1;
 436         }
 437
 438         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 439         if (!prp_list) {
 440                 iod->first_dma = dma_addr;
 441                 iod->npages = -1;
 442                 return BLK_STS_RESOURCE;
 443         }
 444         list[0] = prp_list;
 445         iod->first_dma = prp_dma;
 446         i = 0;
 447         for (;;) {
 448                 if (i == NVME_CTRL_PAGE_SIZE >> 3) {
 449                         __le64 *old_prp_list = prp_list;
 450
 451                         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 452                         if (!prp_list)
 453                                 goto free_prps;
 454                         list[iod->npages++] = prp_list;
 455                         prp_list[0] = old_prp_list[i - 1];
 456                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 457                         i = 1;
 458                 }
 459                 prp_list[i++] = cpu_to_le64(dma_addr);
 460                 dma_len -= NVME_CTRL_PAGE_SIZE;
 461                 dma_addr += NVME_CTRL_PAGE_SIZE;
 462                 length -= NVME_CTRL_PAGE_SIZE;
 463                 if (length <= 0)
 464                         break;
 465                 if (dma_len > 0)
 466                         continue;
 467                 if (unlikely(dma_len < 0))
 468                         goto bad_sgl;
 469                 sg = sg_next(sg);
 470                 dma_addr = sg_dma_address(sg);
 471                 dma_len = sg_dma_len(sg);
 472         }
 473 done:
 474         cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
 475         cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
 476         return BLK_STS_OK;
 477 free_prps:
 478         apple_nvme_free_prps(anv, req);
 479         return BLK_STS_RESOURCE;
 480 bad_sgl:
 481         WARN(DO_ONCE(apple_nvme_print_sgl, iod->sg, iod->nents),
 482              "Invalid SGL for payload:%d nents:%d\n", blk_rq_payload_bytes(req),
 483              iod->nents);
 484         return BLK_STS_IOERR;
 485 }
 486
 487 static blk_status_t apple_nvme_setup_prp_simple(struct apple_nvme *anv,
 488                                                 struct request *req,
 489                                                 struct nvme_rw_command *cmnd,
 490                                                 struct bio_vec *bv)
 491 {
 492         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 493         unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
 494         unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;
 495
 496         iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0);
 497         if (dma_mapping_error(anv->dev, iod->first_dma))
 498                 return BLK_STS_RESOURCE;
 499         iod->dma_len = bv->bv_len;
 500
 501         cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
 502         if (bv->bv_len > first_prp_len)
 503                 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
 504         return BLK_STS_OK;
 505 }
 506
 507 static blk_status_t apple_nvme_map_data(struct apple_nvme *anv,
 508                                         struct request *req,
 509                                         struct nvme_command *cmnd)
 510 {
 511         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 512         blk_status_t ret = BLK_STS_RESOURCE;
 513         int nr_mapped;
 514
 515         if (blk_rq_nr_phys_segments(req) == 1) {
 516                 struct bio_vec bv = req_bvec(req);
 517
 518                 if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)
 519                         return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw,
 520                                                            &bv);
 521         }
 522
 523         iod->dma_len = 0;
 524         iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC);
 525         if (!iod->sg)
 526                 return BLK_STS_RESOURCE;
 527         sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
 528         iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
 529         if (!iod->nents)
 530                 goto out_free_sg;
 531
 532         nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents,
 533                                      rq_dma_dir(req), DMA_ATTR_NO_WARN);
 534         if (!nr_mapped)
 535                 goto out_free_sg;
 536
 537         ret = apple_nvme_setup_prps(anv, req, &cmnd->rw);
 538         if (ret != BLK_STS_OK)
 539                 goto out_unmap_sg;
 540         return BLK_STS_OK;
 541
 542 out_unmap_sg:
 543         dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
 544 out_free_sg:
 545         mempool_free(iod->sg, anv->iod_mempool);
 546         return ret;
 547 }
 548
 549 static __always_inline void apple_nvme_unmap_rq(struct request *req)
 550 {
 551         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 552         struct apple_nvme *anv = queue_to_apple_nvme(iod->q);
 553
 554         if (blk_rq_nr_phys_segments(req))
 555                 apple_nvme_unmap_data(anv, req);
 556 }
 557
 558 static void apple_nvme_complete_rq(struct request *req)
 559 {
 560         apple_nvme_unmap_rq(req);
 561         nvme_complete_rq(req);
 562 }
 563
 564 static void apple_nvme_complete_batch(struct io_comp_batch *iob)
 565 {
 566         nvme_complete_batch(iob, apple_nvme_unmap_rq);
 567 }
 568
 569 static inline bool apple_nvme_cqe_pending(struct apple_nvme_queue *q)
 570 {
 571         struct nvme_completion *hcqe = &q->cqes[q->cq_head];
 572
 573         return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == q->cq_phase;
 574 }
 575
 576 static inline struct blk_mq_tags *
 577 apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q)
 578 {
 579         if (q->is_adminq)
 580                 return anv->admin_tagset.tags[0];
 581         else
 582                 return anv->tagset.tags[0];
 583 }
 584
 585 static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q,
 586                                          struct io_comp_batch *iob, u16 idx)
 587 {
 588         struct apple_nvme *anv = queue_to_apple_nvme(q);
 589         struct nvme_completion *cqe = &q->cqes[idx];
 590         __u16 command_id = READ_ONCE(cqe->command_id);
 591         struct request *req;
 592
 593         apple_nvmmu_inval(q, command_id);
 594
 595         req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id);
 596         if (unlikely(!req)) {
 597                 dev_warn(anv->dev, "invalid id %d completed", command_id);
 598                 return;
 599         }
 600
 601         if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
 602             !blk_mq_add_to_batch(req, iob, nvme_req(req)->status,
 603                                  apple_nvme_complete_batch))
 604                 apple_nvme_complete_rq(req);
 605 }
 606
 607 static inline void apple_nvme_update_cq_head(struct apple_nvme_queue *q)
 608 {
 609         u32 tmp = q->cq_head + 1;
 610
 611         if (tmp == apple_nvme_queue_depth(q)) {
 612                 q->cq_head = 0;
 613                 q->cq_phase ^= 1;
 614         } else {
 615                 q->cq_head = tmp;
 616         }
 617 }
 618
 619 static bool apple_nvme_poll_cq(struct apple_nvme_queue *q,
 620                                struct io_comp_batch *iob)
 621 {
 622         bool found = false;
 623
 624         while (apple_nvme_cqe_pending(q)) {
 625                 found = true;
 626
 627                 /*
 628                  * load-load control dependency between phase and the rest of
 629                  * the cqe requires a full read memory barrier
 630                  */
 631                 dma_rmb();
 632                 apple_nvme_handle_cqe(q, iob, q->cq_head);
 633                 apple_nvme_update_cq_head(q);
 634         }
 635
 636         if (found)
 637                 writel(q->cq_head, q->cq_db);
 638
 639         return found;
 640 }
 641
 642 static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force)
 643 {
 644         bool found;
 645         DEFINE_IO_COMP_BATCH(iob);
 646
 647         if (!READ_ONCE(q->enabled) && !force)
 648                 return false;
 649
 650         found = apple_nvme_poll_cq(q, &iob);
 651
 652         if (!rq_list_empty(iob.req_list))
 653                 apple_nvme_complete_batch(&iob);
 654
 655         return found;
 656 }
 657
 658 static irqreturn_t apple_nvme_irq(int irq, void *data)
 659 {
 660         struct apple_nvme *anv = data;
 661         bool handled = false;
 662         unsigned long flags;
 663
 664         spin_lock_irqsave(&anv->lock, flags);
 665         if (apple_nvme_handle_cq(&anv->ioq, false))
 666                 handled = true;
 667         if (apple_nvme_handle_cq(&anv->adminq, false))
 668                 handled = true;
 669         spin_unlock_irqrestore(&anv->lock, flags);
 670
 671         if (handled)
 672                 return IRQ_HANDLED;
 673         return IRQ_NONE;
 674 }
 675
 676 static int apple_nvme_create_cq(struct apple_nvme *anv)
 677 {
 678         struct nvme_command c = {};
 679
 680         /*
 681          * Note: we (ab)use the fact that the prp fields survive if no data
 682          * is attached to the request.
 683          */
 684         c.create_cq.opcode = nvme_admin_create_cq;
 685         c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr);
 686         c.create_cq.cqid = cpu_to_le16(1);
 687         c.create_cq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1);
 688         c.create_cq.cq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED);
 689         c.create_cq.irq_vector = cpu_to_le16(0);
 690
 691         return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
 692 }
 693
 694 static int apple_nvme_remove_cq(struct apple_nvme *anv)
 695 {
 696         struct nvme_command c = {};
 697
 698         c.delete_queue.opcode = nvme_admin_delete_cq;
 699         c.delete_queue.qid = cpu_to_le16(1);
 700
 701         return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
 702 }
 703
 704 static int apple_nvme_create_sq(struct apple_nvme *anv)
 705 {
 706         struct nvme_command c = {};
 707
 708         /*
 709          * Note: we (ab)use the fact that the prp fields survive if no data
 710          * is attached to the request.
 711          */
 712         c.create_sq.opcode = nvme_admin_create_sq;
 713         c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr);
 714         c.create_sq.sqid = cpu_to_le16(1);
 715         c.create_sq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1);
 716         c.create_sq.sq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG);
 717         c.create_sq.cqid = cpu_to_le16(1);
 718
 719         return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
 720 }
 721
 722 static int apple_nvme_remove_sq(struct apple_nvme *anv)
 723 {
 724         struct nvme_command c = {};
 725
 726         c.delete_queue.opcode = nvme_admin_delete_sq;
 727         c.delete_queue.qid = cpu_to_le16(1);
 728
 729         return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
 730 }
 731
 732 static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 733                                         const struct blk_mq_queue_data *bd)
 734 {
 735         struct nvme_ns *ns = hctx->queue->queuedata;
 736         struct apple_nvme_queue *q = hctx->driver_data;
 737         struct apple_nvme *anv = queue_to_apple_nvme(q);
 738         struct request *req = bd->rq;
 739         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 740         struct nvme_command *cmnd = &iod->cmd;
 741         blk_status_t ret;
 742
 743         iod->npages = -1;
 744         iod->nents = 0;
 745
 746         /*
 747          * We should not need to do this, but we're still using this to
 748          * ensure we can drain requests on a dying queue.
 749          */
 750         if (unlikely(!READ_ONCE(q->enabled)))
 751                 return BLK_STS_IOERR;
 752
 753         if (!nvme_check_ready(&anv->ctrl, req, true))
 754                 return nvme_fail_nonready_command(&anv->ctrl, req);
 755
 756         ret = nvme_setup_cmd(ns, req);
 757         if (ret)
 758                 return ret;
 759
 760         if (blk_rq_nr_phys_segments(req)) {
 761                 ret = apple_nvme_map_data(anv, req, cmnd);
 762                 if (ret)
 763                         goto out_free_cmd;
 764         }
 765
 766         nvme_start_request(req);
 767         apple_nvme_submit_cmd(q, cmnd);
 768         return BLK_STS_OK;
 769
 770 out_free_cmd:
 771         nvme_cleanup_cmd(req);
 772         return ret;
 773 }
 774
 775 static int apple_nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 776                                 unsigned int hctx_idx)
 777 {
 778         hctx->driver_data = data;
 779         return 0;
 780 }
 781
 782 static int apple_nvme_init_request(struct blk_mq_tag_set *set,
 783                                    struct request *req, unsigned int hctx_idx,
 784                                    unsigned int numa_node)
 785 {
 786         struct apple_nvme_queue *q = set->driver_data;
 787         struct apple_nvme *anv = queue_to_apple_nvme(q);
 788         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 789         struct nvme_request *nreq = nvme_req(req);
 790
 791         iod->q = q;
 792         nreq->ctrl = &anv->ctrl;
 793         nreq->cmd = &iod->cmd;
 794
 795         return 0;
 796 }
 797
 798 static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown)
 799 {
 800         enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl);
 801         u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
 802         bool dead = false, freeze = false;
 803         unsigned long flags;
 804
 805         if (apple_rtkit_is_crashed(anv->rtk))
 806                 dead = true;
 807         if (!(csts & NVME_CSTS_RDY))
 808                 dead = true;
 809         if (csts & NVME_CSTS_CFS)
 810                 dead = true;
 811
 812         if (state == NVME_CTRL_LIVE ||
 813             state == NVME_CTRL_RESETTING) {
 814                 freeze = true;
 815                 nvme_start_freeze(&anv->ctrl);
 816         }
 817
 818         /*
 819          * Give the controller a chance to complete all entered requests if
 820          * doing a safe shutdown.
 821          */
 822         if (!dead && shutdown && freeze)
 823                 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT);
 824
 825         nvme_quiesce_io_queues(&anv->ctrl);
 826
 827         if (!dead) {
 828                 if (READ_ONCE(anv->ioq.enabled)) {
 829                         apple_nvme_remove_sq(anv);
 830                         apple_nvme_remove_cq(anv);
 831                 }
 832
 833                 /*
 834                  * Always disable the NVMe controller after shutdown.
 835                  * We need to do this to bring it back up later anyway, and we
 836                  * can't do it while the firmware is not running (e.g. in the
 837                  * resume reset path before RTKit is initialized), so for Apple
 838                  * controllers it makes sense to unconditionally do it here.
 839                  * Additionally, this sequence of events is reliable, while
 840                  * others (like disabling after bringing back the firmware on
 841                  * resume) seem to run into trouble under some circumstances.
 842                  *
 843                  * Both U-Boot and m1n1 also use this convention (i.e. an ANS
 844                  * NVMe controller is handed off with firmware shut down, in an
 845                  * NVMe disabled state, after a clean shutdown).
 846                  */
 847                 if (shutdown)
 848                         nvme_disable_ctrl(&anv->ctrl, shutdown);
 849                 nvme_disable_ctrl(&anv->ctrl, false);
 850         }
 851
 852         WRITE_ONCE(anv->ioq.enabled, false);
 853         WRITE_ONCE(anv->adminq.enabled, false);
 854         mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */
 855         nvme_quiesce_admin_queue(&anv->ctrl);
 856
 857         /* last chance to complete any requests before nvme_cancel_request */
 858         spin_lock_irqsave(&anv->lock, flags);
 859         apple_nvme_handle_cq(&anv->ioq, true);
 860         apple_nvme_handle_cq(&anv->adminq, true);
 861         spin_unlock_irqrestore(&anv->lock, flags);
 862
 863         nvme_cancel_tagset(&anv->ctrl);
 864         nvme_cancel_admin_tagset(&anv->ctrl);
 865
 866         /*
 867          * The driver will not be starting up queues again if shutting down so
 868          * must flush all entered requests to their failed completion to avoid
 869          * deadlocking blk-mq hot-cpu notifier.
 870          */
 871         if (shutdown) {
 872                 nvme_unquiesce_io_queues(&anv->ctrl);
 873                 nvme_unquiesce_admin_queue(&anv->ctrl);
 874         }
 875 }
 876
 877 static enum blk_eh_timer_return apple_nvme_timeout(struct request *req)
 878 {
 879         struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
 880         struct apple_nvme_queue *q = iod->q;
 881         struct apple_nvme *anv = queue_to_apple_nvme(q);
 882         unsigned long flags;
 883         u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
 884
 885         if (nvme_ctrl_state(&anv->ctrl) != NVME_CTRL_LIVE) {
 886                 /*
 887                  * From rdma.c:
 888                  * If we are resetting, connecting or deleting we should
 889                  * complete immediately because we may block controller
 890                  * teardown or setup sequence
 891                  * - ctrl disable/shutdown fabrics requests
 892                  * - connect requests
 893                  * - initialization admin requests
 894                  * - I/O requests that entered after unquiescing and
 895                  *   the controller stopped responding
 896                  *
 897                  * All other requests should be cancelled by the error
 898                  * recovery work, so it's fine that we fail it here.
 899                  */
 900                 dev_warn(anv->dev,
 901                          "I/O %d(aq:%d) timeout while not in live state\n",
 902                          req->tag, q->is_adminq);
 903                 if (blk_mq_request_started(req) &&
 904                     !blk_mq_request_completed(req)) {
 905                         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 906                         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 907                         blk_mq_complete_request(req);
 908                 }
 909                 return BLK_EH_DONE;
 910         }
 911
 912         /* check if we just missed an interrupt if we're still alive */
 913         if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) {
 914                 spin_lock_irqsave(&anv->lock, flags);
 915                 apple_nvme_handle_cq(q, false);
 916                 spin_unlock_irqrestore(&anv->lock, flags);
 917                 if (blk_mq_request_completed(req)) {
 918                         dev_warn(anv->dev,
 919                                  "I/O %d(aq:%d) timeout: completion polled\n",
 920                                  req->tag, q->is_adminq);
 921                         return BLK_EH_DONE;
 922                 }
 923         }
 924
 925         /*
 926          * aborting commands isn't supported which leaves a full reset as our
 927          * only option here
 928          */
 929         dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n",
 930                  req->tag, q->is_adminq);
 931         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 932         apple_nvme_disable(anv, false);
 933         nvme_reset_ctrl(&anv->ctrl);
 934         return BLK_EH_DONE;
 935 }
 936
 937 static int apple_nvme_poll(struct blk_mq_hw_ctx *hctx,
 938                            struct io_comp_batch *iob)
 939 {
 940         struct apple_nvme_queue *q = hctx->driver_data;
 941         struct apple_nvme *anv = queue_to_apple_nvme(q);
 942         bool found;
 943         unsigned long flags;
 944
 945         spin_lock_irqsave(&anv->lock, flags);
 946         found = apple_nvme_poll_cq(q, iob);
 947         spin_unlock_irqrestore(&anv->lock, flags);
 948
 949         return found;
 950 }
 951
 952 static const struct blk_mq_ops apple_nvme_mq_admin_ops = {
 953         .queue_rq = apple_nvme_queue_rq,
 954         .complete = apple_nvme_complete_rq,
 955         .init_hctx = apple_nvme_init_hctx,
 956         .init_request = apple_nvme_init_request,
 957         .timeout = apple_nvme_timeout,
 958 };
 959
 960 static const struct blk_mq_ops apple_nvme_mq_ops = {
 961         .queue_rq = apple_nvme_queue_rq,
 962         .complete = apple_nvme_complete_rq,
 963         .init_hctx = apple_nvme_init_hctx,
 964         .init_request = apple_nvme_init_request,
 965         .timeout = apple_nvme_timeout,
 966         .poll = apple_nvme_poll,
 967 };
 968
 969 static void apple_nvme_init_queue(struct apple_nvme_queue *q)
 970 {
 971         unsigned int depth = apple_nvme_queue_depth(q);
 972
 973         q->cq_head = 0;
 974         q->cq_phase = 1;
 975         memset(q->tcbs, 0,
 976                APPLE_ANS_MAX_QUEUE_DEPTH * sizeof(struct apple_nvmmu_tcb));
 977         memset(q->cqes, 0, depth * sizeof(struct nvme_completion));
 978         WRITE_ONCE(q->enabled, true);
 979         wmb(); /* ensure the first interrupt sees the initialization */
 980 }
 981
 982 static void apple_nvme_reset_work(struct work_struct *work)
 983 {
 984         unsigned int nr_io_queues = 1;
 985         int ret;
 986         u32 boot_status, aqa;
 987         struct apple_nvme *anv =
 988                 container_of(work, struct apple_nvme, ctrl.reset_work);
 989         enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl);
 990
 991         if (state != NVME_CTRL_RESETTING) {
 992                 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state);
 993                 ret = -ENODEV;
 994                 goto out;
 995         }
 996
 997         /* there's unfortunately no known way to recover if RTKit crashed :( */
 998         if (apple_rtkit_is_crashed(anv->rtk)) {
 999                 dev_err(anv->dev,
1000                         "RTKit has crashed without any way to recover.");
1001                 ret = -EIO;
1002                 goto out;
1003         }
1004
1005         /* RTKit must be shut down cleanly for the (soft)-reset to work */
1006         if (apple_rtkit_is_running(anv->rtk)) {
1007                 /* reset the controller if it is enabled */
1008                 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE)
1009                         apple_nvme_disable(anv, false);
1010                 dev_dbg(anv->dev, "Trying to shut down RTKit before reset.");
1011                 ret = apple_rtkit_shutdown(anv->rtk);
1012                 if (ret)
1013                         goto out;
1014         }
1015
1016         writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1017
1018         ret = reset_control_assert(anv->reset);
1019         if (ret)
1020                 goto out;
1021
1022         ret = apple_rtkit_reinit(anv->rtk);
1023         if (ret)
1024                 goto out;
1025
1026         ret = reset_control_deassert(anv->reset);
1027         if (ret)
1028                 goto out;
1029
1030         writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN,
1031                anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1032         ret = apple_rtkit_boot(anv->rtk);
1033         if (ret) {
1034                 dev_err(anv->dev, "ANS did not boot");
1035                 goto out;
1036         }
1037
1038         ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS,
1039                                  boot_status,
1040                                  boot_status == APPLE_ANS_BOOT_STATUS_OK,
1041                                  USEC_PER_MSEC, APPLE_ANS_BOOT_TIMEOUT);
1042         if (ret) {
1043                 dev_err(anv->dev, "ANS did not initialize");
1044                 goto out;
1045         }
1046
1047         dev_dbg(anv->dev, "ANS booted successfully.");
1048
1049         /*
1050          * Limit the max command size to prevent iod->sg allocations going
1051          * over a single page.
1052          */
1053         anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1,
1054                                          dma_max_mapping_size(anv->dev) >> 9);
1055         anv->ctrl.max_segments = NVME_MAX_SEGS;
1056
1057         dma_set_max_seg_size(anv->dev, 0xffffffff);
1058
1059         /*
1060          * Enable NVMMU and linear submission queues.
1061          * While we could keep those disabled and pretend this is slightly
1062          * more common NVMe controller we'd still need some quirks (e.g.
1063          * sq entries will be 128 bytes) and Apple might drop support for
1064          * that mode in the future.
1065          */
1066         writel(APPLE_ANS_LINEAR_SQ_EN,
1067                anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL);
1068
1069         /* Allow as many pending command as possible for both queues */
1070         writel(APPLE_ANS_MAX_QUEUE_DEPTH | (APPLE_ANS_MAX_QUEUE_DEPTH << 16),
1071                anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL);
1072
1073         /* Setup the NVMMU for the maximum admin and IO queue depth */
1074         writel(APPLE_ANS_MAX_QUEUE_DEPTH - 1,
1075                anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS);
1076
1077         /*
1078          * This is probably a chicken bit: without it all commands where any PRP
1079          * is set to zero (including those that don't use that field) fail and
1080          * the co-processor complains about "completed with err BAD_CMD-" or
1081          * a "NULL_PRP_PTR_ERR" in the syslog
1082          */
1083         writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) &
1084                        ~APPLE_ANS_PRP_NULL_CHECK,
1085                anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL);
1086
1087         /* Setup the admin queue */
1088         aqa = APPLE_NVME_AQ_DEPTH - 1;
1089         aqa |= aqa << 16;
1090         writel(aqa, anv->mmio_nvme + NVME_REG_AQA);
1091         writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ);
1092         writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ);
1093
1094         /* Setup NVMMU for both queues */
1095         writeq(anv->adminq.tcb_dma_addr,
1096                anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE);
1097         writeq(anv->ioq.tcb_dma_addr,
1098                anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE);
1099
1100         anv->ctrl.sqsize =
1101                 APPLE_ANS_MAX_QUEUE_DEPTH - 1; /* 0's based queue depth */
1102         anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP);
1103
1104         dev_dbg(anv->dev, "Enabling controller now");
1105         ret = nvme_enable_ctrl(&anv->ctrl);
1106         if (ret)
1107                 goto out;
1108
1109         dev_dbg(anv->dev, "Starting admin queue");
1110         apple_nvme_init_queue(&anv->adminq);
1111         nvme_unquiesce_admin_queue(&anv->ctrl);
1112
1113         if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) {
1114                 dev_warn(anv->ctrl.device,
1115                          "failed to mark controller CONNECTING\n");
1116                 ret = -ENODEV;
1117                 goto out;
1118         }
1119
1120         ret = nvme_init_ctrl_finish(&anv->ctrl, false);
1121         if (ret)
1122                 goto out;
1123
1124         dev_dbg(anv->dev, "Creating IOCQ");
1125         ret = apple_nvme_create_cq(anv);
1126         if (ret)
1127                 goto out;
1128         dev_dbg(anv->dev, "Creating IOSQ");
1129         ret = apple_nvme_create_sq(anv);
1130         if (ret)
1131                 goto out_remove_cq;
1132
1133         apple_nvme_init_queue(&anv->ioq);
1134         nr_io_queues = 1;
1135         ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues);
1136         if (ret)
1137                 goto out_remove_sq;
1138         if (nr_io_queues != 1) {
1139                 ret = -ENXIO;
1140                 goto out_remove_sq;
1141         }
1142
1143         anv->ctrl.queue_count = nr_io_queues + 1;
1144
1145         nvme_unquiesce_io_queues(&anv->ctrl);
1146         nvme_wait_freeze(&anv->ctrl);
1147         blk_mq_update_nr_hw_queues(&anv->tagset, 1);
1148         nvme_unfreeze(&anv->ctrl);
1149
1150         if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) {
1151                 dev_warn(anv->ctrl.device,
1152                          "failed to mark controller live state\n");
1153                 ret = -ENODEV;
1154                 goto out_remove_sq;
1155         }
1156
1157         nvme_start_ctrl(&anv->ctrl);
1158
1159         dev_dbg(anv->dev, "ANS boot and NVMe init completed.");
1160         return;
1161
1162 out_remove_sq:
1163         apple_nvme_remove_sq(anv);
1164 out_remove_cq:
1165         apple_nvme_remove_cq(anv);
1166 out:
1167         dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret);
1168         nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1169         nvme_get_ctrl(&anv->ctrl);
1170         apple_nvme_disable(anv, false);
1171         nvme_mark_namespaces_dead(&anv->ctrl);
1172         if (!queue_work(nvme_wq, &anv->remove_work))
1173                 nvme_put_ctrl(&anv->ctrl);
1174 }
1175
1176 static void apple_nvme_remove_dead_ctrl_work(struct work_struct *work)
1177 {
1178         struct apple_nvme *anv =
1179                 container_of(work, struct apple_nvme, remove_work);
1180
1181         nvme_put_ctrl(&anv->ctrl);
1182         device_release_driver(anv->dev);
1183 }
1184
1185 static int apple_nvme_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
1186 {
1187         *val = readl(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1188         return 0;
1189 }
1190
1191 static int apple_nvme_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
1192 {
1193         writel(val, ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1194         return 0;
1195 }
1196
1197 static int apple_nvme_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
1198 {
1199         *val = readq(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1200         return 0;
1201 }
1202
1203 static int apple_nvme_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
1204 {
1205         struct device *dev = ctrl_to_apple_nvme(ctrl)->dev;
1206
1207         return snprintf(buf, size, "%s\n", dev_name(dev));
1208 }
1209
1210 static void apple_nvme_free_ctrl(struct nvme_ctrl *ctrl)
1211 {
1212         struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl);
1213
1214         if (anv->ctrl.admin_q)
1215                 blk_put_queue(anv->ctrl.admin_q);
1216         put_device(anv->dev);
1217 }
1218
1219 static const struct nvme_ctrl_ops nvme_ctrl_ops = {
1220         .name = "apple-nvme",
1221         .module = THIS_MODULE,
1222         .flags = 0,
1223         .reg_read32 = apple_nvme_reg_read32,
1224         .reg_write32 = apple_nvme_reg_write32,
1225         .reg_read64 = apple_nvme_reg_read64,
1226         .free_ctrl = apple_nvme_free_ctrl,
1227         .get_address = apple_nvme_get_address,
1228 };
1229
1230 static void apple_nvme_async_probe(void *data, async_cookie_t cookie)
1231 {
1232         struct apple_nvme *anv = data;
1233
1234         flush_work(&anv->ctrl.reset_work);
1235         flush_work(&anv->ctrl.scan_work);
1236         nvme_put_ctrl(&anv->ctrl);
1237 }
1238
1239 static void devm_apple_nvme_put_tag_set(void *data)
1240 {
1241         blk_mq_free_tag_set(data);
1242 }
1243
1244 static int apple_nvme_alloc_tagsets(struct apple_nvme *anv)
1245 {
1246         int ret;
1247
1248         anv->admin_tagset.ops = &apple_nvme_mq_admin_ops;
1249         anv->admin_tagset.nr_hw_queues = 1;
1250         anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH;
1251         anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT;
1252         anv->admin_tagset.numa_node = NUMA_NO_NODE;
1253         anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod);
1254         anv->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
1255         anv->admin_tagset.driver_data = &anv->adminq;
1256
1257         ret = blk_mq_alloc_tag_set(&anv->admin_tagset);
1258         if (ret)
1259                 return ret;
1260         ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set,
1261                                        &anv->admin_tagset);
1262         if (ret)
1263                 return ret;
1264
1265         anv->tagset.ops = &apple_nvme_mq_ops;
1266         anv->tagset.nr_hw_queues = 1;
1267         anv->tagset.nr_maps = 1;
1268         /*
1269          * Tags are used as an index to the NVMMU and must be unique across
1270          * both queues. The admin queue gets the first APPLE_NVME_AQ_DEPTH which
1271          * must be marked as reserved in the IO queue.
1272          */
1273         anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH;
1274         anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1;
1275         anv->tagset.timeout = NVME_IO_TIMEOUT;
1276         anv->tagset.numa_node = NUMA_NO_NODE;
1277         anv->tagset.cmd_size = sizeof(struct apple_nvme_iod);
1278         anv->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1279         anv->tagset.driver_data = &anv->ioq;
1280
1281         ret = blk_mq_alloc_tag_set(&anv->tagset);
1282         if (ret)
1283                 return ret;
1284         ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set,
1285                                         &anv->tagset);
1286         if (ret)
1287                 return ret;
1288
1289         anv->ctrl.admin_tagset = &anv->admin_tagset;
1290         anv->ctrl.tagset = &anv->tagset;
1291
1292         return 0;
1293 }
1294
1295 static int apple_nvme_queue_alloc(struct apple_nvme *anv,
1296                                   struct apple_nvme_queue *q)
1297 {
1298         unsigned int depth = apple_nvme_queue_depth(q);
1299
1300         q->cqes = dmam_alloc_coherent(anv->dev,
1301                                       depth * sizeof(struct nvme_completion),
1302                                       &q->cq_dma_addr, GFP_KERNEL);
1303         if (!q->cqes)
1304                 return -ENOMEM;
1305
1306         q->sqes = dmam_alloc_coherent(anv->dev,
1307                                       depth * sizeof(struct nvme_command),
1308                                       &q->sq_dma_addr, GFP_KERNEL);
1309         if (!q->sqes)
1310                 return -ENOMEM;
1311
1312         /*
1313          * We need the maximum queue depth here because the NVMMU only has a
1314          * single depth configuration shared between both queues.
1315          */
1316         q->tcbs = dmam_alloc_coherent(anv->dev,
1317                                       APPLE_ANS_MAX_QUEUE_DEPTH *
1318                                               sizeof(struct apple_nvmmu_tcb),
1319                                       &q->tcb_dma_addr, GFP_KERNEL);
1320         if (!q->tcbs)
1321                 return -ENOMEM;
1322
1323         /*
1324          * initialize phase to make sure the allocated and empty memory
1325          * doesn't look like a full cq already.
1326          */
1327         q->cq_phase = 1;
1328         return 0;
1329 }
1330
1331 static void apple_nvme_detach_genpd(struct apple_nvme *anv)
1332 {
1333         int i;
1334
1335         if (anv->pd_count <= 1)
1336                 return;
1337
1338         for (i = anv->pd_count - 1; i >= 0; i--) {
1339                 if (anv->pd_link[i])
1340                         device_link_del(anv->pd_link[i]);
1341                 if (!IS_ERR_OR_NULL(anv->pd_dev[i]))
1342                         dev_pm_domain_detach(anv->pd_dev[i], true);
1343         }
1344 }
1345
1346 static int apple_nvme_attach_genpd(struct apple_nvme *anv)
1347 {
1348         struct device *dev = anv->dev;
1349         int i;
1350
1351         anv->pd_count = of_count_phandle_with_args(
1352                 dev->of_node, "power-domains", "#power-domain-cells");
1353         if (anv->pd_count <= 1)
1354                 return 0;
1355
1356         anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev),
1357                                    GFP_KERNEL);
1358         if (!anv->pd_dev)
1359                 return -ENOMEM;
1360
1361         anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link),
1362                                     GFP_KERNEL);
1363         if (!anv->pd_link)
1364                 return -ENOMEM;
1365
1366         for (i = 0; i < anv->pd_count; i++) {
1367                 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i);
1368                 if (IS_ERR(anv->pd_dev[i])) {
1369                         apple_nvme_detach_genpd(anv);
1370                         return PTR_ERR(anv->pd_dev[i]);
1371                 }
1372
1373                 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i],
1374                                                   DL_FLAG_STATELESS |
1375                                                   DL_FLAG_PM_RUNTIME |
1376                                                   DL_FLAG_RPM_ACTIVE);
1377                 if (!anv->pd_link[i]) {
1378                         apple_nvme_detach_genpd(anv);
1379                         return -EINVAL;
1380                 }
1381         }
1382
1383         return 0;
1384 }
1385
1386 static void devm_apple_nvme_mempool_destroy(void *data)
1387 {
1388         mempool_destroy(data);
1389 }
1390
1391 static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev)
1392 {
1393         struct device *dev = &pdev->dev;
1394         struct apple_nvme *anv;
1395         int ret;
1396
1397         anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL);
1398         if (!anv)
1399                 return ERR_PTR(-ENOMEM);
1400
1401         anv->dev = get_device(dev);
1402         anv->adminq.is_adminq = true;
1403         platform_set_drvdata(pdev, anv);
1404
1405         ret = apple_nvme_attach_genpd(anv);
1406         if (ret < 0) {
1407                 dev_err_probe(dev, ret, "Failed to attach power domains");
1408                 goto put_dev;
1409         }
1410         if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) {
1411                 ret = -ENXIO;
1412                 goto put_dev;
1413         }
1414
1415         anv->irq = platform_get_irq(pdev, 0);
1416         if (anv->irq < 0) {
1417                 ret = anv->irq;
1418                 goto put_dev;
1419         }
1420         if (!anv->irq) {
1421                 ret = -ENXIO;
1422                 goto put_dev;
1423         }
1424
1425         anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans");
1426         if (IS_ERR(anv->mmio_coproc)) {
1427                 ret = PTR_ERR(anv->mmio_coproc);
1428                 goto put_dev;
1429         }
1430         anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme");
1431         if (IS_ERR(anv->mmio_nvme)) {
1432                 ret = PTR_ERR(anv->mmio_nvme);
1433                 goto put_dev;
1434         }
1435
1436         anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB;
1437         anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB;
1438         anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB;
1439         anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB;
1440
1441         anv->sart = devm_apple_sart_get(dev);
1442         if (IS_ERR(anv->sart)) {
1443                 ret = dev_err_probe(dev, PTR_ERR(anv->sart),
1444                                     "Failed to initialize SART");
1445                 goto put_dev;
1446         }
1447
1448         anv->reset = devm_reset_control_array_get_exclusive(anv->dev);
1449         if (IS_ERR(anv->reset)) {
1450                 ret = dev_err_probe(dev, PTR_ERR(anv->reset),
1451                                     "Failed to get reset control");
1452                 goto put_dev;
1453         }
1454
1455         INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work);
1456         INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work);
1457         spin_lock_init(&anv->lock);
1458
1459         ret = apple_nvme_queue_alloc(anv, &anv->adminq);
1460         if (ret)
1461                 goto put_dev;
1462         ret = apple_nvme_queue_alloc(anv, &anv->ioq);
1463         if (ret)
1464                 goto put_dev;
1465
1466         anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev,
1467                                               NVME_CTRL_PAGE_SIZE,
1468                                               NVME_CTRL_PAGE_SIZE, 0);
1469         if (!anv->prp_page_pool) {
1470                 ret = -ENOMEM;
1471                 goto put_dev;
1472         }
1473
1474         anv->prp_small_pool =
1475                 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0);
1476         if (!anv->prp_small_pool) {
1477                 ret = -ENOMEM;
1478                 goto put_dev;
1479         }
1480
1481         WARN_ON_ONCE(apple_nvme_iod_alloc_size() > PAGE_SIZE);
1482         anv->iod_mempool =
1483                 mempool_create_kmalloc_pool(1, apple_nvme_iod_alloc_size());
1484         if (!anv->iod_mempool) {
1485                 ret = -ENOMEM;
1486                 goto put_dev;
1487         }
1488         ret = devm_add_action_or_reset(anv->dev,
1489                         devm_apple_nvme_mempool_destroy, anv->iod_mempool);
1490         if (ret)
1491                 goto put_dev;
1492
1493         ret = apple_nvme_alloc_tagsets(anv);
1494         if (ret)
1495                 goto put_dev;
1496
1497         ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0,
1498                                "nvme-apple", anv);
1499         if (ret) {
1500                 dev_err_probe(dev, ret, "Failed to request IRQ");
1501                 goto put_dev;
1502         }
1503
1504         anv->rtk =
1505                 devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops);
1506         if (IS_ERR(anv->rtk)) {
1507                 ret = dev_err_probe(dev, PTR_ERR(anv->rtk),
1508                                     "Failed to initialize RTKit");
1509                 goto put_dev;
1510         }
1511
1512         ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops,
1513                              NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS);
1514         if (ret) {
1515                 dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl");
1516                 goto put_dev;
1517         }
1518
1519         return anv;
1520 put_dev:
1521         put_device(anv->dev);
1522         return ERR_PTR(ret);
1523 }
1524
1525 static int apple_nvme_probe(struct platform_device *pdev)
1526 {
1527         struct apple_nvme *anv;
1528         int ret;
1529
1530         anv = apple_nvme_alloc(pdev);
1531         if (IS_ERR(anv))
1532                 return PTR_ERR(anv);
1533
1534         ret = nvme_add_ctrl(&anv->ctrl);
1535         if (ret)
1536                 goto out_put_ctrl;
1537
1538         anv->ctrl.admin_q = blk_mq_alloc_queue(&anv->admin_tagset, NULL, NULL);
1539         if (IS_ERR(anv->ctrl.admin_q)) {
1540                 ret = -ENOMEM;
1541                 anv->ctrl.admin_q = NULL;
1542                 goto out_uninit_ctrl;
1543         }
1544
1545         nvme_reset_ctrl(&anv->ctrl);
1546         async_schedule(apple_nvme_async_probe, anv);
1547
1548         return 0;
1549
1550 out_uninit_ctrl:
1551         nvme_uninit_ctrl(&anv->ctrl);
1552 out_put_ctrl:
1553         nvme_put_ctrl(&anv->ctrl);
1554         return ret;
1555 }
1556
1557 static void apple_nvme_remove(struct platform_device *pdev)
1558 {
1559         struct apple_nvme *anv = platform_get_drvdata(pdev);
1560
1561         nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1562         flush_work(&anv->ctrl.reset_work);
1563         nvme_stop_ctrl(&anv->ctrl);
1564         nvme_remove_namespaces(&anv->ctrl);
1565         apple_nvme_disable(anv, true);
1566         nvme_uninit_ctrl(&anv->ctrl);
1567
1568         if (apple_rtkit_is_running(anv->rtk))
1569                 apple_rtkit_shutdown(anv->rtk);
1570
1571         apple_nvme_detach_genpd(anv);
1572 }
1573
1574 static void apple_nvme_shutdown(struct platform_device *pdev)
1575 {
1576         struct apple_nvme *anv = platform_get_drvdata(pdev);
1577
1578         apple_nvme_disable(anv, true);
1579         if (apple_rtkit_is_running(anv->rtk))
1580                 apple_rtkit_shutdown(anv->rtk);
1581 }
1582
1583 static int apple_nvme_resume(struct device *dev)
1584 {
1585         struct apple_nvme *anv = dev_get_drvdata(dev);
1586
1587         return nvme_reset_ctrl(&anv->ctrl);
1588 }
1589
1590 static int apple_nvme_suspend(struct device *dev)
1591 {
1592         struct apple_nvme *anv = dev_get_drvdata(dev);
1593         int ret = 0;
1594
1595         apple_nvme_disable(anv, true);
1596
1597         if (apple_rtkit_is_running(anv->rtk))
1598                 ret = apple_rtkit_shutdown(anv->rtk);
1599
1600         writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1601
1602         return ret;
1603 }
1604
1605 static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend,
1606                                 apple_nvme_resume);
1607
1608 static const struct of_device_id apple_nvme_of_match[] = {
1609         { .compatible = "apple,nvme-ans2" },
1610         {},
1611 };
1612 MODULE_DEVICE_TABLE(of, apple_nvme_of_match);
1613
1614 static struct platform_driver apple_nvme_driver = {
1615         .driver = {
1616                 .name = "nvme-apple",
1617                 .of_match_table = apple_nvme_of_match,
1618                 .pm = pm_sleep_ptr(&apple_nvme_pm_ops),
1619         },
1620         .probe = apple_nvme_probe,
1621         .remove_new = apple_nvme_remove,
1622         .shutdown = apple_nvme_shutdown,
1623 };
1624 module_platform_driver(apple_nvme_driver);
1625
1626 MODULE_AUTHOR("Sven Peter <[email protected]>");
1627 MODULE_DESCRIPTION("Apple ANS NVM Express device driver");
1628 MODULE_LICENSE("GPL");