drivers/nvme/nvme_apple.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * (C) Copyright 2021 Mark Kettenis <[email protected]>
   4  */
   5
   6 #include <dm.h>
   7 #include <mailbox.h>
   8 #include <mapmem.h>
   9 #include "nvme.h"
  10 #include <reset.h>
  11
  12 #include <asm/io.h>
  13 #include <asm/arch/rtkit.h>
  14 #include <asm/arch/sart.h>
  15 #include <linux/iopoll.h>
  16
  17 /* ASC registers */
  18 #define REG_CPU_CTRL            0x0044
  19 #define  REG_CPU_CTRL_RUN       BIT(4)
  20
  21 /* Apple NVMe registers */
  22 #define ANS_MAX_PEND_CMDS_CTRL  0x01210
  23 #define  ANS_MAX_QUEUE_DEPTH    64
  24 #define ANS_BOOT_STATUS         0x01300
  25 #define  ANS_BOOT_STATUS_OK     0xde71ce55
  26 #define ANS_MODESEL             0x01304
  27 #define ANS_UNKNOWN_CTRL        0x24008
  28 #define  ANS_PRP_NULL_CHECK     (1 << 11)
  29 #define ANS_LINEAR_SQ_CTRL      0x24908
  30 #define  ANS_LINEAR_SQ_CTRL_EN  (1 << 0)
  31 #define ANS_ASQ_DB              0x2490c
  32 #define ANS_IOSQ_DB             0x24910
  33 #define ANS_NVMMU_NUM           0x28100
  34 #define ANS_NVMMU_BASE_ASQ      0x28108
  35 #define ANS_NVMMU_BASE_IOSQ     0x28110
  36 #define ANS_NVMMU_TCB_INVAL     0x28118
  37 #define ANS_NVMMU_TCB_STAT      0x28120
  38
  39 #define ANS_NVMMU_TCB_SIZE      0x4000
  40 #define ANS_NVMMU_TCB_PITCH     0x80
  41
  42 /*
  43  * The Apple NVMe controller includes an IOMMU known as NVMMU.  The
  44  * NVMMU is programmed through an array of TCBs. These TCBs are paired
  45  * with the corresponding slot in the submission queues and need to be
  46  * configured with the command details before a command is allowed to
  47  * execute. This is necessary even for commands that don't do DMA.
  48  */
  49 struct ans_nvmmu_tcb {
  50         u8 opcode;
  51         u8 flags;
  52         u8 slot;
  53         u8 pad0;
  54         u32 prpl_len;
  55         u8 pad1[16];
  56         u64 prp1;
  57         u64 prp2;
  58 };
  59
  60 #define ANS_NVMMU_TCB_WRITE     BIT(0)
  61 #define ANS_NVMMU_TCB_READ      BIT(1)
  62
  63 struct apple_nvme_priv {
  64         struct nvme_dev ndev;
  65         void *base;             /* NVMe registers */
  66         void *asc;              /* ASC registers */
  67         struct reset_ctl_bulk resets; /* ASC reset */
  68         struct mbox_chan chan;
  69         struct apple_sart *sart;
  70         struct apple_rtkit *rtk;
  71         struct ans_nvmmu_tcb *tcbs[NVME_Q_NUM]; /* Submission queue TCBs */
  72         u32 __iomem *q_db[NVME_Q_NUM]; /* Submission queue doorbell */
  73 };
  74
  75 static int apple_nvme_setup_queue(struct nvme_queue *nvmeq)
  76 {
  77         struct apple_nvme_priv *priv =
  78                 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
  79         struct nvme_dev *dev = nvmeq->dev;
  80
  81         switch (nvmeq->qid) {
  82         case NVME_ADMIN_Q:
  83         case NVME_IO_Q:
  84                 break;
  85         default:
  86                 return -EINVAL;
  87         }
  88
  89         priv->tcbs[nvmeq->qid] = (void *)memalign(4096, ANS_NVMMU_TCB_SIZE);
  90         memset((void *)priv->tcbs[nvmeq->qid], 0, ANS_NVMMU_TCB_SIZE);
  91
  92         switch (nvmeq->qid) {
  93         case NVME_ADMIN_Q:
  94                 priv->q_db[nvmeq->qid] =
  95                         ((void __iomem *)dev->bar) + ANS_ASQ_DB;
  96                 nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
  97                             ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_ASQ);
  98                 break;
  99         case NVME_IO_Q:
 100                 priv->q_db[nvmeq->qid] =
 101                         ((void __iomem *)dev->bar) + ANS_IOSQ_DB;
 102                 nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
 103                             ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_IOSQ);
 104                 break;
 105         }
 106
 107         return 0;
 108 }
 109
 110 static void apple_nvme_submit_cmd(struct nvme_queue *nvmeq,
 111                                   struct nvme_command *cmd)
 112 {
 113         struct apple_nvme_priv *priv =
 114                 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
 115         struct ans_nvmmu_tcb *tcb;
 116         u16 tail = nvmeq->sq_tail;
 117
 118         tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
 119         memset(tcb, 0, sizeof(*tcb));
 120         tcb->opcode = cmd->common.opcode;
 121         tcb->flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ;
 122         tcb->slot = tail;
 123         tcb->prpl_len = cmd->rw.length;
 124         tcb->prp1 = cmd->common.prp1;
 125         tcb->prp2 = cmd->common.prp2;
 126
 127         writel(tail, priv->q_db[nvmeq->qid]);
 128 }
 129
 130 static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq,
 131                                     struct nvme_command *cmd)
 132 {
 133         struct apple_nvme_priv *priv =
 134                 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
 135         struct ans_nvmmu_tcb *tcb;
 136         u16 tail = nvmeq->sq_tail;
 137
 138         tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
 139         memset(tcb, 0, sizeof(*tcb));
 140         writel(tail, ((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_INVAL);
 141         readl(((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_STAT);
 142
 143         if (++tail == nvmeq->q_depth)
 144                 tail = 0;
 145         nvmeq->sq_tail = tail;
 146 }
 147
 148 static int nvme_shmem_setup(void *cookie, struct apple_rtkit_buffer *buf)
 149 {
 150         struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
 151
 152         if (!buf || buf->dva || !buf->size)
 153                 return -1;
 154
 155         buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K));
 156         if (!buf->buffer)
 157                 return -ENOMEM;
 158
 159         if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) {
 160                 free(buf->buffer);
 161                 buf->buffer = NULL;
 162                 buf->size = 0;
 163                 return -1;
 164         }
 165
 166         buf->dva = (u64)buf->buffer;
 167
 168         return 0;
 169 }
 170
 171 static void nvme_shmem_destroy(void *cookie, struct apple_rtkit_buffer *buf)
 172 {
 173         struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
 174
 175         if (!buf)
 176                 return;
 177
 178         if (buf->buffer) {
 179                 sart_remove_allowed_region(priv->sart, buf->buffer, buf->size);
 180                 free(buf->buffer);
 181                 buf->buffer = NULL;
 182                 buf->size = 0;
 183                 buf->dva = 0;
 184         }
 185 }
 186
 187 static int apple_nvme_probe(struct udevice *dev)
 188 {
 189         struct apple_nvme_priv *priv = dev_get_priv(dev);
 190         fdt_addr_t addr;
 191         ofnode of_sart;
 192         u32 ctrl, stat, phandle;
 193         int ret;
 194
 195         priv->base = dev_read_addr_ptr(dev);
 196         if (!priv->base)
 197                 return -EINVAL;
 198
 199         addr = dev_read_addr_index(dev, 1);
 200         if (addr == FDT_ADDR_T_NONE)
 201                 return -EINVAL;
 202         priv->asc = map_sysmem(addr, 0);
 203
 204         ret = reset_get_bulk(dev, &priv->resets);
 205         if (ret < 0)
 206                 return ret;
 207
 208         ret = mbox_get_by_index(dev, 0, &priv->chan);
 209         if (ret < 0)
 210                 return ret;
 211
 212         ret = dev_read_u32(dev, "apple,sart", &phandle);
 213         if (ret < 0)
 214                 return ret;
 215
 216         of_sart = ofnode_get_by_phandle(phandle);
 217         priv->sart = sart_init(of_sart);
 218         if (!priv->sart)
 219                 return -EINVAL;
 220
 221         ctrl = readl(priv->asc + REG_CPU_CTRL);
 222         writel(ctrl | REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
 223
 224         priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy);
 225         if (!priv->rtk)
 226                 return -ENOMEM;
 227
 228         ret = apple_rtkit_boot(priv->rtk);
 229         if (ret < 0) {
 230                 printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret);
 231                 return ret;
 232         }
 233
 234         ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat,
 235                                        (stat == ANS_BOOT_STATUS_OK), 100,
 236                                        500000);
 237         if (ret < 0) {
 238                 printf("%s: NVMe firmware didn't boot\n", __func__);
 239                 return -ETIMEDOUT;
 240         }
 241
 242         writel(ANS_LINEAR_SQ_CTRL_EN, priv->base + ANS_LINEAR_SQ_CTRL);
 243         writel(((ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH),
 244                priv->base + ANS_MAX_PEND_CMDS_CTRL);
 245
 246         writel(readl(priv->base + ANS_UNKNOWN_CTRL) & ~ANS_PRP_NULL_CHECK,
 247                priv->base + ANS_UNKNOWN_CTRL);
 248
 249         strcpy(priv->ndev.vendor, "Apple");
 250
 251         writel((ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1,
 252                priv->base + ANS_NVMMU_NUM);
 253         writel(0, priv->base + ANS_MODESEL);
 254
 255         priv->ndev.bar = priv->base;
 256         return nvme_init(dev);
 257 }
 258
 259 static int apple_nvme_remove(struct udevice *dev)
 260 {
 261         struct apple_nvme_priv *priv = dev_get_priv(dev);
 262         u32 ctrl;
 263
 264         nvme_shutdown(dev);
 265
 266         apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
 267
 268         ctrl = readl(priv->asc + REG_CPU_CTRL);
 269         writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
 270
 271         apple_rtkit_free(priv->rtk);
 272         priv->rtk = NULL;
 273
 274         sart_free(priv->sart);
 275         priv->sart = NULL;
 276
 277         reset_assert_bulk(&priv->resets);
 278         reset_deassert_bulk(&priv->resets);
 279
 280         return 0;
 281 }
 282
 283 static const struct nvme_ops apple_nvme_ops = {
 284         .setup_queue = apple_nvme_setup_queue,
 285         .submit_cmd = apple_nvme_submit_cmd,
 286         .complete_cmd = apple_nvme_complete_cmd,
 287 };
 288
 289 static const struct udevice_id apple_nvme_ids[] = {
 290         { .compatible = "apple,nvme-ans2" },
 291         { /* sentinel */ }
 292 };
 293
 294 U_BOOT_DRIVER(apple_nvme) = {
 295         .name = "apple_nvme",
 296         .id = UCLASS_NVME,
 297         .of_match = apple_nvme_ids,
 298         .priv_auto = sizeof(struct apple_nvme_priv),
 299         .probe = apple_nvme_probe,
 300         .remove = apple_nvme_remove,
 301         .ops = &apple_nvme_ops,
 302         .flags = DM_FLAG_OS_PREPARE,
 303 };