drivers/iommu/apple_dart.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (C) 2021 Mark Kettenis <[email protected]>
   4  */
   5
   6 #include <cpu_func.h>
   7 #include <dm.h>
   8 #include <iommu.h>
   9 #include <lmb.h>
  10 #include <memalign.h>
  11 #include <asm/io.h>
  12
  13 #define DART_PARAMS2            0x0004
  14 #define  DART_PARAMS2_BYPASS_SUPPORT    BIT(0)
  15
  16 #define DART_T8020_TLB_CMD              0x0020
  17 #define  DART_T8020_TLB_CMD_FLUSH               BIT(20)
  18 #define  DART_T8020_TLB_CMD_BUSY                BIT(2)
  19 #define DART_T8020_TLB_SIDMASK          0x0034
  20 #define DART_T8020_ERROR                0x0040
  21 #define DART_T8020_ERROR_ADDR_LO        0x0050
  22 #define DART_T8020_ERROR_ADDR_HI        0x0054
  23 #define DART_T8020_CONFIG               0x0060
  24 #define  DART_T8020_CONFIG_LOCK                 BIT(15)
  25 #define DART_T8020_SID_ENABLE           0x00fc
  26 #define DART_T8020_TCR_BASE             0x0100
  27 #define  DART_T8020_TCR_TRANSLATE_ENABLE        BIT(7)
  28 #define  DART_T8020_TCR_BYPASS_DART             BIT(8)
  29 #define  DART_T8020_TCR_BYPASS_DAPF             BIT(12)
  30 #define DART_T8020_TTBR_BASE            0x0200
  31 #define  DART_T8020_TTBR_VALID                  BIT(31)
  32
  33 #define DART_T8110_PARAMS4              0x000c
  34 #define  DART_T8110_PARAMS4_NSID_MASK           (0x1ff << 0)
  35 #define DART_T8110_TLB_CMD              0x0080
  36 #define  DART_T8110_TLB_CMD_BUSY                BIT(31)
  37 #define  DART_T8110_TLB_CMD_FLUSH_ALL           BIT(8)
  38 #define DART_T8110_ERROR                0x0100
  39 #define DART_T8110_ERROR_MASK           0x0104
  40 #define DART_T8110_ERROR_ADDR_LO        0x0170
  41 #define DART_T8110_ERROR_ADDR_HI        0x0174
  42 #define DART_T8110_PROTECT              0x0200
  43 #define  DART_T8110_PROTECT_TTBR_TCR            BIT(0)
  44 #define DART_T8110_SID_ENABLE_BASE      0x0c00
  45 #define DART_T8110_TCR_BASE             0x1000
  46 #define  DART_T8110_TCR_BYPASS_DAPF             BIT(2)
  47 #define  DART_T8110_TCR_BYPASS_DART             BIT(1)
  48 #define  DART_T8110_TCR_TRANSLATE_ENABLE        BIT(0)
  49 #define DART_T8110_TTBR_BASE            0x1400
  50 #define  DART_T8110_TTBR_VALID                  BIT(0)
  51
  52 #define DART_SID_ENABLE(priv, idx) \
  53         ((priv)->sid_enable_base + 4 * (idx))
  54 #define DART_TCR(priv, sid)     ((priv)->tcr_base + 4 * (sid))
  55 #define DART_TTBR(priv, sid, idx)       \
  56         ((priv)->ttbr_base + 4 * (priv)->nttbr * (sid) + 4 * (idx))
  57 #define  DART_TTBR_SHIFT        12
  58
  59 #define DART_ALL_STREAMS(priv)  ((1U << (priv)->nsid) - 1)
  60
  61 #define DART_PAGE_SIZE          SZ_16K
  62 #define DART_PAGE_MASK          (DART_PAGE_SIZE - 1)
  63
  64 #define DART_L1_TABLE           0x3
  65 #define DART_L2_INVAL           0
  66 #define DART_L2_VALID           BIT(0)
  67 #define DART_L2_FULL_PAGE       BIT(1)
  68 #define DART_L2_START(addr)     ((((addr) & DART_PAGE_MASK) >> 2) << 52)
  69 #define DART_L2_END(addr)       ((((addr) & DART_PAGE_MASK) >> 2) << 40)
  70
  71 struct apple_dart_priv {
  72         void *base;
  73         struct lmb lmb;
  74         u64 *l1, *l2;
  75         int bypass, shift;
  76
  77         dma_addr_t dvabase;
  78         dma_addr_t dvaend;
  79
  80         int nsid;
  81         int nttbr;
  82         int sid_enable_base;
  83         int tcr_base;
  84         u32 tcr_translate_enable;
  85         u32 tcr_bypass;
  86         int ttbr_base;
  87         u32 ttbr_valid;
  88         void (*flush_tlb)(struct apple_dart_priv *priv);
  89 };
  90
  91 static void apple_dart_t8020_flush_tlb(struct apple_dart_priv *priv)
  92 {
  93         dsb();
  94
  95         writel(DART_ALL_STREAMS(priv), priv->base + DART_T8020_TLB_SIDMASK);
  96         writel(DART_T8020_TLB_CMD_FLUSH, priv->base + DART_T8020_TLB_CMD);
  97         while (readl(priv->base + DART_T8020_TLB_CMD) &
  98                DART_T8020_TLB_CMD_BUSY)
  99                 continue;
 100 }
 101
 102 static void apple_dart_t8110_flush_tlb(struct apple_dart_priv *priv)
 103 {
 104         dsb();
 105
 106         writel(DART_T8110_TLB_CMD_FLUSH_ALL,
 107                priv->base + DART_T8110_TLB_CMD_FLUSH_ALL);
 108         while (readl(priv->base + DART_T8110_TLB_CMD) &
 109                DART_T8110_TLB_CMD_BUSY)
 110                 continue;
 111 }
 112
 113 static dma_addr_t apple_dart_map(struct udevice *dev, void *addr, size_t size)
 114 {
 115         struct apple_dart_priv *priv = dev_get_priv(dev);
 116         phys_addr_t paddr, dva;
 117         phys_size_t psize, off;
 118         int i, idx;
 119
 120         if (priv->bypass)
 121                 return (phys_addr_t)addr;
 122
 123         paddr = ALIGN_DOWN((phys_addr_t)addr, DART_PAGE_SIZE);
 124         off = (phys_addr_t)addr - paddr;
 125         psize = ALIGN(size + off, DART_PAGE_SIZE);
 126
 127         dva = lmb_alloc(&priv->lmb, psize, DART_PAGE_SIZE);
 128
 129         idx = dva / DART_PAGE_SIZE;
 130         for (i = 0; i < psize / DART_PAGE_SIZE; i++) {
 131                 priv->l2[idx + i] = (paddr  >> priv->shift) | DART_L2_VALID |
 132                         DART_L2_START(0LL) | DART_L2_END(~0LL);
 133                 paddr += DART_PAGE_SIZE;
 134         }
 135         flush_dcache_range((unsigned long)&priv->l2[idx],
 136                            (unsigned long)&priv->l2[idx + i]);
 137         priv->flush_tlb(priv);
 138
 139         return dva + off;
 140 }
 141
 142 static void apple_dart_unmap(struct udevice *dev, dma_addr_t addr, size_t size)
 143 {
 144         struct apple_dart_priv *priv = dev_get_priv(dev);
 145         phys_addr_t dva;
 146         phys_size_t psize;
 147         int i, idx;
 148
 149         if (priv->bypass)
 150                 return;
 151
 152         dva = ALIGN_DOWN(addr, DART_PAGE_SIZE);
 153         psize = size + (addr - dva);
 154         psize = ALIGN(psize, DART_PAGE_SIZE);
 155
 156         idx = dva / DART_PAGE_SIZE;
 157         for (i = 0; i < psize / DART_PAGE_SIZE; i++)
 158                 priv->l2[idx + i] = DART_L2_INVAL;
 159         flush_dcache_range((unsigned long)&priv->l2[idx],
 160                            (unsigned long)&priv->l2[idx + i]);
 161         priv->flush_tlb(priv);
 162
 163         lmb_free(&priv->lmb, dva, psize);
 164 }
 165
 166 static struct iommu_ops apple_dart_ops = {
 167         .map = apple_dart_map,
 168         .unmap = apple_dart_unmap,
 169 };
 170
 171 static int apple_dart_probe(struct udevice *dev)
 172 {
 173         struct apple_dart_priv *priv = dev_get_priv(dev);
 174         dma_addr_t addr;
 175         phys_addr_t l2;
 176         int ntte, nl1, nl2;
 177         int sid, i;
 178         u32 params2, params4;
 179
 180         priv->base = dev_read_addr_ptr(dev);
 181         if (!priv->base)
 182                 return -EINVAL;
 183
 184         if (device_is_compatible(dev, "apple,t8110-dart")) {
 185                 params4 = readl(priv->base + DART_T8110_PARAMS4);
 186                 priv->nsid = params4 & DART_T8110_PARAMS4_NSID_MASK;
 187                 priv->nttbr = 1;
 188                 priv->sid_enable_base = DART_T8110_SID_ENABLE_BASE;
 189                 priv->tcr_base = DART_T8110_TCR_BASE;
 190                 priv->tcr_translate_enable = DART_T8110_TCR_TRANSLATE_ENABLE;
 191                 priv->tcr_bypass =
 192                     DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART;
 193                 priv->ttbr_base = DART_T8110_TTBR_BASE;
 194                 priv->ttbr_valid = DART_T8110_TTBR_VALID;
 195                 priv->flush_tlb = apple_dart_t8110_flush_tlb;
 196         } else {
 197                 priv->nsid = 16;
 198                 priv->nttbr = 4;
 199                 priv->sid_enable_base = DART_T8020_SID_ENABLE;
 200                 priv->tcr_base = DART_T8020_TCR_BASE;
 201                 priv->tcr_translate_enable = DART_T8020_TCR_TRANSLATE_ENABLE;
 202                 priv->tcr_bypass =
 203                     DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART;
 204                 priv->ttbr_base = DART_T8020_TTBR_BASE;
 205                 priv->ttbr_valid = DART_T8020_TTBR_VALID;
 206                 priv->flush_tlb = apple_dart_t8020_flush_tlb;
 207         }
 208
 209         if (device_is_compatible(dev, "apple,t6000-dart") ||
 210             device_is_compatible(dev, "apple,t8110-dart"))
 211                 priv->shift = 4;
 212
 213         priv->dvabase = DART_PAGE_SIZE;
 214         priv->dvaend = SZ_4G - DART_PAGE_SIZE;
 215
 216         lmb_init(&priv->lmb);
 217         lmb_add(&priv->lmb, priv->dvabase, priv->dvaend - priv->dvabase);
 218
 219         /* Disable translations. */
 220         for (sid = 0; sid < priv->nsid; sid++)
 221                 writel(0, priv->base + DART_TCR(priv, sid));
 222
 223         /* Remove page tables. */
 224         for (sid = 0; sid < priv->nsid; sid++) {
 225                 for (i = 0; i < priv->nttbr; i++)
 226                         writel(0, priv->base + DART_TTBR(priv, sid, i));
 227         }
 228         priv->flush_tlb(priv);
 229
 230         params2 = readl(priv->base + DART_PARAMS2);
 231         if (params2 & DART_PARAMS2_BYPASS_SUPPORT) {
 232                 for (sid = 0; sid < priv->nsid; sid++) {
 233                         writel(priv->tcr_bypass,
 234                                priv->base + DART_TCR(priv, sid));
 235                 }
 236                 priv->bypass = 1;
 237                 return 0;
 238         }
 239
 240         ntte = DIV_ROUND_UP(priv->dvaend, DART_PAGE_SIZE);
 241         nl2 = DIV_ROUND_UP(ntte, DART_PAGE_SIZE / sizeof(u64));
 242         nl1 = DIV_ROUND_UP(nl2, DART_PAGE_SIZE / sizeof(u64));
 243
 244         priv->l2 = memalign(DART_PAGE_SIZE, nl2 * DART_PAGE_SIZE);
 245         memset(priv->l2, 0, nl2 * DART_PAGE_SIZE);
 246         flush_dcache_range((unsigned long)priv->l2,
 247                            (unsigned long)priv->l2 + nl2 * DART_PAGE_SIZE);
 248
 249         priv->l1 = memalign(DART_PAGE_SIZE, nl1 * DART_PAGE_SIZE);
 250         memset(priv->l1, 0, nl1 * DART_PAGE_SIZE);
 251         l2 = (phys_addr_t)priv->l2;
 252         for (i = 0; i < nl2; i++) {
 253                 priv->l1[i] = (l2 >> priv->shift) | DART_L1_TABLE;
 254                 l2 += DART_PAGE_SIZE;
 255         }
 256         flush_dcache_range((unsigned long)priv->l1,
 257                            (unsigned long)priv->l1 + nl1 * DART_PAGE_SIZE);
 258
 259         /* Install page tables. */
 260         for (sid = 0; sid < priv->nsid; sid++) {
 261                 addr = (phys_addr_t)priv->l1;
 262                 for (i = 0; i < nl1; i++) {
 263                         writel(addr >> DART_TTBR_SHIFT | priv->ttbr_valid,
 264                                priv->base + DART_TTBR(priv, sid, i));
 265                         addr += DART_PAGE_SIZE;
 266                 }
 267         }
 268         priv->flush_tlb(priv);
 269
 270         /* Enable all streams. */
 271         for (i = 0; i < priv->nsid / 32; i++)
 272                 writel(~0, priv->base + DART_SID_ENABLE(priv, i));
 273
 274         /* Enable translations. */
 275         for (sid = 0; sid < priv->nsid; sid++) {
 276                 writel(priv->tcr_translate_enable,
 277                        priv->base + DART_TCR(priv, sid));
 278         }
 279
 280         return 0;
 281 }
 282
 283 static int apple_dart_remove(struct udevice *dev)
 284 {
 285         struct apple_dart_priv *priv = dev_get_priv(dev);
 286         int sid, i;
 287
 288         /* Disable translations. */
 289         for (sid = 0; sid < priv->nsid; sid++)
 290                 writel(0, priv->base + DART_TCR(priv, sid));
 291
 292         /* Remove page tables. */
 293         for (sid = 0; sid < priv->nsid; sid++) {
 294                 for (i = 0; i < priv->nttbr; i++)
 295                         writel(0, priv->base + DART_TTBR(priv, sid, i));
 296         }
 297         priv->flush_tlb(priv);
 298
 299         return 0;
 300 }
 301
 302 static const struct udevice_id apple_dart_ids[] = {
 303         { .compatible = "apple,t8103-dart" },
 304         { .compatible = "apple,t6000-dart" },
 305         { .compatible = "apple,t8110-dart" },
 306         { /* sentinel */ }
 307 };
 308
 309 U_BOOT_DRIVER(apple_dart) = {
 310         .name = "apple_dart",
 311         .id = UCLASS_IOMMU,
 312         .of_match = apple_dart_ids,
 313         .priv_auto = sizeof(struct apple_dart_priv),
 314         .ops = &apple_dart_ops,
 315         .probe = apple_dart_probe,
 316         .remove = apple_dart_remove,
 317         .flags  = DM_FLAG_OS_PREPARE
 318 };