drivers/iommu/fsl_pamu_domain.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright (C) 2013 Freescale Semiconductor, Inc.
   5  * Author: Varun Sethi <[email protected]>
   6  */
   7
   8 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
   9
  10 #include "fsl_pamu_domain.h"
  11
  12 #include <sysdev/fsl_pci.h>
  13
  14 /*
  15  * Global spinlock that needs to be held while
  16  * configuring PAMU.
  17  */
  18 static DEFINE_SPINLOCK(iommu_lock);
  19
  20 static struct kmem_cache *fsl_pamu_domain_cache;
  21 static struct kmem_cache *iommu_devinfo_cache;
  22 static DEFINE_SPINLOCK(device_domain_lock);
  23
  24 struct iommu_device pamu_iommu; /* IOMMU core code handle */
  25
  26 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
  27 {
  28         return container_of(dom, struct fsl_dma_domain, iommu_domain);
  29 }
  30
  31 static int __init iommu_init_mempool(void)
  32 {
  33         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  34                                                   sizeof(struct fsl_dma_domain),
  35                                                   0,
  36                                                   SLAB_HWCACHE_ALIGN,
  37                                                   NULL);
  38         if (!fsl_pamu_domain_cache) {
  39                 pr_debug("Couldn't create fsl iommu_domain cache\n");
  40                 return -ENOMEM;
  41         }
  42
  43         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  44                                                 sizeof(struct device_domain_info),
  45                                                 0,
  46                                                 SLAB_HWCACHE_ALIGN,
  47                                                 NULL);
  48         if (!iommu_devinfo_cache) {
  49                 pr_debug("Couldn't create devinfo cache\n");
  50                 kmem_cache_destroy(fsl_pamu_domain_cache);
  51                 return -ENOMEM;
  52         }
  53
  54         return 0;
  55 }
  56
  57 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
  58 {
  59         u32 win_cnt = dma_domain->win_cnt;
  60         struct dma_window *win_ptr = &dma_domain->win_arr[0];
  61         struct iommu_domain_geometry *geom;
  62
  63         geom = &dma_domain->iommu_domain.geometry;
  64
  65         if (!win_cnt || !dma_domain->geom_size) {
  66                 pr_debug("Number of windows/geometry not configured for the domain\n");
  67                 return 0;
  68         }
  69
  70         if (win_cnt > 1) {
  71                 u64 subwin_size;
  72                 dma_addr_t subwin_iova;
  73                 u32 wnd;
  74
  75                 subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
  76                 subwin_iova = iova & ~(subwin_size - 1);
  77                 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
  78                 win_ptr = &dma_domain->win_arr[wnd];
  79         }
  80
  81         if (win_ptr->valid)
  82                 return win_ptr->paddr + (iova & (win_ptr->size - 1));
  83
  84         return 0;
  85 }
  86
  87 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
  88 {
  89         struct dma_window *sub_win_ptr = &dma_domain->win_arr[0];
  90         int i, ret;
  91         unsigned long rpn, flags;
  92
  93         for (i = 0; i < dma_domain->win_cnt; i++) {
  94                 if (sub_win_ptr[i].valid) {
  95                         rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT;
  96                         spin_lock_irqsave(&iommu_lock, flags);
  97                         ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
  98                                                  sub_win_ptr[i].size,
  99                                                  ~(u32)0,
 100                                                  rpn,
 101                                                  dma_domain->snoop_id,
 102                                                  dma_domain->stash_id,
 103                                                  (i > 0) ? 1 : 0,
 104                                                  sub_win_ptr[i].prot);
 105                         spin_unlock_irqrestore(&iommu_lock, flags);
 106                         if (ret) {
 107                                 pr_debug("SPAACE configuration failed for liodn %d\n",
 108                                          liodn);
 109                                 return ret;
 110                         }
 111                 }
 112         }
 113
 114         return ret;
 115 }
 116
 117 static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
 118 {
 119         int ret;
 120         struct dma_window *wnd = &dma_domain->win_arr[0];
 121         phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
 122         unsigned long flags;
 123
 124         spin_lock_irqsave(&iommu_lock, flags);
 125         ret = pamu_config_ppaace(liodn, wnd_addr,
 126                                  wnd->size,
 127                                  ~(u32)0,
 128                                  wnd->paddr >> PAMU_PAGE_SHIFT,
 129                                  dma_domain->snoop_id, dma_domain->stash_id,
 130                                  0, wnd->prot);
 131         spin_unlock_irqrestore(&iommu_lock, flags);
 132         if (ret)
 133                 pr_debug("PAACE configuration failed for liodn %d\n", liodn);
 134
 135         return ret;
 136 }
 137
 138 /* Map the DMA window corresponding to the LIODN */
 139 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
 140 {
 141         if (dma_domain->win_cnt > 1)
 142                 return map_subwins(liodn, dma_domain);
 143         else
 144                 return map_win(liodn, dma_domain);
 145 }
 146
 147 /* Update window/subwindow mapping for the LIODN */
 148 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 149 {
 150         int ret;
 151         struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
 152         unsigned long flags;
 153
 154         spin_lock_irqsave(&iommu_lock, flags);
 155         if (dma_domain->win_cnt > 1) {
 156                 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
 157                                          wnd->size,
 158                                          ~(u32)0,
 159                                          wnd->paddr >> PAMU_PAGE_SHIFT,
 160                                          dma_domain->snoop_id,
 161                                          dma_domain->stash_id,
 162                                          (wnd_nr > 0) ? 1 : 0,
 163                                          wnd->prot);
 164                 if (ret)
 165                         pr_debug("Subwindow reconfiguration failed for liodn %d\n",
 166                                  liodn);
 167         } else {
 168                 phys_addr_t wnd_addr;
 169
 170                 wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
 171
 172                 ret = pamu_config_ppaace(liodn, wnd_addr,
 173                                          wnd->size,
 174                                          ~(u32)0,
 175                                          wnd->paddr >> PAMU_PAGE_SHIFT,
 176                                          dma_domain->snoop_id, dma_domain->stash_id,
 177                                          0, wnd->prot);
 178                 if (ret)
 179                         pr_debug("Window reconfiguration failed for liodn %d\n",
 180                                  liodn);
 181         }
 182
 183         spin_unlock_irqrestore(&iommu_lock, flags);
 184
 185         return ret;
 186 }
 187
 188 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
 189                               u32 val)
 190 {
 191         int ret = 0, i;
 192         unsigned long flags;
 193
 194         spin_lock_irqsave(&iommu_lock, flags);
 195         if (!dma_domain->win_arr) {
 196                 pr_debug("Windows not configured, stash destination update failed for liodn %d\n",
 197                          liodn);
 198                 spin_unlock_irqrestore(&iommu_lock, flags);
 199                 return -EINVAL;
 200         }
 201
 202         for (i = 0; i < dma_domain->win_cnt; i++) {
 203                 ret = pamu_update_paace_stash(liodn, i, val);
 204                 if (ret) {
 205                         pr_debug("Failed to update SPAACE %d field for liodn %d\n ",
 206                                  i, liodn);
 207                         spin_unlock_irqrestore(&iommu_lock, flags);
 208                         return ret;
 209                 }
 210         }
 211
 212         spin_unlock_irqrestore(&iommu_lock, flags);
 213
 214         return ret;
 215 }
 216
 217 /* Set the geometry parameters for a LIODN */
 218 static int pamu_set_liodn(int liodn, struct device *dev,
 219                           struct fsl_dma_domain *dma_domain,
 220                           struct iommu_domain_geometry *geom_attr,
 221                           u32 win_cnt)
 222 {
 223         phys_addr_t window_addr, window_size;
 224         phys_addr_t subwin_size;
 225         int ret = 0, i;
 226         u32 omi_index = ~(u32)0;
 227         unsigned long flags;
 228
 229         /*
 230          * Configure the omi_index at the geometry setup time.
 231          * This is a static value which depends on the type of
 232          * device and would not change thereafter.
 233          */
 234         get_ome_index(&omi_index, dev);
 235
 236         window_addr = geom_attr->aperture_start;
 237         window_size = dma_domain->geom_size;
 238
 239         spin_lock_irqsave(&iommu_lock, flags);
 240         ret = pamu_disable_liodn(liodn);
 241         if (!ret)
 242                 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
 243                                          0, dma_domain->snoop_id,
 244                                          dma_domain->stash_id, win_cnt, 0);
 245         spin_unlock_irqrestore(&iommu_lock, flags);
 246         if (ret) {
 247                 pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n",
 248                          liodn, win_cnt);
 249                 return ret;
 250         }
 251
 252         if (win_cnt > 1) {
 253                 subwin_size = window_size >> ilog2(win_cnt);
 254                 for (i = 0; i < win_cnt; i++) {
 255                         spin_lock_irqsave(&iommu_lock, flags);
 256                         ret = pamu_disable_spaace(liodn, i);
 257                         if (!ret)
 258                                 ret = pamu_config_spaace(liodn, win_cnt, i,
 259                                                          subwin_size, omi_index,
 260                                                          0, dma_domain->snoop_id,
 261                                                          dma_domain->stash_id,
 262                                                          0, 0);
 263                         spin_unlock_irqrestore(&iommu_lock, flags);
 264                         if (ret) {
 265                                 pr_debug("SPAACE configuration failed for liodn %d\n",
 266                                          liodn);
 267                                 return ret;
 268                         }
 269                 }
 270         }
 271
 272         return ret;
 273 }
 274
 275 static int check_size(u64 size, dma_addr_t iova)
 276 {
 277         /*
 278          * Size must be a power of two and at least be equal
 279          * to PAMU page size.
 280          */
 281         if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) {
 282                 pr_debug("Size too small or not a power of two\n");
 283                 return -EINVAL;
 284         }
 285
 286         /* iova must be page size aligned */
 287         if (iova & (size - 1)) {
 288                 pr_debug("Address is not aligned with window size\n");
 289                 return -EINVAL;
 290         }
 291
 292         return 0;
 293 }
 294
 295 static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
 296 {
 297         struct fsl_dma_domain *domain;
 298
 299         domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 300         if (!domain)
 301                 return NULL;
 302
 303         domain->stash_id = ~(u32)0;
 304         domain->snoop_id = ~(u32)0;
 305         domain->win_cnt = pamu_get_max_subwin_cnt();
 306         domain->geom_size = 0;
 307
 308         INIT_LIST_HEAD(&domain->devices);
 309
 310         spin_lock_init(&domain->domain_lock);
 311
 312         return domain;
 313 }
 314
 315 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
 316 {
 317         unsigned long flags;
 318
 319         list_del(&info->link);
 320         spin_lock_irqsave(&iommu_lock, flags);
 321         if (win_cnt > 1)
 322                 pamu_free_subwins(info->liodn);
 323         pamu_disable_liodn(info->liodn);
 324         spin_unlock_irqrestore(&iommu_lock, flags);
 325         spin_lock_irqsave(&device_domain_lock, flags);
 326         info->dev->archdata.iommu_domain = NULL;
 327         kmem_cache_free(iommu_devinfo_cache, info);
 328         spin_unlock_irqrestore(&device_domain_lock, flags);
 329 }
 330
 331 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 332 {
 333         struct device_domain_info *info, *tmp;
 334         unsigned long flags;
 335
 336         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 337         /* Remove the device from the domain device list */
 338         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 339                 if (!dev || (info->dev == dev))
 340                         remove_device_ref(info, dma_domain->win_cnt);
 341         }
 342         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 343 }
 344
 345 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 346 {
 347         struct device_domain_info *info, *old_domain_info;
 348         unsigned long flags;
 349
 350         spin_lock_irqsave(&device_domain_lock, flags);
 351         /*
 352          * Check here if the device is already attached to domain or not.
 353          * If the device is already attached to a domain detach it.
 354          */
 355         old_domain_info = dev->archdata.iommu_domain;
 356         if (old_domain_info && old_domain_info->domain != dma_domain) {
 357                 spin_unlock_irqrestore(&device_domain_lock, flags);
 358                 detach_device(dev, old_domain_info->domain);
 359                 spin_lock_irqsave(&device_domain_lock, flags);
 360         }
 361
 362         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 363
 364         info->dev = dev;
 365         info->liodn = liodn;
 366         info->domain = dma_domain;
 367
 368         list_add(&info->link, &dma_domain->devices);
 369         /*
 370          * In case of devices with multiple LIODNs just store
 371          * the info for the first LIODN as all
 372          * LIODNs share the same domain
 373          */
 374         if (!dev->archdata.iommu_domain)
 375                 dev->archdata.iommu_domain = info;
 376         spin_unlock_irqrestore(&device_domain_lock, flags);
 377 }
 378
 379 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 380                                          dma_addr_t iova)
 381 {
 382         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 383
 384         if (iova < domain->geometry.aperture_start ||
 385             iova > domain->geometry.aperture_end)
 386                 return 0;
 387
 388         return get_phys_addr(dma_domain, iova);
 389 }
 390
 391 static bool fsl_pamu_capable(enum iommu_cap cap)
 392 {
 393         return cap == IOMMU_CAP_CACHE_COHERENCY;
 394 }
 395
 396 static void fsl_pamu_domain_free(struct iommu_domain *domain)
 397 {
 398         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 399
 400         /* remove all the devices from the device list */
 401         detach_device(NULL, dma_domain);
 402
 403         dma_domain->enabled = 0;
 404         dma_domain->mapped = 0;
 405
 406         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 407 }
 408
 409 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
 410 {
 411         struct fsl_dma_domain *dma_domain;
 412
 413         if (type != IOMMU_DOMAIN_UNMANAGED)
 414                 return NULL;
 415
 416         dma_domain = iommu_alloc_dma_domain();
 417         if (!dma_domain) {
 418                 pr_debug("dma_domain allocation failed\n");
 419                 return NULL;
 420         }
 421         /* defaul geometry 64 GB i.e. maximum system address */
 422         dma_domain->iommu_domain. geometry.aperture_start = 0;
 423         dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
 424         dma_domain->iommu_domain.geometry.force_aperture = true;
 425
 426         return &dma_domain->iommu_domain;
 427 }
 428
 429 /* Configure geometry settings for all LIODNs associated with domain */
 430 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
 431                                     struct iommu_domain_geometry *geom_attr,
 432                                     u32 win_cnt)
 433 {
 434         struct device_domain_info *info;
 435         int ret = 0;
 436
 437         list_for_each_entry(info, &dma_domain->devices, link) {
 438                 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
 439                                      geom_attr, win_cnt);
 440                 if (ret)
 441                         break;
 442         }
 443
 444         return ret;
 445 }
 446
 447 /* Update stash destination for all LIODNs associated with the domain */
 448 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 449 {
 450         struct device_domain_info *info;
 451         int ret = 0;
 452
 453         list_for_each_entry(info, &dma_domain->devices, link) {
 454                 ret = update_liodn_stash(info->liodn, dma_domain, val);
 455                 if (ret)
 456                         break;
 457         }
 458
 459         return ret;
 460 }
 461
 462 /* Update domain mappings for all LIODNs associated with the domain */
 463 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 464 {
 465         struct device_domain_info *info;
 466         int ret = 0;
 467
 468         list_for_each_entry(info, &dma_domain->devices, link) {
 469                 ret = update_liodn(info->liodn, dma_domain, wnd_nr);
 470                 if (ret)
 471                         break;
 472         }
 473         return ret;
 474 }
 475
 476 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 477 {
 478         struct device_domain_info *info;
 479         int ret = 0;
 480
 481         list_for_each_entry(info, &dma_domain->devices, link) {
 482                 if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
 483                         ret = pamu_disable_liodn(info->liodn);
 484                         if (!ret)
 485                                 dma_domain->enabled = 0;
 486                 } else {
 487                         ret = pamu_disable_spaace(info->liodn, wnd_nr);
 488                 }
 489         }
 490
 491         return ret;
 492 }
 493
 494 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
 495 {
 496         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 497         unsigned long flags;
 498         int ret;
 499
 500         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 501         if (!dma_domain->win_arr) {
 502                 pr_debug("Number of windows not configured\n");
 503                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 504                 return;
 505         }
 506
 507         if (wnd_nr >= dma_domain->win_cnt) {
 508                 pr_debug("Invalid window index\n");
 509                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 510                 return;
 511         }
 512
 513         if (dma_domain->win_arr[wnd_nr].valid) {
 514                 ret = disable_domain_win(dma_domain, wnd_nr);
 515                 if (!ret) {
 516                         dma_domain->win_arr[wnd_nr].valid = 0;
 517                         dma_domain->mapped--;
 518                 }
 519         }
 520
 521         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 522 }
 523
 524 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
 525                                   phys_addr_t paddr, u64 size, int prot)
 526 {
 527         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 528         struct dma_window *wnd;
 529         int pamu_prot = 0;
 530         int ret;
 531         unsigned long flags;
 532         u64 win_size;
 533
 534         if (prot & IOMMU_READ)
 535                 pamu_prot |= PAACE_AP_PERMS_QUERY;
 536         if (prot & IOMMU_WRITE)
 537                 pamu_prot |= PAACE_AP_PERMS_UPDATE;
 538
 539         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 540         if (!dma_domain->win_arr) {
 541                 pr_debug("Number of windows not configured\n");
 542                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 543                 return -ENODEV;
 544         }
 545
 546         if (wnd_nr >= dma_domain->win_cnt) {
 547                 pr_debug("Invalid window index\n");
 548                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 549                 return -EINVAL;
 550         }
 551
 552         win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
 553         if (size > win_size) {
 554                 pr_debug("Invalid window size\n");
 555                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 556                 return -EINVAL;
 557         }
 558
 559         if (dma_domain->win_cnt == 1) {
 560                 if (dma_domain->enabled) {
 561                         pr_debug("Disable the window before updating the mapping\n");
 562                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 563                         return -EBUSY;
 564                 }
 565
 566                 ret = check_size(size, domain->geometry.aperture_start);
 567                 if (ret) {
 568                         pr_debug("Aperture start not aligned to the size\n");
 569                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 570                         return -EINVAL;
 571                 }
 572         }
 573
 574         wnd = &dma_domain->win_arr[wnd_nr];
 575         if (!wnd->valid) {
 576                 wnd->paddr = paddr;
 577                 wnd->size = size;
 578                 wnd->prot = pamu_prot;
 579
 580                 ret = update_domain_mapping(dma_domain, wnd_nr);
 581                 if (!ret) {
 582                         wnd->valid = 1;
 583                         dma_domain->mapped++;
 584                 }
 585         } else {
 586                 pr_debug("Disable the window before updating the mapping\n");
 587                 ret = -EBUSY;
 588         }
 589
 590         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 591
 592         return ret;
 593 }
 594
 595 /*
 596  * Attach the LIODN to the DMA domain and configure the geometry
 597  * and window mappings.
 598  */
 599 static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 600                                 struct device *dev, const u32 *liodn,
 601                                 int num)
 602 {
 603         unsigned long flags;
 604         struct iommu_domain *domain = &dma_domain->iommu_domain;
 605         int ret = 0;
 606         int i;
 607
 608         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 609         for (i = 0; i < num; i++) {
 610                 /* Ensure that LIODN value is valid */
 611                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 612                         pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
 613                                  liodn[i], dev->of_node);
 614                         ret = -EINVAL;
 615                         break;
 616                 }
 617
 618                 attach_device(dma_domain, liodn[i], dev);
 619                 /*
 620                  * Check if geometry has already been configured
 621                  * for the domain. If yes, set the geometry for
 622                  * the LIODN.
 623                  */
 624                 if (dma_domain->win_arr) {
 625                         u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
 626
 627                         ret = pamu_set_liodn(liodn[i], dev, dma_domain,
 628                                              &domain->geometry, win_cnt);
 629                         if (ret)
 630                                 break;
 631                         if (dma_domain->mapped) {
 632                                 /*
 633                                  * Create window/subwindow mapping for
 634                                  * the LIODN.
 635                                  */
 636                                 ret = map_liodn(liodn[i], dma_domain);
 637                                 if (ret)
 638                                         break;
 639                         }
 640                 }
 641         }
 642         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 643
 644         return ret;
 645 }
 646
 647 static int fsl_pamu_attach_device(struct iommu_domain *domain,
 648                                   struct device *dev)
 649 {
 650         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 651         const u32 *liodn;
 652         u32 liodn_cnt;
 653         int len, ret = 0;
 654         struct pci_dev *pdev = NULL;
 655         struct pci_controller *pci_ctl;
 656
 657         /*
 658          * Use LIODN of the PCI controller while attaching a
 659          * PCI device.
 660          */
 661         if (dev_is_pci(dev)) {
 662                 pdev = to_pci_dev(dev);
 663                 pci_ctl = pci_bus_to_host(pdev->bus);
 664                 /*
 665                  * make dev point to pci controller device
 666                  * so we can get the LIODN programmed by
 667                  * u-boot.
 668                  */
 669                 dev = pci_ctl->parent;
 670         }
 671
 672         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 673         if (liodn) {
 674                 liodn_cnt = len / sizeof(u32);
 675                 ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
 676         } else {
 677                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 678                 ret = -EINVAL;
 679         }
 680
 681         return ret;
 682 }
 683
 684 static void fsl_pamu_detach_device(struct iommu_domain *domain,
 685                                    struct device *dev)
 686 {
 687         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 688         const u32 *prop;
 689         int len;
 690         struct pci_dev *pdev = NULL;
 691         struct pci_controller *pci_ctl;
 692
 693         /*
 694          * Use LIODN of the PCI controller while detaching a
 695          * PCI device.
 696          */
 697         if (dev_is_pci(dev)) {
 698                 pdev = to_pci_dev(dev);
 699                 pci_ctl = pci_bus_to_host(pdev->bus);
 700                 /*
 701                  * make dev point to pci controller device
 702                  * so we can get the LIODN programmed by
 703                  * u-boot.
 704                  */
 705                 dev = pci_ctl->parent;
 706         }
 707
 708         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 709         if (prop)
 710                 detach_device(dev, dma_domain);
 711         else
 712                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 713 }
 714
 715 static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
 716 {
 717         struct iommu_domain_geometry *geom_attr = data;
 718         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 719         dma_addr_t geom_size;
 720         unsigned long flags;
 721
 722         geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
 723         /*
 724          * Sanity check the geometry size. Also, we do not support
 725          * DMA outside of the geometry.
 726          */
 727         if (check_size(geom_size, geom_attr->aperture_start) ||
 728             !geom_attr->force_aperture) {
 729                 pr_debug("Invalid PAMU geometry attributes\n");
 730                 return -EINVAL;
 731         }
 732
 733         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 734         if (dma_domain->enabled) {
 735                 pr_debug("Can't set geometry attributes as domain is active\n");
 736                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 737                 return  -EBUSY;
 738         }
 739
 740         /* Copy the domain geometry information */
 741         memcpy(&domain->geometry, geom_attr,
 742                sizeof(struct iommu_domain_geometry));
 743         dma_domain->geom_size = geom_size;
 744
 745         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 746
 747         return 0;
 748 }
 749
 750 /* Set the domain stash attribute */
 751 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
 752 {
 753         struct pamu_stash_attribute *stash_attr = data;
 754         unsigned long flags;
 755         int ret;
 756
 757         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 758
 759         memcpy(&dma_domain->dma_stash, stash_attr,
 760                sizeof(struct pamu_stash_attribute));
 761
 762         dma_domain->stash_id = get_stash_id(stash_attr->cache,
 763                                             stash_attr->cpu);
 764         if (dma_domain->stash_id == ~(u32)0) {
 765                 pr_debug("Invalid stash attributes\n");
 766                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 767                 return -EINVAL;
 768         }
 769
 770         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 771
 772         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 773
 774         return ret;
 775 }
 776
 777 /* Configure domain dma state i.e. enable/disable DMA */
 778 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
 779 {
 780         struct device_domain_info *info;
 781         unsigned long flags;
 782         int ret;
 783
 784         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 785
 786         if (enable && !dma_domain->mapped) {
 787                 pr_debug("Can't enable DMA domain without valid mapping\n");
 788                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 789                 return -ENODEV;
 790         }
 791
 792         dma_domain->enabled = enable;
 793         list_for_each_entry(info, &dma_domain->devices, link) {
 794                 ret = (enable) ? pamu_enable_liodn(info->liodn) :
 795                         pamu_disable_liodn(info->liodn);
 796                 if (ret)
 797                         pr_debug("Unable to set dma state for liodn %d",
 798                                  info->liodn);
 799         }
 800         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 801
 802         return 0;
 803 }
 804
 805 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
 806 {
 807         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 808         unsigned long flags;
 809         int ret;
 810
 811         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 812         /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
 813         if (dma_domain->enabled) {
 814                 pr_debug("Can't set geometry attributes as domain is active\n");
 815                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 816                 return  -EBUSY;
 817         }
 818
 819         /* Ensure that the geometry has been set for the domain */
 820         if (!dma_domain->geom_size) {
 821                 pr_debug("Please configure geometry before setting the number of windows\n");
 822                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 823                 return -EINVAL;
 824         }
 825
 826         /*
 827          * Ensure we have valid window count i.e. it should be less than
 828          * maximum permissible limit and should be a power of two.
 829          */
 830         if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
 831                 pr_debug("Invalid window count\n");
 832                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 833                 return -EINVAL;
 834         }
 835
 836         ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
 837                                        w_count > 1 ? w_count : 0);
 838         if (!ret) {
 839                 kfree(dma_domain->win_arr);
 840                 dma_domain->win_arr = kcalloc(w_count,
 841                                               sizeof(*dma_domain->win_arr),
 842                                               GFP_ATOMIC);
 843                 if (!dma_domain->win_arr) {
 844                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 845                         return -ENOMEM;
 846                 }
 847                 dma_domain->win_cnt = w_count;
 848         }
 849         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 850
 851         return ret;
 852 }
 853
 854 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
 855                                     enum iommu_attr attr_type, void *data)
 856 {
 857         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 858         int ret = 0;
 859
 860         switch (attr_type) {
 861         case DOMAIN_ATTR_GEOMETRY:
 862                 ret = configure_domain_geometry(domain, data);
 863                 break;
 864         case DOMAIN_ATTR_FSL_PAMU_STASH:
 865                 ret = configure_domain_stash(dma_domain, data);
 866                 break;
 867         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 868                 ret = configure_domain_dma_state(dma_domain, *(int *)data);
 869                 break;
 870         case DOMAIN_ATTR_WINDOWS:
 871                 ret = fsl_pamu_set_windows(domain, *(u32 *)data);
 872                 break;
 873         default:
 874                 pr_debug("Unsupported attribute type\n");
 875                 ret = -EINVAL;
 876                 break;
 877         }
 878
 879         return ret;
 880 }
 881
 882 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
 883                                     enum iommu_attr attr_type, void *data)
 884 {
 885         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 886         int ret = 0;
 887
 888         switch (attr_type) {
 889         case DOMAIN_ATTR_FSL_PAMU_STASH:
 890                 memcpy(data, &dma_domain->dma_stash,
 891                        sizeof(struct pamu_stash_attribute));
 892                 break;
 893         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 894                 *(int *)data = dma_domain->enabled;
 895                 break;
 896         case DOMAIN_ATTR_FSL_PAMUV1:
 897                 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
 898                 break;
 899         case DOMAIN_ATTR_WINDOWS:
 900                 *(u32 *)data = dma_domain->win_cnt;
 901                 break;
 902         default:
 903                 pr_debug("Unsupported attribute type\n");
 904                 ret = -EINVAL;
 905                 break;
 906         }
 907
 908         return ret;
 909 }
 910
 911 static struct iommu_group *get_device_iommu_group(struct device *dev)
 912 {
 913         struct iommu_group *group;
 914
 915         group = iommu_group_get(dev);
 916         if (!group)
 917                 group = iommu_group_alloc();
 918
 919         return group;
 920 }
 921
 922 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 923 {
 924         u32 version;
 925
 926         /* Check the PCI controller version number by readding BRR1 register */
 927         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 928         version &= PCI_FSL_BRR1_VER;
 929         /* If PCI controller version is >= 0x204 we can partition endpoints */
 930         return version >= 0x204;
 931 }
 932
 933 /* Get iommu group information from peer devices or devices on the parent bus */
 934 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
 935 {
 936         struct pci_dev *tmp;
 937         struct iommu_group *group;
 938         struct pci_bus *bus = pdev->bus;
 939
 940         /*
 941          * Traverese the pci bus device list to get
 942          * the shared iommu group.
 943          */
 944         while (bus) {
 945                 list_for_each_entry(tmp, &bus->devices, bus_list) {
 946                         if (tmp == pdev)
 947                                 continue;
 948                         group = iommu_group_get(&tmp->dev);
 949                         if (group)
 950                                 return group;
 951                 }
 952
 953                 bus = bus->parent;
 954         }
 955
 956         return NULL;
 957 }
 958
 959 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
 960 {
 961         struct pci_controller *pci_ctl;
 962         bool pci_endpt_partitioning;
 963         struct iommu_group *group = NULL;
 964
 965         pci_ctl = pci_bus_to_host(pdev->bus);
 966         pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
 967         /* We can partition PCIe devices so assign device group to the device */
 968         if (pci_endpt_partitioning) {
 969                 group = pci_device_group(&pdev->dev);
 970
 971                 /*
 972                  * PCIe controller is not a paritionable entity
 973                  * free the controller device iommu_group.
 974                  */
 975                 if (pci_ctl->parent->iommu_group)
 976                         iommu_group_remove_device(pci_ctl->parent);
 977         } else {
 978                 /*
 979                  * All devices connected to the controller will share the
 980                  * PCI controllers device group. If this is the first
 981                  * device to be probed for the pci controller, copy the
 982                  * device group information from the PCI controller device
 983                  * node and remove the PCI controller iommu group.
 984                  * For subsequent devices, the iommu group information can
 985                  * be obtained from sibling devices (i.e. from the bus_devices
 986                  * link list).
 987                  */
 988                 if (pci_ctl->parent->iommu_group) {
 989                         group = get_device_iommu_group(pci_ctl->parent);
 990                         iommu_group_remove_device(pci_ctl->parent);
 991                 } else {
 992                         group = get_shared_pci_device_group(pdev);
 993                 }
 994         }
 995
 996         if (!group)
 997                 group = ERR_PTR(-ENODEV);
 998
 999         return group;
1000 }
1001
1002 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
1003 {
1004         struct iommu_group *group = ERR_PTR(-ENODEV);
1005         int len;
1006
1007         /*
1008          * For platform devices we allocate a separate group for
1009          * each of the devices.
1010          */
1011         if (dev_is_pci(dev))
1012                 group = get_pci_device_group(to_pci_dev(dev));
1013         else if (of_get_property(dev->of_node, "fsl,liodn", &len))
1014                 group = get_device_iommu_group(dev);
1015
1016         return group;
1017 }
1018
1019 static int fsl_pamu_add_device(struct device *dev)
1020 {
1021         struct iommu_group *group;
1022
1023         group = iommu_group_get_for_dev(dev);
1024         if (IS_ERR(group))
1025                 return PTR_ERR(group);
1026
1027         iommu_group_put(group);
1028
1029         iommu_device_link(&pamu_iommu, dev);
1030
1031         return 0;
1032 }
1033
1034 static void fsl_pamu_remove_device(struct device *dev)
1035 {
1036         iommu_device_unlink(&pamu_iommu, dev);
1037         iommu_group_remove_device(dev);
1038 }
1039
1040 static const struct iommu_ops fsl_pamu_ops = {
1041         .capable        = fsl_pamu_capable,
1042         .domain_alloc   = fsl_pamu_domain_alloc,
1043         .domain_free    = fsl_pamu_domain_free,
1044         .attach_dev     = fsl_pamu_attach_device,
1045         .detach_dev     = fsl_pamu_detach_device,
1046         .domain_window_enable = fsl_pamu_window_enable,
1047         .domain_window_disable = fsl_pamu_window_disable,
1048         .iova_to_phys   = fsl_pamu_iova_to_phys,
1049         .domain_set_attr = fsl_pamu_set_domain_attr,
1050         .domain_get_attr = fsl_pamu_get_domain_attr,
1051         .add_device     = fsl_pamu_add_device,
1052         .remove_device  = fsl_pamu_remove_device,
1053         .device_group   = fsl_pamu_device_group,
1054 };
1055
1056 int __init pamu_domain_init(void)
1057 {
1058         int ret = 0;
1059
1060         ret = iommu_init_mempool();
1061         if (ret)
1062                 return ret;
1063
1064         ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
1065         if (ret)
1066                 return ret;
1067
1068         iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
1069
1070         ret = iommu_device_register(&pamu_iommu);
1071         if (ret) {
1072                 iommu_device_sysfs_remove(&pamu_iommu);
1073                 pr_err("Can't register iommu device\n");
1074                 return ret;
1075         }
1076
1077         bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
1078         bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
1079
1080         return ret;
1081 }