]> Git Repo - linux.git/blob - arch/powerpc/platforms/pseries/iommu.c
scsi: zfcp: Trace when request remove fails after qdio send fails
[linux.git] / arch / powerpc / platforms / pseries / iommu.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
4  *
5  * Rewrite, cleanup:
6  *
7  * Copyright (C) 2004 Olof Johansson <[email protected]>, IBM Corporation
8  * Copyright (C) 2006 Olof Johansson <[email protected]>
9  *
10  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
11  */
12
13 #include <linux/init.h>
14 #include <linux/types.h>
15 #include <linux/slab.h>
16 #include <linux/mm.h>
17 #include <linux/memblock.h>
18 #include <linux/spinlock.h>
19 #include <linux/string.h>
20 #include <linux/pci.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/crash_dump.h>
23 #include <linux/memory.h>
24 #include <linux/of.h>
25 #include <linux/iommu.h>
26 #include <linux/rculist.h>
27 #include <asm/io.h>
28 #include <asm/prom.h>
29 #include <asm/rtas.h>
30 #include <asm/iommu.h>
31 #include <asm/pci-bridge.h>
32 #include <asm/machdep.h>
33 #include <asm/firmware.h>
34 #include <asm/tce.h>
35 #include <asm/ppc-pci.h>
36 #include <asm/udbg.h>
37 #include <asm/mmzone.h>
38 #include <asm/plpar_wrappers.h>
39
40 #include "pseries.h"
41
42 enum {
43         DDW_QUERY_PE_DMA_WIN  = 0,
44         DDW_CREATE_PE_DMA_WIN = 1,
45         DDW_REMOVE_PE_DMA_WIN = 2,
46
47         DDW_APPLICABLE_SIZE
48 };
49
50 enum {
51         DDW_EXT_SIZE = 0,
52         DDW_EXT_RESET_DMA_WIN = 1,
53         DDW_EXT_QUERY_OUT_SIZE = 2
54 };
55
56 static struct iommu_table *iommu_pseries_alloc_table(int node)
57 {
58         struct iommu_table *tbl;
59
60         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);
61         if (!tbl)
62                 return NULL;
63
64         INIT_LIST_HEAD_RCU(&tbl->it_group_list);
65         kref_init(&tbl->it_kref);
66         return tbl;
67 }
68
69 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
70 {
71         struct iommu_table_group *table_group;
72
73         table_group = kzalloc_node(sizeof(*table_group), GFP_KERNEL, node);
74         if (!table_group)
75                 return NULL;
76
77         table_group->tables[0] = iommu_pseries_alloc_table(node);
78         if (table_group->tables[0])
79                 return table_group;
80
81         kfree(table_group);
82         return NULL;
83 }
84
85 static void iommu_pseries_free_group(struct iommu_table_group *table_group,
86                 const char *node_name)
87 {
88         struct iommu_table *tbl;
89
90         if (!table_group)
91                 return;
92
93         tbl = table_group->tables[0];
94 #ifdef CONFIG_IOMMU_API
95         if (table_group->group) {
96                 iommu_group_put(table_group->group);
97                 BUG_ON(table_group->group);
98         }
99 #endif
100         iommu_tce_table_put(tbl);
101
102         kfree(table_group);
103 }
104
105 static int tce_build_pSeries(struct iommu_table *tbl, long index,
106                               long npages, unsigned long uaddr,
107                               enum dma_data_direction direction,
108                               unsigned long attrs)
109 {
110         u64 proto_tce;
111         __be64 *tcep;
112         u64 rpn;
113         const unsigned long tceshift = tbl->it_page_shift;
114         const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
115
116         proto_tce = TCE_PCI_READ; // Read allowed
117
118         if (direction != DMA_TO_DEVICE)
119                 proto_tce |= TCE_PCI_WRITE;
120
121         tcep = ((__be64 *)tbl->it_base) + index;
122
123         while (npages--) {
124                 /* can't move this out since we might cross MEMBLOCK boundary */
125                 rpn = __pa(uaddr) >> tceshift;
126                 *tcep = cpu_to_be64(proto_tce | rpn << tceshift);
127
128                 uaddr += pagesize;
129                 tcep++;
130         }
131         return 0;
132 }
133
134
135 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
136 {
137         __be64 *tcep;
138
139         tcep = ((__be64 *)tbl->it_base) + index;
140
141         while (npages--)
142                 *(tcep++) = 0;
143 }
144
145 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
146 {
147         __be64 *tcep;
148
149         tcep = ((__be64 *)tbl->it_base) + index;
150
151         return be64_to_cpu(*tcep);
152 }
153
154 static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
155 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
156
157 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
158                                 long npages, unsigned long uaddr,
159                                 enum dma_data_direction direction,
160                                 unsigned long attrs)
161 {
162         u64 rc = 0;
163         u64 proto_tce, tce;
164         u64 rpn;
165         int ret = 0;
166         long tcenum_start = tcenum, npages_start = npages;
167
168         rpn = __pa(uaddr) >> tceshift;
169         proto_tce = TCE_PCI_READ;
170         if (direction != DMA_TO_DEVICE)
171                 proto_tce |= TCE_PCI_WRITE;
172
173         while (npages--) {
174                 tce = proto_tce | rpn << tceshift;
175                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
176
177                 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
178                         ret = (int)rc;
179                         tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
180                                            (npages_start - (npages + 1)));
181                         break;
182                 }
183
184                 if (rc && printk_ratelimit()) {
185                         printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
186                         printk("\tindex   = 0x%llx\n", (u64)liobn);
187                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
188                         printk("\ttce val = 0x%llx\n", tce );
189                         dump_stack();
190                 }
191
192                 tcenum++;
193                 rpn++;
194         }
195         return ret;
196 }
197
198 static DEFINE_PER_CPU(__be64 *, tce_page);
199
200 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
201                                      long npages, unsigned long uaddr,
202                                      enum dma_data_direction direction,
203                                      unsigned long attrs)
204 {
205         u64 rc = 0;
206         u64 proto_tce;
207         __be64 *tcep;
208         u64 rpn;
209         long l, limit;
210         long tcenum_start = tcenum, npages_start = npages;
211         int ret = 0;
212         unsigned long flags;
213         const unsigned long tceshift = tbl->it_page_shift;
214
215         if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
216                 return tce_build_pSeriesLP(tbl->it_index, tcenum,
217                                            tceshift, npages, uaddr,
218                                            direction, attrs);
219         }
220
221         local_irq_save(flags);  /* to protect tcep and the page behind it */
222
223         tcep = __this_cpu_read(tce_page);
224
225         /* This is safe to do since interrupts are off when we're called
226          * from iommu_alloc{,_sg}()
227          */
228         if (!tcep) {
229                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
230                 /* If allocation fails, fall back to the loop implementation */
231                 if (!tcep) {
232                         local_irq_restore(flags);
233                         return tce_build_pSeriesLP(tbl->it_index, tcenum,
234                                         tceshift,
235                                         npages, uaddr, direction, attrs);
236                 }
237                 __this_cpu_write(tce_page, tcep);
238         }
239
240         rpn = __pa(uaddr) >> tceshift;
241         proto_tce = TCE_PCI_READ;
242         if (direction != DMA_TO_DEVICE)
243                 proto_tce |= TCE_PCI_WRITE;
244
245         /* We can map max one pageful of TCEs at a time */
246         do {
247                 /*
248                  * Set up the page with TCE data, looping through and setting
249                  * the values.
250                  */
251                 limit = min_t(long, npages, 4096 / TCE_ENTRY_SIZE);
252
253                 for (l = 0; l < limit; l++) {
254                         tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
255                         rpn++;
256                 }
257
258                 rc = plpar_tce_put_indirect((u64)tbl->it_index,
259                                             (u64)tcenum << tceshift,
260                                             (u64)__pa(tcep),
261                                             limit);
262
263                 npages -= limit;
264                 tcenum += limit;
265         } while (npages > 0 && !rc);
266
267         local_irq_restore(flags);
268
269         if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
270                 ret = (int)rc;
271                 tce_freemulti_pSeriesLP(tbl, tcenum_start,
272                                         (npages_start - (npages + limit)));
273                 return ret;
274         }
275
276         if (rc && printk_ratelimit()) {
277                 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
278                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
279                 printk("\tnpages  = 0x%llx\n", (u64)npages);
280                 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
281                 dump_stack();
282         }
283         return ret;
284 }
285
286 static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
287                                long npages)
288 {
289         u64 rc;
290
291         while (npages--) {
292                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
293
294                 if (rc && printk_ratelimit()) {
295                         printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
296                         printk("\tindex   = 0x%llx\n", (u64)liobn);
297                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
298                         dump_stack();
299                 }
300
301                 tcenum++;
302         }
303 }
304
305
306 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
307 {
308         u64 rc;
309
310         if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
311                 return tce_free_pSeriesLP(tbl->it_index, tcenum,
312                                           tbl->it_page_shift, npages);
313
314         rc = plpar_tce_stuff((u64)tbl->it_index,
315                              (u64)tcenum << tbl->it_page_shift, 0, npages);
316
317         if (rc && printk_ratelimit()) {
318                 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
319                 printk("\trc      = %lld\n", rc);
320                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
321                 printk("\tnpages  = 0x%llx\n", (u64)npages);
322                 dump_stack();
323         }
324 }
325
326 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
327 {
328         u64 rc;
329         unsigned long tce_ret;
330
331         rc = plpar_tce_get((u64)tbl->it_index,
332                            (u64)tcenum << tbl->it_page_shift, &tce_ret);
333
334         if (rc && printk_ratelimit()) {
335                 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
336                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
337                 printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
338                 dump_stack();
339         }
340
341         return tce_ret;
342 }
343
344 /* this is compatible with cells for the device tree property */
345 struct dynamic_dma_window_prop {
346         __be32  liobn;          /* tce table number */
347         __be64  dma_base;       /* address hi,lo */
348         __be32  tce_shift;      /* ilog2(tce_page_size) */
349         __be32  window_shift;   /* ilog2(tce_window_size) */
350 };
351
352 struct dma_win {
353         struct device_node *device;
354         const struct dynamic_dma_window_prop *prop;
355         struct list_head list;
356 };
357
358 /* Dynamic DMA Window support */
359 struct ddw_query_response {
360         u32 windows_available;
361         u64 largest_available_block;
362         u32 page_size;
363         u32 migration_capable;
364 };
365
366 struct ddw_create_response {
367         u32 liobn;
368         u32 addr_hi;
369         u32 addr_lo;
370 };
371
372 static LIST_HEAD(dma_win_list);
373 /* prevents races between memory on/offline and window creation */
374 static DEFINE_SPINLOCK(dma_win_list_lock);
375 /* protects initializing window twice for same device */
376 static DEFINE_MUTEX(dma_win_init_mutex);
377 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
378 #define DMA64_PROPNAME "linux,dma64-ddr-window-info"
379
380 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
381                                         unsigned long num_pfn, const void *arg)
382 {
383         const struct dynamic_dma_window_prop *maprange = arg;
384         int rc;
385         u64 tce_size, num_tce, dma_offset, next;
386         u32 tce_shift;
387         long limit;
388
389         tce_shift = be32_to_cpu(maprange->tce_shift);
390         tce_size = 1ULL << tce_shift;
391         next = start_pfn << PAGE_SHIFT;
392         num_tce = num_pfn << PAGE_SHIFT;
393
394         /* round back to the beginning of the tce page size */
395         num_tce += next & (tce_size - 1);
396         next &= ~(tce_size - 1);
397
398         /* covert to number of tces */
399         num_tce |= tce_size - 1;
400         num_tce >>= tce_shift;
401
402         do {
403                 /*
404                  * Set up the page with TCE data, looping through and setting
405                  * the values.
406                  */
407                 limit = min_t(long, num_tce, 512);
408                 dma_offset = next + be64_to_cpu(maprange->dma_base);
409
410                 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
411                                              dma_offset,
412                                              0, limit);
413                 next += limit * tce_size;
414                 num_tce -= limit;
415         } while (num_tce > 0 && !rc);
416
417         return rc;
418 }
419
420 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
421                                         unsigned long num_pfn, const void *arg)
422 {
423         const struct dynamic_dma_window_prop *maprange = arg;
424         u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn;
425         __be64 *tcep;
426         u32 tce_shift;
427         u64 rc = 0;
428         long l, limit;
429
430         if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
431                 unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
432                 unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
433                                 be64_to_cpu(maprange->dma_base);
434                 unsigned long tcenum = dmastart >> tceshift;
435                 unsigned long npages = num_pfn << PAGE_SHIFT >> tceshift;
436                 void *uaddr = __va(start_pfn << PAGE_SHIFT);
437
438                 return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn),
439                                 tcenum, tceshift, npages, (unsigned long) uaddr,
440                                 DMA_BIDIRECTIONAL, 0);
441         }
442
443         local_irq_disable();    /* to protect tcep and the page behind it */
444         tcep = __this_cpu_read(tce_page);
445
446         if (!tcep) {
447                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
448                 if (!tcep) {
449                         local_irq_enable();
450                         return -ENOMEM;
451                 }
452                 __this_cpu_write(tce_page, tcep);
453         }
454
455         proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
456
457         liobn = (u64)be32_to_cpu(maprange->liobn);
458         tce_shift = be32_to_cpu(maprange->tce_shift);
459         tce_size = 1ULL << tce_shift;
460         next = start_pfn << PAGE_SHIFT;
461         num_tce = num_pfn << PAGE_SHIFT;
462
463         /* round back to the beginning of the tce page size */
464         num_tce += next & (tce_size - 1);
465         next &= ~(tce_size - 1);
466
467         /* covert to number of tces */
468         num_tce |= tce_size - 1;
469         num_tce >>= tce_shift;
470
471         /* We can map max one pageful of TCEs at a time */
472         do {
473                 /*
474                  * Set up the page with TCE data, looping through and setting
475                  * the values.
476                  */
477                 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
478                 dma_offset = next + be64_to_cpu(maprange->dma_base);
479
480                 for (l = 0; l < limit; l++) {
481                         tcep[l] = cpu_to_be64(proto_tce | next);
482                         next += tce_size;
483                 }
484
485                 rc = plpar_tce_put_indirect(liobn,
486                                             dma_offset,
487                                             (u64)__pa(tcep),
488                                             limit);
489
490                 num_tce -= limit;
491         } while (num_tce > 0 && !rc);
492
493         /* error cleanup: caller will clear whole range */
494
495         local_irq_enable();
496         return rc;
497 }
498
499 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
500                 unsigned long num_pfn, void *arg)
501 {
502         return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
503 }
504
505 static void iommu_table_setparms_common(struct iommu_table *tbl, unsigned long busno,
506                                         unsigned long liobn, unsigned long win_addr,
507                                         unsigned long window_size, unsigned long page_shift,
508                                         void *base, struct iommu_table_ops *table_ops)
509 {
510         tbl->it_busno = busno;
511         tbl->it_index = liobn;
512         tbl->it_offset = win_addr >> page_shift;
513         tbl->it_size = window_size >> page_shift;
514         tbl->it_page_shift = page_shift;
515         tbl->it_base = (unsigned long)base;
516         tbl->it_blocksize = 16;
517         tbl->it_type = TCE_PCI;
518         tbl->it_ops = table_ops;
519 }
520
521 struct iommu_table_ops iommu_table_pseries_ops;
522
523 static void iommu_table_setparms(struct pci_controller *phb,
524                                  struct device_node *dn,
525                                  struct iommu_table *tbl)
526 {
527         struct device_node *node;
528         const unsigned long *basep;
529         const u32 *sizep;
530
531         /* Test if we are going over 2GB of DMA space */
532         if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) {
533                 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
534                 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
535         }
536
537         node = phb->dn;
538         basep = of_get_property(node, "linux,tce-base", NULL);
539         sizep = of_get_property(node, "linux,tce-size", NULL);
540         if (basep == NULL || sizep == NULL) {
541                 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has "
542                                 "missing tce entries !\n", dn);
543                 return;
544         }
545
546         iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur,
547                                     phb->dma_window_size, IOMMU_PAGE_SHIFT_4K,
548                                     __va(*basep), &iommu_table_pseries_ops);
549
550         if (!is_kdump_kernel())
551                 memset((void *)tbl->it_base, 0, *sizep);
552
553         phb->dma_window_base_cur += phb->dma_window_size;
554 }
555
556 struct iommu_table_ops iommu_table_lpar_multi_ops;
557
558 /*
559  * iommu_table_setparms_lpar
560  *
561  * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
562  */
563 static void iommu_table_setparms_lpar(struct pci_controller *phb,
564                                       struct device_node *dn,
565                                       struct iommu_table *tbl,
566                                       struct iommu_table_group *table_group,
567                                       const __be32 *dma_window)
568 {
569         unsigned long offset, size, liobn;
570
571         of_parse_dma_window(dn, dma_window, &liobn, &offset, &size);
572
573         iommu_table_setparms_common(tbl, phb->bus->number, liobn, offset, size, IOMMU_PAGE_SHIFT_4K, NULL,
574                                     &iommu_table_lpar_multi_ops);
575
576
577         table_group->tce32_start = offset;
578         table_group->tce32_size = size;
579 }
580
581 struct iommu_table_ops iommu_table_pseries_ops = {
582         .set = tce_build_pSeries,
583         .clear = tce_free_pSeries,
584         .get = tce_get_pseries
585 };
586
587 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
588 {
589         struct device_node *dn;
590         struct iommu_table *tbl;
591         struct device_node *isa_dn, *isa_dn_orig;
592         struct device_node *tmp;
593         struct pci_dn *pci;
594         int children;
595
596         dn = pci_bus_to_OF_node(bus);
597
598         pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn);
599
600         if (bus->self) {
601                 /* This is not a root bus, any setup will be done for the
602                  * device-side of the bridge in iommu_dev_setup_pSeries().
603                  */
604                 return;
605         }
606         pci = PCI_DN(dn);
607
608         /* Check if the ISA bus on the system is under
609          * this PHB.
610          */
611         isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
612
613         while (isa_dn && isa_dn != dn)
614                 isa_dn = isa_dn->parent;
615
616         of_node_put(isa_dn_orig);
617
618         /* Count number of direct PCI children of the PHB. */
619         for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
620                 children++;
621
622         pr_debug("Children: %d\n", children);
623
624         /* Calculate amount of DMA window per slot. Each window must be
625          * a power of two (due to pci_alloc_consistent requirements).
626          *
627          * Keep 256MB aside for PHBs with ISA.
628          */
629
630         if (!isa_dn) {
631                 /* No ISA/IDE - just set window size and return */
632                 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
633
634                 while (pci->phb->dma_window_size * children > 0x80000000ul)
635                         pci->phb->dma_window_size >>= 1;
636                 pr_debug("No ISA/IDE, window size is 0x%llx\n",
637                          pci->phb->dma_window_size);
638                 pci->phb->dma_window_base_cur = 0;
639
640                 return;
641         }
642
643         /* If we have ISA, then we probably have an IDE
644          * controller too. Allocate a 128MB table but
645          * skip the first 128MB to avoid stepping on ISA
646          * space.
647          */
648         pci->phb->dma_window_size = 0x8000000ul;
649         pci->phb->dma_window_base_cur = 0x8000000ul;
650
651         pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
652         tbl = pci->table_group->tables[0];
653
654         iommu_table_setparms(pci->phb, dn, tbl);
655
656         if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
657                 panic("Failed to initialize iommu table");
658
659         /* Divide the rest (1.75GB) among the children */
660         pci->phb->dma_window_size = 0x80000000ul;
661         while (pci->phb->dma_window_size * children > 0x70000000ul)
662                 pci->phb->dma_window_size >>= 1;
663
664         pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
665 }
666
667 #ifdef CONFIG_IOMMU_API
668 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
669                                 long *tce, enum dma_data_direction *direction)
670 {
671         long rc;
672         unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
673         unsigned long flags, oldtce = 0;
674         u64 proto_tce = iommu_direction_to_tce_perm(*direction);
675         unsigned long newtce = *tce | proto_tce;
676
677         spin_lock_irqsave(&tbl->large_pool.lock, flags);
678
679         rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce);
680         if (!rc)
681                 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce);
682
683         if (!rc) {
684                 *direction = iommu_tce_direction(oldtce);
685                 *tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
686         }
687
688         spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
689
690         return rc;
691 }
692 #endif
693
694 struct iommu_table_ops iommu_table_lpar_multi_ops = {
695         .set = tce_buildmulti_pSeriesLP,
696 #ifdef CONFIG_IOMMU_API
697         .xchg_no_kill = tce_exchange_pseries,
698 #endif
699         .clear = tce_freemulti_pSeriesLP,
700         .get = tce_get_pSeriesLP
701 };
702
703 /*
704  * Find nearest ibm,dma-window (default DMA window) or direct DMA window or
705  * dynamic 64bit DMA window, walking up the device tree.
706  */
707 static struct device_node *pci_dma_find(struct device_node *dn,
708                                         const __be32 **dma_window)
709 {
710         const __be32 *dw = NULL;
711
712         for ( ; dn && PCI_DN(dn); dn = dn->parent) {
713                 dw = of_get_property(dn, "ibm,dma-window", NULL);
714                 if (dw) {
715                         if (dma_window)
716                                 *dma_window = dw;
717                         return dn;
718                 }
719                 dw = of_get_property(dn, DIRECT64_PROPNAME, NULL);
720                 if (dw)
721                         return dn;
722                 dw = of_get_property(dn, DMA64_PROPNAME, NULL);
723                 if (dw)
724                         return dn;
725         }
726
727         return NULL;
728 }
729
730 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
731 {
732         struct iommu_table *tbl;
733         struct device_node *dn, *pdn;
734         struct pci_dn *ppci;
735         const __be32 *dma_window = NULL;
736
737         dn = pci_bus_to_OF_node(bus);
738
739         pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n",
740                  dn);
741
742         pdn = pci_dma_find(dn, &dma_window);
743
744         if (dma_window == NULL)
745                 pr_debug("  no ibm,dma-window property !\n");
746
747         ppci = PCI_DN(pdn);
748
749         pr_debug("  parent is %pOF, iommu_table: 0x%p\n",
750                  pdn, ppci->table_group);
751
752         if (!ppci->table_group) {
753                 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node);
754                 tbl = ppci->table_group->tables[0];
755                 if (dma_window) {
756                         iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
757                                                   ppci->table_group, dma_window);
758
759                         if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
760                                 panic("Failed to initialize iommu table");
761                 }
762                 iommu_register_group(ppci->table_group,
763                                 pci_domain_nr(bus), 0);
764                 pr_debug("  created table: %p\n", ppci->table_group);
765         }
766 }
767
768
769 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
770 {
771         struct device_node *dn;
772         struct iommu_table *tbl;
773
774         pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
775
776         dn = dev->dev.of_node;
777
778         /* If we're the direct child of a root bus, then we need to allocate
779          * an iommu table ourselves. The bus setup code should have setup
780          * the window sizes already.
781          */
782         if (!dev->bus->self) {
783                 struct pci_controller *phb = PCI_DN(dn)->phb;
784
785                 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
786                 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node);
787                 tbl = PCI_DN(dn)->table_group->tables[0];
788                 iommu_table_setparms(phb, dn, tbl);
789
790                 if (!iommu_init_table(tbl, phb->node, 0, 0))
791                         panic("Failed to initialize iommu table");
792
793                 set_iommu_table_base(&dev->dev, tbl);
794                 return;
795         }
796
797         /* If this device is further down the bus tree, search upwards until
798          * an already allocated iommu table is found and use that.
799          */
800
801         while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
802                 dn = dn->parent;
803
804         if (dn && PCI_DN(dn))
805                 set_iommu_table_base(&dev->dev,
806                                 PCI_DN(dn)->table_group->tables[0]);
807         else
808                 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
809                        pci_name(dev));
810 }
811
812 static int __read_mostly disable_ddw;
813
814 static int __init disable_ddw_setup(char *str)
815 {
816         disable_ddw = 1;
817         printk(KERN_INFO "ppc iommu: disabling ddw.\n");
818
819         return 0;
820 }
821
822 early_param("disable_ddw", disable_ddw_setup);
823
824 static void clean_dma_window(struct device_node *np, struct dynamic_dma_window_prop *dwp)
825 {
826         int ret;
827
828         ret = tce_clearrange_multi_pSeriesLP(0,
829                 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
830         if (ret)
831                 pr_warn("%pOF failed to clear tces in window.\n",
832                         np);
833         else
834                 pr_debug("%pOF successfully cleared tces in window.\n",
835                          np);
836 }
837
838 /*
839  * Call only if DMA window is clean.
840  */
841 static void __remove_dma_window(struct device_node *np, u32 *ddw_avail, u64 liobn)
842 {
843         int ret;
844
845         ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
846         if (ret)
847                 pr_warn("%pOF: failed to remove DMA window: rtas returned "
848                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
849                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
850         else
851                 pr_debug("%pOF: successfully removed DMA window: rtas returned "
852                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
853                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
854 }
855
856 static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
857                               struct property *win)
858 {
859         struct dynamic_dma_window_prop *dwp;
860         u64 liobn;
861
862         dwp = win->value;
863         liobn = (u64)be32_to_cpu(dwp->liobn);
864
865         clean_dma_window(np, dwp);
866         __remove_dma_window(np, ddw_avail, liobn);
867 }
868
869 static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_name)
870 {
871         struct property *win;
872         u32 ddw_avail[DDW_APPLICABLE_SIZE];
873         int ret = 0;
874
875         win = of_find_property(np, win_name, NULL);
876         if (!win)
877                 return -EINVAL;
878
879         ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
880                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
881         if (ret)
882                 return 0;
883
884
885         if (win->length >= sizeof(struct dynamic_dma_window_prop))
886                 remove_dma_window(np, ddw_avail, win);
887
888         if (!remove_prop)
889                 return 0;
890
891         ret = of_remove_property(np, win);
892         if (ret)
893                 pr_warn("%pOF: failed to remove DMA window property: %d\n",
894                         np, ret);
895         return 0;
896 }
897
898 static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift)
899 {
900         struct dma_win *window;
901         const struct dynamic_dma_window_prop *dma64;
902         bool found = false;
903
904         spin_lock(&dma_win_list_lock);
905         /* check if we already created a window and dupe that config if so */
906         list_for_each_entry(window, &dma_win_list, list) {
907                 if (window->device == pdn) {
908                         dma64 = window->prop;
909                         *dma_addr = be64_to_cpu(dma64->dma_base);
910                         *window_shift = be32_to_cpu(dma64->window_shift);
911                         found = true;
912                         break;
913                 }
914         }
915         spin_unlock(&dma_win_list_lock);
916
917         return found;
918 }
919
920 static struct dma_win *ddw_list_new_entry(struct device_node *pdn,
921                                           const struct dynamic_dma_window_prop *dma64)
922 {
923         struct dma_win *window;
924
925         window = kzalloc(sizeof(*window), GFP_KERNEL);
926         if (!window)
927                 return NULL;
928
929         window->device = pdn;
930         window->prop = dma64;
931
932         return window;
933 }
934
935 static void find_existing_ddw_windows_named(const char *name)
936 {
937         int len;
938         struct device_node *pdn;
939         struct dma_win *window;
940         const struct dynamic_dma_window_prop *dma64;
941
942         for_each_node_with_property(pdn, name) {
943                 dma64 = of_get_property(pdn, name, &len);
944                 if (!dma64 || len < sizeof(*dma64)) {
945                         remove_ddw(pdn, true, name);
946                         continue;
947                 }
948
949                 window = ddw_list_new_entry(pdn, dma64);
950                 if (!window) {
951                         of_node_put(pdn);
952                         break;
953                 }
954
955                 spin_lock(&dma_win_list_lock);
956                 list_add(&window->list, &dma_win_list);
957                 spin_unlock(&dma_win_list_lock);
958         }
959 }
960
961 static int find_existing_ddw_windows(void)
962 {
963         if (!firmware_has_feature(FW_FEATURE_LPAR))
964                 return 0;
965
966         find_existing_ddw_windows_named(DIRECT64_PROPNAME);
967         find_existing_ddw_windows_named(DMA64_PROPNAME);
968
969         return 0;
970 }
971 machine_arch_initcall(pseries, find_existing_ddw_windows);
972
973 /**
974  * ddw_read_ext - Get the value of an DDW extension
975  * @np:         device node from which the extension value is to be read.
976  * @extnum:     index number of the extension.
977  * @value:      pointer to return value, modified when extension is available.
978  *
979  * Checks if "ibm,ddw-extensions" exists for this node, and get the value
980  * on index 'extnum'.
981  * It can be used only to check if a property exists, passing value == NULL.
982  *
983  * Returns:
984  *      0 if extension successfully read
985  *      -EINVAL if the "ibm,ddw-extensions" does not exist,
986  *      -ENODATA if "ibm,ddw-extensions" does not have a value, and
987  *      -EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
988  */
989 static inline int ddw_read_ext(const struct device_node *np, int extnum,
990                                u32 *value)
991 {
992         static const char propname[] = "ibm,ddw-extensions";
993         u32 count;
994         int ret;
995
996         ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
997         if (ret)
998                 return ret;
999
1000         if (count < extnum)
1001                 return -EOVERFLOW;
1002
1003         if (!value)
1004                 value = &count;
1005
1006         return of_property_read_u32_index(np, propname, extnum, value);
1007 }
1008
1009 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1010                      struct ddw_query_response *query,
1011                      struct device_node *parent)
1012 {
1013         struct device_node *dn;
1014         struct pci_dn *pdn;
1015         u32 cfg_addr, ext_query, query_out[5];
1016         u64 buid;
1017         int ret, out_sz;
1018
1019         /*
1020          * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
1021          * output parameters ibm,query-pe-dma-windows will have, ranging from
1022          * 5 to 6.
1023          */
1024         ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
1025         if (!ret && ext_query == 1)
1026                 out_sz = 6;
1027         else
1028                 out_sz = 5;
1029
1030         /*
1031          * Get the config address and phb buid of the PE window.
1032          * Rely on eeh to retrieve this for us.
1033          * Retrieve them from the pci device, not the node with the
1034          * dma-window property
1035          */
1036         dn = pci_device_to_OF_node(dev);
1037         pdn = PCI_DN(dn);
1038         buid = pdn->phb->buid;
1039         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1040
1041         ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
1042                         cfg_addr, BUID_HI(buid), BUID_LO(buid));
1043
1044         switch (out_sz) {
1045         case 5:
1046                 query->windows_available = query_out[0];
1047                 query->largest_available_block = query_out[1];
1048                 query->page_size = query_out[2];
1049                 query->migration_capable = query_out[3];
1050                 break;
1051         case 6:
1052                 query->windows_available = query_out[0];
1053                 query->largest_available_block = ((u64)query_out[1] << 32) |
1054                                                  query_out[2];
1055                 query->page_size = query_out[3];
1056                 query->migration_capable = query_out[4];
1057                 break;
1058         }
1059
1060         dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d, lb=%llx ps=%x wn=%d\n",
1061                  ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1062                  BUID_LO(buid), ret, query->largest_available_block,
1063                  query->page_size, query->windows_available);
1064
1065         return ret;
1066 }
1067
1068 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1069                         struct ddw_create_response *create, int page_shift,
1070                         int window_shift)
1071 {
1072         struct device_node *dn;
1073         struct pci_dn *pdn;
1074         u32 cfg_addr;
1075         u64 buid;
1076         int ret;
1077
1078         /*
1079          * Get the config address and phb buid of the PE window.
1080          * Rely on eeh to retrieve this for us.
1081          * Retrieve them from the pci device, not the node with the
1082          * dma-window property
1083          */
1084         dn = pci_device_to_OF_node(dev);
1085         pdn = PCI_DN(dn);
1086         buid = pdn->phb->buid;
1087         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1088
1089         do {
1090                 /* extra outputs are LIOBN and dma-addr (hi, lo) */
1091                 ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1092                                 (u32 *)create, cfg_addr, BUID_HI(buid),
1093                                 BUID_LO(buid), page_shift, window_shift);
1094         } while (rtas_busy_delay(ret));
1095         dev_info(&dev->dev,
1096                 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
1097                 "(liobn = 0x%x starting addr = %x %x)\n",
1098                  ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1099                  BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1100                  create->addr_hi, create->addr_lo);
1101
1102         return ret;
1103 }
1104
1105 struct failed_ddw_pdn {
1106         struct device_node *pdn;
1107         struct list_head list;
1108 };
1109
1110 static LIST_HEAD(failed_ddw_pdn_list);
1111
1112 static phys_addr_t ddw_memory_hotplug_max(void)
1113 {
1114         phys_addr_t max_addr = memory_hotplug_max();
1115         struct device_node *memory;
1116
1117         for_each_node_by_type(memory, "memory") {
1118                 unsigned long start, size;
1119                 int n_mem_addr_cells, n_mem_size_cells, len;
1120                 const __be32 *memcell_buf;
1121
1122                 memcell_buf = of_get_property(memory, "reg", &len);
1123                 if (!memcell_buf || len <= 0)
1124                         continue;
1125
1126                 n_mem_addr_cells = of_n_addr_cells(memory);
1127                 n_mem_size_cells = of_n_size_cells(memory);
1128
1129                 start = of_read_number(memcell_buf, n_mem_addr_cells);
1130                 memcell_buf += n_mem_addr_cells;
1131                 size = of_read_number(memcell_buf, n_mem_size_cells);
1132                 memcell_buf += n_mem_size_cells;
1133
1134                 max_addr = max_t(phys_addr_t, max_addr, start + size);
1135         }
1136
1137         return max_addr;
1138 }
1139
1140 /*
1141  * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1142  * ibm,ddw-extensions, which carries the rtas token for
1143  * ibm,reset-pe-dma-windows.
1144  * That rtas-call can be used to restore the default DMA window for the device.
1145  */
1146 static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1147 {
1148         int ret;
1149         u32 cfg_addr, reset_dma_win;
1150         u64 buid;
1151         struct device_node *dn;
1152         struct pci_dn *pdn;
1153
1154         ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1155         if (ret)
1156                 return;
1157
1158         dn = pci_device_to_OF_node(dev);
1159         pdn = PCI_DN(dn);
1160         buid = pdn->phb->buid;
1161         cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1162
1163         ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1164                         BUID_LO(buid));
1165         if (ret)
1166                 dev_info(&dev->dev,
1167                          "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1168                          reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1169                          ret);
1170 }
1171
1172 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1173 static int iommu_get_page_shift(u32 query_page_size)
1174 {
1175         /* Supported IO page-sizes according to LoPAR, note that 2M is out of order */
1176         const int shift[] = {
1177                 __builtin_ctzll(SZ_4K),   __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
1178                 __builtin_ctzll(SZ_32M),  __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
1179                 __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G), __builtin_ctzll(SZ_2M)
1180         };
1181
1182         int i = ARRAY_SIZE(shift) - 1;
1183         int ret = 0;
1184
1185         /*
1186          * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
1187          * - bit 31 means 4k pages are supported,
1188          * - bit 30 means 64k pages are supported, and so on.
1189          * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
1190          */
1191         for (; i >= 0 ; i--) {
1192                 if (query_page_size & (1 << i))
1193                         ret = max(ret, shift[i]);
1194         }
1195
1196         return ret;
1197 }
1198
1199 static struct property *ddw_property_create(const char *propname, u32 liobn, u64 dma_addr,
1200                                             u32 page_shift, u32 window_shift)
1201 {
1202         struct dynamic_dma_window_prop *ddwprop;
1203         struct property *win64;
1204
1205         win64 = kzalloc(sizeof(*win64), GFP_KERNEL);
1206         if (!win64)
1207                 return NULL;
1208
1209         win64->name = kstrdup(propname, GFP_KERNEL);
1210         ddwprop = kzalloc(sizeof(*ddwprop), GFP_KERNEL);
1211         win64->value = ddwprop;
1212         win64->length = sizeof(*ddwprop);
1213         if (!win64->name || !win64->value) {
1214                 kfree(win64->name);
1215                 kfree(win64->value);
1216                 kfree(win64);
1217                 return NULL;
1218         }
1219
1220         ddwprop->liobn = cpu_to_be32(liobn);
1221         ddwprop->dma_base = cpu_to_be64(dma_addr);
1222         ddwprop->tce_shift = cpu_to_be32(page_shift);
1223         ddwprop->window_shift = cpu_to_be32(window_shift);
1224
1225         return win64;
1226 }
1227
1228 /*
1229  * If the PE supports dynamic dma windows, and there is space for a table
1230  * that can map all pages in a linear offset, then setup such a table,
1231  * and record the dma-offset in the struct device.
1232  *
1233  * dev: the pci device we are checking
1234  * pdn: the parent pe node with the ibm,dma_window property
1235  * Future: also check if we can remap the base window for our base page size
1236  *
1237  * returns true if can map all pages (direct mapping), false otherwise..
1238  */
1239 static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
1240 {
1241         int len = 0, ret;
1242         int max_ram_len = order_base_2(ddw_memory_hotplug_max());
1243         struct ddw_query_response query;
1244         struct ddw_create_response create;
1245         int page_shift;
1246         u64 win_addr;
1247         const char *win_name;
1248         struct device_node *dn;
1249         u32 ddw_avail[DDW_APPLICABLE_SIZE];
1250         struct dma_win *window;
1251         struct property *win64;
1252         struct failed_ddw_pdn *fpdn;
1253         bool default_win_removed = false, direct_mapping = false;
1254         bool pmem_present;
1255         struct pci_dn *pci = PCI_DN(pdn);
1256         struct property *default_win = NULL;
1257
1258         dn = of_find_node_by_type(NULL, "ibm,pmemory");
1259         pmem_present = dn != NULL;
1260         of_node_put(dn);
1261
1262         mutex_lock(&dma_win_init_mutex);
1263
1264         if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) {
1265                 direct_mapping = (len >= max_ram_len);
1266                 goto out_unlock;
1267         }
1268
1269         /*
1270          * If we already went through this for a previous function of
1271          * the same device and failed, we don't want to muck with the
1272          * DMA window again, as it will race with in-flight operations
1273          * and can lead to EEHs. The above mutex protects access to the
1274          * list.
1275          */
1276         list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
1277                 if (fpdn->pdn == pdn)
1278                         goto out_unlock;
1279         }
1280
1281         /*
1282          * the ibm,ddw-applicable property holds the tokens for:
1283          * ibm,query-pe-dma-window
1284          * ibm,create-pe-dma-window
1285          * ibm,remove-pe-dma-window
1286          * for the given node in that order.
1287          * the property is actually in the parent, not the PE
1288          */
1289         ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1290                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
1291         if (ret)
1292                 goto out_failed;
1293
1294        /*
1295          * Query if there is a second window of size to map the
1296          * whole partition.  Query returns number of windows, largest
1297          * block assigned to PE (partition endpoint), and two bitmasks
1298          * of page sizes: supported and supported for migrate-dma.
1299          */
1300         dn = pci_device_to_OF_node(dev);
1301         ret = query_ddw(dev, ddw_avail, &query, pdn);
1302         if (ret != 0)
1303                 goto out_failed;
1304
1305         /*
1306          * If there is no window available, remove the default DMA window,
1307          * if it's present. This will make all the resources available to the
1308          * new DDW window.
1309          * If anything fails after this, we need to restore it, so also check
1310          * for extensions presence.
1311          */
1312         if (query.windows_available == 0) {
1313                 int reset_win_ext;
1314
1315                 /* DDW + IOMMU on single window may fail if there is any allocation */
1316                 if (iommu_table_in_use(pci->table_group->tables[0])) {
1317                         dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n");
1318                         goto out_failed;
1319                 }
1320
1321                 default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1322                 if (!default_win)
1323                         goto out_failed;
1324
1325                 reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1326                 if (reset_win_ext)
1327                         goto out_failed;
1328
1329                 remove_dma_window(pdn, ddw_avail, default_win);
1330                 default_win_removed = true;
1331
1332                 /* Query again, to check if the window is available */
1333                 ret = query_ddw(dev, ddw_avail, &query, pdn);
1334                 if (ret != 0)
1335                         goto out_failed;
1336
1337                 if (query.windows_available == 0) {
1338                         /* no windows are available for this device. */
1339                         dev_dbg(&dev->dev, "no free dynamic windows");
1340                         goto out_failed;
1341                 }
1342         }
1343
1344         page_shift = iommu_get_page_shift(query.page_size);
1345         if (!page_shift) {
1346                 dev_dbg(&dev->dev, "no supported page size in mask %x",
1347                         query.page_size);
1348                 goto out_failed;
1349         }
1350
1351
1352         /*
1353          * The "ibm,pmemory" can appear anywhere in the address space.
1354          * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
1355          * for the upper limit and fallback to max RAM otherwise but this
1356          * disables device::dma_ops_bypass.
1357          */
1358         len = max_ram_len;
1359         if (pmem_present) {
1360                 if (query.largest_available_block >=
1361                     (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1362                         len = MAX_PHYSMEM_BITS;
1363                 else
1364                         dev_info(&dev->dev, "Skipping ibm,pmemory");
1365         }
1366
1367         /* check if the available block * number of ptes will map everything */
1368         if (query.largest_available_block < (1ULL << (len - page_shift))) {
1369                 dev_dbg(&dev->dev,
1370                         "can't map partition max 0x%llx with %llu %llu-sized pages\n",
1371                         1ULL << len,
1372                         query.largest_available_block,
1373                         1ULL << page_shift);
1374
1375                 len = order_base_2(query.largest_available_block << page_shift);
1376                 win_name = DMA64_PROPNAME;
1377         } else {
1378                 direct_mapping = !default_win_removed ||
1379                         (len == MAX_PHYSMEM_BITS) ||
1380                         (!pmem_present && (len == max_ram_len));
1381                 win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
1382         }
1383
1384         ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1385         if (ret != 0)
1386                 goto out_failed;
1387
1388         dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n",
1389                   create.liobn, dn);
1390
1391         win_addr = ((u64)create.addr_hi << 32) | create.addr_lo;
1392         win64 = ddw_property_create(win_name, create.liobn, win_addr, page_shift, len);
1393
1394         if (!win64) {
1395                 dev_info(&dev->dev,
1396                          "couldn't allocate property, property name, or value\n");
1397                 goto out_remove_win;
1398         }
1399
1400         ret = of_add_property(pdn, win64);
1401         if (ret) {
1402                 dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d",
1403                         pdn, ret);
1404                 goto out_free_prop;
1405         }
1406
1407         window = ddw_list_new_entry(pdn, win64->value);
1408         if (!window)
1409                 goto out_del_prop;
1410
1411         if (direct_mapping) {
1412                 /* DDW maps the whole partition, so enable direct DMA mapping */
1413                 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1414                                             win64->value, tce_setrange_multi_pSeriesLP_walk);
1415                 if (ret) {
1416                         dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n",
1417                                  dn, ret);
1418
1419                         /* Make sure to clean DDW if any TCE was set*/
1420                         clean_dma_window(pdn, win64->value);
1421                         goto out_del_list;
1422                 }
1423         } else {
1424                 struct iommu_table *newtbl;
1425                 int i;
1426                 unsigned long start = 0, end = 0;
1427
1428                 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) {
1429                         const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM;
1430
1431                         /* Look for MMIO32 */
1432                         if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) {
1433                                 start = pci->phb->mem_resources[i].start;
1434                                 end = pci->phb->mem_resources[i].end;
1435                                 break;
1436                         }
1437                 }
1438
1439                 /* New table for using DDW instead of the default DMA window */
1440                 newtbl = iommu_pseries_alloc_table(pci->phb->node);
1441                 if (!newtbl) {
1442                         dev_dbg(&dev->dev, "couldn't create new IOMMU table\n");
1443                         goto out_del_list;
1444                 }
1445
1446                 iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, win_addr,
1447                                             1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops);
1448                 iommu_init_table(newtbl, pci->phb->node, start, end);
1449
1450                 pci->table_group->tables[1] = newtbl;
1451
1452                 set_iommu_table_base(&dev->dev, newtbl);
1453         }
1454
1455         if (default_win_removed) {
1456                 iommu_tce_table_put(pci->table_group->tables[0]);
1457                 pci->table_group->tables[0] = NULL;
1458
1459                 /* default_win is valid here because default_win_removed == true */
1460                 of_remove_property(pdn, default_win);
1461                 dev_info(&dev->dev, "Removed default DMA window for %pOF\n", pdn);
1462         }
1463
1464         spin_lock(&dma_win_list_lock);
1465         list_add(&window->list, &dma_win_list);
1466         spin_unlock(&dma_win_list_lock);
1467
1468         dev->dev.archdata.dma_offset = win_addr;
1469         goto out_unlock;
1470
1471 out_del_list:
1472         kfree(window);
1473
1474 out_del_prop:
1475         of_remove_property(pdn, win64);
1476
1477 out_free_prop:
1478         kfree(win64->name);
1479         kfree(win64->value);
1480         kfree(win64);
1481
1482 out_remove_win:
1483         /* DDW is clean, so it's ok to call this directly. */
1484         __remove_dma_window(pdn, ddw_avail, create.liobn);
1485
1486 out_failed:
1487         if (default_win_removed)
1488                 reset_dma_window(dev, pdn);
1489
1490         fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1491         if (!fpdn)
1492                 goto out_unlock;
1493         fpdn->pdn = pdn;
1494         list_add(&fpdn->list, &failed_ddw_pdn_list);
1495
1496 out_unlock:
1497         mutex_unlock(&dma_win_init_mutex);
1498
1499         /*
1500          * If we have persistent memory and the window size is only as big
1501          * as RAM, then we failed to create a window to cover persistent
1502          * memory and need to set the DMA limit.
1503          */
1504         if (pmem_present && direct_mapping && len == max_ram_len)
1505                 dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << len);
1506
1507         return direct_mapping;
1508 }
1509
1510 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1511 {
1512         struct device_node *pdn, *dn;
1513         struct iommu_table *tbl;
1514         const __be32 *dma_window = NULL;
1515         struct pci_dn *pci;
1516
1517         pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1518
1519         /* dev setup for LPAR is a little tricky, since the device tree might
1520          * contain the dma-window properties per-device and not necessarily
1521          * for the bus. So we need to search upwards in the tree until we
1522          * either hit a dma-window property, OR find a parent with a table
1523          * already allocated.
1524          */
1525         dn = pci_device_to_OF_node(dev);
1526         pr_debug("  node is %pOF\n", dn);
1527
1528         pdn = pci_dma_find(dn, &dma_window);
1529         if (!pdn || !PCI_DN(pdn)) {
1530                 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1531                        "no DMA window found for pci dev=%s dn=%pOF\n",
1532                                  pci_name(dev), dn);
1533                 return;
1534         }
1535         pr_debug("  parent is %pOF\n", pdn);
1536
1537         pci = PCI_DN(pdn);
1538         if (!pci->table_group) {
1539                 pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
1540                 tbl = pci->table_group->tables[0];
1541                 iommu_table_setparms_lpar(pci->phb, pdn, tbl,
1542                                 pci->table_group, dma_window);
1543
1544                 iommu_init_table(tbl, pci->phb->node, 0, 0);
1545                 iommu_register_group(pci->table_group,
1546                                 pci_domain_nr(pci->phb->bus), 0);
1547                 pr_debug("  created table: %p\n", pci->table_group);
1548         } else {
1549                 pr_debug("  found DMA window, table: %p\n", pci->table_group);
1550         }
1551
1552         set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1553         iommu_add_device(pci->table_group, &dev->dev);
1554 }
1555
1556 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
1557 {
1558         struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
1559
1560         /* only attempt to use a new window if 64-bit DMA is requested */
1561         if (dma_mask < DMA_BIT_MASK(64))
1562                 return false;
1563
1564         dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1565
1566         /*
1567          * the device tree might contain the dma-window properties
1568          * per-device and not necessarily for the bus. So we need to
1569          * search upwards in the tree until we either hit a dma-window
1570          * property, OR find a parent with a table already allocated.
1571          */
1572         pdn = pci_dma_find(dn, NULL);
1573         if (pdn && PCI_DN(pdn))
1574                 return enable_ddw(pdev, pdn);
1575
1576         return false;
1577 }
1578
1579 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1580                 void *data)
1581 {
1582         struct dma_win *window;
1583         struct memory_notify *arg = data;
1584         int ret = 0;
1585
1586         switch (action) {
1587         case MEM_GOING_ONLINE:
1588                 spin_lock(&dma_win_list_lock);
1589                 list_for_each_entry(window, &dma_win_list, list) {
1590                         ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1591                                         arg->nr_pages, window->prop);
1592                         /* XXX log error */
1593                 }
1594                 spin_unlock(&dma_win_list_lock);
1595                 break;
1596         case MEM_CANCEL_ONLINE:
1597         case MEM_OFFLINE:
1598                 spin_lock(&dma_win_list_lock);
1599                 list_for_each_entry(window, &dma_win_list, list) {
1600                         ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1601                                         arg->nr_pages, window->prop);
1602                         /* XXX log error */
1603                 }
1604                 spin_unlock(&dma_win_list_lock);
1605                 break;
1606         default:
1607                 break;
1608         }
1609         if (ret && action != MEM_CANCEL_ONLINE)
1610                 return NOTIFY_BAD;
1611
1612         return NOTIFY_OK;
1613 }
1614
1615 static struct notifier_block iommu_mem_nb = {
1616         .notifier_call = iommu_mem_notifier,
1617 };
1618
1619 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data)
1620 {
1621         int err = NOTIFY_OK;
1622         struct of_reconfig_data *rd = data;
1623         struct device_node *np = rd->dn;
1624         struct pci_dn *pci = PCI_DN(np);
1625         struct dma_win *window;
1626
1627         switch (action) {
1628         case OF_RECONFIG_DETACH_NODE:
1629                 /*
1630                  * Removing the property will invoke the reconfig
1631                  * notifier again, which causes dead-lock on the
1632                  * read-write semaphore of the notifier chain. So
1633                  * we have to remove the property when releasing
1634                  * the device node.
1635                  */
1636                 if (remove_ddw(np, false, DIRECT64_PROPNAME))
1637                         remove_ddw(np, false, DMA64_PROPNAME);
1638
1639                 if (pci && pci->table_group)
1640                         iommu_pseries_free_group(pci->table_group,
1641                                         np->full_name);
1642
1643                 spin_lock(&dma_win_list_lock);
1644                 list_for_each_entry(window, &dma_win_list, list) {
1645                         if (window->device == np) {
1646                                 list_del(&window->list);
1647                                 kfree(window);
1648                                 break;
1649                         }
1650                 }
1651                 spin_unlock(&dma_win_list_lock);
1652                 break;
1653         default:
1654                 err = NOTIFY_DONE;
1655                 break;
1656         }
1657         return err;
1658 }
1659
1660 static struct notifier_block iommu_reconfig_nb = {
1661         .notifier_call = iommu_reconfig_notifier,
1662 };
1663
1664 /* These are called very early. */
1665 void __init iommu_init_early_pSeries(void)
1666 {
1667         if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1668                 return;
1669
1670         if (firmware_has_feature(FW_FEATURE_LPAR)) {
1671                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1672                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1673                 if (!disable_ddw)
1674                         pseries_pci_controller_ops.iommu_bypass_supported =
1675                                 iommu_bypass_supported_pSeriesLP;
1676         } else {
1677                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
1678                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
1679         }
1680
1681
1682         of_reconfig_notifier_register(&iommu_reconfig_nb);
1683         register_memory_notifier(&iommu_mem_nb);
1684
1685         set_pci_dma_ops(&dma_iommu_ops);
1686 }
1687
1688 static int __init disable_multitce(char *str)
1689 {
1690         if (strcmp(str, "off") == 0 &&
1691             firmware_has_feature(FW_FEATURE_LPAR) &&
1692             (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1693              firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
1694                 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1695                 powerpc_firmware_features &=
1696                         ~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
1697         }
1698         return 1;
1699 }
1700
1701 __setup("multitce=", disable_multitce);
1702
1703 static int tce_iommu_bus_notifier(struct notifier_block *nb,
1704                 unsigned long action, void *data)
1705 {
1706         struct device *dev = data;
1707
1708         switch (action) {
1709         case BUS_NOTIFY_DEL_DEVICE:
1710                 iommu_del_device(dev);
1711                 return 0;
1712         default:
1713                 return 0;
1714         }
1715 }
1716
1717 static struct notifier_block tce_iommu_bus_nb = {
1718         .notifier_call = tce_iommu_bus_notifier,
1719 };
1720
1721 static int __init tce_iommu_bus_notifier_init(void)
1722 {
1723         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1724         return 0;
1725 }
1726 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init);
This page took 0.138328 seconds and 4 git commands to generate.