1 From 89a1f0d7826df69d8e02268b97bc3da02e07203f Mon Sep 17 00:00:00 2001
2 From: Yangbo Lu <yangbo.lu@nxp.com>
3 Date: Thu, 5 Jul 2018 17:35:15 +0800
4 Subject: [PATCH 22/32] iommu: support layerscape
6 This is an integrated patch for layerscape smmu support.
8 Signed-off-by: Eric Auger <eric.auger@redhat.com>
9 Signed-off-by: Robin Murphy <robin.murphy@arm.com>
10 Signed-off-by: Nipun Gupta <nipun.gupta@nxp.com>
11 Signed-off-by: Sunil Goutham <sgoutham@cavium.com>
12 Signed-off-by: Yangbo Lu <yangbo.lu@nxp.com>
14 drivers/iommu/amd_iommu.c | 56 +++++---
15 drivers/iommu/arm-smmu-v3.c | 111 +++++++++++-----
16 drivers/iommu/arm-smmu.c | 100 ++++++++++++---
17 drivers/iommu/dma-iommu.c | 242 +++++++++++++++++++++++++++++------
18 drivers/iommu/intel-iommu.c | 92 ++++++++++---
19 drivers/iommu/iommu.c | 240 ++++++++++++++++++++++++++++++++--
20 drivers/iommu/mtk_iommu.c | 2 +
21 drivers/iommu/mtk_iommu_v1.c | 2 +
22 include/linux/dma-iommu.h | 11 ++
23 include/linux/iommu.h | 57 +++++++--
24 10 files changed, 762 insertions(+), 151 deletions(-)
26 --- a/drivers/iommu/amd_iommu.c
27 +++ b/drivers/iommu/amd_iommu.c
28 @@ -379,6 +379,8 @@ static struct iommu_group *acpihid_devic
31 entry->group = generic_device_group(dev);
33 + iommu_group_ref_get(entry->group);
37 @@ -3166,9 +3168,10 @@ static bool amd_iommu_capable(enum iommu
41 -static void amd_iommu_get_dm_regions(struct device *dev,
42 - struct list_head *head)
43 +static void amd_iommu_get_resv_regions(struct device *dev,
44 + struct list_head *head)
46 + struct iommu_resv_region *region;
47 struct unity_map_entry *entry;
50 @@ -3177,41 +3180,56 @@ static void amd_iommu_get_dm_regions(str
53 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
54 - struct iommu_dm_region *region;
58 if (devid < entry->devid_start || devid > entry->devid_end)
61 - region = kzalloc(sizeof(*region), GFP_KERNEL);
62 + length = entry->address_end - entry->address_start;
63 + if (entry->prot & IOMMU_PROT_IR)
65 + if (entry->prot & IOMMU_PROT_IW)
66 + prot |= IOMMU_WRITE;
68 + region = iommu_alloc_resv_region(entry->address_start,
72 pr_err("Out of memory allocating dm-regions for %s\n",
77 - region->start = entry->address_start;
78 - region->length = entry->address_end - entry->address_start;
79 - if (entry->prot & IOMMU_PROT_IR)
80 - region->prot |= IOMMU_READ;
81 - if (entry->prot & IOMMU_PROT_IW)
82 - region->prot |= IOMMU_WRITE;
84 list_add_tail(®ion->list, head);
87 + region = iommu_alloc_resv_region(MSI_RANGE_START,
88 + MSI_RANGE_END - MSI_RANGE_START + 1,
92 + list_add_tail(®ion->list, head);
94 + region = iommu_alloc_resv_region(HT_RANGE_START,
95 + HT_RANGE_END - HT_RANGE_START + 1,
96 + 0, IOMMU_RESV_RESERVED);
99 + list_add_tail(®ion->list, head);
102 -static void amd_iommu_put_dm_regions(struct device *dev,
103 +static void amd_iommu_put_resv_regions(struct device *dev,
104 struct list_head *head)
106 - struct iommu_dm_region *entry, *next;
107 + struct iommu_resv_region *entry, *next;
109 list_for_each_entry_safe(entry, next, head, list)
113 -static void amd_iommu_apply_dm_region(struct device *dev,
114 +static void amd_iommu_apply_resv_region(struct device *dev,
115 struct iommu_domain *domain,
116 - struct iommu_dm_region *region)
117 + struct iommu_resv_region *region)
119 struct dma_ops_domain *dma_dom = to_dma_ops_domain(to_pdomain(domain));
120 unsigned long start, end;
121 @@ -3235,9 +3253,9 @@ static const struct iommu_ops amd_iommu_
122 .add_device = amd_iommu_add_device,
123 .remove_device = amd_iommu_remove_device,
124 .device_group = amd_iommu_device_group,
125 - .get_dm_regions = amd_iommu_get_dm_regions,
126 - .put_dm_regions = amd_iommu_put_dm_regions,
127 - .apply_dm_region = amd_iommu_apply_dm_region,
128 + .get_resv_regions = amd_iommu_get_resv_regions,
129 + .put_resv_regions = amd_iommu_put_resv_regions,
130 + .apply_resv_region = amd_iommu_apply_resv_region,
131 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
134 --- a/drivers/iommu/arm-smmu-v3.c
135 +++ b/drivers/iommu/arm-smmu-v3.c
137 /* High-level queue structures */
138 #define ARM_SMMU_POLL_TIMEOUT_US 100
140 +#define MSI_IOVA_BASE 0x8000000
141 +#define MSI_IOVA_LENGTH 0x100000
143 static bool disable_bypass;
144 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
145 MODULE_PARM_DESC(disable_bypass,
146 @@ -552,9 +555,14 @@ struct arm_smmu_s2_cfg {
149 struct arm_smmu_strtab_ent {
152 - bool bypass; /* Overrides s1/s2 config */
154 + * An STE is "assigned" if the master emitting the corresponding SID
155 + * is attached to a domain. The behaviour of an unassigned STE is
156 + * determined by the disable_bypass parameter, whereas an assigned
157 + * STE behaves according to s1_cfg/s2_cfg, which themselves are
158 + * configured according to the domain type.
161 struct arm_smmu_s1_cfg *s1_cfg;
162 struct arm_smmu_s2_cfg *s2_cfg;
164 @@ -627,6 +635,7 @@ enum arm_smmu_domain_stage {
165 ARM_SMMU_DOMAIN_S1 = 0,
167 ARM_SMMU_DOMAIN_NESTED,
168 + ARM_SMMU_DOMAIN_BYPASS,
171 struct arm_smmu_domain {
172 @@ -1000,9 +1009,9 @@ static void arm_smmu_write_strtab_ent(st
173 * This is hideously complicated, but we only really care about
174 * three cases at the moment:
176 - * 1. Invalid (all zero) -> bypass (init)
177 - * 2. Bypass -> translation (attach)
178 - * 3. Translation -> bypass (detach)
179 + * 1. Invalid (all zero) -> bypass/fault (init)
180 + * 2. Bypass/fault -> translation/bypass (attach)
181 + * 3. Translation/bypass -> bypass/fault (detach)
183 * Given that we can't update the STE atomically and the SMMU
184 * doesn't read the thing in a defined order, that leaves us
185 @@ -1041,11 +1050,15 @@ static void arm_smmu_write_strtab_ent(st
188 /* Nuke the existing STE_0 value, as we're going to rewrite it */
189 - val = ste->valid ? STRTAB_STE_0_V : 0;
190 + val = STRTAB_STE_0_V;
193 + if (!ste->assigned || !(ste->s1_cfg || ste->s2_cfg)) {
194 + if (!ste->assigned && disable_bypass)
195 + val |= STRTAB_STE_0_CFG_ABORT;
197 + val |= STRTAB_STE_0_CFG_BYPASS;
200 - val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
201 - : STRTAB_STE_0_CFG_BYPASS;
202 dst[0] = cpu_to_le64(val);
203 dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
204 << STRTAB_STE_1_SHCFG_SHIFT);
205 @@ -1108,10 +1121,7 @@ static void arm_smmu_write_strtab_ent(st
206 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
209 - struct arm_smmu_strtab_ent ste = {
213 + struct arm_smmu_strtab_ent ste = { .assigned = false };
215 for (i = 0; i < nent; ++i) {
216 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
217 @@ -1365,8 +1375,6 @@ static bool arm_smmu_capable(enum iommu_
219 case IOMMU_CAP_CACHE_COHERENCY:
221 - case IOMMU_CAP_INTR_REMAP:
222 - return true; /* MSIs are just memory writes */
223 case IOMMU_CAP_NOEXEC:
226 @@ -1378,7 +1386,9 @@ static struct iommu_domain *arm_smmu_dom
228 struct arm_smmu_domain *smmu_domain;
230 - if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
231 + if (type != IOMMU_DOMAIN_UNMANAGED &&
232 + type != IOMMU_DOMAIN_DMA &&
233 + type != IOMMU_DOMAIN_IDENTITY)
237 @@ -1509,6 +1519,11 @@ static int arm_smmu_domain_finalise(stru
238 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
239 struct arm_smmu_device *smmu = smmu_domain->smmu;
241 + if (domain->type == IOMMU_DOMAIN_IDENTITY) {
242 + smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
246 /* Restrict the stage to what we can actually support */
247 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
248 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
249 @@ -1581,7 +1596,7 @@ static __le64 *arm_smmu_get_step_for_sid
253 -static int arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
254 +static void arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
257 struct arm_smmu_master_data *master = fwspec->iommu_priv;
258 @@ -1600,17 +1615,14 @@ static int arm_smmu_install_ste_for_dev(
260 arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste);
266 static void arm_smmu_detach_dev(struct device *dev)
268 struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
270 - master->ste.bypass = true;
271 - if (arm_smmu_install_ste_for_dev(dev->iommu_fwspec) < 0)
272 - dev_warn(dev, "failed to install bypass STE\n");
273 + master->ste.assigned = false;
274 + arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
277 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
278 @@ -1629,7 +1641,7 @@ static int arm_smmu_attach_dev(struct io
281 /* Already attached to a different domain? */
284 arm_smmu_detach_dev(dev);
286 mutex_lock(&smmu_domain->init_mutex);
287 @@ -1650,10 +1662,12 @@ static int arm_smmu_attach_dev(struct io
291 - ste->bypass = false;
293 + ste->assigned = true;
295 - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
296 + if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) {
297 + ste->s1_cfg = NULL;
298 + ste->s2_cfg = NULL;
299 + } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
300 ste->s1_cfg = &smmu_domain->s1_cfg;
302 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
303 @@ -1662,10 +1676,7 @@ static int arm_smmu_attach_dev(struct io
304 ste->s2_cfg = &smmu_domain->s2_cfg;
307 - ret = arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
309 - ste->valid = false;
311 + arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
313 mutex_unlock(&smmu_domain->init_mutex);
315 @@ -1696,6 +1707,9 @@ arm_smmu_unmap(struct iommu_domain *doma
316 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
317 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
319 + if (domain->type == IOMMU_DOMAIN_IDENTITY)
325 @@ -1811,7 +1825,7 @@ static void arm_smmu_remove_device(struc
328 master = fwspec->iommu_priv;
329 - if (master && master->ste.valid)
330 + if (master && master->ste.assigned)
331 arm_smmu_detach_dev(dev);
332 iommu_group_remove_device(dev);
334 @@ -1840,6 +1854,9 @@ static int arm_smmu_domain_get_attr(stru
336 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
338 + if (domain->type != IOMMU_DOMAIN_UNMANAGED)
342 case DOMAIN_ATTR_NESTING:
343 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
344 @@ -1855,6 +1872,9 @@ static int arm_smmu_domain_set_attr(stru
346 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
348 + if (domain->type != IOMMU_DOMAIN_UNMANAGED)
351 mutex_lock(&smmu_domain->init_mutex);
354 @@ -1884,6 +1904,31 @@ static int arm_smmu_of_xlate(struct devi
355 return iommu_fwspec_add_ids(dev, args->args, 1);
358 +static void arm_smmu_get_resv_regions(struct device *dev,
359 + struct list_head *head)
361 + struct iommu_resv_region *region;
362 + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
364 + region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
365 + prot, IOMMU_RESV_SW_MSI);
369 + list_add_tail(®ion->list, head);
371 + iommu_dma_get_resv_regions(dev, head);
374 +static void arm_smmu_put_resv_regions(struct device *dev,
375 + struct list_head *head)
377 + struct iommu_resv_region *entry, *next;
379 + list_for_each_entry_safe(entry, next, head, list)
383 static struct iommu_ops arm_smmu_ops = {
384 .capable = arm_smmu_capable,
385 .domain_alloc = arm_smmu_domain_alloc,
386 @@ -1899,6 +1944,8 @@ static struct iommu_ops arm_smmu_ops = {
387 .domain_get_attr = arm_smmu_domain_get_attr,
388 .domain_set_attr = arm_smmu_domain_set_attr,
389 .of_xlate = arm_smmu_of_xlate,
390 + .get_resv_regions = arm_smmu_get_resv_regions,
391 + .put_resv_regions = arm_smmu_put_resv_regions,
392 .pgsize_bitmap = -1UL, /* Restricted during device attach */
395 --- a/drivers/iommu/arm-smmu.c
396 +++ b/drivers/iommu/arm-smmu.c
398 #include <linux/spinlock.h>
400 #include <linux/amba/bus.h>
401 +#include <linux/fsl/mc.h>
403 #include "io-pgtable.h"
405 @@ -247,6 +248,7 @@ enum arm_smmu_s2cr_privcfg {
406 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
408 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
409 +#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
411 #define CB_PAR_F (1 << 0)
413 @@ -278,6 +280,9 @@ enum arm_smmu_s2cr_privcfg {
415 #define FSYNR0_WNR (1 << 4)
417 +#define MSI_IOVA_BASE 0x8000000
418 +#define MSI_IOVA_LENGTH 0x100000
420 static int force_stage;
421 module_param(force_stage, int, S_IRUGO);
422 MODULE_PARM_DESC(force_stage,
423 @@ -401,6 +406,7 @@ enum arm_smmu_domain_stage {
424 ARM_SMMU_DOMAIN_S1 = 0,
426 ARM_SMMU_DOMAIN_NESTED,
427 + ARM_SMMU_DOMAIN_BYPASS,
430 struct arm_smmu_domain {
431 @@ -821,6 +827,12 @@ static int arm_smmu_init_domain_context(
432 if (smmu_domain->smmu)
435 + if (domain->type == IOMMU_DOMAIN_IDENTITY) {
436 + smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
437 + smmu_domain->smmu = smmu;
442 * Mapping the requested stage onto what we support is surprisingly
443 * complicated, mainly because the spec allows S1+S2 SMMUs without
444 @@ -981,7 +993,7 @@ static void arm_smmu_destroy_domain_cont
445 void __iomem *cb_base;
449 + if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
453 @@ -1004,7 +1016,9 @@ static struct iommu_domain *arm_smmu_dom
455 struct arm_smmu_domain *smmu_domain;
457 - if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
458 + if (type != IOMMU_DOMAIN_UNMANAGED &&
459 + type != IOMMU_DOMAIN_DMA &&
460 + type != IOMMU_DOMAIN_IDENTITY)
463 * Allocate the domain and initialise some of its data structures.
464 @@ -1202,10 +1216,15 @@ static int arm_smmu_domain_add_master(st
466 struct arm_smmu_device *smmu = smmu_domain->smmu;
467 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
468 - enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
469 u8 cbndx = smmu_domain->cfg.cbndx;
470 + enum arm_smmu_s2cr_type type;
473 + if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
474 + type = S2CR_TYPE_BYPASS;
476 + type = S2CR_TYPE_TRANS;
478 for_each_cfg_sme(fwspec, i, idx) {
479 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
481 @@ -1343,6 +1362,9 @@ static phys_addr_t arm_smmu_iova_to_phys
482 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
483 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
485 + if (domain->type == IOMMU_DOMAIN_IDENTITY)
491 @@ -1368,8 +1390,6 @@ static bool arm_smmu_capable(enum iommu_
495 - case IOMMU_CAP_INTR_REMAP:
496 - return true; /* MSIs are just memory writes */
497 case IOMMU_CAP_NOEXEC:
500 @@ -1478,10 +1498,12 @@ static struct iommu_group *arm_smmu_devi
505 + return iommu_group_ref_get(group);
508 group = pci_device_group(dev);
509 + else if (dev_is_fsl_mc(dev))
510 + group = fsl_mc_device_group(dev);
512 group = generic_device_group(dev);
514 @@ -1493,6 +1515,9 @@ static int arm_smmu_domain_get_attr(stru
516 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
518 + if (domain->type != IOMMU_DOMAIN_UNMANAGED)
522 case DOMAIN_ATTR_NESTING:
523 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
524 @@ -1508,6 +1533,9 @@ static int arm_smmu_domain_set_attr(stru
526 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
528 + if (domain->type != IOMMU_DOMAIN_UNMANAGED)
531 mutex_lock(&smmu_domain->init_mutex);
534 @@ -1534,17 +1562,44 @@ out_unlock:
536 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
539 + u32 mask, fwid = 0;
541 if (args->args_count > 0)
542 fwid |= (u16)args->args[0];
544 if (args->args_count > 1)
545 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
546 + else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
547 + fwid |= (u16)mask << SMR_MASK_SHIFT;
549 return iommu_fwspec_add_ids(dev, &fwid, 1);
552 +static void arm_smmu_get_resv_regions(struct device *dev,
553 + struct list_head *head)
555 + struct iommu_resv_region *region;
556 + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
558 + region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
559 + prot, IOMMU_RESV_SW_MSI);
563 + list_add_tail(®ion->list, head);
565 + iommu_dma_get_resv_regions(dev, head);
568 +static void arm_smmu_put_resv_regions(struct device *dev,
569 + struct list_head *head)
571 + struct iommu_resv_region *entry, *next;
573 + list_for_each_entry_safe(entry, next, head, list)
577 static struct iommu_ops arm_smmu_ops = {
578 .capable = arm_smmu_capable,
579 .domain_alloc = arm_smmu_domain_alloc,
580 @@ -1560,6 +1615,8 @@ static struct iommu_ops arm_smmu_ops = {
581 .domain_get_attr = arm_smmu_domain_get_attr,
582 .domain_set_attr = arm_smmu_domain_set_attr,
583 .of_xlate = arm_smmu_of_xlate,
584 + .get_resv_regions = arm_smmu_get_resv_regions,
585 + .put_resv_regions = arm_smmu_put_resv_regions,
586 .pgsize_bitmap = -1UL, /* Restricted during device attach */
589 @@ -1581,16 +1638,22 @@ static void arm_smmu_device_reset(struct
590 for (i = 0; i < smmu->num_mapping_groups; ++i)
591 arm_smmu_write_sme(smmu, i);
594 - * Before clearing ARM_MMU500_ACTLR_CPRE, need to
595 - * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
596 - * bit is only present in MMU-500r2 onwards.
598 - reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
599 - major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
600 - if ((smmu->model == ARM_MMU500) && (major >= 2)) {
601 + if (smmu->model == ARM_MMU500) {
603 + * Before clearing ARM_MMU500_ACTLR_CPRE, need to
604 + * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
605 + * bit is only present in MMU-500r2 onwards.
607 + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
608 + major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
609 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
610 - reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
612 + reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
614 + * Allow unmatched Stream IDs to allocate bypass
615 + * TLB entries for reduced latency.
617 + reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
618 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
621 @@ -2024,6 +2087,11 @@ static int arm_smmu_device_dt_probe(stru
622 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
625 +#ifdef CONFIG_FSL_MC_BUS
626 + if (!iommu_present(&fsl_mc_bus_type))
627 + bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
633 --- a/drivers/iommu/dma-iommu.c
634 +++ b/drivers/iommu/dma-iommu.c
635 @@ -37,15 +37,50 @@ struct iommu_dma_msi_page {
639 +enum iommu_dma_cookie_type {
640 + IOMMU_DMA_IOVA_COOKIE,
641 + IOMMU_DMA_MSI_COOKIE,
644 struct iommu_dma_cookie {
645 - struct iova_domain iovad;
646 - struct list_head msi_page_list;
647 - spinlock_t msi_lock;
648 + enum iommu_dma_cookie_type type;
650 + /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
651 + struct iova_domain iovad;
652 + /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
653 + dma_addr_t msi_iova;
655 + struct list_head msi_page_list;
656 + spinlock_t msi_lock;
659 +static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
661 + if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
662 + return cookie->iovad.granule;
666 static inline struct iova_domain *cookie_iovad(struct iommu_domain *domain)
668 - return &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;
669 + struct iommu_dma_cookie *cookie = domain->iova_cookie;
671 + if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
672 + return &cookie->iovad;
676 +static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
678 + struct iommu_dma_cookie *cookie;
680 + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
682 + spin_lock_init(&cookie->msi_lock);
683 + INIT_LIST_HEAD(&cookie->msi_page_list);
684 + cookie->type = type;
689 int iommu_dma_init(void)
690 @@ -62,25 +97,53 @@ int iommu_dma_init(void)
692 int iommu_get_dma_cookie(struct iommu_domain *domain)
694 + if (domain->iova_cookie)
697 + domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
698 + if (!domain->iova_cookie)
703 +EXPORT_SYMBOL(iommu_get_dma_cookie);
706 + * iommu_get_msi_cookie - Acquire just MSI remapping resources
707 + * @domain: IOMMU domain to prepare
708 + * @base: Start address of IOVA region for MSI mappings
710 + * Users who manage their own IOVA allocation and do not want DMA API support,
711 + * but would still like to take advantage of automatic MSI remapping, can use
712 + * this to initialise their own domain appropriately. Users should reserve a
713 + * contiguous IOVA region, starting at @base, large enough to accommodate the
714 + * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address
715 + * used by the devices attached to @domain.
717 +int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
719 struct iommu_dma_cookie *cookie;
721 + if (domain->type != IOMMU_DOMAIN_UNMANAGED)
724 if (domain->iova_cookie)
727 - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
728 + cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
732 - spin_lock_init(&cookie->msi_lock);
733 - INIT_LIST_HEAD(&cookie->msi_page_list);
734 + cookie->msi_iova = base;
735 domain->iova_cookie = cookie;
738 -EXPORT_SYMBOL(iommu_get_dma_cookie);
739 +EXPORT_SYMBOL(iommu_get_msi_cookie);
742 * iommu_put_dma_cookie - Release a domain's DMA mapping resources
743 - * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
744 + * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
745 + * iommu_get_msi_cookie()
747 * IOMMU drivers should normally call this from their domain_free callback.
749 @@ -92,7 +155,7 @@ void iommu_put_dma_cookie(struct iommu_d
753 - if (cookie->iovad.granule)
754 + if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)
755 put_iova_domain(&cookie->iovad);
757 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
758 @@ -104,21 +167,99 @@ void iommu_put_dma_cookie(struct iommu_d
760 EXPORT_SYMBOL(iommu_put_dma_cookie);
762 -static void iova_reserve_pci_windows(struct pci_dev *dev,
763 - struct iova_domain *iovad)
765 + * iommu_dma_get_resv_regions - Reserved region driver helper
766 + * @dev: Device from iommu_get_resv_regions()
767 + * @list: Reserved region list from iommu_get_resv_regions()
769 + * IOMMU drivers can use this to implement their .get_resv_regions callback
770 + * for general non-IOMMU-specific reservations. Currently, this covers host
771 + * bridge windows for PCI devices.
773 +void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
775 - struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
776 + struct pci_host_bridge *bridge;
777 struct resource_entry *window;
778 - unsigned long lo, hi;
780 + if (!dev_is_pci(dev))
783 + bridge = pci_find_host_bridge(to_pci_dev(dev)->bus);
784 resource_list_for_each_entry(window, &bridge->windows) {
785 + struct iommu_resv_region *region;
789 if (resource_type(window->res) != IORESOURCE_MEM)
792 - lo = iova_pfn(iovad, window->res->start - window->offset);
793 - hi = iova_pfn(iovad, window->res->end - window->offset);
794 + start = window->res->start - window->offset;
795 + length = window->res->end - window->res->start + 1;
796 + region = iommu_alloc_resv_region(start, length, 0,
797 + IOMMU_RESV_RESERVED);
801 + list_add_tail(®ion->list, list);
804 +EXPORT_SYMBOL(iommu_dma_get_resv_regions);
806 +static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
807 + phys_addr_t start, phys_addr_t end)
809 + struct iova_domain *iovad = &cookie->iovad;
810 + struct iommu_dma_msi_page *msi_page;
813 + start -= iova_offset(iovad, start);
814 + num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);
816 + msi_page = kcalloc(num_pages, sizeof(*msi_page), GFP_KERNEL);
820 + for (i = 0; i < num_pages; i++) {
821 + msi_page[i].phys = start;
822 + msi_page[i].iova = start;
823 + INIT_LIST_HEAD(&msi_page[i].list);
824 + list_add(&msi_page[i].list, &cookie->msi_page_list);
825 + start += iovad->granule;
831 +static int iova_reserve_iommu_regions(struct device *dev,
832 + struct iommu_domain *domain)
834 + struct iommu_dma_cookie *cookie = domain->iova_cookie;
835 + struct iova_domain *iovad = &cookie->iovad;
836 + struct iommu_resv_region *region;
837 + LIST_HEAD(resv_regions);
840 + iommu_get_resv_regions(dev, &resv_regions);
841 + list_for_each_entry(region, &resv_regions, list) {
842 + unsigned long lo, hi;
844 + /* We ARE the software that manages these! */
845 + if (region->type == IOMMU_RESV_SW_MSI)
848 + lo = iova_pfn(iovad, region->start);
849 + hi = iova_pfn(iovad, region->start + region->length - 1);
850 reserve_iova(iovad, lo, hi);
852 + if (region->type == IOMMU_RESV_MSI)
853 + ret = cookie_init_hw_msi_region(cookie, region->start,
854 + region->start + region->length);
858 + iommu_put_resv_regions(dev, &resv_regions);
864 @@ -136,11 +277,12 @@ static void iova_reserve_pci_windows(str
865 int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
866 u64 size, struct device *dev)
868 - struct iova_domain *iovad = cookie_iovad(domain);
869 + struct iommu_dma_cookie *cookie = domain->iova_cookie;
870 + struct iova_domain *iovad = &cookie->iovad;
871 unsigned long order, base_pfn, end_pfn;
875 + if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
878 /* Use the smallest supported page size for IOVA granularity */
879 order = __ffs(domain->pgsize_bitmap);
880 @@ -160,22 +302,37 @@ int iommu_dma_init_domain(struct iommu_d
881 end_pfn = min_t(unsigned long, end_pfn,
882 domain->geometry.aperture_end >> order);
885 + * PCI devices may have larger DMA masks, but still prefer allocating
886 + * within a 32-bit mask to avoid DAC addressing. Such limitations don't
887 + * apply to the typical platform device, so for those we may as well
888 + * leave the cache limit at the top of their range to save an rb_last()
889 + * traversal on every allocation.
891 + if (dev && dev_is_pci(dev))
892 + end_pfn &= DMA_BIT_MASK(32) >> order;
894 - /* All we can safely do with an existing domain is enlarge it */
895 + /* start_pfn is always nonzero for an already-initialised domain */
896 if (iovad->start_pfn) {
897 if (1UL << order != iovad->granule ||
898 - base_pfn != iovad->start_pfn ||
899 - end_pfn < iovad->dma_32bit_pfn) {
900 + base_pfn != iovad->start_pfn) {
901 pr_warn("Incompatible range for DMA domain\n");
904 - iovad->dma_32bit_pfn = end_pfn;
906 - init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn);
907 - if (dev && dev_is_pci(dev))
908 - iova_reserve_pci_windows(to_pci_dev(dev), iovad);
910 + * If we have devices with different DMA masks, move the free
911 + * area cache limit down for the benefit of the smaller one.
913 + iovad->dma_32bit_pfn = min(end_pfn, iovad->dma_32bit_pfn);
919 + init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn);
923 + return iova_reserve_iommu_regions(dev, domain);
925 EXPORT_SYMBOL(iommu_dma_init_domain);
927 @@ -643,11 +800,12 @@ static struct iommu_dma_msi_page *iommu_
929 struct iommu_dma_cookie *cookie = domain->iova_cookie;
930 struct iommu_dma_msi_page *msi_page;
931 - struct iova_domain *iovad = &cookie->iovad;
932 + struct iova_domain *iovad = cookie_iovad(domain);
934 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
935 + size_t size = cookie_msi_granule(cookie);
937 - msi_addr &= ~(phys_addr_t)iova_mask(iovad);
938 + msi_addr &= ~(phys_addr_t)(size - 1);
939 list_for_each_entry(msi_page, &cookie->msi_page_list, list)
940 if (msi_page->phys == msi_addr)
942 @@ -656,13 +814,18 @@ static struct iommu_dma_msi_page *iommu_
946 - iova = __alloc_iova(domain, iovad->granule, dma_get_mask(dev));
948 - goto out_free_page;
950 msi_page->phys = msi_addr;
951 - msi_page->iova = iova_dma_addr(iovad, iova);
952 - if (iommu_map(domain, msi_page->iova, msi_addr, iovad->granule, prot))
954 + iova = __alloc_iova(domain, size, dma_get_mask(dev));
956 + goto out_free_page;
957 + msi_page->iova = iova_dma_addr(iovad, iova);
959 + msi_page->iova = cookie->msi_iova;
960 + cookie->msi_iova += size;
963 + if (iommu_map(domain, msi_page->iova, msi_addr, size, prot))
966 INIT_LIST_HEAD(&msi_page->list);
967 @@ -670,7 +833,10 @@ static struct iommu_dma_msi_page *iommu_
971 - __free_iova(iovad, iova);
973 + __free_iova(iovad, iova);
975 + cookie->msi_iova -= size;
979 @@ -711,7 +877,7 @@ void iommu_dma_map_msi_msg(int irq, stru
982 msg->address_hi = upper_32_bits(msi_page->iova);
983 - msg->address_lo &= iova_mask(&cookie->iovad);
984 + msg->address_lo &= cookie_msi_granule(cookie) - 1;
985 msg->address_lo += lower_32_bits(msi_page->iova);
988 --- a/drivers/iommu/intel-iommu.c
989 +++ b/drivers/iommu/intel-iommu.c
990 @@ -441,6 +441,7 @@ struct dmar_rmrr_unit {
991 u64 end_address; /* reserved end address */
992 struct dmar_dev_scope *devices; /* target devices */
993 int devices_cnt; /* target device count */
994 + struct iommu_resv_region *resv; /* reserved region handle */
997 struct dmar_atsr_unit {
998 @@ -4267,27 +4268,40 @@ static inline void init_iommu_pm_ops(voi
999 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
1001 struct acpi_dmar_reserved_memory *rmrr;
1002 + int prot = DMA_PTE_READ|DMA_PTE_WRITE;
1003 struct dmar_rmrr_unit *rmrru;
1006 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
1011 rmrru->hdr = header;
1012 rmrr = (struct acpi_dmar_reserved_memory *)header;
1013 rmrru->base_address = rmrr->base_address;
1014 rmrru->end_address = rmrr->end_address;
1016 + length = rmrr->end_address - rmrr->base_address + 1;
1017 + rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
1018 + IOMMU_RESV_DIRECT);
1022 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
1023 ((void *)rmrr) + rmrr->header.length,
1024 &rmrru->devices_cnt);
1025 - if (rmrru->devices_cnt && rmrru->devices == NULL) {
1029 + if (rmrru->devices_cnt && rmrru->devices == NULL)
1032 list_add(&rmrru->list, &dmar_rmrr_units);
1036 + kfree(rmrru->resv);
1043 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
1044 @@ -4501,6 +4515,7 @@ static void intel_iommu_free_dmars(void)
1045 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
1046 list_del(&rmrru->list);
1047 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
1048 + kfree(rmrru->resv);
1052 @@ -5236,6 +5251,45 @@ static void intel_iommu_remove_device(st
1053 iommu_device_unlink(iommu->iommu_dev, dev);
1056 +static void intel_iommu_get_resv_regions(struct device *device,
1057 + struct list_head *head)
1059 + struct iommu_resv_region *reg;
1060 + struct dmar_rmrr_unit *rmrr;
1061 + struct device *i_dev;
1065 + for_each_rmrr_units(rmrr) {
1066 + for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
1068 + if (i_dev != device)
1071 + list_add_tail(&rmrr->resv->list, head);
1074 + rcu_read_unlock();
1076 + reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
1077 + IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
1078 + 0, IOMMU_RESV_MSI);
1081 + list_add_tail(®->list, head);
1084 +static void intel_iommu_put_resv_regions(struct device *dev,
1085 + struct list_head *head)
1087 + struct iommu_resv_region *entry, *next;
1089 + list_for_each_entry_safe(entry, next, head, list) {
1090 + if (entry->type == IOMMU_RESV_RESERVED)
1095 #ifdef CONFIG_INTEL_IOMMU_SVM
1096 #define MAX_NR_PASID_BITS (20)
1097 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
1098 @@ -5366,19 +5420,21 @@ struct intel_iommu *intel_svm_device_to_
1099 #endif /* CONFIG_INTEL_IOMMU_SVM */
1101 static const struct iommu_ops intel_iommu_ops = {
1102 - .capable = intel_iommu_capable,
1103 - .domain_alloc = intel_iommu_domain_alloc,
1104 - .domain_free = intel_iommu_domain_free,
1105 - .attach_dev = intel_iommu_attach_device,
1106 - .detach_dev = intel_iommu_detach_device,
1107 - .map = intel_iommu_map,
1108 - .unmap = intel_iommu_unmap,
1109 - .map_sg = default_iommu_map_sg,
1110 - .iova_to_phys = intel_iommu_iova_to_phys,
1111 - .add_device = intel_iommu_add_device,
1112 - .remove_device = intel_iommu_remove_device,
1113 - .device_group = pci_device_group,
1114 - .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
1115 + .capable = intel_iommu_capable,
1116 + .domain_alloc = intel_iommu_domain_alloc,
1117 + .domain_free = intel_iommu_domain_free,
1118 + .attach_dev = intel_iommu_attach_device,
1119 + .detach_dev = intel_iommu_detach_device,
1120 + .map = intel_iommu_map,
1121 + .unmap = intel_iommu_unmap,
1122 + .map_sg = default_iommu_map_sg,
1123 + .iova_to_phys = intel_iommu_iova_to_phys,
1124 + .add_device = intel_iommu_add_device,
1125 + .remove_device = intel_iommu_remove_device,
1126 + .get_resv_regions = intel_iommu_get_resv_regions,
1127 + .put_resv_regions = intel_iommu_put_resv_regions,
1128 + .device_group = pci_device_group,
1129 + .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
1132 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
1133 --- a/drivers/iommu/iommu.c
1134 +++ b/drivers/iommu/iommu.c
1136 #include <linux/bitops.h>
1137 #include <linux/property.h>
1138 #include <trace/events/iommu.h>
1139 +#include <linux/fsl/mc.h>
1141 static struct kset *iommu_group_kset;
1142 static DEFINE_IDA(iommu_group_ida);
1143 +static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
1145 struct iommu_callback_data {
1146 const struct iommu_ops *ops;
1147 @@ -68,6 +70,13 @@ struct iommu_group_attribute {
1148 const char *buf, size_t count);
1151 +static const char * const iommu_group_resv_type_string[] = {
1152 + [IOMMU_RESV_DIRECT] = "direct",
1153 + [IOMMU_RESV_RESERVED] = "reserved",
1154 + [IOMMU_RESV_MSI] = "msi",
1155 + [IOMMU_RESV_SW_MSI] = "msi",
1158 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
1159 struct iommu_group_attribute iommu_group_attr_##_name = \
1160 __ATTR(_name, _mode, _show, _store)
1161 @@ -86,6 +95,18 @@ static int __iommu_attach_group(struct i
1162 static void __iommu_detach_group(struct iommu_domain *domain,
1163 struct iommu_group *group);
1165 +static int __init iommu_set_def_domain_type(char *str)
1169 + if (!str || strtobool(str, &pt))
1172 + iommu_def_domain_type = pt ? IOMMU_DOMAIN_IDENTITY : IOMMU_DOMAIN_DMA;
1175 +early_param("iommu.passthrough", iommu_set_def_domain_type);
1177 static ssize_t iommu_group_attr_show(struct kobject *kobj,
1178 struct attribute *__attr, char *buf)
1180 @@ -133,8 +154,131 @@ static ssize_t iommu_group_show_name(str
1181 return sprintf(buf, "%s\n", group->name);
1185 + * iommu_insert_resv_region - Insert a new region in the
1186 + * list of reserved regions.
1187 + * @new: new region to insert
1188 + * @regions: list of regions
1190 + * The new element is sorted by address with respect to the other
1191 + * regions of the same type. In case it overlaps with another
1192 + * region of the same type, regions are merged. In case it
1193 + * overlaps with another region of different type, regions are
1196 +static int iommu_insert_resv_region(struct iommu_resv_region *new,
1197 + struct list_head *regions)
1199 + struct iommu_resv_region *region;
1200 + phys_addr_t start = new->start;
1201 + phys_addr_t end = new->start + new->length - 1;
1202 + struct list_head *pos = regions->next;
1204 + while (pos != regions) {
1205 + struct iommu_resv_region *entry =
1206 + list_entry(pos, struct iommu_resv_region, list);
1207 + phys_addr_t a = entry->start;
1208 + phys_addr_t b = entry->start + entry->length - 1;
1209 + int type = entry->type;
1213 + } else if (start > b) {
1215 + } else if ((start >= a) && (end <= b)) {
1216 + if (new->type == type)
1221 + if (new->type == type) {
1222 + phys_addr_t new_start = min(a, start);
1223 + phys_addr_t new_end = max(b, end);
1225 + list_del(&entry->list);
1226 + entry->start = new_start;
1227 + entry->length = new_end - new_start + 1;
1228 + iommu_insert_resv_region(entry, regions);
1235 + region = iommu_alloc_resv_region(new->start, new->length,
1236 + new->prot, new->type);
1240 + list_add_tail(®ion->list, pos);
1246 +iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
1247 + struct list_head *group_resv_regions)
1249 + struct iommu_resv_region *entry;
1252 + list_for_each_entry(entry, dev_resv_regions, list) {
1253 + ret = iommu_insert_resv_region(entry, group_resv_regions);
1260 +int iommu_get_group_resv_regions(struct iommu_group *group,
1261 + struct list_head *head)
1263 + struct iommu_device *device;
1266 + mutex_lock(&group->mutex);
1267 + list_for_each_entry(device, &group->devices, list) {
1268 + struct list_head dev_resv_regions;
1270 + INIT_LIST_HEAD(&dev_resv_regions);
1271 + iommu_get_resv_regions(device->dev, &dev_resv_regions);
1272 + ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
1273 + iommu_put_resv_regions(device->dev, &dev_resv_regions);
1277 + mutex_unlock(&group->mutex);
1280 +EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
1282 +static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
1285 + struct iommu_resv_region *region, *next;
1286 + struct list_head group_resv_regions;
1289 + INIT_LIST_HEAD(&group_resv_regions);
1290 + iommu_get_group_resv_regions(group, &group_resv_regions);
1292 + list_for_each_entry_safe(region, next, &group_resv_regions, list) {
1293 + str += sprintf(str, "0x%016llx 0x%016llx %s\n",
1294 + (long long int)region->start,
1295 + (long long int)(region->start +
1296 + region->length - 1),
1297 + iommu_group_resv_type_string[region->type]);
1301 + return (str - buf);
1304 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
1306 +static IOMMU_GROUP_ATTR(reserved_regions, 0444,
1307 + iommu_group_show_resv_regions, NULL);
1309 static void iommu_group_release(struct kobject *kobj)
1311 struct iommu_group *group = to_iommu_group(kobj);
1312 @@ -212,6 +356,11 @@ struct iommu_group *iommu_group_alloc(vo
1314 kobject_put(&group->kobj);
1316 + ret = iommu_group_create_file(group,
1317 + &iommu_group_attr_reserved_regions);
1319 + return ERR_PTR(ret);
1321 pr_debug("Allocated group %d\n", group->id);
1324 @@ -318,7 +467,7 @@ static int iommu_group_create_direct_map
1327 struct iommu_domain *domain = group->default_domain;
1328 - struct iommu_dm_region *entry;
1329 + struct iommu_resv_region *entry;
1330 struct list_head mappings;
1331 unsigned long pg_size;
1333 @@ -331,18 +480,21 @@ static int iommu_group_create_direct_map
1334 pg_size = 1UL << __ffs(domain->pgsize_bitmap);
1335 INIT_LIST_HEAD(&mappings);
1337 - iommu_get_dm_regions(dev, &mappings);
1338 + iommu_get_resv_regions(dev, &mappings);
1340 /* We need to consider overlapping regions for different devices */
1341 list_for_each_entry(entry, &mappings, list) {
1342 dma_addr_t start, end, addr;
1344 - if (domain->ops->apply_dm_region)
1345 - domain->ops->apply_dm_region(dev, domain, entry);
1346 + if (domain->ops->apply_resv_region)
1347 + domain->ops->apply_resv_region(dev, domain, entry);
1349 start = ALIGN(entry->start, pg_size);
1350 end = ALIGN(entry->start + entry->length, pg_size);
1352 + if (entry->type != IOMMU_RESV_DIRECT)
1355 for (addr = start; addr < end; addr += pg_size) {
1356 phys_addr_t phys_addr;
1358 @@ -358,7 +510,7 @@ static int iommu_group_create_direct_map
1362 - iommu_put_dm_regions(dev, &mappings);
1363 + iommu_put_resv_regions(dev, &mappings);
1367 @@ -563,6 +715,19 @@ struct iommu_group *iommu_group_get(stru
1368 EXPORT_SYMBOL_GPL(iommu_group_get);
1371 + * iommu_group_ref_get - Increment reference on a group
1372 + * @group: the group to use, must not be NULL
1374 + * This function is called by iommu drivers to take additional references on an
1375 + * existing group. Returns the given group for convenience.
1377 +struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
1379 + kobject_get(group->devices_kobj);
1384 * iommu_group_put - Decrement group reference
1385 * @group: the group to use
1387 @@ -812,6 +977,26 @@ struct iommu_group *pci_device_group(str
1391 +/* Get the IOMMU group for device on fsl-mc bus */
1392 +struct iommu_group *fsl_mc_device_group(struct device *dev)
1394 + struct device *cont_dev = fsl_mc_cont_dev(dev);
1395 + struct iommu_group *group;
1397 + /* Container device is responsible for creating the iommu group */
1398 + if (fsl_mc_is_cont_dev(dev)) {
1399 + group = iommu_group_alloc();
1400 + if (IS_ERR(group))
1403 + get_device(cont_dev);
1404 + group = iommu_group_get(cont_dev);
1405 + put_device(cont_dev);
1412 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
1413 * @dev: target device
1414 @@ -845,10 +1030,19 @@ struct iommu_group *iommu_group_get_for_
1417 if (!group->default_domain) {
1418 - group->default_domain = __iommu_domain_alloc(dev->bus,
1419 - IOMMU_DOMAIN_DMA);
1420 + struct iommu_domain *dom;
1422 + dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type);
1423 + if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) {
1425 + "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA",
1426 + iommu_def_domain_type);
1427 + dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA);
1430 + group->default_domain = dom;
1432 - group->domain = group->default_domain;
1433 + group->domain = dom;
1436 ret = iommu_group_add_device(group, dev);
1437 @@ -1557,20 +1751,38 @@ int iommu_domain_set_attr(struct iommu_d
1439 EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
1441 -void iommu_get_dm_regions(struct device *dev, struct list_head *list)
1442 +void iommu_get_resv_regions(struct device *dev, struct list_head *list)
1444 const struct iommu_ops *ops = dev->bus->iommu_ops;
1446 - if (ops && ops->get_dm_regions)
1447 - ops->get_dm_regions(dev, list);
1448 + if (ops && ops->get_resv_regions)
1449 + ops->get_resv_regions(dev, list);
1452 -void iommu_put_dm_regions(struct device *dev, struct list_head *list)
1453 +void iommu_put_resv_regions(struct device *dev, struct list_head *list)
1455 const struct iommu_ops *ops = dev->bus->iommu_ops;
1457 - if (ops && ops->put_dm_regions)
1458 - ops->put_dm_regions(dev, list);
1459 + if (ops && ops->put_resv_regions)
1460 + ops->put_resv_regions(dev, list);
1463 +struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
1464 + size_t length, int prot,
1465 + enum iommu_resv_type type)
1467 + struct iommu_resv_region *region;
1469 + region = kzalloc(sizeof(*region), GFP_KERNEL);
1473 + INIT_LIST_HEAD(®ion->list);
1474 + region->start = start;
1475 + region->length = length;
1476 + region->prot = prot;
1477 + region->type = type;
1481 /* Request that a device is direct mapped by the IOMMU */
1482 --- a/drivers/iommu/mtk_iommu.c
1483 +++ b/drivers/iommu/mtk_iommu.c
1484 @@ -410,6 +410,8 @@ static struct iommu_group *mtk_iommu_dev
1485 data->m4u_group = iommu_group_alloc();
1486 if (IS_ERR(data->m4u_group))
1487 dev_err(dev, "Failed to allocate M4U IOMMU group\n");
1489 + iommu_group_ref_get(data->m4u_group);
1491 return data->m4u_group;
1493 --- a/drivers/iommu/mtk_iommu_v1.c
1494 +++ b/drivers/iommu/mtk_iommu_v1.c
1495 @@ -502,6 +502,8 @@ static struct iommu_group *mtk_iommu_dev
1496 data->m4u_group = iommu_group_alloc();
1497 if (IS_ERR(data->m4u_group))
1498 dev_err(dev, "Failed to allocate M4U IOMMU group\n");
1500 + iommu_group_ref_get(data->m4u_group);
1502 return data->m4u_group;
1504 --- a/include/linux/dma-iommu.h
1505 +++ b/include/linux/dma-iommu.h
1506 @@ -28,6 +28,7 @@ int iommu_dma_init(void);
1508 /* Domain management interface for IOMMU drivers */
1509 int iommu_get_dma_cookie(struct iommu_domain *domain);
1510 +int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
1511 void iommu_put_dma_cookie(struct iommu_domain *domain);
1513 /* Setup call for arch DMA mapping code */
1514 @@ -67,6 +68,7 @@ int iommu_dma_mapping_error(struct devic
1516 /* The DMA API isn't _quite_ the whole story, though... */
1517 void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg);
1518 +void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
1522 @@ -83,6 +85,11 @@ static inline int iommu_get_dma_cookie(s
1526 +static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
1531 static inline void iommu_put_dma_cookie(struct iommu_domain *domain)
1534 @@ -91,6 +98,10 @@ static inline void iommu_dma_map_msi_msg
1538 +static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
1542 #endif /* CONFIG_IOMMU_DMA */
1543 #endif /* __KERNEL__ */
1544 #endif /* __DMA_IOMMU_H */
1545 --- a/include/linux/iommu.h
1546 +++ b/include/linux/iommu.h
1547 @@ -117,18 +117,32 @@ enum iommu_attr {
1551 +/* These are the possible reserved region types */
1552 +enum iommu_resv_type {
1553 + /* Memory regions which must be mapped 1:1 at all times */
1554 + IOMMU_RESV_DIRECT,
1555 + /* Arbitrary "never map this or give it to a device" address ranges */
1556 + IOMMU_RESV_RESERVED,
1557 + /* Hardware MSI region (untranslated) */
1559 + /* Software-managed MSI translation window */
1560 + IOMMU_RESV_SW_MSI,
1564 - * struct iommu_dm_region - descriptor for a direct mapped memory region
1565 + * struct iommu_resv_region - descriptor for a reserved memory region
1566 * @list: Linked list pointers
1567 * @start: System physical start address of the region
1568 * @length: Length of the region in bytes
1569 * @prot: IOMMU Protection flags (READ/WRITE/...)
1570 + * @type: Type of the reserved region
1572 -struct iommu_dm_region {
1573 +struct iommu_resv_region {
1574 struct list_head list;
1578 + enum iommu_resv_type type;
1581 #ifdef CONFIG_IOMMU_API
1582 @@ -150,9 +164,9 @@ struct iommu_dm_region {
1583 * @device_group: find iommu group for a particular device
1584 * @domain_get_attr: Query domain attributes
1585 * @domain_set_attr: Change domain attributes
1586 - * @get_dm_regions: Request list of direct mapping requirements for a device
1587 - * @put_dm_regions: Free list of direct mapping requirements for a device
1588 - * @apply_dm_region: Temporary helper call-back for iova reserved ranges
1589 + * @get_resv_regions: Request list of reserved regions for a device
1590 + * @put_resv_regions: Free list of reserved regions for a device
1591 + * @apply_resv_region: Temporary helper call-back for iova reserved ranges
1592 * @domain_window_enable: Configure and enable a particular window for a domain
1593 * @domain_window_disable: Disable a particular window for a domain
1594 * @domain_set_windows: Set the number of windows for a domain
1595 @@ -184,11 +198,12 @@ struct iommu_ops {
1596 int (*domain_set_attr)(struct iommu_domain *domain,
1597 enum iommu_attr attr, void *data);
1599 - /* Request/Free a list of direct mapping requirements for a device */
1600 - void (*get_dm_regions)(struct device *dev, struct list_head *list);
1601 - void (*put_dm_regions)(struct device *dev, struct list_head *list);
1602 - void (*apply_dm_region)(struct device *dev, struct iommu_domain *domain,
1603 - struct iommu_dm_region *region);
1604 + /* Request/Free a list of reserved regions for a device */
1605 + void (*get_resv_regions)(struct device *dev, struct list_head *list);
1606 + void (*put_resv_regions)(struct device *dev, struct list_head *list);
1607 + void (*apply_resv_region)(struct device *dev,
1608 + struct iommu_domain *domain,
1609 + struct iommu_resv_region *region);
1611 /* Window handling functions */
1612 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr,
1613 @@ -233,9 +248,14 @@ extern phys_addr_t iommu_iova_to_phys(st
1614 extern void iommu_set_fault_handler(struct iommu_domain *domain,
1615 iommu_fault_handler_t handler, void *token);
1617 -extern void iommu_get_dm_regions(struct device *dev, struct list_head *list);
1618 -extern void iommu_put_dm_regions(struct device *dev, struct list_head *list);
1619 +extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
1620 +extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
1621 extern int iommu_request_dm_for_dev(struct device *dev);
1622 +extern struct iommu_resv_region *
1623 +iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
1624 + enum iommu_resv_type type);
1625 +extern int iommu_get_group_resv_regions(struct iommu_group *group,
1626 + struct list_head *head);
1628 extern int iommu_attach_group(struct iommu_domain *domain,
1629 struct iommu_group *group);
1630 @@ -253,6 +273,7 @@ extern void iommu_group_remove_device(st
1631 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data,
1632 int (*fn)(struct device *, void *));
1633 extern struct iommu_group *iommu_group_get(struct device *dev);
1634 +extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group);
1635 extern void iommu_group_put(struct iommu_group *group);
1636 extern int iommu_group_register_notifier(struct iommu_group *group,
1637 struct notifier_block *nb);
1638 @@ -330,6 +351,8 @@ static inline size_t iommu_map_sg(struct
1639 extern struct iommu_group *pci_device_group(struct device *dev);
1640 /* Generic device grouping function */
1641 extern struct iommu_group *generic_device_group(struct device *dev);
1642 +/* FSL-MC device grouping function */
1643 +struct iommu_group *fsl_mc_device_group(struct device *dev);
1646 * struct iommu_fwspec - per-device IOMMU instance data
1647 @@ -439,16 +462,22 @@ static inline void iommu_set_fault_handl
1651 -static inline void iommu_get_dm_regions(struct device *dev,
1652 +static inline void iommu_get_resv_regions(struct device *dev,
1653 struct list_head *list)
1657 -static inline void iommu_put_dm_regions(struct device *dev,
1658 +static inline void iommu_put_resv_regions(struct device *dev,
1659 struct list_head *list)
1663 +static inline int iommu_get_group_resv_regions(struct iommu_group *group,
1664 + struct list_head *head)
1669 static inline int iommu_request_dm_for_dev(struct device *dev)