1 From 5fd6ee7fd084838e09d4e463ae53cd9aaa7fce70 Mon Sep 17 00:00:00 2001
2 From: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>
3 Date: Thu, 11 May 2023 16:37:34 +0100
4 Subject: [PATCH] drivers: iommu: Add BCM2712 IOMMU
6 Add a driver for BCM2712 IOMMUs.
7 There is a small driver for the Shared IOMMU TLB Cache.
8 Each IOMMU instance is a separate device.
10 IOMMUs are set up with a "pass-through" range covering
11 the lowest 40BGytes (which should cover all of SDRAM)
12 for the benefit of non-IOMMU-aware devices that share
13 a physical IOMMU; and translation for addresses in the
16 An optional parameter adds a DMA offset (which otherwise
17 would be lost?) to virtual addresses for DMA masters on a
20 Signed-off-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>
22 drivers/iommu/Kconfig | 7 +
23 drivers/iommu/Makefile | 1 +
24 drivers/iommu/bcm2712-iommu-cache.c | 77 ++++
25 drivers/iommu/bcm2712-iommu.c | 672 ++++++++++++++++++++++++++++
26 drivers/iommu/bcm2712-iommu.h | 45 ++
27 5 files changed, 802 insertions(+)
28 create mode 100644 drivers/iommu/bcm2712-iommu-cache.c
29 create mode 100644 drivers/iommu/bcm2712-iommu.c
30 create mode 100644 drivers/iommu/bcm2712-iommu.h
32 --- a/drivers/iommu/Kconfig
33 +++ b/drivers/iommu/Kconfig
34 @@ -506,4 +506,11 @@ config SPRD_IOMMU
36 Say Y here if you want to use the multimedia devices listed above.
39 + tristate "BCM2712 IOMMU driver"
40 + depends on ARM64 && ARCH_BCM
43 + IOMMU driver for BCM2712
46 --- a/drivers/iommu/Makefile
47 +++ b/drivers/iommu/Makefile
48 @@ -31,3 +31,4 @@ obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iom
49 obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o
50 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
51 obj-$(CONFIG_APPLE_DART) += apple-dart.o
52 +obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o
54 +++ b/drivers/iommu/bcm2712-iommu-cache.c
56 +// SPDX-License-Identifier: GPL-2.0-only
58 + * IOMMU driver for BCM2712
60 + * Copyright (c) 2023 Raspberry Pi Ltd.
63 +#include "bcm2712-iommu.h"
65 +#include <linux/err.h>
66 +#include <linux/of_platform.h>
67 +#include <linux/platform_device.h>
68 +#include <linux/spinlock.h>
70 +#define MMUC_CONTROL_ENABLE 1
71 +#define MMUC_CONTROL_FLUSH 2
72 +#define MMUC_CONTROL_FLUSHING 4
74 +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache)
76 + unsigned long flags;
79 + spin_lock_irqsave(&cache->hw_lock, flags);
80 + if (cache->reg_base) {
81 + /* Enable and flush the TLB cache */
82 + writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH,
85 + /* Wait for flush to complete: it should be very quick */
86 + for (i = 0; i < 1024; i++) {
87 + if (!(MMUC_CONTROL_FLUSHING & readl(cache->reg_base)))
92 + spin_unlock_irqrestore(&cache->hw_lock, flags);
95 +static int bcm2712_iommu_cache_probe(struct platform_device *pdev)
97 + struct bcm2712_iommu_cache *cache;
99 + dev_info(&pdev->dev, __func__);
100 + cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL);
104 + cache->dev = &pdev->dev;
105 + platform_set_drvdata(pdev, cache);
106 + spin_lock_init(&cache->hw_lock);
108 + /* Get IOMMUC registers; we only use the first register (IOMMUC_CTRL) */
109 + cache->reg_base = devm_platform_ioremap_resource(pdev, 0);
110 + if (IS_ERR(cache->reg_base)) {
111 + dev_err(&pdev->dev, "Failed to get IOMMU Cache registers address\n");
112 + cache->reg_base = NULL;
117 +static const struct of_device_id bcm2712_iommu_cache_of_match[] = {
119 + . compatible = "brcm,bcm2712-iommuc"
121 + { /* sentinel */ },
124 +static struct platform_driver bcm2712_iommu_cache_driver = {
125 + .probe = bcm2712_iommu_cache_probe,
127 + .name = "bcm2712-iommu-cache",
128 + .of_match_table = bcm2712_iommu_cache_of_match
132 +builtin_platform_driver(bcm2712_iommu_cache_driver);
134 +++ b/drivers/iommu/bcm2712-iommu.c
136 +// SPDX-License-Identifier: GPL-2.0-only
138 + * IOMMU driver for BCM2712
140 + * Copyright (c) 2023 Raspberry Pi Ltd.
143 +#include "bcm2712-iommu.h"
145 +#include <linux/dma-mapping.h>
146 +#include <linux/err.h>
147 +#include <linux/iommu.h>
148 +#include <linux/of_platform.h>
149 +#include <linux/platform_device.h>
150 +#include <linux/spinlock.h>
152 +#define MMU_WR(off, val) writel(val, mmu->reg_base + (off))
153 +#define MMU_RD(off) readl(mmu->reg_base + (off))
155 +#define domain_to_mmu(d) (container_of(d, struct bcm2712_iommu_domain, base)->mmu)
157 +#define MMMU_CTRL_OFFSET 0x00
158 +#define MMMU_CTRL_CAP_EXCEEDED BIT(27)
159 +#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN BIT(26)
160 +#define MMMU_CTRL_CAP_EXCEEDED_INT_EN BIT(25)
161 +#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN BIT(24)
162 +#define MMMU_CTRL_PT_INVALID BIT(20)
163 +#define MMMU_CTRL_PT_INVALID_ABORT_EN BIT(19)
164 +#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN BIT(18)
165 +#define MMMU_CTRL_PT_INVALID_EN BIT(17)
166 +#define MMMU_CTRL_WRITE_VIOLATION BIT(12)
167 +#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN BIT(11)
168 +#define MMMU_CTRL_WRITE_VIOLATION_INT_EN BIT(10)
169 +#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9)
170 +#define MMMU_CTRL_BYPASS BIT(8)
171 +#define MMMU_CTRL_TLB_CLEARING BIT(7)
172 +#define MMMU_CTRL_STATS_CLEAR BIT(3)
173 +#define MMMU_CTRL_TLB_CLEAR BIT(2)
174 +#define MMMU_CTRL_STATS_ENABLE BIT(1)
175 +#define MMMU_CTRL_ENABLE BIT(0)
177 +#define MMMU_PT_PA_BASE_OFFSET 0x04
179 +#define MMMU_HIT_OFFSET 0x08
180 +#define MMMU_MISS_OFFSET 0x0C
181 +#define MMMU_STALL_OFFSET 0x10
183 +#define MMMU_ADDR_CAP_OFFSET 0x14
184 +#define MMMU_ADDR_CAP_ENABLE BIT(31)
185 +#define ADDR_CAP_SHIFT 28 /* ADDR_CAP is defined to be in 256 MByte units */
187 +#define MMMU_SHOOT_DOWN_OFFSET 0x18
188 +#define MMMU_SHOOT_DOWN_SHOOTING BIT(31)
189 +#define MMMU_SHOOT_DOWN_SHOOT BIT(30)
191 +#define MMMU_BYPASS_START_OFFSET 0x1C
192 +#define MMMU_BYPASS_START_ENABLE BIT(31)
193 +#define MMMU_BYPASS_START_INVERT BIT(30)
195 +#define MMMU_BYPASS_END_OFFSET 0x20
196 +#define MMMU_BYPASS_END_ENABLE BIT(31)
198 +#define MMMU_MISC_OFFSET 0x24
199 +#define MMMU_MISC_SINGLE_TABLE BIT(31)
201 +#define MMMU_ILLEGAL_ADR_OFFSET 0x30
202 +#define MMMU_ILLEGAL_ADR_ENABLE BIT(31)
204 +#define MMMU_DEBUG_INFO_OFFSET 0x38
205 +#define MMMU_DEBUG_INFO_VERSION_MASK 0x0000000Fu
206 +#define MMMU_DEBUG_INFO_VA_WIDTH_MASK 0x000000F0u
207 +#define MMMU_DEBUG_INFO_PA_WIDTH_MASK 0x00000F00u
208 +#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK 0x000FF000u
209 +#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK 0x0FF00000u
210 +#define MMMU_DEBUG_INFO_BYPASS_4M BIT(28)
211 +#define MMMU_DEBUG_INFO_BYPASS BIT(29)
213 +#define MMMU_PTE_PAGESIZE_MASK 0xC0000000u
214 +#define MMMU_PTE_WRITEABLE BIT(29)
215 +#define MMMU_PTE_VALID BIT(28)
218 + * BCM2712 IOMMU is organized around 4Kbyte pages (MMU_PAGE_SIZE).
219 + * Linux PAGE_SIZE must not be smaller but may be larger (e.g. 4K, 16K).
221 + * Unlike many larger MMUs, this one uses a 4-byte word size, allowing
222 + * 1024 entries within each 4K table page, and two-level translation.
224 + * Let's allocate enough table space for 2GB of translated memory (IOVA).
225 + * This requires 512 4K pages (2MB) of level-2 tables, one page of
226 + * top-level table (only half-filled in this particular configuration),
227 + * plus one "default" page to catch illegal requests.
229 + * The translated virtual address region is between 40GB and 42GB;
230 + * addresses below this range pass straight through to the SDRAM.
232 + * Currently we assume a 1:1:1 correspondence of IOMMU, group and domain.
235 +#define MMU_PAGE_SHIFT 12
236 +#define MMU_PAGE_SIZE BIT(MMU_PAGE_SHIFT)
238 +#define PAGEWORDS_SHIFT (MMU_PAGE_SHIFT - 2)
239 +#define HUGEPAGE_SHIFT (MMU_PAGE_SHIFT + PAGEWORDS_SHIFT)
240 +#define L1_CHUNK_SHIFT (MMU_PAGE_SHIFT + 2 * PAGEWORDS_SHIFT)
242 +#define APERTURE_BASE (40ul << 30)
243 +#define APERTURE_SIZE (2ul << 30)
244 +#define APERTURE_TOP (APERTURE_BASE + APERTURE_SIZE)
245 +#define TRANSLATED_PAGES (APERTURE_SIZE >> MMU_PAGE_SHIFT)
246 +#define L2_PAGES (TRANSLATED_PAGES >> PAGEWORDS_SHIFT)
247 +#define TABLES_ALLOC_SIZE (L2_PAGES * MMU_PAGE_SIZE + 2 * PAGE_SIZE)
249 +static void bcm2712_iommu_init(struct bcm2712_iommu *mmu)
251 + unsigned int i, bypass_shift;
252 + struct sg_dma_page_iter it;
253 + u32 u = MMU_RD(MMMU_DEBUG_INFO_OFFSET);
256 + * Check IOMMU version and hardware configuration.
257 + * This driver is for VC IOMMU version >= 4 (with 2-level tables)
258 + * and assumes at least 36 bits of virtual and physical address space.
259 + * Bigpage and superpage sizes are typically 64K and 1M, but may vary
260 + * (hugepage size is fixed at 4M, the range covered by an L2 page).
262 + dev_info(mmu->dev, "%s: DEBUG_INFO = 0x%08x\n", __func__, u);
263 + WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 ||
264 + FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 ||
265 + FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u) < 6 ||
266 + !(u & MMMU_DEBUG_INFO_BYPASS));
268 + mmu->bigpage_mask =
269 + ((1u << FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u)) - 1u) << MMU_PAGE_SHIFT;
270 + mmu->superpage_mask =
271 + ((1u << FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u)) - 1u) << MMU_PAGE_SHIFT;
272 + bypass_shift = (u & MMMU_DEBUG_INFO_BYPASS_4M) ?
273 + HUGEPAGE_SHIFT : ADDR_CAP_SHIFT;
275 + /* Disable MMU and clear sticky flags; meanwhile flush the TLB */
276 + MMU_WR(MMMU_CTRL_OFFSET,
277 + MMMU_CTRL_CAP_EXCEEDED |
278 + MMMU_CTRL_PT_INVALID |
279 + MMMU_CTRL_WRITE_VIOLATION |
280 + MMMU_CTRL_STATS_CLEAR |
281 + MMMU_CTRL_TLB_CLEAR);
284 + * Put MMU into 2-level mode; set address cap and "bypass" range
285 + * (note that some of these registers have unintuitive off-by-ones).
286 + * Addresses below APERTURE_BASE are passed unchanged: this is
287 + * useful for blocks which share an IOMMU with other blocks
288 + * whose drivers are not IOMMU-aware.
290 + MMU_WR(MMMU_MISC_OFFSET,
291 + MMU_RD(MMMU_MISC_OFFSET) & ~MMMU_MISC_SINGLE_TABLE);
292 + MMU_WR(MMMU_ADDR_CAP_OFFSET,
293 + MMMU_ADDR_CAP_ENABLE +
294 + (APERTURE_TOP >> ADDR_CAP_SHIFT) - 1);
295 + if (APERTURE_BASE > 0) {
296 + MMU_WR(MMMU_BYPASS_START_OFFSET,
297 + MMMU_BYPASS_START_ENABLE + MMMU_BYPASS_START_INVERT +
298 + (APERTURE_BASE >> bypass_shift) - 1);
299 + MMU_WR(MMMU_BYPASS_END_OFFSET,
300 + MMMU_BYPASS_END_ENABLE +
301 + (APERTURE_TOP >> bypass_shift));
303 + MMU_WR(MMMU_BYPASS_START_OFFSET, 0);
304 + MMU_WR(MMMU_BYPASS_END_OFFSET, 0);
307 + /* Ensure tables are zeroed (which marks all pages as invalid) */
308 + dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
309 + memset(mmu->tables, 0, TABLES_ALLOC_SIZE);
310 + mmu->nmapped_pages = 0;
312 + /* Initialize the high-level table to point to the low-level pages */
313 + __sg_page_iter_start(&it.base, mmu->sgt->sgl, mmu->sgt->nents, 0);
314 + for (i = 0; i < L2_PAGES; i++) {
315 + if (!(i % (PAGE_SIZE / MMU_PAGE_SIZE))) {
316 + __sg_page_iter_dma_next(&it);
317 + u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
321 + mmu->tables[TRANSLATED_PAGES + i] = MMMU_PTE_VALID + u;
325 + * Configure the addresses of the top-level table (offset because
326 + * the aperture does not start from zero), and of the default page.
327 + * For simplicity, both these regions are whole Linux pages.
329 + __sg_page_iter_dma_next(&it);
330 + u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
331 + MMU_WR(MMMU_PT_PA_BASE_OFFSET, u - (APERTURE_BASE >> L1_CHUNK_SHIFT));
332 + __sg_page_iter_dma_next(&it);
333 + u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
334 + MMU_WR(MMMU_ILLEGAL_ADR_OFFSET, MMMU_ILLEGAL_ADR_ENABLE + u);
335 + dma_sync_sgtable_for_device(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
336 + mmu->dirty = false;
338 + /* Flush (and enable) the shared TLB cache; enable this MMU. */
340 + bcm2712_iommu_cache_flush(mmu->cache);
341 + MMU_WR(MMMU_CTRL_OFFSET,
342 + MMMU_CTRL_CAP_EXCEEDED_ABORT_EN |
343 + MMMU_CTRL_PT_INVALID_ABORT_EN |
344 + MMMU_CTRL_WRITE_VIOLATION_ABORT_EN |
345 + MMMU_CTRL_STATS_ENABLE |
349 +static int bcm2712_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
351 + struct bcm2712_iommu *mmu = dev ? dev_iommu_priv_get(dev) : 0;
352 + struct bcm2712_iommu_domain *mydomain =
353 + container_of(domain, struct bcm2712_iommu_domain, base);
355 + dev_info(dev, "%s: MMU %s\n",
356 + __func__, mmu ? dev_name(mmu->dev) : "");
359 + mydomain->mmu = mmu;
360 + mmu->domain = mydomain;
362 + if (mmu->dma_iova_offset) {
363 + domain->geometry.aperture_start =
364 + mmu->dma_iova_offset + APERTURE_BASE;
365 + domain->geometry.aperture_end =
366 + mmu->dma_iova_offset + APERTURE_TOP - 1ul;
374 +static void bcm2712_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
380 +static int bcm2712_iommu_map(struct iommu_domain *domain, unsigned long iova,
381 + phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
383 + struct bcm2712_iommu *mmu = domain_to_mmu(domain);
386 + iova -= mmu->dma_iova_offset;
387 + if (iova >= APERTURE_BASE && iova + bytes <= APERTURE_TOP) {
389 + u32 entry = MMMU_PTE_VALID | (pa >> MMU_PAGE_SHIFT);
390 + u32 align = (u32)(iova | pa | bytes);
392 + /* large page and write enable flags */
393 + if (!(align & ((1 << HUGEPAGE_SHIFT) - 1)))
394 + entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 3);
395 + else if (!(align & mmu->superpage_mask) && mmu->superpage_mask)
396 + entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 2);
397 + else if (!(align & mmu->bigpage_mask) && mmu->bigpage_mask)
398 + entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 1);
399 + if (prot & IOMMU_WRITE)
400 + entry |= MMMU_PTE_WRITEABLE;
402 + /* Ensure tables are cache-coherent with CPU */
404 + dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
408 + iova -= APERTURE_BASE;
409 + for (p = iova >> MMU_PAGE_SHIFT;
410 + p < (iova + bytes) >> MMU_PAGE_SHIFT; p++) {
411 + mmu->nmapped_pages += !(mmu->tables[p]);
412 + mmu->tables[p] = entry++;
414 + } else if (iova + bytes > APERTURE_BASE || iova != pa) {
415 + dev_warn(mmu->dev, "%s: iova=0x%lx pa=0x%llx size=0x%llx OUT OF RANGE!\n",
417 + (unsigned long long)pa, (unsigned long long)bytes);
424 +static size_t bcm2712_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
425 + size_t bytes, struct iommu_iotlb_gather *gather)
427 + struct bcm2712_iommu *mmu = domain_to_mmu(domain);
429 + if (iova >= mmu->dma_iova_offset + APERTURE_BASE &&
430 + iova + bytes <= mmu->dma_iova_offset + APERTURE_TOP) {
433 + /* Record just the lower and upper bounds in "gather" */
435 + bool empty = (gather->end <= gather->start);
437 + if (empty || gather->start < iova)
438 + gather->start = iova;
439 + if (empty || gather->end < iova + bytes)
440 + gather->end = iova + bytes;
443 + /* Ensure tables are cache-coherent with CPU */
445 + dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
449 + /* Clear table entries, this marks the addresses as illegal */
450 + iova -= (mmu->dma_iova_offset + APERTURE_BASE);
451 + for (p = iova >> MMU_PAGE_SHIFT;
452 + p < (iova + bytes) >> MMU_PAGE_SHIFT;
454 + mmu->nmapped_pages -= !!(mmu->tables[p]);
455 + mmu->tables[p] = 0;
462 +static void bcm2712_iommu_sync_range(struct iommu_domain *domain,
463 + unsigned long iova, size_t size)
465 + struct bcm2712_iommu *mmu = domain_to_mmu(domain);
466 + unsigned long iova_end;
467 + unsigned int i, p4;
469 + if (!mmu || !mmu->dirty)
472 + /* Ensure tables are cleaned from CPU cache or write-buffer */
473 + dma_sync_sgtable_for_device(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
474 + mmu->dirty = false;
476 + /* Flush the shared TLB cache */
478 + bcm2712_iommu_cache_flush(mmu->cache);
481 + * When flushing a large range or when nothing needs to be kept,
482 + * it's quicker to use the"TLB_CLEAR" flag. Otherwise, invalidate
483 + * TLB entries in lines of 4 words each. Each flush/clear operation
484 + * should complete almost instantaneously.
486 + iova -= mmu->dma_iova_offset;
487 + iova_end = min(APERTURE_TOP, iova + size);
488 + iova = max(APERTURE_BASE, iova);
489 + if (mmu->nmapped_pages == 0 || iova_end - iova >= APERTURE_SIZE / 8) {
490 + MMU_WR(MMMU_CTRL_OFFSET,
491 + MMMU_CTRL_CAP_EXCEEDED_ABORT_EN |
492 + MMMU_CTRL_PT_INVALID_ABORT_EN |
493 + MMMU_CTRL_WRITE_VIOLATION_ABORT_EN |
494 + MMMU_CTRL_TLB_CLEAR |
495 + MMMU_CTRL_STATS_ENABLE |
497 + for (i = 0; i < 1024; i++) {
498 + if (!(MMMU_CTRL_TLB_CLEARING & MMU_RD(MMMU_CTRL_OFFSET)))
503 + for (p4 = iova >> (MMU_PAGE_SHIFT + 2);
504 + p4 < (iova_end + 3 * MMU_PAGE_SIZE) >> (MMU_PAGE_SHIFT + 2);
506 + MMU_WR(MMMU_SHOOT_DOWN_OFFSET,
507 + MMMU_SHOOT_DOWN_SHOOT + (p4 << 2));
508 + for (i = 0; i < 1024; i++) {
509 + if (!(MMMU_SHOOT_DOWN_SHOOTING & MMU_RD(MMMU_SHOOT_DOWN_OFFSET)))
517 +static void bcm2712_iommu_sync(struct iommu_domain *domain,
518 + struct iommu_iotlb_gather *gather)
520 + bcm2712_iommu_sync_range(domain, gather->start,
521 + gather->end - gather->start);
524 +static void bcm2712_iommu_sync_all(struct iommu_domain *domain)
526 + bcm2712_iommu_sync_range(domain, APERTURE_BASE, APERTURE_SIZE);
529 +static phys_addr_t bcm2712_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
531 + struct bcm2712_iommu *mmu = domain_to_mmu(domain);
534 + iova -= mmu->dma_iova_offset;
535 + if (iova >= APERTURE_BASE && iova < APERTURE_TOP) {
536 + p = (iova - APERTURE_BASE) >> MMU_PAGE_SHIFT;
537 + p = mmu->tables[p] & 0x0FFFFFFFu;
538 + return (((phys_addr_t)p) << MMU_PAGE_SHIFT) + (iova & (MMU_PAGE_SIZE - 1u));
539 + } else if (iova < APERTURE_BASE) {
540 + return (phys_addr_t)iova;
542 + return (phys_addr_t)-EINVAL;
546 +static void bcm2712_iommu_domain_free(struct iommu_domain *domain)
548 + struct bcm2712_iommu_domain *mydomain =
549 + container_of(domain, struct bcm2712_iommu_domain, base);
554 +static const struct iommu_domain_ops bcm2712_iommu_domain_ops = {
555 + .attach_dev = bcm2712_iommu_attach_dev,
556 + .detach_dev = bcm2712_iommu_detach_dev,
557 + .map = bcm2712_iommu_map,
558 + .unmap = bcm2712_iommu_unmap,
559 + .iotlb_sync = bcm2712_iommu_sync,
560 + .iotlb_sync_map = bcm2712_iommu_sync_range,
561 + .flush_iotlb_all = bcm2712_iommu_sync_all,
562 + .iova_to_phys = bcm2712_iommu_iova_to_phys,
563 + .free = bcm2712_iommu_domain_free,
566 +static struct iommu_domain *bcm2712_iommu_domain_alloc(unsigned int type)
568 + struct bcm2712_iommu_domain *domain;
570 + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
573 + domain = kzalloc(sizeof(*domain), GFP_KERNEL);
577 + domain->base.type = type;
578 + domain->base.ops = &bcm2712_iommu_domain_ops;
579 + domain->base.geometry.aperture_start = APERTURE_BASE;
580 + domain->base.geometry.aperture_end = APERTURE_TOP - 1ul;
581 + domain->base.geometry.force_aperture = true;
582 + return &domain->base;
585 +static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev)
587 + struct bcm2712_iommu *mmu;
590 + * For reasons I don't fully understand, we need to try both
591 + * cases (dev_iommu_priv_get() and platform_get_drvdata())
592 + * in order to get both GPU and ISP-BE to probe successfully.
594 + mmu = dev_iommu_priv_get(dev);
596 + struct device_node *np;
597 + struct platform_device *pdev;
599 + /* Ignore devices that don't have an "iommus" property with exactly one phandle */
600 + if (!dev->of_node ||
601 + of_property_count_elems_of_size(dev->of_node, "iommus", sizeof(phandle)) != 1)
602 + return ERR_PTR(-ENODEV);
604 + np = of_parse_phandle(dev->of_node, "iommus", 0);
606 + return ERR_PTR(-EINVAL);
608 + pdev = of_find_device_by_node(np);
611 + mmu = platform_get_drvdata(pdev);
614 + return ERR_PTR(-ENODEV);
617 + dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
618 + dev_iommu_priv_set(dev, mmu);
619 + return &mmu->iommu;
622 +static void bcm2712_iommu_release_device(struct device *dev)
624 + dev_iommu_priv_set(dev, NULL);
627 +static struct iommu_group *bcm2712_iommu_device_group(struct device *dev)
629 + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
631 + if (!mmu || !mmu->group)
632 + return ERR_PTR(-EINVAL);
634 + dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
635 + return iommu_group_ref_get(mmu->group);
638 +static int bcm2712_iommu_of_xlate(struct device *dev,
639 + struct of_phandle_args *args)
641 + struct platform_device *iommu_dev;
642 + struct bcm2712_iommu *mmu;
644 + iommu_dev = of_find_device_by_node(args->np);
645 + mmu = platform_get_drvdata(iommu_dev);
646 + dev_iommu_priv_set(dev, mmu);
647 + dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
652 +static bool bcm2712_iommu_capable(struct device *dev, enum iommu_cap cap)
657 +static const struct iommu_ops bcm2712_iommu_ops = {
658 + .capable = bcm2712_iommu_capable,
659 + .domain_alloc = bcm2712_iommu_domain_alloc,
660 + .probe_device = bcm2712_iommu_probe_device,
661 + .release_device = bcm2712_iommu_release_device,
662 + .device_group = bcm2712_iommu_device_group,
663 + /* Advertise native page sizes as well as 2M, 16K which Linux may prefer */
664 + .pgsize_bitmap = (SZ_4M | SZ_2M | SZ_1M | SZ_64K | SZ_16K | SZ_4K),
665 + .default_domain_ops = &bcm2712_iommu_domain_ops,
666 + .of_xlate = bcm2712_iommu_of_xlate,
669 +static int bcm2712_iommu_probe(struct platform_device *pdev)
671 + struct bcm2712_iommu *mmu;
672 + struct bcm2712_iommu_cache *cache = NULL;
675 + /* First of all, check for an IOMMU shared cache */
676 + if (pdev->dev.of_node) {
677 + struct device_node *cache_np;
678 + struct platform_device *cache_pdev;
680 + cache_np = of_parse_phandle(pdev->dev.of_node, "cache", 0);
682 + cache_pdev = of_find_device_by_node(cache_np);
683 + of_node_put(cache_np);
684 + if (cache_pdev && !IS_ERR(cache_pdev))
685 + cache = platform_get_drvdata(cache_pdev);
687 + return -EPROBE_DEFER;
691 + /* Allocate private data */
692 + mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
696 + mmu->name = dev_name(&pdev->dev);
697 + mmu->dev = &pdev->dev;
698 + mmu->cache = cache;
699 + platform_set_drvdata(pdev, mmu);
700 + spin_lock_init(&mmu->hw_lock);
703 + * XXX When an IOMMU is downstream of a PCIe RC or some other chip/bus
704 + * and serves some of the masters thereon (others using pass-through),
705 + * we seem to fumble and lose the "dma-ranges" address offset for
706 + * masters using IOMMU. This property restores it, where needed.
708 + if (!pdev->dev.of_node ||
709 + of_property_read_u64(pdev->dev.of_node, "dma-iova-offset",
710 + &mmu->dma_iova_offset))
711 + mmu->dma_iova_offset = 0;
714 + * The IOMMU is itself a device that allocates DMA-able memory
715 + * to hold its translation tables. Provided the IOVA aperture
716 + * is no larger than 4 GBytes (so that the L1 table fits within
717 + * a single 4K page), we don't need the tables to be contiguous.
718 + * Assume we can address at least 36 bits (64 GB).
720 + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36));
722 + mmu->sgt = dma_alloc_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
723 + DMA_TO_DEVICE, GFP_KERNEL,
724 + DMA_ATTR_ALLOC_SINGLE_PAGES);
729 + mmu->tables = dma_vmap_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
731 + if (!mmu->tables) {
736 + /* Get IOMMU registers */
737 + mmu->reg_base = devm_platform_ioremap_resource(pdev, 0);
738 + if (IS_ERR(mmu->reg_base)) {
739 + dev_err(&pdev->dev, "Failed to get IOMMU registers address\n");
740 + ret = PTR_ERR(mmu->reg_base);
745 + mmu->group = iommu_group_alloc();
746 + if (IS_ERR(mmu->group)) {
747 + ret = PTR_ERR(mmu->group);
751 + ret = iommu_device_sysfs_add(&mmu->iommu, mmu->dev, NULL, mmu->name);
755 + /* Initialize table and hardware */
756 + bcm2712_iommu_init(mmu);
757 + ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, &pdev->dev);
759 + dev_info(&pdev->dev, "%s: Success\n", __func__);
763 + dev_info(&pdev->dev, "%s: Failure %d\n", __func__, ret);
765 + iommu_group_put(mmu->group);
767 + dma_vunmap_noncontiguous(&pdev->dev,
768 + (void *)(mmu->tables));
769 + mmu->tables = NULL;
771 + dma_free_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
772 + mmu->sgt, DMA_TO_DEVICE);
778 +static int bcm2712_iommu_remove(struct platform_device *pdev)
780 + struct bcm2712_iommu *mmu = platform_get_drvdata(pdev);
783 + MMU_WR(MMMU_CTRL_OFFSET, 0); /* disable the MMU */
785 + dma_free_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
786 + mmu->sgt, DMA_TO_DEVICE);
791 +static const struct of_device_id bcm2712_iommu_of_match[] = {
793 + . compatible = "brcm,bcm2712-iommu"
795 + { /* sentinel */ },
798 +static struct platform_driver bcm2712_iommu_driver = {
799 + .probe = bcm2712_iommu_probe,
800 + .remove = bcm2712_iommu_remove,
802 + .name = "bcm2712-iommu",
803 + .of_match_table = bcm2712_iommu_of_match
807 +builtin_platform_driver(bcm2712_iommu_driver);
809 +++ b/drivers/iommu/bcm2712-iommu.h
811 +/* SPDX-License-Identifier: GPL-2.0-only */
813 + * IOMMU driver for BCM2712
815 + * Copyright (c) 2023 Raspberry Pi Ltd.
818 +#ifndef _BCM2712_IOMMU_H
819 +#define _BCM2712_IOMMU_H
821 +#include <linux/iommu.h>
822 +#include <linux/scatterlist.h>
824 +struct bcm2712_iommu_cache {
825 + struct device *dev;
826 + spinlock_t hw_lock; /* to protect HW registers */
827 + void __iomem *reg_base;
830 +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache);
832 +struct bcm2712_iommu {
833 + struct device *dev;
834 + struct iommu_device iommu;
835 + struct iommu_group *group;
836 + struct bcm2712_iommu_domain *domain;
838 + struct sg_table *sgt; /* allocated memory for page tables */
839 + u32 *tables; /* kernel mapping for page tables */
840 + struct bcm2712_iommu_cache *cache;
841 + spinlock_t hw_lock; /* to protect HW registers */
842 + void __iomem *reg_base;
843 + u64 dma_iova_offset; /* Hack for IOMMU attached to PCIe RC */
845 + u32 superpage_mask;
846 + unsigned int nmapped_pages;
847 + bool dirty; /* true when tables are oriented towards CPU */
850 +struct bcm2712_iommu_domain {
851 + struct iommu_domain base;
852 + struct bcm2712_iommu *mmu;