1 From 8d8f9e0697227ec7b71cd86d114d83ee2bb6431b Mon Sep 17 00:00:00 2001
2 From: Phil Elwell <phil@raspberrypi.org>
3 Date: Thu, 4 Apr 2019 13:33:47 +0100
4 Subject: [PATCH] bcm2835-dma: Add proper 40-bit DMA support
6 BCM2711 has 4 DMA channels with a 40-bit address range, allowing them
7 to access the full 4GB of memory on a Pi 4.
9 Signed-off-by: Phil Elwell <phil@raspberrypi.org>
11 drivers/dma/bcm2835-dma.c | 485 ++++++++++++++++++++++++++++++++------
12 1 file changed, 412 insertions(+), 73 deletions(-)
14 --- a/drivers/dma/bcm2835-dma.c
15 +++ b/drivers/dma/bcm2835-dma.c
17 #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
18 #define BCM2835_DMA_CHAN_NAME_SIZE 8
19 #define BCM2835_DMA_BULK_MASK BIT(0)
20 +#define BCM2711_DMA_MEMCPY_CHAN 14
22 +struct bcm2835_dma_cfg_data {
23 + u32 chan_40bit_mask;
27 * struct bcm2835_dmadev - BCM2835 DMA controller
28 @@ -50,6 +55,7 @@ struct bcm2835_dmadev {
29 struct dma_device ddev;
32 + const struct bcm2835_dma_cfg_data *cfg_data;
35 struct bcm2835_dma_cb {
36 @@ -62,6 +68,17 @@ struct bcm2835_dma_cb {
40 +struct bcm2711_dma40_scb {
51 struct bcm2835_cb_entry {
52 struct bcm2835_dma_cb *cb;
54 @@ -82,6 +99,7 @@ struct bcm2835_chan {
55 unsigned int irq_flags;
58 + bool is_40bit_channel;
62 @@ -171,13 +189,118 @@ struct bcm2835_desc {
63 #define BCM2835_DMA_DATA_TYPE_S128 16
65 /* Valid only for channels 0 - 14, 15 has its own base address */
66 -#define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */
67 +#define BCM2835_DMA_CHAN_SIZE 0x100
68 +#define BCM2835_DMA_CHAN(n) ((n) * BCM2835_DMA_CHAN_SIZE) /* Base address */
69 #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))
71 /* the max dma length for different channels */
72 #define MAX_DMA_LEN SZ_1G
73 #define MAX_LITE_DMA_LEN (SZ_64K - 4)
75 +/* 40-bit DMA support */
76 +#define BCM2711_DMA40_CS 0x00
77 +#define BCM2711_DMA40_CB 0x04
78 +#define BCM2711_DMA40_DEBUG 0x0c
79 +#define BCM2711_DMA40_TI 0x10
80 +#define BCM2711_DMA40_SRC 0x14
81 +#define BCM2711_DMA40_SRCI 0x18
82 +#define BCM2711_DMA40_DEST 0x1c
83 +#define BCM2711_DMA40_DESTI 0x20
84 +#define BCM2711_DMA40_LEN 0x24
85 +#define BCM2711_DMA40_NEXT_CB 0x28
86 +#define BCM2711_DMA40_DEBUG2 0x2c
88 +#define BCM2711_DMA40_ACTIVE BIT(0)
89 +#define BCM2711_DMA40_END BIT(1)
90 +#define BCM2711_DMA40_INT BIT(2)
91 +#define BCM2711_DMA40_DREQ BIT(3) /* DREQ state */
92 +#define BCM2711_DMA40_RD_PAUSED BIT(4) /* Reading is paused */
93 +#define BCM2711_DMA40_WR_PAUSED BIT(5) /* Writing is paused */
94 +#define BCM2711_DMA40_DREQ_PAUSED BIT(6) /* Is paused by DREQ flow control */
95 +#define BCM2711_DMA40_WAITING_FOR_WRITES BIT(7) /* Waiting for last write */
96 +#define BCM2711_DMA40_ERR BIT(10)
97 +#define BCM2711_DMA40_QOS(x) (((x) & 0x1f) << 16)
98 +#define BCM2711_DMA40_PANIC_QOS(x) (((x) & 0x1f) << 20)
99 +#define BCM2711_DMA40_WAIT_FOR_WRITES BIT(28)
100 +#define BCM2711_DMA40_DISDEBUG BIT(29)
101 +#define BCM2711_DMA40_ABORT BIT(30)
102 +#define BCM2711_DMA40_HALT BIT(31)
103 +#define BCM2711_DMA40_CS_FLAGS(x) (x & (BCM2711_DMA40_QOS(15) | \
104 + BCM2711_DMA40_PANIC_QOS(15) | \
105 + BCM2711_DMA40_WAIT_FOR_WRITES | \
106 + BCM2711_DMA40_DISDEBUG))
108 +/* Transfer information bits */
109 +#define BCM2711_DMA40_INTEN BIT(0)
110 +#define BCM2711_DMA40_TDMODE BIT(1) /* 2D-Mode */
111 +#define BCM2711_DMA40_WAIT_RESP BIT(2) /* wait for AXI write to be acked */
112 +#define BCM2711_DMA40_WAIT_RD_RESP BIT(3) /* wait for AXI read to complete */
113 +#define BCM2711_DMA40_PER_MAP(x) ((x & 31) << 9) /* REQ source */
114 +#define BCM2711_DMA40_S_DREQ BIT(14) /* enable SREQ for source */
115 +#define BCM2711_DMA40_D_DREQ BIT(15) /* enable DREQ for destination */
116 +#define BCM2711_DMA40_S_WAIT(x) ((x & 0xff) << 16) /* add DMA read-wait cycles */
117 +#define BCM2711_DMA40_D_WAIT(x) ((x & 0xff) << 24) /* add DMA write-wait cycles */
119 +/* debug register bits */
120 +#define BCM2711_DMA40_DEBUG_WRITE_ERR BIT(0)
121 +#define BCM2711_DMA40_DEBUG_FIFO_ERR BIT(1)
122 +#define BCM2711_DMA40_DEBUG_READ_ERR BIT(2)
123 +#define BCM2711_DMA40_DEBUG_READ_CB_ERR BIT(3)
124 +#define BCM2711_DMA40_DEBUG_IN_ON_ERR BIT(8)
125 +#define BCM2711_DMA40_DEBUG_ABORT_ON_ERR BIT(9)
126 +#define BCM2711_DMA40_DEBUG_HALT_ON_ERR BIT(10)
127 +#define BCM2711_DMA40_DEBUG_DISABLE_CLK_GATE BIT(11)
128 +#define BCM2711_DMA40_DEBUG_RSTATE_SHIFT 14
129 +#define BCM2711_DMA40_DEBUG_RSTATE_BITS 4
130 +#define BCM2711_DMA40_DEBUG_WSTATE_SHIFT 18
131 +#define BCM2711_DMA40_DEBUG_WSTATE_BITS 4
132 +#define BCM2711_DMA40_DEBUG_RESET BIT(23)
133 +#define BCM2711_DMA40_DEBUG_ID_SHIFT 24
134 +#define BCM2711_DMA40_DEBUG_ID_BITS 4
135 +#define BCM2711_DMA40_DEBUG_VERSION_SHIFT 28
136 +#define BCM2711_DMA40_DEBUG_VERSION_BITS 4
138 +/* Valid only for channels 0 - 3 (11 - 14) */
139 +#define BCM2711_DMA40_CHAN(n) (((n) + 11) << 8) /* Base address */
140 +#define BCM2711_DMA40_CHANIO(base, n) ((base) + BCM2711_DMA_CHAN(n))
142 +/* the max dma length for different channels */
143 +#define MAX_DMA40_LEN SZ_1G
145 +#define BCM2711_DMA40_BURST_LEN(x) ((min(x,16) - 1) << 8)
146 +#define BCM2711_DMA40_INC BIT(12)
147 +#define BCM2711_DMA40_SIZE_32 (0 << 13)
148 +#define BCM2711_DMA40_SIZE_64 (1 << 13)
149 +#define BCM2711_DMA40_SIZE_128 (2 << 13)
150 +#define BCM2711_DMA40_SIZE_256 (3 << 13)
151 +#define BCM2711_DMA40_IGNORE BIT(15)
152 +#define BCM2711_DMA40_STRIDE(x) ((x) << 16) /* For 2D mode */
154 +#define BCM2711_DMA40_MEMCPY_FLAGS \
155 + (BCM2711_DMA40_QOS(0) | \
156 + BCM2711_DMA40_PANIC_QOS(0) | \
157 + BCM2711_DMA40_WAIT_FOR_WRITES | \
158 + BCM2711_DMA40_DISDEBUG)
160 +#define BCM2711_DMA40_MEMCPY_XFER_INFO \
161 + (BCM2711_DMA40_SIZE_128 | \
162 + BCM2711_DMA40_INC | \
163 + BCM2711_DMA40_BURST_LEN(16))
165 +struct bcm2835_dmadev *memcpy_parent;
166 +static void __iomem *memcpy_chan;
167 +static struct bcm2711_dma40_scb *memcpy_scb;
168 +static dma_addr_t memcpy_scb_dma;
169 +DEFINE_SPINLOCK(memcpy_lock);
171 +static const struct bcm2835_dma_cfg_data bcm2835_dma_cfg = {
172 + .chan_40bit_mask = 0,
175 +static const struct bcm2835_dma_cfg_data bcm2711_dma_cfg = {
176 + .chan_40bit_mask = BIT(11) | BIT(12) | BIT(13) | BIT(14),
179 static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)
181 /* lite and normal channels have different max frame length */
182 @@ -207,6 +330,32 @@ static inline struct bcm2835_desc *to_bc
183 return container_of(t, struct bcm2835_desc, vd.tx);
186 +static inline uint32_t to_bcm2711_ti(uint32_t info)
188 + return ((info & BCM2835_DMA_INT_EN) ? BCM2711_DMA40_INTEN : 0) |
189 + ((info & BCM2835_DMA_WAIT_RESP) ? BCM2711_DMA40_WAIT_RESP : 0) |
190 + ((info & BCM2835_DMA_S_DREQ) ?
191 + (BCM2711_DMA40_S_DREQ | BCM2711_DMA40_WAIT_RD_RESP) : 0) |
192 + ((info & BCM2835_DMA_D_DREQ) ? BCM2711_DMA40_D_DREQ : 0) |
193 + BCM2711_DMA40_PER_MAP((info >> 16) & 0x1f);
196 +static inline uint32_t to_bcm2711_srci(uint32_t info)
198 + return ((info & BCM2835_DMA_S_INC) ? BCM2711_DMA40_INC : 0);
201 +static inline uint32_t to_bcm2711_dsti(uint32_t info)
203 + return ((info & BCM2835_DMA_D_INC) ? BCM2711_DMA40_INC : 0);
206 +static inline uint32_t to_bcm2711_cbaddr(dma_addr_t addr)
208 + BUG_ON(addr & 0x1f);
209 + return (addr >> 5);
212 static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)
215 @@ -225,45 +374,53 @@ static void bcm2835_dma_desc_free(struct
218 static void bcm2835_dma_create_cb_set_length(
219 - struct bcm2835_chan *chan,
220 + struct bcm2835_chan *c,
221 struct bcm2835_dma_cb *control_block,
227 - size_t max_len = bcm2835_dma_max_frame_length(chan);
228 + size_t max_len = bcm2835_dma_max_frame_length(c);
231 /* set the length taking lite-channel limitations into account */
232 - control_block->length = min_t(u32, len, max_len);
233 + cb_len = min_t(u32, len, max_len);
235 - /* finished if we have no period_length */
240 + * period_len means: that we need to generate
241 + * transfers that are terminating at every
242 + * multiple of period_len - this is typically
243 + * used to set the interrupt flag in info
244 + * which is required during cyclic transfers
248 - * period_len means: that we need to generate
249 - * transfers that are terminating at every
250 - * multiple of period_len - this is typically
251 - * used to set the interrupt flag in info
252 - * which is required during cyclic transfers
254 + /* have we filled in period_length yet? */
255 + if (*total_len + cb_len < period_len) {
256 + /* update number of bytes in this period so far */
257 + *total_len += cb_len;
259 + /* calculate the length that remains to reach period_len */
260 + cb_len = period_len - *total_len;
262 - /* have we filled in period_length yet? */
263 - if (*total_len + control_block->length < period_len) {
264 - /* update number of bytes in this period so far */
265 - *total_len += control_block->length;
267 + /* reset total_length for next period */
272 - /* calculate the length that remains to reach period_length */
273 - control_block->length = period_len - *total_len;
275 - /* reset total_length for next period */
278 - /* add extrainfo bits in info */
279 - control_block->info |= finalextrainfo;
280 + if (c->is_40bit_channel) {
281 + struct bcm2711_dma40_scb *scb =
282 + (struct bcm2711_dma40_scb *)control_block;
285 + /* add extrainfo bits to ti */
286 + scb->ti |= to_bcm2711_ti(finalextrainfo);
288 + control_block->length = cb_len;
289 + /* add extrainfo bits to info */
290 + control_block->info |= finalextrainfo;
294 static inline size_t bcm2835_dma_count_frames_for_sg(
295 @@ -286,7 +443,7 @@ static inline size_t bcm2835_dma_count_f
297 * bcm2835_dma_create_cb_chain - create a control block and fills data in
299 - * @chan: the @dma_chan for which we run this
300 + * @c: the @bcm2835_chan for which we run this
301 * @direction: the direction in which we transfer
302 * @cyclic: it is a cyclic transfer
303 * @info: the default info bits to apply per controlblock
304 @@ -304,12 +461,11 @@ static inline size_t bcm2835_dma_count_f
305 * @gfp: the GFP flag to use for allocation
307 static struct bcm2835_desc *bcm2835_dma_create_cb_chain(
308 - struct dma_chan *chan, enum dma_transfer_direction direction,
309 + struct bcm2835_chan *c, enum dma_transfer_direction direction,
310 bool cyclic, u32 info, u32 finalextrainfo, size_t frames,
311 dma_addr_t src, dma_addr_t dst, size_t buf_len,
312 size_t period_len, gfp_t gfp)
314 - struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
315 size_t len = buf_len, total_len;
317 struct bcm2835_desc *d;
318 @@ -341,11 +497,23 @@ static struct bcm2835_desc *bcm2835_dma_
320 /* fill in the control block */
321 control_block = cb_entry->cb;
322 - control_block->info = info;
323 - control_block->src = src;
324 - control_block->dst = dst;
325 - control_block->stride = 0;
326 - control_block->next = 0;
327 + if (c->is_40bit_channel) {
328 + struct bcm2711_dma40_scb *scb =
329 + (struct bcm2711_dma40_scb *)control_block;
330 + scb->ti = to_bcm2711_ti(info);
331 + scb->src = lower_32_bits(src);
332 + scb->srci= upper_32_bits(src) | to_bcm2711_srci(info);
333 + scb->dst = lower_32_bits(dst);
334 + scb->dsti = upper_32_bits(dst) | to_bcm2711_dsti(info);
337 + control_block->info = info;
338 + control_block->src = src;
339 + control_block->dst = dst;
340 + control_block->stride = 0;
341 + control_block->next = 0;
344 /* set up length in control_block if requested */
346 /* calculate length honoring period_length */
347 @@ -359,7 +527,11 @@ static struct bcm2835_desc *bcm2835_dma_
350 /* link this the last controlblock */
352 + if (frame && c->is_40bit_channel)
353 + ((struct bcm2711_dma40_scb *)
354 + d->cb_list[frame - 1].cb)->next_cb =
355 + to_bcm2711_cbaddr(cb_entry->paddr);
356 + if (frame && !c->is_40bit_channel)
357 d->cb_list[frame - 1].cb->next = cb_entry->paddr;
359 /* update src and dst and length */
360 @@ -369,11 +541,21 @@ static struct bcm2835_desc *bcm2835_dma_
361 dst += control_block->length;
363 /* Length of total transfer */
364 - d->size += control_block->length;
365 + if (c->is_40bit_channel)
366 + d->size += ((struct bcm2711_dma40_scb *)control_block)->len;
368 + d->size += control_block->length;
371 /* the last frame requires extra flags */
372 - d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
373 + if (c->is_40bit_channel) {
374 + struct bcm2711_dma40_scb *scb =
375 + (struct bcm2711_dma40_scb *)d->cb_list[d->frames-1].cb;
377 + scb->ti |= to_bcm2711_ti(finalextrainfo);
379 + d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
382 /* detect a size missmatch */
383 if (buf_len && (d->size != buf_len))
384 @@ -387,13 +569,12 @@ error_cb:
387 static void bcm2835_dma_fill_cb_chain_with_sg(
388 - struct dma_chan *chan,
389 + struct bcm2835_chan *c,
390 enum dma_transfer_direction direction,
391 struct bcm2835_cb_entry *cb,
392 struct scatterlist *sgl,
395 - struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
399 @@ -401,14 +582,35 @@ static void bcm2835_dma_fill_cb_chain_wi
401 max_len = bcm2835_dma_max_frame_length(c);
402 for_each_sg(sgl, sgent, sg_len, i) {
403 - for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);
405 - addr += cb->cb->length, len -= cb->cb->length, cb++) {
406 - if (direction == DMA_DEV_TO_MEM)
407 - cb->cb->dst = addr;
409 - cb->cb->src = addr;
410 - cb->cb->length = min(len, max_len);
411 + if (c->is_40bit_channel) {
412 + struct bcm2711_dma40_scb *scb;
414 + for (addr = sg_dma_address(sgent),
415 + len = sg_dma_len(sgent);
417 + addr += scb->len, len -= scb->len, cb++) {
418 + scb = (struct bcm2711_dma40_scb *)cb->cb;
419 + if (direction == DMA_DEV_TO_MEM) {
420 + scb->dst = lower_32_bits(addr);
421 + scb->dsti = upper_32_bits(addr) | BCM2711_DMA40_INC;
423 + scb->src = lower_32_bits(addr);
424 + scb->srci = upper_32_bits(addr) | BCM2711_DMA40_INC;
426 + scb->len = min(len, max_len);
429 + for (addr = sg_dma_address(sgent),
430 + len = sg_dma_len(sgent);
432 + addr += cb->cb->length, len -= cb->cb->length,
434 + if (direction == DMA_DEV_TO_MEM)
435 + cb->cb->dst = addr;
437 + cb->cb->src = addr;
438 + cb->cb->length = min(len, max_len);
443 @@ -417,6 +619,10 @@ static void bcm2835_dma_abort(struct bcm
445 void __iomem *chan_base = c->chan_base;
446 long int timeout = 10000;
447 + u32 wait_mask = BCM2835_DMA_WAITING_FOR_WRITES;
449 + if (c->is_40bit_channel)
450 + wait_mask = BCM2711_DMA40_WAITING_FOR_WRITES;
453 * A zero control block address means the channel is idle.
454 @@ -429,8 +635,7 @@ static void bcm2835_dma_abort(struct bcm
455 writel(0, chan_base + BCM2835_DMA_CS);
457 /* Wait for any current AXI transfer to complete */
458 - while ((readl(chan_base + BCM2835_DMA_CS) &
459 - BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)
460 + while ((readl(chan_base + BCM2835_DMA_CS) & wait_mask) && --timeout)
463 /* Peripheral might be stuck and fail to signal AXI write responses */
464 @@ -455,9 +660,16 @@ static void bcm2835_dma_start_desc(struc
466 c->desc = d = to_bcm2835_dma_desc(&vd->tx);
468 - writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
469 - writel(BCM2835_DMA_ACTIVE | BCM2835_DMA_CS_FLAGS(c->dreq),
470 - c->chan_base + BCM2835_DMA_CS);
471 + if (c->is_40bit_channel) {
472 + writel(to_bcm2711_cbaddr(d->cb_list[0].paddr),
473 + c->chan_base + BCM2711_DMA40_CB);
474 + writel(BCM2711_DMA40_ACTIVE | BCM2711_DMA40_CS_FLAGS(c->dreq),
475 + c->chan_base + BCM2711_DMA40_CS);
477 + writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
478 + writel(BCM2835_DMA_ACTIVE | BCM2835_DMA_CS_FLAGS(c->dreq),
479 + c->chan_base + BCM2835_DMA_CS);
483 static irqreturn_t bcm2835_dma_callback(int irq, void *data)
484 @@ -484,8 +696,7 @@ static irqreturn_t bcm2835_dma_callback(
485 * if this IRQ handler is threaded.) If the channel is finished, it
486 * will remain idle despite the ACTIVE flag being set.
488 - writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE |
489 - BCM2835_DMA_CS_FLAGS(c->dreq),
490 + writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,
491 c->chan_base + BCM2835_DMA_CS);
494 @@ -588,9 +799,17 @@ static enum dma_status bcm2835_dma_tx_st
495 struct bcm2835_desc *d = c->desc;
498 - if (d->dir == DMA_MEM_TO_DEV)
499 + if (d->dir == DMA_MEM_TO_DEV && c->is_40bit_channel)
500 + pos = readl(c->chan_base + BCM2711_DMA40_SRC) +
501 + ((readl(c->chan_base + BCM2711_DMA40_SRCI) &
503 + else if (d->dir == DMA_MEM_TO_DEV && !c->is_40bit_channel)
504 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);
505 - else if (d->dir == DMA_DEV_TO_MEM)
506 + else if (d->dir == DMA_DEV_TO_MEM && c->is_40bit_channel)
507 + pos = readl(c->chan_base + BCM2711_DMA40_DEST) +
508 + ((readl(c->chan_base + BCM2711_DMA40_DESTI) &
510 + else if (d->dir == DMA_DEV_TO_MEM && !c->is_40bit_channel)
511 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);
514 @@ -636,7 +855,7 @@ static struct dma_async_tx_descriptor *b
515 frames = bcm2835_dma_frames_for_length(len, max_len);
517 /* allocate the CB chain - this also fills in the pointers */
518 - d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,
519 + d = bcm2835_dma_create_cb_chain(c, DMA_MEM_TO_MEM, false,
521 src, dst, len, 0, GFP_KERNEL);
523 @@ -671,11 +890,21 @@ static struct dma_async_tx_descriptor *b
524 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
526 src = c->cfg.src_addr;
528 + * One would think it ought to be possible to get the physical
529 + * to dma address mapping information from the dma-ranges DT
530 + * property, but I've not found a way yet that doesn't involve
531 + * open-coding the whole thing.
533 + if (c->is_40bit_channel)
534 + src |= 0x400000000ull;
535 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
537 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
539 dst = c->cfg.dst_addr;
540 + if (c->is_40bit_channel)
541 + dst |= 0x400000000ull;
542 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
545 @@ -683,7 +912,7 @@ static struct dma_async_tx_descriptor *b
546 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);
548 /* allocate the CB chain */
549 - d = bcm2835_dma_create_cb_chain(chan, direction, false,
550 + d = bcm2835_dma_create_cb_chain(c, direction, false,
552 frames, src, dst, 0, 0,
554 @@ -691,7 +920,7 @@ static struct dma_async_tx_descriptor *b
557 /* fill in frames with scatterlist pointers */
558 - bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,
559 + bcm2835_dma_fill_cb_chain_with_sg(c, direction, d->cb_list,
562 return vchan_tx_prep(&c->vc, &d->vd, flags);
563 @@ -745,12 +974,16 @@ static struct dma_async_tx_descriptor *b
564 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
566 src = c->cfg.src_addr;
567 + if (c->is_40bit_channel)
568 + src |= 0x400000000ull;
570 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
572 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
574 dst = c->cfg.dst_addr;
575 + if (c->is_40bit_channel)
576 + dst |= 0x400000000ull;
578 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
580 @@ -770,7 +1003,7 @@ static struct dma_async_tx_descriptor *b
581 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine
582 * implementation calls prep_dma_cyclic with interrupts disabled.
584 - d = bcm2835_dma_create_cb_chain(chan, direction, true,
585 + d = bcm2835_dma_create_cb_chain(c, direction, true,
587 frames, src, dst, buf_len,
588 period_len, GFP_NOWAIT);
589 @@ -778,7 +1011,12 @@ static struct dma_async_tx_descriptor *b
592 /* wrap around into a loop */
593 - d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
594 + if (c->is_40bit_channel)
595 + ((struct bcm2711_dma40_scb *)
596 + d->cb_list[frames - 1].cb)->next_cb =
597 + to_bcm2711_cbaddr(d->cb_list[0].paddr);
599 + d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
601 return vchan_tx_prep(&c->vc, &d->vd, flags);
603 @@ -839,9 +1077,11 @@ static int bcm2835_dma_chan_init(struct
605 c->irq_flags = irq_flags;
607 - /* check in DEBUG register if this is a LITE channel */
608 - if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
609 - BCM2835_DMA_DEBUG_LITE)
610 + /* check for 40bit and lite channels */
611 + if (d->cfg_data->chan_40bit_mask & BIT(chan_id))
612 + c->is_40bit_channel = true;
613 + else if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
614 + BCM2835_DMA_DEBUG_LITE)
615 c->is_lite_channel = true;
618 @@ -861,8 +1101,58 @@ static void bcm2835_dma_free(struct bcm2
619 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
622 +int bcm2711_dma40_memcpy_init(void)
624 + if (!memcpy_parent)
625 + return -EPROBE_DEFER;
635 +EXPORT_SYMBOL(bcm2711_dma40_memcpy_init);
637 +void bcm2711_dma40_memcpy(dma_addr_t dst, dma_addr_t src, size_t size)
639 + struct bcm2711_dma40_scb *scb = memcpy_scb;
640 + unsigned long flags;
643 + pr_err("bcm2711_dma40_memcpy not initialised!\n");
647 + spin_lock_irqsave(&memcpy_lock, flags);
650 + scb->src = lower_32_bits(src);
651 + scb->srci = upper_32_bits(src) | BCM2711_DMA40_MEMCPY_XFER_INFO;
652 + scb->dst = lower_32_bits(dst);
653 + scb->dsti = upper_32_bits(dst) | BCM2711_DMA40_MEMCPY_XFER_INFO;
657 + writel((u32)(memcpy_scb_dma >> 5), memcpy_chan + BCM2711_DMA40_CB);
658 + writel(BCM2711_DMA40_MEMCPY_FLAGS + BCM2711_DMA40_ACTIVE,
659 + memcpy_chan + BCM2711_DMA40_CS);
661 + /* Poll for completion */
662 + while (!(readl(memcpy_chan + BCM2711_DMA40_CS) & BCM2711_DMA40_END))
665 + writel(BCM2711_DMA40_END, memcpy_chan + BCM2711_DMA40_CS);
667 + spin_unlock_irqrestore(&memcpy_lock, flags);
669 +EXPORT_SYMBOL(bcm2711_dma40_memcpy);
671 static const struct of_device_id bcm2835_dma_of_match[] = {
672 - { .compatible = "brcm,bcm2835-dma", },
673 + { .compatible = "brcm,bcm2835-dma", .data = &bcm2835_dma_cfg },
674 + { .compatible = "brcm,bcm2711-dma", .data = &bcm2711_dma_cfg },
677 MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);
678 @@ -894,6 +1184,8 @@ static int bcm2835_dma_probe(struct plat
680 uint32_t chans_available;
681 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];
682 + const struct of_device_id *of_id;
683 + int chan_count, chan_start, chan_end;
685 if (!pdev->dev.dma_mask)
686 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
687 @@ -914,9 +1206,13 @@ static int bcm2835_dma_probe(struct plat
688 base = devm_ioremap_resource(&pdev->dev, res);
690 return PTR_ERR(base);
691 - rc = bcm_dmaman_probe(pdev, base, BCM2835_DMA_BULK_MASK);
693 - dev_err(&pdev->dev, "Failed to initialize the legacy API\n");
695 + /* The set of channels can be split across multiple instances. */
696 + chan_start = ((u32)(uintptr_t)base / BCM2835_DMA_CHAN_SIZE) & 0xf;
697 + base -= BCM2835_DMA_CHAN(chan_start);
698 + chan_count = resource_size(res) / BCM2835_DMA_CHAN_SIZE;
699 + chan_end = min(chan_start + chan_count,
700 + BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1);
704 @@ -953,6 +1249,14 @@ static int bcm2835_dma_probe(struct plat
708 + of_id = of_match_node(bcm2835_dma_of_match, pdev->dev.of_node);
710 + dev_err(&pdev->dev, "Failed to match compatible string\n");
714 + od->cfg_data = of_id->data;
716 /* Request DMA channel mask from device tree */
717 if (of_property_read_u32(pdev->dev.of_node,
718 "brcm,dma-channel-mask",
719 @@ -962,11 +1266,34 @@ static int bcm2835_dma_probe(struct plat
723 - /* Channel 0 is used by the legacy API */
724 - chans_available &= ~BCM2835_DMA_BULK_MASK;
725 + /* One channel is reserved for the legacy API */
726 + if (chans_available & BCM2835_DMA_BULK_MASK) {
727 + rc = bcm_dmaman_probe(pdev, base,
728 + chans_available & BCM2835_DMA_BULK_MASK);
730 + dev_err(&pdev->dev,
731 + "Failed to initialize the legacy API\n");
733 + chans_available &= ~BCM2835_DMA_BULK_MASK;
736 + /* And possibly one for the 40-bit DMA memcpy API */
737 + if (chans_available & od->cfg_data->chan_40bit_mask &
738 + BIT(BCM2711_DMA_MEMCPY_CHAN)) {
739 + memcpy_parent = od;
740 + memcpy_chan = BCM2835_DMA_CHANIO(base, BCM2711_DMA_MEMCPY_CHAN);
741 + memcpy_scb = dma_alloc_coherent(memcpy_parent->ddev.dev,
742 + sizeof(*memcpy_scb),
743 + &memcpy_scb_dma, GFP_KERNEL);
745 + dev_warn(&pdev->dev,
746 + "Failed to allocated memcpy scb\n");
748 + chans_available &= ~BIT(BCM2711_DMA_MEMCPY_CHAN);
751 /* get irqs for each channel that we support */
752 - for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
753 + for (i = chan_start; i < chan_end; i++) {
754 /* skip masked out channels */
755 if (!(chans_available & (1 << i))) {
757 @@ -989,13 +1316,17 @@ static int bcm2835_dma_probe(struct plat
758 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);
763 /* get irqs for each channel */
764 - for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
765 + for (i = chan_start; i < chan_end; i++) {
766 /* skip channels without irq */
770 /* check if there are other channels that also use this irq */
771 + /* FIXME: This will fail if interrupts are shared across
774 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
775 if ((i != j) && (irq[j] == irq[i])) {
776 @@ -1007,9 +1338,10 @@ static int bcm2835_dma_probe(struct plat
777 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);
783 - dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);
784 + dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", chan_count);
786 /* Device-tree DMA controller registration */
787 rc = of_dma_controller_register(pdev->dev.of_node,
788 @@ -1041,6 +1373,13 @@ static int bcm2835_dma_remove(struct pla
790 bcm_dmaman_remove(pdev);
791 dma_async_device_unregister(&od->ddev);
792 + if (memcpy_parent == od) {
793 + dma_free_coherent(&pdev->dev, sizeof(*memcpy_scb), memcpy_scb,
795 + memcpy_parent = NULL;
797 + memcpy_chan = NULL;
799 bcm2835_dma_free(od);