1 From d3d537ebe9884e7d945ab74bb02312d0c2c9b08d Mon Sep 17 00:00:00 2001
2 From: Yangbo Lu <yangbo.lu@nxp.com>
3 Date: Thu, 5 Jul 2018 17:32:53 +0800
4 Subject: [PATCH 17/32] dma: support layerscape
6 This is an integrated patch for layerscape dma support.
8 Signed-off-by: jiaheng.fan <jiaheng.fan@nxp.com>
9 Signed-off-by: Yangbo Lu <yangbo.lu@nxp.com>
11 drivers/dma/Kconfig | 31 +
12 drivers/dma/Makefile | 3 +
13 drivers/dma/caam_dma.c | 563 ++++++++++
14 drivers/dma/dpaa2-qdma/Kconfig | 8 +
15 drivers/dma/dpaa2-qdma/Makefile | 8 +
16 drivers/dma/dpaa2-qdma/dpaa2-qdma.c | 940 +++++++++++++++++
17 drivers/dma/dpaa2-qdma/dpaa2-qdma.h | 227 +++++
18 drivers/dma/dpaa2-qdma/dpdmai.c | 515 ++++++++++
19 drivers/dma/dpaa2-qdma/fsl_dpdmai.h | 521 ++++++++++
20 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h | 222 ++++
21 drivers/dma/fsl-qdma.c | 1243 +++++++++++++++++++++++
22 11 files changed, 4281 insertions(+)
23 create mode 100644 drivers/dma/caam_dma.c
24 create mode 100644 drivers/dma/dpaa2-qdma/Kconfig
25 create mode 100644 drivers/dma/dpaa2-qdma/Makefile
26 create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.c
27 create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.h
28 create mode 100644 drivers/dma/dpaa2-qdma/dpdmai.c
29 create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai.h
30 create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
31 create mode 100644 drivers/dma/fsl-qdma.c
33 --- a/drivers/dma/Kconfig
34 +++ b/drivers/dma/Kconfig
35 @@ -192,6 +192,20 @@ config FSL_EDMA
36 multiplexing capability for DMA request sources(slot).
37 This module can be found on Freescale Vybrid and LS-1 SoCs.
40 + tristate "Freescale qDMA engine support"
42 + select DMA_VIRTUAL_CHANNELS
43 + select DMA_ENGINE_RAID
44 + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
46 + Support the Freescale qDMA engine with command queue and legacy mode.
47 + Channel virtualization is supported through enqueuing of DMA jobs to,
48 + or dequeuing DMA jobs from, different work queues.
49 + This module can be found on Freescale LS SoCs.
51 +source drivers/dma/dpaa2-qdma/Kconfig
54 tristate "Freescale RAID engine Support"
55 depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH
56 @@ -564,6 +578,23 @@ config ZX_DMA
58 Support the DMA engine for ZTE ZX296702 platform devices.
60 +config CRYPTO_DEV_FSL_CAAM_DMA
61 + tristate "CAAM DMA engine support"
62 + depends on CRYPTO_DEV_FSL_CAAM_JR
66 + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
68 + Selecting this will offload the DMA operations for users of
69 + the scatter gather memcopy API to the CAAM via job rings. The
70 + CAAM is a hardware module that provides hardware acceleration to
71 + cryptographic operations. It has a built-in DMA controller that can
72 + be programmed to read/write cryptographic data. This module defines
73 + a DMA driver that uses the DMA capabilities of the CAAM.
75 + To compile this as a module, choose M here: the module
76 + will be called caam_dma.
79 source "drivers/dma/bestcomm/Kconfig"
80 --- a/drivers/dma/Makefile
81 +++ b/drivers/dma/Makefile
82 @@ -29,6 +29,8 @@ obj-$(CONFIG_DW_DMAC_CORE) += dw/
83 obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
84 obj-$(CONFIG_FSL_DMA) += fsldma.o
85 obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
86 +obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o
87 +obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma/
88 obj-$(CONFIG_FSL_RAID) += fsl_raid.o
89 obj-$(CONFIG_HSU_DMA) += hsu/
90 obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
91 @@ -67,6 +69,7 @@ obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-
92 obj-$(CONFIG_TI_EDMA) += edma.o
93 obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
94 obj-$(CONFIG_ZX_DMA) += zx296702_dma.o
95 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o
100 +++ b/drivers/dma/caam_dma.c
103 + * caam support for SG DMA
105 + * Copyright 2016 Freescale Semiconductor, Inc
106 + * Copyright 2017 NXP
109 +#include <linux/module.h>
110 +#include <linux/platform_device.h>
111 +#include <linux/dma-mapping.h>
112 +#include <linux/interrupt.h>
113 +#include <linux/slab.h>
114 +#include <linux/debugfs.h>
116 +#include <linux/dmaengine.h>
117 +#include "dmaengine.h"
119 +#include "../crypto/caam/regs.h"
120 +#include "../crypto/caam/jr.h"
121 +#include "../crypto/caam/error.h"
122 +#include "../crypto/caam/intern.h"
123 +#include "../crypto/caam/desc_constr.h"
124 +#include "../crypto/caam/sg_sw_sec4.h"
126 +#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \
129 +/* This is max chunk size of a DMA transfer. If a buffer is larger than this
130 + * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes
131 + * and for each chunk a DMA transfer request is issued.
132 + * This value is the largest number on 16 bits that is a multiple of 256 bytes
133 + * (the largest configurable CAAM DMA burst size).
135 +#define CAAM_DMA_CHUNK_SIZE 65280
137 +struct caam_dma_sh_desc {
138 + u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned;
139 + dma_addr_t desc_dma;
142 +/* caam dma extended descriptor */
143 +struct caam_dma_edesc {
144 + struct dma_async_tx_descriptor async_tx;
145 + struct list_head node;
146 + struct caam_dma_ctx *ctx;
147 + dma_addr_t src_dma;
148 + dma_addr_t dst_dma;
149 + unsigned int src_len;
150 + unsigned int dst_len;
151 + struct sec4_sg_entry *sec4_sg;
152 + u32 jd[] ____cacheline_aligned;
156 + * caam_dma_ctx - per jr/channel context
157 + * @chan: dma channel used by async_tx API
158 + * @node: list_head used to attach to the global dma_ctx_list
159 + * @jrdev: Job Ring device
160 + * @submit_q: queue of pending (submitted, but not enqueued) jobs
161 + * @done_not_acked: jobs that have been completed by jr, but maybe not acked
162 + * @edesc_lock: protects extended descriptor
164 +struct caam_dma_ctx {
165 + struct dma_chan chan;
166 + struct list_head node;
167 + struct device *jrdev;
168 + struct list_head submit_q;
169 + struct list_head done_not_acked;
170 + spinlock_t edesc_lock;
173 +static struct dma_device *dma_dev;
174 +static struct caam_dma_sh_desc *dma_sh_desc;
175 +static LIST_HEAD(dma_ctx_list);
177 +static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx)
179 + struct caam_dma_edesc *edesc = NULL;
180 + struct caam_dma_ctx *ctx = NULL;
181 + dma_cookie_t cookie;
183 + edesc = container_of(tx, struct caam_dma_edesc, async_tx);
184 + ctx = container_of(tx->chan, struct caam_dma_ctx, chan);
186 + spin_lock_bh(&ctx->edesc_lock);
188 + cookie = dma_cookie_assign(tx);
189 + list_add_tail(&edesc->node, &ctx->submit_q);
191 + spin_unlock_bh(&ctx->edesc_lock);
196 +static unsigned int caam_dma_sg_dma_len(struct scatterlist *sg,
197 + unsigned int nents)
201 + for (len = 0; sg && nents; sg = sg_next(sg), nents--)
202 + len += sg_dma_len(sg);
207 +static struct caam_dma_edesc *
208 +caam_dma_sg_edesc_alloc(struct dma_chan *chan,
209 + struct scatterlist *dst_sg, unsigned int dst_nents,
210 + struct scatterlist *src_sg, unsigned int src_nents,
211 + unsigned long flags)
213 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
215 + struct device *jrdev = ctx->jrdev;
216 + struct caam_dma_edesc *edesc;
217 + struct sec4_sg_entry *sec4_sg;
218 + dma_addr_t sec4_sg_dma_src;
219 + unsigned int sec4_sg_bytes;
221 + if (!dst_sg || !src_sg || !dst_nents || !src_nents)
224 + sec4_sg_bytes = (src_nents + dst_nents) * sizeof(*sec4_sg);
226 + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,
227 + GFP_DMA | GFP_NOWAIT);
229 + return ERR_PTR(-ENOMEM);
231 + edesc->src_len = caam_dma_sg_dma_len(src_sg, src_nents);
232 + edesc->dst_len = caam_dma_sg_dma_len(dst_sg, dst_nents);
233 + if (edesc->src_len != edesc->dst_len) {
234 + dev_err(jrdev, "%s: src(%u) and dst(%u) len mismatch.\n",
235 + __func__, edesc->src_len, edesc->dst_len);
237 + return ERR_PTR(-EINVAL);
240 + dma_async_tx_descriptor_init(&edesc->async_tx, chan);
241 + edesc->async_tx.tx_submit = caam_dma_tx_submit;
242 + edesc->async_tx.flags = flags;
243 + edesc->async_tx.cookie = -EBUSY;
245 + /* Prepare SEC SGs */
246 + edesc->sec4_sg = (void *)edesc + offsetof(struct caam_dma_edesc, jd) +
249 + sec4_sg = edesc->sec4_sg;
250 + sg_to_sec4_sg_last(src_sg, src_nents, sec4_sg, 0);
252 + sec4_sg += src_nents;
253 + sg_to_sec4_sg_last(dst_sg, dst_nents, sec4_sg, 0);
255 + sec4_sg_dma_src = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes,
257 + if (dma_mapping_error(jrdev, sec4_sg_dma_src)) {
258 + dev_err(jrdev, "error mapping segments to device\n");
260 + return ERR_PTR(-ENOMEM);
263 + edesc->src_dma = sec4_sg_dma_src;
264 + edesc->dst_dma = sec4_sg_dma_src + src_nents * sizeof(*sec4_sg);
270 +static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc)
272 + struct caam_dma_ctx *ctx = edesc->ctx;
273 + struct caam_dma_edesc *_edesc = NULL;
275 + spin_lock_bh(&ctx->edesc_lock);
277 + list_add_tail(&edesc->node, &ctx->done_not_acked);
278 + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
279 + if (async_tx_test_ack(&edesc->async_tx)) {
280 + list_del(&edesc->node);
285 + spin_unlock_bh(&ctx->edesc_lock);
288 +static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err,
291 + struct caam_dma_edesc *edesc = context;
292 + struct caam_dma_ctx *ctx = edesc->ctx;
293 + dma_async_tx_callback callback;
294 + void *callback_param;
297 + caam_jr_strstatus(ctx->jrdev, err);
299 + dma_run_dependencies(&edesc->async_tx);
301 + spin_lock_bh(&ctx->edesc_lock);
302 + dma_cookie_complete(&edesc->async_tx);
303 + spin_unlock_bh(&ctx->edesc_lock);
305 + callback = edesc->async_tx.callback;
306 + callback_param = edesc->async_tx.callback_param;
308 + dma_descriptor_unmap(&edesc->async_tx);
310 + caam_jr_chan_free_edesc(edesc);
313 + callback(callback_param);
316 +static void caam_dma_sg_init_job_desc(struct caam_dma_edesc *edesc)
318 + u32 *jd = edesc->jd;
319 + u32 *sh_desc = dma_sh_desc->desc;
320 + dma_addr_t desc_dma = dma_sh_desc->desc_dma;
322 + /* init the job descriptor */
323 + init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE);
325 + /* set SEQIN PTR */
326 + append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, LDST_SGF);
328 + /* set SEQOUT PTR */
329 + append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, LDST_SGF);
332 + print_hex_dump(KERN_ERR, "caam dma desc@" __stringify(__LINE__) ": ",
333 + DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1);
337 +/* This function can be called from an interrupt context */
338 +static struct dma_async_tx_descriptor *
339 +caam_dma_prep_sg(struct dma_chan *chan, struct scatterlist *dst_sg,
340 + unsigned int dst_nents, struct scatterlist *src_sg,
341 + unsigned int src_nents, unsigned long flags)
343 + struct caam_dma_edesc *edesc;
345 + /* allocate extended descriptor */
346 + edesc = caam_dma_sg_edesc_alloc(chan, dst_sg, dst_nents, src_sg,
348 + if (IS_ERR_OR_NULL(edesc))
349 + return ERR_CAST(edesc);
351 + /* Initialize job descriptor */
352 + caam_dma_sg_init_job_desc(edesc);
354 + return &edesc->async_tx;
357 +static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc)
359 + u32 *jd = edesc->jd;
360 + u32 *sh_desc = dma_sh_desc->desc;
361 + dma_addr_t desc_dma = dma_sh_desc->desc_dma;
363 + /* init the job descriptor */
364 + init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE);
366 + /* set SEQIN PTR */
367 + append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0);
369 + /* set SEQOUT PTR */
370 + append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0);
373 + print_hex_dump(KERN_ERR, "caam dma desc@" __stringify(__LINE__) ": ",
374 + DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1);
378 +static struct dma_async_tx_descriptor *
379 +caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
380 + size_t len, unsigned long flags)
382 + struct caam_dma_edesc *edesc;
383 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
386 + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT);
388 + return ERR_PTR(-ENOMEM);
390 + dma_async_tx_descriptor_init(&edesc->async_tx, chan);
391 + edesc->async_tx.tx_submit = caam_dma_tx_submit;
392 + edesc->async_tx.flags = flags;
393 + edesc->async_tx.cookie = -EBUSY;
395 + edesc->src_dma = src;
396 + edesc->src_len = len;
397 + edesc->dst_dma = dst;
398 + edesc->dst_len = len;
401 + caam_dma_memcpy_init_job_desc(edesc);
403 + return &edesc->async_tx;
406 +/* This function can be called in an interrupt context */
407 +static void caam_dma_issue_pending(struct dma_chan *chan)
409 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
411 + struct caam_dma_edesc *edesc, *_edesc;
413 + spin_lock_bh(&ctx->edesc_lock);
414 + list_for_each_entry_safe(edesc, _edesc, &ctx->submit_q, node) {
415 + if (caam_jr_enqueue(ctx->jrdev, edesc->jd,
416 + caam_dma_done, edesc) < 0)
418 + list_del(&edesc->node);
420 + spin_unlock_bh(&ctx->edesc_lock);
423 +static void caam_dma_free_chan_resources(struct dma_chan *chan)
425 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
427 + struct caam_dma_edesc *edesc, *_edesc;
429 + spin_lock_bh(&ctx->edesc_lock);
430 + list_for_each_entry_safe(edesc, _edesc, &ctx->submit_q, node) {
431 + list_del(&edesc->node);
434 + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
435 + list_del(&edesc->node);
438 + spin_unlock_bh(&ctx->edesc_lock);
441 +static int caam_dma_jr_chan_bind(void)
443 + struct device *jrdev;
444 + struct caam_dma_ctx *ctx;
448 + for (i = 0; i < caam_jr_driver_probed(); i++) {
449 + jrdev = caam_jridx_alloc(i);
450 + if (IS_ERR(jrdev)) {
451 + pr_err("job ring device %d allocation failed\n", i);
455 + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
457 + caam_jr_free(jrdev);
461 + ctx->chan.device = dma_dev;
462 + ctx->chan.private = ctx;
464 + ctx->jrdev = jrdev;
466 + INIT_LIST_HEAD(&ctx->submit_q);
467 + INIT_LIST_HEAD(&ctx->done_not_acked);
468 + INIT_LIST_HEAD(&ctx->node);
469 + spin_lock_init(&ctx->edesc_lock);
471 + dma_cookie_init(&ctx->chan);
473 + /* add the context of this channel to the context list */
474 + list_add_tail(&ctx->node, &dma_ctx_list);
476 + /* add this channel to the device chan list */
477 + list_add_tail(&ctx->chan.device_node, &dma_dev->channels);
485 +static inline void caam_jr_dma_free(struct dma_chan *chan)
487 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
490 + list_del(&ctx->node);
491 + list_del(&chan->device_node);
492 + caam_jr_free(ctx->jrdev);
496 +static void set_caam_dma_desc(u32 *desc)
500 + /* dma shared descriptor */
501 + init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT));
503 + /* REG1 = CAAM_DMA_CHUNK_SIZE */
504 + append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE);
506 + /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */
507 + append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE);
512 + jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
515 + /* REG1 = SEQINLEN */
516 + append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ);
519 + set_jump_tgt_here(desc, jmp_cmd);
521 + /* VARSEQINLEN = REG1 */
522 + append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ);
524 + /* VARSEQOUTLEN = REG1 */
525 + append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ);
527 + /* do FIFO STORE */
528 + append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF);
531 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
532 + FIFOLD_TYPE_IFIFO | LDST_VLF);
535 + * jmp 0xF8 (after shared desc header)
537 + append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
538 + JUMP_COND_MATH_Z | 0xF8);
541 + print_hex_dump(KERN_ERR, "caam dma shdesc@" __stringify(__LINE__) ": ",
542 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
546 +static int __init caam_dma_probe(struct platform_device *pdev)
548 + struct device *dev = &pdev->dev;
549 + struct device *ctrldev = dev->parent;
550 + struct dma_chan *chan, *_chan;
555 + if (!caam_jr_driver_probed()) {
556 + dev_info(dev, "Defer probing after JR driver probing\n");
557 + return -EPROBE_DEFER;
560 + dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL);
564 + dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA);
568 + sh_desc = dma_sh_desc->desc;
569 + set_caam_dma_desc(sh_desc);
570 + dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc,
571 + desc_bytes(sh_desc),
573 + if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) {
574 + dev_err(dev, "unable to map dma descriptor\n");
578 + INIT_LIST_HEAD(&dma_dev->channels);
580 + bonds = caam_dma_jr_chan_bind();
586 + dma_dev->dev = dev;
587 + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
588 + dma_cap_set(DMA_SG, dma_dev->cap_mask);
589 + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
590 + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
591 + dma_dev->device_tx_status = dma_cookie_status;
592 + dma_dev->device_issue_pending = caam_dma_issue_pending;
593 + dma_dev->device_prep_dma_sg = caam_dma_prep_sg;
594 + dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy;
595 + dma_dev->device_free_chan_resources = caam_dma_free_chan_resources;
597 + err = dma_async_device_register(dma_dev);
599 + dev_err(dev, "Failed to register CAAM DMA engine\n");
603 + dev_info(dev, "caam dma support with %d job rings\n", bonds);
608 + list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node)
609 + caam_jr_dma_free(chan);
611 + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc),
614 + kfree(dma_sh_desc);
620 +static int caam_dma_remove(struct platform_device *pdev)
622 + struct device *dev = &pdev->dev;
623 + struct device *ctrldev = dev->parent;
624 + struct caam_dma_ctx *ctx, *_ctx;
626 + dma_async_device_unregister(dma_dev);
628 + list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) {
629 + list_del(&ctx->node);
630 + caam_jr_free(ctx->jrdev);
634 + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma,
635 + desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE);
637 + kfree(dma_sh_desc);
640 + dev_info(dev, "caam dma support disabled\n");
644 +static const struct of_device_id caam_dma_match[] = {
645 + { .compatible = "fsl,sec-v5.4-dma", },
646 + { .compatible = "fsl,sec-v5.0-dma", },
647 + { .compatible = "fsl,sec-v4.0-dma", },
650 +MODULE_DEVICE_TABLE(of, caam_dma_match);
652 +static struct platform_driver caam_dma_driver = {
654 + .name = "caam-dma",
655 + .of_match_table = caam_dma_match,
657 + .probe = caam_dma_probe,
658 + .remove = caam_dma_remove,
660 +module_platform_driver(caam_dma_driver);
662 +MODULE_LICENSE("Dual BSD/GPL");
663 +MODULE_DESCRIPTION("NXP CAAM support for SG DMA");
664 +MODULE_AUTHOR("NXP Semiconductors");
666 +++ b/drivers/dma/dpaa2-qdma/Kconfig
668 +menuconfig FSL_DPAA2_QDMA
669 + tristate "NXP DPAA2 QDMA"
670 + depends on FSL_MC_BUS && FSL_MC_DPIO
672 + select DMA_VIRTUAL_CHANNELS
674 + NXP Data Path Acceleration Architecture 2 QDMA driver,
675 + using the NXP MC bus driver.
677 +++ b/drivers/dma/dpaa2-qdma/Makefile
680 +# Makefile for the NXP DPAA2 CAAM controllers
682 +ccflags-y += -DVERSION=\"\"
684 +obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma.o
686 +fsl-dpaa2-qdma-objs := dpaa2-qdma.o dpdmai.o
688 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.c
691 + * drivers/dma/dpaa2-qdma/dpaa2-qdma.c
693 + * Copyright 2015-2017 NXP Semiconductor, Inc.
694 + * Author: Changming Huang <jerry.huang@nxp.com>
696 + * Driver for the NXP QDMA engine with QMan mode.
697 + * Channel virtualization is supported through enqueuing of DMA jobs to,
698 + * or dequeuing DMA jobs from different work queues with QMan portal.
699 + * This module can be found on NXP LS2 SoCs.
701 + * This program is free software; you can redistribute it and/or modify it
702 + * under the terms of the GNU General Public License as published by the
703 + * Free Software Foundation; either version 2 of the License, or (at your
704 + * option) any later version.
707 +#include <linux/init.h>
708 +#include <linux/module.h>
709 +#include <linux/interrupt.h>
710 +#include <linux/clk.h>
711 +#include <linux/dma-mapping.h>
712 +#include <linux/dmapool.h>
713 +#include <linux/slab.h>
714 +#include <linux/spinlock.h>
715 +#include <linux/of.h>
716 +#include <linux/of_device.h>
717 +#include <linux/of_address.h>
718 +#include <linux/of_irq.h>
719 +#include <linux/of_dma.h>
720 +#include <linux/types.h>
721 +#include <linux/delay.h>
722 +#include <linux/iommu.h>
724 +#include "../virt-dma.h"
726 +#include <linux/fsl/mc.h>
727 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h"
728 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
729 +#include "fsl_dpdmai_cmd.h"
730 +#include "fsl_dpdmai.h"
731 +#include "dpaa2-qdma.h"
733 +static bool smmu_disable = true;
735 +static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
737 + return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
740 +static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
742 + return container_of(vd, struct dpaa2_qdma_comp, vdesc);
745 +static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
750 +static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
752 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
753 + unsigned long flags;
756 + spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
757 + vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
758 + spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
760 + vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
764 + * Request a command descriptor for enqueue.
766 +static struct dpaa2_qdma_comp *
767 +dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
769 + struct dpaa2_qdma_comp *comp_temp = NULL;
770 + unsigned long flags;
772 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
773 + if (list_empty(&dpaa2_chan->comp_free)) {
774 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
775 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
778 + comp_temp->fd_virt_addr = dma_pool_alloc(dpaa2_chan->fd_pool,
779 + GFP_NOWAIT, &comp_temp->fd_bus_addr);
780 + if (!comp_temp->fd_virt_addr)
783 + comp_temp->fl_virt_addr =
784 + (void *)((struct dpaa2_fd *)
785 + comp_temp->fd_virt_addr + 1);
786 + comp_temp->fl_bus_addr = comp_temp->fd_bus_addr +
787 + sizeof(struct dpaa2_fd);
788 + comp_temp->desc_virt_addr =
789 + (void *)((struct dpaa2_fl_entry *)
790 + comp_temp->fl_virt_addr + 3);
791 + comp_temp->desc_bus_addr = comp_temp->fl_bus_addr +
792 + sizeof(struct dpaa2_fl_entry) * 3;
794 + comp_temp->qchan = dpaa2_chan;
795 + comp_temp->sg_blk_num = 0;
796 + INIT_LIST_HEAD(&comp_temp->sg_src_head);
797 + INIT_LIST_HEAD(&comp_temp->sg_dst_head);
800 + comp_temp = list_first_entry(&dpaa2_chan->comp_free,
801 + struct dpaa2_qdma_comp, list);
802 + list_del(&comp_temp->list);
803 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
805 + comp_temp->qchan = dpaa2_chan;
810 +static void dpaa2_qdma_populate_fd(uint32_t format,
811 + struct dpaa2_qdma_comp *dpaa2_comp)
813 + struct dpaa2_fd *fd;
815 + fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
816 + memset(fd, 0, sizeof(struct dpaa2_fd));
819 + dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);
820 + /* Bypass memory translation, Frame list format, short length disable */
821 + /* we need to disable BMT if fsl-mc use iova addr */
823 + dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);
824 + dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);
826 + dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);
829 +/* first frame list for descriptor buffer */
830 +static void dpaa2_qdma_populate_first_framel(
831 + struct dpaa2_fl_entry *f_list,
832 + struct dpaa2_qdma_comp *dpaa2_comp)
834 + struct dpaa2_qdma_sd_d *sdd;
836 + sdd = (struct dpaa2_qdma_sd_d *)dpaa2_comp->desc_virt_addr;
837 + memset(sdd, 0, 2 * (sizeof(*sdd)));
838 + /* source and destination descriptor */
839 + sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); /* source descriptor CMD */
841 + sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); /* dest descriptor CMD */
843 + memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
844 + /* first frame list to source descriptor */
846 + dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);
847 + dpaa2_fl_set_len(f_list, 0x20);
848 + dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);
851 + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
854 +/* source and destination frame list */
855 +static void dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,
856 + dma_addr_t dst, dma_addr_t src, size_t len, uint8_t fmt)
858 + /* source frame list to source buffer */
859 + memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
862 + dpaa2_fl_set_addr(f_list, src);
863 + dpaa2_fl_set_len(f_list, len);
864 + dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); /* single buffer frame or scatter gather frame */
866 + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
869 + /* destination frame list to destination buffer */
870 + memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
872 + dpaa2_fl_set_addr(f_list, dst);
873 + dpaa2_fl_set_len(f_list, len);
874 + dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
875 + dpaa2_fl_set_final(f_list, QDMA_FL_F); /* single buffer frame or scatter gather frame */
877 + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
880 +static struct dma_async_tx_descriptor *dpaa2_qdma_prep_memcpy(
881 + struct dma_chan *chan, dma_addr_t dst,
882 + dma_addr_t src, size_t len, unsigned long flags)
884 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
885 + struct dpaa2_qdma_comp *dpaa2_comp;
886 + struct dpaa2_fl_entry *f_list;
889 + dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
892 + format = QDMA_FD_LONG_FORMAT;
894 + format = QDMA_FD_SHORT_FORMAT;
896 + /* populate Frame descriptor */
897 + dpaa2_qdma_populate_fd(format, dpaa2_comp);
899 + f_list = (struct dpaa2_fl_entry *)dpaa2_comp->fl_virt_addr;
902 + /* first frame list for descriptor buffer (logn format) */
903 + dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp);
908 + dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
910 + return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
913 +static struct qdma_sg_blk *dpaa2_qdma_get_sg_blk(
914 + struct dpaa2_qdma_comp *dpaa2_comp,
915 + struct dpaa2_qdma_chan *dpaa2_chan)
917 + struct qdma_sg_blk *sg_blk = NULL;
918 + dma_addr_t phy_sgb;
919 + unsigned long flags;
921 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
922 + if (list_empty(&dpaa2_chan->sgb_free)) {
923 + sg_blk = (struct qdma_sg_blk *)dma_pool_alloc(
924 + dpaa2_chan->sg_blk_pool,
925 + GFP_NOWAIT, &phy_sgb);
927 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
930 + sg_blk->blk_virt_addr = (void *)(sg_blk + 1);
931 + sg_blk->blk_bus_addr = phy_sgb + sizeof(*sg_blk);
933 + sg_blk = list_first_entry(&dpaa2_chan->sgb_free,
934 + struct qdma_sg_blk, list);
935 + list_del(&sg_blk->list);
937 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
942 +static uint32_t dpaa2_qdma_populate_sg(struct device *dev,
943 + struct dpaa2_qdma_chan *dpaa2_chan,
944 + struct dpaa2_qdma_comp *dpaa2_comp,
945 + struct scatterlist *dst_sg, u32 dst_nents,
946 + struct scatterlist *src_sg, u32 src_nents)
948 + struct dpaa2_qdma_sg *src_sge;
949 + struct dpaa2_qdma_sg *dst_sge;
950 + struct qdma_sg_blk *sg_blk;
951 + struct qdma_sg_blk *sg_blk_dst;
957 + uint32_t total_len = 0;
960 + num = min(dst_nents, src_nents);
961 + blocks = num / (NUM_SG_PER_BLK - 1);
962 + if (num % (NUM_SG_PER_BLK - 1))
964 + if (dpaa2_comp->sg_blk_num < blocks) {
965 + len = blocks - dpaa2_comp->sg_blk_num;
966 + for (i = 0; i < len; i++) {
967 + /* source sg blocks */
968 + sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan);
971 + list_add_tail(&sg_blk->list, &dpaa2_comp->sg_src_head);
972 + /* destination sg blocks */
973 + sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan);
976 + list_add_tail(&sg_blk->list, &dpaa2_comp->sg_dst_head);
979 + len = dpaa2_comp->sg_blk_num - blocks;
980 + for (i = 0; i < len; i++) {
981 + spin_lock(&dpaa2_chan->queue_lock);
982 + /* handle source sg blocks */
983 + sg_blk = list_first_entry(&dpaa2_comp->sg_src_head,
984 + struct qdma_sg_blk, list);
985 + list_del(&sg_blk->list);
986 + list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free);
987 + /* handle destination sg blocks */
988 + sg_blk = list_first_entry(&dpaa2_comp->sg_dst_head,
989 + struct qdma_sg_blk, list);
990 + list_del(&sg_blk->list);
991 + list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free);
992 + spin_unlock(&dpaa2_chan->queue_lock);
995 + dpaa2_comp->sg_blk_num = blocks;
997 + /* get the first source sg phy address */
998 + sg_blk = list_first_entry(&dpaa2_comp->sg_src_head,
999 + struct qdma_sg_blk, list);
1000 + dpaa2_comp->sge_src_bus_addr = sg_blk->blk_bus_addr;
1001 + /* get the first destinaiton sg phy address */
1002 + sg_blk_dst = list_first_entry(&dpaa2_comp->sg_dst_head,
1003 + struct qdma_sg_blk, list);
1004 + dpaa2_comp->sge_dst_bus_addr = sg_blk_dst->blk_bus_addr;
1006 + for (i = 0; i < blocks; i++) {
1007 + src_sge = (struct dpaa2_qdma_sg *)sg_blk->blk_virt_addr;
1008 + dst_sge = (struct dpaa2_qdma_sg *)sg_blk_dst->blk_virt_addr;
1010 + for (j = 0; j < (NUM_SG_PER_BLK - 1); j++) {
1011 + len = min(sg_dma_len(dst_sg), sg_dma_len(src_sg));
1015 + src = sg_dma_address(src_sg);
1016 + dst = sg_dma_address(dst_sg);
1019 + src_sge->addr_lo = src;
1020 + src_sge->addr_hi = (src >> 32);
1021 + src_sge->data_len.data_len_sl0 = len;
1022 + src_sge->ctrl.sl = QDMA_SG_SL_LONG;
1023 + src_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
1024 + /* destination SG */
1025 + dst_sge->addr_lo = dst;
1026 + dst_sge->addr_hi = (dst >> 32);
1027 + dst_sge->data_len.data_len_sl0 = len;
1028 + dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
1029 + dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
1033 + src_sge->ctrl.f = QDMA_SG_F;
1034 + dst_sge->ctrl.f = QDMA_SG_F;
1037 + dst_sg = sg_next(dst_sg);
1038 + src_sg = sg_next(src_sg);
1041 + if (j == (NUM_SG_PER_BLK - 2)) {
1042 + /* for next blocks, extension */
1043 + sg_blk = list_next_entry(sg_blk, list);
1044 + sg_blk_dst = list_next_entry(sg_blk_dst, list);
1045 + src_sge->addr_lo = sg_blk->blk_bus_addr;
1046 + src_sge->addr_hi = sg_blk->blk_bus_addr >> 32;
1047 + src_sge->ctrl.sl = QDMA_SG_SL_LONG;
1048 + src_sge->ctrl.fmt = QDMA_SG_FMT_SGTE;
1049 + dst_sge->addr_lo = sg_blk_dst->blk_bus_addr;
1050 + dst_sge->addr_hi =
1051 + sg_blk_dst->blk_bus_addr >> 32;
1052 + dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
1053 + dst_sge->ctrl.fmt = QDMA_SG_FMT_SGTE;
1062 +static enum dma_status dpaa2_qdma_tx_status(struct dma_chan *chan,
1063 + dma_cookie_t cookie, struct dma_tx_state *txstate)
1065 + return dma_cookie_status(chan, cookie, txstate);
1068 +static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
1072 +static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
1074 + struct dpaa2_qdma_comp *dpaa2_comp;
1075 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
1076 + struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
1077 + struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
1078 + struct virt_dma_desc *vdesc;
1079 + struct dpaa2_fd *fd;
1081 + unsigned long flags;
1083 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
1084 + spin_lock(&dpaa2_chan->vchan.lock);
1085 + if (vchan_issue_pending(&dpaa2_chan->vchan)) {
1086 + vdesc = vchan_next_desc(&dpaa2_chan->vchan);
1089 + dpaa2_comp = to_fsl_qdma_comp(vdesc);
1091 + fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
1093 + list_del(&vdesc->node);
1094 + list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
1096 + /* TOBO: priority hard-coded to zero */
1097 + err = dpaa2_io_service_enqueue_fq(NULL,
1098 + priv->tx_queue_attr[0].fqid, fd);
1100 + list_del(&dpaa2_comp->list);
1101 + list_add_tail(&dpaa2_comp->list,
1102 + &dpaa2_chan->comp_free);
1107 + spin_unlock(&dpaa2_chan->vchan.lock);
1108 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
1111 +static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
1113 + struct device *dev = &ls_dev->dev;
1114 + struct dpaa2_qdma_priv *priv;
1115 + struct dpaa2_qdma_priv_per_prio *ppriv;
1116 + uint8_t prio_def = DPDMAI_PRIO_NUM;
1120 + priv = dev_get_drvdata(dev);
1123 + priv->dpqdma_id = ls_dev->obj_desc.id;
1125 + /*Get the handle for the DPDMAI this interface is associate with */
1126 + err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
1128 + dev_err(dev, "dpdmai_open() failed\n");
1131 + dev_info(dev, "Opened dpdmai object successfully\n");
1133 + err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
1134 + &priv->dpdmai_attr);
1136 + dev_err(dev, "dpdmai_get_attributes() failed\n");
1140 + if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
1141 + dev_err(dev, "DPDMAI major version mismatch\n"
1142 + "Found %u.%u, supported version is %u.%u\n",
1143 + priv->dpdmai_attr.version.major,
1144 + priv->dpdmai_attr.version.minor,
1145 + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
1148 + if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
1149 + dev_err(dev, "DPDMAI minor version mismatch\n"
1150 + "Found %u.%u, supported version is %u.%u\n",
1151 + priv->dpdmai_attr.version.major,
1152 + priv->dpdmai_attr.version.minor,
1153 + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
1156 + priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
1157 + ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
1159 + dev_err(dev, "kzalloc for ppriv failed\n");
1162 + priv->ppriv = ppriv;
1164 + for (i = 0; i < priv->num_pairs; i++) {
1165 + err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
1166 + i, &priv->rx_queue_attr[i]);
1168 + dev_err(dev, "dpdmai_get_rx_queue() failed\n");
1171 + ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
1173 + err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
1174 + i, &priv->tx_queue_attr[i]);
1176 + dev_err(dev, "dpdmai_get_tx_queue() failed\n");
1179 + ppriv->req_fqid = priv->tx_queue_attr[i].fqid;
1181 + ppriv->priv = priv;
1188 +static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
1190 + struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
1191 + struct dpaa2_qdma_priv_per_prio, nctx);
1192 + struct dpaa2_qdma_priv *priv = ppriv->priv;
1193 + struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
1194 + struct dpaa2_qdma_chan *qchan;
1195 + const struct dpaa2_fd *fd;
1196 + const struct dpaa2_fd *fd_eq;
1197 + struct dpaa2_dq *dq;
1203 + uint32_t n_chans = priv->dpaa2_qdma->n_chans;
1206 + err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
1210 + while (!is_last) {
1212 + dq = dpaa2_io_store_next(ppriv->store, &is_last);
1213 + } while (!is_last && !dq);
1215 + dev_err(priv->dev, "FQID returned no valid frames!\n");
1219 + /* obtain FD and process the error */
1220 + fd = dpaa2_dq_fd(dq);
1222 + status = dpaa2_fd_get_ctrl(fd) & 0xff;
1224 + dev_err(priv->dev, "FD error occurred\n");
1226 + for (i = 0; i < n_chans; i++) {
1227 + qchan = &priv->dpaa2_qdma->chans[i];
1228 + spin_lock(&qchan->queue_lock);
1229 + if (list_empty(&qchan->comp_used)) {
1230 + spin_unlock(&qchan->queue_lock);
1233 + list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
1234 + &qchan->comp_used, list) {
1235 + fd_eq = (struct dpaa2_fd *)
1236 + dpaa2_comp->fd_virt_addr;
1238 + if (le64_to_cpu(fd_eq->simple.addr) ==
1239 + le64_to_cpu(fd->simple.addr)) {
1241 + list_del(&dpaa2_comp->list);
1242 + list_add_tail(&dpaa2_comp->list,
1243 + &qchan->comp_free);
1245 + spin_lock(&qchan->vchan.lock);
1246 + vchan_cookie_complete(
1247 + &dpaa2_comp->vdesc);
1248 + spin_unlock(&qchan->vchan.lock);
1253 + spin_unlock(&qchan->queue_lock);
1259 + dpaa2_io_service_rearm(NULL, ctx);
1262 +static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
1265 + struct device *dev = priv->dev;
1266 + struct dpaa2_qdma_priv_per_prio *ppriv;
1268 + num = priv->num_pairs;
1269 + ppriv = priv->ppriv;
1270 + for (i = 0; i < num; i++) {
1271 + ppriv->nctx.is_cdan = 0;
1272 + ppriv->nctx.desired_cpu = 1;
1273 + ppriv->nctx.id = ppriv->rsp_fqid;
1274 + ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
1275 + err = dpaa2_io_service_register(NULL, &ppriv->nctx);
1277 + dev_err(dev, "Notification register failed\n");
1281 + ppriv->store = dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE,
1283 + if (!ppriv->store) {
1284 + dev_err(dev, "dpaa2_io_store_create() failed\n");
1293 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
1296 + while (ppriv >= priv->ppriv) {
1297 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
1298 + dpaa2_io_store_destroy(ppriv->store);
1304 +static void __cold dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
1306 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1309 + for (i = 0; i < priv->num_pairs; i++) {
1310 + dpaa2_io_store_destroy(ppriv->store);
1315 +static void __cold dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
1317 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1320 + for (i = 0; i < priv->num_pairs; i++) {
1321 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
1326 +static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
1329 + struct dpdmai_rx_queue_cfg rx_queue_cfg;
1330 + struct device *dev = priv->dev;
1331 + struct dpaa2_qdma_priv_per_prio *ppriv;
1332 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
1335 + num = priv->num_pairs;
1336 + ppriv = priv->ppriv;
1337 + for (i = 0; i < num; i++) {
1338 + rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
1339 + DPDMAI_QUEUE_OPT_DEST;
1340 + rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
1341 + rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
1342 + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
1343 + rx_queue_cfg.dest_cfg.priority = ppriv->prio;
1344 + err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
1345 + rx_queue_cfg.dest_cfg.priority, &rx_queue_cfg);
1347 + dev_err(dev, "dpdmai_set_rx_queue() failed\n");
1357 +static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
1360 + struct device *dev = priv->dev;
1361 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
1362 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1365 + for (i = 0; i < priv->num_pairs; i++) {
1366 + ppriv->nctx.qman64 = 0;
1367 + ppriv->nctx.dpio_id = 0;
1371 + err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
1373 + dev_err(dev, "dpdmai_reset() failed\n");
1378 +static void __cold dpaa2_dpdmai_free_pool(struct dpaa2_qdma_chan *qchan,
1379 + struct list_head *head)
1381 + struct qdma_sg_blk *sgb_tmp, *_sgb_tmp;
1382 + /* free the QDMA SG pool block */
1383 + list_for_each_entry_safe(sgb_tmp, _sgb_tmp, head, list) {
1384 + sgb_tmp->blk_virt_addr = (void *)((struct qdma_sg_blk *)
1385 + sgb_tmp->blk_virt_addr - 1);
1386 + sgb_tmp->blk_bus_addr = sgb_tmp->blk_bus_addr
1387 + - sizeof(*sgb_tmp);
1388 + dma_pool_free(qchan->sg_blk_pool, sgb_tmp->blk_virt_addr,
1389 + sgb_tmp->blk_bus_addr);
1394 +static void __cold dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
1395 + struct list_head *head)
1397 + struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
1398 + /* free the QDMA comp resource */
1399 + list_for_each_entry_safe(comp_tmp, _comp_tmp,
1401 + dma_pool_free(qchan->fd_pool,
1402 + comp_tmp->fd_virt_addr,
1403 + comp_tmp->fd_bus_addr);
1404 + /* free the SG source block on comp */
1405 + dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_src_head);
1406 + /* free the SG destination block on comp */
1407 + dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_dst_head);
1408 + list_del(&comp_tmp->list);
1414 +static void __cold dpaa2_dpdmai_free_channels(
1415 + struct dpaa2_qdma_engine *dpaa2_qdma)
1417 + struct dpaa2_qdma_chan *qchan;
1420 + num = dpaa2_qdma->n_chans;
1421 + for (i = 0; i < num; i++) {
1422 + qchan = &dpaa2_qdma->chans[i];
1423 + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
1424 + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
1425 + dpaa2_dpdmai_free_pool(qchan, &qchan->sgb_free);
1426 + dma_pool_destroy(qchan->fd_pool);
1427 + dma_pool_destroy(qchan->sg_blk_pool);
1431 +static int dpaa2_dpdmai_alloc_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
1433 + struct dpaa2_qdma_chan *dpaa2_chan;
1434 + struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
1437 + INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
1438 + for (i = 0; i < dpaa2_qdma->n_chans; i++) {
1439 + dpaa2_chan = &dpaa2_qdma->chans[i];
1440 + dpaa2_chan->qdma = dpaa2_qdma;
1441 + dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
1442 + vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
1444 + dpaa2_chan->fd_pool = dma_pool_create("fd_pool",
1445 + dev, FD_POOL_SIZE, 32, 0);
1446 + if (!dpaa2_chan->fd_pool)
1448 + dpaa2_chan->sg_blk_pool = dma_pool_create("sg_blk_pool",
1449 + dev, SG_POOL_SIZE, 32, 0);
1450 + if (!dpaa2_chan->sg_blk_pool)
1453 + spin_lock_init(&dpaa2_chan->queue_lock);
1454 + INIT_LIST_HEAD(&dpaa2_chan->comp_used);
1455 + INIT_LIST_HEAD(&dpaa2_chan->comp_free);
1456 + INIT_LIST_HEAD(&dpaa2_chan->sgb_free);
1461 +static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
1463 + struct dpaa2_qdma_priv *priv;
1464 + struct device *dev = &dpdmai_dev->dev;
1465 + struct dpaa2_qdma_engine *dpaa2_qdma;
1468 + priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1471 + dev_set_drvdata(dev, priv);
1472 + priv->dpdmai_dev = dpdmai_dev;
1474 + priv->iommu_domain = iommu_get_domain_for_dev(dev);
1475 + if (priv->iommu_domain)
1476 + smmu_disable = false;
1478 + /* obtain a MC portal */
1479 + err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
1481 + dev_err(dev, "MC portal allocation failed\n");
1482 + goto err_mcportal;
1485 + /* DPDMAI initialization */
1486 + err = dpaa2_qdma_setup(dpdmai_dev);
1488 + dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
1489 + goto err_dpdmai_setup;
1493 + err = dpaa2_qdma_dpio_setup(priv);
1495 + dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
1496 + goto err_dpio_setup;
1499 + /* DPDMAI binding to DPIO */
1500 + err = dpaa2_dpdmai_bind(priv);
1502 + dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
1506 + /* DPDMAI enable */
1507 + err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
1509 + dev_err(dev, "dpdmai_enable() faile\n");
1513 + dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
1514 + if (!dpaa2_qdma) {
1519 + priv->dpaa2_qdma = dpaa2_qdma;
1520 + dpaa2_qdma->priv = priv;
1522 + dpaa2_qdma->n_chans = NUM_CH;
1524 + err = dpaa2_dpdmai_alloc_channels(dpaa2_qdma);
1526 + dev_err(dev, "QDMA alloc channels faile\n");
1530 + dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
1531 + dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
1532 + dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
1534 + dpaa2_qdma->dma_dev.dev = dev;
1535 + dpaa2_qdma->dma_dev.device_alloc_chan_resources
1536 + = dpaa2_qdma_alloc_chan_resources;
1537 + dpaa2_qdma->dma_dev.device_free_chan_resources
1538 + = dpaa2_qdma_free_chan_resources;
1539 + dpaa2_qdma->dma_dev.device_tx_status = dpaa2_qdma_tx_status;
1540 + dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
1541 + dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
1543 + err = dma_async_device_register(&dpaa2_qdma->dma_dev);
1545 + dev_err(dev, "Can't register NXP QDMA engine.\n");
1552 + dpaa2_dpdmai_free_channels(dpaa2_qdma);
1553 + kfree(dpaa2_qdma);
1555 + dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
1557 + dpaa2_dpdmai_dpio_unbind(priv);
1559 + dpaa2_dpmai_store_free(priv);
1560 + dpaa2_dpdmai_dpio_free(priv);
1562 + dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
1564 + fsl_mc_portal_free(priv->mc_io);
1566 + kfree(priv->ppriv);
1568 + dev_set_drvdata(dev, NULL);
1572 +static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
1574 + struct device *dev;
1575 + struct dpaa2_qdma_priv *priv;
1576 + struct dpaa2_qdma_engine *dpaa2_qdma;
1578 + dev = &ls_dev->dev;
1579 + priv = dev_get_drvdata(dev);
1580 + dpaa2_qdma = priv->dpaa2_qdma;
1582 + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
1583 + dpaa2_dpdmai_dpio_unbind(priv);
1584 + dpaa2_dpmai_store_free(priv);
1585 + dpaa2_dpdmai_dpio_free(priv);
1586 + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
1587 + fsl_mc_portal_free(priv->mc_io);
1588 + dev_set_drvdata(dev, NULL);
1589 + dpaa2_dpdmai_free_channels(dpaa2_qdma);
1591 + dma_async_device_unregister(&dpaa2_qdma->dma_dev);
1593 + kfree(dpaa2_qdma);
1598 +static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
1600 + .vendor = FSL_MC_VENDOR_FREESCALE,
1601 + .obj_type = "dpdmai",
1606 +static struct fsl_mc_driver dpaa2_qdma_driver = {
1608 + .name = "dpaa2-qdma",
1609 + .owner = THIS_MODULE,
1611 + .probe = dpaa2_qdma_probe,
1612 + .remove = dpaa2_qdma_remove,
1613 + .match_id_table = dpaa2_qdma_id_table
1616 +static int __init dpaa2_qdma_driver_init(void)
1618 + return fsl_mc_driver_register(&(dpaa2_qdma_driver));
1620 +late_initcall(dpaa2_qdma_driver_init);
1622 +static void __exit fsl_qdma_exit(void)
1624 + fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
1626 +module_exit(fsl_qdma_exit);
1628 +MODULE_DESCRIPTION("NXP DPAA2 qDMA driver");
1629 +MODULE_LICENSE("Dual BSD/GPL");
1631 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.h
1633 +/* Copyright 2015 NXP Semiconductor Inc.
1635 + * Redistribution and use in source and binary forms, with or without
1636 + * modification, are permitted provided that the following conditions are met:
1637 + * * Redistributions of source code must retain the above copyright
1638 + * notice, this list of conditions and the following disclaimer.
1639 + * * Redistributions in binary form must reproduce the above copyright
1640 + * notice, this list of conditions and the following disclaimer in the
1641 + * documentation and/or other materials provided with the distribution.
1642 + * * Neither the name of NXP Semiconductor nor the
1643 + * names of its contributors may be used to endorse or promote products
1644 + * derived from this software without specific prior written permission.
1647 + * ALTERNATIVELY, this software may be distributed under the terms of the
1648 + * GNU General Public License ("GPL") as published by the Free Software
1649 + * Foundation, either version 2 of that License or (at your option) any
1652 + * THIS SOFTWARE IS PROVIDED BY NXP Semiconductor ``AS IS'' AND ANY
1653 + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1654 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1655 + * DISCLAIMED. IN NO EVENT SHALL NXP Semiconductor BE LIABLE FOR ANY
1656 + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1657 + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1658 + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1659 + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1660 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1661 + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1664 +#ifndef __DPAA2_QDMA_H
1665 +#define __DPAA2_QDMA_H
1667 +#define LONG_FORMAT 1
1669 +#define DPAA2_QDMA_STORE_SIZE 16
1671 +#define NUM_SG_PER_BLK 16
1673 +#define QDMA_DMR_OFFSET 0x0
1674 +#define QDMA_DQ_EN (0 << 30)
1675 +#define QDMA_DQ_DIS (1 << 30)
1677 +#define QDMA_DSR_M_OFFSET 0x10004
1679 +struct dpaa2_qdma_sd_d {
1683 + uint32_t ssd:12; /* souce stride distance */
1684 + uint32_t sss:12; /* souce stride size */
1688 + uint32_t dsd:12; /* Destination stride distance */
1689 + uint32_t dss:12; /* Destination stride size */
1693 + uint32_t rbpcmd; /* Route-by-port command */
1695 +} __attribute__((__packed__));
1696 +/* Source descriptor command read transaction type for RBP=0:
1697 + coherent copy of cacheable memory */
1698 +#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28)
1699 +/* Destination descriptor command write transaction type for RBP=0:
1700 + coherent copy of cacheable memory */
1701 +#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28)
1703 +#define QDMA_SG_FMT_SDB 0x0 /* single data buffer */
1704 +#define QDMA_SG_FMT_FDS 0x1 /* frame data section */
1705 +#define QDMA_SG_FMT_SGTE 0x2 /* SGT extension */
1706 +#define QDMA_SG_SL_SHORT 0x1 /* short length */
1707 +#define QDMA_SG_SL_LONG 0x0 /* short length */
1708 +#define QDMA_SG_F 0x1 /* last sg entry */
1709 +struct dpaa2_qdma_sg {
1710 + uint32_t addr_lo; /* address 0:31 */
1711 + uint32_t addr_hi:17; /* address 32:48 */
1714 + uint32_t data_len_sl0; /* SL=0, the long format */
1716 + uint32_t len:17; /* SL=1, the short format */
1717 + uint32_t reserve:3;
1720 + uint32_t size:10; /* buff size */
1722 + } data_len; /* AVAIL_LENGTH */
1727 + uint32_t offset:12;
1732 +} __attribute__((__packed__));
1734 +#define QMAN_FD_FMT_ENABLE (1) /* frame list table enable */
1735 +#define QMAN_FD_BMT_ENABLE (1 << 15) /* bypass memory translation */
1736 +#define QMAN_FD_BMT_DISABLE (0 << 15) /* bypass memory translation */
1737 +#define QMAN_FD_SL_DISABLE (0 << 14) /* short lengthe disabled */
1738 +#define QMAN_FD_SL_ENABLE (1 << 14) /* short lengthe enabled */
1740 +#define QDMA_SB_FRAME (0 << 28) /* single frame */
1741 +#define QDMA_SG_FRAME (2 << 28) /* scatter gather frames */
1742 +#define QDMA_FINAL_BIT_DISABLE (0 << 31) /* final bit disable */
1743 +#define QDMA_FINAL_BIT_ENABLE (1 << 31) /* final bit enable */
1745 +#define QDMA_FD_SHORT_FORMAT (1 << 11) /* short format */
1746 +#define QDMA_FD_LONG_FORMAT (0 << 11) /* long format */
1747 +#define QDMA_SER_DISABLE (0 << 8) /* no notification */
1748 +#define QDMA_SER_CTX (1 << 8) /* notification by FQD_CTX[fqid] */
1749 +#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */
1750 +#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */
1751 +#define QDMA_FD_SPF_ENALBE (1 << 30) /* source prefetch enable */
1753 +#define QMAN_FD_VA_ENABLE (1 << 14) /* Address used is virtual address */
1754 +#define QMAN_FD_VA_DISABLE (0 << 14)/* Address used is a real address */
1755 +#define QMAN_FD_CBMT_ENABLE (1 << 15) /* Flow Context: 49bit physical address */
1756 +#define QMAN_FD_CBMT_DISABLE (0 << 15) /* Flow Context: 64bit virtual address */
1757 +#define QMAN_FD_SC_DISABLE (0 << 27) /* stashing control */
1759 +#define QDMA_FL_FMT_SBF (0x0) /* Single buffer frame */
1760 +#define QDMA_FL_FMT_SGE 0x2 /* Scatter gather frame */
1761 +#define QDMA_FL_BMT_ENABLE (0x1 << 15)/* enable bypass memory translation */
1762 +#define QDMA_FL_BMT_DISABLE 0x0 /* enable bypass memory translation */
1763 +#define QDMA_FL_SL_LONG (0x0 << 2)/* long length */
1764 +#define QDMA_FL_SL_SHORT 0x1 /* short length */
1765 +#define QDMA_FL_F (0x1)/* last frame list bit */
1766 +/*Description of Frame list table structure*/
1768 +struct dpaa2_qdma_chan {
1769 + struct virt_dma_chan vchan;
1770 + struct virt_dma_desc vdesc;
1771 + enum dma_status status;
1772 + struct dpaa2_qdma_engine *qdma;
1774 + struct mutex dpaa2_queue_mutex;
1775 + spinlock_t queue_lock;
1776 + struct dma_pool *fd_pool;
1777 + struct dma_pool *sg_blk_pool;
1779 + struct list_head comp_used;
1780 + struct list_head comp_free;
1782 + struct list_head sgb_free;
1785 +struct qdma_sg_blk {
1786 + dma_addr_t blk_bus_addr;
1787 + void *blk_virt_addr;
1788 + struct list_head list;
1791 +struct dpaa2_qdma_comp {
1792 + dma_addr_t fd_bus_addr;
1793 + dma_addr_t fl_bus_addr;
1794 + dma_addr_t desc_bus_addr;
1795 + dma_addr_t sge_src_bus_addr;
1796 + dma_addr_t sge_dst_bus_addr;
1797 + void *fd_virt_addr;
1798 + void *fl_virt_addr;
1799 + void *desc_virt_addr;
1800 + void *sg_src_virt_addr;
1801 + void *sg_dst_virt_addr;
1802 + struct qdma_sg_blk *sg_blk;
1803 + uint32_t sg_blk_num;
1804 + struct list_head sg_src_head;
1805 + struct list_head sg_dst_head;
1806 + struct dpaa2_qdma_chan *qchan;
1807 + struct virt_dma_desc vdesc;
1808 + struct list_head list;
1811 +struct dpaa2_qdma_engine {
1812 + struct dma_device dma_dev;
1814 + struct dpaa2_qdma_chan chans[NUM_CH];
1816 + struct dpaa2_qdma_priv *priv;
1820 + * dpaa2_qdma_priv - driver private data
1822 +struct dpaa2_qdma_priv {
1825 + struct iommu_domain *iommu_domain;
1826 + struct dpdmai_attr dpdmai_attr;
1827 + struct device *dev;
1828 + struct fsl_mc_io *mc_io;
1829 + struct fsl_mc_device *dpdmai_dev;
1831 + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM];
1832 + struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_PRIO_NUM];
1834 + uint8_t num_pairs;
1836 + struct dpaa2_qdma_engine *dpaa2_qdma;
1837 + struct dpaa2_qdma_priv_per_prio *ppriv;
1840 +struct dpaa2_qdma_priv_per_prio {
1845 + struct dpaa2_io_store *store;
1846 + struct dpaa2_io_notification_ctx nctx;
1848 + struct dpaa2_qdma_priv *priv;
1851 +/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */
1852 +#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \
1853 + sizeof(struct dpaa2_fl_entry) * 3 + \
1854 + sizeof(struct dpaa2_qdma_sd_d) * 2)
1856 +/* qdma_sg_blk + 16 SGs */
1857 +#define SG_POOL_SIZE (sizeof(struct qdma_sg_blk) +\
1858 + sizeof(struct dpaa2_qdma_sg) * NUM_SG_PER_BLK)
1859 +#endif /* __DPAA2_QDMA_H */
1861 +++ b/drivers/dma/dpaa2-qdma/dpdmai.c
1863 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
1865 + * Redistribution and use in source and binary forms, with or without
1866 + * modification, are permitted provided that the following conditions are met:
1867 + * * Redistributions of source code must retain the above copyright
1868 + * notice, this list of conditions and the following disclaimer.
1869 + * * Redistributions in binary form must reproduce the above copyright
1870 + * notice, this list of conditions and the following disclaimer in the
1871 + * documentation and/or other materials provided with the distribution.
1872 + * * Neither the name of the above-listed copyright holders nor the
1873 + * names of any contributors may be used to endorse or promote products
1874 + * derived from this software without specific prior written permission.
1877 + * ALTERNATIVELY, this software may be distributed under the terms of the
1878 + * GNU General Public License ("GPL") as published by the Free Software
1879 + * Foundation, either version 2 of that License or (at your option) any
1882 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1883 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1884 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1885 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
1886 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1887 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1888 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1889 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1890 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1891 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1892 + * POSSIBILITY OF SUCH DAMAGE.
1894 +#include <linux/types.h>
1895 +#include <linux/io.h>
1896 +#include "fsl_dpdmai.h"
1897 +#include "fsl_dpdmai_cmd.h"
1898 +#include <linux/fsl/mc.h>
1900 +struct dpdmai_cmd_open {
1904 +struct dpdmai_rsp_get_attributes {
1906 + u8 num_of_priorities;
1913 +struct dpdmai_cmd_queue {
1926 +struct dpdmai_rsp_get_tx_queue {
1932 +int dpdmai_open(struct fsl_mc_io *mc_io,
1933 + uint32_t cmd_flags,
1937 + struct fsl_mc_command cmd = { 0 };
1938 + struct dpdmai_cmd_open *cmd_params;
1941 + /* prepare command */
1942 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN,
1946 + cmd_params = (struct dpdmai_cmd_open *)cmd.params;
1947 + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id);
1949 + /* send command to mc*/
1950 + err = mc_send_command(mc_io, &cmd);
1954 + /* retrieve response parameters */
1955 + *token = mc_cmd_hdr_read_token(&cmd);
1959 +int dpdmai_close(struct fsl_mc_io *mc_io,
1960 + uint32_t cmd_flags,
1963 + struct fsl_mc_command cmd = { 0 };
1965 + /* prepare command */
1966 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE,
1967 + cmd_flags, token);
1969 + /* send command to mc*/
1970 + return mc_send_command(mc_io, &cmd);
1973 +int dpdmai_create(struct fsl_mc_io *mc_io,
1974 + uint32_t cmd_flags,
1975 + const struct dpdmai_cfg *cfg,
1978 + struct fsl_mc_command cmd = { 0 };
1981 + /* prepare command */
1982 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE,
1985 + DPDMAI_CMD_CREATE(cmd, cfg);
1987 + /* send command to mc*/
1988 + err = mc_send_command(mc_io, &cmd);
1992 + /* retrieve response parameters */
1993 + *token = MC_CMD_HDR_READ_TOKEN(cmd.header);
1998 +int dpdmai_destroy(struct fsl_mc_io *mc_io,
1999 + uint32_t cmd_flags,
2002 + struct fsl_mc_command cmd = { 0 };
2004 + /* prepare command */
2005 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY,
2009 + /* send command to mc*/
2010 + return mc_send_command(mc_io, &cmd);
2013 +int dpdmai_enable(struct fsl_mc_io *mc_io,
2014 + uint32_t cmd_flags,
2017 + struct fsl_mc_command cmd = { 0 };
2019 + /* prepare command */
2020 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE,
2024 + /* send command to mc*/
2025 + return mc_send_command(mc_io, &cmd);
2028 +int dpdmai_disable(struct fsl_mc_io *mc_io,
2029 + uint32_t cmd_flags,
2032 + struct fsl_mc_command cmd = { 0 };
2034 + /* prepare command */
2035 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE,
2039 + /* send command to mc*/
2040 + return mc_send_command(mc_io, &cmd);
2043 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
2044 + uint32_t cmd_flags,
2048 + struct fsl_mc_command cmd = { 0 };
2050 + /* prepare command */
2051 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED,
2055 + /* send command to mc*/
2056 + err = mc_send_command(mc_io, &cmd);
2060 + /* retrieve response parameters */
2061 + DPDMAI_RSP_IS_ENABLED(cmd, *en);
2066 +int dpdmai_reset(struct fsl_mc_io *mc_io,
2067 + uint32_t cmd_flags,
2070 + struct fsl_mc_command cmd = { 0 };
2072 + /* prepare command */
2073 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET,
2077 + /* send command to mc*/
2078 + return mc_send_command(mc_io, &cmd);
2081 +int dpdmai_get_irq(struct fsl_mc_io *mc_io,
2082 + uint32_t cmd_flags,
2084 + uint8_t irq_index,
2086 + struct dpdmai_irq_cfg *irq_cfg)
2088 + struct fsl_mc_command cmd = { 0 };
2091 + /* prepare command */
2092 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ,
2095 + DPDMAI_CMD_GET_IRQ(cmd, irq_index);
2097 + /* send command to mc*/
2098 + err = mc_send_command(mc_io, &cmd);
2102 + /* retrieve response parameters */
2103 + DPDMAI_RSP_GET_IRQ(cmd, *type, irq_cfg);
2108 +int dpdmai_set_irq(struct fsl_mc_io *mc_io,
2109 + uint32_t cmd_flags,
2111 + uint8_t irq_index,
2112 + struct dpdmai_irq_cfg *irq_cfg)
2114 + struct fsl_mc_command cmd = { 0 };
2116 + /* prepare command */
2117 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ,
2120 + DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg);
2122 + /* send command to mc*/
2123 + return mc_send_command(mc_io, &cmd);
2126 +int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io,
2127 + uint32_t cmd_flags,
2129 + uint8_t irq_index,
2132 + struct fsl_mc_command cmd = { 0 };
2135 + /* prepare command */
2136 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_ENABLE,
2139 + DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index);
2141 + /* send command to mc*/
2142 + err = mc_send_command(mc_io, &cmd);
2146 + /* retrieve response parameters */
2147 + DPDMAI_RSP_GET_IRQ_ENABLE(cmd, *en);
2152 +int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io,
2153 + uint32_t cmd_flags,
2155 + uint8_t irq_index,
2158 + struct fsl_mc_command cmd = { 0 };
2160 + /* prepare command */
2161 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_ENABLE,
2164 + DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en);
2166 + /* send command to mc*/
2167 + return mc_send_command(mc_io, &cmd);
2170 +int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io,
2171 + uint32_t cmd_flags,
2173 + uint8_t irq_index,
2176 + struct fsl_mc_command cmd = { 0 };
2179 + /* prepare command */
2180 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_MASK,
2183 + DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index);
2185 + /* send command to mc*/
2186 + err = mc_send_command(mc_io, &cmd);
2190 + /* retrieve response parameters */
2191 + DPDMAI_RSP_GET_IRQ_MASK(cmd, *mask);
2196 +int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io,
2197 + uint32_t cmd_flags,
2199 + uint8_t irq_index,
2202 + struct fsl_mc_command cmd = { 0 };
2204 + /* prepare command */
2205 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_MASK,
2208 + DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask);
2210 + /* send command to mc*/
2211 + return mc_send_command(mc_io, &cmd);
2214 +int dpdmai_get_irq_status(struct fsl_mc_io *mc_io,
2215 + uint32_t cmd_flags,
2217 + uint8_t irq_index,
2220 + struct fsl_mc_command cmd = { 0 };
2223 + /* prepare command */
2224 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_STATUS,
2227 + DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status);
2229 + /* send command to mc*/
2230 + err = mc_send_command(mc_io, &cmd);
2234 + /* retrieve response parameters */
2235 + DPDMAI_RSP_GET_IRQ_STATUS(cmd, *status);
2240 +int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io,
2241 + uint32_t cmd_flags,
2243 + uint8_t irq_index,
2246 + struct fsl_mc_command cmd = { 0 };
2248 + /* prepare command */
2249 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLEAR_IRQ_STATUS,
2252 + DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status);
2254 + /* send command to mc*/
2255 + return mc_send_command(mc_io, &cmd);
2258 +int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
2259 + uint32_t cmd_flags,
2261 + struct dpdmai_attr *attr)
2263 + struct fsl_mc_command cmd = { 0 };
2265 + struct dpdmai_rsp_get_attributes *rsp_params;
2267 + /* prepare command */
2268 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR,
2272 + /* send command to mc*/
2273 + err = mc_send_command(mc_io, &cmd);
2277 + /* retrieve response parameters */
2278 + rsp_params = (struct dpdmai_rsp_get_attributes *)cmd.params;
2279 + attr->id = le32_to_cpu(rsp_params->id);
2280 + attr->version.major = le16_to_cpu(rsp_params->major);
2281 + attr->version.minor = le16_to_cpu(rsp_params->minor);
2282 + attr->num_of_priorities = rsp_params->num_of_priorities;
2288 +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
2289 + uint32_t cmd_flags,
2292 + const struct dpdmai_rx_queue_cfg *cfg)
2294 + struct fsl_mc_command cmd = { 0 };
2295 + struct dpdmai_cmd_queue *cmd_params;
2297 + /* prepare command */
2298 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE,
2302 + cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2303 + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
2304 + cmd_params->priority = cfg->dest_cfg.priority;
2305 + cmd_params->queue = priority;
2306 + cmd_params->dest_type = cfg->dest_cfg.dest_type;
2307 + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
2308 + cmd_params->options = cpu_to_le32(cfg->options);
2311 + /* send command to mc*/
2312 + return mc_send_command(mc_io, &cmd);
2315 +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
2316 + uint32_t cmd_flags,
2318 + uint8_t priority, struct dpdmai_rx_queue_attr *attr)
2320 + struct fsl_mc_command cmd = { 0 };
2321 + struct dpdmai_cmd_queue *cmd_params;
2324 + /* prepare command */
2325 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE,
2329 + cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2330 + cmd_params->queue = priority;
2332 + /* send command to mc*/
2333 + err = mc_send_command(mc_io, &cmd);
2337 + /* retrieve response parameters */
2338 + attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id);
2339 + attr->dest_cfg.priority = cmd_params->priority;
2340 + attr->dest_cfg.dest_type = cmd_params->dest_type;
2341 + attr->user_ctx = le64_to_cpu(cmd_params->user_ctx);
2342 + attr->fqid = le32_to_cpu(cmd_params->fqid);
2347 +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
2348 + uint32_t cmd_flags,
2351 + struct dpdmai_tx_queue_attr *attr)
2353 + struct fsl_mc_command cmd = { 0 };
2354 + struct dpdmai_cmd_queue *cmd_params;
2355 + struct dpdmai_rsp_get_tx_queue *rsp_params;
2358 + /* prepare command */
2359 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE,
2363 + cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2364 + cmd_params->queue = priority;
2366 + /* send command to mc*/
2367 + err = mc_send_command(mc_io, &cmd);
2371 + /* retrieve response parameters */
2373 + rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params;
2374 + attr->fqid = le32_to_cpu(rsp_params->fqid);
2379 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai.h
2381 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
2383 + * Redistribution and use in source and binary forms, with or without
2384 + * modification, are permitted provided that the following conditions are met:
2385 + * * Redistributions of source code must retain the above copyright
2386 + * notice, this list of conditions and the following disclaimer.
2387 + * * Redistributions in binary form must reproduce the above copyright
2388 + * notice, this list of conditions and the following disclaimer in the
2389 + * documentation and/or other materials provided with the distribution.
2390 + * * Neither the name of the above-listed copyright holders nor the
2391 + * names of any contributors may be used to endorse or promote products
2392 + * derived from this software without specific prior written permission.
2395 + * ALTERNATIVELY, this software may be distributed under the terms of the
2396 + * GNU General Public License ("GPL") as published by the Free Software
2397 + * Foundation, either version 2 of that License or (at your option) any
2400 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2401 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2402 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2403 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
2404 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2405 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2406 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2407 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2408 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
2409 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
2410 + * POSSIBILITY OF SUCH DAMAGE.
2412 +#ifndef __FSL_DPDMAI_H
2413 +#define __FSL_DPDMAI_H
2417 +/* Data Path DMA Interface API
2418 + * Contains initialization APIs and runtime control APIs for DPDMAI
2421 +/* General DPDMAI macros */
2424 + * Maximum number of Tx/Rx priorities per DPDMAI object
2426 +#define DPDMAI_PRIO_NUM 2
2429 + * All queues considered; see dpdmai_set_rx_queue()
2431 +#define DPDMAI_ALL_QUEUES (uint8_t)(-1)
2434 + * dpdmai_open() - Open a control session for the specified object
2435 + * @mc_io: Pointer to MC portal's I/O object
2436 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2437 + * @dpdmai_id: DPDMAI unique ID
2438 + * @token: Returned token; use in subsequent API calls
2440 + * This function can be used to open a control session for an
2441 + * already created object; an object may have been declared in
2442 + * the DPL or by calling the dpdmai_create() function.
2443 + * This function returns a unique authentication token,
2444 + * associated with the specific object ID and the specific MC
2445 + * portal; this token must be used in all subsequent commands for
2446 + * this specific object.
2448 + * Return: '0' on Success; Error code otherwise.
2450 +int dpdmai_open(struct fsl_mc_io *mc_io,
2451 + uint32_t cmd_flags,
2456 + * dpdmai_close() - Close the control session of the object
2457 + * @mc_io: Pointer to MC portal's I/O object
2458 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2459 + * @token: Token of DPDMAI object
2461 + * After this function is called, no further operations are
2462 + * allowed on the object without opening a new control session.
2464 + * Return: '0' on Success; Error code otherwise.
2466 +int dpdmai_close(struct fsl_mc_io *mc_io,
2467 + uint32_t cmd_flags,
2471 + * struct dpdmai_cfg - Structure representing DPDMAI configuration
2472 + * @priorities: Priorities for the DMA hardware processing; valid priorities are
2473 + * configured with values 1-8; the entry following last valid entry
2474 + * should be configured with 0
2476 +struct dpdmai_cfg {
2477 + uint8_t priorities[DPDMAI_PRIO_NUM];
2481 + * dpdmai_create() - Create the DPDMAI object
2482 + * @mc_io: Pointer to MC portal's I/O object
2483 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2484 + * @cfg: Configuration structure
2485 + * @token: Returned token; use in subsequent API calls
2487 + * Create the DPDMAI object, allocate required resources and
2488 + * perform required initialization.
2490 + * The object can be created either by declaring it in the
2491 + * DPL file, or by calling this function.
2493 + * This function returns a unique authentication token,
2494 + * associated with the specific object ID and the specific MC
2495 + * portal; this token must be used in all subsequent calls to
2496 + * this specific object. For objects that are created using the
2497 + * DPL file, call dpdmai_open() function to get an authentication
2500 + * Return: '0' on Success; Error code otherwise.
2502 +int dpdmai_create(struct fsl_mc_io *mc_io,
2503 + uint32_t cmd_flags,
2504 + const struct dpdmai_cfg *cfg,
2508 + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources.
2509 + * @mc_io: Pointer to MC portal's I/O object
2510 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2511 + * @token: Token of DPDMAI object
2513 + * Return: '0' on Success; error code otherwise.
2515 +int dpdmai_destroy(struct fsl_mc_io *mc_io,
2516 + uint32_t cmd_flags,
2520 + * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames.
2521 + * @mc_io: Pointer to MC portal's I/O object
2522 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2523 + * @token: Token of DPDMAI object
2525 + * Return: '0' on Success; Error code otherwise.
2527 +int dpdmai_enable(struct fsl_mc_io *mc_io,
2528 + uint32_t cmd_flags,
2532 + * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames.
2533 + * @mc_io: Pointer to MC portal's I/O object
2534 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2535 + * @token: Token of DPDMAI object
2537 + * Return: '0' on Success; Error code otherwise.
2539 +int dpdmai_disable(struct fsl_mc_io *mc_io,
2540 + uint32_t cmd_flags,
2544 + * dpdmai_is_enabled() - Check if the DPDMAI is enabled.
2545 + * @mc_io: Pointer to MC portal's I/O object
2546 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2547 + * @token: Token of DPDMAI object
2548 + * @en: Returns '1' if object is enabled; '0' otherwise
2550 + * Return: '0' on Success; Error code otherwise.
2552 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
2553 + uint32_t cmd_flags,
2558 + * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state.
2559 + * @mc_io: Pointer to MC portal's I/O object
2560 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2561 + * @token: Token of DPDMAI object
2563 + * Return: '0' on Success; Error code otherwise.
2565 +int dpdmai_reset(struct fsl_mc_io *mc_io,
2566 + uint32_t cmd_flags,
2570 + * struct dpdmai_irq_cfg - IRQ configuration
2571 + * @addr: Address that must be written to signal a message-based interrupt
2572 + * @val: Value to write into irq_addr address
2573 + * @irq_num: A user defined number associated with this IRQ
2575 +struct dpdmai_irq_cfg {
2582 + * dpdmai_set_irq() - Set IRQ information for the DPDMAI to trigger an interrupt.
2583 + * @mc_io: Pointer to MC portal's I/O object
2584 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2585 + * @token: Token of DPDMAI object
2586 + * @irq_index: Identifies the interrupt index to configure
2587 + * @irq_cfg: IRQ configuration
2589 + * Return: '0' on Success; Error code otherwise.
2591 +int dpdmai_set_irq(struct fsl_mc_io *mc_io,
2592 + uint32_t cmd_flags,
2594 + uint8_t irq_index,
2595 + struct dpdmai_irq_cfg *irq_cfg);
2598 + * dpdmai_get_irq() - Get IRQ information from the DPDMAI
2600 + * @mc_io: Pointer to MC portal's I/O object
2601 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2602 + * @token: Token of DPDMAI object
2603 + * @irq_index: The interrupt index to configure
2604 + * @type: Interrupt type: 0 represents message interrupt
2605 + * type (both irq_addr and irq_val are valid)
2606 + * @irq_cfg: IRQ attributes
2608 + * Return: '0' on Success; Error code otherwise.
2610 +int dpdmai_get_irq(struct fsl_mc_io *mc_io,
2611 + uint32_t cmd_flags,
2613 + uint8_t irq_index,
2615 + struct dpdmai_irq_cfg *irq_cfg);
2618 + * dpdmai_set_irq_enable() - Set overall interrupt state.
2619 + * @mc_io: Pointer to MC portal's I/O object
2620 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2621 + * @token: Token of DPDMAI object
2622 + * @irq_index: The interrupt index to configure
2623 + * @en: Interrupt state - enable = 1, disable = 0
2625 + * Allows GPP software to control when interrupts are generated.
2626 + * Each interrupt can have up to 32 causes. The enable/disable control's the
2627 + * overall interrupt state. if the interrupt is disabled no causes will cause
2630 + * Return: '0' on Success; Error code otherwise.
2632 +int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io,
2633 + uint32_t cmd_flags,
2635 + uint8_t irq_index,
2639 + * dpdmai_get_irq_enable() - Get overall interrupt state
2640 + * @mc_io: Pointer to MC portal's I/O object
2641 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2642 + * @token: Token of DPDMAI object
2643 + * @irq_index: The interrupt index to configure
2644 + * @en: Returned Interrupt state - enable = 1, disable = 0
2646 + * Return: '0' on Success; Error code otherwise.
2648 +int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io,
2649 + uint32_t cmd_flags,
2651 + uint8_t irq_index,
2655 + * dpdmai_set_irq_mask() - Set interrupt mask.
2656 + * @mc_io: Pointer to MC portal's I/O object
2657 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2658 + * @token: Token of DPDMAI object
2659 + * @irq_index: The interrupt index to configure
2660 + * @mask: event mask to trigger interrupt;
2662 + * 0 = ignore event
2663 + * 1 = consider event for asserting IRQ
2665 + * Every interrupt can have up to 32 causes and the interrupt model supports
2666 + * masking/unmasking each cause independently
2668 + * Return: '0' on Success; Error code otherwise.
2670 +int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io,
2671 + uint32_t cmd_flags,
2673 + uint8_t irq_index,
2677 + * dpdmai_get_irq_mask() - Get interrupt mask.
2678 + * @mc_io: Pointer to MC portal's I/O object
2679 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2680 + * @token: Token of DPDMAI object
2681 + * @irq_index: The interrupt index to configure
2682 + * @mask: Returned event mask to trigger interrupt
2684 + * Every interrupt can have up to 32 causes and the interrupt model supports
2685 + * masking/unmasking each cause independently
2687 + * Return: '0' on Success; Error code otherwise.
2689 +int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io,
2690 + uint32_t cmd_flags,
2692 + uint8_t irq_index,
2696 + * dpdmai_get_irq_status() - Get the current status of any pending interrupts
2697 + * @mc_io: Pointer to MC portal's I/O object
2698 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2699 + * @token: Token of DPDMAI object
2700 + * @irq_index: The interrupt index to configure
2701 + * @status: Returned interrupts status - one bit per cause:
2702 + * 0 = no interrupt pending
2703 + * 1 = interrupt pending
2705 + * Return: '0' on Success; Error code otherwise.
2707 +int dpdmai_get_irq_status(struct fsl_mc_io *mc_io,
2708 + uint32_t cmd_flags,
2710 + uint8_t irq_index,
2711 + uint32_t *status);
2714 + * dpdmai_clear_irq_status() - Clear a pending interrupt's status
2715 + * @mc_io: Pointer to MC portal's I/O object
2716 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2717 + * @token: Token of DPDMAI object
2718 + * @irq_index: The interrupt index to configure
2719 + * @status: bits to clear (W1C) - one bit per cause:
2720 + * 0 = don't change
2721 + * 1 = clear status bit
2723 + * Return: '0' on Success; Error code otherwise.
2725 +int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io,
2726 + uint32_t cmd_flags,
2728 + uint8_t irq_index,
2732 + * struct dpdmai_attr - Structure representing DPDMAI attributes
2733 + * @id: DPDMAI object ID
2734 + * @version: DPDMAI version
2735 + * @num_of_priorities: number of priorities
2737 +struct dpdmai_attr {
2740 + * struct version - DPDMAI version
2741 + * @major: DPDMAI major version
2742 + * @minor: DPDMAI minor version
2748 + uint8_t num_of_priorities;
2752 + * dpdmai_get_attributes() - Retrieve DPDMAI attributes.
2753 + * @mc_io: Pointer to MC portal's I/O object
2754 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2755 + * @token: Token of DPDMAI object
2756 + * @attr: Returned object's attributes
2758 + * Return: '0' on Success; Error code otherwise.
2760 +int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
2761 + uint32_t cmd_flags,
2763 + struct dpdmai_attr *attr);
2766 + * enum dpdmai_dest - DPDMAI destination types
2767 + * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode
2768 + * and does not generate FQDAN notifications; user is expected to dequeue
2769 + * from the queue based on polling or other user-defined method
2770 + * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
2771 + * notifications to the specified DPIO; user is expected to dequeue
2772 + * from the queue only after notification is received
2773 + * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate
2774 + * FQDAN notifications, but is connected to the specified DPCON object;
2775 + * user is expected to dequeue from the DPCON channel
2778 + DPDMAI_DEST_NONE = 0,
2779 + DPDMAI_DEST_DPIO = 1,
2780 + DPDMAI_DEST_DPCON = 2
2784 + * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters
2785 + * @dest_type: Destination type
2786 + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
2787 + * @priority: Priority selection within the DPIO or DPCON channel; valid values
2788 + * are 0-1 or 0-7, depending on the number of priorities in that
2789 + * channel; not relevant for 'DPDMAI_DEST_NONE' option
2791 +struct dpdmai_dest_cfg {
2792 + enum dpdmai_dest dest_type;
2797 +/* DPDMAI queue modification options */
2800 + * Select to modify the user's context associated with the queue
2802 +#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001
2805 + * Select to modify the queue's destination
2807 +#define DPDMAI_QUEUE_OPT_DEST 0x00000002
2810 + * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration
2811 + * @options: Flags representing the suggested modifications to the queue;
2812 + * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags
2813 + * @user_ctx: User context value provided in the frame descriptor of each
2815 + * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options'
2816 + * @dest_cfg: Queue destination parameters;
2817 + * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options'
2819 +struct dpdmai_rx_queue_cfg {
2821 + uint64_t user_ctx;
2822 + struct dpdmai_dest_cfg dest_cfg;
2827 + * dpdmai_set_rx_queue() - Set Rx queue configuration
2828 + * @mc_io: Pointer to MC portal's I/O object
2829 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2830 + * @token: Token of DPDMAI object
2831 + * @priority: Select the queue relative to number of
2832 + * priorities configured at DPDMAI creation; use
2833 + * DPDMAI_ALL_QUEUES to configure all Rx queues
2835 + * @cfg: Rx queue configuration
2837 + * Return: '0' on Success; Error code otherwise.
2839 +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
2840 + uint32_t cmd_flags,
2843 + const struct dpdmai_rx_queue_cfg *cfg);
2846 + * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues
2847 + * @user_ctx: User context value provided in the frame descriptor of each
2849 + * @dest_cfg: Queue destination configuration
2850 + * @fqid: Virtual FQID value to be used for dequeue operations
2852 +struct dpdmai_rx_queue_attr {
2853 + uint64_t user_ctx;
2854 + struct dpdmai_dest_cfg dest_cfg;
2859 + * dpdmai_get_rx_queue() - Retrieve Rx queue attributes.
2860 + * @mc_io: Pointer to MC portal's I/O object
2861 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2862 + * @token: Token of DPDMAI object
2863 + * @priority: Select the queue relative to number of
2864 + * priorities configured at DPDMAI creation
2865 + * @attr: Returned Rx queue attributes
2867 + * Return: '0' on Success; Error code otherwise.
2869 +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
2870 + uint32_t cmd_flags,
2873 + struct dpdmai_rx_queue_attr *attr);
2876 + * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues
2877 + * @fqid: Virtual FQID to be used for sending frames to DMA hardware
2880 +struct dpdmai_tx_queue_attr {
2885 + * dpdmai_get_tx_queue() - Retrieve Tx queue attributes.
2886 + * @mc_io: Pointer to MC portal's I/O object
2887 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2888 + * @token: Token of DPDMAI object
2889 + * @priority: Select the queue relative to number of
2890 + * priorities configured at DPDMAI creation
2891 + * @attr: Returned Tx queue attributes
2893 + * Return: '0' on Success; Error code otherwise.
2895 +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
2896 + uint32_t cmd_flags,
2899 + struct dpdmai_tx_queue_attr *attr);
2901 +#endif /* __FSL_DPDMAI_H */
2903 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
2905 +/* Copyright 2013-2016 Freescale Semiconductor Inc.
2907 + * Redistribution and use in source and binary forms, with or without
2908 + * modification, are permitted provided that the following conditions are met:
2909 + * * Redistributions of source code must retain the above copyright
2910 + * notice, this list of conditions and the following disclaimer.
2911 + * * Redistributions in binary form must reproduce the above copyright
2912 + * notice, this list of conditions and the following disclaimer in the
2913 + * documentation and/or other materials provided with the distribution.
2914 + * * Neither the name of the above-listed copyright holders nor the
2915 + * names of any contributors may be used to endorse or promote products
2916 + * derived from this software without specific prior written permission.
2919 + * ALTERNATIVELY, this software may be distributed under the terms of the
2920 + * GNU General Public License ("GPL") as published by the Free Software
2921 + * Foundation, either version 2 of that License or (at your option) any
2924 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2925 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2926 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2927 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
2928 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2929 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2930 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2931 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2932 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
2933 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
2934 + * POSSIBILITY OF SUCH DAMAGE.
2936 +#ifndef _FSL_DPDMAI_CMD_H
2937 +#define _FSL_DPDMAI_CMD_H
2939 +/* DPDMAI Version */
2940 +#define DPDMAI_VER_MAJOR 2
2941 +#define DPDMAI_VER_MINOR 2
2943 +#define DPDMAI_CMD_BASE_VERSION 0
2944 +#define DPDMAI_CMD_ID_OFFSET 4
2947 +#define DPDMAI_CMDID_CLOSE ((0x800 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2948 +#define DPDMAI_CMDID_OPEN ((0x80E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2949 +#define DPDMAI_CMDID_CREATE ((0x90E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2950 +#define DPDMAI_CMDID_DESTROY ((0x900 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2952 +#define DPDMAI_CMDID_ENABLE ((0x002 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2953 +#define DPDMAI_CMDID_DISABLE ((0x003 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2954 +#define DPDMAI_CMDID_GET_ATTR ((0x004 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2955 +#define DPDMAI_CMDID_RESET ((0x005 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2956 +#define DPDMAI_CMDID_IS_ENABLED ((0x006 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2958 +#define DPDMAI_CMDID_SET_IRQ ((0x010 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2959 +#define DPDMAI_CMDID_GET_IRQ ((0x011 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2960 +#define DPDMAI_CMDID_SET_IRQ_ENABLE ((0x012 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2961 +#define DPDMAI_CMDID_GET_IRQ_ENABLE ((0x013 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2962 +#define DPDMAI_CMDID_SET_IRQ_MASK ((0x014 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2963 +#define DPDMAI_CMDID_GET_IRQ_MASK ((0x015 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2964 +#define DPDMAI_CMDID_GET_IRQ_STATUS ((0x016 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2965 +#define DPDMAI_CMDID_CLEAR_IRQ_STATUS ((0x017 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2967 +#define DPDMAI_CMDID_SET_RX_QUEUE ((0x1A0 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2968 +#define DPDMAI_CMDID_GET_RX_QUEUE ((0x1A1 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2969 +#define DPDMAI_CMDID_GET_TX_QUEUE ((0x1A2 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2972 +#define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */
2973 +#define MC_CMD_HDR_TOKEN_S 16 /* Token field size */
2976 +#define MAKE_UMASK64(_width) \
2977 + ((uint64_t)((_width) < 64 ? ((uint64_t)1 << (_width)) - 1 : \
2980 +static inline uint64_t mc_enc(int lsoffset, int width, uint64_t val)
2982 + return (uint64_t)(((uint64_t)val & MAKE_UMASK64(width)) << lsoffset);
2985 +static inline uint64_t mc_dec(uint64_t val, int lsoffset, int width)
2987 + return (uint64_t)((val >> lsoffset) & MAKE_UMASK64(width));
2990 +#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \
2991 + ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg))
2993 +#define MC_RSP_OP(_cmd, _param, _offset, _width, _type, _arg) \
2994 + (_arg = (_type)mc_dec(_cmd.params[_param], (_offset), (_width)))
2996 +#define MC_CMD_HDR_READ_TOKEN(_hdr) \
2997 + ((uint16_t)mc_dec((_hdr), MC_CMD_HDR_TOKEN_O, MC_CMD_HDR_TOKEN_S))
2999 +/* cmd, param, offset, width, type, arg_name */
3000 +#define DPDMAI_CMD_OPEN(cmd, dpdmai_id) \
3001 + MC_CMD_OP(cmd, 0, 0, 32, int, dpdmai_id)
3003 +/* cmd, param, offset, width, type, arg_name */
3004 +#define DPDMAI_CMD_CREATE(cmd, cfg) \
3006 + MC_CMD_OP(cmd, 0, 8, 8, uint8_t, cfg->priorities[0]);\
3007 + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, cfg->priorities[1]);\
3010 +/* cmd, param, offset, width, type, arg_name */
3011 +#define DPDMAI_RSP_IS_ENABLED(cmd, en) \
3012 + MC_RSP_OP(cmd, 0, 0, 1, int, en)
3014 +/* cmd, param, offset, width, type, arg_name */
3015 +#define DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \
3017 + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\
3018 + MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\
3019 + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\
3020 + MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \
3023 +/* cmd, param, offset, width, type, arg_name */
3024 +#define DPDMAI_CMD_GET_IRQ(cmd, irq_index) \
3025 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
3027 +/* cmd, param, offset, width, type, arg_name */
3028 +#define DPDMAI_RSP_GET_IRQ(cmd, type, irq_cfg) \
3030 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val); \
3031 + MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\
3032 + MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \
3033 + MC_RSP_OP(cmd, 2, 32, 32, int, type); \
3036 +/* cmd, param, offset, width, type, arg_name */
3037 +#define DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \
3039 + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, enable_state); \
3040 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
3043 +/* cmd, param, offset, width, type, arg_name */
3044 +#define DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \
3045 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
3047 +/* cmd, param, offset, width, type, arg_name */
3048 +#define DPDMAI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \
3049 + MC_RSP_OP(cmd, 0, 0, 8, uint8_t, enable_state)
3051 +/* cmd, param, offset, width, type, arg_name */
3052 +#define DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \
3054 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \
3055 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
3058 +/* cmd, param, offset, width, type, arg_name */
3059 +#define DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index) \
3060 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
3062 +/* cmd, param, offset, width, type, arg_name */
3063 +#define DPDMAI_RSP_GET_IRQ_MASK(cmd, mask) \
3064 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask)
3066 +/* cmd, param, offset, width, type, arg_name */
3067 +#define DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \
3069 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\
3070 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\
3073 +/* cmd, param, offset, width, type, arg_name */
3074 +#define DPDMAI_RSP_GET_IRQ_STATUS(cmd, status) \
3075 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status)
3077 +/* cmd, param, offset, width, type, arg_name */
3078 +#define DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \
3080 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \
3081 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
3084 +/* cmd, param, offset, width, type, arg_name */
3085 +#define DPDMAI_RSP_GET_ATTR(cmd, attr) \
3087 + MC_RSP_OP(cmd, 0, 0, 32, int, attr->id); \
3088 + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->num_of_priorities); \
3089 + MC_RSP_OP(cmd, 1, 0, 16, uint16_t, attr->version.major);\
3090 + MC_RSP_OP(cmd, 1, 16, 16, uint16_t, attr->version.minor);\
3093 +/* cmd, param, offset, width, type, arg_name */
3094 +#define DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \
3096 + MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id); \
3097 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority); \
3098 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority); \
3099 + MC_CMD_OP(cmd, 0, 48, 4, enum dpdmai_dest, cfg->dest_cfg.dest_type); \
3100 + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx); \
3101 + MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\
3104 +/* cmd, param, offset, width, type, arg_name */
3105 +#define DPDMAI_CMD_GET_RX_QUEUE(cmd, priority) \
3106 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority)
3108 +/* cmd, param, offset, width, type, arg_name */
3109 +#define DPDMAI_RSP_GET_RX_QUEUE(cmd, attr) \
3111 + MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\
3112 + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\
3113 + MC_RSP_OP(cmd, 0, 48, 4, enum dpdmai_dest, attr->dest_cfg.dest_type);\
3114 + MC_RSP_OP(cmd, 1, 0, 64, uint64_t, attr->user_ctx);\
3115 + MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\
3118 +/* cmd, param, offset, width, type, arg_name */
3119 +#define DPDMAI_CMD_GET_TX_QUEUE(cmd, priority) \
3120 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority)
3122 +/* cmd, param, offset, width, type, arg_name */
3123 +#define DPDMAI_RSP_GET_TX_QUEUE(cmd, attr) \
3124 + MC_RSP_OP(cmd, 1, 0, 32, uint32_t, attr->fqid)
3126 +#endif /* _FSL_DPDMAI_CMD_H */
3128 +++ b/drivers/dma/fsl-qdma.c
3131 + * drivers/dma/fsl-qdma.c
3133 + * Copyright 2014-2015 Freescale Semiconductor, Inc.
3135 + * Driver for the Freescale qDMA engine with software command queue mode.
3136 + * Channel virtualization is supported through enqueuing of DMA jobs to,
3137 + * or dequeuing DMA jobs from, different work queues.
3138 + * This module can be found on Freescale LS SoCs.
3140 + * This program is free software; you can redistribute it and/or modify it
3141 + * under the terms of the GNU General Public License as published by the
3142 + * Free Software Foundation; either version 2 of the License, or (at your
3143 + * option) any later version.
3146 +#include <asm/cacheflush.h>
3147 +#include <linux/clk.h>
3148 +#include <linux/delay.h>
3149 +#include <linux/dma-mapping.h>
3150 +#include <linux/dmapool.h>
3151 +#include <linux/init.h>
3152 +#include <linux/interrupt.h>
3153 +#include <linux/module.h>
3154 +#include <linux/of.h>
3155 +#include <linux/of_address.h>
3156 +#include <linux/of_device.h>
3157 +#include <linux/of_dma.h>
3158 +#include <linux/of_irq.h>
3159 +#include <linux/slab.h>
3160 +#include <linux/spinlock.h>
3162 +#include "virt-dma.h"
3164 +#define FSL_QDMA_DMR 0x0
3165 +#define FSL_QDMA_DSR 0x4
3166 +#define FSL_QDMA_DEIER 0xe00
3167 +#define FSL_QDMA_DEDR 0xe04
3168 +#define FSL_QDMA_DECFDW0R 0xe10
3169 +#define FSL_QDMA_DECFDW1R 0xe14
3170 +#define FSL_QDMA_DECFDW2R 0xe18
3171 +#define FSL_QDMA_DECFDW3R 0xe1c
3172 +#define FSL_QDMA_DECFQIDR 0xe30
3173 +#define FSL_QDMA_DECBR 0xe34
3175 +#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x))
3176 +#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x))
3177 +#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x))
3178 +#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x))
3179 +#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x))
3180 +#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x))
3181 +#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x))
3182 +#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x))
3184 +#define FSL_QDMA_SQDPAR 0x80c
3185 +#define FSL_QDMA_SQEPAR 0x814
3186 +#define FSL_QDMA_BSQMR 0x800
3187 +#define FSL_QDMA_BSQSR 0x804
3188 +#define FSL_QDMA_BSQICR 0x828
3189 +#define FSL_QDMA_CQMR 0xa00
3190 +#define FSL_QDMA_CQDSCR1 0xa08
3191 +#define FSL_QDMA_CQDSCR2 0xa0c
3192 +#define FSL_QDMA_CQIER 0xa10
3193 +#define FSL_QDMA_CQEDR 0xa14
3194 +#define FSL_QDMA_SQCCMR 0xa20
3196 +#define FSL_QDMA_SQICR_ICEN
3198 +#define FSL_QDMA_CQIDR_CQT 0xff000000
3199 +#define FSL_QDMA_CQIDR_SQPE 0x800000
3200 +#define FSL_QDMA_CQIDR_SQT 0x8000
3202 +#define FSL_QDMA_BCQIER_CQTIE 0x8000
3203 +#define FSL_QDMA_BCQIER_CQPEIE 0x800000
3204 +#define FSL_QDMA_BSQICR_ICEN 0x80000000
3205 +#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16)
3206 +#define FSL_QDMA_CQIER_MEIE 0x80000000
3207 +#define FSL_QDMA_CQIER_TEIE 0x1
3208 +#define FSL_QDMA_SQCCMR_ENTER_WM 0x200000
3210 +#define FSL_QDMA_QUEUE_MAX 8
3212 +#define FSL_QDMA_BCQMR_EN 0x80000000
3213 +#define FSL_QDMA_BCQMR_EI 0x40000000
3214 +#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20)
3215 +#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16)
3217 +#define FSL_QDMA_BCQSR_QF 0x10000
3218 +#define FSL_QDMA_BCQSR_XOFF 0x1
3220 +#define FSL_QDMA_BSQMR_EN 0x80000000
3221 +#define FSL_QDMA_BSQMR_DI 0x40000000
3222 +#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16)
3224 +#define FSL_QDMA_BSQSR_QE 0x20000
3226 +#define FSL_QDMA_DMR_DQD 0x40000000
3227 +#define FSL_QDMA_DSR_DB 0x80000000
3229 +#define FSL_QDMA_BASE_BUFFER_SIZE 96
3230 +#define FSL_QDMA_EXPECT_SG_ENTRY_NUM 16
3231 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64
3232 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384
3233 +#define FSL_QDMA_QUEUE_NUM_MAX 8
3235 +#define FSL_QDMA_CMD_RWTTYPE 0x4
3236 +#define FSL_QDMA_CMD_LWC 0x2
3238 +#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28
3239 +#define FSL_QDMA_CMD_NS_OFFSET 27
3240 +#define FSL_QDMA_CMD_DQOS_OFFSET 24
3241 +#define FSL_QDMA_CMD_WTHROTL_OFFSET 20
3242 +#define FSL_QDMA_CMD_DSEN_OFFSET 19
3243 +#define FSL_QDMA_CMD_LWC_OFFSET 16
3245 +#define FSL_QDMA_E_SG_TABLE 1
3246 +#define FSL_QDMA_E_DATA_BUFFER 0
3247 +#define FSL_QDMA_F_LAST_ENTRY 1
3249 +u64 pre_addr, pre_queue;
3251 +/* qDMA Command Descriptor Fotmats */
3253 +/* Compound Command Descriptor Fotmat */
3254 +struct fsl_qdma_ccdf {
3255 + __le32 status; /* ser, status */
3256 + __le32 cfg; /* format, offset */
3259 + __le32 addr_lo; /* low 32-bits of 40-bit address */
3260 + u8 addr_hi; /* high 8-bits of 40-bit address */
3261 + u8 __reserved1[2];
3262 + u8 cfg8b_w1; /* dd, queue*/
3268 +#define QDMA_CCDF_STATUS 20
3269 +#define QDMA_CCDF_OFFSET 20
3270 +#define QDMA_CCDF_MASK GENMASK(28, 20)
3271 +#define QDMA_CCDF_FOTMAT BIT(29)
3272 +#define QDMA_CCDF_SER BIT(30)
3274 +static inline u64 qdma_ccdf_addr_get64(const struct fsl_qdma_ccdf *ccdf)
3276 + return le64_to_cpu(ccdf->data) & 0xffffffffffLLU;
3278 +static inline u64 qdma_ccdf_get_queue(const struct fsl_qdma_ccdf *ccdf)
3280 + return ccdf->cfg8b_w1 & 0xff;
3282 +static inline void qdma_ccdf_addr_set64(struct fsl_qdma_ccdf *ccdf, u64 addr)
3284 + ccdf->addr_hi = upper_32_bits(addr);
3285 + ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr));
3287 +static inline int qdma_ccdf_get_offset(const struct fsl_qdma_ccdf *ccdf)
3289 + return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET;
3291 +static inline void qdma_ccdf_set_format(struct fsl_qdma_ccdf *ccdf, int offset)
3293 + ccdf->cfg = cpu_to_le32(QDMA_CCDF_FOTMAT | offset);
3295 +static inline int qdma_ccdf_get_status(const struct fsl_qdma_ccdf *ccdf)
3297 + return (le32_to_cpu(ccdf->status) & QDMA_CCDF_MASK) >> QDMA_CCDF_STATUS;
3299 +static inline void qdma_ccdf_set_ser(struct fsl_qdma_ccdf *ccdf, int status)
3301 + ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status);
3303 +/* qDMA Compound S/G Format */
3304 +struct fsl_qdma_csgf {
3305 + __le32 offset; /* offset */
3306 + __le32 cfg; /* E bit, F bit, length */
3309 + __le32 addr_lo; /* low 32-bits of 40-bit address */
3310 + u8 addr_hi; /* high 8-bits of 40-bit address */
3311 + u8 __reserved1[3];
3317 +#define QDMA_SG_FIN BIT(30)
3318 +#define QDMA_SG_EXT BIT(31)
3319 +#define QDMA_SG_LEN_MASK GENMASK(29, 0)
3320 +static inline u64 qdma_csgf_addr_get64(const struct fsl_qdma_csgf *sg)
3322 + return be64_to_cpu(sg->data) & 0xffffffffffLLU;
3324 +static inline void qdma_csgf_addr_set64(struct fsl_qdma_csgf *sg, u64 addr)
3326 + sg->addr_hi = upper_32_bits(addr);
3327 + sg->addr_lo = cpu_to_le32(lower_32_bits(addr));
3329 +static inline void qdma_csgf_set_len(struct fsl_qdma_csgf *csgf, int len)
3331 + csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK);
3333 +static inline void qdma_csgf_set_f(struct fsl_qdma_csgf *csgf, int len)
3335 + csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));
3337 +static inline void qdma_csgf_set_e(struct fsl_qdma_csgf *csgf, int len)
3339 + csgf->cfg = cpu_to_le32(QDMA_SG_EXT | (len & QDMA_SG_LEN_MASK));
3342 +/* qDMA Source Descriptor Format */
3343 +struct fsl_qdma_sdf {
3345 + __le32 cfg; /* rev4, bit[0-11] - ssd, bit[12-23] sss */
3350 +/*qDMA Destination Descriptor Format*/
3351 +struct fsl_qdma_ddf {
3353 + __le32 cfg; /* rev2, bit[0-11] - dsd, bit[12-23] - dss */
3358 +struct fsl_qdma_chan {
3359 + struct virt_dma_chan vchan;
3360 + struct virt_dma_desc vdesc;
3361 + enum dma_status status;
3363 + struct fsl_qdma_engine *qdma;
3364 + struct fsl_qdma_queue *queue;
3365 + struct list_head qcomp;
3368 +struct fsl_qdma_queue {
3369 + struct fsl_qdma_ccdf *virt_head;
3370 + struct fsl_qdma_ccdf *virt_tail;
3371 + struct list_head comp_used;
3372 + struct list_head comp_free;
3373 + struct dma_pool *comp_pool;
3374 + struct dma_pool *sg_pool;
3375 + spinlock_t queue_lock;
3376 + dma_addr_t bus_addr;
3379 + struct fsl_qdma_ccdf *cq;
3382 +struct fsl_qdma_sg {
3383 + dma_addr_t bus_addr;
3387 +struct fsl_qdma_comp {
3388 + dma_addr_t bus_addr;
3390 + struct fsl_qdma_chan *qchan;
3391 + struct fsl_qdma_sg *sg_block;
3392 + struct virt_dma_desc vdesc;
3393 + struct list_head list;
3398 +struct fsl_qdma_engine {
3399 + struct dma_device dma_dev;
3400 + void __iomem *ctrl_base;
3401 + void __iomem *status_base;
3402 + void __iomem *block_base;
3405 + struct mutex fsl_qdma_mutex;
3409 + struct fsl_qdma_queue *queue;
3410 + struct fsl_qdma_queue *status;
3411 + struct fsl_qdma_chan chans[];
3415 +static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr)
3417 + if (qdma->big_endian)
3418 + return ioread32be(addr);
3420 + return ioread32(addr);
3423 +static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val,
3424 + void __iomem *addr)
3426 + if (qdma->big_endian)
3427 + iowrite32be(val, addr);
3429 + iowrite32(val, addr);
3432 +static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan)
3434 + return container_of(chan, struct fsl_qdma_chan, vchan.chan);
3437 +static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
3439 + return container_of(vd, struct fsl_qdma_comp, vdesc);
3442 +static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan)
3445 + * In QDMA mode, We don't need to do anything.
3450 +static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
3452 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3453 + unsigned long flags;
3456 + spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
3457 + vchan_get_all_descriptors(&fsl_chan->vchan, &head);
3458 + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
3460 + vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
3463 +static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
3464 + dma_addr_t dst, dma_addr_t src, u32 len)
3466 + struct fsl_qdma_ccdf *ccdf;
3467 + struct fsl_qdma_csgf *csgf_desc, *csgf_src, *csgf_dest;
3468 + struct fsl_qdma_sdf *sdf;
3469 + struct fsl_qdma_ddf *ddf;
3471 + ccdf = (struct fsl_qdma_ccdf *)fsl_comp->virt_addr;
3472 + csgf_desc = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 1;
3473 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 2;
3474 + csgf_dest = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 3;
3475 + sdf = (struct fsl_qdma_sdf *)fsl_comp->virt_addr + 4;
3476 + ddf = (struct fsl_qdma_ddf *)fsl_comp->virt_addr + 5;
3478 + memset(fsl_comp->virt_addr, 0, FSL_QDMA_BASE_BUFFER_SIZE);
3479 + /* Head Command Descriptor(Frame Descriptor) */
3480 + qdma_ccdf_addr_set64(ccdf, fsl_comp->bus_addr + 16);
3481 + qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf));
3482 + qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf));
3483 + /* Status notification is enqueued to status queue. */
3484 + /* Compound Command Descriptor(Frame List Table) */
3485 + qdma_csgf_addr_set64(csgf_desc, fsl_comp->bus_addr + 64);
3486 + /* It must be 32 as Compound S/G Descriptor */
3487 + qdma_csgf_set_len(csgf_desc, 32);
3488 + qdma_csgf_addr_set64(csgf_src, src);
3489 + qdma_csgf_set_len(csgf_src, len);
3490 + qdma_csgf_addr_set64(csgf_dest, dst);
3491 + qdma_csgf_set_len(csgf_dest, len);
3492 + /* This entry is the last entry. */
3493 + qdma_csgf_set_f(csgf_dest, len);
3494 + /* Descriptor Buffer */
3495 + sdf->cmd = cpu_to_le32(
3496 + FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET);
3497 + ddf->cmd = cpu_to_le32(
3498 + FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET);
3499 + ddf->cmd |= cpu_to_le32(
3500 + FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET);
3503 +static void fsl_qdma_comp_fill_sg(
3504 + struct fsl_qdma_comp *fsl_comp,
3505 + struct scatterlist *dst_sg, unsigned int dst_nents,
3506 + struct scatterlist *src_sg, unsigned int src_nents)
3508 + struct fsl_qdma_ccdf *ccdf;
3509 + struct fsl_qdma_csgf *csgf_desc, *csgf_src, *csgf_dest, *csgf_sg;
3510 + struct fsl_qdma_sdf *sdf;
3511 + struct fsl_qdma_ddf *ddf;
3512 + struct fsl_qdma_sg *sg_block, *temp;
3513 + struct scatterlist *sg;
3514 + u64 total_src_len = 0;
3515 + u64 total_dst_len = 0;
3518 + ccdf = (struct fsl_qdma_ccdf *)fsl_comp->virt_addr;
3519 + csgf_desc = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 1;
3520 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 2;
3521 + csgf_dest = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 3;
3522 + sdf = (struct fsl_qdma_sdf *)fsl_comp->virt_addr + 4;
3523 + ddf = (struct fsl_qdma_ddf *)fsl_comp->virt_addr + 5;
3524 + memset(fsl_comp->virt_addr, 0, FSL_QDMA_BASE_BUFFER_SIZE);
3525 + /* Head Command Descriptor(Frame Descriptor) */
3526 + qdma_ccdf_addr_set64(ccdf, fsl_comp->bus_addr + 16);
3527 + qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf));
3528 + /* Status notification is enqueued to status queue. */
3529 + qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf));
3531 + /* Compound Command Descriptor(Frame List Table) */
3532 + qdma_csgf_addr_set64(csgf_desc, fsl_comp->bus_addr + 64);
3533 + /* It must be 32 as Compound S/G Descriptor */
3534 + qdma_csgf_set_len(csgf_desc, 32);
3536 + sg_block = fsl_comp->sg_block;
3537 + qdma_csgf_addr_set64(csgf_src, sg_block->bus_addr);
3538 + /* This entry link to the s/g entry. */
3539 + qdma_csgf_set_e(csgf_src, 32);
3541 + temp = sg_block + fsl_comp->sg_block_src;
3542 + qdma_csgf_addr_set64(csgf_dest, temp->bus_addr);
3543 + /* This entry is the last entry. */
3544 + qdma_csgf_set_f(csgf_dest, 32);
3545 + /* This entry link to the s/g entry. */
3546 + qdma_csgf_set_e(csgf_dest, 32);
3548 + for_each_sg(src_sg, sg, src_nents, i) {
3549 + temp = sg_block + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
3550 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
3551 + i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
3552 + qdma_csgf_addr_set64(csgf_sg, sg_dma_address(sg));
3553 + qdma_csgf_set_len(csgf_sg, sg_dma_len(sg));
3554 + total_src_len += sg_dma_len(sg);
3556 + if (i == src_nents - 1)
3557 + qdma_csgf_set_f(csgf_sg, sg_dma_len(sg));
3558 + if (i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) ==
3559 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 2) {
3560 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
3561 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1;
3563 + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3564 + qdma_csgf_addr_set64(csgf_sg, temp->bus_addr);
3565 + qdma_csgf_set_e(csgf_sg, sg_dma_len(sg));
3569 + sg_block += fsl_comp->sg_block_src;
3570 + for_each_sg(dst_sg, sg, dst_nents, i) {
3571 + temp = sg_block + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
3572 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
3573 + i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
3574 + qdma_csgf_addr_set64(csgf_sg, sg_dma_address(sg));
3575 + qdma_csgf_set_len(csgf_sg, sg_dma_len(sg));
3576 + total_dst_len += sg_dma_len(sg);
3578 + if (i == dst_nents - 1)
3579 + qdma_csgf_set_f(csgf_sg, sg_dma_len(sg));
3580 + if (i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) ==
3581 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 2) {
3582 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
3583 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1;
3585 + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3586 + qdma_csgf_addr_set64(csgf_sg, temp->bus_addr);
3587 + qdma_csgf_set_e(csgf_sg, sg_dma_len(sg));
3591 + if (total_src_len != total_dst_len)
3592 + dev_err(&fsl_comp->qchan->vchan.chan.dev->device,
3593 + "The data length for src and dst isn't match.\n");
3595 + qdma_csgf_set_len(csgf_src, total_src_len);
3596 + qdma_csgf_set_len(csgf_dest, total_dst_len);
3598 + /* Descriptor Buffer */
3602 + * Prei-request full command descriptor for enqueue.
3604 +static int fsl_qdma_pre_request_enqueue_desc(struct fsl_qdma_queue *queue)
3606 + struct fsl_qdma_comp *comp_temp;
3609 + for (i = 0; i < queue->n_cq; i++) {
3610 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
3613 + comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool,
3615 + &comp_temp->bus_addr);
3616 + if (!comp_temp->virt_addr)
3618 + list_add_tail(&comp_temp->list, &queue->comp_free);
3624 + * Request a command descriptor for enqueue.
3626 +static struct fsl_qdma_comp *fsl_qdma_request_enqueue_desc(
3627 + struct fsl_qdma_chan *fsl_chan,
3628 + unsigned int dst_nents,
3629 + unsigned int src_nents)
3631 + struct fsl_qdma_comp *comp_temp;
3632 + struct fsl_qdma_sg *sg_block;
3633 + struct fsl_qdma_queue *queue = fsl_chan->queue;
3634 + unsigned long flags;
3635 + unsigned int dst_sg_entry_block, src_sg_entry_block, sg_entry_total, i;
3637 + spin_lock_irqsave(&queue->queue_lock, flags);
3638 + if (list_empty(&queue->comp_free)) {
3639 + spin_unlock_irqrestore(&queue->queue_lock, flags);
3640 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
3643 + comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool,
3645 + &comp_temp->bus_addr);
3646 + if (!comp_temp->virt_addr)
3649 + comp_temp = list_first_entry(&queue->comp_free,
3650 + struct fsl_qdma_comp,
3652 + list_del(&comp_temp->list);
3653 + spin_unlock_irqrestore(&queue->queue_lock, flags);
3656 + if (dst_nents != 0)
3657 + dst_sg_entry_block = dst_nents /
3658 + (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3660 + dst_sg_entry_block = 0;
3662 + if (src_nents != 0)
3663 + src_sg_entry_block = src_nents /
3664 + (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3666 + src_sg_entry_block = 0;
3668 + sg_entry_total = dst_sg_entry_block + src_sg_entry_block;
3669 + if (sg_entry_total) {
3670 + sg_block = kzalloc(sizeof(*sg_block) *
3675 + comp_temp->sg_block = sg_block;
3676 + for (i = 0; i < sg_entry_total; i++) {
3677 + sg_block->virt_addr = dma_pool_alloc(queue->sg_pool,
3679 + &sg_block->bus_addr);
3680 + memset(sg_block->virt_addr, 0,
3681 + FSL_QDMA_EXPECT_SG_ENTRY_NUM * 16);
3686 + comp_temp->sg_block_src = src_sg_entry_block;
3687 + comp_temp->sg_block_dst = dst_sg_entry_block;
3688 + comp_temp->qchan = fsl_chan;
3693 +static struct fsl_qdma_queue *fsl_qdma_alloc_queue_resources(
3694 + struct platform_device *pdev,
3695 + unsigned int queue_num)
3697 + struct device_node *np = pdev->dev.of_node;
3698 + struct fsl_qdma_queue *queue_head, *queue_temp;
3700 + unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
3702 + if (queue_num > FSL_QDMA_QUEUE_MAX)
3703 + queue_num = FSL_QDMA_QUEUE_MAX;
3704 + len = sizeof(*queue_head) * queue_num;
3705 + queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
3709 + ret = of_property_read_u32_array(np, "queue-sizes", queue_size,
3712 + dev_err(&pdev->dev, "Can't get queue-sizes.\n");
3716 + for (i = 0; i < queue_num; i++) {
3717 + if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX
3718 + || queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3719 + dev_err(&pdev->dev, "Get wrong queue-sizes.\n");
3722 + queue_temp = queue_head + i;
3723 + queue_temp->cq = dma_alloc_coherent(&pdev->dev,
3724 + sizeof(struct fsl_qdma_ccdf) *
3726 + &queue_temp->bus_addr,
3728 + if (!queue_temp->cq)
3730 + queue_temp->n_cq = queue_size[i];
3731 + queue_temp->id = i;
3732 + queue_temp->virt_head = queue_temp->cq;
3733 + queue_temp->virt_tail = queue_temp->cq;
3735 + * The dma pool for queue command buffer
3737 + queue_temp->comp_pool = dma_pool_create("comp_pool",
3739 + FSL_QDMA_BASE_BUFFER_SIZE,
3741 + if (!queue_temp->comp_pool) {
3742 + dma_free_coherent(&pdev->dev,
3743 + sizeof(struct fsl_qdma_ccdf) *
3746 + queue_temp->bus_addr);
3750 + * The dma pool for queue command buffer
3752 + queue_temp->sg_pool = dma_pool_create("sg_pool",
3754 + FSL_QDMA_EXPECT_SG_ENTRY_NUM * 16,
3756 + if (!queue_temp->sg_pool) {
3757 + dma_free_coherent(&pdev->dev,
3758 + sizeof(struct fsl_qdma_ccdf) *
3761 + queue_temp->bus_addr);
3762 + dma_pool_destroy(queue_temp->comp_pool);
3766 + * List for queue command buffer
3768 + INIT_LIST_HEAD(&queue_temp->comp_used);
3769 + INIT_LIST_HEAD(&queue_temp->comp_free);
3770 + spin_lock_init(&queue_temp->queue_lock);
3773 + return queue_head;
3776 +static struct fsl_qdma_queue *fsl_qdma_prep_status_queue(
3777 + struct platform_device *pdev)
3779 + struct device_node *np = pdev->dev.of_node;
3780 + struct fsl_qdma_queue *status_head;
3781 + unsigned int status_size;
3784 + ret = of_property_read_u32(np, "status-sizes", &status_size);
3786 + dev_err(&pdev->dev, "Can't get status-sizes.\n");
3789 + if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX
3790 + || status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3791 + dev_err(&pdev->dev, "Get wrong status_size.\n");
3794 + status_head = devm_kzalloc(&pdev->dev, sizeof(*status_head),
3800 + * Buffer for queue command
3802 + status_head->cq = dma_alloc_coherent(&pdev->dev,
3803 + sizeof(struct fsl_qdma_ccdf) *
3805 + &status_head->bus_addr,
3807 + if (!status_head->cq)
3809 + status_head->n_cq = status_size;
3810 + status_head->virt_head = status_head->cq;
3811 + status_head->virt_tail = status_head->cq;
3812 + status_head->comp_pool = NULL;
3814 + return status_head;
3817 +static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
3819 + void __iomem *ctrl = fsl_qdma->ctrl_base;
3820 + void __iomem *block = fsl_qdma->block_base;
3824 + /* Disable the command queue and wait for idle state. */
3825 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3826 + reg |= FSL_QDMA_DMR_DQD;
3827 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3828 + for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
3829 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i));
3832 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR);
3833 + if (!(reg & FSL_QDMA_DSR_DB))
3840 + /* Disable status queue. */
3841 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR);
3844 + * Clear the command queue interrupt detect register for all queues.
3846 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3851 +static int fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma)
3853 + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
3854 + struct fsl_qdma_queue *fsl_status = fsl_qdma->status;
3855 + struct fsl_qdma_queue *temp_queue;
3856 + struct fsl_qdma_comp *fsl_comp;
3857 + struct fsl_qdma_ccdf *status_addr;
3858 + struct fsl_qdma_csgf *csgf_src;
3859 + void __iomem *block = fsl_qdma->block_base;
3861 + bool duplicate, duplicate_handle;
3865 + duplicate_handle = 0;
3866 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR);
3867 + if (reg & FSL_QDMA_BSQSR_QE)
3869 + status_addr = fsl_status->virt_head;
3870 + if (qdma_ccdf_get_queue(status_addr) == pre_queue &&
3871 + qdma_ccdf_addr_get64(status_addr) == pre_addr)
3873 + i = qdma_ccdf_get_queue(status_addr);
3874 + pre_queue = qdma_ccdf_get_queue(status_addr);
3875 + pre_addr = qdma_ccdf_addr_get64(status_addr);
3876 + temp_queue = fsl_queue + i;
3877 + spin_lock(&temp_queue->queue_lock);
3878 + if (list_empty(&temp_queue->comp_used)) {
3880 + duplicate_handle = 1;
3882 + spin_unlock(&temp_queue->queue_lock);
3886 + fsl_comp = list_first_entry(&temp_queue->comp_used,
3887 + struct fsl_qdma_comp,
3889 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr
3891 + if (fsl_comp->bus_addr + 16 != pre_addr) {
3893 + duplicate_handle = 1;
3895 + spin_unlock(&temp_queue->queue_lock);
3901 + if (duplicate_handle) {
3902 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3903 + reg |= FSL_QDMA_BSQMR_DI;
3904 + qdma_ccdf_addr_set64(status_addr, 0x0);
3905 + fsl_status->virt_head++;
3906 + if (fsl_status->virt_head == fsl_status->cq
3907 + + fsl_status->n_cq)
3908 + fsl_status->virt_head = fsl_status->cq;
3909 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3910 + spin_unlock(&temp_queue->queue_lock);
3913 + list_del(&fsl_comp->list);
3915 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3916 + reg |= FSL_QDMA_BSQMR_DI;
3917 + qdma_ccdf_addr_set64(status_addr, 0x0);
3918 + fsl_status->virt_head++;
3919 + if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
3920 + fsl_status->virt_head = fsl_status->cq;
3921 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3922 + spin_unlock(&temp_queue->queue_lock);
3924 + spin_lock(&fsl_comp->qchan->vchan.lock);
3925 + vchan_cookie_complete(&fsl_comp->vdesc);
3926 + fsl_comp->qchan->status = DMA_COMPLETE;
3927 + spin_unlock(&fsl_comp->qchan->vchan.lock);
3932 +static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id)
3934 + struct fsl_qdma_engine *fsl_qdma = dev_id;
3935 + unsigned int intr;
3936 + void __iomem *status = fsl_qdma->status_base;
3938 + intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR);
3941 + dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n");
3943 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
3944 + return IRQ_HANDLED;
3947 +static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id)
3949 + struct fsl_qdma_engine *fsl_qdma = dev_id;
3950 + unsigned int intr, reg;
3951 + void __iomem *block = fsl_qdma->block_base;
3952 + void __iomem *ctrl = fsl_qdma->ctrl_base;
3954 + intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0));
3956 + if ((intr & FSL_QDMA_CQIDR_SQT) != 0)
3957 + intr = fsl_qdma_queue_transfer_complete(fsl_qdma);
3960 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3961 + reg |= FSL_QDMA_DMR_DQD;
3962 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3963 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0));
3964 + dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n");
3967 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3969 + return IRQ_HANDLED;
3973 +fsl_qdma_irq_init(struct platform_device *pdev,
3974 + struct fsl_qdma_engine *fsl_qdma)
3978 + fsl_qdma->error_irq = platform_get_irq_byname(pdev,
3980 + if (fsl_qdma->error_irq < 0) {
3981 + dev_err(&pdev->dev, "Can't get qdma controller irq.\n");
3982 + return fsl_qdma->error_irq;
3985 + fsl_qdma->queue_irq = platform_get_irq_byname(pdev, "qdma-queue");
3986 + if (fsl_qdma->queue_irq < 0) {
3987 + dev_err(&pdev->dev, "Can't get qdma queue irq.\n");
3988 + return fsl_qdma->queue_irq;
3991 + ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,
3992 + fsl_qdma_error_handler, 0, "qDMA error", fsl_qdma);
3994 + dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n");
3997 + ret = devm_request_irq(&pdev->dev, fsl_qdma->queue_irq,
3998 + fsl_qdma_queue_handler, 0, "qDMA queue", fsl_qdma);
4000 + dev_err(&pdev->dev, "Can't register qDMA queue IRQ.\n");
4007 +static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
4009 + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
4010 + struct fsl_qdma_queue *temp;
4011 + void __iomem *ctrl = fsl_qdma->ctrl_base;
4012 + void __iomem *status = fsl_qdma->status_base;
4013 + void __iomem *block = fsl_qdma->block_base;
4017 + /* Try to halt the qDMA engine first. */
4018 + ret = fsl_qdma_halt(fsl_qdma);
4020 + dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!");
4025 + * Clear the command queue interrupt detect register for all queues.
4027 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
4029 + for (i = 0; i < fsl_qdma->n_queues; i++) {
4030 + temp = fsl_queue + i;
4032 + * Initialize Command Queue registers to point to the first
4033 + * command descriptor in memory.
4034 + * Dequeue Pointer Address Registers
4035 + * Enqueue Pointer Address Registers
4037 + qdma_writel(fsl_qdma, temp->bus_addr,
4038 + block + FSL_QDMA_BCQDPA_SADDR(i));
4039 + qdma_writel(fsl_qdma, temp->bus_addr,
4040 + block + FSL_QDMA_BCQEPA_SADDR(i));
4042 + /* Initialize the queue mode. */
4043 + reg = FSL_QDMA_BCQMR_EN;
4044 + reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq)-4);
4045 + reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq)-6);
4046 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i));
4050 + * Workaround for erratum: ERR010812.
4051 + * We must enable XOFF to avoid the enqueue rejection occurs.
4052 + * Setting SQCCMR ENTER_WM to 0x20.
4054 + qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM,
4055 + block + FSL_QDMA_SQCCMR);
4057 + * Initialize status queue registers to point to the first
4058 + * command descriptor in memory.
4059 + * Dequeue Pointer Address Registers
4060 + * Enqueue Pointer Address Registers
4062 + qdma_writel(fsl_qdma, fsl_qdma->status->bus_addr,
4063 + block + FSL_QDMA_SQEPAR);
4064 + qdma_writel(fsl_qdma, fsl_qdma->status->bus_addr,
4065 + block + FSL_QDMA_SQDPAR);
4066 + /* Initialize status queue interrupt. */
4067 + qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE,
4068 + block + FSL_QDMA_BCQIER(0));
4069 + qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN | FSL_QDMA_BSQICR_ICST(5)
4071 + block + FSL_QDMA_BSQICR);
4072 + qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE | FSL_QDMA_CQIER_TEIE,
4073 + block + FSL_QDMA_CQIER);
4074 + /* Initialize controller interrupt register. */
4075 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
4076 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEIER);
4078 + /* Initialize the status queue mode. */
4079 + reg = FSL_QDMA_BSQMR_EN;
4080 + reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2(fsl_qdma->status->n_cq)-6);
4081 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
4083 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
4084 + reg &= ~FSL_QDMA_DMR_DQD;
4085 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
4090 +static struct dma_async_tx_descriptor *fsl_qdma_prep_dma_sg(
4091 + struct dma_chan *chan,
4092 + struct scatterlist *dst_sg, unsigned int dst_nents,
4093 + struct scatterlist *src_sg, unsigned int src_nents,
4094 + unsigned long flags)
4096 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
4097 + struct fsl_qdma_comp *fsl_comp;
4099 + fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan,
4102 + fsl_qdma_comp_fill_sg(fsl_comp, dst_sg, dst_nents, src_sg, src_nents);
4104 + return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
4107 +static struct dma_async_tx_descriptor *
4108 +fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
4109 + dma_addr_t src, size_t len, unsigned long flags)
4111 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
4112 + struct fsl_qdma_comp *fsl_comp;
4114 + fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan, 0, 0);
4115 + fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
4117 + return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
4120 +static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
4122 + void __iomem *block = fsl_chan->qdma->block_base;
4123 + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
4124 + struct fsl_qdma_comp *fsl_comp;
4125 + struct virt_dma_desc *vdesc;
4128 + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id));
4129 + if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF))
4131 + vdesc = vchan_next_desc(&fsl_chan->vchan);
4134 + list_del(&vdesc->node);
4135 + fsl_comp = to_fsl_qdma_comp(vdesc);
4137 + memcpy(fsl_queue->virt_head++, fsl_comp->virt_addr, 16);
4138 + if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
4139 + fsl_queue->virt_head = fsl_queue->cq;
4141 + list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
4143 + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id));
4144 + reg |= FSL_QDMA_BCQMR_EI;
4145 + qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
4146 + fsl_chan->status = DMA_IN_PROGRESS;
4149 +static enum dma_status fsl_qdma_tx_status(struct dma_chan *chan,
4150 + dma_cookie_t cookie, struct dma_tx_state *txstate)
4152 + return dma_cookie_status(chan, cookie, txstate);
4155 +static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc)
4157 + struct fsl_qdma_comp *fsl_comp;
4158 + struct fsl_qdma_queue *fsl_queue;
4159 + struct fsl_qdma_sg *sg_block;
4160 + unsigned long flags;
4163 + fsl_comp = to_fsl_qdma_comp(vdesc);
4164 + fsl_queue = fsl_comp->qchan->queue;
4166 + if (fsl_comp->sg_block) {
4167 + for (i = 0; i < fsl_comp->sg_block_src +
4168 + fsl_comp->sg_block_dst; i++) {
4169 + sg_block = fsl_comp->sg_block + i;
4170 + dma_pool_free(fsl_queue->sg_pool,
4171 + sg_block->virt_addr,
4172 + sg_block->bus_addr);
4174 + kfree(fsl_comp->sg_block);
4177 + spin_lock_irqsave(&fsl_queue->queue_lock, flags);
4178 + list_add_tail(&fsl_comp->list, &fsl_queue->comp_free);
4179 + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
4182 +static void fsl_qdma_issue_pending(struct dma_chan *chan)
4184 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
4185 + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
4186 + unsigned long flags;
4188 + spin_lock_irqsave(&fsl_queue->queue_lock, flags);
4189 + spin_lock(&fsl_chan->vchan.lock);
4190 + if (vchan_issue_pending(&fsl_chan->vchan))
4191 + fsl_qdma_enqueue_desc(fsl_chan);
4192 + spin_unlock(&fsl_chan->vchan.lock);
4193 + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
4196 +static int fsl_qdma_probe(struct platform_device *pdev)
4198 + struct device_node *np = pdev->dev.of_node;
4199 + struct fsl_qdma_engine *fsl_qdma;
4200 + struct fsl_qdma_chan *fsl_chan;
4201 + struct resource *res;
4202 + unsigned int len, chans, queues;
4205 + ret = of_property_read_u32(np, "channels", &chans);
4207 + dev_err(&pdev->dev, "Can't get channels.\n");
4211 + len = sizeof(*fsl_qdma) + sizeof(*fsl_chan) * chans;
4212 + fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
4216 + ret = of_property_read_u32(np, "queues", &queues);
4218 + dev_err(&pdev->dev, "Can't get queues.\n");
4222 + fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, queues);
4223 + if (!fsl_qdma->queue)
4226 + fsl_qdma->status = fsl_qdma_prep_status_queue(pdev);
4227 + if (!fsl_qdma->status)
4230 + fsl_qdma->n_chans = chans;
4231 + fsl_qdma->n_queues = queues;
4232 + mutex_init(&fsl_qdma->fsl_qdma_mutex);
4234 + res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4235 + fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res);
4236 + if (IS_ERR(fsl_qdma->ctrl_base))
4237 + return PTR_ERR(fsl_qdma->ctrl_base);
4239 + res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
4240 + fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res);
4241 + if (IS_ERR(fsl_qdma->status_base))
4242 + return PTR_ERR(fsl_qdma->status_base);
4244 + res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
4245 + fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res);
4246 + if (IS_ERR(fsl_qdma->block_base))
4247 + return PTR_ERR(fsl_qdma->block_base);
4249 + ret = fsl_qdma_irq_init(pdev, fsl_qdma);
4253 + fsl_qdma->big_endian = of_property_read_bool(np, "big-endian");
4254 + INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels);
4255 + for (i = 0; i < fsl_qdma->n_chans; i++) {
4256 + struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
4258 + fsl_chan->qdma = fsl_qdma;
4259 + fsl_chan->queue = fsl_qdma->queue + i % fsl_qdma->n_queues;
4260 + fsl_chan->vchan.desc_free = fsl_qdma_free_desc;
4261 + INIT_LIST_HEAD(&fsl_chan->qcomp);
4262 + vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev);
4264 + for (i = 0; i < fsl_qdma->n_queues; i++)
4265 + fsl_qdma_pre_request_enqueue_desc(fsl_qdma->queue + i);
4267 + dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask);
4268 + dma_cap_set(DMA_SG, fsl_qdma->dma_dev.cap_mask);
4270 + fsl_qdma->dma_dev.dev = &pdev->dev;
4271 + fsl_qdma->dma_dev.device_alloc_chan_resources
4272 + = fsl_qdma_alloc_chan_resources;
4273 + fsl_qdma->dma_dev.device_free_chan_resources
4274 + = fsl_qdma_free_chan_resources;
4275 + fsl_qdma->dma_dev.device_tx_status = fsl_qdma_tx_status;
4276 + fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy;
4277 + fsl_qdma->dma_dev.device_prep_dma_sg = fsl_qdma_prep_dma_sg;
4278 + fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending;
4280 + dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
4282 + platform_set_drvdata(pdev, fsl_qdma);
4284 + ret = dma_async_device_register(&fsl_qdma->dma_dev);
4286 + dev_err(&pdev->dev, "Can't register Freescale qDMA engine.\n");
4290 + ret = fsl_qdma_reg_init(fsl_qdma);
4292 + dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");
4300 +static int fsl_qdma_remove(struct platform_device *pdev)
4302 + struct device_node *np = pdev->dev.of_node;
4303 + struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev);
4304 + struct fsl_qdma_queue *queue_temp;
4305 + struct fsl_qdma_queue *status = fsl_qdma->status;
4306 + struct fsl_qdma_comp *comp_temp, *_comp_temp;
4309 + of_dma_controller_free(np);
4310 + dma_async_device_unregister(&fsl_qdma->dma_dev);
4312 + /* Free descriptor areas */
4313 + for (i = 0; i < fsl_qdma->n_queues; i++) {
4314 + queue_temp = fsl_qdma->queue + i;
4315 + list_for_each_entry_safe(comp_temp, _comp_temp,
4316 + &queue_temp->comp_used, list) {
4317 + dma_pool_free(queue_temp->comp_pool,
4318 + comp_temp->virt_addr,
4319 + comp_temp->bus_addr);
4320 + list_del(&comp_temp->list);
4323 + list_for_each_entry_safe(comp_temp, _comp_temp,
4324 + &queue_temp->comp_free, list) {
4325 + dma_pool_free(queue_temp->comp_pool,
4326 + comp_temp->virt_addr,
4327 + comp_temp->bus_addr);
4328 + list_del(&comp_temp->list);
4331 + dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_ccdf) *
4332 + queue_temp->n_cq, queue_temp->cq,
4333 + queue_temp->bus_addr);
4334 + dma_pool_destroy(queue_temp->comp_pool);
4337 + dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_ccdf) *
4338 + status->n_cq, status->cq, status->bus_addr);
4342 +static const struct of_device_id fsl_qdma_dt_ids[] = {
4343 + { .compatible = "fsl,ls1021a-qdma", },
4344 + { /* sentinel */ }
4346 +MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids);
4348 +static struct platform_driver fsl_qdma_driver = {
4350 + .name = "fsl-qdma",
4351 + .owner = THIS_MODULE,
4352 + .of_match_table = fsl_qdma_dt_ids,
4354 + .probe = fsl_qdma_probe,
4355 + .remove = fsl_qdma_remove,
4358 +static int __init fsl_qdma_init(void)
4360 + return platform_driver_register(&fsl_qdma_driver);
4362 +subsys_initcall(fsl_qdma_init);
4364 +static void __exit fsl_qdma_exit(void)
4366 + platform_driver_unregister(&fsl_qdma_driver);
4368 +module_exit(fsl_qdma_exit);
4370 +MODULE_ALIAS("platform:fsl-qdma");
4371 +MODULE_DESCRIPTION("Freescale qDMA engine driver");
4372 +MODULE_LICENSE("GPL v2");