1 From 27aa9f97887f599267c345075e61979de785c770 Mon Sep 17 00:00:00 2001
2 From: Peng Ma <peng.ma@nxp.com>
3 Date: Thu, 11 Oct 2018 16:49:41 +0800
4 Subject: [PATCH] dma: caam: add dma memcpy driver
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
9 This module introduces a memcpy DMA driver based on the DMA capabilities
10 of the CAAM hardware block. CAAM DMA is a platform driver that is only
11 probed if the device is defined in the device tree. The driver creates
12 a DMA channel for each JR of the CAAM. This introduces a dependency on
13 the JR driver. Therefore a defering mechanism was used to ensure that
14 the CAAM DMA driver is probed only after the JR driver.
16 Signed-off-by: Radu Alexe <radu.alexe@nxp.com>
17 Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com>
18 Signed-off-by: Rajiv Vishwakarma <rajiv.vishwakarma@nxp.com>
19 Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
21 Signed-off-by: Yangbo Lu <yangbo.lu@nxp.com>
23 drivers/dma/Kconfig | 19 +-
24 drivers/dma/Makefile | 1 +
25 drivers/dma/caam_dma.c | 462 +++++++++++++++++++++++++++++++++++++++++++++++++
26 3 files changed, 481 insertions(+), 1 deletion(-)
27 create mode 100644 drivers/dma/caam_dma.c
29 --- a/drivers/dma/Kconfig
30 +++ b/drivers/dma/Kconfig
31 @@ -132,6 +132,24 @@ config COH901318
33 Enable support for ST-Ericsson COH 901 318 DMA.
35 +config CRYPTO_DEV_FSL_CAAM_DMA
36 + tristate "CAAM DMA engine support"
37 + depends on CRYPTO_DEV_FSL_CAAM_JR
41 + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
43 + Selecting this will offload the DMA operations for users of
44 + the scatter gather memcopy API to the CAAM via job rings. The
45 + CAAM is a hardware module that provides hardware acceleration to
46 + cryptographic operations. It has a built-in DMA controller that can
47 + be programmed to read/write cryptographic data. This module defines
48 + a DMA driver that uses the DMA capabilities of the CAAM.
50 + To compile this as a module, choose M here: the module
51 + will be called caam_dma.
54 tristate "BCM2835 DMA engine support"
55 depends on ARCH_BCM2835
56 @@ -663,7 +681,6 @@ config ZX_DMA
58 Support the DMA engine for ZTE ZX family platform devices.
62 source "drivers/dma/bestcomm/Kconfig"
64 --- a/drivers/dma/Makefile
65 +++ b/drivers/dma/Makefile
66 @@ -77,6 +77,7 @@ obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
67 obj-$(CONFIG_ZX_DMA) += zx_dma.o
68 obj-$(CONFIG_ST_FDMA) += st_fdma.o
69 obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/
70 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o
75 +++ b/drivers/dma/caam_dma.c
78 + * caam support for SG DMA
80 + * Copyright 2016 Freescale Semiconductor, Inc
81 + * Copyright 2017 NXP
83 + * Redistribution and use in source and binary forms, with or without
84 + * modification, are permitted provided that the following conditions are met:
85 + * * Redistributions of source code must retain the above copyright
86 + * notice, this list of conditions and the following disclaimer.
87 + * * Redistributions in binary form must reproduce the above copyright
88 + * notice, this list of conditions and the following disclaimer in the
89 + * documentation and/or other materials provided with the distribution.
90 + * * Neither the names of the above-listed copyright holders nor the
91 + * names of any contributors may be used to endorse or promote products
92 + * derived from this software without specific prior written permission.
95 + * ALTERNATIVELY, this software may be distributed under the terms of the
96 + * GNU General Public License ("GPL") as published by the Free Software
97 + * Foundation, either version 2 of that License or (at your option) any
100 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
101 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
102 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
103 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
104 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
105 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
106 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
107 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
108 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
109 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
110 + * POSSIBILITY OF SUCH DAMAGE.
113 +#include <linux/dma-mapping.h>
114 +#include <linux/dmaengine.h>
115 +#include <linux/module.h>
116 +#include <linux/platform_device.h>
117 +#include <linux/slab.h>
119 +#include "dmaengine.h"
121 +#include "../crypto/caam/regs.h"
122 +#include "../crypto/caam/jr.h"
123 +#include "../crypto/caam/error.h"
124 +#include "../crypto/caam/desc_constr.h"
126 +#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \
130 + * This is max chunk size of a DMA transfer. If a buffer is larger than this
131 + * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes
132 + * and for each chunk a DMA transfer request is issued.
133 + * This value is the largest number on 16 bits that is a multiple of 256 bytes
134 + * (the largest configurable CAAM DMA burst size).
136 +#define CAAM_DMA_CHUNK_SIZE 65280
138 +struct caam_dma_sh_desc {
139 + u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned;
140 + dma_addr_t desc_dma;
143 +/* caam dma extended descriptor */
144 +struct caam_dma_edesc {
145 + struct dma_async_tx_descriptor async_tx;
146 + struct list_head node;
147 + struct caam_dma_ctx *ctx;
148 + dma_addr_t src_dma;
149 + dma_addr_t dst_dma;
150 + unsigned int src_len;
151 + unsigned int dst_len;
152 + u32 jd[] ____cacheline_aligned;
156 + * caam_dma_ctx - per jr/channel context
157 + * @chan: dma channel used by async_tx API
158 + * @node: list_head used to attach to the global dma_ctx_list
159 + * @jrdev: Job Ring device
160 + * @pending_q: queue of pending (submitted, but not enqueued) jobs
161 + * @done_not_acked: jobs that have been completed by jr, but maybe not acked
162 + * @edesc_lock: protects extended descriptor
164 +struct caam_dma_ctx {
165 + struct dma_chan chan;
166 + struct list_head node;
167 + struct device *jrdev;
168 + struct list_head pending_q;
169 + struct list_head done_not_acked;
170 + spinlock_t edesc_lock;
173 +static struct dma_device *dma_dev;
174 +static struct caam_dma_sh_desc *dma_sh_desc;
175 +static LIST_HEAD(dma_ctx_list);
177 +static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx)
179 + struct caam_dma_edesc *edesc = NULL;
180 + struct caam_dma_ctx *ctx = NULL;
181 + dma_cookie_t cookie;
183 + edesc = container_of(tx, struct caam_dma_edesc, async_tx);
184 + ctx = container_of(tx->chan, struct caam_dma_ctx, chan);
186 + spin_lock_bh(&ctx->edesc_lock);
188 + cookie = dma_cookie_assign(tx);
189 + list_add_tail(&edesc->node, &ctx->pending_q);
191 + spin_unlock_bh(&ctx->edesc_lock);
196 +static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc)
198 + struct caam_dma_ctx *ctx = edesc->ctx;
199 + struct caam_dma_edesc *_edesc = NULL;
201 + spin_lock_bh(&ctx->edesc_lock);
203 + list_add_tail(&edesc->node, &ctx->done_not_acked);
204 + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
205 + if (async_tx_test_ack(&edesc->async_tx)) {
206 + list_del(&edesc->node);
211 + spin_unlock_bh(&ctx->edesc_lock);
214 +static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err,
217 + struct caam_dma_edesc *edesc = context;
218 + struct caam_dma_ctx *ctx = edesc->ctx;
219 + dma_async_tx_callback callback;
220 + void *callback_param;
223 + caam_jr_strstatus(ctx->jrdev, err);
225 + dma_run_dependencies(&edesc->async_tx);
227 + spin_lock_bh(&ctx->edesc_lock);
228 + dma_cookie_complete(&edesc->async_tx);
229 + spin_unlock_bh(&ctx->edesc_lock);
231 + callback = edesc->async_tx.callback;
232 + callback_param = edesc->async_tx.callback_param;
234 + dma_descriptor_unmap(&edesc->async_tx);
236 + caam_jr_chan_free_edesc(edesc);
239 + callback(callback_param);
242 +static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc)
244 + u32 *jd = edesc->jd;
245 + u32 *sh_desc = dma_sh_desc->desc;
246 + dma_addr_t desc_dma = dma_sh_desc->desc_dma;
248 + /* init the job descriptor */
249 + init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE);
251 + /* set SEQIN PTR */
252 + append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0);
254 + /* set SEQOUT PTR */
255 + append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0);
257 + print_hex_dump_debug("caam dma desc@" __stringify(__LINE__) ": ",
258 + DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1);
261 +static struct dma_async_tx_descriptor *
262 +caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
263 + size_t len, unsigned long flags)
265 + struct caam_dma_edesc *edesc;
266 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
269 + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT);
271 + return ERR_PTR(-ENOMEM);
273 + dma_async_tx_descriptor_init(&edesc->async_tx, chan);
274 + edesc->async_tx.tx_submit = caam_dma_tx_submit;
275 + edesc->async_tx.flags = flags;
276 + edesc->async_tx.cookie = -EBUSY;
278 + edesc->src_dma = src;
279 + edesc->src_len = len;
280 + edesc->dst_dma = dst;
281 + edesc->dst_len = len;
284 + caam_dma_memcpy_init_job_desc(edesc);
286 + return &edesc->async_tx;
289 +/* This function can be called in an interrupt context */
290 +static void caam_dma_issue_pending(struct dma_chan *chan)
292 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
294 + struct caam_dma_edesc *edesc, *_edesc;
296 + spin_lock_bh(&ctx->edesc_lock);
297 + list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) {
298 + if (caam_jr_enqueue(ctx->jrdev, edesc->jd,
299 + caam_dma_done, edesc) < 0)
301 + list_del(&edesc->node);
303 + spin_unlock_bh(&ctx->edesc_lock);
306 +static void caam_dma_free_chan_resources(struct dma_chan *chan)
308 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
310 + struct caam_dma_edesc *edesc, *_edesc;
312 + spin_lock_bh(&ctx->edesc_lock);
313 + list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) {
314 + list_del(&edesc->node);
317 + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
318 + list_del(&edesc->node);
321 + spin_unlock_bh(&ctx->edesc_lock);
324 +static int caam_dma_jr_chan_bind(void)
326 + struct device *jrdev;
327 + struct caam_dma_ctx *ctx;
331 + for (i = 0; i < caam_jr_driver_probed(); i++) {
332 + jrdev = caam_jridx_alloc(i);
333 + if (IS_ERR(jrdev)) {
334 + pr_err("job ring device %d allocation failed\n", i);
338 + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
340 + caam_jr_free(jrdev);
344 + ctx->chan.device = dma_dev;
345 + ctx->chan.private = ctx;
347 + ctx->jrdev = jrdev;
349 + INIT_LIST_HEAD(&ctx->pending_q);
350 + INIT_LIST_HEAD(&ctx->done_not_acked);
351 + INIT_LIST_HEAD(&ctx->node);
352 + spin_lock_init(&ctx->edesc_lock);
354 + dma_cookie_init(&ctx->chan);
356 + /* add the context of this channel to the context list */
357 + list_add_tail(&ctx->node, &dma_ctx_list);
359 + /* add this channel to the device chan list */
360 + list_add_tail(&ctx->chan.device_node, &dma_dev->channels);
368 +static inline void caam_jr_dma_free(struct dma_chan *chan)
370 + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
373 + list_del(&ctx->node);
374 + list_del(&chan->device_node);
375 + caam_jr_free(ctx->jrdev);
379 +static void set_caam_dma_desc(u32 *desc)
383 + /* dma shared descriptor */
384 + init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT));
386 + /* REG1 = CAAM_DMA_CHUNK_SIZE */
387 + append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE);
389 + /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */
390 + append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE);
396 + jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
399 + /* REG1 = SEQINLEN */
400 + append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ);
403 + set_jump_tgt_here(desc, jmp_cmd);
405 + /* VARSEQINLEN = REG1 */
406 + append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ);
408 + /* VARSEQOUTLEN = REG1 */
409 + append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ);
411 + /* do FIFO STORE */
412 + append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF);
415 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
416 + FIFOLD_TYPE_IFIFO | LDST_VLF);
420 + * jmp 0xF8 (after shared desc header)
422 + append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
423 + JUMP_COND_MATH_Z | 0xF8);
425 + print_hex_dump_debug("caam dma shdesc@" __stringify(__LINE__) ": ",
426 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
430 +static int caam_dma_probe(struct platform_device *pdev)
432 + struct device *dev = &pdev->dev;
433 + struct device *ctrldev = dev->parent;
434 + struct dma_chan *chan, *_chan;
439 + if (!caam_jr_driver_probed()) {
440 + dev_info(dev, "Defer probing after JR driver probing\n");
441 + return -EPROBE_DEFER;
444 + dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL);
448 + dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA);
452 + sh_desc = dma_sh_desc->desc;
453 + set_caam_dma_desc(sh_desc);
454 + dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc,
455 + desc_bytes(sh_desc),
457 + if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) {
458 + dev_err(dev, "unable to map dma descriptor\n");
462 + INIT_LIST_HEAD(&dma_dev->channels);
464 + bonds = caam_dma_jr_chan_bind();
470 + dma_dev->dev = dev;
471 + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
472 + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
473 + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
474 + dma_dev->device_tx_status = dma_cookie_status;
475 + dma_dev->device_issue_pending = caam_dma_issue_pending;
476 + dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy;
477 + dma_dev->device_free_chan_resources = caam_dma_free_chan_resources;
479 + err = dma_async_device_register(dma_dev);
481 + dev_err(dev, "Failed to register CAAM DMA engine\n");
485 + dev_info(dev, "caam dma support with %d job rings\n", bonds);
490 + list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node)
491 + caam_jr_dma_free(chan);
493 + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc),
496 + kfree(dma_sh_desc);
502 +static int caam_dma_remove(struct platform_device *pdev)
504 + struct device *dev = &pdev->dev;
505 + struct device *ctrldev = dev->parent;
506 + struct caam_dma_ctx *ctx, *_ctx;
508 + dma_async_device_unregister(dma_dev);
510 + list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) {
511 + list_del(&ctx->node);
512 + caam_jr_free(ctx->jrdev);
516 + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma,
517 + desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE);
519 + kfree(dma_sh_desc);
522 + dev_info(dev, "caam dma support disabled\n");
526 +static struct platform_driver caam_dma_driver = {
528 + .name = "caam-dma",
530 + .probe = caam_dma_probe,
531 + .remove = caam_dma_remove,
533 +module_platform_driver(caam_dma_driver);
535 +MODULE_LICENSE("Dual BSD/GPL");
536 +MODULE_DESCRIPTION("NXP CAAM support for DMA engine");
537 +MODULE_AUTHOR("NXP Semiconductors");
538 +MODULE_ALIAS("platform:caam-dma");