1 From 8322bafdcee1d7eaf15540ff013415bff1eacb28 Mon Sep 17 00:00:00 2001
2 From: Andy Gross <agross@codeaurora.org>
3 Date: Thu, 26 Jun 2014 10:50:24 -0500
4 Subject: [PATCH 179/182] spi: qup: Add DMA capabilities
6 This patch adds DMA capabilities to the spi-qup driver. If DMA channels are
7 present, the QUP will use DMA instead of block mode for transfers to/from SPI
8 peripherals for transactions larger greater than the length of a block.
10 Signed-off-by: Andy Gross <agross@codeaurora.org>
12 drivers/spi/spi-qup.c | 361 ++++++++++++++++++++++++++++++++++++++++++++++---
13 1 file changed, 340 insertions(+), 21 deletions(-)
15 --- a/drivers/spi/spi-qup.c
16 +++ b/drivers/spi/spi-qup.c
18 #include <linux/platform_device.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/spi/spi.h>
21 +#include <linux/dmaengine.h>
22 +#include <linux/dma-mapping.h>
24 #define QUP_CONFIG 0x0000
25 #define QUP_STATE 0x0004
28 #define SPI_NUM_CHIPSELECTS 4
30 +#define SPI_MAX_XFER (SZ_64K - 64)
32 /* high speed mode is when bus rate is greater then 26MHz */
33 #define SPI_HS_MIN_RATE 26000000
34 #define SPI_MAX_RATE 50000000
35 @@ -143,6 +147,14 @@ struct spi_qup {
41 + struct dma_chan *rx_chan;
42 + struct dma_slave_config rx_conf;
43 + struct dma_chan *tx_chan;
44 + struct dma_slave_config tx_conf;
46 + atomic_t dma_outstanding;
50 @@ -266,6 +278,221 @@ static void spi_qup_fifo_write(struct sp
54 +static void qup_dma_callback(void *data)
56 + struct spi_qup *controller = data;
58 + if (atomic_dec_and_test(&controller->dma_outstanding))
59 + complete(&controller->done);
62 +static int spi_qup_do_dma(struct spi_qup *controller, struct spi_transfer *xfer)
64 + struct dma_async_tx_descriptor *rxd, *txd;
65 + dma_cookie_t rx_cookie, tx_cookie;
66 + u32 xfer_len, rx_align = 0, tx_align = 0, n_words;
67 + struct scatterlist tx_sg[2], rx_sg[2];
69 + u32 bytes_to_xfer = xfer->len;
71 + u32 rx_nents = 0, tx_nents = 0;
72 + dma_addr_t rx_dma = 0, tx_dma = 0, rx_dummy_dma = 0, tx_dummy_dma = 0;
76 + rx_dma = dma_map_single(controller->dev, xfer->rx_buf,
77 + xfer->len, DMA_FROM_DEVICE);
79 + if (dma_mapping_error(controller->dev, rx_dma)) {
84 + /* check to see if we need dummy buffer for leftover bytes */
85 + rx_align = xfer->len % controller->in_blk_sz;
87 + rx_dummy_dma = dma_map_single(controller->dev,
88 + controller->dummy, controller->in_fifo_sz,
91 + if (dma_mapping_error(controller->dev, rx_dummy_dma)) {
93 + goto err_map_rx_dummy;
99 + tx_dma = dma_map_single(controller->dev,
100 + (void *)xfer->tx_buf, xfer->len, DMA_TO_DEVICE);
102 + if (dma_mapping_error(controller->dev, tx_dma)) {
107 + /* check to see if we need dummy buffer for leftover bytes */
108 + tx_align = xfer->len % controller->out_blk_sz;
110 + memcpy(controller->dummy + SZ_1K,
111 + xfer->tx_buf + xfer->len - tx_align,
113 + memset(controller->dummy + SZ_1K + tx_align, 0,
114 + controller->out_blk_sz - tx_align);
116 + tx_dummy_dma = dma_map_single(controller->dev,
117 + controller->dummy + SZ_1K,
118 + controller->out_blk_sz, DMA_TO_DEVICE);
120 + if (dma_mapping_error(controller->dev, tx_dummy_dma)) {
122 + goto err_map_tx_dummy;
127 + atomic_set(&controller->dma_outstanding, 0);
129 + while (bytes_to_xfer > 0) {
130 + xfer_len = min_t(u32, bytes_to_xfer, SPI_MAX_XFER);
131 + n_words = DIV_ROUND_UP(xfer_len, controller->w_size);
133 + /* write out current word count to controller */
134 + writel_relaxed(n_words, controller->base + QUP_MX_INPUT_CNT);
135 + writel_relaxed(n_words, controller->base + QUP_MX_OUTPUT_CNT);
137 + reinit_completion(&controller->done);
139 + if (xfer->tx_buf) {
140 + /* recalc align for each transaction */
141 + tx_align = xfer_len % controller->out_blk_sz;
148 + /* initialize scatterlists */
149 + sg_init_table(tx_sg, tx_nents);
150 + sg_dma_len(&tx_sg[0]) = xfer_len - tx_align;
151 + sg_dma_address(&tx_sg[0]) = tx_dma + offset;
153 + /* account for non block size transfer */
155 + sg_dma_len(&tx_sg[1]) = controller->out_blk_sz;
156 + sg_dma_address(&tx_sg[1]) = tx_dummy_dma;
159 + txd = dmaengine_prep_slave_sg(controller->tx_chan,
160 + tx_sg, tx_nents, DMA_MEM_TO_DEV, 0);
166 + atomic_inc(&controller->dma_outstanding);
168 + txd->callback = qup_dma_callback;
169 + txd->callback_param = controller;
171 + tx_cookie = dmaengine_submit(txd);
173 + dma_async_issue_pending(controller->tx_chan);
176 + if (xfer->rx_buf) {
177 + /* recalc align for each transaction */
178 + rx_align = xfer_len % controller->in_blk_sz;
185 + /* initialize scatterlists */
186 + sg_init_table(rx_sg, rx_nents);
187 + sg_dma_address(&rx_sg[0]) = rx_dma + offset;
188 + sg_dma_len(&rx_sg[0]) = xfer_len - rx_align;
190 + /* account for non block size transfer */
192 + sg_dma_len(&rx_sg[1]) = controller->in_blk_sz;
193 + sg_dma_address(&rx_sg[1]) = rx_dummy_dma;
196 + rxd = dmaengine_prep_slave_sg(controller->rx_chan,
197 + rx_sg, rx_nents, DMA_DEV_TO_MEM, 0);
203 + atomic_inc(&controller->dma_outstanding);
205 + rxd->callback = qup_dma_callback;
206 + rxd->callback_param = controller;
208 + rx_cookie = dmaengine_submit(rxd);
210 + dma_async_issue_pending(controller->rx_chan);
213 + if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
214 + dev_warn(controller->dev, "cannot set EXECUTE state\n");
218 + if (!wait_for_completion_timeout(&controller->done,
219 + msecs_to_jiffies(1000))) {
222 + /* clear out all the DMA transactions */
224 + dmaengine_terminate_all(controller->tx_chan);
226 + dmaengine_terminate_all(controller->rx_chan);
232 + memcpy(xfer->rx_buf + offset + xfer->len - rx_align,
233 + controller->dummy, rx_align);
235 + /* adjust remaining bytes to transfer */
236 + bytes_to_xfer -= xfer_len;
237 + offset += xfer_len;
240 + /* reset mini-core state so we can program next transaction */
241 + if (spi_qup_set_state(controller, QUP_STATE_RESET)) {
242 + dev_err(controller->dev, "cannot set RESET state\n");
251 + dma_unmap_single(controller->dev, tx_dummy_dma,
252 + controller->out_fifo_sz, DMA_TO_DEVICE);
255 + dma_unmap_single(controller->dev, tx_dma, xfer->len,
259 + dma_unmap_single(controller->dev, rx_dummy_dma,
260 + controller->in_fifo_sz, DMA_FROM_DEVICE);
263 + dma_unmap_single(controller->dev, rx_dma, xfer->len,
269 static irqreturn_t spi_qup_qup_irq(int irq, void *dev_id)
271 struct spi_qup *controller = dev_id;
272 @@ -315,11 +542,13 @@ static irqreturn_t spi_qup_qup_irq(int i
276 - if (opflags & QUP_OP_IN_SERVICE_FLAG)
277 - spi_qup_fifo_read(controller, xfer);
278 + if (!controller->use_dma) {
279 + if (opflags & QUP_OP_IN_SERVICE_FLAG)
280 + spi_qup_fifo_read(controller, xfer);
282 - if (opflags & QUP_OP_OUT_SERVICE_FLAG)
283 - spi_qup_fifo_write(controller, xfer);
284 + if (opflags & QUP_OP_OUT_SERVICE_FLAG)
285 + spi_qup_fifo_write(controller, xfer);
288 spin_lock_irqsave(&controller->lock, flags);
289 controller->error = error;
290 @@ -339,6 +568,8 @@ static int spi_qup_io_config(struct spi_
291 struct spi_qup *controller = spi_master_get_devdata(spi->master);
292 u32 config, iomode, mode;
293 int ret, n_words, w_size;
294 + size_t dma_align = dma_get_cache_alignment();
295 + u32 dma_available = 0;
297 if (spi->mode & SPI_LOOP && xfer->len > controller->in_fifo_sz) {
298 dev_err(controller->dev, "too big size for loopback %d > %d\n",
299 @@ -367,6 +598,13 @@ static int spi_qup_io_config(struct spi_
300 n_words = xfer->len / w_size;
301 controller->w_size = w_size;
303 + if (controller->rx_chan &&
304 + IS_ALIGNED((size_t)xfer->tx_buf, dma_align) &&
305 + IS_ALIGNED((size_t)xfer->rx_buf, dma_align) &&
306 + !is_vmalloc_addr(xfer->tx_buf) &&
307 + !is_vmalloc_addr(xfer->rx_buf))
310 if (n_words <= (controller->in_fifo_sz / sizeof(u32))) {
311 mode = QUP_IO_M_MODE_FIFO;
312 writel_relaxed(n_words, controller->base + QUP_MX_READ_CNT);
313 @@ -374,19 +612,30 @@ static int spi_qup_io_config(struct spi_
314 /* must be zero for FIFO */
315 writel_relaxed(0, controller->base + QUP_MX_INPUT_CNT);
316 writel_relaxed(0, controller->base + QUP_MX_OUTPUT_CNT);
318 + controller->use_dma = 0;
319 + } else if (!dma_available) {
320 mode = QUP_IO_M_MODE_BLOCK;
321 writel_relaxed(n_words, controller->base + QUP_MX_INPUT_CNT);
322 writel_relaxed(n_words, controller->base + QUP_MX_OUTPUT_CNT);
323 /* must be zero for BLOCK and BAM */
324 writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
325 writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
326 + controller->use_dma = 0;
328 + mode = QUP_IO_M_MODE_DMOV;
329 + writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
330 + writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
331 + controller->use_dma = 1;
334 iomode = readl_relaxed(controller->base + QUP_IO_M_MODES);
335 /* Set input and output transfer mode */
336 iomode &= ~(QUP_IO_M_INPUT_MODE_MASK | QUP_IO_M_OUTPUT_MODE_MASK);
337 - iomode &= ~(QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN);
338 + if (!controller->use_dma)
339 + iomode &= ~(QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN);
341 + iomode |= QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN;
343 iomode |= (mode << QUP_IO_M_OUTPUT_MODE_MASK_SHIFT);
344 iomode |= (mode << QUP_IO_M_INPUT_MODE_MASK_SHIFT);
346 @@ -419,11 +668,20 @@ static int spi_qup_io_config(struct spi_
347 config &= ~(QUP_CONFIG_NO_INPUT | QUP_CONFIG_NO_OUTPUT | QUP_CONFIG_N);
348 config |= xfer->bits_per_word - 1;
349 config |= QUP_CONFIG_SPI_MODE;
351 + if (controller->use_dma) {
353 + config |= QUP_CONFIG_NO_OUTPUT;
355 + config |= QUP_CONFIG_NO_INPUT;
358 writel_relaxed(config, controller->base + QUP_CONFIG);
360 /* only write to OPERATIONAL_MASK when register is present */
361 if (!controller->qup_v1)
362 writel_relaxed(0, controller->base + QUP_OPERATIONAL_MASK);
367 @@ -452,26 +710,32 @@ static int spi_qup_transfer_one(struct s
368 controller->tx_bytes = 0;
369 spin_unlock_irqrestore(&controller->lock, flags);
371 - if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
372 - dev_warn(controller->dev, "cannot set RUN state\n");
375 + if (controller->use_dma) {
376 + ret = spi_qup_do_dma(controller, xfer);
378 + if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
379 + dev_warn(controller->dev, "cannot set RUN state\n");
383 - if (spi_qup_set_state(controller, QUP_STATE_PAUSE)) {
384 - dev_warn(controller->dev, "cannot set PAUSE state\n");
387 + if (spi_qup_set_state(controller, QUP_STATE_PAUSE)) {
388 + dev_warn(controller->dev, "cannot set PAUSE state\n");
392 - spi_qup_fifo_write(controller, xfer);
393 + spi_qup_fifo_write(controller, xfer);
395 - if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
396 - dev_warn(controller->dev, "cannot set EXECUTE state\n");
399 + if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
400 + dev_warn(controller->dev, "cannot set EXECUTE state\n");
404 - if (!wait_for_completion_timeout(&controller->done, timeout))
406 + if (!ret && !wait_for_completion_timeout(&controller->done,
412 spi_qup_set_state(controller, QUP_STATE_RESET);
413 spin_lock_irqsave(&controller->lock, flags);
414 controller->xfer = NULL;
415 @@ -553,6 +817,7 @@ static int spi_qup_probe(struct platform
416 master->transfer_one = spi_qup_transfer_one;
417 master->dev.of_node = pdev->dev.of_node;
418 master->auto_runtime_pm = true;
419 + master->dma_alignment = dma_get_cache_alignment();
421 platform_set_drvdata(pdev, master);
423 @@ -612,6 +877,55 @@ static int spi_qup_probe(struct platform
424 writel_relaxed(SPI_ERROR_CLK_UNDER_RUN | SPI_ERROR_CLK_OVER_RUN,
425 base + SPI_ERROR_FLAGS_EN);
427 + /* allocate dma resources, if available */
428 + controller->rx_chan = dma_request_slave_channel(&pdev->dev, "rx");
429 + if (controller->rx_chan) {
430 + controller->tx_chan =
431 + dma_request_slave_channel(&pdev->dev, "tx");
433 + if (!controller->tx_chan) {
434 + dev_err(&pdev->dev, "Failed to allocate dma tx chan");
435 + dma_release_channel(controller->rx_chan);
438 + /* set DMA parameters */
439 + controller->rx_conf.device_fc = 1;
440 + controller->rx_conf.src_addr = res->start + QUP_INPUT_FIFO;
441 + controller->rx_conf.src_maxburst = controller->in_blk_sz;
443 + controller->tx_conf.device_fc = 1;
444 + controller->tx_conf.dst_addr = res->start + QUP_OUTPUT_FIFO;
445 + controller->tx_conf.dst_maxburst = controller->out_blk_sz;
447 + if (dmaengine_slave_config(controller->rx_chan,
448 + &controller->rx_conf)) {
449 + dev_err(&pdev->dev, "failed to configure RX channel\n");
451 + dma_release_channel(controller->rx_chan);
452 + dma_release_channel(controller->tx_chan);
453 + controller->tx_chan = NULL;
454 + controller->rx_chan = NULL;
455 + } else if (dmaengine_slave_config(controller->tx_chan,
456 + &controller->tx_conf)) {
457 + dev_err(&pdev->dev, "failed to configure TX channel\n");
459 + dma_release_channel(controller->rx_chan);
460 + dma_release_channel(controller->tx_chan);
461 + controller->tx_chan = NULL;
462 + controller->rx_chan = NULL;
465 + controller->dummy = devm_kmalloc(controller->dev, PAGE_SIZE,
468 + if (!controller->dummy) {
469 + dma_release_channel(controller->rx_chan);
470 + dma_release_channel(controller->tx_chan);
471 + controller->tx_chan = NULL;
472 + controller->rx_chan = NULL;
476 /* if earlier version of the QUP, disable INPUT_OVERRUN */
477 if (controller->qup_v1)
478 writel_relaxed(QUP_ERROR_OUTPUT_OVER_RUN |
479 @@ -730,6 +1044,11 @@ static int spi_qup_remove(struct platfor
483 + if (controller->rx_chan)
484 + dma_release_channel(controller->rx_chan);
485 + if (controller->tx_chan)
486 + dma_release_channel(controller->tx_chan);
488 clk_disable_unprepare(controller->cclk);
489 clk_disable_unprepare(controller->iclk);