dca516e8783fcb5fcdd285b6d7f5090e7fa52177
[openwrt/openwrt.git] / target / linux / ipq40xx / patches-4.14 / 040-dmaengine-qcom-bam-Process-multiple-pending-descript.patch
1 From 6b4faeac05bc0b91616b921191cb054d1376f3b4 Mon Sep 17 00:00:00 2001
2 From: Sricharan R <sricharan@codeaurora.org>
3 Date: Mon, 28 Aug 2017 20:30:24 +0530
4 Subject: [PATCH] dmaengine: qcom-bam: Process multiple pending descriptors
5
6 The bam dmaengine has a circular FIFO to which we
7 add hw descriptors that describes the transaction.
8 The FIFO has space for about 4096 hw descriptors.
9
10 Currently we add one descriptor and wait for it to
11 complete with interrupt and then add the next pending
12 descriptor. In this way, the FIFO is underutilized
13 since only one descriptor is processed at a time, although
14 there is space in FIFO for the BAM to process more.
15
16 Instead keep adding descriptors to FIFO till its full,
17 that allows BAM to continue to work on the next descriptor
18 immediately after signalling completion interrupt for the
19 previous descriptor.
20
21 Also when the client has not set the DMA_PREP_INTERRUPT for
22 a descriptor, then do not configure BAM to trigger a interrupt
23 upon completion of that descriptor. This way we get a interrupt
24 only for the descriptor for which DMA_PREP_INTERRUPT was
25 requested and there signal completion of all the previous completed
26 descriptors. So we still do callbacks for all requested descriptors,
27 but just that the number of interrupts are reduced.
28
29 CURRENT:
30
31 ------ ------- ---------------
32 |DES 0| |DESC 1| |DESC 2 + INT |
33 ------ ------- ---------------
34 | | |
35 | | |
36 INTERRUPT: (INT) (INT) (INT)
37 CALLBACK: (CB) (CB) (CB)
38
39 MTD_SPEEDTEST READ PAGE: 3560 KiB/s
40 MTD_SPEEDTEST WRITE PAGE: 2664 KiB/s
41 IOZONE READ: 2456 KB/s
42 IOZONE WRITE: 1230 KB/s
43
44 bam dma interrupts (after tests): 96508
45
46 CHANGE:
47
48 ------ ------- -------------
49 |DES 0| |DESC 1 |DESC 2 + INT |
50 ------ ------- --------------
51 |
52 |
53 (INT)
54 (CB for 0, 1, 2)
55
56 MTD_SPEEDTEST READ PAGE: 3860 KiB/s
57 MTD_SPEEDTEST WRITE PAGE: 2837 KiB/s
58 IOZONE READ: 2677 KB/s
59 IOZONE WRITE: 1308 KB/s
60
61 bam dma interrupts (after tests): 58806
62
63 Signed-off-by: Sricharan R <sricharan@codeaurora.org>
64 Reviewed-by: Andy Gross <andy.gross@linaro.org>
65 Tested-by: Abhishek Sahu <absahu@codeaurora.org>
66 Signed-off-by: Vinod Koul <vinod.koul@intel.com>
67 ---
68 drivers/dma/qcom/bam_dma.c | 169 +++++++++++++++++++++++++++++----------------
69 1 file changed, 109 insertions(+), 60 deletions(-)
70
71 --- a/drivers/dma/qcom/bam_dma.c
72 +++ b/drivers/dma/qcom/bam_dma.c
73 @@ -46,6 +46,7 @@
74 #include <linux/of_address.h>
75 #include <linux/of_irq.h>
76 #include <linux/of_dma.h>
77 +#include <linux/circ_buf.h>
78 #include <linux/clk.h>
79 #include <linux/dmaengine.h>
80 #include <linux/pm_runtime.h>
81 @@ -78,6 +79,8 @@ struct bam_async_desc {
82
83 struct bam_desc_hw *curr_desc;
84
85 + /* list node for the desc in the bam_chan list of descriptors */
86 + struct list_head desc_node;
87 enum dma_transfer_direction dir;
88 size_t length;
89 struct bam_desc_hw desc[0];
90 @@ -347,6 +350,8 @@ static const struct reg_offset_data bam_
91 #define BAM_DESC_FIFO_SIZE SZ_32K
92 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
93 #define BAM_FIFO_SIZE (SZ_32K - 8)
94 +#define IS_BUSY(chan) (CIRC_SPACE(bchan->tail, bchan->head,\
95 + MAX_DESCRIPTORS + 1) == 0)
96
97 struct bam_chan {
98 struct virt_dma_chan vc;
99 @@ -356,8 +361,6 @@ struct bam_chan {
100 /* configuration from device tree */
101 u32 id;
102
103 - struct bam_async_desc *curr_txd; /* current running dma */
104 -
105 /* runtime configuration */
106 struct dma_slave_config slave;
107
108 @@ -372,6 +375,8 @@ struct bam_chan {
109 unsigned int initialized; /* is the channel hw initialized? */
110 unsigned int paused; /* is the channel paused? */
111 unsigned int reconfigure; /* new slave config? */
112 + /* list of descriptors currently processed */
113 + struct list_head desc_list;
114
115 struct list_head node;
116 };
117 @@ -539,7 +544,7 @@ static void bam_free_chan(struct dma_cha
118
119 vchan_free_chan_resources(to_virt_chan(chan));
120
121 - if (bchan->curr_txd) {
122 + if (!list_empty(&bchan->desc_list)) {
123 dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
124 goto err;
125 }
126 @@ -632,8 +637,6 @@ static struct dma_async_tx_descriptor *b
127
128 if (flags & DMA_PREP_INTERRUPT)
129 async_desc->flags |= DESC_FLAG_EOT;
130 - else
131 - async_desc->flags |= DESC_FLAG_INT;
132
133 async_desc->num_desc = num_alloc;
134 async_desc->curr_desc = async_desc->desc;
135 @@ -684,14 +687,16 @@ err_out:
136 static int bam_dma_terminate_all(struct dma_chan *chan)
137 {
138 struct bam_chan *bchan = to_bam_chan(chan);
139 + struct bam_async_desc *async_desc, *tmp;
140 unsigned long flag;
141 LIST_HEAD(head);
142
143 /* remove all transactions, including active transaction */
144 spin_lock_irqsave(&bchan->vc.lock, flag);
145 - if (bchan->curr_txd) {
146 - list_add(&bchan->curr_txd->vd.node, &bchan->vc.desc_issued);
147 - bchan->curr_txd = NULL;
148 + list_for_each_entry_safe(async_desc, tmp,
149 + &bchan->desc_list, desc_node) {
150 + list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
151 + list_del(&async_desc->desc_node);
152 }
153
154 vchan_get_all_descriptors(&bchan->vc, &head);
155 @@ -763,9 +768,9 @@ static int bam_resume(struct dma_chan *c
156 */
157 static u32 process_channel_irqs(struct bam_device *bdev)
158 {
159 - u32 i, srcs, pipe_stts;
160 + u32 i, srcs, pipe_stts, offset, avail;
161 unsigned long flags;
162 - struct bam_async_desc *async_desc;
163 + struct bam_async_desc *async_desc, *tmp;
164
165 srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
166
167 @@ -785,27 +790,40 @@ static u32 process_channel_irqs(struct b
168 writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
169
170 spin_lock_irqsave(&bchan->vc.lock, flags);
171 - async_desc = bchan->curr_txd;
172
173 - if (async_desc) {
174 - async_desc->num_desc -= async_desc->xfer_len;
175 - async_desc->curr_desc += async_desc->xfer_len;
176 - bchan->curr_txd = NULL;
177 + offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
178 + P_SW_OFSTS_MASK;
179 + offset /= sizeof(struct bam_desc_hw);
180 +
181 + /* Number of bytes available to read */
182 + avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);
183 +
184 + list_for_each_entry_safe(async_desc, tmp,
185 + &bchan->desc_list, desc_node) {
186 + /* Not enough data to read */
187 + if (avail < async_desc->xfer_len)
188 + break;
189
190 /* manage FIFO */
191 bchan->head += async_desc->xfer_len;
192 bchan->head %= MAX_DESCRIPTORS;
193
194 + async_desc->num_desc -= async_desc->xfer_len;
195 + async_desc->curr_desc += async_desc->xfer_len;
196 + avail -= async_desc->xfer_len;
197 +
198 /*
199 - * if complete, process cookie. Otherwise
200 + * if complete, process cookie. Otherwise
201 * push back to front of desc_issued so that
202 * it gets restarted by the tasklet
203 */
204 - if (!async_desc->num_desc)
205 + if (!async_desc->num_desc) {
206 vchan_cookie_complete(&async_desc->vd);
207 - else
208 + } else {
209 list_add(&async_desc->vd.node,
210 - &bchan->vc.desc_issued);
211 + &bchan->vc.desc_issued);
212 + }
213 + list_del(&async_desc->desc_node);
214 }
215
216 spin_unlock_irqrestore(&bchan->vc.lock, flags);
217 @@ -867,6 +885,7 @@ static enum dma_status bam_tx_status(str
218 struct dma_tx_state *txstate)
219 {
220 struct bam_chan *bchan = to_bam_chan(chan);
221 + struct bam_async_desc *async_desc;
222 struct virt_dma_desc *vd;
223 int ret;
224 size_t residue = 0;
225 @@ -882,11 +901,17 @@ static enum dma_status bam_tx_status(str
226
227 spin_lock_irqsave(&bchan->vc.lock, flags);
228 vd = vchan_find_desc(&bchan->vc, cookie);
229 - if (vd)
230 + if (vd) {
231 residue = container_of(vd, struct bam_async_desc, vd)->length;
232 - else if (bchan->curr_txd && bchan->curr_txd->vd.tx.cookie == cookie)
233 - for (i = 0; i < bchan->curr_txd->num_desc; i++)
234 - residue += bchan->curr_txd->curr_desc[i].size;
235 + } else {
236 + list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
237 + if (async_desc->vd.tx.cookie != cookie)
238 + continue;
239 +
240 + for (i = 0; i < async_desc->num_desc; i++)
241 + residue += async_desc->curr_desc[i].size;
242 + }
243 + }
244
245 spin_unlock_irqrestore(&bchan->vc.lock, flags);
246
247 @@ -927,63 +952,86 @@ static void bam_start_dma(struct bam_cha
248 {
249 struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
250 struct bam_device *bdev = bchan->bdev;
251 - struct bam_async_desc *async_desc;
252 + struct bam_async_desc *async_desc = NULL;
253 struct bam_desc_hw *desc;
254 struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
255 sizeof(struct bam_desc_hw));
256 int ret;
257 + unsigned int avail;
258 + struct dmaengine_desc_callback cb;
259
260 lockdep_assert_held(&bchan->vc.lock);
261
262 if (!vd)
263 return;
264
265 - list_del(&vd->node);
266 -
267 - async_desc = container_of(vd, struct bam_async_desc, vd);
268 - bchan->curr_txd = async_desc;
269 -
270 ret = pm_runtime_get_sync(bdev->dev);
271 if (ret < 0)
272 return;
273
274 - /* on first use, initialize the channel hardware */
275 - if (!bchan->initialized)
276 - bam_chan_init_hw(bchan, async_desc->dir);
277 -
278 - /* apply new slave config changes, if necessary */
279 - if (bchan->reconfigure)
280 - bam_apply_new_config(bchan, async_desc->dir);
281 + while (vd && !IS_BUSY(bchan)) {
282 + list_del(&vd->node);
283
284 - desc = bchan->curr_txd->curr_desc;
285 + async_desc = container_of(vd, struct bam_async_desc, vd);
286
287 - if (async_desc->num_desc > MAX_DESCRIPTORS)
288 - async_desc->xfer_len = MAX_DESCRIPTORS;
289 - else
290 - async_desc->xfer_len = async_desc->num_desc;
291 + /* on first use, initialize the channel hardware */
292 + if (!bchan->initialized)
293 + bam_chan_init_hw(bchan, async_desc->dir);
294
295 - /* set any special flags on the last descriptor */
296 - if (async_desc->num_desc == async_desc->xfer_len)
297 - desc[async_desc->xfer_len - 1].flags |=
298 - cpu_to_le16(async_desc->flags);
299 - else
300 - desc[async_desc->xfer_len - 1].flags |=
301 - cpu_to_le16(DESC_FLAG_INT);
302 + /* apply new slave config changes, if necessary */
303 + if (bchan->reconfigure)
304 + bam_apply_new_config(bchan, async_desc->dir);
305 +
306 + desc = async_desc->curr_desc;
307 + avail = CIRC_SPACE(bchan->tail, bchan->head,
308 + MAX_DESCRIPTORS + 1);
309 +
310 + if (async_desc->num_desc > avail)
311 + async_desc->xfer_len = avail;
312 + else
313 + async_desc->xfer_len = async_desc->num_desc;
314 +
315 + /* set any special flags on the last descriptor */
316 + if (async_desc->num_desc == async_desc->xfer_len)
317 + desc[async_desc->xfer_len - 1].flags |=
318 + cpu_to_le16(async_desc->flags);
319
320 - if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
321 - u32 partial = MAX_DESCRIPTORS - bchan->tail;
322 + vd = vchan_next_desc(&bchan->vc);
323
324 - memcpy(&fifo[bchan->tail], desc,
325 - partial * sizeof(struct bam_desc_hw));
326 - memcpy(fifo, &desc[partial], (async_desc->xfer_len - partial) *
327 + dmaengine_desc_get_callback(&async_desc->vd.tx, &cb);
328 +
329 + /*
330 + * An interrupt is generated at this desc, if
331 + * - FIFO is FULL.
332 + * - No more descriptors to add.
333 + * - If a callback completion was requested for this DESC,
334 + * In this case, BAM will deliver the completion callback
335 + * for this desc and continue processing the next desc.
336 + */
337 + if (((avail <= async_desc->xfer_len) || !vd ||
338 + dmaengine_desc_callback_valid(&cb)) &&
339 + !(async_desc->flags & DESC_FLAG_EOT))
340 + desc[async_desc->xfer_len - 1].flags |=
341 + cpu_to_le16(DESC_FLAG_INT);
342 +
343 + if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
344 + u32 partial = MAX_DESCRIPTORS - bchan->tail;
345 +
346 + memcpy(&fifo[bchan->tail], desc,
347 + partial * sizeof(struct bam_desc_hw));
348 + memcpy(fifo, &desc[partial],
349 + (async_desc->xfer_len - partial) *
350 sizeof(struct bam_desc_hw));
351 - } else {
352 - memcpy(&fifo[bchan->tail], desc,
353 - async_desc->xfer_len * sizeof(struct bam_desc_hw));
354 - }
355 + } else {
356 + memcpy(&fifo[bchan->tail], desc,
357 + async_desc->xfer_len *
358 + sizeof(struct bam_desc_hw));
359 + }
360
361 - bchan->tail += async_desc->xfer_len;
362 - bchan->tail %= MAX_DESCRIPTORS;
363 + bchan->tail += async_desc->xfer_len;
364 + bchan->tail %= MAX_DESCRIPTORS;
365 + list_add_tail(&async_desc->desc_node, &bchan->desc_list);
366 + }
367
368 /* ensure descriptor writes and dma start not reordered */
369 wmb();
370 @@ -1012,7 +1060,7 @@ static void dma_tasklet(unsigned long da
371 bchan = &bdev->channels[i];
372 spin_lock_irqsave(&bchan->vc.lock, flags);
373
374 - if (!list_empty(&bchan->vc.desc_issued) && !bchan->curr_txd)
375 + if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
376 bam_start_dma(bchan);
377 spin_unlock_irqrestore(&bchan->vc.lock, flags);
378 }
379 @@ -1033,7 +1081,7 @@ static void bam_issue_pending(struct dma
380 spin_lock_irqsave(&bchan->vc.lock, flags);
381
382 /* if work pending and idle, start a transaction */
383 - if (vchan_issue_pending(&bchan->vc) && !bchan->curr_txd)
384 + if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
385 bam_start_dma(bchan);
386
387 spin_unlock_irqrestore(&bchan->vc.lock, flags);
388 @@ -1133,6 +1181,7 @@ static void bam_channel_init(struct bam_
389
390 vchan_init(&bchan->vc, &bdev->common);
391 bchan->vc.desc_free = bam_dma_free_desc;
392 + INIT_LIST_HEAD(&bchan->desc_list);
393 }
394
395 static const struct of_device_id bam_of_match[] = {