1 From 545bbd2db8451d8e81a83aa1bbc0e8e5227a0503 Mon Sep 17 00:00:00 2001
2 From: Phil Elwell <phil@raspberrypi.org>
3 Date: Thu, 11 Feb 2016 16:51:01 +0000
4 Subject: [PATCH] bcm2835-sdhost: Major revision
6 This is a significant revision of the bcm2835-sdhost driver. It
7 improves on the original in a number of ways:
9 1) Through the use of CMD23 for reads it appears to avoid problems
10 reading some sectors on certain high speed cards.
11 2) Better atomicity to prevent crashes.
12 3) Higher performance.
13 4) Activity logging included, for easier diagnosis in the event
16 Signed-off-by: Phil Elwell <phil@raspberrypi.org>
18 drivers/mmc/host/bcm2835-sdhost.c | 1284 ++++++++++++++++++++-----------------
19 1 file changed, 686 insertions(+), 598 deletions(-)
21 --- a/drivers/mmc/host/bcm2835-sdhost.c
22 +++ b/drivers/mmc/host/bcm2835-sdhost.c
24 * BCM2835 SD host driver.
26 * Author: Phil Elwell <phil@raspberrypi.org>
28 + * Copyright (C) 2015-2016 Raspberry Pi (Trading) Ltd.
31 * mmc-bcm2835.c by Gellert Weisz
33 * along with this program. If not, see <http://www.gnu.org/licenses/>.
36 -#define SAFE_READ_THRESHOLD 4
37 -#define SAFE_WRITE_THRESHOLD 4
39 -#define ALLOW_CMD23 0
41 -#define USE_BLOCK_IRQ 1
42 +#define FIFO_READ_THRESHOLD 4
43 +#define FIFO_WRITE_THRESHOLD 4
44 +#define ALLOW_CMD23_READ 1
45 +#define ALLOW_CMD23_WRITE 0
47 +#define SDDATA_FIFO_PIO_BURST 8
48 +#define CMD_DALLY_US 1
50 #include <linux/delay.h>
51 #include <linux/module.h>
53 #include <linux/dma-mapping.h>
54 #include <linux/of_dma.h>
55 #include <linux/time.h>
56 +#include <linux/workqueue.h>
58 #define DRIVER_NAME "sdhost-bcm2835"
61 #define SDEDM_READ_THRESHOLD_SHIFT 14
62 #define SDEDM_THRESHOLD_MASK 0x1f
64 +#define SDEDM_FSM_MASK 0xf
65 +#define SDEDM_FSM_IDENTMODE 0x0
66 +#define SDEDM_FSM_DATAMODE 0x1
67 +#define SDEDM_FSM_READDATA 0x2
68 +#define SDEDM_FSM_WRITEDATA 0x3
69 +#define SDEDM_FSM_READWAIT 0x4
70 +#define SDEDM_FSM_READCRC 0x5
71 +#define SDEDM_FSM_WRITECRC 0x6
72 +#define SDEDM_FSM_WRITEWAIT1 0x7
73 +#define SDEDM_FSM_POWERDOWN 0x8
74 +#define SDEDM_FSM_POWERUP 0x9
75 +#define SDEDM_FSM_WRITESTART1 0xa
76 +#define SDEDM_FSM_WRITESTART2 0xb
77 +#define SDEDM_FSM_GENPULSES 0xc
78 +#define SDEDM_FSM_WRITEWAIT2 0xd
79 +#define SDEDM_FSM_STARTPOWDOWN 0xf
81 +#define SDDATA_FIFO_WORDS 16
83 +#define USE_CMD23_FLAGS ((ALLOW_CMD23_READ * MMC_DATA_READ) | \
84 + (ALLOW_CMD23_WRITE * MMC_DATA_WRITE))
89 @@ -131,15 +155,17 @@ struct bcm2835_host {
91 struct tasklet_struct finish_tasklet; /* Tasklet structures */
93 - struct timer_list timer; /* Timer for timeouts */
94 + struct work_struct cmd_wait_wq; /* Workqueue function */
96 - struct timer_list pio_timer; /* PIO error detection timer */
97 + struct timer_list timer; /* Timer for timeouts */
99 struct sg_mapping_iter sg_miter; /* SG state for PIO */
100 unsigned int blocks; /* remaining PIO blocks */
102 int irq; /* Device IRQ */
104 + u32 cmd_quick_poll_retries;
105 + u32 ns_per_fifo_word;
107 /* cached registers */
109 @@ -154,16 +180,21 @@ struct bcm2835_host {
111 unsigned int use_busy:1; /* Wait for busy interrupt */
113 - unsigned int debug:1; /* Enable debug output */
114 + unsigned int use_sbc:1; /* Send CMD23 */
117 + unsigned int debug:1; /* Enable debug output */
120 struct dma_chan *dma_chan_rx; /* DMA channel for reads */
121 struct dma_chan *dma_chan_tx; /* DMA channel for writes */
122 + struct dma_chan *dma_chan; /* Channel in used */
123 + struct dma_async_tx_descriptor *dma_desc;
126 + struct page *drain_page;
134 @@ -173,13 +204,98 @@ struct bcm2835_host {
135 u32 overclock_50; /* frequency to use when 50MHz is requested (in MHz) */
136 u32 overclock; /* Current frequency if overclocked, else zero */
137 u32 pio_limit; /* Maximum block count for PIO (0 = always DMA) */
143 - u32 sectors; /* Cached card size in sectors */
144 - u32 single_read_sectors[8];
145 +struct log_entry_struct {
152 +typedef struct log_entry_struct LOG_ENTRY_T;
154 +LOG_ENTRY_T *sdhost_log_buf;
155 +dma_addr_t sdhost_log_addr;
156 +static u32 sdhost_log_idx;
157 +static spinlock_t log_lock;
158 +static void __iomem *timer_base;
160 +#define LOG_ENTRIES (256*1)
161 +#define LOG_SIZE (sizeof(LOG_ENTRY_T)*LOG_ENTRIES)
163 +static void log_init(u32 bus_to_phys)
165 + spin_lock_init(&log_lock);
166 + sdhost_log_buf = dma_zalloc_coherent(NULL, LOG_SIZE, &sdhost_log_addr,
168 + if (sdhost_log_buf) {
169 + pr_err("sdhost: log_buf @ %p (%x)\n",
170 + sdhost_log_buf, sdhost_log_addr);
171 + timer_base = ioremap_nocache(bus_to_phys + 0x7e003000, SZ_4K);
173 + pr_err("sdhost: failed to remap timer\n");
176 + pr_err("sdhost: failed to allocate log buf\n");
179 +static void log_event_impl(const char *event, u32 param1, u32 param2)
181 + if (sdhost_log_buf) {
182 + LOG_ENTRY_T *entry;
183 + unsigned long flags;
185 + spin_lock_irqsave(&log_lock, flags);
187 + entry = sdhost_log_buf + sdhost_log_idx;
188 + memcpy(entry->event, event, 4);
189 + entry->timestamp = (readl(timer_base + 4) & 0x3fffffff) +
190 + (smp_processor_id()<<30);
191 + entry->param1 = param1;
192 + entry->param2 = param2;
193 + sdhost_log_idx = (sdhost_log_idx + 1) % LOG_ENTRIES;
195 + spin_unlock_irqrestore(&log_lock, flags);
199 +static void log_dump(void)
201 + if (sdhost_log_buf) {
202 + LOG_ENTRY_T *entry;
203 + unsigned long flags;
206 + spin_lock_irqsave(&log_lock, flags);
208 + idx = sdhost_log_idx;
210 + entry = sdhost_log_buf + idx;
211 + if (entry->event[0] != '\0')
212 + pr_err("[%08x] %.4s %x %x\n",
217 + idx = (idx + 1) % LOG_ENTRIES;
218 + } while (idx != sdhost_log_idx);
220 + spin_unlock_irqrestore(&log_lock, flags);
224 +#define log_event(event, param1, param2) log_event_impl(event, param1, param2)
228 +#define log_init(x) (void)0
229 +#define log_event(event, param1, param2) (void)0
230 +#define log_dump() (void)0
234 static inline void bcm2835_sdhost_write(struct bcm2835_host *host, u32 val, int reg)
236 @@ -201,7 +317,7 @@ static void bcm2835_sdhost_dumpcmd(struc
240 - pr_info("%s:%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
241 + pr_err("%s:%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
242 mmc_hostname(host->mmc),
243 (cmd == host->cmd) ? '>' : ' ',
244 label, cmd->opcode, cmd->arg, cmd->flags,
245 @@ -211,73 +327,74 @@ static void bcm2835_sdhost_dumpcmd(struc
247 static void bcm2835_sdhost_dumpregs(struct bcm2835_host *host)
249 - bcm2835_sdhost_dumpcmd(host, host->mrq->sbc, "sbc");
250 - bcm2835_sdhost_dumpcmd(host, host->mrq->cmd, "cmd");
251 - if (host->mrq->data)
252 - pr_err("%s: data blocks %x blksz %x - err %d\n",
253 - mmc_hostname(host->mmc),
254 - host->mrq->data->blocks,
255 - host->mrq->data->blksz,
256 - host->mrq->data->error);
257 - bcm2835_sdhost_dumpcmd(host, host->mrq->stop, "stop");
260 + bcm2835_sdhost_dumpcmd(host, host->mrq->sbc, "sbc");
261 + bcm2835_sdhost_dumpcmd(host, host->mrq->cmd, "cmd");
262 + if (host->mrq->data)
263 + pr_err("%s: data blocks %x blksz %x - err %d\n",
264 + mmc_hostname(host->mmc),
265 + host->mrq->data->blocks,
266 + host->mrq->data->blksz,
267 + host->mrq->data->error);
268 + bcm2835_sdhost_dumpcmd(host, host->mrq->stop, "stop");
271 - pr_info("%s: =========== REGISTER DUMP ===========\n",
272 + pr_err("%s: =========== REGISTER DUMP ===========\n",
273 mmc_hostname(host->mmc));
275 - pr_info("%s: SDCMD 0x%08x\n",
276 + pr_err("%s: SDCMD 0x%08x\n",
277 mmc_hostname(host->mmc),
278 bcm2835_sdhost_read(host, SDCMD));
279 - pr_info("%s: SDARG 0x%08x\n",
280 + pr_err("%s: SDARG 0x%08x\n",
281 mmc_hostname(host->mmc),
282 bcm2835_sdhost_read(host, SDARG));
283 - pr_info("%s: SDTOUT 0x%08x\n",
284 + pr_err("%s: SDTOUT 0x%08x\n",
285 mmc_hostname(host->mmc),
286 bcm2835_sdhost_read(host, SDTOUT));
287 - pr_info("%s: SDCDIV 0x%08x\n",
288 + pr_err("%s: SDCDIV 0x%08x\n",
289 mmc_hostname(host->mmc),
290 bcm2835_sdhost_read(host, SDCDIV));
291 - pr_info("%s: SDRSP0 0x%08x\n",
292 + pr_err("%s: SDRSP0 0x%08x\n",
293 mmc_hostname(host->mmc),
294 bcm2835_sdhost_read(host, SDRSP0));
295 - pr_info("%s: SDRSP1 0x%08x\n",
296 + pr_err("%s: SDRSP1 0x%08x\n",
297 mmc_hostname(host->mmc),
298 bcm2835_sdhost_read(host, SDRSP1));
299 - pr_info("%s: SDRSP2 0x%08x\n",
300 + pr_err("%s: SDRSP2 0x%08x\n",
301 mmc_hostname(host->mmc),
302 bcm2835_sdhost_read(host, SDRSP2));
303 - pr_info("%s: SDRSP3 0x%08x\n",
304 + pr_err("%s: SDRSP3 0x%08x\n",
305 mmc_hostname(host->mmc),
306 bcm2835_sdhost_read(host, SDRSP3));
307 - pr_info("%s: SDHSTS 0x%08x\n",
308 + pr_err("%s: SDHSTS 0x%08x\n",
309 mmc_hostname(host->mmc),
310 bcm2835_sdhost_read(host, SDHSTS));
311 - pr_info("%s: SDVDD 0x%08x\n",
312 + pr_err("%s: SDVDD 0x%08x\n",
313 mmc_hostname(host->mmc),
314 bcm2835_sdhost_read(host, SDVDD));
315 - pr_info("%s: SDEDM 0x%08x\n",
316 + pr_err("%s: SDEDM 0x%08x\n",
317 mmc_hostname(host->mmc),
318 bcm2835_sdhost_read(host, SDEDM));
319 - pr_info("%s: SDHCFG 0x%08x\n",
320 + pr_err("%s: SDHCFG 0x%08x\n",
321 mmc_hostname(host->mmc),
322 bcm2835_sdhost_read(host, SDHCFG));
323 - pr_info("%s: SDHBCT 0x%08x\n",
324 + pr_err("%s: SDHBCT 0x%08x\n",
325 mmc_hostname(host->mmc),
326 bcm2835_sdhost_read(host, SDHBCT));
327 - pr_info("%s: SDHBLC 0x%08x\n",
328 + pr_err("%s: SDHBLC 0x%08x\n",
329 mmc_hostname(host->mmc),
330 bcm2835_sdhost_read(host, SDHBLC));
332 - pr_info("%s: ===========================================\n",
333 + pr_err("%s: ===========================================\n",
334 mmc_hostname(host->mmc));
338 static void bcm2835_sdhost_set_power(struct bcm2835_host *host, bool on)
340 bcm2835_sdhost_write(host, on ? 1 : 0, SDVDD);
344 static void bcm2835_sdhost_reset_internal(struct bcm2835_host *host)
347 @@ -300,26 +417,24 @@ static void bcm2835_sdhost_reset_interna
348 temp = bcm2835_sdhost_read(host, SDEDM);
349 temp &= ~((SDEDM_THRESHOLD_MASK<<SDEDM_READ_THRESHOLD_SHIFT) |
350 (SDEDM_THRESHOLD_MASK<<SDEDM_WRITE_THRESHOLD_SHIFT));
351 - temp |= (SAFE_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
352 - (SAFE_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
353 + temp |= (FIFO_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
354 + (FIFO_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
355 bcm2835_sdhost_write(host, temp, SDEDM);
357 bcm2835_sdhost_set_power(host, true);
361 - host->single_read_sectors[0] = ~0;
362 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
363 bcm2835_sdhost_write(host, host->cdiv, SDCDIV);
368 static void bcm2835_sdhost_reset(struct mmc_host *mmc)
370 struct bcm2835_host *host = mmc_priv(mmc);
372 spin_lock_irqsave(&host->lock, flags);
373 + log_event("RST<", 0, 0);
375 bcm2835_sdhost_reset_internal(host);
377 @@ -344,82 +459,48 @@ static void bcm2835_sdhost_init(struct b
381 -static bool bcm2835_sdhost_is_write_complete(struct bcm2835_host *host)
382 +static void bcm2835_sdhost_wait_transfer_complete(struct bcm2835_host *host)
384 - bool write_complete = ((bcm2835_sdhost_read(host, SDEDM) & 0xf) == 1);
386 + u32 alternate_idle;
389 - if (!write_complete) {
390 - /* Request an IRQ for the last block */
391 - host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
392 - bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
393 - if ((bcm2835_sdhost_read(host, SDEDM) & 0xf) == 1) {
394 - /* The write has now completed. Disable the interrupt
395 - and clear the status flag */
396 - host->hcfg &= ~SDHCFG_BLOCK_IRPT_EN;
397 - bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
398 - bcm2835_sdhost_write(host, SDHSTS_BLOCK_IRPT, SDHSTS);
399 - write_complete = true;
402 + alternate_idle = (host->mrq->data->flags & MMC_DATA_READ) ?
403 + SDEDM_FSM_READWAIT : SDEDM_FSM_WRITESTART1;
405 - return write_complete;
407 + edm = bcm2835_sdhost_read(host, SDEDM);
409 -static void bcm2835_sdhost_wait_write_complete(struct bcm2835_host *host)
413 - static struct timeval start_time;
414 - static int max_stall_time = 0;
415 - static int total_stall_time = 0;
416 - struct timeval before, after;
418 - do_gettimeofday(&before);
419 - if (max_stall_time == 0)
420 - start_time = before;
422 + log_event("WTC<", edm, 0);
427 - u32 edm = bcm2835_sdhost_read(host, SDEDM);
428 - if ((edm & 0xf) == 1)
429 + u32 fsm = edm & SDEDM_FSM_MASK;
430 + if ((fsm == SDEDM_FSM_IDENTMODE) ||
431 + (fsm == SDEDM_FSM_DATAMODE))
434 - if (timediff > 5000000) {
436 - do_gettimeofday(&after);
437 - timediff = (after.tv_sec - before.tv_sec)*1000000 +
438 - (after.tv_usec - before.tv_usec);
439 + if (fsm == alternate_idle) {
440 + bcm2835_sdhost_write(host,
441 + edm | SDEDM_FORCE_DATA_MODE,
446 - pr_err(" wait_write_complete - still waiting after %dus\n",
449 - pr_err(" wait_write_complete - still waiting after %d retries\n",
451 + if (timediff == 100000) {
452 + pr_err("%s: wait_transfer_complete - still waiting after %d retries\n",
453 + mmc_hostname(host->mmc),
457 bcm2835_sdhost_dumpregs(host);
458 - host->data->error = -ETIMEDOUT;
459 + host->mrq->data->error = -ETIMEDOUT;
460 + log_event("WTC!", edm, 0);
464 + edm = bcm2835_sdhost_read(host, SDEDM);
468 - do_gettimeofday(&after);
469 - timediff = (after.tv_sec - before.tv_sec)*1000000 + (after.tv_usec - before.tv_usec);
471 - total_stall_time += timediff;
472 - if (timediff > max_stall_time)
473 - max_stall_time = timediff;
475 - if ((after.tv_sec - start_time.tv_sec) > 10) {
476 - pr_debug(" wait_write_complete - max wait %dus, total %dus\n",
477 - max_stall_time, total_stall_time);
478 - start_time = after;
479 - max_stall_time = 0;
480 - total_stall_time = 0;
483 + log_event("WTC>", edm, 0);
486 static void bcm2835_sdhost_finish_data(struct bcm2835_host *host);
487 @@ -427,65 +508,44 @@ static void bcm2835_sdhost_finish_data(s
488 static void bcm2835_sdhost_dma_complete(void *param)
490 struct bcm2835_host *host = param;
491 - struct dma_chan *dma_chan;
492 + struct mmc_data *data = host->data;
496 spin_lock_irqsave(&host->lock, flags);
497 + log_event("DMA<", (u32)host->data, bcm2835_sdhost_read(host, SDHSTS));
498 + log_event("DMA ", bcm2835_sdhost_read(host, SDCMD),
499 + bcm2835_sdhost_read(host, SDEDM));
502 - bool write_complete;
504 - write_complete = bcm2835_sdhost_is_write_complete(host);
506 - bcm2835_sdhost_wait_write_complete(host);
507 - write_complete = true;
509 - pr_debug("dma_complete() - write_complete=%d\n",
512 - if (write_complete || (host->data->flags & MMC_DATA_READ))
514 - if (write_complete) {
515 - dma_chan = host->dma_chan_tx;
516 - dir_data = DMA_TO_DEVICE;
518 - dma_chan = host->dma_chan_rx;
519 - dir_data = DMA_FROM_DEVICE;
522 - dma_unmap_sg(dma_chan->device->dev,
523 - host->data->sg, host->data->sg_len,
525 + if (host->dma_chan) {
526 + dma_unmap_sg(host->dma_chan->device->dev,
527 + data->sg, data->sg_len,
530 - bcm2835_sdhost_finish_data(host);
532 + host->dma_chan = NULL;
535 - spin_unlock_irqrestore(&host->lock, flags);
537 + if (host->drain_words) {
541 -static bool data_transfer_wait(struct bcm2835_host *host)
543 - unsigned long timeout = 1000000;
546 - u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
547 - if (sdhsts & SDHSTS_DATA_FLAG) {
548 - bcm2835_sdhost_write(host, SDHSTS_DATA_FLAG, SDHSTS);
550 + page = kmap_atomic(host->drain_page);
551 + buf = page + host->drain_offset;
553 + while (host->drain_words) {
554 + u32 edm = bcm2835_sdhost_read(host, SDEDM);
555 + if ((edm >> 4) & 0x1f)
556 + *(buf++) = bcm2835_sdhost_read(host,
558 + host->drain_words--;
562 - if (timeout == 0) {
563 - pr_err("%s: Data %s timeout\n",
564 - mmc_hostname(host->mmc),
565 - (host->data->flags & MMC_DATA_READ) ? "read" : "write");
566 - bcm2835_sdhost_dumpregs(host);
567 - host->data->error = -ETIMEDOUT;
570 + kunmap_atomic(page);
574 + bcm2835_sdhost_finish_data(host);
576 + log_event("DMA>", (u32)host->data, 0);
577 + spin_unlock_irqrestore(&host->lock, flags);
580 static void bcm2835_sdhost_read_block_pio(struct bcm2835_host *host)
581 @@ -493,32 +553,83 @@ static void bcm2835_sdhost_read_block_pi
585 + unsigned long wait_max;
587 blksize = host->data->blksz;
589 + wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
591 local_irq_save(flags);
594 - if (!sg_miter_next(&host->sg_miter))
599 + if (!sg_miter_next(&host->sg_miter)) {
600 + host->data->error = -EINVAL;
604 len = min(host->sg_miter.length, blksize);
607 + host->data->error = -EINVAL;
612 host->sg_miter.consumed = len;
614 buf = (u32 *)host->sg_miter.addr;
617 - if (!data_transfer_wait(host))
619 + copy_words = len/4;
621 + while (copy_words) {
622 + int burst_words, words;
625 + burst_words = SDDATA_FIFO_PIO_BURST;
626 + if (burst_words > copy_words)
627 + burst_words = copy_words;
628 + edm = bcm2835_sdhost_read(host, SDEDM);
629 + words = ((edm >> 4) & 0x1f);
631 + if (words < burst_words) {
632 + int fsm_state = (edm & SDEDM_FSM_MASK);
633 + if ((fsm_state != SDEDM_FSM_READDATA) &&
634 + (fsm_state != SDEDM_FSM_READWAIT) &&
635 + (fsm_state != SDEDM_FSM_READCRC)) {
636 + hsts = bcm2835_sdhost_read(host,
638 + pr_err("%s: fsm %x, hsts %x\n",
639 + mmc_hostname(host->mmc),
641 + if (hsts & SDHSTS_ERROR_MASK)
645 + if (time_after(jiffies, wait_max)) {
646 + pr_err("%s: PIO read timeout - EDM %x\n",
647 + mmc_hostname(host->mmc),
649 + hsts = SDHSTS_REW_TIME_OUT;
652 + ndelay((burst_words - words) *
653 + host->ns_per_fifo_word);
655 + } else if (words > copy_words) {
656 + words = copy_words;
659 + copy_words -= words;
661 - *(buf++) = bcm2835_sdhost_read(host, SDDATA);
664 + *(buf++) = bcm2835_sdhost_read(host, SDDATA);
669 - if (host->data->error)
670 + if (hsts & SDHSTS_ERROR_MASK)
674 @@ -532,32 +643,83 @@ static void bcm2835_sdhost_write_block_p
678 + unsigned long wait_max;
680 blksize = host->data->blksz;
682 + wait_max = jiffies + msecs_to_jiffies(host->pio_timeout);
684 local_irq_save(flags);
687 - if (!sg_miter_next(&host->sg_miter))
692 + if (!sg_miter_next(&host->sg_miter)) {
693 + host->data->error = -EINVAL;
697 len = min(host->sg_miter.length, blksize);
700 + host->data->error = -EINVAL;
705 host->sg_miter.consumed = len;
707 - buf = host->sg_miter.addr;
708 + buf = (u32 *)host->sg_miter.addr;
711 - if (!data_transfer_wait(host))
713 + copy_words = len/4;
715 + while (copy_words) {
716 + int burst_words, words;
719 + burst_words = SDDATA_FIFO_PIO_BURST;
720 + if (burst_words > copy_words)
721 + burst_words = copy_words;
722 + edm = bcm2835_sdhost_read(host, SDEDM);
723 + words = SDDATA_FIFO_WORDS - ((edm >> 4) & 0x1f);
725 + if (words < burst_words) {
726 + int fsm_state = (edm & SDEDM_FSM_MASK);
727 + if ((fsm_state != SDEDM_FSM_WRITEDATA) &&
728 + (fsm_state != SDEDM_FSM_WRITESTART1) &&
729 + (fsm_state != SDEDM_FSM_WRITESTART2)) {
730 + hsts = bcm2835_sdhost_read(host,
732 + pr_err("%s: fsm %x, hsts %x\n",
733 + mmc_hostname(host->mmc),
735 + if (hsts & SDHSTS_ERROR_MASK)
739 - bcm2835_sdhost_write(host, *(buf++), SDDATA);
741 + if (time_after(jiffies, wait_max)) {
742 + pr_err("%s: PIO write timeout - EDM %x\n",
743 + mmc_hostname(host->mmc),
745 + hsts = SDHSTS_REW_TIME_OUT;
748 + ndelay((burst_words - words) *
749 + host->ns_per_fifo_word);
751 + } else if (words > copy_words) {
752 + words = copy_words;
755 + copy_words -= words;
758 + bcm2835_sdhost_write(host, *(buf++), SDDATA);
763 - if (host->data->error)
764 + if (hsts & SDHSTS_ERROR_MASK)
768 @@ -566,12 +728,12 @@ static void bcm2835_sdhost_write_block_p
769 local_irq_restore(flags);
773 static void bcm2835_sdhost_transfer_pio(struct bcm2835_host *host)
778 + log_event("XFP<", (u32)host->data, host->blocks);
780 is_read = (host->data->flags & MMC_DATA_READ) != 0;
782 @@ -595,28 +757,21 @@ static void bcm2835_sdhost_transfer_pio(
783 is_read ? "read" : "write",
785 host->data->error = -ETIMEDOUT;
786 - } else if (!is_read && !host->data->error) {
787 - /* Start a timer in case a transfer error occurs because
788 - there is no error interrupt */
789 - mod_timer(&host->pio_timer, jiffies + host->pio_timeout);
791 + log_event("XFP>", (u32)host->data, host->blocks);
795 -static void bcm2835_sdhost_transfer_dma(struct bcm2835_host *host)
796 +static void bcm2835_sdhost_prepare_dma(struct bcm2835_host *host,
797 + struct mmc_data *data)
799 - u32 len, dir_data, dir_slave;
800 + int len, dir_data, dir_slave;
801 struct dma_async_tx_descriptor *desc = NULL;
802 struct dma_chan *dma_chan;
804 - pr_debug("bcm2835_sdhost_transfer_dma()\n");
806 - WARN_ON(!host->data);
807 + log_event("PRD<", (u32)data, 0);
808 + pr_debug("bcm2835_sdhost_prepare_dma()\n");
813 - if (host->data->flags & MMC_DATA_READ) {
814 + if (data->flags & MMC_DATA_READ) {
815 dma_chan = host->dma_chan_rx;
816 dir_data = DMA_FROM_DEVICE;
817 dir_slave = DMA_DEV_TO_MEM;
818 @@ -625,35 +780,71 @@ static void bcm2835_sdhost_transfer_dma(
819 dir_data = DMA_TO_DEVICE;
820 dir_slave = DMA_MEM_TO_DEV;
822 + log_event("PRD1", (u32)dma_chan, 0);
824 BUG_ON(!dma_chan->device);
825 BUG_ON(!dma_chan->device->dev);
826 - BUG_ON(!host->data->sg);
829 - len = dma_map_sg(dma_chan->device->dev, host->data->sg,
830 - host->data->sg_len, dir_data);
832 - desc = dmaengine_prep_slave_sg(dma_chan, host->data->sg,
833 + /* The block doesn't manage the FIFO DREQs properly for multi-block
834 + transfers, so don't attempt to DMA the final few words.
835 + Unfortunately this requires the final sg entry to be trimmed.
836 + N.B. This code demands that the overspill is contained in
840 + host->drain_words = 0;
841 + if ((data->blocks > 1) && (dir_data == DMA_FROM_DEVICE)) {
842 + struct scatterlist *sg;
846 + len = min((u32)(FIFO_READ_THRESHOLD - 1) * 4,
847 + (u32)data->blocks * data->blksz);
849 + for_each_sg(data->sg, sg, data->sg_len, i) {
850 + if (sg_is_last(sg)) {
851 + BUG_ON(sg->length < len);
853 + host->drain_page = (struct page *)sg->page_link;
854 + host->drain_offset = sg->offset + sg->length;
857 + host->drain_words = len/4;
860 + len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
863 + log_event("PRD2", len, 0);
865 + desc = dmaengine_prep_slave_sg(dma_chan, data->sg,
867 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
869 - dev_err(mmc_dev(host->mmc), "dma_map_sg returned zero length\n");
871 + log_event("PRD3", (u32)desc, 0);
874 desc->callback = bcm2835_sdhost_dma_complete;
875 desc->callback_param = host;
876 - dmaengine_submit(desc);
877 - dma_async_issue_pending(dma_chan);
878 + host->dma_desc = desc;
879 + host->dma_chan = dma_chan;
880 + host->dma_dir = dir_data;
883 + log_event("PDM>", (u32)data, 0);
886 +static void bcm2835_sdhost_start_dma(struct bcm2835_host *host)
888 + log_event("SDMA", (u32)host->data, (u32)host->dma_chan);
889 + dmaengine_submit(host->dma_desc);
890 + dma_async_issue_pending(host->dma_chan);
893 static void bcm2835_sdhost_set_transfer_irqs(struct bcm2835_host *host)
895 u32 all_irqs = SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN |
898 + if (host->dma_desc)
899 host->hcfg = (host->hcfg & ~all_irqs) |
902 @@ -664,13 +855,13 @@ static void bcm2835_sdhost_set_transfer_
903 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
907 static void bcm2835_sdhost_prepare_data(struct bcm2835_host *host, struct mmc_command *cmd)
909 struct mmc_data *data = cmd->data;
917 @@ -679,46 +870,19 @@ static void bcm2835_sdhost_prepare_data(
918 BUG_ON(data->blksz > host->mmc->max_blk_size);
919 BUG_ON(data->blocks > 65535);
922 host->data_complete = 0;
923 host->flush_fifo = 0;
924 host->data->bytes_xfered = 0;
926 - if (!host->sectors && host->mmc->card && !(host->debug_flags & 1))
928 - struct mmc_card *card = host->mmc->card;
929 - if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
931 - * The EXT_CSD sector count is in number of 512 byte
934 - host->sectors = card->ext_csd.sectors;
935 - pr_err("%s: using ext_csd!\n", mmc_hostname(host->mmc));
938 - * The CSD capacity field is in units of read_blkbits.
939 - * set_capacity takes units of 512 bytes.
941 - host->sectors = card->csd.capacity <<
942 - (card->csd.read_blkbits - 9);
944 - host->single_read_sectors[0] = host->sectors - 65;
945 - host->single_read_sectors[1] = host->sectors - 64;
946 - host->single_read_sectors[2] = host->sectors - 33;
947 - host->single_read_sectors[3] = host->sectors - 32;
948 - host->single_read_sectors[4] = host->sectors - 1;
949 - host->single_read_sectors[5] = ~0; /* Safety net */
952 - host->use_dma = host->have_dma && (data->blocks > host->pio_limit);
953 - if (!host->use_dma) {
954 + if (!host->dma_desc) {
958 - flags = SG_MITER_ATOMIC;
959 if (data->flags & MMC_DATA_READ)
960 - flags |= SG_MITER_TO_SG;
961 + flags = SG_MITER_TO_SG;
963 - flags |= SG_MITER_FROM_SG;
964 + flags = SG_MITER_FROM_SG;
965 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
966 host->blocks = data->blocks;
968 @@ -726,19 +890,20 @@ static void bcm2835_sdhost_prepare_data(
969 bcm2835_sdhost_set_transfer_irqs(host);
971 bcm2835_sdhost_write(host, data->blksz, SDHBCT);
972 - bcm2835_sdhost_write(host, host->use_dma ? data->blocks : 0, SDHBLC);
973 + bcm2835_sdhost_write(host, data->blocks, SDHBLC);
979 -void bcm2835_sdhost_send_command(struct bcm2835_host *host, struct mmc_command *cmd)
980 +bool bcm2835_sdhost_send_command(struct bcm2835_host *host,
981 + struct mmc_command *cmd)
984 unsigned long timeout;
988 + log_event("CMD<", cmd->opcode, cmd->arg);
991 pr_debug("%s: send_command %d 0x%x "
992 @@ -761,9 +926,9 @@ void bcm2835_sdhost_send_command(struct
993 pr_err("%s: previous command never completed.\n",
994 mmc_hostname(host->mmc));
995 bcm2835_sdhost_dumpregs(host);
997 + cmd->error = -EILSEQ;
998 tasklet_schedule(&host->finish_tasklet);
1004 @@ -791,23 +956,24 @@ void bcm2835_sdhost_send_command(struct
1005 if (sdhsts & SDHSTS_ERROR_MASK)
1006 bcm2835_sdhost_write(host, sdhsts, SDHSTS);
1008 - bcm2835_sdhost_prepare_data(host, cmd);
1010 - bcm2835_sdhost_write(host, cmd->arg, SDARG);
1012 if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1013 pr_err("%s: unsupported response type!\n",
1014 mmc_hostname(host->mmc));
1015 cmd->error = -EINVAL;
1016 tasklet_schedule(&host->finish_tasklet);
1021 + bcm2835_sdhost_prepare_data(host, cmd);
1023 + bcm2835_sdhost_write(host, cmd->arg, SDARG);
1025 sdcmd = cmd->opcode & SDCMD_CMD_MASK;
1027 - if (!(cmd->flags & MMC_RSP_PRESENT))
1028 + host->use_busy = 0;
1029 + if (!(cmd->flags & MMC_RSP_PRESENT)) {
1030 sdcmd |= SDCMD_NO_RESPONSE;
1033 if (cmd->flags & MMC_RSP_136)
1034 sdcmd |= SDCMD_LONG_RESPONSE;
1035 if (cmd->flags & MMC_RSP_BUSY) {
1036 @@ -817,6 +983,7 @@ void bcm2835_sdhost_send_command(struct
1040 + log_event("CMDD", cmd->data->blocks, cmd->data->blksz);
1041 if (host->delay_after_stop) {
1043 int time_since_stop;
1044 @@ -839,10 +1006,12 @@ void bcm2835_sdhost_send_command(struct
1047 bcm2835_sdhost_write(host, sdcmd | SDCMD_NEW_FLAG, SDCMD);
1053 -static void bcm2835_sdhost_finish_command(struct bcm2835_host *host);
1054 +static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1055 + unsigned long *irq_flags);
1056 static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host);
1058 static void bcm2835_sdhost_finish_data(struct bcm2835_host *host)
1059 @@ -852,6 +1021,7 @@ static void bcm2835_sdhost_finish_data(s
1063 + log_event("FDA<", (u32)host->mrq, (u32)host->cmd);
1064 pr_debug("finish_data(error %d, stop %d, sbc %d)\n",
1065 data->error, data->stop ? 1 : 0,
1066 host->mrq->sbc ? 1 : 0);
1067 @@ -859,10 +1029,7 @@ static void bcm2835_sdhost_finish_data(s
1068 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
1069 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1071 - if (data->error) {
1072 - data->bytes_xfered = 0;
1074 - data->bytes_xfered = data->blksz * data->blocks;
1075 + data->bytes_xfered = data->error ? 0 : (data->blksz * data->blocks);
1077 host->data_complete = 1;
1079 @@ -877,9 +1044,9 @@ static void bcm2835_sdhost_finish_data(s
1082 bcm2835_sdhost_transfer_complete(host);
1083 + log_event("FDA>", (u32)host->mrq, (u32)host->cmd);
1087 static void bcm2835_sdhost_transfer_complete(struct bcm2835_host *host)
1089 struct mmc_data *data;
1090 @@ -891,6 +1058,7 @@ static void bcm2835_sdhost_transfer_comp
1094 + log_event("TCM<", (u32)data, data->error);
1095 pr_debug("transfer_complete(error %d, stop %d)\n",
1096 data->error, data->stop ? 1 : 0);
1098 @@ -899,88 +1067,114 @@ static void bcm2835_sdhost_transfer_comp
1099 * a) open-ended multiblock transfer (no CMD23)
1100 * b) error in multiblock transfer
1104 - !host->mrq->sbc)) {
1105 - host->flush_fifo = 1;
1106 - bcm2835_sdhost_send_command(host, data->stop);
1107 - if (host->delay_after_stop)
1108 - do_gettimeofday(&host->stop_time);
1109 - if (!host->use_busy)
1110 - bcm2835_sdhost_finish_command(host);
1111 + if (host->mrq->stop && (data->error || !host->use_sbc)) {
1112 + if (bcm2835_sdhost_send_command(host, host->mrq->stop)) {
1113 + /* No busy, so poll for completion */
1114 + if (!host->use_busy)
1115 + bcm2835_sdhost_finish_command(host, NULL);
1117 + if (host->delay_after_stop)
1118 + do_gettimeofday(&host->stop_time);
1121 + bcm2835_sdhost_wait_transfer_complete(host);
1122 tasklet_schedule(&host->finish_tasklet);
1124 + log_event("TCM>", (u32)data, 0);
1127 -static void bcm2835_sdhost_finish_command(struct bcm2835_host *host)
1128 +/* If irq_flags is valid, the caller is in a thread context and is allowed
1130 +static void bcm2835_sdhost_finish_command(struct bcm2835_host *host,
1131 + unsigned long *irq_flags)
1134 - unsigned long timeout;
1137 struct timeval before, after;
1141 + log_event("FCM<", (u32)host->mrq, (u32)host->cmd);
1142 pr_debug("finish_command(%x)\n", bcm2835_sdhost_read(host, SDCMD));
1144 BUG_ON(!host->cmd || !host->mrq);
1147 - do_gettimeofday(&before);
1149 - /* Wait max 100 ms */
1151 + /* Poll quickly at first */
1153 + retries = host->cmd_quick_poll_retries;
1155 + /* Work out how many polls take 1us by timing 10us */
1156 + struct timeval start, now;
1165 + do_gettimeofday(&start);
1167 + for (i = 0; i < retries; i++) {
1169 + sdcmd = bcm2835_sdhost_read(host, SDCMD);
1172 + do_gettimeofday(&now);
1173 + us_diff = (now.tv_sec - start.tv_sec) * 1000000 +
1174 + (now.tv_usec - start.tv_usec);
1175 + } while (us_diff < 10);
1177 + host->cmd_quick_poll_retries = ((retries * us_diff + 9)*CMD_DALLY_US)/10 + 1;
1178 + retries = 1; // We've already waited long enough this time
1181 + retries = host->cmd_quick_poll_retries;
1182 for (sdcmd = bcm2835_sdhost_read(host, SDCMD);
1183 - (sdcmd & SDCMD_NEW_FLAG) && timeout;
1185 - if (host->flush_fifo) {
1186 - while (bcm2835_sdhost_read(host, SDHSTS) &
1188 - (void)bcm2835_sdhost_read(host, SDDATA);
1191 + (sdcmd & SDCMD_NEW_FLAG) && !(sdcmd & SDCMD_FAIL_FLAG) && retries;
1194 sdcmd = bcm2835_sdhost_read(host, SDCMD);
1197 - do_gettimeofday(&after);
1198 - timediff = (after.tv_sec - before.tv_sec)*1000000 +
1199 - (after.tv_usec - before.tv_usec);
1201 - pr_debug(" finish_command - waited %dus\n", timediff);
1204 + unsigned long wait_max;
1207 + /* Schedule the work */
1208 + log_event("CWWQ", 0, 0);
1209 + schedule_work(&host->cmd_wait_wq);
1213 + /* Wait max 100 ms */
1214 + wait_max = jiffies + msecs_to_jiffies(100);
1215 + while (time_before(jiffies, wait_max)) {
1216 + spin_unlock_irqrestore(&host->lock, *irq_flags);
1217 + usleep_range(1, 10);
1218 + spin_lock_irqsave(&host->lock, *irq_flags);
1219 + sdcmd = bcm2835_sdhost_read(host, SDCMD);
1220 + if (!(sdcmd & SDCMD_NEW_FLAG) ||
1221 + (sdcmd & SDCMD_FAIL_FLAG))
1226 - if (timeout == 0) {
1227 + /* Check for errors */
1228 + if (sdcmd & SDCMD_NEW_FLAG) {
1229 pr_err("%s: command never completed.\n",
1230 mmc_hostname(host->mmc));
1231 bcm2835_sdhost_dumpregs(host);
1232 host->cmd->error = -EIO;
1233 tasklet_schedule(&host->finish_tasklet);
1237 - if (host->flush_fifo) {
1238 - for (timeout = 100;
1239 - (bcm2835_sdhost_read(host, SDHSTS) & SDHSTS_DATA_FLAG) && timeout;
1241 - (void)bcm2835_sdhost_read(host, SDDATA);
1243 - host->flush_fifo = 0;
1244 - if (timeout == 0) {
1245 - pr_err("%s: FIFO never drained.\n",
1246 - mmc_hostname(host->mmc));
1247 - bcm2835_sdhost_dumpregs(host);
1248 - host->cmd->error = -EIO;
1249 - tasklet_schedule(&host->finish_tasklet);
1254 - /* Check for errors */
1255 - if (sdcmd & SDCMD_FAIL_FLAG)
1257 + } else if (sdcmd & SDCMD_FAIL_FLAG) {
1258 u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1260 + /* Clear the errors */
1261 + bcm2835_sdhost_write(host, SDHSTS_ERROR_MASK, SDHSTS);
1264 pr_info("%s: error detected - CMD %x, HSTS %03x, EDM %x\n",
1265 mmc_hostname(host->mmc), sdcmd, sdhsts,
1266 @@ -1003,7 +1197,7 @@ static void bcm2835_sdhost_finish_comman
1267 mmc_hostname(host->mmc),
1269 bcm2835_sdhost_dumpregs(host);
1270 - host->cmd->error = -EIO;
1271 + host->cmd->error = -EILSEQ;
1273 tasklet_schedule(&host->finish_tasklet);
1275 @@ -1018,31 +1212,31 @@ static void bcm2835_sdhost_finish_comman
1276 pr_debug("%s: finish_command %08x %08x %08x %08x\n",
1277 mmc_hostname(host->mmc),
1278 host->cmd->resp[0], host->cmd->resp[1], host->cmd->resp[2], host->cmd->resp[3]);
1279 + log_event("RSP ", host->cmd->resp[0], host->cmd->resp[1]);
1281 host->cmd->resp[0] = bcm2835_sdhost_read(host, SDRSP0);
1282 pr_debug("%s: finish_command %08x\n",
1283 mmc_hostname(host->mmc),
1284 host->cmd->resp[0]);
1285 + log_event("RSP ", host->cmd->resp[0], 0);
1289 - host->cmd->error = 0;
1291 if (host->cmd == host->mrq->sbc) {
1292 /* Finished CMD23, now send actual command. */
1294 - bcm2835_sdhost_send_command(host, host->mrq->cmd);
1295 + if (bcm2835_sdhost_send_command(host, host->mrq->cmd)) {
1296 + if (host->data && host->dma_desc)
1297 + /* DMA transfer starts now, PIO starts after irq */
1298 + bcm2835_sdhost_start_dma(host);
1300 - if (host->cmd->data && host->use_dma)
1301 - /* DMA transfer starts now, PIO starts after irq */
1302 - bcm2835_sdhost_transfer_dma(host);
1304 - if (!host->use_busy)
1305 - bcm2835_sdhost_finish_command(host);
1306 - } else if (host->cmd == host->mrq->stop)
1307 + if (!host->use_busy)
1308 + bcm2835_sdhost_finish_command(host, NULL);
1310 + } else if (host->cmd == host->mrq->stop) {
1311 /* Finished CMD12 */
1312 tasklet_schedule(&host->finish_tasklet);
1315 /* Processed actual command. */
1318 @@ -1050,6 +1244,7 @@ static void bcm2835_sdhost_finish_comman
1319 else if (host->data_complete)
1320 bcm2835_sdhost_transfer_complete(host);
1322 + log_event("FCM>", (u32)host->mrq, (u32)host->cmd);
1325 static void bcm2835_sdhost_timeout(unsigned long data)
1326 @@ -1060,10 +1255,12 @@ static void bcm2835_sdhost_timeout(unsig
1327 host = (struct bcm2835_host *)data;
1329 spin_lock_irqsave(&host->lock, flags);
1330 + log_event("TIM<", 0, 0);
1333 pr_err("%s: timeout waiting for hardware interrupt.\n",
1334 mmc_hostname(host->mmc));
1336 bcm2835_sdhost_dumpregs(host);
1339 @@ -1084,74 +1281,15 @@ static void bcm2835_sdhost_timeout(unsig
1340 spin_unlock_irqrestore(&host->lock, flags);
1343 -static void bcm2835_sdhost_pio_timeout(unsigned long data)
1345 - struct bcm2835_host *host;
1346 - unsigned long flags;
1348 - host = (struct bcm2835_host *)data;
1350 - spin_lock_irqsave(&host->lock, flags);
1353 - u32 sdhsts = bcm2835_sdhost_read(host, SDHSTS);
1355 - if (sdhsts & SDHSTS_REW_TIME_OUT) {
1356 - pr_err("%s: transfer timeout\n",
1357 - mmc_hostname(host->mmc));
1359 - bcm2835_sdhost_dumpregs(host);
1361 - pr_err("%s: unexpected transfer timeout\n",
1362 - mmc_hostname(host->mmc));
1363 - bcm2835_sdhost_dumpregs(host);
1366 - bcm2835_sdhost_write(host, SDHSTS_TRANSFER_ERROR_MASK,
1369 - host->data->error = -ETIMEDOUT;
1371 - bcm2835_sdhost_finish_data(host);
1375 - spin_unlock_irqrestore(&host->lock, flags);
1378 -static void bcm2835_sdhost_enable_sdio_irq_nolock(struct bcm2835_host *host, int enable)
1381 - host->hcfg |= SDHCFG_SDIO_IRPT_EN;
1383 - host->hcfg &= ~SDHCFG_SDIO_IRPT_EN;
1384 - bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1388 -static void bcm2835_sdhost_enable_sdio_irq(struct mmc_host *mmc, int enable)
1390 - struct bcm2835_host *host = mmc_priv(mmc);
1391 - unsigned long flags;
1393 - pr_debug("%s: enable_sdio_irq(%d)\n", mmc_hostname(mmc), enable);
1394 - spin_lock_irqsave(&host->lock, flags);
1395 - bcm2835_sdhost_enable_sdio_irq_nolock(host, enable);
1396 - spin_unlock_irqrestore(&host->lock, flags);
1399 -static u32 bcm2835_sdhost_busy_irq(struct bcm2835_host *host, u32 intmask)
1400 +static void bcm2835_sdhost_busy_irq(struct bcm2835_host *host, u32 intmask)
1402 - const u32 handled = (SDHSTS_REW_TIME_OUT | SDHSTS_CMD_TIME_OUT |
1403 - SDHSTS_CRC16_ERROR | SDHSTS_CRC7_ERROR |
1404 - SDHSTS_FIFO_ERROR);
1406 + log_event("IRQB", (u32)host->cmd, intmask);
1408 pr_err("%s: got command busy interrupt 0x%08x even "
1409 "though no command operation was in progress.\n",
1410 mmc_hostname(host->mmc), (unsigned)intmask);
1411 bcm2835_sdhost_dumpregs(host);
1416 if (!host->use_busy) {
1417 @@ -1159,7 +1297,7 @@ static u32 bcm2835_sdhost_busy_irq(struc
1418 "though not expecting one.\n",
1419 mmc_hostname(host->mmc), (unsigned)intmask);
1420 bcm2835_sdhost_dumpregs(host);
1426 @@ -1182,28 +1320,23 @@ static u32 bcm2835_sdhost_busy_irq(struc
1427 } else if (intmask & SDHSTS_CMD_TIME_OUT)
1428 host->cmd->error = -ETIMEDOUT;
1431 bcm2835_sdhost_dumpregs(host);
1432 - tasklet_schedule(&host->finish_tasklet);
1435 - bcm2835_sdhost_finish_command(host);
1438 + bcm2835_sdhost_finish_command(host, NULL);
1441 -static u32 bcm2835_sdhost_data_irq(struct bcm2835_host *host, u32 intmask)
1442 +static void bcm2835_sdhost_data_irq(struct bcm2835_host *host, u32 intmask)
1444 - const u32 handled = (SDHSTS_REW_TIME_OUT |
1445 - SDHSTS_CRC16_ERROR |
1446 - SDHSTS_FIFO_ERROR);
1448 /* There are no dedicated data/space available interrupt
1449 status bits, so it is necessary to use the single shared
1450 data/space available FIFO status bits. It is therefore not
1451 an error to get here when there is no data transfer in
1453 + log_event("IRQD", (u32)host->data, intmask);
1458 if (intmask & (SDHSTS_CRC16_ERROR |
1460 @@ -1214,46 +1347,37 @@ static u32 bcm2835_sdhost_data_irq(struc
1462 host->data->error = -ETIMEDOUT;
1464 - bcm2835_sdhost_dumpregs(host);
1465 - tasklet_schedule(&host->finish_tasklet);
1467 + if (host->debug) {
1469 + bcm2835_sdhost_dumpregs(host);
1473 - /* Use the block interrupt for writes after the first block */
1474 - if (host->data->flags & MMC_DATA_WRITE) {
1475 + if (host->data->error) {
1476 + bcm2835_sdhost_finish_data(host);
1477 + } else if (host->data->flags & MMC_DATA_WRITE) {
1478 + /* Use the block interrupt for writes after the first block */
1479 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN);
1480 host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
1481 bcm2835_sdhost_write(host, host->hcfg, SDHCFG);
1482 - if (host->data->error)
1483 - bcm2835_sdhost_finish_data(host);
1485 - bcm2835_sdhost_transfer_pio(host);
1486 + bcm2835_sdhost_transfer_pio(host);
1488 - if (!host->data->error) {
1489 - bcm2835_sdhost_transfer_pio(host);
1492 + bcm2835_sdhost_transfer_pio(host);
1494 if ((host->blocks == 0) || host->data->error)
1495 bcm2835_sdhost_finish_data(host);
1501 -static u32 bcm2835_sdhost_block_irq(struct bcm2835_host *host, u32 intmask)
1502 +static void bcm2835_sdhost_block_irq(struct bcm2835_host *host, u32 intmask)
1504 - struct dma_chan *dma_chan;
1506 - const u32 handled = (SDHSTS_REW_TIME_OUT |
1507 - SDHSTS_CRC16_ERROR |
1508 - SDHSTS_FIFO_ERROR);
1510 + log_event("IRQK", (u32)host->data, intmask);
1512 pr_err("%s: got block interrupt 0x%08x even "
1513 "though no data operation was in progress.\n",
1514 mmc_hostname(host->mmc), (unsigned)intmask);
1515 bcm2835_sdhost_dumpregs(host);
1520 if (intmask & (SDHSTS_CRC16_ERROR |
1521 @@ -1265,149 +1389,69 @@ static u32 bcm2835_sdhost_block_irq(stru
1523 host->data->error = -ETIMEDOUT;
1526 + if (host->debug) {
1528 bcm2835_sdhost_dumpregs(host);
1529 - tasklet_schedule(&host->finish_tasklet);
1534 - if (!host->use_dma) {
1535 + if (!host->dma_desc) {
1536 BUG_ON(!host->blocks);
1538 - if ((host->blocks == 0) || host->data->error) {
1539 - /* Cancel the timer */
1540 - del_timer(&host->pio_timer);
1542 + if (host->data->error || (--host->blocks == 0)) {
1543 bcm2835_sdhost_finish_data(host);
1545 - /* Reset the timer */
1546 - mod_timer(&host->pio_timer,
1547 - jiffies + host->pio_timeout);
1549 bcm2835_sdhost_transfer_pio(host);
1551 - /* Reset the timer */
1552 - mod_timer(&host->pio_timer,
1553 - jiffies + host->pio_timeout);
1555 } else if (host->data->flags & MMC_DATA_WRITE) {
1556 - dma_chan = host->dma_chan_tx;
1557 - dir_data = DMA_TO_DEVICE;
1558 - dma_unmap_sg(dma_chan->device->dev,
1559 - host->data->sg, host->data->sg_len,
1562 bcm2835_sdhost_finish_data(host);
1569 static irqreturn_t bcm2835_sdhost_irq(int irq, void *dev_id)
1571 irqreturn_t result = IRQ_NONE;
1572 struct bcm2835_host *host = dev_id;
1573 - u32 unexpected = 0, early = 0;
1577 spin_lock(&host->lock);
1579 - for (loops = 0; loops < 1; loops++) {
1580 - u32 intmask, handled;
1582 - intmask = bcm2835_sdhost_read(host, SDHSTS);
1583 - handled = intmask & (SDHSTS_BUSY_IRPT |
1584 - SDHSTS_BLOCK_IRPT |
1585 - SDHSTS_SDIO_IRPT |
1586 - SDHSTS_DATA_FLAG);
1587 - if ((handled == SDHSTS_DATA_FLAG) &&
1588 - (loops == 0) && !host->data) {
1589 - pr_err("%s: sdhost_irq data interrupt 0x%08x even "
1590 - "though no data operation was in progress.\n",
1591 - mmc_hostname(host->mmc),
1592 - (unsigned)intmask);
1594 - bcm2835_sdhost_dumpregs(host);
1599 + intmask = bcm2835_sdhost_read(host, SDHSTS);
1600 + log_event("IRQ<", intmask, 0);
1604 + bcm2835_sdhost_write(host,
1605 + SDHSTS_BUSY_IRPT |
1606 + SDHSTS_BLOCK_IRPT |
1607 + SDHSTS_SDIO_IRPT |
1611 + if (intmask & SDHSTS_BLOCK_IRPT) {
1612 + bcm2835_sdhost_block_irq(host, intmask);
1613 result = IRQ_HANDLED;
1616 - /* Clear all interrupts and notifications */
1617 - bcm2835_sdhost_write(host, intmask, SDHSTS);
1619 - if (intmask & SDHSTS_BUSY_IRPT)
1620 - handled |= bcm2835_sdhost_busy_irq(host, intmask);
1622 - /* There is no true data interrupt status bit, so it is
1623 - necessary to qualify the data flag with the interrupt
1625 - if ((intmask & SDHSTS_DATA_FLAG) &&
1626 - (host->hcfg & SDHCFG_DATA_IRPT_EN))
1627 - handled |= bcm2835_sdhost_data_irq(host, intmask);
1629 - if (intmask & SDHSTS_BLOCK_IRPT)
1630 - handled |= bcm2835_sdhost_block_irq(host, intmask);
1632 - if (intmask & SDHSTS_SDIO_IRPT) {
1633 - bcm2835_sdhost_enable_sdio_irq_nolock(host, false);
1634 - host->thread_isr |= SDHSTS_SDIO_IRPT;
1635 - result = IRQ_WAKE_THREAD;
1637 + if (intmask & SDHSTS_BUSY_IRPT) {
1638 + bcm2835_sdhost_busy_irq(host, intmask);
1639 + result = IRQ_HANDLED;
1642 - unexpected |= (intmask & ~handled);
1643 + /* There is no true data interrupt status bit, so it is
1644 + necessary to qualify the data flag with the interrupt
1646 + if ((intmask & SDHSTS_DATA_FLAG) &&
1647 + (host->hcfg & SDHCFG_DATA_IRPT_EN)) {
1648 + bcm2835_sdhost_data_irq(host, intmask);
1649 + result = IRQ_HANDLED;
1654 + log_event("IRQ>", bcm2835_sdhost_read(host, SDHSTS), 0);
1655 spin_unlock(&host->lock);
1658 - pr_debug("%s: early %x (loops %d)\n",
1659 - mmc_hostname(host->mmc), early, loops);
1662 - pr_err("%s: unexpected interrupt 0x%08x.\n",
1663 - mmc_hostname(host->mmc), unexpected);
1664 - bcm2835_sdhost_dumpregs(host);
1670 -static irqreturn_t bcm2835_sdhost_thread_irq(int irq, void *dev_id)
1672 - struct bcm2835_host *host = dev_id;
1673 - unsigned long flags;
1676 - spin_lock_irqsave(&host->lock, flags);
1677 - isr = host->thread_isr;
1678 - host->thread_isr = 0;
1679 - spin_unlock_irqrestore(&host->lock, flags);
1681 - if (isr & SDHSTS_SDIO_IRPT) {
1682 - sdio_run_irqs(host->mmc);
1684 -/* Is this necessary? Why re-enable an interrupt which is enabled?
1685 - spin_lock_irqsave(&host->lock, flags);
1686 - if (host->flags & SDHSTS_SDIO_IRPT_ENABLED)
1687 - bcm2835_sdhost_enable_sdio_irq_nolock(host, true);
1688 - spin_unlock_irqrestore(&host->lock, flags);
1692 - return isr ? IRQ_HANDLED : IRQ_NONE;
1697 void bcm2835_sdhost_set_clock(struct bcm2835_host *host, unsigned int clock)
1699 int div = 0; /* Initialized for compiler warning */
1700 @@ -1417,9 +1461,8 @@ void bcm2835_sdhost_set_clock(struct bcm
1701 pr_info("%s: set_clock(%d)\n", mmc_hostname(host->mmc), clock);
1703 if ((host->overclock_50 > 50) &&
1704 - (clock == 50*MHZ)) {
1705 + (clock == 50*MHZ))
1706 clock = host->overclock_50 * MHZ + (MHZ - 1);
1709 /* The SDCDIV register has 11 bits, and holds (div - 2).
1710 But in data mode the max is 50MHz wihout a minimum, and only the
1711 @@ -1466,6 +1509,11 @@ void bcm2835_sdhost_set_clock(struct bcm
1712 clock = host->max_clk / (div + 2);
1713 host->mmc->actual_clock = clock;
1715 + /* Calibrate some delays */
1717 + host->ns_per_fifo_word = (1000000000/clock) *
1718 + ((host->mmc->caps & MMC_CAP_4_BIT_DATA) ? 8 : 32);
1720 if (clock > input_clock) {
1721 /* Save the closest value, to make it easier
1722 to reduce in the event of error */
1723 @@ -1501,6 +1549,7 @@ static void bcm2835_sdhost_request(struc
1725 struct bcm2835_host *host;
1726 unsigned long flags;
1729 host = mmc_priv(mmc);
1731 @@ -1521,6 +1570,8 @@ static void bcm2835_sdhost_request(struc
1734 /* Reset the error statuses in case this is a retry */
1736 + mrq->sbc->error = 0;
1738 mrq->cmd->error = 0;
1740 @@ -1536,28 +1587,58 @@ static void bcm2835_sdhost_request(struc
1744 + if (host->use_dma && mrq->data &&
1745 + (mrq->data->blocks > host->pio_limit))
1746 + bcm2835_sdhost_prepare_dma(host, mrq->data);
1748 spin_lock_irqsave(&host->lock, flags);
1750 WARN_ON(host->mrq != NULL);
1755 - bcm2835_sdhost_send_command(host, mrq->sbc);
1757 - bcm2835_sdhost_send_command(host, mrq->cmd);
1758 + edm = bcm2835_sdhost_read(host, SDEDM);
1759 + fsm = edm & SDEDM_FSM_MASK;
1762 - spin_unlock_irqrestore(&host->lock, flags);
1763 + log_event("REQ<", (u32)mrq, edm);
1764 + if ((fsm != SDEDM_FSM_IDENTMODE) &&
1765 + (fsm != SDEDM_FSM_DATAMODE)) {
1766 + pr_err("%s: previous command (%d) not complete (EDM %x)\n",
1767 + mmc_hostname(host->mmc),
1768 + bcm2835_sdhost_read(host, SDCMD) & SDCMD_CMD_MASK,
1770 + log_event("REQ!", (u32)mrq, edm);
1772 + bcm2835_sdhost_dumpregs(host);
1773 + mrq->cmd->error = -EILSEQ;
1774 + tasklet_schedule(&host->finish_tasklet);
1776 + spin_unlock_irqrestore(&host->lock, flags);
1780 + host->use_sbc = !!mrq->sbc &&
1781 + (host->mrq->data->flags & USE_CMD23_FLAGS);
1782 + if (host->use_sbc) {
1783 + if (bcm2835_sdhost_send_command(host, mrq->sbc)) {
1784 + if (!host->use_busy)
1785 + bcm2835_sdhost_finish_command(host, &flags);
1787 + } else if (bcm2835_sdhost_send_command(host, mrq->cmd)) {
1788 + if (host->data && host->dma_desc)
1789 + /* DMA transfer starts now, PIO starts after irq */
1790 + bcm2835_sdhost_start_dma(host);
1792 - if (!mrq->sbc && mrq->cmd->data && host->use_dma)
1793 - /* DMA transfer starts now, PIO starts after irq */
1794 - bcm2835_sdhost_transfer_dma(host);
1795 + if (!host->use_busy)
1796 + bcm2835_sdhost_finish_command(host, &flags);
1799 - if (!host->use_busy)
1800 - bcm2835_sdhost_finish_command(host);
1802 + log_event("CMD ", (u32)mrq->cmd->opcode,
1803 + mrq->data ? (u32)mrq->data->blksz : 0);
1806 + log_event("REQ>", (u32)mrq, 0);
1807 + spin_unlock_irqrestore(&host->lock, flags);
1810 static void bcm2835_sdhost_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1812 @@ -1574,6 +1655,8 @@ static void bcm2835_sdhost_set_ios(struc
1814 spin_lock_irqsave(&host->lock, flags);
1816 + log_event("IOS<", ios->clock, 0);
1818 if (!ios->clock || ios->clock != host->clock) {
1819 bcm2835_sdhost_set_clock(host, ios->clock);
1820 host->clock = ios->clock;
1821 @@ -1596,59 +1679,53 @@ static void bcm2835_sdhost_set_ios(struc
1822 spin_unlock_irqrestore(&host->lock, flags);
1825 -static int bcm2835_sdhost_multi_io_quirk(struct mmc_card *card,
1826 - unsigned int direction,
1827 - u32 blk_pos, int blk_size)
1829 - /* There is a bug in the host controller hardware that makes
1830 - reading the final sector of the card as part of a multiple read
1831 - problematic. Detect that case and shorten the read accordingly.
1833 +static struct mmc_host_ops bcm2835_sdhost_ops = {
1834 + .request = bcm2835_sdhost_request,
1835 + .set_ios = bcm2835_sdhost_set_ios,
1836 + .hw_reset = bcm2835_sdhost_reset,
1839 +static void bcm2835_sdhost_cmd_wait_work(struct work_struct *work)
1841 struct bcm2835_host *host;
1842 + unsigned long flags;
1844 - host = mmc_priv(card->host);
1845 + host = container_of(work, struct bcm2835_host, cmd_wait_wq);
1847 - if (!host->sectors) {
1848 - /* csd.capacity is in weird units - convert to sectors */
1849 - u32 card_sectors = (card->csd.capacity << (card->csd.read_blkbits - 9));
1850 - if ((direction == MMC_DATA_READ) &&
1851 - ((blk_pos + blk_size) == card_sectors))
1855 + spin_lock_irqsave(&host->lock, flags);
1857 - if (direction == MMC_DATA_READ) {
1860 - for (i = 0; blk_pos > (sector = host->single_read_sectors[i]); i++)
1862 + log_event("CWK<", (u32)host->cmd, (u32)host->mrq);
1864 - if ((blk_pos + blk_size) > sector)
1865 - blk_size = (blk_pos == sector) ? 1 : (sector - blk_pos);
1867 + * If this tasklet gets rescheduled while running, it will
1868 + * be run again afterwards but without any active request.
1871 + spin_unlock_irqrestore(&host->lock, flags);
1877 + bcm2835_sdhost_finish_command(host, &flags);
1879 -static struct mmc_host_ops bcm2835_sdhost_ops = {
1880 - .request = bcm2835_sdhost_request,
1881 - .set_ios = bcm2835_sdhost_set_ios,
1882 - .enable_sdio_irq = bcm2835_sdhost_enable_sdio_irq,
1883 - .hw_reset = bcm2835_sdhost_reset,
1884 - .multi_io_quirk = bcm2835_sdhost_multi_io_quirk,
1888 + log_event("CWK>", (u32)host->cmd, 0);
1890 + spin_unlock_irqrestore(&host->lock, flags);
1893 static void bcm2835_sdhost_tasklet_finish(unsigned long param)
1895 struct bcm2835_host *host;
1896 unsigned long flags;
1897 struct mmc_request *mrq;
1898 + struct dma_chan *terminate_chan = NULL;
1900 host = (struct bcm2835_host *)param;
1902 spin_lock_irqsave(&host->lock, flags);
1904 + log_event("TSK<", (u32)host->mrq, 0);
1906 * If this tasklet gets rescheduled while running, it will
1907 * be run again afterwards but without any active request.
1908 @@ -1683,11 +1760,23 @@ static void bcm2835_sdhost_tasklet_finis
1912 + host->dma_desc = NULL;
1913 + terminate_chan = host->dma_chan;
1914 + host->dma_chan = NULL;
1916 spin_unlock_irqrestore(&host->lock, flags);
1917 - mmc_request_done(host->mmc, mrq);
1920 + if (terminate_chan)
1922 + int err = dmaengine_terminate_all(terminate_chan);
1924 + pr_err("%s: failed to terminate DMA (%d)\n",
1925 + mmc_hostname(host->mmc), err);
1928 + mmc_request_done(host->mmc, mrq);
1929 + log_event("TSK>", (u32)mrq, 0);
1932 int bcm2835_sdhost_add_host(struct bcm2835_host *host)
1934 @@ -1709,10 +1798,10 @@ int bcm2835_sdhost_add_host(struct bcm28
1935 mmc->f_max, mmc->f_min, mmc->max_busy_timeout);
1937 /* host controller capabilities */
1938 - mmc->caps |= /* MMC_CAP_SDIO_IRQ |*/ MMC_CAP_4_BIT_DATA |
1940 MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
1941 MMC_CAP_NEEDS_POLL | MMC_CAP_HW_RESET | MMC_CAP_ERASE |
1942 - (ALLOW_CMD23 * MMC_CAP_CMD23);
1943 + ((ALLOW_CMD23_READ|ALLOW_CMD23_WRITE) * MMC_CAP_CMD23);
1945 spin_lock_init(&host->lock);
1947 @@ -1722,9 +1811,9 @@ int bcm2835_sdhost_add_host(struct bcm28
1948 pr_err("%s: unable to initialise DMA channels. "
1949 "Falling back to PIO\n",
1951 - host->have_dma = false;
1952 + host->use_dma = false;
1954 - host->have_dma = true;
1955 + host->use_dma = true;
1957 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1958 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1959 @@ -1741,7 +1830,7 @@ int bcm2835_sdhost_add_host(struct bcm28
1960 ret = dmaengine_slave_config(host->dma_chan_rx, &cfg);
1963 - host->have_dma = false;
1964 + host->use_dma = false;
1967 mmc->max_segs = 128;
1968 @@ -1756,16 +1845,15 @@ int bcm2835_sdhost_add_host(struct bcm28
1969 tasklet_init(&host->finish_tasklet,
1970 bcm2835_sdhost_tasklet_finish, (unsigned long)host);
1972 - setup_timer(&host->timer, bcm2835_sdhost_timeout,
1973 - (unsigned long)host);
1974 + INIT_WORK(&host->cmd_wait_wq, bcm2835_sdhost_cmd_wait_work);
1976 - setup_timer(&host->pio_timer, bcm2835_sdhost_pio_timeout,
1977 + setup_timer(&host->timer, bcm2835_sdhost_timeout,
1978 (unsigned long)host);
1980 bcm2835_sdhost_init(host, 0);
1981 - ret = request_threaded_irq(host->irq, bcm2835_sdhost_irq,
1982 - bcm2835_sdhost_thread_irq,
1983 - IRQF_SHARED, mmc_hostname(mmc), host);
1985 + ret = request_irq(host->irq, bcm2835_sdhost_irq, 0 /*IRQF_SHARED*/,
1986 + mmc_hostname(mmc), host);
1988 pr_err("%s: failed to request IRQ %d: %d\n",
1989 mmc_hostname(mmc), host->irq, ret);
1990 @@ -1776,11 +1864,11 @@ int bcm2835_sdhost_add_host(struct bcm28
1993 pio_limit_string[0] = '\0';
1994 - if (host->have_dma && (host->pio_limit > 0))
1995 + if (host->use_dma && (host->pio_limit > 0))
1996 sprintf(pio_limit_string, " (>%d)", host->pio_limit);
1997 pr_info("%s: %s loaded - DMA %s%s\n",
1998 mmc_hostname(mmc), DRIVER_NAME,
1999 - host->have_dma ? "enabled" : "disabled",
2000 + host->use_dma ? "enabled" : "disabled",
2004 @@ -1810,8 +1898,11 @@ static int bcm2835_sdhost_probe(struct p
2005 mmc->ops = &bcm2835_sdhost_ops;
2006 host = mmc_priv(mmc);
2008 + host->cmd_quick_poll_retries = 0;
2009 host->pio_timeout = msecs_to_jiffies(500);
2010 + host->pio_limit = 1;
2011 host->max_delay = 1; /* Warn if over 1ms */
2012 + host->allow_dma = 1;
2013 spin_lock_init(&host->lock);
2015 iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2016 @@ -1827,13 +1918,12 @@ static int bcm2835_sdhost_probe(struct p
2019 host->bus_addr = be32_to_cpup(addr);
2020 + log_init(iomem->start - host->bus_addr);
2021 pr_debug(" - ioaddr %lx, iomem->start %lx, bus_addr %lx\n",
2022 (unsigned long)host->ioaddr,
2023 (unsigned long)iomem->start,
2024 (unsigned long)host->bus_addr);
2026 - host->allow_dma = ALLOW_DMA;
2029 /* Read any custom properties */
2030 of_property_read_u32(node,
2031 @@ -1845,16 +1935,17 @@ static int bcm2835_sdhost_probe(struct p
2032 of_property_read_u32(node,
2035 - host->allow_dma = ALLOW_DMA &&
2037 !of_property_read_bool(node, "brcm,force-pio");
2038 host->debug = of_property_read_bool(node, "brcm,debug");
2039 - of_property_read_u32(node,
2040 - "brcm,debug-flags",
2041 - &host->debug_flags);
2044 - if (host->debug_flags)
2045 - dev_err(dev, "debug_flags=%x\n", host->debug_flags);
2046 + host->dma_chan = NULL;
2047 + host->dma_desc = NULL;
2049 + /* Formally recognise the other way of disabling DMA */
2050 + if (host->pio_limit == 0x7fffffff)
2051 + host->allow_dma = false;
2053 if (host->allow_dma) {
2055 @@ -1940,15 +2031,12 @@ static int bcm2835_sdhost_remove(struct
2060 static const struct of_device_id bcm2835_sdhost_match[] = {
2061 { .compatible = "brcm,bcm2835-sdhost" },
2064 MODULE_DEVICE_TABLE(of, bcm2835_sdhost_match);
2068 static struct platform_driver bcm2835_sdhost_driver = {
2069 .probe = bcm2835_sdhost_probe,
2070 .remove = bcm2835_sdhost_remove,