1 From ed83da894caf28e267eab3a01ef037a7198391a1 Mon Sep 17 00:00:00 2001
2 From: popcornmix <popcornmix@gmail.com>
3 Date: Mon, 19 Nov 2012 18:27:05 +0000
4 Subject: [PATCH 041/196] Add Simon Hall's dma helper module, useful in future
8 arch/arm/mach-bcm2708/Kconfig | 8 +
9 arch/arm/mach-bcm2708/Makefile | 3 +
10 arch/arm/mach-bcm2708/dmaer.c | 887 ++++++++++++++++++++++++
11 arch/arm/mach-bcm2708/include/mach/vc_support.h | 69 ++
12 arch/arm/mach-bcm2708/vc_support.c | 319 +++++++++
13 5 files changed, 1286 insertions(+)
14 create mode 100755 arch/arm/mach-bcm2708/dmaer.c
15 create mode 100755 arch/arm/mach-bcm2708/include/mach/vc_support.h
16 create mode 100755 arch/arm/mach-bcm2708/vc_support.c
18 diff --git a/arch/arm/mach-bcm2708/Kconfig b/arch/arm/mach-bcm2708/Kconfig
19 index a35ff89..b85bb8d 100644
20 --- a/arch/arm/mach-bcm2708/Kconfig
21 +++ b/arch/arm/mach-bcm2708/Kconfig
22 @@ -38,4 +38,12 @@ config BCM2708_SPIDEV
25 Binds spidev driver to the SPI0 master
28 + tristate "BCM2708 DMA helper"
29 + depends on MACH_BCM2708
32 + Enable DMA helper for accelerating X composition
35 diff --git a/arch/arm/mach-bcm2708/Makefile b/arch/arm/mach-bcm2708/Makefile
36 index 164ecb2..0da162c 100644
37 --- a/arch/arm/mach-bcm2708/Makefile
38 +++ b/arch/arm/mach-bcm2708/Makefile
39 @@ -6,3 +6,6 @@ obj-$(CONFIG_MACH_BCM2708) += clock.o bcm2708.o armctrl.o vcio.o power.o dma.o
40 obj-$(CONFIG_BCM2708_GPIO) += bcm2708_gpio.o
41 obj-$(CONFIG_BCM2708_VCMEM) += vc_mem.o
43 +obj-$(CONFIG_BCM2708_DMAER) += dmaer_master.o
44 +dmaer_master-objs := dmaer.o vc_support.o
46 diff --git a/arch/arm/mach-bcm2708/dmaer.c b/arch/arm/mach-bcm2708/dmaer.c
48 index 0000000..d1bc0fa
50 +++ b/arch/arm/mach-bcm2708/dmaer.c
52 +#include <linux/init.h>
53 +#include <linux/sched.h>
54 +#include <linux/module.h>
55 +#include <linux/types.h>
56 +#include <linux/kdev_t.h>
57 +#include <linux/fs.h>
58 +#include <linux/cdev.h>
59 +#include <linux/mm.h>
60 +#include <linux/slab.h>
61 +#include <linux/pagemap.h>
62 +#include <linux/device.h>
63 +#include <linux/jiffies.h>
64 +#include <linux/timex.h>
65 +#include <linux/dma-mapping.h>
67 +#include <asm/uaccess.h>
68 +#include <asm/atomic.h>
69 +#include <asm/cacheflush.h>
72 +#include <mach/dma.h>
73 +#include <mach/vc_support.h>
75 +#ifdef ECLIPSE_IGNORE
85 +#define _IOWR(a, b, c) b
86 +#define _IOW(a, b, c) b
93 +#define PRINTK(args...) printk(args)
94 +//#define PRINTK_VERBOSE(args...) printk(args)
95 +//#define PRINTK(args...)
96 +#define PRINTK_VERBOSE(args...)
99 +#define PAGES_PER_LIST 500
102 + struct page *m_pPages[PAGES_PER_LIST];
103 + unsigned int m_used;
104 + struct PageList *m_pNext;
109 + //each vma has a linked list of pages associated with it
110 + struct PageList *m_pPageHead;
111 + struct PageList *m_pPageTail;
112 + unsigned int m_refCount;
115 +struct DmaControlBlock
117 + unsigned int m_transferInfo;
118 + void __user *m_pSourceAddr;
119 + void __user *m_pDestAddr;
120 + unsigned int m_xferLen;
121 + unsigned int m_tdStride;
122 + struct DmaControlBlock *m_pNext;
123 + unsigned int m_blank1, m_blank2;
126 +/***** DEFINES ******/
127 +//magic number defining the module
128 +#define DMA_MAGIC 0xdd
130 +//do user virtual to physical translation of the CB chain
131 +#define DMA_PREPARE _IOWR(DMA_MAGIC, 0, struct DmaControlBlock *)
133 +//kick the pre-prepared CB chain
134 +#define DMA_KICK _IOW(DMA_MAGIC, 1, struct DmaControlBlock *)
136 +//prepare it, kick it, wait for it
137 +#define DMA_PREPARE_KICK_WAIT _IOWR(DMA_MAGIC, 2, struct DmaControlBlock *)
139 +//prepare it, kick it, don't wait for it
140 +#define DMA_PREPARE_KICK _IOWR(DMA_MAGIC, 3, struct DmaControlBlock *)
142 +//not currently implemented
143 +#define DMA_WAIT_ONE _IO(DMA_MAGIC, 4, struct DmaControlBlock *)
145 +//wait on all kicked CB chains
146 +#define DMA_WAIT_ALL _IO(DMA_MAGIC, 5)
148 +//in order to discover the largest AXI burst that should be programmed into the transfer params
149 +#define DMA_MAX_BURST _IO(DMA_MAGIC, 6)
151 +//set the address range through which the user address is assumed to already by a physical address
152 +#define DMA_SET_MIN_PHYS _IOW(DMA_MAGIC, 7, unsigned long)
153 +#define DMA_SET_MAX_PHYS _IOW(DMA_MAGIC, 8, unsigned long)
154 +#define DMA_SET_PHYS_OFFSET _IOW(DMA_MAGIC, 9, unsigned long)
156 +//used to define the size for the CMA-based allocation *in pages*, can only be done once once the file is opened
157 +#define DMA_CMA_SET_SIZE _IOW(DMA_MAGIC, 10, unsigned long)
159 +//used to get the version of the module, to test for a capability
160 +#define DMA_GET_VERSION _IO(DMA_MAGIC, 99)
162 +#define VERSION_NUMBER 1
164 +#define VIRT_TO_BUS_CACHE_SIZE 8
166 +/***** FILE OPS *****/
167 +static int Open(struct inode *pInode, struct file *pFile);
168 +static int Release(struct inode *pInode, struct file *pFile);
169 +static long Ioctl(struct file *pFile, unsigned int cmd, unsigned long arg);
170 +static ssize_t Read(struct file *pFile, char __user *pUser, size_t count, loff_t *offp);
171 +static int Mmap(struct file *pFile, struct vm_area_struct *pVma);
173 +/***** VMA OPS ****/
174 +static void VmaOpen4k(struct vm_area_struct *pVma);
175 +static void VmaClose4k(struct vm_area_struct *pVma);
176 +static int VmaFault4k(struct vm_area_struct *pVma, struct vm_fault *pVmf);
178 +/**** DMA PROTOTYPES */
179 +static struct DmaControlBlock __user *DmaPrepare(struct DmaControlBlock __user *pUserCB, int *pError);
180 +static int DmaKick(struct DmaControlBlock __user *pUserCB);
181 +static void DmaWaitAll(void);
184 +static int __init dmaer_init(void);
185 +static void __exit dmaer_exit(void);
188 +static struct vm_operations_struct g_vmOps4k = {
190 + .close = VmaClose4k,
191 + .fault = VmaFault4k,
194 +static struct file_operations g_fOps = {
195 + .owner = THIS_MODULE,
199 + .unlocked_ioctl = Ioctl,
201 + .release = Release,
205 +/***** GLOBALS ******/
206 +static dev_t g_majorMinor;
208 +//tracking usage of the two files
209 +static atomic_t g_oneLock4k = ATOMIC_INIT(1);
212 +static struct cdev g_cDev;
213 +static int g_trackedPages = 0;
216 +static unsigned int *g_pDmaChanBase;
217 +static int g_dmaIrq;
218 +static int g_dmaChan;
221 +static int g_cmaHandle;
223 +//user virtual to bus address translation acceleration
224 +static unsigned long g_virtAddr[VIRT_TO_BUS_CACHE_SIZE];
225 +static unsigned long g_busAddr[VIRT_TO_BUS_CACHE_SIZE];
226 +static unsigned long g_cbVirtAddr;
227 +static unsigned long g_cbBusAddr;
228 +static int g_cacheInsertAt;
229 +static int g_cacheHit, g_cacheMiss;
232 +static void __user *g_pMinPhys;
233 +static void __user *g_pMaxPhys;
234 +static unsigned long g_physOffset;
236 +/****** CACHE OPERATIONS ********/
237 +static inline void FlushAddrCache(void)
240 + for (count = 0; count < VIRT_TO_BUS_CACHE_SIZE; count++)
241 + g_virtAddr[count] = 0xffffffff; //never going to match as we always chop the bottom bits anyway
243 + g_cbVirtAddr = 0xffffffff;
245 + g_cacheInsertAt = 0;
248 +//translate from a user virtual address to a bus address by mapping the page
249 +//NB this won't lock a page in memory, so to avoid potential paging issues using kernel logical addresses
250 +static inline void __iomem *UserVirtualToBus(void __user *pUser)
253 + struct page *pPage;
256 + //map it (requiring that the pointer points to something that does not hang off the page boundary)
257 + mapped = get_user_pages(current, current->mm,
258 + (unsigned long)pUser, 1,
263 + if (mapped <= 0) //error
266 + PRINTK_VERBOSE(KERN_DEBUG "user virtual %p arm phys %p bus %p\n",
267 + pUser, page_address(pPage), (void __iomem *)__virt_to_bus(page_address(pPage)));
269 + //get the arm physical address
270 + phys = page_address(pPage) + offset_in_page(pUser);
271 + page_cache_release(pPage);
273 + //and now the bus address
274 + return (void __iomem *)__virt_to_bus(phys);
277 +static inline void __iomem *UserVirtualToBusViaCbCache(void __user *pUser)
279 + unsigned long virtual_page = (unsigned long)pUser & ~4095;
280 + unsigned long page_offset = (unsigned long)pUser & 4095;
281 + unsigned long bus_addr;
283 + if (g_cbVirtAddr == virtual_page)
285 + bus_addr = g_cbBusAddr + page_offset;
287 + return (void __iomem *)bus_addr;
291 + bus_addr = (unsigned long)UserVirtualToBus(pUser);
296 + g_cbVirtAddr = virtual_page;
297 + g_cbBusAddr = bus_addr & ~4095;
300 + return (void __iomem *)bus_addr;
304 +//do the same as above, by query our virt->bus cache
305 +static inline void __iomem *UserVirtualToBusViaCache(void __user *pUser)
308 + //get the page and its offset
309 + unsigned long virtual_page = (unsigned long)pUser & ~4095;
310 + unsigned long page_offset = (unsigned long)pUser & 4095;
311 + unsigned long bus_addr;
313 + if (pUser >= g_pMinPhys && pUser < g_pMaxPhys)
315 + PRINTK_VERBOSE(KERN_DEBUG "user->phys passthrough on %p\n", pUser);
316 + return (void __iomem *)((unsigned long)pUser + g_physOffset);
319 + //check the cache for our entry
320 + for (count = 0; count < VIRT_TO_BUS_CACHE_SIZE; count++)
321 + if (g_virtAddr[count] == virtual_page)
323 + bus_addr = g_busAddr[count] + page_offset;
325 + return (void __iomem *)bus_addr;
328 + //not found, look up manually and then insert its page address
329 + bus_addr = (unsigned long)UserVirtualToBus(pUser);
334 + g_virtAddr[g_cacheInsertAt] = virtual_page;
335 + g_busAddr[g_cacheInsertAt] = bus_addr & ~4095;
339 + if (g_cacheInsertAt == VIRT_TO_BUS_CACHE_SIZE)
340 + g_cacheInsertAt = 0;
344 + return (void __iomem *)bus_addr;
347 +/***** FILE OPERATIONS ****/
348 +static int Open(struct inode *pInode, struct file *pFile)
350 + PRINTK(KERN_DEBUG "file opening: %d/%d\n", imajor(pInode), iminor(pInode));
352 + //check which device we are
353 + if (iminor(pInode) == 0) //4k
355 + //only one at a time
356 + if (!atomic_dec_and_test(&g_oneLock4k))
358 + atomic_inc(&g_oneLock4k);
365 + //todo there will be trouble if two different processes open the files
367 + //reset after any file is opened
368 + g_pMinPhys = (void __user *)-1;
369 + g_pMaxPhys = (void __user *)0;
376 +static int Release(struct inode *pInode, struct file *pFile)
378 + PRINTK(KERN_DEBUG "file closing, %d pages tracked\n", g_trackedPages);
379 + if (g_trackedPages)
380 + PRINTK(KERN_ERR "we\'re leaking memory!\n");
382 + //wait for any dmas to finish
385 + //free this memory on the application closing the file or it crashing (implicitly closing the file)
388 + PRINTK(KERN_DEBUG "unlocking vc memory\n");
389 + if (UnlockVcMemory(g_cmaHandle))
390 + PRINTK(KERN_ERR "uh-oh, unable to unlock vc memory!\n");
391 + PRINTK(KERN_DEBUG "releasing vc memory\n");
392 + if (ReleaseVcMemory(g_cmaHandle))
393 + PRINTK(KERN_ERR "uh-oh, unable to release vc memory!\n");
396 + if (iminor(pInode) == 0)
397 + atomic_inc(&g_oneLock4k);
404 +static struct DmaControlBlock __user *DmaPrepare(struct DmaControlBlock __user *pUserCB, int *pError)
406 + struct DmaControlBlock kernCB;
407 + struct DmaControlBlock __user *pUNext;
408 + void __iomem *pSourceBus, __iomem *pDestBus;
410 + //get the control block into kernel memory so we can work on it
411 + if (copy_from_user(&kernCB, pUserCB, sizeof(struct DmaControlBlock)) != 0)
413 + PRINTK(KERN_ERR "copy_from_user failed for user cb %p\n", pUserCB);
418 + if (kernCB.m_pSourceAddr == 0 || kernCB.m_pDestAddr == 0)
420 + PRINTK(KERN_ERR "faulty source (%p) dest (%p) addresses for user cb %p\n",
421 + kernCB.m_pSourceAddr, kernCB.m_pDestAddr, pUserCB);
426 + pSourceBus = UserVirtualToBusViaCache(kernCB.m_pSourceAddr);
427 + pDestBus = UserVirtualToBusViaCache(kernCB.m_pDestAddr);
429 + if (!pSourceBus || !pDestBus)
431 + PRINTK(KERN_ERR "virtual to bus translation failure for source/dest %p/%p->%p/%p\n",
432 + kernCB.m_pSourceAddr, kernCB.m_pDestAddr,
433 + pSourceBus, pDestBus);
438 + //update the user structure with the new bus addresses
439 + kernCB.m_pSourceAddr = pSourceBus;
440 + kernCB.m_pDestAddr = pDestBus;
442 + PRINTK_VERBOSE(KERN_DEBUG "final source %p dest %p\n", kernCB.m_pSourceAddr, kernCB.m_pDestAddr);
444 + //sort out the bus address for the next block
445 + pUNext = kernCB.m_pNext;
447 + if (kernCB.m_pNext)
449 + void __iomem *pNextBus;
450 + pNextBus = UserVirtualToBusViaCbCache(kernCB.m_pNext);
454 + PRINTK(KERN_ERR "virtual to bus translation failure for m_pNext\n");
459 + //update the pointer with the bus address
460 + kernCB.m_pNext = pNextBus;
463 + //write it back to user space
464 + if (copy_to_user(pUserCB, &kernCB, sizeof(struct DmaControlBlock)) != 0)
466 + PRINTK(KERN_ERR "copy_to_user failed for cb %p\n", pUserCB);
471 + __cpuc_flush_dcache_area(pUserCB, 32);
477 +static int DmaKick(struct DmaControlBlock __user *pUserCB)
479 + void __iomem *pBusCB;
481 + pBusCB = UserVirtualToBusViaCbCache(pUserCB);
484 + PRINTK(KERN_ERR "virtual to bus translation failure for cb\n");
488 + //flush_cache_all();
490 + bcm_dma_start(g_pDmaChanBase, (dma_addr_t)pBusCB);
495 +static void DmaWaitAll(void)
498 + volatile int inner_count;
499 + volatile unsigned int cs;
500 + unsigned long time_before, time_after;
502 + time_before = jiffies;
503 + //bcm_dma_wait_idle(g_pDmaChanBase);
506 + cs = readl(g_pDmaChanBase);
508 + while ((cs & 1) == 1)
510 + cs = readl(g_pDmaChanBase);
513 + for (inner_count = 0; inner_count < 32; inner_count++);
515 + asm volatile ("MCR p15,0,r0,c7,c0,4 \n");
517 + if (counter >= 1000000)
519 + PRINTK(KERN_WARNING "DMA failed to finish in a timely fashion\n");
523 + time_after = jiffies;
524 + PRINTK_VERBOSE(KERN_DEBUG "done, counter %d, cs %08x", counter, cs);
525 + PRINTK_VERBOSE(KERN_DEBUG "took %ld jiffies, %d HZ\n", time_after - time_before, HZ);
528 +static long Ioctl(struct file *pFile, unsigned int cmd, unsigned long arg)
531 + PRINTK_VERBOSE(KERN_DEBUG "ioctl cmd %x arg %lx\n", cmd, arg);
536 + case DMA_PREPARE_KICK:
537 + case DMA_PREPARE_KICK_WAIT:
539 + struct DmaControlBlock __user *pUCB = (struct DmaControlBlock *)arg;
541 + unsigned long start_time = jiffies;
544 + //flush our address cache
547 + PRINTK_VERBOSE(KERN_DEBUG "dma prepare\n");
549 + //do virtual to bus translation for each entry
552 + pUCB = DmaPrepare(pUCB, &error);
553 + } while (error == 0 && ++steps && pUCB);
554 + PRINTK_VERBOSE(KERN_DEBUG "prepare done in %d steps, %ld\n", steps, jiffies - start_time);
556 + //carry straight on if we want to kick too
557 + if (cmd == DMA_PREPARE || error)
559 + PRINTK_VERBOSE(KERN_DEBUG "falling out\n");
560 + return error ? -EINVAL : 0;
564 + PRINTK_VERBOSE(KERN_DEBUG "dma begin\n");
566 + if (cmd == DMA_KICK)
569 + DmaKick((struct DmaControlBlock __user *)arg);
571 + if (cmd != DMA_PREPARE_KICK_WAIT)
573 +/* case DMA_WAIT_ONE:
574 + //PRINTK(KERN_DEBUG "dma wait one\n");
577 + //PRINTK(KERN_DEBUG "dma wait all\n");
580 + case DMA_MAX_BURST:
581 + if (g_dmaChan == 0)
585 + case DMA_SET_MIN_PHYS:
586 + g_pMinPhys = (void __user *)arg;
587 + PRINTK(KERN_DEBUG "min/max user/phys bypass set to %p %p\n", g_pMinPhys, g_pMaxPhys);
589 + case DMA_SET_MAX_PHYS:
590 + g_pMaxPhys = (void __user *)arg;
591 + PRINTK(KERN_DEBUG "min/max user/phys bypass set to %p %p\n", g_pMinPhys, g_pMaxPhys);
593 + case DMA_SET_PHYS_OFFSET:
594 + g_physOffset = arg;
595 + PRINTK(KERN_DEBUG "user/phys bypass offset set to %ld\n", g_physOffset);
597 + case DMA_CMA_SET_SIZE:
599 + unsigned int pBusAddr;
603 + PRINTK(KERN_ERR "memory has already been allocated (handle %d)\n", g_cmaHandle);
607 + PRINTK(KERN_INFO "allocating %ld bytes of VC memory\n", arg * 4096);
610 + if (AllocateVcMemory(&g_cmaHandle, arg * 4096, 4096, MEM_FLAG_L1_NONALLOCATING | MEM_FLAG_NO_INIT | MEM_FLAG_HINT_PERMALOCK))
612 + PRINTK(KERN_ERR "failed to allocate %ld bytes of VC memory\n", arg * 4096);
617 + //get an address for it
618 + PRINTK(KERN_INFO "trying to map VC memory\n");
620 + if (LockVcMemory(&pBusAddr, g_cmaHandle))
622 + PRINTK(KERN_ERR "failed to map CMA handle %d, releasing memory\n", g_cmaHandle);
623 + ReleaseVcMemory(g_cmaHandle);
627 + PRINTK(KERN_INFO "bus address for CMA memory is %x\n", pBusAddr);
630 + case DMA_GET_VERSION:
631 + PRINTK(KERN_DEBUG "returning version number, %d\n", VERSION_NUMBER);
632 + return VERSION_NUMBER;
634 + PRINTK(KERN_DEBUG "unknown ioctl: %d\n", cmd);
641 +static ssize_t Read(struct file *pFile, char __user *pUser, size_t count, loff_t *offp)
646 +static int Mmap(struct file *pFile, struct vm_area_struct *pVma)
648 + struct PageList *pPages;
649 + struct VmaPageList *pVmaList;
651 + PRINTK_VERBOSE(KERN_DEBUG "MMAP vma %p, length %ld (%s %d)\n",
652 + pVma, pVma->vm_end - pVma->vm_start,
653 + current->comm, current->pid);
654 + PRINTK_VERBOSE(KERN_DEBUG "MMAP %p %d (tracked %d)\n", pVma, current->pid, g_trackedPages);
656 + //make a new page list
657 + pPages = (struct PageList *)kmalloc(sizeof(struct PageList), GFP_KERNEL);
660 + PRINTK(KERN_ERR "couldn\'t allocate a new page list (%s %d)\n",
661 + current->comm, current->pid);
665 + //clear the page list
666 + pPages->m_used = 0;
667 + pPages->m_pNext = 0;
669 + //insert our vma and new page list somewhere
670 + if (!pVma->vm_private_data)
672 + struct VmaPageList *pList;
674 + PRINTK_VERBOSE(KERN_DEBUG "new vma list, making new one (%s %d)\n",
675 + current->comm, current->pid);
677 + //make a new vma list
678 + pList = (struct VmaPageList *)kmalloc(sizeof(struct VmaPageList), GFP_KERNEL);
681 + PRINTK(KERN_ERR "couldn\'t allocate vma page list (%s %d)\n",
682 + current->comm, current->pid);
688 + pVma->vm_private_data = (void *)pList;
689 + pList->m_refCount = 0;
692 + pVmaList = (struct VmaPageList *)pVma->vm_private_data;
694 + //add it to the vma list
695 + pVmaList->m_pPageHead = pPages;
696 + pVmaList->m_pPageTail = pPages;
698 + pVma->vm_ops = &g_vmOps4k;
699 + pVma->vm_flags |= VM_IO;
706 +/****** VMA OPERATIONS ******/
708 +static void VmaOpen4k(struct vm_area_struct *pVma)
710 + struct VmaPageList *pVmaList;
712 + PRINTK_VERBOSE(KERN_DEBUG "vma open %p private %p (%s %d), %d live pages\n", pVma, pVma->vm_private_data, current->comm, current->pid, g_trackedPages);
713 + PRINTK_VERBOSE(KERN_DEBUG "OPEN %p %d %ld pages (tracked pages %d)\n",
714 + pVma, current->pid, (pVma->vm_end - pVma->vm_start) >> 12,
717 + pVmaList = (struct VmaPageList *)pVma->vm_private_data;
721 + pVmaList->m_refCount++;
722 + PRINTK_VERBOSE(KERN_DEBUG "ref count is now %d\n", pVmaList->m_refCount);
726 + PRINTK_VERBOSE(KERN_DEBUG "err, open but no vma page list\n");
730 +static void VmaClose4k(struct vm_area_struct *pVma)
732 + struct VmaPageList *pVmaList;
735 + PRINTK_VERBOSE(KERN_DEBUG "vma close %p private %p (%s %d)\n", pVma, pVma->vm_private_data, current->comm, current->pid);
737 + //wait for any dmas to finish
740 + //find our vma in the list
741 + pVmaList = (struct VmaPageList *)pVma->vm_private_data;
746 + struct PageList *pPages;
748 + pVmaList->m_refCount--;
750 + if (pVmaList->m_refCount == 0)
752 + PRINTK_VERBOSE(KERN_DEBUG "found vma, freeing pages (%s %d)\n",
753 + current->comm, current->pid);
755 + pPages = pVmaList->m_pPageHead;
759 + PRINTK(KERN_ERR "no page list (%s %d)!\n",
760 + current->comm, current->pid);
766 + struct PageList *next;
769 + PRINTK_VERBOSE(KERN_DEBUG "page list (%s %d)\n",
770 + current->comm, current->pid);
772 + next = pPages->m_pNext;
773 + for (count = 0; count < pPages->m_used; count++)
775 + PRINTK_VERBOSE(KERN_DEBUG "freeing page %p (%s %d)\n",
776 + pPages->m_pPages[count],
777 + current->comm, current->pid);
778 + __free_pages(pPages->m_pPages[count], 0);
783 + PRINTK_VERBOSE(KERN_DEBUG "freeing page list (%s %d)\n",
784 + current->comm, current->pid);
789 + //remove our vma from the list
791 + pVma->vm_private_data = 0;
795 + PRINTK_VERBOSE(KERN_DEBUG "ref count is %d, not closing\n", pVmaList->m_refCount);
800 + PRINTK_VERBOSE(KERN_ERR "uh-oh, vma %p not found (%s %d)!\n", pVma, current->comm, current->pid);
801 + PRINTK_VERBOSE(KERN_ERR "CLOSE ERR\n");
804 + PRINTK_VERBOSE(KERN_DEBUG "CLOSE %p %d %d pages (tracked pages %d)",
805 + pVma, current->pid, freed, g_trackedPages);
807 + PRINTK_VERBOSE(KERN_DEBUG "%d pages open\n", g_trackedPages);
810 +static int VmaFault4k(struct vm_area_struct *pVma, struct vm_fault *pVmf)
812 + PRINTK_VERBOSE(KERN_DEBUG "vma fault for vma %p private %p at offset %ld (%s %d)\n", pVma, pVma->vm_private_data, pVmf->pgoff,
813 + current->comm, current->pid);
814 + PRINTK_VERBOSE(KERN_DEBUG "FAULT\n");
815 + pVmf->page = alloc_page(GFP_KERNEL);
819 + PRINTK_VERBOSE(KERN_DEBUG "alloc page virtual %p\n", page_address(pVmf->page));
824 + PRINTK(KERN_ERR "vma fault oom (%s %d)\n", current->comm, current->pid);
825 + return VM_FAULT_OOM;
829 + struct VmaPageList *pVmaList;
831 + get_page(pVmf->page);
834 + //find our vma in the list
835 + pVmaList = (struct VmaPageList *)pVma->vm_private_data;
839 + PRINTK_VERBOSE(KERN_DEBUG "vma found (%s %d)\n", current->comm, current->pid);
841 + if (pVmaList->m_pPageTail->m_used == PAGES_PER_LIST)
843 + PRINTK_VERBOSE(KERN_DEBUG "making new page list (%s %d)\n", current->comm, current->pid);
844 + //making a new page list
845 + pVmaList->m_pPageTail->m_pNext = (struct PageList *)kmalloc(sizeof(struct PageList), GFP_KERNEL);
846 + if (!pVmaList->m_pPageTail->m_pNext)
849 + //update the tail pointer
850 + pVmaList->m_pPageTail = pVmaList->m_pPageTail->m_pNext;
851 + pVmaList->m_pPageTail->m_used = 0;
852 + pVmaList->m_pPageTail->m_pNext = 0;
855 + PRINTK_VERBOSE(KERN_DEBUG "adding page to list (%s %d)\n", current->comm, current->pid);
857 + pVmaList->m_pPageTail->m_pPages[pVmaList->m_pPageTail->m_used] = pVmf->page;
858 + pVmaList->m_pPageTail->m_used++;
861 + PRINTK(KERN_ERR "returned page for vma we don\'t know %p (%s %d)\n", pVma, current->comm, current->pid);
867 +/****** GENERIC FUNCTIONS ******/
868 +static int __init dmaer_init(void)
870 + int result = alloc_chrdev_region(&g_majorMinor, 0, 1, "dmaer");
873 + PRINTK(KERN_ERR "unable to get major device number\n");
877 + PRINTK(KERN_DEBUG "major device number %d\n", MAJOR(g_majorMinor));
879 + PRINTK(KERN_DEBUG "vma list size %d, page list size %d, page size %ld\n",
880 + sizeof(struct VmaPageList), sizeof(struct PageList), PAGE_SIZE);
882 + //get a dma channel to work with
883 + result = bcm_dma_chan_alloc(BCM_DMA_FEATURE_FAST, (void **)&g_pDmaChanBase, &g_dmaIrq);
885 + //uncomment to force to channel 0
887 + //g_pDmaChanBase = 0xce808000;
891 + PRINTK(KERN_ERR "failed to allocate dma channel\n");
893 + unregister_chrdev_region(g_majorMinor, 1);
896 + //reset the channel
897 + PRINTK(KERN_DEBUG "allocated dma channel %d (%p), initial state %08x\n", result, g_pDmaChanBase, *g_pDmaChanBase);
898 + *g_pDmaChanBase = 1 << 31;
899 + PRINTK(KERN_DEBUG "post-reset %08x\n", *g_pDmaChanBase);
901 + g_dmaChan = result;
903 + //clear the cache stats
907 + //register our device - after this we are go go go
908 + cdev_init(&g_cDev, &g_fOps);
909 + g_cDev.owner = THIS_MODULE;
910 + g_cDev.ops = &g_fOps;
912 + result = cdev_add(&g_cDev, g_majorMinor, 1);
915 + PRINTK(KERN_ERR "failed to add character device\n");
916 + unregister_chrdev_region(g_majorMinor, 1);
917 + bcm_dma_chan_free(g_dmaChan);
924 +static void __exit dmaer_exit(void)
926 + PRINTK(KERN_INFO "closing dmaer device, cache stats: %d hits %d misses\n", g_cacheHit, g_cacheMiss);
927 + //unregister the device
929 + unregister_chrdev_region(g_majorMinor, 1);
930 + //free the dma channel
931 + bcm_dma_chan_free(g_dmaChan);
934 +MODULE_LICENSE("Dual BSD/GPL");
935 +MODULE_AUTHOR("Simon Hall");
936 +module_init(dmaer_init);
937 +module_exit(dmaer_exit);
939 diff --git a/arch/arm/mach-bcm2708/include/mach/vc_support.h b/arch/arm/mach-bcm2708/include/mach/vc_support.h
941 index 0000000..70e809f
943 +++ b/arch/arm/mach-bcm2708/include/mach/vc_support.h
945 +#ifndef _VC_SUPPORT_H_
946 +#define _VC_SUPPORT_H_
951 + * Created on: 25 Nov 2012
957 + If a MEM_HANDLE_T is discardable, the memory manager may resize it to size
958 + 0 at any time when it is not locked or retained.
960 + MEM_FLAG_DISCARDABLE = 1 << 0,
963 + If a MEM_HANDLE_T is allocating (or normal), its block of memory will be
964 + accessed in an allocating fashion through the cache.
966 + MEM_FLAG_NORMAL = 0 << 2,
967 + MEM_FLAG_ALLOCATING = MEM_FLAG_NORMAL,
970 + If a MEM_HANDLE_T is direct, its block of memory will be accessed
971 + directly, bypassing the cache.
973 + MEM_FLAG_DIRECT = 1 << 2,
976 + If a MEM_HANDLE_T is coherent, its block of memory will be accessed in a
977 + non-allocating fashion through the cache.
979 + MEM_FLAG_COHERENT = 2 << 2,
982 + If a MEM_HANDLE_T is L1-nonallocating, its block of memory will be accessed by
983 + the VPU in a fashion which is allocating in L2, but only coherent in L1.
985 + MEM_FLAG_L1_NONALLOCATING = (MEM_FLAG_DIRECT | MEM_FLAG_COHERENT),
988 + If a MEM_HANDLE_T is zero'd, its contents are set to 0 rather than
989 + MEM_HANDLE_INVALID on allocation and resize up.
991 + MEM_FLAG_ZERO = 1 << 4,
994 + If a MEM_HANDLE_T is uninitialised, it will not be reset to a defined value
995 + (either zero, or all 1's) on allocation.
997 + MEM_FLAG_NO_INIT = 1 << 5,
1002 + MEM_FLAG_HINT_PERMALOCK = 1 << 6, /* Likely to be locked for long periods of time. */
1005 +unsigned int AllocateVcMemory(unsigned int *pHandle, unsigned int size, unsigned int alignment, unsigned int flags);
1006 +unsigned int ReleaseVcMemory(unsigned int handle);
1007 +unsigned int LockVcMemory(unsigned int *pBusAddress, unsigned int handle);
1008 +unsigned int UnlockVcMemory(unsigned int handle);
1010 +unsigned int ExecuteVcCode(unsigned int code,
1011 + unsigned int r0, unsigned int r1, unsigned int r2, unsigned int r3, unsigned int r4, unsigned int r5);
1014 diff --git a/arch/arm/mach-bcm2708/vc_support.c b/arch/arm/mach-bcm2708/vc_support.c
1015 new file mode 100755
1016 index 0000000..5cb1335
1018 +++ b/arch/arm/mach-bcm2708/vc_support.c
1023 + * Created on: 25 Nov 2012
1027 +#include <linux/module.h>
1028 +#include <mach/vcio.h>
1030 +#ifdef ECLIPSE_IGNORE
1038 +#define KERN_WARNING
1040 +#define _IOWR(a, b, c) b
1041 +#define _IOW(a, b, c) b
1042 +#define _IO(a, b) b
1046 +/****** VC MAILBOX FUNCTIONALITY ******/
1047 +unsigned int AllocateVcMemory(unsigned int *pHandle, unsigned int size, unsigned int alignment, unsigned int flags)
1051 + unsigned int m_msgSize;
1052 + unsigned int m_response;
1056 + unsigned int m_tagId;
1057 + unsigned int m_sendBufferSize;
1059 + unsigned int m_sendDataSize;
1060 + unsigned int m_recvDataSize;
1066 + unsigned int m_size;
1067 + unsigned int m_handle;
1069 + unsigned int m_alignment;
1070 + unsigned int m_flags;
1074 + unsigned int m_endTag;
1078 + msg.m_msgSize = sizeof(msg);
1079 + msg.m_response = 0;
1082 + //fill in the tag for the allocation command
1083 + msg.m_tag.m_tagId = 0x3000c;
1084 + msg.m_tag.m_sendBufferSize = 12;
1085 + msg.m_tag.m_sendDataSize = 12;
1087 + //fill in our args
1088 + msg.m_tag.m_args.m_size = size;
1089 + msg.m_tag.m_args.m_alignment = alignment;
1090 + msg.m_tag.m_args.m_flags = flags;
1093 + s = bcm_mailbox_property(&msg, sizeof(msg));
1095 + if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004)
1097 + *pHandle = msg.m_tag.m_args.m_handle;
1102 + printk(KERN_ERR "failed to allocate vc memory: s=%d response=%08x recv data size=%08x\n",
1103 + s, msg.m_response, msg.m_tag.m_recvDataSize);
1108 +unsigned int ReleaseVcMemory(unsigned int handle)
1112 + unsigned int m_msgSize;
1113 + unsigned int m_response;
1117 + unsigned int m_tagId;
1118 + unsigned int m_sendBufferSize;
1120 + unsigned int m_sendDataSize;
1121 + unsigned int m_recvDataSize;
1127 + unsigned int m_handle;
1128 + unsigned int m_error;
1133 + unsigned int m_endTag;
1137 + msg.m_msgSize = sizeof(msg);
1138 + msg.m_response = 0;
1141 + //fill in the tag for the release command
1142 + msg.m_tag.m_tagId = 0x3000f;
1143 + msg.m_tag.m_sendBufferSize = 4;
1144 + msg.m_tag.m_sendDataSize = 4;
1146 + //pass across the handle
1147 + msg.m_tag.m_args.m_handle = handle;
1149 + s = bcm_mailbox_property(&msg, sizeof(msg));
1151 + if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004 && msg.m_tag.m_args.m_error == 0)
1155 + printk(KERN_ERR "failed to release vc memory: s=%d response=%08x recv data size=%08x error=%08x\n",
1156 + s, msg.m_response, msg.m_tag.m_recvDataSize, msg.m_tag.m_args.m_error);
1161 +unsigned int LockVcMemory(unsigned int *pBusAddress, unsigned int handle)
1165 + unsigned int m_msgSize;
1166 + unsigned int m_response;
1170 + unsigned int m_tagId;
1171 + unsigned int m_sendBufferSize;
1173 + unsigned int m_sendDataSize;
1174 + unsigned int m_recvDataSize;
1180 + unsigned int m_handle;
1181 + unsigned int m_busAddress;
1186 + unsigned int m_endTag;
1190 + msg.m_msgSize = sizeof(msg);
1191 + msg.m_response = 0;
1194 + //fill in the tag for the lock command
1195 + msg.m_tag.m_tagId = 0x3000d;
1196 + msg.m_tag.m_sendBufferSize = 4;
1197 + msg.m_tag.m_sendDataSize = 4;
1199 + //pass across the handle
1200 + msg.m_tag.m_args.m_handle = handle;
1202 + s = bcm_mailbox_property(&msg, sizeof(msg));
1204 + if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004)
1206 + //pick out the bus address
1207 + *pBusAddress = msg.m_tag.m_args.m_busAddress;
1212 + printk(KERN_ERR "failed to lock vc memory: s=%d response=%08x recv data size=%08x\n",
1213 + s, msg.m_response, msg.m_tag.m_recvDataSize);
1218 +unsigned int UnlockVcMemory(unsigned int handle)
1222 + unsigned int m_msgSize;
1223 + unsigned int m_response;
1227 + unsigned int m_tagId;
1228 + unsigned int m_sendBufferSize;
1230 + unsigned int m_sendDataSize;
1231 + unsigned int m_recvDataSize;
1237 + unsigned int m_handle;
1238 + unsigned int m_error;
1243 + unsigned int m_endTag;
1247 + msg.m_msgSize = sizeof(msg);
1248 + msg.m_response = 0;
1251 + //fill in the tag for the unlock command
1252 + msg.m_tag.m_tagId = 0x3000e;
1253 + msg.m_tag.m_sendBufferSize = 4;
1254 + msg.m_tag.m_sendDataSize = 4;
1256 + //pass across the handle
1257 + msg.m_tag.m_args.m_handle = handle;
1259 + s = bcm_mailbox_property(&msg, sizeof(msg));
1261 + //check the error code too
1262 + if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004 && msg.m_tag.m_args.m_error == 0)
1266 + printk(KERN_ERR "failed to unlock vc memory: s=%d response=%08x recv data size=%08x error%08x\n",
1267 + s, msg.m_response, msg.m_tag.m_recvDataSize, msg.m_tag.m_args.m_error);
1272 +unsigned int ExecuteVcCode(unsigned int code,
1273 + unsigned int r0, unsigned int r1, unsigned int r2, unsigned int r3, unsigned int r4, unsigned int r5)
1277 + unsigned int m_msgSize;
1278 + unsigned int m_response;
1282 + unsigned int m_tagId;
1283 + unsigned int m_sendBufferSize;
1285 + unsigned int m_sendDataSize;
1286 + unsigned int m_recvDataSize;
1292 + unsigned int m_pCode;
1293 + unsigned int m_return;
1295 + unsigned int m_r0;
1296 + unsigned int m_r1;
1297 + unsigned int m_r2;
1298 + unsigned int m_r3;
1299 + unsigned int m_r4;
1300 + unsigned int m_r5;
1304 + unsigned int m_endTag;
1308 + msg.m_msgSize = sizeof(msg);
1309 + msg.m_response = 0;
1312 + //fill in the tag for the unlock command
1313 + msg.m_tag.m_tagId = 0x30010;
1314 + msg.m_tag.m_sendBufferSize = 28;
1315 + msg.m_tag.m_sendDataSize = 28;
1317 + //pass across the handle
1318 + msg.m_tag.m_args.m_pCode = code;
1319 + msg.m_tag.m_args.m_r0 = r0;
1320 + msg.m_tag.m_args.m_r1 = r1;
1321 + msg.m_tag.m_args.m_r2 = r2;
1322 + msg.m_tag.m_args.m_r3 = r3;
1323 + msg.m_tag.m_args.m_r4 = r4;
1324 + msg.m_tag.m_args.m_r5 = r5;
1326 + s = bcm_mailbox_property(&msg, sizeof(msg));
1328 + //check the error code too
1329 + if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004)
1330 + return msg.m_tag.m_args.m_return;
1333 + printk(KERN_ERR "failed to execute: s=%d response=%08x recv data size=%08x\n",
1334 + s, msg.m_response, msg.m_tag.m_recvDataSize);