2 * Copyright (C) 2007 PA Semi, Inc
4 * Driver for the PA Semi PWRficient DMA Crypto Engine
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/version.h>
21 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33))
22 #include <generated/autoconf.h>
24 #include <linux/autoconf.h>
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/interrupt.h>
29 #include <linux/timer.h>
30 #include <linux/random.h>
31 #include <linux/skbuff.h>
32 #include <asm/scatterlist.h>
33 #include <linux/moduleparam.h>
34 #include <linux/pci.h>
35 #include <cryptodev.h>
37 #include "pasemi_fnu.h"
39 #define DRV_NAME "pasemi"
41 #define TIMER_INTERVAL 1000
43 static void __devexit
pasemi_dma_remove(struct pci_dev
*pdev
);
44 static struct pasdma_status
volatile * dma_status
;
47 module_param(debug
, int, 0644);
48 MODULE_PARM_DESC(debug
, "Enable debug");
50 static void pasemi_desc_start(struct pasemi_desc
*desc
, u64 hdr
)
58 static void pasemi_desc_build(struct pasemi_desc
*desc
, u64 val
)
60 desc
->quad
[desc
->quad_cnt
++] = val
;
61 desc
->size
= (desc
->quad_cnt
+ 1) / 2;
64 static void pasemi_desc_hdr(struct pasemi_desc
*desc
, u64 hdr
)
69 static int pasemi_desc_size(struct pasemi_desc
*desc
)
74 static void pasemi_ring_add_desc(
75 struct pasemi_fnu_txring
*ring
,
76 struct pasemi_desc
*desc
,
77 struct cryptop
*crp
) {
79 int ring_index
= 2 * (ring
->next_to_fill
& (TX_RING_SIZE
-1));
81 TX_DESC_INFO(ring
, ring
->next_to_fill
).desc_size
= desc
->size
;
82 TX_DESC_INFO(ring
, ring
->next_to_fill
).desc_postop
= desc
->postop
;
83 TX_DESC_INFO(ring
, ring
->next_to_fill
).cf_crp
= crp
;
85 for (i
= 0; i
< desc
->quad_cnt
; i
+= 2) {
86 ring_index
= 2 * (ring
->next_to_fill
& (TX_RING_SIZE
-1));
87 ring
->desc
[ring_index
] = desc
->quad
[i
];
88 ring
->desc
[ring_index
+ 1] = desc
->quad
[i
+ 1];
92 if (desc
->quad_cnt
& 1)
93 ring
->desc
[ring_index
+ 1] = 0;
96 static void pasemi_ring_incr(struct pasemi_softc
*sc
, int chan_index
, int incr
)
98 out_le32(sc
->dma_regs
+ PAS_DMA_TXCHAN_INCR(sc
->base_chan
+ chan_index
),
103 * Generate a new software session.
106 pasemi_newsession(device_t dev
, u_int32_t
*sidp
, struct cryptoini
*cri
)
108 struct cryptoini
*c
, *encini
= NULL
, *macini
= NULL
;
109 struct pasemi_softc
*sc
= device_get_softc(dev
);
110 struct pasemi_session
*ses
= NULL
, **sespp
;
114 struct pasemi_desc init_desc
;
115 struct pasemi_fnu_txring
*txring
;
117 DPRINTF("%s()\n", __FUNCTION__
);
118 if (sidp
== NULL
|| cri
== NULL
|| sc
== NULL
) {
119 DPRINTF("%s,%d - EINVAL\n", __FILE__
, __LINE__
);
122 for (c
= cri
; c
!= NULL
; c
= c
->cri_next
) {
123 if (ALG_IS_SIG(c
->cri_alg
)) {
127 } else if (ALG_IS_CIPHER(c
->cri_alg
)) {
132 DPRINTF("UNKNOWN c->cri_alg %d\n", c
->cri_alg
);
136 if (encini
== NULL
&& macini
== NULL
)
139 /* validate key length */
140 switch (encini
->cri_alg
) {
142 if (encini
->cri_klen
!= 64)
144 ccmd
= DMA_CALGO_DES
;
146 case CRYPTO_3DES_CBC
:
147 if (encini
->cri_klen
!= 192)
149 ccmd
= DMA_CALGO_3DES
;
152 if (encini
->cri_klen
!= 128 &&
153 encini
->cri_klen
!= 192 &&
154 encini
->cri_klen
!= 256)
156 ccmd
= DMA_CALGO_AES
;
159 if (encini
->cri_klen
!= 128)
161 ccmd
= DMA_CALGO_ARC
;
164 DPRINTF("UNKNOWN encini->cri_alg %d\n",
171 switch (macini
->cri_alg
) {
173 case CRYPTO_MD5_HMAC
:
177 case CRYPTO_SHA1_HMAC
:
181 DPRINTF("UNKNOWN macini->cri_alg %d\n",
185 if (((macini
->cri_klen
+ 7) / 8) > blksz
) {
186 DPRINTF("key length %d bigger than blksize %d not supported\n",
187 ((macini
->cri_klen
+ 7) / 8), blksz
);
192 for (sesn
= 0; sesn
< sc
->sc_nsessions
; sesn
++) {
193 if (sc
->sc_sessions
[sesn
] == NULL
) {
194 sc
->sc_sessions
[sesn
] = (struct pasemi_session
*)
195 kzalloc(sizeof(struct pasemi_session
), GFP_ATOMIC
);
196 ses
= sc
->sc_sessions
[sesn
];
198 } else if (sc
->sc_sessions
[sesn
]->used
== 0) {
199 ses
= sc
->sc_sessions
[sesn
];
205 sespp
= (struct pasemi_session
**)
206 kzalloc(sc
->sc_nsessions
* 2 *
207 sizeof(struct pasemi_session
*), GFP_ATOMIC
);
210 memcpy(sespp
, sc
->sc_sessions
,
211 sc
->sc_nsessions
* sizeof(struct pasemi_session
*));
212 kfree(sc
->sc_sessions
);
213 sc
->sc_sessions
= sespp
;
214 sesn
= sc
->sc_nsessions
;
215 ses
= sc
->sc_sessions
[sesn
] = (struct pasemi_session
*)
216 kzalloc(sizeof(struct pasemi_session
), GFP_ATOMIC
);
219 sc
->sc_nsessions
*= 2;
224 ses
->dma_addr
= pci_map_single(sc
->dma_pdev
, (void *) ses
->civ
,
225 sizeof(struct pasemi_session
), DMA_TO_DEVICE
);
227 /* enter the channel scheduler */
228 spin_lock_irqsave(&sc
->sc_chnlock
, flags
);
230 /* ARC4 has to be processed by the even channel */
231 if (encini
&& (encini
->cri_alg
== CRYPTO_ARC4
))
232 ses
->chan
= sc
->sc_lastchn
& ~1;
234 ses
->chan
= sc
->sc_lastchn
;
235 sc
->sc_lastchn
= (sc
->sc_lastchn
+ 1) % sc
->sc_num_channels
;
237 spin_unlock_irqrestore(&sc
->sc_chnlock
, flags
);
239 txring
= &sc
->tx
[ses
->chan
];
245 /* XXX may read fewer than requested */
246 get_random_bytes(ses
->civ
, sizeof(ses
->civ
));
248 ses
->keysz
= (encini
->cri_klen
- 63) / 64;
249 memcpy(ses
->key
, encini
->cri_key
, (ses
->keysz
+ 1) * 8);
251 pasemi_desc_start(&init_desc
,
252 XCT_CTRL_HDR(ses
->chan
, (encini
&& macini
) ? 0x68 : 0x40, DMA_FN_CIV0
));
253 pasemi_desc_build(&init_desc
,
254 XCT_FUN_SRC_PTR((encini
&& macini
) ? 0x68 : 0x40, ses
->dma_addr
));
257 if (macini
->cri_alg
== CRYPTO_MD5_HMAC
||
258 macini
->cri_alg
== CRYPTO_SHA1_HMAC
)
259 memcpy(ses
->hkey
, macini
->cri_key
, blksz
);
261 /* Load initialization constants(RFC 1321, 3174) */
262 ses
->hiv
[0] = 0x67452301efcdab89ULL
;
263 ses
->hiv
[1] = 0x98badcfe10325476ULL
;
264 ses
->hiv
[2] = 0xc3d2e1f000000000ULL
;
269 spin_lock_irqsave(&txring
->fill_lock
, flags
);
271 if (((txring
->next_to_fill
+ pasemi_desc_size(&init_desc
)) -
272 txring
->next_to_clean
) > TX_RING_SIZE
) {
273 spin_unlock_irqrestore(&txring
->fill_lock
, flags
);
278 pasemi_ring_add_desc(txring
, &init_desc
, NULL
);
279 pasemi_ring_incr(sc
, ses
->chan
,
280 pasemi_desc_size(&init_desc
));
284 spin_unlock_irqrestore(&txring
->fill_lock
, flags
);
286 *sidp
= PASEMI_SID(sesn
);
291 * Deallocate a session.
294 pasemi_freesession(device_t dev
, u_int64_t tid
)
296 struct pasemi_softc
*sc
= device_get_softc(dev
);
298 u_int32_t sid
= ((u_int32_t
) tid
) & 0xffffffff;
300 DPRINTF("%s()\n", __FUNCTION__
);
304 session
= PASEMI_SESSION(sid
);
305 if (session
>= sc
->sc_nsessions
|| !sc
->sc_sessions
[session
])
308 pci_unmap_single(sc
->dma_pdev
,
309 sc
->sc_sessions
[session
]->dma_addr
,
310 sizeof(struct pasemi_session
), DMA_TO_DEVICE
);
311 memset(sc
->sc_sessions
[session
], 0,
312 sizeof(struct pasemi_session
));
318 pasemi_process(device_t dev
, struct cryptop
*crp
, int hint
)
321 int err
= 0, ivsize
, srclen
= 0, reinit
= 0, reinit_size
= 0, chsel
;
322 struct pasemi_softc
*sc
= device_get_softc(dev
);
323 struct cryptodesc
*crd1
, *crd2
, *maccrd
, *enccrd
;
325 struct pasemi_desc init_desc
, work_desc
;
326 struct pasemi_session
*ses
;
330 struct pasemi_fnu_txring
*txring
;
332 DPRINTF("%s()\n", __FUNCTION__
);
334 if (crp
== NULL
|| crp
->crp_callback
== NULL
|| sc
== NULL
)
338 if (PASEMI_SESSION(crp
->crp_sid
) >= sc
->sc_nsessions
)
341 ses
= sc
->sc_sessions
[PASEMI_SESSION(crp
->crp_sid
)];
343 crd1
= crp
->crp_desc
;
348 crd2
= crd1
->crd_next
;
350 if (ALG_IS_SIG(crd1
->crd_alg
)) {
354 else if (ALG_IS_CIPHER(crd2
->crd_alg
) &&
355 (crd2
->crd_flags
& CRD_F_ENCRYPT
) == 0)
359 } else if (ALG_IS_CIPHER(crd1
->crd_alg
)) {
363 else if (ALG_IS_SIG(crd2
->crd_alg
) &&
364 (crd1
->crd_flags
& CRD_F_ENCRYPT
))
373 txring
= &sc
->tx
[chsel
];
375 if (enccrd
&& !maccrd
) {
376 if (enccrd
->crd_alg
== CRYPTO_ARC4
)
379 srclen
= crp
->crp_ilen
;
381 pasemi_desc_start(&work_desc
, XCT_FUN_O
| XCT_FUN_I
382 | XCT_FUN_FUN(chsel
));
383 if (enccrd
->crd_flags
& CRD_F_ENCRYPT
)
384 pasemi_desc_hdr(&work_desc
, XCT_FUN_CRM_ENC
);
386 pasemi_desc_hdr(&work_desc
, XCT_FUN_CRM_DEC
);
387 } else if (enccrd
&& maccrd
) {
388 if (enccrd
->crd_alg
== CRYPTO_ARC4
)
392 if (enccrd
->crd_flags
& CRD_F_ENCRYPT
) {
393 /* Encrypt -> Authenticate */
394 pasemi_desc_start(&work_desc
, XCT_FUN_O
| XCT_FUN_I
| XCT_FUN_CRM_ENC_SIG
395 | XCT_FUN_A
| XCT_FUN_FUN(chsel
));
396 srclen
= maccrd
->crd_skip
+ maccrd
->crd_len
;
398 /* Authenticate -> Decrypt */
399 pasemi_desc_start(&work_desc
, XCT_FUN_O
| XCT_FUN_I
| XCT_FUN_CRM_SIG_DEC
400 | XCT_FUN_24BRES
| XCT_FUN_FUN(chsel
));
401 pasemi_desc_build(&work_desc
, 0);
402 pasemi_desc_build(&work_desc
, 0);
403 pasemi_desc_build(&work_desc
, 0);
404 work_desc
.postop
= PASEMI_CHECK_SIG
;
405 srclen
= crp
->crp_ilen
;
408 pasemi_desc_hdr(&work_desc
, XCT_FUN_SHL(maccrd
->crd_skip
/ 4));
409 pasemi_desc_hdr(&work_desc
, XCT_FUN_CHL(enccrd
->crd_skip
- maccrd
->crd_skip
));
410 } else if (!enccrd
&& maccrd
) {
411 srclen
= maccrd
->crd_len
;
413 pasemi_desc_start(&init_desc
,
414 XCT_CTRL_HDR(chsel
, 0x58, DMA_FN_HKEY0
));
415 pasemi_desc_build(&init_desc
,
416 XCT_FUN_SRC_PTR(0x58, ((struct pasemi_session
*)ses
->dma_addr
)->hkey
));
418 pasemi_desc_start(&work_desc
, XCT_FUN_O
| XCT_FUN_I
| XCT_FUN_CRM_SIG
419 | XCT_FUN_A
| XCT_FUN_FUN(chsel
));
423 switch (enccrd
->crd_alg
) {
424 case CRYPTO_3DES_CBC
:
425 pasemi_desc_hdr(&work_desc
, XCT_FUN_ALG_3DES
|
427 ivsize
= sizeof(u64
);
430 pasemi_desc_hdr(&work_desc
, XCT_FUN_ALG_DES
|
432 ivsize
= sizeof(u64
);
435 pasemi_desc_hdr(&work_desc
, XCT_FUN_ALG_AES
|
437 ivsize
= 2 * sizeof(u64
);
440 pasemi_desc_hdr(&work_desc
, XCT_FUN_ALG_ARC
);
444 printk(DRV_NAME
": unimplemented enccrd->crd_alg %d\n",
450 ivp
= (ivsize
== sizeof(u64
)) ? (caddr_t
) &ses
->civ
[1] : (caddr_t
) &ses
->civ
[0];
451 if (enccrd
->crd_flags
& CRD_F_ENCRYPT
) {
452 if (enccrd
->crd_flags
& CRD_F_IV_EXPLICIT
)
453 memcpy(ivp
, enccrd
->crd_iv
, ivsize
);
454 /* If IV is not present in the buffer already, it has to be copied there */
455 if ((enccrd
->crd_flags
& CRD_F_IV_PRESENT
) == 0)
456 crypto_copyback(crp
->crp_flags
, crp
->crp_buf
,
457 enccrd
->crd_inject
, ivsize
, ivp
);
459 if (enccrd
->crd_flags
& CRD_F_IV_EXPLICIT
)
460 /* IV is provided expicitly in descriptor */
461 memcpy(ivp
, enccrd
->crd_iv
, ivsize
);
463 /* IV is provided in the packet */
464 crypto_copydata(crp
->crp_flags
, crp
->crp_buf
,
465 enccrd
->crd_inject
, ivsize
,
471 switch (maccrd
->crd_alg
) {
473 pasemi_desc_hdr(&work_desc
, XCT_FUN_SIG_MD5
|
474 XCT_FUN_HSZ((crp
->crp_ilen
- maccrd
->crd_inject
) / 4));
477 pasemi_desc_hdr(&work_desc
, XCT_FUN_SIG_SHA1
|
478 XCT_FUN_HSZ((crp
->crp_ilen
- maccrd
->crd_inject
) / 4));
480 case CRYPTO_MD5_HMAC
:
481 pasemi_desc_hdr(&work_desc
, XCT_FUN_SIG_HMAC_MD5
|
482 XCT_FUN_HSZ((crp
->crp_ilen
- maccrd
->crd_inject
) / 4));
484 case CRYPTO_SHA1_HMAC
:
485 pasemi_desc_hdr(&work_desc
, XCT_FUN_SIG_HMAC_SHA1
|
486 XCT_FUN_HSZ((crp
->crp_ilen
- maccrd
->crd_inject
) / 4));
489 printk(DRV_NAME
": unimplemented maccrd->crd_alg %d\n",
496 if (crp
->crp_flags
& CRYPTO_F_SKBUF
) {
497 /* using SKB buffers */
498 skb
= (struct sk_buff
*)crp
->crp_buf
;
499 if (skb_shinfo(skb
)->nr_frags
) {
500 printk(DRV_NAME
": skb frags unimplemented\n");
506 XCT_FUN_DST_PTR(skb
->len
, pci_map_single(
507 sc
->dma_pdev
, skb
->data
,
508 skb
->len
, DMA_TO_DEVICE
)));
512 srclen
, pci_map_single(
513 sc
->dma_pdev
, skb
->data
,
514 srclen
, DMA_TO_DEVICE
)));
515 pasemi_desc_hdr(&work_desc
, XCT_FUN_LLEN(srclen
));
516 } else if (crp
->crp_flags
& CRYPTO_F_IOV
) {
517 /* using IOV buffers */
518 uiop
= (struct uio
*)crp
->crp_buf
;
519 if (uiop
->uio_iovcnt
> 1) {
520 printk(DRV_NAME
": iov frags unimplemented\n");
525 /* crp_olen is never set; always use crp_ilen */
528 XCT_FUN_DST_PTR(crp
->crp_ilen
, pci_map_single(
530 uiop
->uio_iov
->iov_base
,
531 crp
->crp_ilen
, DMA_TO_DEVICE
)));
532 pasemi_desc_hdr(&work_desc
, XCT_FUN_LLEN(srclen
));
536 XCT_FUN_SRC_PTR(srclen
, pci_map_single(
538 uiop
->uio_iov
->iov_base
,
539 srclen
, DMA_TO_DEVICE
)));
541 /* using contig buffers */
544 XCT_FUN_DST_PTR(crp
->crp_ilen
, pci_map_single(
547 crp
->crp_ilen
, DMA_TO_DEVICE
)));
550 XCT_FUN_SRC_PTR(srclen
, pci_map_single(
552 crp
->crp_buf
, srclen
,
554 pasemi_desc_hdr(&work_desc
, XCT_FUN_LLEN(srclen
));
557 spin_lock_irqsave(&txring
->fill_lock
, flags
);
559 if (txring
->sesn
!= PASEMI_SESSION(crp
->crp_sid
)) {
560 txring
->sesn
= PASEMI_SESSION(crp
->crp_sid
);
565 pasemi_desc_start(&init_desc
,
566 XCT_CTRL_HDR(chsel
, reinit
? reinit_size
: 0x10, DMA_FN_CIV0
));
567 pasemi_desc_build(&init_desc
,
568 XCT_FUN_SRC_PTR(reinit
? reinit_size
: 0x10, ses
->dma_addr
));
571 if (((txring
->next_to_fill
+ pasemi_desc_size(&init_desc
) +
572 pasemi_desc_size(&work_desc
)) -
573 txring
->next_to_clean
) > TX_RING_SIZE
) {
574 spin_unlock_irqrestore(&txring
->fill_lock
, flags
);
579 pasemi_ring_add_desc(txring
, &init_desc
, NULL
);
580 pasemi_ring_add_desc(txring
, &work_desc
, crp
);
582 pasemi_ring_incr(sc
, chsel
,
583 pasemi_desc_size(&init_desc
) +
584 pasemi_desc_size(&work_desc
));
586 spin_unlock_irqrestore(&txring
->fill_lock
, flags
);
588 mod_timer(&txring
->crypto_timer
, jiffies
+ TIMER_INTERVAL
);
593 printk(DRV_NAME
": unsupported algorithm or algorithm order alg1 %d alg2 %d\n",
594 crd1
->crd_alg
, crd2
->crd_alg
);
598 if (err
!= ERESTART
) {
599 crp
->crp_etype
= err
;
605 static int pasemi_clean_tx(struct pasemi_softc
*sc
, int chan
)
608 struct pasemi_fnu_txring
*ring
= &sc
->tx
[chan
];
610 int flags
, loops
= 10;
614 spin_lock_irqsave(&ring
->clean_lock
, flags
);
616 while ((delta_cnt
= (dma_status
->tx_sta
[sc
->base_chan
+ chan
]
617 & PAS_STATUS_PCNT_M
) - ring
->total_pktcnt
)
620 for (i
= 0; i
< delta_cnt
; i
++) {
621 desc_size
= TX_DESC_INFO(ring
, ring
->next_to_clean
).desc_size
;
622 crp
= TX_DESC_INFO(ring
, ring
->next_to_clean
).cf_crp
;
624 ring_idx
= 2 * (ring
->next_to_clean
& (TX_RING_SIZE
-1));
625 if (TX_DESC_INFO(ring
, ring
->next_to_clean
).desc_postop
& PASEMI_CHECK_SIG
) {
626 /* Need to make sure signature matched,
627 * if not - return error */
628 if (!(ring
->desc
[ring_idx
+ 1] & (1ULL << 63)))
629 crp
->crp_etype
= -EINVAL
;
631 crypto_done(TX_DESC_INFO(ring
,
632 ring
->next_to_clean
).cf_crp
);
633 TX_DESC_INFO(ring
, ring
->next_to_clean
).cf_crp
= NULL
;
636 XCT_PTR_ADDR_LEN(ring
->desc
[ring_idx
+ 1]),
639 ring
->desc
[ring_idx
] = ring
->desc
[ring_idx
+ 1] = 0;
641 ring
->next_to_clean
++;
642 for (j
= 1; j
< desc_size
; j
++) {
644 (ring
->next_to_clean
&
648 XCT_PTR_ADDR_LEN(ring
->desc
[ring_idx
]),
650 if (ring
->desc
[ring_idx
+ 1])
657 ring
->desc
[ring_idx
] =
658 ring
->desc
[ring_idx
+ 1] = 0;
659 ring
->next_to_clean
++;
662 for (j
= 0; j
< desc_size
; j
++) {
663 ring_idx
= 2 * (ring
->next_to_clean
& (TX_RING_SIZE
-1));
664 ring
->desc
[ring_idx
] =
665 ring
->desc
[ring_idx
+ 1] = 0;
666 ring
->next_to_clean
++;
671 ring
->total_pktcnt
+= delta_cnt
;
673 spin_unlock_irqrestore(&ring
->clean_lock
, flags
);
678 static void sweepup_tx(struct pasemi_softc
*sc
)
682 for (i
= 0; i
< sc
->sc_num_channels
; i
++)
683 pasemi_clean_tx(sc
, i
);
686 static irqreturn_t
pasemi_intr(int irq
, void *arg
, struct pt_regs
*regs
)
688 struct pasemi_softc
*sc
= arg
;
690 int chan
= irq
- sc
->base_irq
;
691 int chan_index
= sc
->base_chan
+ chan
;
692 u64 stat
= dma_status
->tx_sta
[chan_index
];
694 DPRINTF("%s()\n", __FUNCTION__
);
696 if (!(stat
& PAS_STATUS_CAUSE_M
))
699 pasemi_clean_tx(sc
, chan
);
701 stat
= dma_status
->tx_sta
[chan_index
];
703 reg
= PAS_IOB_DMA_TXCH_RESET_PINTC
|
704 PAS_IOB_DMA_TXCH_RESET_PCNT(sc
->tx
[chan
].total_pktcnt
);
706 if (stat
& PAS_STATUS_SOFT
)
707 reg
|= PAS_IOB_DMA_RXCH_RESET_SINTC
;
709 out_le32(sc
->iob_regs
+ PAS_IOB_DMA_TXCH_RESET(chan_index
), reg
);
715 static int pasemi_dma_setup_tx_resources(struct pasemi_softc
*sc
, int chan
)
718 int chan_index
= chan
+ sc
->base_chan
;
720 struct pasemi_fnu_txring
*ring
;
722 ring
= &sc
->tx
[chan
];
724 spin_lock_init(&ring
->fill_lock
);
725 spin_lock_init(&ring
->clean_lock
);
727 ring
->desc_info
= kzalloc(sizeof(struct pasemi_desc_info
) *
728 TX_RING_SIZE
, GFP_KERNEL
);
729 if (!ring
->desc_info
)
732 /* Allocate descriptors */
733 ring
->desc
= dma_alloc_coherent(&sc
->dma_pdev
->dev
,
736 &ring
->dma
, GFP_KERNEL
);
740 memset((void *) ring
->desc
, 0, TX_RING_SIZE
* 2 * sizeof(u64
));
742 out_le32(sc
->iob_regs
+ PAS_IOB_DMA_TXCH_RESET(chan_index
), 0x30);
744 ring
->total_pktcnt
= 0;
746 out_le32(sc
->dma_regs
+ PAS_DMA_TXCHAN_BASEL(chan_index
),
747 PAS_DMA_TXCHAN_BASEL_BRBL(ring
->dma
));
749 val
= PAS_DMA_TXCHAN_BASEU_BRBH(ring
->dma
>> 32);
750 val
|= PAS_DMA_TXCHAN_BASEU_SIZ(TX_RING_SIZE
>> 2);
752 out_le32(sc
->dma_regs
+ PAS_DMA_TXCHAN_BASEU(chan_index
), val
);
754 out_le32(sc
->dma_regs
+ PAS_DMA_TXCHAN_CFG(chan_index
),
755 PAS_DMA_TXCHAN_CFG_TY_FUNC
|
756 PAS_DMA_TXCHAN_CFG_TATTR(chan
) |
757 PAS_DMA_TXCHAN_CFG_WT(2));
759 /* enable tx channel */
760 out_le32(sc
->dma_regs
+
761 PAS_DMA_TXCHAN_TCMDSTA(chan_index
),
762 PAS_DMA_TXCHAN_TCMDSTA_EN
);
764 out_le32(sc
->iob_regs
+ PAS_IOB_DMA_TXCH_CFG(chan_index
),
765 PAS_IOB_DMA_TXCH_CFG_CNTTH(1000));
767 ring
->next_to_fill
= 0;
768 ring
->next_to_clean
= 0;
770 snprintf(ring
->irq_name
, sizeof(ring
->irq_name
),
771 "%s%d", "crypto", chan
);
773 ring
->irq
= irq_create_mapping(NULL
, sc
->base_irq
+ chan
);
774 ret
= request_irq(ring
->irq
, (irq_handler_t
)
775 pasemi_intr
, IRQF_DISABLED
, ring
->irq_name
, sc
);
777 printk(KERN_ERR DRV_NAME
": failed to hook irq %d ret %d\n",
783 setup_timer(&ring
->crypto_timer
, (void *) sweepup_tx
, (unsigned long) sc
);
788 static device_method_t pasemi_methods
= {
789 /* crypto device methods */
790 DEVMETHOD(cryptodev_newsession
, pasemi_newsession
),
791 DEVMETHOD(cryptodev_freesession
, pasemi_freesession
),
792 DEVMETHOD(cryptodev_process
, pasemi_process
),
795 /* Set up the crypto device structure, private data,
796 * and anything else we need before we start */
799 pasemi_dma_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
801 struct pasemi_softc
*sc
;
804 DPRINTF(KERN_ERR
"%s()\n", __FUNCTION__
);
806 sc
= kzalloc(sizeof(*sc
), GFP_KERNEL
);
810 softc_device_init(sc
, DRV_NAME
, 1, pasemi_methods
);
812 pci_set_drvdata(pdev
, sc
);
814 spin_lock_init(&sc
->sc_chnlock
);
816 sc
->sc_sessions
= (struct pasemi_session
**)
817 kzalloc(PASEMI_INITIAL_SESSIONS
*
818 sizeof(struct pasemi_session
*), GFP_ATOMIC
);
819 if (sc
->sc_sessions
== NULL
) {
824 sc
->sc_nsessions
= PASEMI_INITIAL_SESSIONS
;
826 sc
->base_irq
= pdev
->irq
+ 6;
831 sc
->iob_pdev
= pci_get_device(PCI_VENDOR_ID_PASEMI
, 0xa001, NULL
);
833 dev_err(&pdev
->dev
, "Can't find I/O Bridge\n");
838 /* This is hardcoded and ugly, but we have some firmware versions
839 * who don't provide the register space in the device tree. Luckily
840 * they are at well-known locations so we can just do the math here.
843 ioremap(0xe0000000 + (sc
->dma_pdev
->devfn
<< 12), 0x2000);
845 ioremap(0xe0000000 + (sc
->iob_pdev
->devfn
<< 12), 0x2000);
846 if (!sc
->dma_regs
|| !sc
->iob_regs
) {
847 dev_err(&pdev
->dev
, "Can't map registers\n");
852 dma_status
= __ioremap(0xfd800000, 0x1000, 0);
855 dev_err(&pdev
->dev
, "Can't map dmastatus space\n");
859 sc
->tx
= (struct pasemi_fnu_txring
*)
860 kzalloc(sizeof(struct pasemi_fnu_txring
)
867 /* Initialize the h/w */
868 out_le32(sc
->dma_regs
+ PAS_DMA_COM_CFG
,
869 (in_le32(sc
->dma_regs
+ PAS_DMA_COM_CFG
) |
870 PAS_DMA_COM_CFG_FWF
));
871 out_le32(sc
->dma_regs
+ PAS_DMA_COM_TXCMD
, PAS_DMA_COM_TXCMD_EN
);
873 for (i
= 0; i
< PASEMI_FNU_CHANNELS
; i
++) {
874 sc
->sc_num_channels
++;
875 ret
= pasemi_dma_setup_tx_resources(sc
, i
);
880 sc
->sc_cid
= crypto_get_driverid(softc_get_device(sc
),
881 CRYPTOCAP_F_HARDWARE
);
882 if (sc
->sc_cid
< 0) {
883 printk(KERN_ERR DRV_NAME
": could not get crypto driver id\n");
888 /* register algorithms with the framework */
889 printk(DRV_NAME
":");
891 crypto_register(sc
->sc_cid
, CRYPTO_DES_CBC
, 0, 0);
892 crypto_register(sc
->sc_cid
, CRYPTO_3DES_CBC
, 0, 0);
893 crypto_register(sc
->sc_cid
, CRYPTO_AES_CBC
, 0, 0);
894 crypto_register(sc
->sc_cid
, CRYPTO_ARC4
, 0, 0);
895 crypto_register(sc
->sc_cid
, CRYPTO_SHA1
, 0, 0);
896 crypto_register(sc
->sc_cid
, CRYPTO_MD5
, 0, 0);
897 crypto_register(sc
->sc_cid
, CRYPTO_SHA1_HMAC
, 0, 0);
898 crypto_register(sc
->sc_cid
, CRYPTO_MD5_HMAC
, 0, 0);
903 pasemi_dma_remove(pdev
);
907 #define MAX_RETRIES 5000
909 static void pasemi_free_tx_resources(struct pasemi_softc
*sc
, int chan
)
911 struct pasemi_fnu_txring
*ring
= &sc
->tx
[chan
];
912 int chan_index
= chan
+ sc
->base_chan
;
916 /* Stop the channel */
917 out_le32(sc
->dma_regs
+
918 PAS_DMA_TXCHAN_TCMDSTA(chan_index
),
919 PAS_DMA_TXCHAN_TCMDSTA_ST
);
921 for (retries
= 0; retries
< MAX_RETRIES
; retries
++) {
922 stat
= in_le32(sc
->dma_regs
+
923 PAS_DMA_TXCHAN_TCMDSTA(chan_index
));
924 if (!(stat
& PAS_DMA_TXCHAN_TCMDSTA_ACT
))
929 if (stat
& PAS_DMA_TXCHAN_TCMDSTA_ACT
)
930 dev_err(&sc
->dma_pdev
->dev
, "Failed to stop tx channel %d\n",
933 /* Disable the channel */
934 out_le32(sc
->dma_regs
+
935 PAS_DMA_TXCHAN_TCMDSTA(chan_index
),
939 kfree((void *) ring
->desc_info
);
941 dma_free_coherent(&sc
->dma_pdev
->dev
,
944 (void *) ring
->desc
, ring
->dma
);
946 free_irq(ring
->irq
, sc
);
948 del_timer(&ring
->crypto_timer
);
951 static void __devexit
pasemi_dma_remove(struct pci_dev
*pdev
)
953 struct pasemi_softc
*sc
= pci_get_drvdata(pdev
);
956 DPRINTF("%s()\n", __FUNCTION__
);
958 if (sc
->sc_cid
>= 0) {
959 crypto_unregister_all(sc
->sc_cid
);
963 for (i
= 0; i
< sc
->sc_num_channels
; i
++)
964 pasemi_free_tx_resources(sc
, i
);
968 if (sc
->sc_sessions
) {
969 for (i
= 0; i
< sc
->sc_nsessions
; i
++)
970 kfree(sc
->sc_sessions
[i
]);
971 kfree(sc
->sc_sessions
);
974 pci_dev_put(sc
->iob_pdev
);
976 iounmap(sc
->dma_regs
);
978 iounmap(sc
->iob_regs
);
982 static struct pci_device_id pasemi_dma_pci_tbl
[] = {
983 { PCI_DEVICE(PCI_VENDOR_ID_PASEMI
, 0xa007) },
986 MODULE_DEVICE_TABLE(pci
, pasemi_dma_pci_tbl
);
988 static struct pci_driver pasemi_dma_driver
= {
989 .name
= "pasemi_dma",
990 .id_table
= pasemi_dma_pci_tbl
,
991 .probe
= pasemi_dma_probe
,
992 .remove
= __devexit_p(pasemi_dma_remove
),
995 static void __exit
pasemi_dma_cleanup_module(void)
997 pci_unregister_driver(&pasemi_dma_driver
);
998 __iounmap(dma_status
);
1002 int pasemi_dma_init_module(void)
1004 return pci_register_driver(&pasemi_dma_driver
);
1007 module_init(pasemi_dma_init_module
);
1008 module_exit(pasemi_dma_cleanup_module
);
1010 MODULE_LICENSE("Dual BSD/GPL");
1011 MODULE_AUTHOR("Egor Martovetsky egor@pasemi.com");
1012 MODULE_DESCRIPTION("OCF driver for PA Semi PWRficient DMA Crypto Engine");