projects / openwrt / svn-archive / archive.git / blob
? search:


de7743154f921444d6bcbe29ff63b2291a2d713b
[openwrt/svn-archive/archive.git] / target / linux / generic / files / crypto / ocf / safe / safe.c
1 /*-
2 * Linux port done by David McCullough <david_mccullough@mcafee.com>
3 * Copyright (C) 2004-2010 David McCullough
4 * The license and original author are listed below.
5 *
6 * Copyright (c) 2003 Sam Leffler, Errno Consulting
7 * Copyright (c) 2003 Global Technology Associates, Inc.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 __FBSDID("$FreeBSD: src/sys/dev/safe/safe.c,v 1.18 2007/03/21 03:42:50 sam Exp $");
32 */
33
34 #include <linux/version.h>
35 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33))
36 #include <generated/autoconf.h>
37 #else
38 #include <linux/autoconf.h>
39 #endif
40 #include <linux/module.h>
41 #include <linux/kernel.h>
42 #include <linux/init.h>
43 #include <linux/list.h>
44 #include <linux/slab.h>
45 #include <linux/wait.h>
46 #include <linux/sched.h>
47 #include <linux/pci.h>
48 #include <linux/delay.h>
49 #include <linux/interrupt.h>
50 #include <linux/spinlock.h>
51 #include <linux/random.h>
52 #include <linux/version.h>
53 #include <linux/skbuff.h>
54 #include <asm/io.h>
55
56 /*
57 * SafeNet SafeXcel-1141 hardware crypto accelerator
58 */
59
60 #include <cryptodev.h>
61 #include <uio.h>
62 #include <safe/safereg.h>
63 #include <safe/safevar.h>
64
65 #if 1
66 #define DPRINTF(a) do { \
67 if (debug) { \
68 printk("%s: ", sc ? \
69 device_get_nameunit(sc->sc_dev) : "safe"); \
70 printk a; \
71 } \
72 } while (0)
73 #else
74 #define DPRINTF(a)
75 #endif
76
77 /*
78 * until we find a cleaner way, include the BSD md5/sha1 code
79 * here
80 */
81 #define HMAC_HACK 1
82 #ifdef HMAC_HACK
83 #define LITTLE_ENDIAN 1234
84 #define BIG_ENDIAN 4321
85 #ifdef __LITTLE_ENDIAN
86 #define BYTE_ORDER LITTLE_ENDIAN
87 #endif
88 #ifdef __BIG_ENDIAN
89 #define BYTE_ORDER BIG_ENDIAN
90 #endif
91 #include <safe/md5.h>
92 #include <safe/md5.c>
93 #include <safe/sha1.h>
94 #include <safe/sha1.c>
95
96 u_int8_t hmac_ipad_buffer[64] = {
97 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
98 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
99 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
100 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
101 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
102 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
103 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
104 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36
105 };
106
107 u_int8_t hmac_opad_buffer[64] = {
108 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
109 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
110 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
111 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
112 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
113 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
114 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
115 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C
116 };
117 #endif /* HMAC_HACK */
118
119 /* add proc entry for this */
120 struct safe_stats safestats;
121
122 #define debug safe_debug
123 int safe_debug = 0;
124 module_param(safe_debug, int, 0644);
125 MODULE_PARM_DESC(safe_debug, "Enable debug");
126
127 static void safe_callback(struct safe_softc *, struct safe_ringentry *);
128 static void safe_feed(struct safe_softc *, struct safe_ringentry *);
129 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
130 static void safe_rng_init(struct safe_softc *);
131 int safe_rngbufsize = 8; /* 32 bytes each read */
132 module_param(safe_rngbufsize, int, 0644);
133 MODULE_PARM_DESC(safe_rngbufsize, "RNG polling buffer size (32-bit words)");
134 int safe_rngmaxalarm = 8; /* max alarms before reset */
135 module_param(safe_rngmaxalarm, int, 0644);
136 MODULE_PARM_DESC(safe_rngmaxalarm, "RNG max alarms before reset");
137 #endif /* SAFE_NO_RNG */
138
139 static void safe_totalreset(struct safe_softc *sc);
140 static int safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op);
141 static int safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op);
142 static int safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re);
143 static int safe_kprocess(device_t dev, struct cryptkop *krp, int hint);
144 static int safe_kstart(struct safe_softc *sc);
145 static int safe_ksigbits(struct safe_softc *sc, struct crparam *cr);
146 static void safe_kfeed(struct safe_softc *sc);
147 static void safe_kpoll(unsigned long arg);
148 static void safe_kload_reg(struct safe_softc *sc, u_int32_t off,
149 u_int32_t len, struct crparam *n);
150
151 static int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
152 static int safe_freesession(device_t, u_int64_t);
153 static int safe_process(device_t, struct cryptop *, int);
154
155 static device_method_t safe_methods = {
156 /* crypto device methods */
157 DEVMETHOD(cryptodev_newsession, safe_newsession),
158 DEVMETHOD(cryptodev_freesession,safe_freesession),
159 DEVMETHOD(cryptodev_process, safe_process),
160 DEVMETHOD(cryptodev_kprocess, safe_kprocess),
161 };
162
163 #define READ_REG(sc,r) readl((sc)->sc_base_addr + (r))
164 #define WRITE_REG(sc,r,val) writel((val), (sc)->sc_base_addr + (r))
165
166 #define SAFE_MAX_CHIPS 8
167 static struct safe_softc *safe_chip_idx[SAFE_MAX_CHIPS];
168
169 /*
170 * split our buffers up into safe DMAable byte fragments to avoid lockup
171 * bug in 1141 HW on rev 1.0.
172 */
173
174 static int
175 pci_map_linear(
176 struct safe_softc *sc,
177 struct safe_operand *buf,
178 void *addr,
179 int len)
180 {
181 dma_addr_t tmp;
182 int chunk, tlen = len;
183
184 tmp = pci_map_single(sc->sc_pcidev, addr, len, PCI_DMA_BIDIRECTIONAL);
185
186 buf->mapsize += len;
187 while (len > 0) {
188 chunk = (len > sc->sc_max_dsize) ? sc->sc_max_dsize : len;
189 buf->segs[buf->nsegs].ds_addr = tmp;
190 buf->segs[buf->nsegs].ds_len = chunk;
191 buf->segs[buf->nsegs].ds_tlen = tlen;
192 buf->nsegs++;
193 tmp += chunk;
194 len -= chunk;
195 tlen = 0;
196 }
197 return 0;
198 }
199
200 /*
201 * map in a given uio buffer (great on some arches :-)
202 */
203
204 static int
205 pci_map_uio(struct safe_softc *sc, struct safe_operand *buf, struct uio *uio)
206 {
207 struct iovec *iov = uio->uio_iov;
208 int n;
209
210 DPRINTF(("%s()\n", __FUNCTION__));
211
212 buf->mapsize = 0;
213 buf->nsegs = 0;
214
215 for (n = 0; n < uio->uio_iovcnt; n++) {
216 pci_map_linear(sc, buf, iov->iov_base, iov->iov_len);
217 iov++;
218 }
219
220 /* identify this buffer by the first segment */
221 buf->map = (void *) buf->segs[0].ds_addr;
222 return(0);
223 }
224
225 /*
226 * map in a given sk_buff
227 */
228
229 static int
230 pci_map_skb(struct safe_softc *sc,struct safe_operand *buf,struct sk_buff *skb)
231 {
232 int i;
233
234 DPRINTF(("%s()\n", __FUNCTION__));
235
236 buf->mapsize = 0;
237 buf->nsegs = 0;
238
239 pci_map_linear(sc, buf, skb->data, skb_headlen(skb));
240
241 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
242 pci_map_linear(sc, buf,
243 page_address(skb_shinfo(skb)->frags[i].page) +
244 skb_shinfo(skb)->frags[i].page_offset,
245 skb_shinfo(skb)->frags[i].size);
246 }
247
248 /* identify this buffer by the first segment */
249 buf->map = (void *) buf->segs[0].ds_addr;
250 return(0);
251 }
252
253
254 #if 0 /* not needed at this time */
255 static void
256 pci_sync_operand(struct safe_softc *sc, struct safe_operand *buf)
257 {
258 int i;
259
260 DPRINTF(("%s()\n", __FUNCTION__));
261 for (i = 0; i < buf->nsegs; i++)
262 pci_dma_sync_single_for_cpu(sc->sc_pcidev, buf->segs[i].ds_addr,
263 buf->segs[i].ds_len, PCI_DMA_BIDIRECTIONAL);
264 }
265 #endif
266
267 static void
268 pci_unmap_operand(struct safe_softc *sc, struct safe_operand *buf)
269 {
270 int i;
271 DPRINTF(("%s()\n", __FUNCTION__));
272 for (i = 0; i < buf->nsegs; i++) {
273 if (buf->segs[i].ds_tlen) {
274 DPRINTF(("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
275 pci_unmap_single(sc->sc_pcidev, buf->segs[i].ds_addr,
276 buf->segs[i].ds_tlen, PCI_DMA_BIDIRECTIONAL);
277 DPRINTF(("%s - unmap %d 0x%x %d done\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
278 }
279 buf->segs[i].ds_addr = 0;
280 buf->segs[i].ds_len = 0;
281 buf->segs[i].ds_tlen = 0;
282 }
283 buf->nsegs = 0;
284 buf->mapsize = 0;
285 buf->map = 0;
286 }
287
288
289 /*
290 * SafeXcel Interrupt routine
291 */
292 static irqreturn_t
293 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
294 safe_intr(int irq, void *arg)
295 #else
296 safe_intr(int irq, void *arg, struct pt_regs *regs)
297 #endif
298 {
299 struct safe_softc *sc = arg;
300 int stat;
301 unsigned long flags;
302
303 stat = READ_REG(sc, SAFE_HM_STAT);
304
305 DPRINTF(("%s(stat=0x%x)\n", __FUNCTION__, stat));
306
307 if (stat == 0) /* shared irq, not for us */
308 return IRQ_NONE;
309
310 WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
311
312 if ((stat & SAFE_INT_PE_DDONE)) {
313 /*
314 * Descriptor(s) done; scan the ring and
315 * process completed operations.
316 */
317 spin_lock_irqsave(&sc->sc_ringmtx, flags);
318 while (sc->sc_back != sc->sc_front) {
319 struct safe_ringentry *re = sc->sc_back;
320
321 #ifdef SAFE_DEBUG
322 if (debug) {
323 safe_dump_ringstate(sc, __func__);
324 safe_dump_request(sc, __func__, re);
325 }
326 #endif
327 /*
328 * safe_process marks ring entries that were allocated
329 * but not used with a csr of zero. This insures the
330 * ring front pointer never needs to be set backwards
331 * in the event that an entry is allocated but not used
332 * because of a setup error.
333 */
334 DPRINTF(("%s re->re_desc.d_csr=0x%x\n", __FUNCTION__, re->re_desc.d_csr));
335 if (re->re_desc.d_csr != 0) {
336 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) {
337 DPRINTF(("%s !CSR_IS_DONE\n", __FUNCTION__));
338 break;
339 }
340 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) {
341 DPRINTF(("%s !LEN_IS_DONE\n", __FUNCTION__));
342 break;
343 }
344 sc->sc_nqchip--;
345 safe_callback(sc, re);
346 }
347 if (++(sc->sc_back) == sc->sc_ringtop)
348 sc->sc_back = sc->sc_ring;
349 }
350 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
351 }
352
353 /*
354 * Check to see if we got any DMA Error
355 */
356 if (stat & SAFE_INT_PE_ERROR) {
357 printk("%s: dmaerr dmastat %08x\n", device_get_nameunit(sc->sc_dev),
358 (int)READ_REG(sc, SAFE_PE_DMASTAT));
359 safestats.st_dmaerr++;
360 safe_totalreset(sc);
361 #if 0
362 safe_feed(sc);
363 #endif
364 }
365
366 if (sc->sc_needwakeup) { /* XXX check high watermark */
367 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
368 DPRINTF(("%s: wakeup crypto %x\n", __func__,
369 sc->sc_needwakeup));
370 sc->sc_needwakeup &= ~wakeup;
371 crypto_unblock(sc->sc_cid, wakeup);
372 }
373
374 return IRQ_HANDLED;
375 }
376
377 /*
378 * safe_feed() - post a request to chip
379 */
380 static void
381 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
382 {
383 DPRINTF(("%s()\n", __FUNCTION__));
384 #ifdef SAFE_DEBUG
385 if (debug) {
386 safe_dump_ringstate(sc, __func__);
387 safe_dump_request(sc, __func__, re);
388 }
389 #endif
390 sc->sc_nqchip++;
391 if (sc->sc_nqchip > safestats.st_maxqchip)
392 safestats.st_maxqchip = sc->sc_nqchip;
393 /* poke h/w to check descriptor ring, any value can be written */
394 WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
395 }
396
397 #define N(a) (sizeof(a) / sizeof (a[0]))
398 static void
399 safe_setup_enckey(struct safe_session *ses, caddr_t key)
400 {
401 int i;
402
403 bcopy(key, ses->ses_key, ses->ses_klen / 8);
404
405 /* PE is little-endian, insure proper byte order */
406 for (i = 0; i < N(ses->ses_key); i++)
407 ses->ses_key[i] = htole32(ses->ses_key[i]);
408 }
409
410 static void
411 safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
412 {
413 #ifdef HMAC_HACK
414 MD5_CTX md5ctx;
415 SHA1_CTX sha1ctx;
416 int i;
417
418
419 for (i = 0; i < klen; i++)
420 key[i] ^= HMAC_IPAD_VAL;
421
422 if (algo == CRYPTO_MD5_HMAC) {
423 MD5Init(&md5ctx);
424 MD5Update(&md5ctx, key, klen);
425 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
426 bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
427 } else {
428 SHA1Init(&sha1ctx);
429 SHA1Update(&sha1ctx, key, klen);
430 SHA1Update(&sha1ctx, hmac_ipad_buffer,
431 SHA1_HMAC_BLOCK_LEN - klen);
432 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
433 }
434
435 for (i = 0; i < klen; i++)
436 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
437
438 if (algo == CRYPTO_MD5_HMAC) {
439 MD5Init(&md5ctx);
440 MD5Update(&md5ctx, key, klen);
441 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
442 bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
443 } else {
444 SHA1Init(&sha1ctx);
445 SHA1Update(&sha1ctx, key, klen);
446 SHA1Update(&sha1ctx, hmac_opad_buffer,
447 SHA1_HMAC_BLOCK_LEN - klen);
448 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
449 }
450
451 for (i = 0; i < klen; i++)
452 key[i] ^= HMAC_OPAD_VAL;
453
454 #if 0
455 /*
456 * this code prevents SHA working on a BE host,
457 * so it is obviously wrong. I think the byte
458 * swap setup we do with the chip fixes this for us
459 */
460
461 /* PE is little-endian, insure proper byte order */
462 for (i = 0; i < N(ses->ses_hminner); i++) {
463 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
464 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
465 }
466 #endif
467 #else /* HMAC_HACK */
468 printk("safe: md5/sha not implemented\n");
469 #endif /* HMAC_HACK */
470 }
471 #undef N
472
473 /*
474 * Allocate a new 'session' and return an encoded session id. 'sidp'
475 * contains our registration id, and should contain an encoded session
476 * id on successful allocation.
477 */
478 static int
479 safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
480 {
481 struct safe_softc *sc = device_get_softc(dev);
482 struct cryptoini *c, *encini = NULL, *macini = NULL;
483 struct safe_session *ses = NULL;
484 int sesn;
485
486 DPRINTF(("%s()\n", __FUNCTION__));
487
488 if (sidp == NULL || cri == NULL || sc == NULL)
489 return (EINVAL);
490
491 for (c = cri; c != NULL; c = c->cri_next) {
492 if (c->cri_alg == CRYPTO_MD5_HMAC ||
493 c->cri_alg == CRYPTO_SHA1_HMAC ||
494 c->cri_alg == CRYPTO_NULL_HMAC) {
495 if (macini)
496 return (EINVAL);
497 macini = c;
498 } else if (c->cri_alg == CRYPTO_DES_CBC ||
499 c->cri_alg == CRYPTO_3DES_CBC ||
500 c->cri_alg == CRYPTO_AES_CBC ||
501 c->cri_alg == CRYPTO_NULL_CBC) {
502 if (encini)
503 return (EINVAL);
504 encini = c;
505 } else
506 return (EINVAL);
507 }
508 if (encini == NULL && macini == NULL)
509 return (EINVAL);
510 if (encini) { /* validate key length */
511 switch (encini->cri_alg) {
512 case CRYPTO_DES_CBC:
513 if (encini->cri_klen != 64)
514 return (EINVAL);
515 break;
516 case CRYPTO_3DES_CBC:
517 if (encini->cri_klen != 192)
518 return (EINVAL);
519 break;
520 case CRYPTO_AES_CBC:
521 if (encini->cri_klen != 128 &&
522 encini->cri_klen != 192 &&
523 encini->cri_klen != 256)
524 return (EINVAL);
525 break;
526 }
527 }
528
529 if (sc->sc_sessions == NULL) {
530 ses = sc->sc_sessions = (struct safe_session *)
531 kmalloc(sizeof(struct safe_session), SLAB_ATOMIC);
532 if (ses == NULL)
533 return (ENOMEM);
534 memset(ses, 0, sizeof(struct safe_session));
535 sesn = 0;
536 sc->sc_nsessions = 1;
537 } else {
538 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
539 if (sc->sc_sessions[sesn].ses_used == 0) {
540 ses = &sc->sc_sessions[sesn];
541 break;
542 }
543 }
544
545 if (ses == NULL) {
546 sesn = sc->sc_nsessions;
547 ses = (struct safe_session *)
548 kmalloc((sesn + 1) * sizeof(struct safe_session), SLAB_ATOMIC);
549 if (ses == NULL)
550 return (ENOMEM);
551 memset(ses, 0, (sesn + 1) * sizeof(struct safe_session));
552 bcopy(sc->sc_sessions, ses, sesn *
553 sizeof(struct safe_session));
554 bzero(sc->sc_sessions, sesn *
555 sizeof(struct safe_session));
556 kfree(sc->sc_sessions);
557 sc->sc_sessions = ses;
558 ses = &sc->sc_sessions[sesn];
559 sc->sc_nsessions++;
560 }
561 }
562
563 bzero(ses, sizeof(struct safe_session));
564 ses->ses_used = 1;
565
566 if (encini) {
567 /* get an IV */
568 /* XXX may read fewer than requested */
569 read_random(ses->ses_iv, sizeof(ses->ses_iv));
570
571 ses->ses_klen = encini->cri_klen;
572 if (encini->cri_key != NULL)
573 safe_setup_enckey(ses, encini->cri_key);
574 }
575
576 if (macini) {
577 ses->ses_mlen = macini->cri_mlen;
578 if (ses->ses_mlen == 0) {
579 if (macini->cri_alg == CRYPTO_MD5_HMAC)
580 ses->ses_mlen = MD5_HASH_LEN;
581 else
582 ses->ses_mlen = SHA1_HASH_LEN;
583 }
584
585 if (macini->cri_key != NULL) {
586 safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
587 macini->cri_klen / 8);
588 }
589 }
590
591 *sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
592 return (0);
593 }
594
595 /*
596 * Deallocate a session.
597 */
598 static int
599 safe_freesession(device_t dev, u_int64_t tid)
600 {
601 struct safe_softc *sc = device_get_softc(dev);
602 int session, ret;
603 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
604
605 DPRINTF(("%s()\n", __FUNCTION__));
606
607 if (sc == NULL)
608 return (EINVAL);
609
610 session = SAFE_SESSION(sid);
611 if (session < sc->sc_nsessions) {
612 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
613 ret = 0;
614 } else
615 ret = EINVAL;
616 return (ret);
617 }
618
619
620 static int
621 safe_process(device_t dev, struct cryptop *crp, int hint)
622 {
623 struct safe_softc *sc = device_get_softc(dev);
624 int err = 0, i, nicealign, uniform;
625 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
626 int bypass, oplen, ivsize;
627 caddr_t iv;
628 int16_t coffset;
629 struct safe_session *ses;
630 struct safe_ringentry *re;
631 struct safe_sarec *sa;
632 struct safe_pdesc *pd;
633 u_int32_t cmd0, cmd1, staterec;
634 unsigned long flags;
635
636 DPRINTF(("%s()\n", __FUNCTION__));
637
638 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
639 safestats.st_invalid++;
640 return (EINVAL);
641 }
642 if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
643 safestats.st_badsession++;
644 return (EINVAL);
645 }
646
647 spin_lock_irqsave(&sc->sc_ringmtx, flags);
648 if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
649 safestats.st_ringfull++;
650 sc->sc_needwakeup |= CRYPTO_SYMQ;
651 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
652 return (ERESTART);
653 }
654 re = sc->sc_front;
655
656 staterec = re->re_sa.sa_staterec; /* save */
657 /* NB: zero everything but the PE descriptor */
658 bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
659 re->re_sa.sa_staterec = staterec; /* restore */
660
661 re->re_crp = crp;
662 re->re_sesn = SAFE_SESSION(crp->crp_sid);
663
664 re->re_src.nsegs = 0;
665 re->re_dst.nsegs = 0;
666
667 if (crp->crp_flags & CRYPTO_F_SKBUF) {
668 re->re_src_skb = (struct sk_buff *)crp->crp_buf;
669 re->re_dst_skb = (struct sk_buff *)crp->crp_buf;
670 } else if (crp->crp_flags & CRYPTO_F_IOV) {
671 re->re_src_io = (struct uio *)crp->crp_buf;
672 re->re_dst_io = (struct uio *)crp->crp_buf;
673 } else {
674 safestats.st_badflags++;
675 err = EINVAL;
676 goto errout; /* XXX we don't handle contiguous blocks! */
677 }
678
679 sa = &re->re_sa;
680 ses = &sc->sc_sessions[re->re_sesn];
681
682 crd1 = crp->crp_desc;
683 if (crd1 == NULL) {
684 safestats.st_nodesc++;
685 err = EINVAL;
686 goto errout;
687 }
688 crd2 = crd1->crd_next;
689
690 cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
691 cmd1 = 0;
692 if (crd2 == NULL) {
693 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
694 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
695 crd1->crd_alg == CRYPTO_NULL_HMAC) {
696 maccrd = crd1;
697 enccrd = NULL;
698 cmd0 |= SAFE_SA_CMD0_OP_HASH;
699 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
700 crd1->crd_alg == CRYPTO_3DES_CBC ||
701 crd1->crd_alg == CRYPTO_AES_CBC ||
702 crd1->crd_alg == CRYPTO_NULL_CBC) {
703 maccrd = NULL;
704 enccrd = crd1;
705 cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
706 } else {
707 safestats.st_badalg++;
708 err = EINVAL;
709 goto errout;
710 }
711 } else {
712 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
713 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
714 crd1->crd_alg == CRYPTO_NULL_HMAC) &&
715 (crd2->crd_alg == CRYPTO_DES_CBC ||
716 crd2->crd_alg == CRYPTO_3DES_CBC ||
717 crd2->crd_alg == CRYPTO_AES_CBC ||
718 crd2->crd_alg == CRYPTO_NULL_CBC) &&
719 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
720 maccrd = crd1;
721 enccrd = crd2;
722 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
723 crd1->crd_alg == CRYPTO_3DES_CBC ||
724 crd1->crd_alg == CRYPTO_AES_CBC ||
725 crd1->crd_alg == CRYPTO_NULL_CBC) &&
726 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
727 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
728 crd2->crd_alg == CRYPTO_NULL_HMAC) &&
729 (crd1->crd_flags & CRD_F_ENCRYPT)) {
730 enccrd = crd1;
731 maccrd = crd2;
732 } else {
733 safestats.st_badalg++;
734 err = EINVAL;
735 goto errout;
736 }
737 cmd0 |= SAFE_SA_CMD0_OP_BOTH;
738 }
739
740 if (enccrd) {
741 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
742 safe_setup_enckey(ses, enccrd->crd_key);
743
744 if (enccrd->crd_alg == CRYPTO_DES_CBC) {
745 cmd0 |= SAFE_SA_CMD0_DES;
746 cmd1 |= SAFE_SA_CMD1_CBC;
747 ivsize = 2*sizeof(u_int32_t);
748 } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
749 cmd0 |= SAFE_SA_CMD0_3DES;
750 cmd1 |= SAFE_SA_CMD1_CBC;
751 ivsize = 2*sizeof(u_int32_t);
752 } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
753 cmd0 |= SAFE_SA_CMD0_AES;
754 cmd1 |= SAFE_SA_CMD1_CBC;
755 if (ses->ses_klen == 128)
756 cmd1 |= SAFE_SA_CMD1_AES128;
757 else if (ses->ses_klen == 192)
758 cmd1 |= SAFE_SA_CMD1_AES192;
759 else
760 cmd1 |= SAFE_SA_CMD1_AES256;
761 ivsize = 4*sizeof(u_int32_t);
762 } else {
763 cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
764 ivsize = 0;
765 }
766
767 /*
768 * Setup encrypt/decrypt state. When using basic ops
769 * we can't use an inline IV because hash/crypt offset
770 * must be from the end of the IV to the start of the
771 * crypt data and this leaves out the preceding header
772 * from the hash calculation. Instead we place the IV
773 * in the state record and set the hash/crypt offset to
774 * copy both the header+IV.
775 */
776 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
777 cmd0 |= SAFE_SA_CMD0_OUTBOUND;
778
779 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
780 iv = enccrd->crd_iv;
781 else
782 iv = (caddr_t) ses->ses_iv;
783 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
784 crypto_copyback(crp->crp_flags, crp->crp_buf,
785 enccrd->crd_inject, ivsize, iv);
786 }
787 bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
788 /* make iv LE */
789 for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
790 re->re_sastate.sa_saved_iv[i] =
791 cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
792 cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
793 re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
794 } else {
795 cmd0 |= SAFE_SA_CMD0_INBOUND;
796
797 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
798 bcopy(enccrd->crd_iv,
799 re->re_sastate.sa_saved_iv, ivsize);
800 } else {
801 crypto_copydata(crp->crp_flags, crp->crp_buf,
802 enccrd->crd_inject, ivsize,
803 (caddr_t)re->re_sastate.sa_saved_iv);
804 }
805 /* make iv LE */
806 for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
807 re->re_sastate.sa_saved_iv[i] =
808 cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
809 cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
810 }
811 /*
812 * For basic encryption use the zero pad algorithm.
813 * This pads results to an 8-byte boundary and
814 * suppresses padding verification for inbound (i.e.
815 * decrypt) operations.
816 *
817 * NB: Not sure if the 8-byte pad boundary is a problem.
818 */
819 cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
820
821 /* XXX assert key bufs have the same size */
822 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
823 }
824
825 if (maccrd) {
826 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
827 safe_setup_mackey(ses, maccrd->crd_alg,
828 maccrd->crd_key, maccrd->crd_klen / 8);
829 }
830
831 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
832 cmd0 |= SAFE_SA_CMD0_MD5;
833 cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
834 } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
835 cmd0 |= SAFE_SA_CMD0_SHA1;
836 cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
837 } else {
838 cmd0 |= SAFE_SA_CMD0_HASH_NULL;
839 }
840 /*
841 * Digest data is loaded from the SA and the hash
842 * result is saved to the state block where we
843 * retrieve it for return to the caller.
844 */
845 /* XXX assert digest bufs have the same size */
846 bcopy(ses->ses_hminner, sa->sa_indigest,
847 sizeof(sa->sa_indigest));
848 bcopy(ses->ses_hmouter, sa->sa_outdigest,
849 sizeof(sa->sa_outdigest));
850
851 cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
852 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
853 }
854
855 if (enccrd && maccrd) {
856 /*
857 * The offset from hash data to the start of
858 * crypt data is the difference in the skips.
859 */
860 bypass = maccrd->crd_skip;
861 coffset = enccrd->crd_skip - maccrd->crd_skip;
862 if (coffset < 0) {
863 DPRINTF(("%s: hash does not precede crypt; "
864 "mac skip %u enc skip %u\n",
865 __func__, maccrd->crd_skip, enccrd->crd_skip));
866 safestats.st_skipmismatch++;
867 err = EINVAL;
868 goto errout;
869 }
870 oplen = enccrd->crd_skip + enccrd->crd_len;
871 if (maccrd->crd_skip + maccrd->crd_len != oplen) {
872 DPRINTF(("%s: hash amount %u != crypt amount %u\n",
873 __func__, maccrd->crd_skip + maccrd->crd_len,
874 oplen));
875 safestats.st_lenmismatch++;
876 err = EINVAL;
877 goto errout;
878 }
879 #ifdef SAFE_DEBUG
880 if (debug) {
881 printf("mac: skip %d, len %d, inject %d\n",
882 maccrd->crd_skip, maccrd->crd_len,
883 maccrd->crd_inject);
884 printf("enc: skip %d, len %d, inject %d\n",
885 enccrd->crd_skip, enccrd->crd_len,
886 enccrd->crd_inject);
887 printf("bypass %d coffset %d oplen %d\n",
888 bypass, coffset, oplen);
889 }
890 #endif
891 if (coffset & 3) { /* offset must be 32-bit aligned */
892 DPRINTF(("%s: coffset %u misaligned\n",
893 __func__, coffset));
894 safestats.st_coffmisaligned++;
895 err = EINVAL;
896 goto errout;
897 }
898 coffset >>= 2;
899 if (coffset > 255) { /* offset must be <256 dwords */
900 DPRINTF(("%s: coffset %u too big\n",
901 __func__, coffset));
902 safestats.st_cofftoobig++;
903 err = EINVAL;
904 goto errout;
905 }
906 /*
907 * Tell the hardware to copy the header to the output.
908 * The header is defined as the data from the end of
909 * the bypass to the start of data to be encrypted.
910 * Typically this is the inline IV. Note that you need
911 * to do this even if src+dst are the same; it appears
912 * that w/o this bit the crypted data is written
913 * immediately after the bypass data.
914 */
915 cmd1 |= SAFE_SA_CMD1_HDRCOPY;
916 /*
917 * Disable IP header mutable bit handling. This is
918 * needed to get correct HMAC calculations.
919 */
920 cmd1 |= SAFE_SA_CMD1_MUTABLE;
921 } else {
922 if (enccrd) {
923 bypass = enccrd->crd_skip;
924 oplen = bypass + enccrd->crd_len;
925 } else {
926 bypass = maccrd->crd_skip;
927 oplen = bypass + maccrd->crd_len;
928 }
929 coffset = 0;
930 }
931 /* XXX verify multiple of 4 when using s/g */
932 if (bypass > 96) { /* bypass offset must be <= 96 bytes */
933 DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
934 safestats.st_bypasstoobig++;
935 err = EINVAL;
936 goto errout;
937 }
938
939 if (crp->crp_flags & CRYPTO_F_SKBUF) {
940 if (pci_map_skb(sc, &re->re_src, re->re_src_skb)) {
941 safestats.st_noload++;
942 err = ENOMEM;
943 goto errout;
944 }
945 } else if (crp->crp_flags & CRYPTO_F_IOV) {
946 if (pci_map_uio(sc, &re->re_src, re->re_src_io)) {
947 safestats.st_noload++;
948 err = ENOMEM;
949 goto errout;
950 }
951 }
952 nicealign = safe_dmamap_aligned(sc, &re->re_src);
953 uniform = safe_dmamap_uniform(sc, &re->re_src);
954
955 DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
956 nicealign, uniform, re->re_src.nsegs));
957 if (re->re_src.nsegs > 1) {
958 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
959 ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
960 for (i = 0; i < re->re_src_nsegs; i++) {
961 /* NB: no need to check if there's space */
962 pd = sc->sc_spfree;
963 if (++(sc->sc_spfree) == sc->sc_springtop)
964 sc->sc_spfree = sc->sc_spring;
965
966 KASSERT((pd->pd_flags&3) == 0 ||
967 (pd->pd_flags&3) == SAFE_PD_DONE,
968 ("bogus source particle descriptor; flags %x",
969 pd->pd_flags));
970 pd->pd_addr = re->re_src_segs[i].ds_addr;
971 pd->pd_size = re->re_src_segs[i].ds_len;
972 pd->pd_flags = SAFE_PD_READY;
973 }
974 cmd0 |= SAFE_SA_CMD0_IGATHER;
975 } else {
976 /*
977 * No need for gather, reference the operand directly.
978 */
979 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
980 }
981
982 if (enccrd == NULL && maccrd != NULL) {
983 /*
984 * Hash op; no destination needed.
985 */
986 } else {
987 if (crp->crp_flags & (CRYPTO_F_IOV|CRYPTO_F_SKBUF)) {
988 if (!nicealign) {
989 safestats.st_iovmisaligned++;
990 err = EINVAL;
991 goto errout;
992 }
993 if (uniform != 1) {
994 device_printf(sc->sc_dev, "!uniform source\n");
995 if (!uniform) {
996 /*
997 * There's no way to handle the DMA
998 * requirements with this uio. We
999 * could create a separate DMA area for
1000 * the result and then copy it back,
1001 * but for now we just bail and return
1002 * an error. Note that uio requests
1003 * > SAFE_MAX_DSIZE are handled because
1004 * the DMA map and segment list for the
1005 * destination wil result in a
1006 * destination particle list that does
1007 * the necessary scatter DMA.
1008 */
1009 safestats.st_iovnotuniform++;
1010 err = EINVAL;
1011 goto errout;
1012 }
1013 } else
1014 re->re_dst = re->re_src;
1015 } else {
1016 safestats.st_badflags++;
1017 err = EINVAL;
1018 goto errout;
1019 }
1020
1021 if (re->re_dst.nsegs > 1) {
1022 re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
1023 ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
1024 for (i = 0; i < re->re_dst_nsegs; i++) {
1025 pd = sc->sc_dpfree;
1026 KASSERT((pd->pd_flags&3) == 0 ||
1027 (pd->pd_flags&3) == SAFE_PD_DONE,
1028 ("bogus dest particle descriptor; flags %x",
1029 pd->pd_flags));
1030 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
1031 sc->sc_dpfree = sc->sc_dpring;
1032 pd->pd_addr = re->re_dst_segs[i].ds_addr;
1033 pd->pd_flags = SAFE_PD_READY;
1034 }
1035 cmd0 |= SAFE_SA_CMD0_OSCATTER;
1036 } else {
1037 /*
1038 * No need for scatter, reference the operand directly.
1039 */
1040 re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1041 }
1042 }
1043
1044 /*
1045 * All done with setup; fillin the SA command words
1046 * and the packet engine descriptor. The operation
1047 * is now ready for submission to the hardware.
1048 */
1049 sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1050 sa->sa_cmd1 = cmd1
1051 | (coffset << SAFE_SA_CMD1_OFFSET_S)
1052 | SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
1053 | SAFE_SA_CMD1_SRPCI
1054 ;
1055 /*
1056 * NB: the order of writes is important here. In case the
1057 * chip is scanning the ring because of an outstanding request
1058 * it might nab this one too. In that case we need to make
1059 * sure the setup is complete before we write the length
1060 * field of the descriptor as it signals the descriptor is
1061 * ready for processing.
1062 */
1063 re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1064 if (maccrd)
1065 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1066 wmb();
1067 re->re_desc.d_len = oplen
1068 | SAFE_PE_LEN_READY
1069 | (bypass << SAFE_PE_LEN_BYPASS_S)
1070 ;
1071
1072 safestats.st_ipackets++;
1073 safestats.st_ibytes += oplen;
1074
1075 if (++(sc->sc_front) == sc->sc_ringtop)
1076 sc->sc_front = sc->sc_ring;
1077
1078 /* XXX honor batching */
1079 safe_feed(sc, re);
1080 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1081 return (0);
1082
1083 errout:
1084 if (re->re_src.map != re->re_dst.map)
1085 pci_unmap_operand(sc, &re->re_dst);
1086 if (re->re_src.map)
1087 pci_unmap_operand(sc, &re->re_src);
1088 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1089 if (err != ERESTART) {
1090 crp->crp_etype = err;
1091 crypto_done(crp);
1092 } else {
1093 sc->sc_needwakeup |= CRYPTO_SYMQ;
1094 }
1095 return (err);
1096 }
1097
1098 static void
1099 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1100 {
1101 struct cryptop *crp = (struct cryptop *)re->re_crp;
1102 struct cryptodesc *crd;
1103
1104 DPRINTF(("%s()\n", __FUNCTION__));
1105
1106 safestats.st_opackets++;
1107 safestats.st_obytes += re->re_dst.mapsize;
1108
1109 if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1110 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1111 re->re_desc.d_csr,
1112 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1113 safestats.st_peoperr++;
1114 crp->crp_etype = EIO; /* something more meaningful? */
1115 }
1116
1117 if (re->re_dst.map != NULL && re->re_dst.map != re->re_src.map)
1118 pci_unmap_operand(sc, &re->re_dst);
1119 pci_unmap_operand(sc, &re->re_src);
1120
1121 /*
1122 * If result was written to a differet mbuf chain, swap
1123 * it in as the return value and reclaim the original.
1124 */
1125 if ((crp->crp_flags & CRYPTO_F_SKBUF) && re->re_src_skb != re->re_dst_skb) {
1126 device_printf(sc->sc_dev, "no CRYPTO_F_SKBUF swapping support\n");
1127 /* kfree_skb(skb) */
1128 /* crp->crp_buf = (caddr_t)re->re_dst_skb */
1129 return;
1130 }
1131
1132 if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
1133 /* copy out IV for future use */
1134 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1135 int i;
1136 int ivsize;
1137
1138 if (crd->crd_alg == CRYPTO_DES_CBC ||
1139 crd->crd_alg == CRYPTO_3DES_CBC) {
1140 ivsize = 2*sizeof(u_int32_t);
1141 } else if (crd->crd_alg == CRYPTO_AES_CBC) {
1142 ivsize = 4*sizeof(u_int32_t);
1143 } else
1144 continue;
1145 crypto_copydata(crp->crp_flags, crp->crp_buf,
1146 crd->crd_skip + crd->crd_len - ivsize, ivsize,
1147 (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
1148 for (i = 0;
1149 i < ivsize/sizeof(sc->sc_sessions[re->re_sesn].ses_iv[0]);
1150 i++)
1151 sc->sc_sessions[re->re_sesn].ses_iv[i] =
1152 cpu_to_le32(sc->sc_sessions[re->re_sesn].ses_iv[i]);
1153 break;
1154 }
1155 }
1156
1157 if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1158 /* copy out ICV result */
1159 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1160 if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1161 crd->crd_alg == CRYPTO_SHA1_HMAC ||
1162 crd->crd_alg == CRYPTO_NULL_HMAC))
1163 continue;
1164 if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1165 /*
1166 * SHA-1 ICV's are byte-swapped; fix 'em up
1167 * before copy them to their destination.
1168 */
1169 re->re_sastate.sa_saved_indigest[0] =
1170 cpu_to_be32(re->re_sastate.sa_saved_indigest[0]);
1171 re->re_sastate.sa_saved_indigest[1] =
1172 cpu_to_be32(re->re_sastate.sa_saved_indigest[1]);
1173 re->re_sastate.sa_saved_indigest[2] =
1174 cpu_to_be32(re->re_sastate.sa_saved_indigest[2]);
1175 } else {
1176 re->re_sastate.sa_saved_indigest[0] =
1177 cpu_to_le32(re->re_sastate.sa_saved_indigest[0]);
1178 re->re_sastate.sa_saved_indigest[1] =
1179 cpu_to_le32(re->re_sastate.sa_saved_indigest[1]);
1180 re->re_sastate.sa_saved_indigest[2] =
1181 cpu_to_le32(re->re_sastate.sa_saved_indigest[2]);
1182 }
1183 crypto_copyback(crp->crp_flags, crp->crp_buf,
1184 crd->crd_inject,
1185 sc->sc_sessions[re->re_sesn].ses_mlen,
1186 (caddr_t)re->re_sastate.sa_saved_indigest);
1187 break;
1188 }
1189 }
1190 crypto_done(crp);
1191 }
1192
1193
1194 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
1195 #define SAFE_RNG_MAXWAIT 1000
1196
1197 static void
1198 safe_rng_init(struct safe_softc *sc)
1199 {
1200 u_int32_t w, v;
1201 int i;
1202
1203 DPRINTF(("%s()\n", __FUNCTION__));
1204
1205 WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1206 /* use default value according to the manual */
1207 WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
1208 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1209
1210 /*
1211 * There is a bug in rev 1.0 of the 1140 that when the RNG
1212 * is brought out of reset the ready status flag does not
1213 * work until the RNG has finished its internal initialization.
1214 *
1215 * So in order to determine the device is through its
1216 * initialization we must read the data register, using the
1217 * status reg in the read in case it is initialized. Then read
1218 * the data register until it changes from the first read.
1219 * Once it changes read the data register until it changes
1220 * again. At this time the RNG is considered initialized.
1221 * This could take between 750ms - 1000ms in time.
1222 */
1223 i = 0;
1224 w = READ_REG(sc, SAFE_RNG_OUT);
1225 do {
1226 v = READ_REG(sc, SAFE_RNG_OUT);
1227 if (v != w) {
1228 w = v;
1229 break;
1230 }
1231 DELAY(10);
1232 } while (++i < SAFE_RNG_MAXWAIT);
1233
1234 /* Wait Until data changes again */
1235 i = 0;
1236 do {
1237 v = READ_REG(sc, SAFE_RNG_OUT);
1238 if (v != w)
1239 break;
1240 DELAY(10);
1241 } while (++i < SAFE_RNG_MAXWAIT);
1242 }
1243
1244 static __inline void
1245 safe_rng_disable_short_cycle(struct safe_softc *sc)
1246 {
1247 DPRINTF(("%s()\n", __FUNCTION__));
1248
1249 WRITE_REG(sc, SAFE_RNG_CTRL,
1250 READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1251 }
1252
1253 static __inline void
1254 safe_rng_enable_short_cycle(struct safe_softc *sc)
1255 {
1256 DPRINTF(("%s()\n", __FUNCTION__));
1257
1258 WRITE_REG(sc, SAFE_RNG_CTRL,
1259 READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1260 }
1261
1262 static __inline u_int32_t
1263 safe_rng_read(struct safe_softc *sc)
1264 {
1265 int i;
1266
1267 i = 0;
1268 while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1269 ;
1270 return READ_REG(sc, SAFE_RNG_OUT);
1271 }
1272
1273 static int
1274 safe_read_random(void *arg, u_int32_t *buf, int maxwords)
1275 {
1276 struct safe_softc *sc = (struct safe_softc *) arg;
1277 int i, rc;
1278
1279 DPRINTF(("%s()\n", __FUNCTION__));
1280
1281 safestats.st_rng++;
1282 /*
1283 * Fetch the next block of data.
1284 */
1285 if (maxwords > safe_rngbufsize)
1286 maxwords = safe_rngbufsize;
1287 if (maxwords > SAFE_RNG_MAXBUFSIZ)
1288 maxwords = SAFE_RNG_MAXBUFSIZ;
1289 retry:
1290 /* read as much as we can */
1291 for (rc = 0; rc < maxwords; rc++) {
1292 if (READ_REG(sc, SAFE_RNG_STAT) != 0)
1293 break;
1294 buf[rc] = READ_REG(sc, SAFE_RNG_OUT);
1295 }
1296 if (rc == 0)
1297 return 0;
1298 /*
1299 * Check the comparator alarm count and reset the h/w if
1300 * it exceeds our threshold. This guards against the
1301 * hardware oscillators resonating with external signals.
1302 */
1303 if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1304 u_int32_t freq_inc, w;
1305
1306 DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1307 (unsigned)READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1308 safestats.st_rngalarm++;
1309 safe_rng_enable_short_cycle(sc);
1310 freq_inc = 18;
1311 for (i = 0; i < 64; i++) {
1312 w = READ_REG(sc, SAFE_RNG_CNFG);
1313 freq_inc = ((w + freq_inc) & 0x3fL);
1314 w = ((w & ~0x3fL) | freq_inc);
1315 WRITE_REG(sc, SAFE_RNG_CNFG, w);
1316
1317 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1318
1319 (void) safe_rng_read(sc);
1320 DELAY(25);
1321
1322 if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1323 safe_rng_disable_short_cycle(sc);
1324 goto retry;
1325 }
1326 freq_inc = 1;
1327 }
1328 safe_rng_disable_short_cycle(sc);
1329 } else
1330 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1331
1332 return(rc);
1333 }
1334 #endif /* defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG) */
1335
1336
1337 /*
1338 * Resets the board. Values in the regesters are left as is
1339 * from the reset (i.e. initial values are assigned elsewhere).
1340 */
1341 static void
1342 safe_reset_board(struct safe_softc *sc)
1343 {
1344 u_int32_t v;
1345 /*
1346 * Reset the device. The manual says no delay
1347 * is needed between marking and clearing reset.
1348 */
1349 DPRINTF(("%s()\n", __FUNCTION__));
1350
1351 v = READ_REG(sc, SAFE_PE_DMACFG) &~
1352 (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1353 SAFE_PE_DMACFG_SGRESET);
1354 WRITE_REG(sc, SAFE_PE_DMACFG, v
1355 | SAFE_PE_DMACFG_PERESET
1356 | SAFE_PE_DMACFG_PDRRESET
1357 | SAFE_PE_DMACFG_SGRESET);
1358 WRITE_REG(sc, SAFE_PE_DMACFG, v);
1359 }
1360
1361 /*
1362 * Initialize registers we need to touch only once.
1363 */
1364 static void
1365 safe_init_board(struct safe_softc *sc)
1366 {
1367 u_int32_t v, dwords;
1368
1369 DPRINTF(("%s()\n", __FUNCTION__));
1370
1371 v = READ_REG(sc, SAFE_PE_DMACFG);
1372 v &=~ ( SAFE_PE_DMACFG_PEMODE
1373 | SAFE_PE_DMACFG_FSENA /* failsafe enable */
1374 | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1375 | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1376 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1377 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1378 | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1379 | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1380 );
1381 v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
1382 | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1383 | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1384 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1385 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1386 | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */
1387 #if 0
1388 | SAFE_PE_DMACFG_ESPACKET /* swap the packet data */
1389 #endif
1390 ;
1391 WRITE_REG(sc, SAFE_PE_DMACFG, v);
1392
1393 #ifdef __BIG_ENDIAN
1394 /* tell the safenet that we are 4321 and not 1234 */
1395 WRITE_REG(sc, SAFE_ENDIAN, 0xe4e41b1b);
1396 #endif
1397
1398 if (sc->sc_chiprev == SAFE_REV(1,0)) {
1399 /*
1400 * Avoid large PCI DMA transfers. Rev 1.0 has a bug where
1401 * "target mode transfers" done while the chip is DMA'ing
1402 * >1020 bytes cause the hardware to lockup. To avoid this
1403 * we reduce the max PCI transfer size and use small source
1404 * particle descriptors (<= 256 bytes).
1405 */
1406 WRITE_REG(sc, SAFE_DMA_CFG, 256);
1407 device_printf(sc->sc_dev,
1408 "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1409 (unsigned) ((READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff),
1410 (unsigned) SAFE_REV_MAJ(sc->sc_chiprev),
1411 (unsigned) SAFE_REV_MIN(sc->sc_chiprev));
1412 sc->sc_max_dsize = 256;
1413 } else {
1414 sc->sc_max_dsize = SAFE_MAX_DSIZE;
1415 }
1416
1417 /* NB: operands+results are overlaid */
1418 WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1419 WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1420 /*
1421 * Configure ring entry size and number of items in the ring.
1422 */
1423 KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1424 ("PE ring entry not 32-bit aligned!"));
1425 dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1426 WRITE_REG(sc, SAFE_PE_RINGCFG,
1427 (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1428 WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
1429
1430 WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1431 WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1432 WRITE_REG(sc, SAFE_PE_PARTSIZE,
1433 (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1434 /*
1435 * NB: destination particles are fixed size. We use
1436 * an mbuf cluster and require all results go to
1437 * clusters or smaller.
1438 */
1439 WRITE_REG(sc, SAFE_PE_PARTCFG, sc->sc_max_dsize);
1440
1441 /* it's now safe to enable PE mode, do it */
1442 WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1443
1444 /*
1445 * Configure hardware to use level-triggered interrupts and
1446 * to interrupt after each descriptor is processed.
1447 */
1448 WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1449 WRITE_REG(sc, SAFE_HI_CLR, 0xffffffff);
1450 WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1451 WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1452 }
1453
1454
1455 /*
1456 * Clean up after a chip crash.
1457 * It is assumed that the caller in splimp()
1458 */
1459 static void
1460 safe_cleanchip(struct safe_softc *sc)
1461 {
1462 DPRINTF(("%s()\n", __FUNCTION__));
1463
1464 if (sc->sc_nqchip != 0) {
1465 struct safe_ringentry *re = sc->sc_back;
1466
1467 while (re != sc->sc_front) {
1468 if (re->re_desc.d_csr != 0)
1469 safe_free_entry(sc, re);
1470 if (++re == sc->sc_ringtop)
1471 re = sc->sc_ring;
1472 }
1473 sc->sc_back = re;
1474 sc->sc_nqchip = 0;
1475 }
1476 }
1477
1478 /*
1479 * free a safe_q
1480 * It is assumed that the caller is within splimp().
1481 */
1482 static int
1483 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1484 {
1485 struct cryptop *crp;
1486
1487 DPRINTF(("%s()\n", __FUNCTION__));
1488
1489 /*
1490 * Free header MCR
1491 */
1492 if ((re->re_dst_skb != NULL) && (re->re_src_skb != re->re_dst_skb))
1493 #ifdef NOTYET
1494 m_freem(re->re_dst_m);
1495 #else
1496 printk("%s,%d: SKB not supported\n", __FILE__, __LINE__);
1497 #endif
1498
1499 crp = (struct cryptop *)re->re_crp;
1500
1501 re->re_desc.d_csr = 0;
1502
1503 crp->crp_etype = EFAULT;
1504 crypto_done(crp);
1505 return(0);
1506 }
1507
1508 /*
1509 * Routine to reset the chip and clean up.
1510 * It is assumed that the caller is in splimp()
1511 */
1512 static void
1513 safe_totalreset(struct safe_softc *sc)
1514 {
1515 DPRINTF(("%s()\n", __FUNCTION__));
1516
1517 safe_reset_board(sc);
1518 safe_init_board(sc);
1519 safe_cleanchip(sc);
1520 }
1521
1522 /*
1523 * Is the operand suitable aligned for direct DMA. Each
1524 * segment must be aligned on a 32-bit boundary and all
1525 * but the last segment must be a multiple of 4 bytes.
1526 */
1527 static int
1528 safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op)
1529 {
1530 int i;
1531
1532 DPRINTF(("%s()\n", __FUNCTION__));
1533
1534 for (i = 0; i < op->nsegs; i++) {
1535 if (op->segs[i].ds_addr & 3)
1536 return (0);
1537 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1538 return (0);
1539 }
1540 return (1);
1541 }
1542
1543 /*
1544 * Is the operand suitable for direct DMA as the destination
1545 * of an operation. The hardware requires that each ``particle''
1546 * but the last in an operation result have the same size. We
1547 * fix that size at SAFE_MAX_DSIZE bytes. This routine returns
1548 * 0 if some segment is not a multiple of of this size, 1 if all
1549 * segments are exactly this size, or 2 if segments are at worst
1550 * a multple of this size.
1551 */
1552 static int
1553 safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op)
1554 {
1555 int result = 1;
1556
1557 DPRINTF(("%s()\n", __FUNCTION__));
1558
1559 if (op->nsegs > 0) {
1560 int i;
1561
1562 for (i = 0; i < op->nsegs-1; i++) {
1563 if (op->segs[i].ds_len % sc->sc_max_dsize)
1564 return (0);
1565 if (op->segs[i].ds_len != sc->sc_max_dsize)
1566 result = 2;
1567 }
1568 }
1569 return (result);
1570 }
1571
1572 static int
1573 safe_kprocess(device_t dev, struct cryptkop *krp, int hint)
1574 {
1575 struct safe_softc *sc = device_get_softc(dev);
1576 struct safe_pkq *q;
1577 unsigned long flags;
1578
1579 DPRINTF(("%s()\n", __FUNCTION__));
1580
1581 if (sc == NULL) {
1582 krp->krp_status = EINVAL;
1583 goto err;
1584 }
1585
1586 if (krp->krp_op != CRK_MOD_EXP) {
1587 krp->krp_status = EOPNOTSUPP;
1588 goto err;
1589 }
1590
1591 q = (struct safe_pkq *) kmalloc(sizeof(*q), GFP_KERNEL);
1592 if (q == NULL) {
1593 krp->krp_status = ENOMEM;
1594 goto err;
1595 }
1596 memset(q, 0, sizeof(*q));
1597 q->pkq_krp = krp;
1598 INIT_LIST_HEAD(&q->pkq_list);
1599
1600 spin_lock_irqsave(&sc->sc_pkmtx, flags);
1601 list_add_tail(&q->pkq_list, &sc->sc_pkq);
1602 safe_kfeed(sc);
1603 spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1604 return (0);
1605
1606 err:
1607 crypto_kdone(krp);
1608 return (0);
1609 }
1610
1611 #define SAFE_CRK_PARAM_BASE 0
1612 #define SAFE_CRK_PARAM_EXP 1
1613 #define SAFE_CRK_PARAM_MOD 2
1614
1615 static int
1616 safe_kstart(struct safe_softc *sc)
1617 {
1618 struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1619 int exp_bits, mod_bits, base_bits;
1620 u_int32_t op, a_off, b_off, c_off, d_off;
1621
1622 DPRINTF(("%s()\n", __FUNCTION__));
1623
1624 if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1625 krp->krp_status = EINVAL;
1626 return (1);
1627 }
1628
1629 base_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1630 if (base_bits > 2048)
1631 goto too_big;
1632 if (base_bits <= 0) /* 5. base not zero */
1633 goto too_small;
1634
1635 exp_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1636 if (exp_bits > 2048)
1637 goto too_big;
1638 if (exp_bits <= 0) /* 1. exponent word length > 0 */
1639 goto too_small; /* 4. exponent not zero */
1640
1641 mod_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1642 if (mod_bits > 2048)
1643 goto too_big;
1644 if (mod_bits <= 32) /* 2. modulus word length > 1 */
1645 goto too_small; /* 8. MSW of modulus != zero */
1646 if (mod_bits < exp_bits) /* 3 modulus len >= exponent len */
1647 goto too_small;
1648 if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1649 goto bad_domain; /* 6. modulus is odd */
1650 if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1651 goto too_small; /* make sure result will fit */
1652
1653 /* 7. modulus > base */
1654 if (mod_bits < base_bits)
1655 goto too_small;
1656 if (mod_bits == base_bits) {
1657 u_int8_t *basep, *modp;
1658 int i;
1659
1660 basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1661 ((base_bits + 7) / 8) - 1;
1662 modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1663 ((mod_bits + 7) / 8) - 1;
1664
1665 for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1666 if (*modp < *basep)
1667 goto too_small;
1668 if (*modp > *basep)
1669 break;
1670 }
1671 }
1672
1673 /* And on the 9th step, he rested. */
1674
1675 WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1676 WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1677 if (mod_bits > 1024) {
1678 op = SAFE_PK_FUNC_EXP4;
1679 a_off = 0x000;
1680 b_off = 0x100;
1681 c_off = 0x200;
1682 d_off = 0x300;
1683 } else {
1684 op = SAFE_PK_FUNC_EXP16;
1685 a_off = 0x000;
1686 b_off = 0x080;
1687 c_off = 0x100;
1688 d_off = 0x180;
1689 }
1690 sc->sc_pk_reslen = b_off - a_off;
1691 sc->sc_pk_resoff = d_off;
1692
1693 /* A is exponent, B is modulus, C is base, D is result */
1694 safe_kload_reg(sc, a_off, b_off - a_off,
1695 &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1696 WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1697 safe_kload_reg(sc, b_off, b_off - a_off,
1698 &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1699 WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1700 safe_kload_reg(sc, c_off, b_off - a_off,
1701 &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1702 WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1703 WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1704
1705 WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1706
1707 return (0);
1708
1709 too_big:
1710 krp->krp_status = E2BIG;
1711 return (1);
1712 too_small:
1713 krp->krp_status = ERANGE;
1714 return (1);
1715 bad_domain:
1716 krp->krp_status = EDOM;
1717 return (1);
1718 }
1719
1720 static int
1721 safe_ksigbits(struct safe_softc *sc, struct crparam *cr)
1722 {
1723 u_int plen = (cr->crp_nbits + 7) / 8;
1724 int i, sig = plen * 8;
1725 u_int8_t c, *p = cr->crp_p;
1726
1727 DPRINTF(("%s()\n", __FUNCTION__));
1728
1729 for (i = plen - 1; i >= 0; i--) {
1730 c = p[i];
1731 if (c != 0) {
1732 while ((c & 0x80) == 0) {
1733 sig--;
1734 c <<= 1;
1735 }
1736 break;
1737 }
1738 sig -= 8;
1739 }
1740 return (sig);
1741 }
1742
1743 static void
1744 safe_kfeed(struct safe_softc *sc)
1745 {
1746 struct safe_pkq *q, *tmp;
1747
1748 DPRINTF(("%s()\n", __FUNCTION__));
1749
1750 if (list_empty(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1751 return;
1752 if (sc->sc_pkq_cur != NULL)
1753 return;
1754 list_for_each_entry_safe(q, tmp, &sc->sc_pkq, pkq_list) {
1755 sc->sc_pkq_cur = q;
1756 list_del(&q->pkq_list);
1757 if (safe_kstart(sc) != 0) {
1758 crypto_kdone(q->pkq_krp);
1759 kfree(q);
1760 sc->sc_pkq_cur = NULL;
1761 } else {
1762 /* op started, start polling */
1763 mod_timer(&sc->sc_pkto, jiffies + 1);
1764 break;
1765 }
1766 }
1767 }
1768
1769 static void
1770 safe_kpoll(unsigned long arg)
1771 {
1772 struct safe_softc *sc = NULL;
1773 struct safe_pkq *q;
1774 struct crparam *res;
1775 int i;
1776 u_int32_t buf[64];
1777 unsigned long flags;
1778
1779 DPRINTF(("%s()\n", __FUNCTION__));
1780
1781 if (arg >= SAFE_MAX_CHIPS)
1782 return;
1783 sc = safe_chip_idx[arg];
1784 if (!sc) {
1785 DPRINTF(("%s() - bad callback\n", __FUNCTION__));
1786 return;
1787 }
1788
1789 spin_lock_irqsave(&sc->sc_pkmtx, flags);
1790 if (sc->sc_pkq_cur == NULL)
1791 goto out;
1792 if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
1793 /* still running, check back later */
1794 mod_timer(&sc->sc_pkto, jiffies + 1);
1795 goto out;
1796 }
1797
1798 q = sc->sc_pkq_cur;
1799 res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
1800 bzero(buf, sizeof(buf));
1801 bzero(res->crp_p, (res->crp_nbits + 7) / 8);
1802 for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
1803 buf[i] = le32_to_cpu(READ_REG(sc, SAFE_PK_RAM_START +
1804 sc->sc_pk_resoff + (i << 2)));
1805 bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
1806 /*
1807 * reduce the bits that need copying if possible
1808 */
1809 res->crp_nbits = min(res->crp_nbits,sc->sc_pk_reslen * 8);
1810 res->crp_nbits = safe_ksigbits(sc, res);
1811
1812 for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
1813 WRITE_REG(sc, i, 0);
1814
1815 crypto_kdone(q->pkq_krp);
1816 kfree(q);
1817 sc->sc_pkq_cur = NULL;
1818
1819 safe_kfeed(sc);
1820 out:
1821 spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1822 }
1823
1824 static void
1825 safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
1826 struct crparam *n)
1827 {
1828 u_int32_t buf[64], i;
1829
1830 DPRINTF(("%s()\n", __FUNCTION__));
1831
1832 bzero(buf, sizeof(buf));
1833 bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
1834
1835 for (i = 0; i < len >> 2; i++)
1836 WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
1837 cpu_to_le32(buf[i]));
1838 }
1839
1840 #ifdef SAFE_DEBUG
1841 static void
1842 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1843 {
1844 printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1845 , tag
1846 , READ_REG(sc, SAFE_DMA_ENDIAN)
1847 , READ_REG(sc, SAFE_DMA_SRCADDR)
1848 , READ_REG(sc, SAFE_DMA_DSTADDR)
1849 , READ_REG(sc, SAFE_DMA_STAT)
1850 );
1851 }
1852
1853 static void
1854 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1855 {
1856 printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1857 , tag
1858 , READ_REG(sc, SAFE_HI_CFG)
1859 , READ_REG(sc, SAFE_HI_MASK)
1860 , READ_REG(sc, SAFE_HI_DESC_CNT)
1861 , READ_REG(sc, SAFE_HU_STAT)
1862 , READ_REG(sc, SAFE_HM_STAT)
1863 );
1864 }
1865
1866 static void
1867 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1868 {
1869 u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1870
1871 /* NB: assume caller has lock on ring */
1872 printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1873 tag,
1874 estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1875 (unsigned long)(sc->sc_back - sc->sc_ring),
1876 (unsigned long)(sc->sc_front - sc->sc_ring));
1877 }
1878
1879 static void
1880 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1881 {
1882 int ix, nsegs;
1883
1884 ix = re - sc->sc_ring;
1885 printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1886 , tag
1887 , re, ix
1888 , re->re_desc.d_csr
1889 , re->re_desc.d_src
1890 , re->re_desc.d_dst
1891 , re->re_desc.d_sa
1892 , re->re_desc.d_len
1893 );
1894 if (re->re_src.nsegs > 1) {
1895 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1896 sizeof(struct safe_pdesc);
1897 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1898 printf(" spd[%u] %p: %p size %u flags %x"
1899 , ix, &sc->sc_spring[ix]
1900 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1901 , sc->sc_spring[ix].pd_size
1902 , sc->sc_spring[ix].pd_flags
1903 );
1904 if (sc->sc_spring[ix].pd_size == 0)
1905 printf(" (zero!)");
1906 printf("\n");
1907 if (++ix == SAFE_TOTAL_SPART)
1908 ix = 0;
1909 }
1910 }
1911 if (re->re_dst.nsegs > 1) {
1912 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1913 sizeof(struct safe_pdesc);
1914 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1915 printf(" dpd[%u] %p: %p flags %x\n"
1916 , ix, &sc->sc_dpring[ix]
1917 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1918 , sc->sc_dpring[ix].pd_flags
1919 );
1920 if (++ix == SAFE_TOTAL_DPART)
1921 ix = 0;
1922 }
1923 }
1924 printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1925 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1926 printf("sa: key %x %x %x %x %x %x %x %x\n"
1927 , re->re_sa.sa_key[0]
1928 , re->re_sa.sa_key[1]
1929 , re->re_sa.sa_key[2]
1930 , re->re_sa.sa_key[3]
1931 , re->re_sa.sa_key[4]
1932 , re->re_sa.sa_key[5]
1933 , re->re_sa.sa_key[6]
1934 , re->re_sa.sa_key[7]
1935 );
1936 printf("sa: indigest %x %x %x %x %x\n"
1937 , re->re_sa.sa_indigest[0]
1938 , re->re_sa.sa_indigest[1]
1939 , re->re_sa.sa_indigest[2]
1940 , re->re_sa.sa_indigest[3]
1941 , re->re_sa.sa_indigest[4]
1942 );
1943 printf("sa: outdigest %x %x %x %x %x\n"
1944 , re->re_sa.sa_outdigest[0]
1945 , re->re_sa.sa_outdigest[1]
1946 , re->re_sa.sa_outdigest[2]
1947 , re->re_sa.sa_outdigest[3]
1948 , re->re_sa.sa_outdigest[4]
1949 );
1950 printf("sr: iv %x %x %x %x\n"
1951 , re->re_sastate.sa_saved_iv[0]
1952 , re->re_sastate.sa_saved_iv[1]
1953 , re->re_sastate.sa_saved_iv[2]
1954 , re->re_sastate.sa_saved_iv[3]
1955 );
1956 printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1957 , re->re_sastate.sa_saved_hashbc
1958 , re->re_sastate.sa_saved_indigest[0]
1959 , re->re_sastate.sa_saved_indigest[1]
1960 , re->re_sastate.sa_saved_indigest[2]
1961 , re->re_sastate.sa_saved_indigest[3]
1962 , re->re_sastate.sa_saved_indigest[4]
1963 );
1964 }
1965
1966 static void
1967 safe_dump_ring(struct safe_softc *sc, const char *tag)
1968 {
1969 unsigned long flags;
1970
1971 spin_lock_irqsave(&sc->sc_ringmtx, flags);
1972 printf("\nSafeNet Ring State:\n");
1973 safe_dump_intrstate(sc, tag);
1974 safe_dump_dmastatus(sc, tag);
1975 safe_dump_ringstate(sc, tag);
1976 if (sc->sc_nqchip) {
1977 struct safe_ringentry *re = sc->sc_back;
1978 do {
1979 safe_dump_request(sc, tag, re);
1980 if (++re == sc->sc_ringtop)
1981 re = sc->sc_ring;
1982 } while (re != sc->sc_front);
1983 }
1984 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1985 }
1986 #endif /* SAFE_DEBUG */
1987
1988
1989 static int safe_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1990 {
1991 struct safe_softc *sc = NULL;
1992 u32 mem_start, mem_len, cmd;
1993 int i, rc, devinfo;
1994 dma_addr_t raddr;
1995 static int num_chips = 0;
1996
1997 DPRINTF(("%s()\n", __FUNCTION__));
1998
1999 if (pci_enable_device(dev) < 0)
2000 return(-ENODEV);
2001
2002 if (!dev->irq) {
2003 printk("safe: found device with no IRQ assigned. check BIOS settings!");
2004 pci_disable_device(dev);
2005 return(-ENODEV);
2006 }
2007
2008 if (pci_set_mwi(dev)) {
2009 printk("safe: pci_set_mwi failed!");
2010 return(-ENODEV);
2011 }
2012
2013 sc = (struct safe_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
2014 if (!sc)
2015 return(-ENOMEM);
2016 memset(sc, 0, sizeof(*sc));
2017
2018 softc_device_init(sc, "safe", num_chips, safe_methods);
2019
2020 sc->sc_irq = -1;
2021 sc->sc_cid = -1;
2022 sc->sc_pcidev = dev;
2023 if (num_chips < SAFE_MAX_CHIPS) {
2024 safe_chip_idx[device_get_unit(sc->sc_dev)] = sc;
2025 num_chips++;
2026 }
2027
2028 INIT_LIST_HEAD(&sc->sc_pkq);
2029 spin_lock_init(&sc->sc_pkmtx);
2030
2031 pci_set_drvdata(sc->sc_pcidev, sc);
2032
2033 /* we read its hardware registers as memory */
2034 mem_start = pci_resource_start(sc->sc_pcidev, 0);
2035 mem_len = pci_resource_len(sc->sc_pcidev, 0);
2036
2037 sc->sc_base_addr = (ocf_iomem_t) ioremap(mem_start, mem_len);
2038 if (!sc->sc_base_addr) {
2039 device_printf(sc->sc_dev, "failed to ioremap 0x%x-0x%x\n",
2040 mem_start, mem_start + mem_len - 1);
2041 goto out;
2042 }
2043
2044 /* fix up the bus size */
2045 if (pci_set_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2046 device_printf(sc->sc_dev, "No usable DMA configuration, aborting.\n");
2047 goto out;
2048 }
2049 if (pci_set_consistent_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2050 device_printf(sc->sc_dev, "No usable consistent DMA configuration, aborting.\n");
2051 goto out;
2052 }
2053
2054 pci_set_master(sc->sc_pcidev);
2055
2056 pci_read_config_dword(sc->sc_pcidev, PCI_COMMAND, &cmd);
2057
2058 if (!(cmd & PCI_COMMAND_MEMORY)) {
2059 device_printf(sc->sc_dev, "failed to enable memory mapping\n");
2060 goto out;
2061 }
2062
2063 if (!(cmd & PCI_COMMAND_MASTER)) {
2064 device_printf(sc->sc_dev, "failed to enable bus mastering\n");
2065 goto out;
2066 }
2067
2068 rc = request_irq(dev->irq, safe_intr, IRQF_SHARED, "safe", sc);
2069 if (rc) {
2070 device_printf(sc->sc_dev, "failed to hook irq %d\n", sc->sc_irq);
2071 goto out;
2072 }
2073 sc->sc_irq = dev->irq;
2074
2075 sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
2076 (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
2077
2078 /*
2079 * Allocate packet engine descriptors.
2080 */
2081 sc->sc_ringalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2082 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2083 &sc->sc_ringalloc.dma_paddr);
2084 if (!sc->sc_ringalloc.dma_vaddr) {
2085 device_printf(sc->sc_dev, "cannot allocate PE descriptor ring\n");
2086 goto out;
2087 }
2088
2089 /*
2090 * Hookup the static portion of all our data structures.
2091 */
2092 sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
2093 sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
2094 sc->sc_front = sc->sc_ring;
2095 sc->sc_back = sc->sc_ring;
2096 raddr = sc->sc_ringalloc.dma_paddr;
2097 bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
2098 for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
2099 struct safe_ringentry *re = &sc->sc_ring[i];
2100
2101 re->re_desc.d_sa = raddr +
2102 offsetof(struct safe_ringentry, re_sa);
2103 re->re_sa.sa_staterec = raddr +
2104 offsetof(struct safe_ringentry, re_sastate);
2105
2106 raddr += sizeof (struct safe_ringentry);
2107 }
2108 spin_lock_init(&sc->sc_ringmtx);
2109
2110 /*
2111 * Allocate scatter and gather particle descriptors.
2112 */
2113 sc->sc_spalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2114 SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
2115 &sc->sc_spalloc.dma_paddr);
2116 if (!sc->sc_spalloc.dma_vaddr) {
2117 device_printf(sc->sc_dev, "cannot allocate source particle descriptor ring\n");
2118 goto out;
2119 }
2120 sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
2121 sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
2122 sc->sc_spfree = sc->sc_spring;
2123 bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
2124
2125 sc->sc_dpalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2126 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2127 &sc->sc_dpalloc.dma_paddr);
2128 if (!sc->sc_dpalloc.dma_vaddr) {
2129 device_printf(sc->sc_dev, "cannot allocate destination particle descriptor ring\n");
2130 goto out;
2131 }
2132 sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
2133 sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
2134 sc->sc_dpfree = sc->sc_dpring;
2135 bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
2136
2137 sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
2138 if (sc->sc_cid < 0) {
2139 device_printf(sc->sc_dev, "could not get crypto driver id\n");
2140 goto out;
2141 }
2142
2143 printf("%s:", device_get_nameunit(sc->sc_dev));
2144
2145 devinfo = READ_REG(sc, SAFE_DEVINFO);
2146 if (devinfo & SAFE_DEVINFO_RNG) {
2147 sc->sc_flags |= SAFE_FLAGS_RNG;
2148 printf(" rng");
2149 }
2150 if (devinfo & SAFE_DEVINFO_PKEY) {
2151 printf(" key");
2152 sc->sc_flags |= SAFE_FLAGS_KEY;
2153 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
2154 #if 0
2155 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
2156 #endif
2157 init_timer(&sc->sc_pkto);
2158 sc->sc_pkto.function = safe_kpoll;
2159 sc->sc_pkto.data = (unsigned long) device_get_unit(sc->sc_dev);
2160 }
2161 if (devinfo & SAFE_DEVINFO_DES) {
2162 printf(" des/3des");
2163 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
2164 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
2165 }
2166 if (devinfo & SAFE_DEVINFO_AES) {
2167 printf(" aes");
2168 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
2169 }
2170 if (devinfo & SAFE_DEVINFO_MD5) {
2171 printf(" md5");
2172 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
2173 }
2174 if (devinfo & SAFE_DEVINFO_SHA1) {
2175 printf(" sha1");
2176 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
2177 }
2178 printf(" null");
2179 crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
2180 crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
2181 /* XXX other supported algorithms */
2182 printf("\n");
2183
2184 safe_reset_board(sc); /* reset h/w */
2185 safe_init_board(sc); /* init h/w */
2186
2187 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
2188 if (sc->sc_flags & SAFE_FLAGS_RNG) {
2189 safe_rng_init(sc);
2190 crypto_rregister(sc->sc_cid, safe_read_random, sc);
2191 }
2192 #endif /* SAFE_NO_RNG */
2193
2194 return (0);
2195
2196 out:
2197 if (sc->sc_cid >= 0)
2198 crypto_unregister_all(sc->sc_cid);
2199 if (sc->sc_irq != -1)
2200 free_irq(sc->sc_irq, sc);
2201 if (sc->sc_ringalloc.dma_vaddr)
2202 pci_free_consistent(sc->sc_pcidev,
2203 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2204 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2205 if (sc->sc_spalloc.dma_vaddr)
2206 pci_free_consistent(sc->sc_pcidev,
2207 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2208 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2209 if (sc->sc_dpalloc.dma_vaddr)
2210 pci_free_consistent(sc->sc_pcidev,
2211 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2212 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2213 kfree(sc);
2214 return(-ENODEV);
2215 }
2216
2217 static void safe_remove(struct pci_dev *dev)
2218 {
2219 struct safe_softc *sc = pci_get_drvdata(dev);
2220
2221 DPRINTF(("%s()\n", __FUNCTION__));
2222
2223 /* XXX wait/abort active ops */
2224
2225 WRITE_REG(sc, SAFE_HI_MASK, 0); /* disable interrupts */
2226
2227 del_timer_sync(&sc->sc_pkto);
2228
2229 crypto_unregister_all(sc->sc_cid);
2230
2231 safe_cleanchip(sc);
2232
2233 if (sc->sc_irq != -1)
2234 free_irq(sc->sc_irq, sc);
2235 if (sc->sc_ringalloc.dma_vaddr)
2236 pci_free_consistent(sc->sc_pcidev,
2237 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2238 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2239 if (sc->sc_spalloc.dma_vaddr)
2240 pci_free_consistent(sc->sc_pcidev,
2241 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2242 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2243 if (sc->sc_dpalloc.dma_vaddr)
2244 pci_free_consistent(sc->sc_pcidev,
2245 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2246 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2247 sc->sc_irq = -1;
2248 sc->sc_ringalloc.dma_vaddr = NULL;
2249 sc->sc_spalloc.dma_vaddr = NULL;
2250 sc->sc_dpalloc.dma_vaddr = NULL;
2251 }
2252
2253 static struct pci_device_id safe_pci_tbl[] = {
2254 { PCI_VENDOR_SAFENET, PCI_PRODUCT_SAFEXCEL,
2255 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
2256 { },
2257 };
2258 MODULE_DEVICE_TABLE(pci, safe_pci_tbl);
2259
2260 static struct pci_driver safe_driver = {
2261 .name = "safe",
2262 .id_table = safe_pci_tbl,
2263 .probe = safe_probe,
2264 .remove = safe_remove,
2265 /* add PM stuff here one day */
2266 };
2267
2268 static int __init safe_init (void)
2269 {
2270 struct safe_softc *sc = NULL;
2271 int rc;
2272
2273 DPRINTF(("%s(%p)\n", __FUNCTION__, safe_init));
2274
2275 rc = pci_register_driver(&safe_driver);
2276 pci_register_driver_compat(&safe_driver, rc);
2277
2278 return rc;
2279 }
2280
2281 static void __exit safe_exit (void)
2282 {
2283 pci_unregister_driver(&safe_driver);
2284 }
2285
2286 module_init(safe_init);
2287 module_exit(safe_exit);
2288
2289 MODULE_LICENSE("BSD");
2290 MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>");
2291 MODULE_DESCRIPTION("OCF driver for safenet PCI crypto devices");
OpenWrt SVN history