projects / openwrt / openwrt.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
remove linux 2.4 specific build system code
[openwrt/openwrt.git] / target / linux / generic-2.6 / files / crypto / ocf / cryptocteon / cavium_crypto.c
1 /*
2 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
3 *
4 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
5 * reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 * 1. Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 * must display the following acknowledgement:
16 * This product includes software developed by Cavium Networks
17 * 4. Cavium Networks' name may not be used to endorse or promote products
18 * derived from this software without specific prior written permission.
19 *
20 * This Software, including technical data, may be subject to U.S. export
21 * control laws, including the U.S. Export Administration Act and its
22 * associated regulations, and may be subject to export or import regulations
23 * in other countries. You warrant that You will comply strictly in all
24 * respects with all such regulations and acknowledge that you have the
25 * responsibility to obtain licenses to export, re-export or import the
26 * Software.
27 *
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
29 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
30 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
31 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
32 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
33 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
34 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
35 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
36 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
37 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 */
39 /****************************************************************************/
40
41 #include <linux/scatterlist.h>
42 #include <asm/octeon/octeon.h>
43 #include "octeon-asm.h"
44
45 /****************************************************************************/
46
47 extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *);
48 extern void octeon_crypto_disable(struct octeon_cop2_state *, unsigned long);
49
50 #define SG_INIT(s, p, i, l) \
51 { \
52 (i) = 0; \
53 (l) = (s)[0].length; \
54 (p) = (typeof(p)) sg_virt((s)); \
55 CVMX_PREFETCH0((p)); \
56 }
57
58 #define SG_CONSUME(s, p, i, l) \
59 { \
60 (p)++; \
61 (l) -= sizeof(*(p)); \
62 if ((l) < 0) { \
63 dprintk("%s, %d: l = %d\n", __FILE__, __LINE__, l); \
64 } else if ((l) == 0) { \
65 (i)++; \
66 (l) = (s)[0].length; \
67 (p) = (typeof(p)) sg_virt(s); \
68 CVMX_PREFETCH0((p)); \
69 } \
70 }
71
72 #define ESP_HEADER_LENGTH 8
73 #define DES_CBC_IV_LENGTH 8
74 #define AES_CBC_IV_LENGTH 16
75 #define ESP_HMAC_LEN 12
76
77 #define ESP_HEADER_LENGTH 8
78 #define DES_CBC_IV_LENGTH 8
79
80 /****************************************************************************/
81
82 #define CVM_LOAD_SHA_UNIT(dat, next) { \
83 if (next == 0) { \
84 next = 1; \
85 CVMX_MT_HSH_DAT (dat, 0); \
86 } else if (next == 1) { \
87 next = 2; \
88 CVMX_MT_HSH_DAT (dat, 1); \
89 } else if (next == 2) { \
90 next = 3; \
91 CVMX_MT_HSH_DAT (dat, 2); \
92 } else if (next == 3) { \
93 next = 4; \
94 CVMX_MT_HSH_DAT (dat, 3); \
95 } else if (next == 4) { \
96 next = 5; \
97 CVMX_MT_HSH_DAT (dat, 4); \
98 } else if (next == 5) { \
99 next = 6; \
100 CVMX_MT_HSH_DAT (dat, 5); \
101 } else if (next == 6) { \
102 next = 7; \
103 CVMX_MT_HSH_DAT (dat, 6); \
104 } else { \
105 CVMX_MT_HSH_STARTSHA (dat); \
106 next = 0; \
107 } \
108 }
109
110 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \
111 if (next == 0) { \
112 CVMX_MT_HSH_DAT (dat1, 0); \
113 CVMX_MT_HSH_DAT (dat2, 1); \
114 next = 2; \
115 } else if (next == 1) { \
116 CVMX_MT_HSH_DAT (dat1, 1); \
117 CVMX_MT_HSH_DAT (dat2, 2); \
118 next = 3; \
119 } else if (next == 2) { \
120 CVMX_MT_HSH_DAT (dat1, 2); \
121 CVMX_MT_HSH_DAT (dat2, 3); \
122 next = 4; \
123 } else if (next == 3) { \
124 CVMX_MT_HSH_DAT (dat1, 3); \
125 CVMX_MT_HSH_DAT (dat2, 4); \
126 next = 5; \
127 } else if (next == 4) { \
128 CVMX_MT_HSH_DAT (dat1, 4); \
129 CVMX_MT_HSH_DAT (dat2, 5); \
130 next = 6; \
131 } else if (next == 5) { \
132 CVMX_MT_HSH_DAT (dat1, 5); \
133 CVMX_MT_HSH_DAT (dat2, 6); \
134 next = 7; \
135 } else if (next == 6) { \
136 CVMX_MT_HSH_DAT (dat1, 6); \
137 CVMX_MT_HSH_STARTSHA (dat2); \
138 next = 0; \
139 } else { \
140 CVMX_MT_HSH_STARTSHA (dat1); \
141 CVMX_MT_HSH_DAT (dat2, 0); \
142 next = 1; \
143 } \
144 }
145
146 /****************************************************************************/
147
148 #define CVM_LOAD_MD5_UNIT(dat, next) { \
149 if (next == 0) { \
150 next = 1; \
151 CVMX_MT_HSH_DAT (dat, 0); \
152 } else if (next == 1) { \
153 next = 2; \
154 CVMX_MT_HSH_DAT (dat, 1); \
155 } else if (next == 2) { \
156 next = 3; \
157 CVMX_MT_HSH_DAT (dat, 2); \
158 } else if (next == 3) { \
159 next = 4; \
160 CVMX_MT_HSH_DAT (dat, 3); \
161 } else if (next == 4) { \
162 next = 5; \
163 CVMX_MT_HSH_DAT (dat, 4); \
164 } else if (next == 5) { \
165 next = 6; \
166 CVMX_MT_HSH_DAT (dat, 5); \
167 } else if (next == 6) { \
168 next = 7; \
169 CVMX_MT_HSH_DAT (dat, 6); \
170 } else { \
171 CVMX_MT_HSH_STARTMD5 (dat); \
172 next = 0; \
173 } \
174 }
175
176 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \
177 if (next == 0) { \
178 CVMX_MT_HSH_DAT (dat1, 0); \
179 CVMX_MT_HSH_DAT (dat2, 1); \
180 next = 2; \
181 } else if (next == 1) { \
182 CVMX_MT_HSH_DAT (dat1, 1); \
183 CVMX_MT_HSH_DAT (dat2, 2); \
184 next = 3; \
185 } else if (next == 2) { \
186 CVMX_MT_HSH_DAT (dat1, 2); \
187 CVMX_MT_HSH_DAT (dat2, 3); \
188 next = 4; \
189 } else if (next == 3) { \
190 CVMX_MT_HSH_DAT (dat1, 3); \
191 CVMX_MT_HSH_DAT (dat2, 4); \
192 next = 5; \
193 } else if (next == 4) { \
194 CVMX_MT_HSH_DAT (dat1, 4); \
195 CVMX_MT_HSH_DAT (dat2, 5); \
196 next = 6; \
197 } else if (next == 5) { \
198 CVMX_MT_HSH_DAT (dat1, 5); \
199 CVMX_MT_HSH_DAT (dat2, 6); \
200 next = 7; \
201 } else if (next == 6) { \
202 CVMX_MT_HSH_DAT (dat1, 6); \
203 CVMX_MT_HSH_STARTMD5 (dat2); \
204 next = 0; \
205 } else { \
206 CVMX_MT_HSH_STARTMD5 (dat1); \
207 CVMX_MT_HSH_DAT (dat2, 0); \
208 next = 1; \
209 } \
210 }
211
212 /****************************************************************************/
213
214 static inline uint64_t
215 swap64(uint64_t a)
216 {
217 return ((a >> 56) |
218 (((a >> 48) & 0xfful) << 8) |
219 (((a >> 40) & 0xfful) << 16) |
220 (((a >> 32) & 0xfful) << 24) |
221 (((a >> 24) & 0xfful) << 32) |
222 (((a >> 16) & 0xfful) << 40) |
223 (((a >> 8) & 0xfful) << 48) | (((a >> 0) & 0xfful) << 56));
224 }
225
226 /****************************************************************************/
227
228 void
229 octo_calc_hash(__u8 auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
230 {
231 uint8_t hash_key[64];
232 uint64_t *key1;
233 register uint64_t xor1 = 0x3636363636363636ULL;
234 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
235 struct octeon_cop2_state state;
236 unsigned long flags;
237
238 dprintk("%s()\n", __FUNCTION__);
239
240 memset(hash_key, 0, sizeof(hash_key));
241 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
242 key1 = (uint64_t *) hash_key;
243 flags = octeon_crypto_enable(&state);
244 if (auth) {
245 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
246 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
247 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
248 } else {
249 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
250 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
251 }
252
253 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
254 key1++;
255 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
256 key1++;
257 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
258 key1++;
259 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
260 key1++;
261 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
262 key1++;
263 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
264 key1++;
265 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
266 key1++;
267 if (auth)
268 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
269 else
270 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
271
272 CVMX_MF_HSH_IV(inner[0], 0);
273 CVMX_MF_HSH_IV(inner[1], 1);
274 if (auth) {
275 inner[2] = 0;
276 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
277 }
278
279 memset(hash_key, 0, sizeof(hash_key));
280 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
281 key1 = (uint64_t *) hash_key;
282 if (auth) {
283 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
284 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
285 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
286 } else {
287 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
288 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
289 }
290
291 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
292 key1++;
293 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
294 key1++;
295 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
296 key1++;
297 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
298 key1++;
299 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
300 key1++;
301 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
302 key1++;
303 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
304 key1++;
305 if (auth)
306 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
307 else
308 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
309
310 CVMX_MF_HSH_IV(outer[0], 0);
311 CVMX_MF_HSH_IV(outer[1], 1);
312 if (auth) {
313 outer[2] = 0;
314 CVMX_MF_HSH_IV(outer[2], 2);
315 }
316 octeon_crypto_disable(&state, flags);
317 return;
318 }
319
320 /****************************************************************************/
321 /* DES functions */
322
323 int
324 octo_des_cbc_encrypt(
325 struct octo_sess *od,
326 struct scatterlist *sg, int sg_len,
327 int auth_off, int auth_len,
328 int crypt_off, int crypt_len,
329 int icv_off, uint8_t *ivp)
330 {
331 uint64_t *data;
332 int data_i, data_l;
333 struct octeon_cop2_state state;
334 unsigned long flags;
335
336 dprintk("%s()\n", __FUNCTION__);
337
338 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
339 (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) {
340 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
341 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
342 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
343 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
344 return -EINVAL;
345 }
346
347 SG_INIT(sg, data, data_i, data_l);
348
349 CVMX_PREFETCH0(ivp);
350 CVMX_PREFETCH0(od->octo_enckey);
351
352 flags = octeon_crypto_enable(&state);
353
354 /* load 3DES Key */
355 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
356 if (od->octo_encklen == 24) {
357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
358 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
359 } else if (od->octo_encklen == 8) {
360 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
361 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
362 } else {
363 octeon_crypto_disable(&state, flags);
364 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
365 return -EINVAL;
366 }
367
368 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
369
370 while (crypt_off > 0) {
371 SG_CONSUME(sg, data, data_i, data_l);
372 crypt_off -= 8;
373 }
374
375 while (crypt_len > 0) {
376 CVMX_MT_3DES_ENC_CBC(*data);
377 CVMX_MF_3DES_RESULT(*data);
378 SG_CONSUME(sg, data, data_i, data_l);
379 crypt_len -= 8;
380 }
381
382 octeon_crypto_disable(&state, flags);
383 return 0;
384 }
385
386
387 int
388 octo_des_cbc_decrypt(
389 struct octo_sess *od,
390 struct scatterlist *sg, int sg_len,
391 int auth_off, int auth_len,
392 int crypt_off, int crypt_len,
393 int icv_off, uint8_t *ivp)
394 {
395 uint64_t *data;
396 int data_i, data_l;
397 struct octeon_cop2_state state;
398 unsigned long flags;
399
400 dprintk("%s()\n", __FUNCTION__);
401
402 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
403 (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) {
404 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
405 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
406 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
407 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
408 return -EINVAL;
409 }
410
411 SG_INIT(sg, data, data_i, data_l);
412
413 CVMX_PREFETCH0(ivp);
414 CVMX_PREFETCH0(od->octo_enckey);
415
416 flags = octeon_crypto_enable(&state);
417
418 /* load 3DES Key */
419 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
420 if (od->octo_encklen == 24) {
421 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
422 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
423 } else if (od->octo_encklen == 8) {
424 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
425 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
426 } else {
427 octeon_crypto_disable(&state, flags);
428 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
429 return -EINVAL;
430 }
431
432 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
433
434 while (crypt_off > 0) {
435 SG_CONSUME(sg, data, data_i, data_l);
436 crypt_off -= 8;
437 }
438
439 while (crypt_len > 0) {
440 CVMX_MT_3DES_DEC_CBC(*data);
441 CVMX_MF_3DES_RESULT(*data);
442 SG_CONSUME(sg, data, data_i, data_l);
443 crypt_len -= 8;
444 }
445
446 octeon_crypto_disable(&state, flags);
447 return 0;
448 }
449
450 /****************************************************************************/
451 /* AES functions */
452
453 int
454 octo_aes_cbc_encrypt(
455 struct octo_sess *od,
456 struct scatterlist *sg, int sg_len,
457 int auth_off, int auth_len,
458 int crypt_off, int crypt_len,
459 int icv_off, uint8_t *ivp)
460 {
461 uint64_t *data, *pdata;
462 int data_i, data_l;
463 struct octeon_cop2_state state;
464 unsigned long flags;
465
466 dprintk("%s()\n", __FUNCTION__);
467
468 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
469 (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) {
470 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
471 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
472 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
473 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
474 return -EINVAL;
475 }
476
477 SG_INIT(sg, data, data_i, data_l);
478
479 CVMX_PREFETCH0(ivp);
480 CVMX_PREFETCH0(od->octo_enckey);
481
482 flags = octeon_crypto_enable(&state);
483
484 /* load AES Key */
485 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
486 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
487
488 if (od->octo_encklen == 16) {
489 CVMX_MT_AES_KEY(0x0, 2);
490 CVMX_MT_AES_KEY(0x0, 3);
491 } else if (od->octo_encklen == 24) {
492 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
493 CVMX_MT_AES_KEY(0x0, 3);
494 } else if (od->octo_encklen == 32) {
495 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
496 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
497 } else {
498 octeon_crypto_disable(&state, flags);
499 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
500 return -EINVAL;
501 }
502 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
503
504 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
505 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
506
507 while (crypt_off > 0) {
508 SG_CONSUME(sg, data, data_i, data_l);
509 crypt_off -= 8;
510 }
511
512 while (crypt_len > 0) {
513 pdata = data;
514 CVMX_MT_AES_ENC_CBC0(*data);
515 SG_CONSUME(sg, data, data_i, data_l);
516 CVMX_MT_AES_ENC_CBC1(*data);
517 CVMX_MF_AES_RESULT(*pdata, 0);
518 CVMX_MF_AES_RESULT(*data, 1);
519 SG_CONSUME(sg, data, data_i, data_l);
520 crypt_len -= 16;
521 }
522
523 octeon_crypto_disable(&state, flags);
524 return 0;
525 }
526
527
528 int
529 octo_aes_cbc_decrypt(
530 struct octo_sess *od,
531 struct scatterlist *sg, int sg_len,
532 int auth_off, int auth_len,
533 int crypt_off, int crypt_len,
534 int icv_off, uint8_t *ivp)
535 {
536 uint64_t *data, *pdata;
537 int data_i, data_l;
538 struct octeon_cop2_state state;
539 unsigned long flags;
540
541 dprintk("%s()\n", __FUNCTION__);
542
543 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
544 (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) {
545 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
546 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
547 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
548 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
549 return -EINVAL;
550 }
551
552 SG_INIT(sg, data, data_i, data_l);
553
554 CVMX_PREFETCH0(ivp);
555 CVMX_PREFETCH0(od->octo_enckey);
556
557 flags = octeon_crypto_enable(&state);
558
559 /* load AES Key */
560 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
561 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
562
563 if (od->octo_encklen == 16) {
564 CVMX_MT_AES_KEY(0x0, 2);
565 CVMX_MT_AES_KEY(0x0, 3);
566 } else if (od->octo_encklen == 24) {
567 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
568 CVMX_MT_AES_KEY(0x0, 3);
569 } else if (od->octo_encklen == 32) {
570 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
571 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
572 } else {
573 octeon_crypto_disable(&state, flags);
574 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
575 return -EINVAL;
576 }
577 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
578
579 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
580 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
581
582 while (crypt_off > 0) {
583 SG_CONSUME(sg, data, data_i, data_l);
584 crypt_off -= 8;
585 }
586
587 while (crypt_len > 0) {
588 pdata = data;
589 CVMX_MT_AES_DEC_CBC0(*data);
590 SG_CONSUME(sg, data, data_i, data_l);
591 CVMX_MT_AES_DEC_CBC1(*data);
592 CVMX_MF_AES_RESULT(*pdata, 0);
593 CVMX_MF_AES_RESULT(*data, 1);
594 SG_CONSUME(sg, data, data_i, data_l);
595 crypt_len -= 16;
596 }
597
598 octeon_crypto_disable(&state, flags);
599 return 0;
600 }
601
602 /****************************************************************************/
603 /* MD5 */
604
605 int
606 octo_null_md5_encrypt(
607 struct octo_sess *od,
608 struct scatterlist *sg, int sg_len,
609 int auth_off, int auth_len,
610 int crypt_off, int crypt_len,
611 int icv_off, uint8_t *ivp)
612 {
613 register int next = 0;
614 uint64_t *data;
615 uint64_t tmp1, tmp2;
616 int data_i, data_l, alen = auth_len;
617 struct octeon_cop2_state state;
618 unsigned long flags;
619
620 dprintk("%s()\n", __FUNCTION__);
621
622 if (unlikely(od == NULL || sg==NULL || sg_len==0 ||
623 (auth_off & 0x7) || (auth_off + auth_len > sg_len))) {
624 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
625 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
626 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
627 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
628 return -EINVAL;
629 }
630
631 SG_INIT(sg, data, data_i, data_l);
632
633 flags = octeon_crypto_enable(&state);
634
635 /* Load MD5 IV */
636 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
637 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
638
639 while (auth_off > 0) {
640 SG_CONSUME(sg, data, data_i, data_l);
641 auth_off -= 8;
642 }
643
644 while (auth_len > 0) {
645 CVM_LOAD_MD5_UNIT(*data, next);
646 auth_len -= 8;
647 SG_CONSUME(sg, data, data_i, data_l);
648 }
649
650 /* finish the hash */
651 CVMX_PREFETCH0(od->octo_hmouter);
652 #if 0
653 if (unlikely(inplen)) {
654 uint64_t tmp = 0;
655 uint8_t *p = (uint8_t *) & tmp;
656 p[inplen] = 0x80;
657 do {
658 inplen--;
659 p[inplen] = ((uint8_t *) data)[inplen];
660 } while (inplen);
661 CVM_LOAD_MD5_UNIT(tmp, next);
662 } else {
663 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
664 }
665 #else
666 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
667 #endif
668
669 /* Finish Inner hash */
670 while (next != 7) {
671 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
672 }
673 CVMX_ES64(tmp1, ((alen + 64) << 3));
674 CVM_LOAD_MD5_UNIT(tmp1, next);
675
676 /* Get the inner hash of HMAC */
677 CVMX_MF_HSH_IV(tmp1, 0);
678 CVMX_MF_HSH_IV(tmp2, 1);
679
680 /* Initialize hash unit */
681 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
682 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
683
684 CVMX_MT_HSH_DAT(tmp1, 0);
685 CVMX_MT_HSH_DAT(tmp2, 1);
686 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
687 CVMX_MT_HSH_DATZ(3);
688 CVMX_MT_HSH_DATZ(4);
689 CVMX_MT_HSH_DATZ(5);
690 CVMX_MT_HSH_DATZ(6);
691 CVMX_ES64(tmp1, ((64 + 16) << 3));
692 CVMX_MT_HSH_STARTMD5(tmp1);
693
694 /* save the HMAC */
695 SG_INIT(sg, data, data_i, data_l);
696 while (icv_off > 0) {
697 SG_CONSUME(sg, data, data_i, data_l);
698 icv_off -= 8;
699 }
700 CVMX_MF_HSH_IV(*data, 0);
701 SG_CONSUME(sg, data, data_i, data_l);
702 CVMX_MF_HSH_IV(tmp1, 1);
703 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
704
705 octeon_crypto_disable(&state, flags);
706 return 0;
707 }
708
709 /****************************************************************************/
710 /* SHA1 */
711
712 int
713 octo_null_sha1_encrypt(
714 struct octo_sess *od,
715 struct scatterlist *sg, int sg_len,
716 int auth_off, int auth_len,
717 int crypt_off, int crypt_len,
718 int icv_off, uint8_t *ivp)
719 {
720 register int next = 0;
721 uint64_t *data;
722 uint64_t tmp1, tmp2, tmp3;
723 int data_i, data_l, alen = auth_len;
724 struct octeon_cop2_state state;
725 unsigned long flags;
726
727 dprintk("%s()\n", __FUNCTION__);
728
729 if (unlikely(od == NULL || sg==NULL || sg_len==0 ||
730 (auth_off & 0x7) || (auth_off + auth_len > sg_len))) {
731 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
732 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
733 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
734 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
735 return -EINVAL;
736 }
737
738 SG_INIT(sg, data, data_i, data_l);
739
740 flags = octeon_crypto_enable(&state);
741
742 /* Load SHA1 IV */
743 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
744 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
745 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
746
747 while (auth_off > 0) {
748 SG_CONSUME(sg, data, data_i, data_l);
749 auth_off -= 8;
750 }
751
752 while (auth_len > 0) {
753 CVM_LOAD_SHA_UNIT(*data, next);
754 auth_len -= 8;
755 SG_CONSUME(sg, data, data_i, data_l);
756 }
757
758 /* finish the hash */
759 CVMX_PREFETCH0(od->octo_hmouter);
760 #if 0
761 if (unlikely(inplen)) {
762 uint64_t tmp = 0;
763 uint8_t *p = (uint8_t *) & tmp;
764 p[inplen] = 0x80;
765 do {
766 inplen--;
767 p[inplen] = ((uint8_t *) data)[inplen];
768 } while (inplen);
769 CVM_LOAD_MD5_UNIT(tmp, next);
770 } else {
771 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
772 }
773 #else
774 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
775 #endif
776
777 /* Finish Inner hash */
778 while (next != 7) {
779 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
780 }
781 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
782
783 /* Get the inner hash of HMAC */
784 CVMX_MF_HSH_IV(tmp1, 0);
785 CVMX_MF_HSH_IV(tmp2, 1);
786 tmp3 = 0;
787 CVMX_MF_HSH_IV(tmp3, 2);
788
789 /* Initialize hash unit */
790 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
791 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
792 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
793
794 CVMX_MT_HSH_DAT(tmp1, 0);
795 CVMX_MT_HSH_DAT(tmp2, 1);
796 tmp3 |= 0x0000000080000000;
797 CVMX_MT_HSH_DAT(tmp3, 2);
798 CVMX_MT_HSH_DATZ(3);
799 CVMX_MT_HSH_DATZ(4);
800 CVMX_MT_HSH_DATZ(5);
801 CVMX_MT_HSH_DATZ(6);
802 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
803
804 /* save the HMAC */
805 SG_INIT(sg, data, data_i, data_l);
806 while (icv_off > 0) {
807 SG_CONSUME(sg, data, data_i, data_l);
808 icv_off -= 8;
809 }
810 CVMX_MF_HSH_IV(*data, 0);
811 SG_CONSUME(sg, data, data_i, data_l);
812 CVMX_MF_HSH_IV(tmp1, 1);
813 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
814
815 octeon_crypto_disable(&state, flags);
816 return 0;
817 }
818
819 /****************************************************************************/
820 /* DES MD5 */
821
822 int
823 octo_des_cbc_md5_encrypt(
824 struct octo_sess *od,
825 struct scatterlist *sg, int sg_len,
826 int auth_off, int auth_len,
827 int crypt_off, int crypt_len,
828 int icv_off, uint8_t *ivp)
829 {
830 register int next = 0;
831 union {
832 uint32_t data32[2];
833 uint64_t data64[1];
834 } mydata;
835 uint64_t *data = &mydata.data64[0];
836 uint32_t *data32;
837 uint64_t tmp1, tmp2;
838 int data_i, data_l, alen = auth_len;
839 struct octeon_cop2_state state;
840 unsigned long flags;
841
842 dprintk("%s()\n", __FUNCTION__);
843
844 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
845 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
846 (crypt_len & 0x7) ||
847 (auth_len & 0x7) ||
848 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
849 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
850 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
851 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
852 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
853 return -EINVAL;
854 }
855
856 SG_INIT(sg, data32, data_i, data_l);
857
858 CVMX_PREFETCH0(ivp);
859 CVMX_PREFETCH0(od->octo_enckey);
860
861 flags = octeon_crypto_enable(&state);
862
863 /* load 3DES Key */
864 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
865 if (od->octo_encklen == 24) {
866 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
867 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
868 } else if (od->octo_encklen == 8) {
869 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
870 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
871 } else {
872 octeon_crypto_disable(&state, flags);
873 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
874 return -EINVAL;
875 }
876
877 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
878
879 /* Load MD5 IV */
880 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
881 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
882
883 while (crypt_off > 0 && auth_off > 0) {
884 SG_CONSUME(sg, data32, data_i, data_l);
885 crypt_off -= 4;
886 auth_off -= 4;
887 }
888
889 while (crypt_len > 0 || auth_len > 0) {
890 uint32_t *first = data32;
891 mydata.data32[0] = *first;
892 SG_CONSUME(sg, data32, data_i, data_l);
893 mydata.data32[1] = *data32;
894 if (crypt_off <= 0) {
895 if (crypt_len > 0) {
896 CVMX_MT_3DES_ENC_CBC(*data);
897 CVMX_MF_3DES_RESULT(*data);
898 crypt_len -= 8;
899 }
900 } else
901 crypt_off -= 8;
902 if (auth_off <= 0) {
903 if (auth_len > 0) {
904 CVM_LOAD_MD5_UNIT(*data, next);
905 auth_len -= 8;
906 }
907 } else
908 auth_off -= 8;
909 *first = mydata.data32[0];
910 *data32 = mydata.data32[1];
911 SG_CONSUME(sg, data32, data_i, data_l);
912 }
913
914 /* finish the hash */
915 CVMX_PREFETCH0(od->octo_hmouter);
916 #if 0
917 if (unlikely(inplen)) {
918 uint64_t tmp = 0;
919 uint8_t *p = (uint8_t *) & tmp;
920 p[inplen] = 0x80;
921 do {
922 inplen--;
923 p[inplen] = ((uint8_t *) data)[inplen];
924 } while (inplen);
925 CVM_LOAD_MD5_UNIT(tmp, next);
926 } else {
927 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
928 }
929 #else
930 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
931 #endif
932
933 /* Finish Inner hash */
934 while (next != 7) {
935 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
936 }
937 CVMX_ES64(tmp1, ((alen + 64) << 3));
938 CVM_LOAD_MD5_UNIT(tmp1, next);
939
940 /* Get the inner hash of HMAC */
941 CVMX_MF_HSH_IV(tmp1, 0);
942 CVMX_MF_HSH_IV(tmp2, 1);
943
944 /* Initialize hash unit */
945 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
946 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
947
948 CVMX_MT_HSH_DAT(tmp1, 0);
949 CVMX_MT_HSH_DAT(tmp2, 1);
950 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
951 CVMX_MT_HSH_DATZ(3);
952 CVMX_MT_HSH_DATZ(4);
953 CVMX_MT_HSH_DATZ(5);
954 CVMX_MT_HSH_DATZ(6);
955 CVMX_ES64(tmp1, ((64 + 16) << 3));
956 CVMX_MT_HSH_STARTMD5(tmp1);
957
958 /* save the HMAC */
959 SG_INIT(sg, data32, data_i, data_l);
960 while (icv_off > 0) {
961 SG_CONSUME(sg, data32, data_i, data_l);
962 icv_off -= 4;
963 }
964 CVMX_MF_HSH_IV(tmp1, 0);
965 *data32 = (uint32_t) (tmp1 >> 32);
966 SG_CONSUME(sg, data32, data_i, data_l);
967 *data32 = (uint32_t) tmp1;
968 SG_CONSUME(sg, data32, data_i, data_l);
969 CVMX_MF_HSH_IV(tmp1, 1);
970 *data32 = (uint32_t) (tmp1 >> 32);
971
972 octeon_crypto_disable(&state, flags);
973 return 0;
974 }
975
976 int
977 octo_des_cbc_md5_decrypt(
978 struct octo_sess *od,
979 struct scatterlist *sg, int sg_len,
980 int auth_off, int auth_len,
981 int crypt_off, int crypt_len,
982 int icv_off, uint8_t *ivp)
983 {
984 register int next = 0;
985 union {
986 uint32_t data32[2];
987 uint64_t data64[1];
988 } mydata;
989 uint64_t *data = &mydata.data64[0];
990 uint32_t *data32;
991 uint64_t tmp1, tmp2;
992 int data_i, data_l, alen = auth_len;
993 struct octeon_cop2_state state;
994 unsigned long flags;
995
996 dprintk("%s()\n", __FUNCTION__);
997
998 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
999 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1000 (crypt_len & 0x7) ||
1001 (auth_len & 0x7) ||
1002 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1003 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1004 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1005 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1006 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1007 return -EINVAL;
1008 }
1009
1010 SG_INIT(sg, data32, data_i, data_l);
1011
1012 CVMX_PREFETCH0(ivp);
1013 CVMX_PREFETCH0(od->octo_enckey);
1014
1015 flags = octeon_crypto_enable(&state);
1016
1017 /* load 3DES Key */
1018 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1019 if (od->octo_encklen == 24) {
1020 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1021 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1022 } else if (od->octo_encklen == 8) {
1023 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1024 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1025 } else {
1026 octeon_crypto_disable(&state, flags);
1027 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1028 return -EINVAL;
1029 }
1030
1031 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1032
1033 /* Load MD5 IV */
1034 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1035 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1036
1037 while (crypt_off > 0 && auth_off > 0) {
1038 SG_CONSUME(sg, data32, data_i, data_l);
1039 crypt_off -= 4;
1040 auth_off -= 4;
1041 }
1042
1043 while (crypt_len > 0 || auth_len > 0) {
1044 uint32_t *first = data32;
1045 mydata.data32[0] = *first;
1046 SG_CONSUME(sg, data32, data_i, data_l);
1047 mydata.data32[1] = *data32;
1048 if (auth_off <= 0) {
1049 if (auth_len > 0) {
1050 CVM_LOAD_MD5_UNIT(*data, next);
1051 auth_len -= 8;
1052 }
1053 } else
1054 auth_off -= 8;
1055 if (crypt_off <= 0) {
1056 if (crypt_len > 0) {
1057 CVMX_MT_3DES_DEC_CBC(*data);
1058 CVMX_MF_3DES_RESULT(*data);
1059 crypt_len -= 8;
1060 }
1061 } else
1062 crypt_off -= 8;
1063 *first = mydata.data32[0];
1064 *data32 = mydata.data32[1];
1065 SG_CONSUME(sg, data32, data_i, data_l);
1066 }
1067
1068 /* finish the hash */
1069 CVMX_PREFETCH0(od->octo_hmouter);
1070 #if 0
1071 if (unlikely(inplen)) {
1072 uint64_t tmp = 0;
1073 uint8_t *p = (uint8_t *) & tmp;
1074 p[inplen] = 0x80;
1075 do {
1076 inplen--;
1077 p[inplen] = ((uint8_t *) data)[inplen];
1078 } while (inplen);
1079 CVM_LOAD_MD5_UNIT(tmp, next);
1080 } else {
1081 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1082 }
1083 #else
1084 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1085 #endif
1086
1087 /* Finish Inner hash */
1088 while (next != 7) {
1089 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1090 }
1091 CVMX_ES64(tmp1, ((alen + 64) << 3));
1092 CVM_LOAD_MD5_UNIT(tmp1, next);
1093
1094 /* Get the inner hash of HMAC */
1095 CVMX_MF_HSH_IV(tmp1, 0);
1096 CVMX_MF_HSH_IV(tmp2, 1);
1097
1098 /* Initialize hash unit */
1099 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1100 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1101
1102 CVMX_MT_HSH_DAT(tmp1, 0);
1103 CVMX_MT_HSH_DAT(tmp2, 1);
1104 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1105 CVMX_MT_HSH_DATZ(3);
1106 CVMX_MT_HSH_DATZ(4);
1107 CVMX_MT_HSH_DATZ(5);
1108 CVMX_MT_HSH_DATZ(6);
1109 CVMX_ES64(tmp1, ((64 + 16) << 3));
1110 CVMX_MT_HSH_STARTMD5(tmp1);
1111
1112 /* save the HMAC */
1113 SG_INIT(sg, data32, data_i, data_l);
1114 while (icv_off > 0) {
1115 SG_CONSUME(sg, data32, data_i, data_l);
1116 icv_off -= 4;
1117 }
1118 CVMX_MF_HSH_IV(tmp1, 0);
1119 *data32 = (uint32_t) (tmp1 >> 32);
1120 SG_CONSUME(sg, data32, data_i, data_l);
1121 *data32 = (uint32_t) tmp1;
1122 SG_CONSUME(sg, data32, data_i, data_l);
1123 CVMX_MF_HSH_IV(tmp1, 1);
1124 *data32 = (uint32_t) (tmp1 >> 32);
1125
1126 octeon_crypto_disable(&state, flags);
1127 return 0;
1128 }
1129
1130 /****************************************************************************/
1131 /* DES SHA */
1132
1133 int
1134 octo_des_cbc_sha1_encrypt(
1135 struct octo_sess *od,
1136 struct scatterlist *sg, int sg_len,
1137 int auth_off, int auth_len,
1138 int crypt_off, int crypt_len,
1139 int icv_off, uint8_t *ivp)
1140 {
1141 register int next = 0;
1142 union {
1143 uint32_t data32[2];
1144 uint64_t data64[1];
1145 } mydata;
1146 uint64_t *data = &mydata.data64[0];
1147 uint32_t *data32;
1148 uint64_t tmp1, tmp2, tmp3;
1149 int data_i, data_l, alen = auth_len;
1150 struct octeon_cop2_state state;
1151 unsigned long flags;
1152
1153 dprintk("%s()\n", __FUNCTION__);
1154
1155 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
1156 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1157 (crypt_len & 0x7) ||
1158 (auth_len & 0x7) ||
1159 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1160 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1161 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1162 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1163 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1164 return -EINVAL;
1165 }
1166
1167 SG_INIT(sg, data32, data_i, data_l);
1168
1169 CVMX_PREFETCH0(ivp);
1170 CVMX_PREFETCH0(od->octo_enckey);
1171
1172 flags = octeon_crypto_enable(&state);
1173
1174 /* load 3DES Key */
1175 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1176 if (od->octo_encklen == 24) {
1177 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1178 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1179 } else if (od->octo_encklen == 8) {
1180 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1181 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1182 } else {
1183 octeon_crypto_disable(&state, flags);
1184 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1185 return -EINVAL;
1186 }
1187
1188 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1189
1190 /* Load SHA1 IV */
1191 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1192 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1193 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1194
1195 while (crypt_off > 0 && auth_off > 0) {
1196 SG_CONSUME(sg, data32, data_i, data_l);
1197 crypt_off -= 4;
1198 auth_off -= 4;
1199 }
1200
1201 while (crypt_len > 0 || auth_len > 0) {
1202 uint32_t *first = data32;
1203 mydata.data32[0] = *first;
1204 SG_CONSUME(sg, data32, data_i, data_l);
1205 mydata.data32[1] = *data32;
1206 if (crypt_off <= 0) {
1207 if (crypt_len > 0) {
1208 CVMX_MT_3DES_ENC_CBC(*data);
1209 CVMX_MF_3DES_RESULT(*data);
1210 crypt_len -= 8;
1211 }
1212 } else
1213 crypt_off -= 8;
1214 if (auth_off <= 0) {
1215 if (auth_len > 0) {
1216 CVM_LOAD_SHA_UNIT(*data, next);
1217 auth_len -= 8;
1218 }
1219 } else
1220 auth_off -= 8;
1221 *first = mydata.data32[0];
1222 *data32 = mydata.data32[1];
1223 SG_CONSUME(sg, data32, data_i, data_l);
1224 }
1225
1226 /* finish the hash */
1227 CVMX_PREFETCH0(od->octo_hmouter);
1228 #if 0
1229 if (unlikely(inplen)) {
1230 uint64_t tmp = 0;
1231 uint8_t *p = (uint8_t *) & tmp;
1232 p[inplen] = 0x80;
1233 do {
1234 inplen--;
1235 p[inplen] = ((uint8_t *) data)[inplen];
1236 } while (inplen);
1237 CVM_LOAD_SHA_UNIT(tmp, next);
1238 } else {
1239 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1240 }
1241 #else
1242 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1243 #endif
1244
1245 /* Finish Inner hash */
1246 while (next != 7) {
1247 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1248 }
1249 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1250
1251 /* Get the inner hash of HMAC */
1252 CVMX_MF_HSH_IV(tmp1, 0);
1253 CVMX_MF_HSH_IV(tmp2, 1);
1254 tmp3 = 0;
1255 CVMX_MF_HSH_IV(tmp3, 2);
1256
1257 /* Initialize hash unit */
1258 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1259 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1260 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1261
1262 CVMX_MT_HSH_DAT(tmp1, 0);
1263 CVMX_MT_HSH_DAT(tmp2, 1);
1264 tmp3 |= 0x0000000080000000;
1265 CVMX_MT_HSH_DAT(tmp3, 2);
1266 CVMX_MT_HSH_DATZ(3);
1267 CVMX_MT_HSH_DATZ(4);
1268 CVMX_MT_HSH_DATZ(5);
1269 CVMX_MT_HSH_DATZ(6);
1270 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1271
1272 /* save the HMAC */
1273 SG_INIT(sg, data32, data_i, data_l);
1274 while (icv_off > 0) {
1275 SG_CONSUME(sg, data32, data_i, data_l);
1276 icv_off -= 4;
1277 }
1278 CVMX_MF_HSH_IV(tmp1, 0);
1279 *data32 = (uint32_t) (tmp1 >> 32);
1280 SG_CONSUME(sg, data32, data_i, data_l);
1281 *data32 = (uint32_t) tmp1;
1282 SG_CONSUME(sg, data32, data_i, data_l);
1283 CVMX_MF_HSH_IV(tmp1, 1);
1284 *data32 = (uint32_t) (tmp1 >> 32);
1285
1286 octeon_crypto_disable(&state, flags);
1287 return 0;
1288 }
1289
1290 int
1291 octo_des_cbc_sha1_decrypt(
1292 struct octo_sess *od,
1293 struct scatterlist *sg, int sg_len,
1294 int auth_off, int auth_len,
1295 int crypt_off, int crypt_len,
1296 int icv_off, uint8_t *ivp)
1297 {
1298 register int next = 0;
1299 union {
1300 uint32_t data32[2];
1301 uint64_t data64[1];
1302 } mydata;
1303 uint64_t *data = &mydata.data64[0];
1304 uint32_t *data32;
1305 uint64_t tmp1, tmp2, tmp3;
1306 int data_i, data_l, alen = auth_len;
1307 struct octeon_cop2_state state;
1308 unsigned long flags;
1309
1310 dprintk("%s()\n", __FUNCTION__);
1311
1312 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
1313 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1314 (crypt_len & 0x7) ||
1315 (auth_len & 0x7) ||
1316 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1317 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1318 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1319 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1320 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1321 return -EINVAL;
1322 }
1323
1324 SG_INIT(sg, data32, data_i, data_l);
1325
1326 CVMX_PREFETCH0(ivp);
1327 CVMX_PREFETCH0(od->octo_enckey);
1328
1329 flags = octeon_crypto_enable(&state);
1330
1331 /* load 3DES Key */
1332 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1333 if (od->octo_encklen == 24) {
1334 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1335 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1336 } else if (od->octo_encklen == 8) {
1337 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1338 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1339 } else {
1340 octeon_crypto_disable(&state, flags);
1341 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1342 return -EINVAL;
1343 }
1344
1345 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1346
1347 /* Load SHA1 IV */
1348 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1349 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1350 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1351
1352 while (crypt_off > 0 && auth_off > 0) {
1353 SG_CONSUME(sg, data32, data_i, data_l);
1354 crypt_off -= 4;
1355 auth_off -= 4;
1356 }
1357
1358 while (crypt_len > 0 || auth_len > 0) {
1359 uint32_t *first = data32;
1360 mydata.data32[0] = *first;
1361 SG_CONSUME(sg, data32, data_i, data_l);
1362 mydata.data32[1] = *data32;
1363 if (auth_off <= 0) {
1364 if (auth_len > 0) {
1365 CVM_LOAD_SHA_UNIT(*data, next);
1366 auth_len -= 8;
1367 }
1368 } else
1369 auth_off -= 8;
1370 if (crypt_off <= 0) {
1371 if (crypt_len > 0) {
1372 CVMX_MT_3DES_DEC_CBC(*data);
1373 CVMX_MF_3DES_RESULT(*data);
1374 crypt_len -= 8;
1375 }
1376 } else
1377 crypt_off -= 8;
1378 *first = mydata.data32[0];
1379 *data32 = mydata.data32[1];
1380 SG_CONSUME(sg, data32, data_i, data_l);
1381 }
1382
1383 /* finish the hash */
1384 CVMX_PREFETCH0(od->octo_hmouter);
1385 #if 0
1386 if (unlikely(inplen)) {
1387 uint64_t tmp = 0;
1388 uint8_t *p = (uint8_t *) & tmp;
1389 p[inplen] = 0x80;
1390 do {
1391 inplen--;
1392 p[inplen] = ((uint8_t *) data)[inplen];
1393 } while (inplen);
1394 CVM_LOAD_SHA_UNIT(tmp, next);
1395 } else {
1396 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1397 }
1398 #else
1399 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1400 #endif
1401
1402 /* Finish Inner hash */
1403 while (next != 7) {
1404 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1405 }
1406 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1407
1408 /* Get the inner hash of HMAC */
1409 CVMX_MF_HSH_IV(tmp1, 0);
1410 CVMX_MF_HSH_IV(tmp2, 1);
1411 tmp3 = 0;
1412 CVMX_MF_HSH_IV(tmp3, 2);
1413
1414 /* Initialize hash unit */
1415 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1416 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1417 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1418
1419 CVMX_MT_HSH_DAT(tmp1, 0);
1420 CVMX_MT_HSH_DAT(tmp2, 1);
1421 tmp3 |= 0x0000000080000000;
1422 CVMX_MT_HSH_DAT(tmp3, 2);
1423 CVMX_MT_HSH_DATZ(3);
1424 CVMX_MT_HSH_DATZ(4);
1425 CVMX_MT_HSH_DATZ(5);
1426 CVMX_MT_HSH_DATZ(6);
1427 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1428 /* save the HMAC */
1429 SG_INIT(sg, data32, data_i, data_l);
1430 while (icv_off > 0) {
1431 SG_CONSUME(sg, data32, data_i, data_l);
1432 icv_off -= 4;
1433 }
1434 CVMX_MF_HSH_IV(tmp1, 0);
1435 *data32 = (uint32_t) (tmp1 >> 32);
1436 SG_CONSUME(sg, data32, data_i, data_l);
1437 *data32 = (uint32_t) tmp1;
1438 SG_CONSUME(sg, data32, data_i, data_l);
1439 CVMX_MF_HSH_IV(tmp1, 1);
1440 *data32 = (uint32_t) (tmp1 >> 32);
1441
1442 octeon_crypto_disable(&state, flags);
1443 return 0;
1444 }
1445
1446 /****************************************************************************/
1447 /* AES MD5 */
1448
1449 int
1450 octo_aes_cbc_md5_encrypt(
1451 struct octo_sess *od,
1452 struct scatterlist *sg, int sg_len,
1453 int auth_off, int auth_len,
1454 int crypt_off, int crypt_len,
1455 int icv_off, uint8_t *ivp)
1456 {
1457 register int next = 0;
1458 union {
1459 uint32_t data32[2];
1460 uint64_t data64[1];
1461 } mydata[2];
1462 uint64_t *pdata = &mydata[0].data64[0];
1463 uint64_t *data = &mydata[1].data64[0];
1464 uint32_t *data32;
1465 uint64_t tmp1, tmp2;
1466 int data_i, data_l, alen = auth_len;
1467 struct octeon_cop2_state state;
1468 unsigned long flags;
1469
1470 dprintk("%s()\n", __FUNCTION__);
1471
1472 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
1473 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1474 (crypt_len & 0x7) ||
1475 (auth_len & 0x7) ||
1476 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1477 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1478 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1479 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1480 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1481 return -EINVAL;
1482 }
1483
1484 SG_INIT(sg, data32, data_i, data_l);
1485
1486 CVMX_PREFETCH0(ivp);
1487 CVMX_PREFETCH0(od->octo_enckey);
1488
1489 flags = octeon_crypto_enable(&state);
1490
1491 /* load AES Key */
1492 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1493 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1494
1495 if (od->octo_encklen == 16) {
1496 CVMX_MT_AES_KEY(0x0, 2);
1497 CVMX_MT_AES_KEY(0x0, 3);
1498 } else if (od->octo_encklen == 24) {
1499 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1500 CVMX_MT_AES_KEY(0x0, 3);
1501 } else if (od->octo_encklen == 32) {
1502 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1503 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1504 } else {
1505 octeon_crypto_disable(&state, flags);
1506 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1507 return -EINVAL;
1508 }
1509 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1510
1511 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1512 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1513
1514 /* Load MD5 IV */
1515 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1516 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1517
1518 while (crypt_off > 0 && auth_off > 0) {
1519 SG_CONSUME(sg, data32, data_i, data_l);
1520 crypt_off -= 4;
1521 auth_off -= 4;
1522 }
1523
1524 /* align auth and crypt */
1525 while (crypt_off > 0 && auth_len > 0) {
1526 mydata[0].data32[0] = *data32;
1527 SG_CONSUME(sg, data32, data_i, data_l);
1528 mydata[0].data32[1] = *data32;
1529 SG_CONSUME(sg, data32, data_i, data_l);
1530 CVM_LOAD_MD5_UNIT(*pdata, next);
1531 crypt_off -= 8;
1532 auth_len -= 8;
1533 }
1534
1535 while (crypt_len > 0) {
1536 uint32_t *pdata32[3];
1537
1538 pdata32[0] = data32;
1539 mydata[0].data32[0] = *data32;
1540 SG_CONSUME(sg, data32, data_i, data_l);
1541
1542 pdata32[1] = data32;
1543 mydata[0].data32[1] = *data32;
1544 SG_CONSUME(sg, data32, data_i, data_l);
1545
1546 pdata32[2] = data32;
1547 mydata[1].data32[0] = *data32;
1548 SG_CONSUME(sg, data32, data_i, data_l);
1549
1550 mydata[1].data32[1] = *data32;
1551
1552 CVMX_MT_AES_ENC_CBC0(*pdata);
1553 CVMX_MT_AES_ENC_CBC1(*data);
1554 CVMX_MF_AES_RESULT(*pdata, 0);
1555 CVMX_MF_AES_RESULT(*data, 1);
1556 crypt_len -= 16;
1557
1558 if (auth_len > 0) {
1559 CVM_LOAD_MD5_UNIT(*pdata, next);
1560 auth_len -= 8;
1561 }
1562 if (auth_len > 0) {
1563 CVM_LOAD_MD5_UNIT(*data, next);
1564 auth_len -= 8;
1565 }
1566
1567 *pdata32[0] = mydata[0].data32[0];
1568 *pdata32[1] = mydata[0].data32[1];
1569 *pdata32[2] = mydata[1].data32[0];
1570 *data32 = mydata[1].data32[1];
1571
1572 SG_CONSUME(sg, data32, data_i, data_l);
1573 }
1574
1575 /* finish any left over hashing */
1576 while (auth_len > 0) {
1577 mydata[0].data32[0] = *data32;
1578 SG_CONSUME(sg, data32, data_i, data_l);
1579 mydata[0].data32[1] = *data32;
1580 SG_CONSUME(sg, data32, data_i, data_l);
1581 CVM_LOAD_MD5_UNIT(*pdata, next);
1582 auth_len -= 8;
1583 }
1584
1585 /* finish the hash */
1586 CVMX_PREFETCH0(od->octo_hmouter);
1587 #if 0
1588 if (unlikely(inplen)) {
1589 uint64_t tmp = 0;
1590 uint8_t *p = (uint8_t *) & tmp;
1591 p[inplen] = 0x80;
1592 do {
1593 inplen--;
1594 p[inplen] = ((uint8_t *) data)[inplen];
1595 } while (inplen);
1596 CVM_LOAD_MD5_UNIT(tmp, next);
1597 } else {
1598 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1599 }
1600 #else
1601 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1602 #endif
1603
1604 /* Finish Inner hash */
1605 while (next != 7) {
1606 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1607 }
1608 CVMX_ES64(tmp1, ((alen + 64) << 3));
1609 CVM_LOAD_MD5_UNIT(tmp1, next);
1610
1611 /* Get the inner hash of HMAC */
1612 CVMX_MF_HSH_IV(tmp1, 0);
1613 CVMX_MF_HSH_IV(tmp2, 1);
1614
1615 /* Initialize hash unit */
1616 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1617 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1618
1619 CVMX_MT_HSH_DAT(tmp1, 0);
1620 CVMX_MT_HSH_DAT(tmp2, 1);
1621 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1622 CVMX_MT_HSH_DATZ(3);
1623 CVMX_MT_HSH_DATZ(4);
1624 CVMX_MT_HSH_DATZ(5);
1625 CVMX_MT_HSH_DATZ(6);
1626 CVMX_ES64(tmp1, ((64 + 16) << 3));
1627 CVMX_MT_HSH_STARTMD5(tmp1);
1628
1629 /* save the HMAC */
1630 SG_INIT(sg, data32, data_i, data_l);
1631 while (icv_off > 0) {
1632 SG_CONSUME(sg, data32, data_i, data_l);
1633 icv_off -= 4;
1634 }
1635 CVMX_MF_HSH_IV(tmp1, 0);
1636 *data32 = (uint32_t) (tmp1 >> 32);
1637 SG_CONSUME(sg, data32, data_i, data_l);
1638 *data32 = (uint32_t) tmp1;
1639 SG_CONSUME(sg, data32, data_i, data_l);
1640 CVMX_MF_HSH_IV(tmp1, 1);
1641 *data32 = (uint32_t) (tmp1 >> 32);
1642
1643 octeon_crypto_disable(&state, flags);
1644 return 0;
1645 }
1646
1647 int
1648 octo_aes_cbc_md5_decrypt(
1649 struct octo_sess *od,
1650 struct scatterlist *sg, int sg_len,
1651 int auth_off, int auth_len,
1652 int crypt_off, int crypt_len,
1653 int icv_off, uint8_t *ivp)
1654 {
1655 register int next = 0;
1656 union {
1657 uint32_t data32[2];
1658 uint64_t data64[1];
1659 } mydata[2];
1660 uint64_t *pdata = &mydata[0].data64[0];
1661 uint64_t *data = &mydata[1].data64[0];
1662 uint32_t *data32;
1663 uint64_t tmp1, tmp2;
1664 int data_i, data_l, alen = auth_len;
1665 struct octeon_cop2_state state;
1666 unsigned long flags;
1667
1668 dprintk("%s()\n", __FUNCTION__);
1669
1670 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
1671 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1672 (crypt_len & 0x7) ||
1673 (auth_len & 0x7) ||
1674 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1675 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1676 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1677 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1678 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1679 return -EINVAL;
1680 }
1681
1682 SG_INIT(sg, data32, data_i, data_l);
1683
1684 CVMX_PREFETCH0(ivp);
1685 CVMX_PREFETCH0(od->octo_enckey);
1686
1687 flags = octeon_crypto_enable(&state);
1688
1689 /* load AES Key */
1690 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1691 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1692
1693 if (od->octo_encklen == 16) {
1694 CVMX_MT_AES_KEY(0x0, 2);
1695 CVMX_MT_AES_KEY(0x0, 3);
1696 } else if (od->octo_encklen == 24) {
1697 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1698 CVMX_MT_AES_KEY(0x0, 3);
1699 } else if (od->octo_encklen == 32) {
1700 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1701 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1702 } else {
1703 octeon_crypto_disable(&state, flags);
1704 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1705 return -EINVAL;
1706 }
1707 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1708
1709 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1710 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1711
1712 /* Load MD5 IV */
1713 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1714 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1715
1716 while (crypt_off > 0 && auth_off > 0) {
1717 SG_CONSUME(sg, data32, data_i, data_l);
1718 crypt_off -= 4;
1719 auth_off -= 4;
1720 }
1721
1722 /* align auth and crypt */
1723 while (crypt_off > 0 && auth_len > 0) {
1724 mydata[0].data32[0] = *data32;
1725 SG_CONSUME(sg, data32, data_i, data_l);
1726 mydata[0].data32[1] = *data32;
1727 SG_CONSUME(sg, data32, data_i, data_l);
1728 CVM_LOAD_MD5_UNIT(*pdata, next);
1729 crypt_off -= 8;
1730 auth_len -= 8;
1731 }
1732
1733 while (crypt_len > 0) {
1734 uint32_t *pdata32[3];
1735
1736 pdata32[0] = data32;
1737 mydata[0].data32[0] = *data32;
1738 SG_CONSUME(sg, data32, data_i, data_l);
1739 pdata32[1] = data32;
1740 mydata[0].data32[1] = *data32;
1741 SG_CONSUME(sg, data32, data_i, data_l);
1742 pdata32[2] = data32;
1743 mydata[1].data32[0] = *data32;
1744 SG_CONSUME(sg, data32, data_i, data_l);
1745 mydata[1].data32[1] = *data32;
1746
1747 if (auth_len > 0) {
1748 CVM_LOAD_MD5_UNIT(*pdata, next);
1749 auth_len -= 8;
1750 }
1751
1752 if (auth_len > 0) {
1753 CVM_LOAD_MD5_UNIT(*data, next);
1754 auth_len -= 8;
1755 }
1756
1757 CVMX_MT_AES_DEC_CBC0(*pdata);
1758 CVMX_MT_AES_DEC_CBC1(*data);
1759 CVMX_MF_AES_RESULT(*pdata, 0);
1760 CVMX_MF_AES_RESULT(*data, 1);
1761 crypt_len -= 16;
1762
1763 *pdata32[0] = mydata[0].data32[0];
1764 *pdata32[1] = mydata[0].data32[1];
1765 *pdata32[2] = mydata[1].data32[0];
1766 *data32 = mydata[1].data32[1];
1767
1768 SG_CONSUME(sg, data32, data_i, data_l);
1769 }
1770
1771 /* finish left over hash if any */
1772 while (auth_len > 0) {
1773 mydata[0].data32[0] = *data32;
1774 SG_CONSUME(sg, data32, data_i, data_l);
1775 mydata[0].data32[1] = *data32;
1776 SG_CONSUME(sg, data32, data_i, data_l);
1777 CVM_LOAD_MD5_UNIT(*pdata, next);
1778 auth_len -= 8;
1779 }
1780
1781
1782 /* finish the hash */
1783 CVMX_PREFETCH0(od->octo_hmouter);
1784 #if 0
1785 if (unlikely(inplen)) {
1786 uint64_t tmp = 0;
1787 uint8_t *p = (uint8_t *) & tmp;
1788 p[inplen] = 0x80;
1789 do {
1790 inplen--;
1791 p[inplen] = ((uint8_t *) data)[inplen];
1792 } while (inplen);
1793 CVM_LOAD_MD5_UNIT(tmp, next);
1794 } else {
1795 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1796 }
1797 #else
1798 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1799 #endif
1800
1801 /* Finish Inner hash */
1802 while (next != 7) {
1803 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1804 }
1805 CVMX_ES64(tmp1, ((alen + 64) << 3));
1806 CVM_LOAD_MD5_UNIT(tmp1, next);
1807
1808 /* Get the inner hash of HMAC */
1809 CVMX_MF_HSH_IV(tmp1, 0);
1810 CVMX_MF_HSH_IV(tmp2, 1);
1811
1812 /* Initialize hash unit */
1813 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1814 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1815
1816 CVMX_MT_HSH_DAT(tmp1, 0);
1817 CVMX_MT_HSH_DAT(tmp2, 1);
1818 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1819 CVMX_MT_HSH_DATZ(3);
1820 CVMX_MT_HSH_DATZ(4);
1821 CVMX_MT_HSH_DATZ(5);
1822 CVMX_MT_HSH_DATZ(6);
1823 CVMX_ES64(tmp1, ((64 + 16) << 3));
1824 CVMX_MT_HSH_STARTMD5(tmp1);
1825
1826 /* save the HMAC */
1827 SG_INIT(sg, data32, data_i, data_l);
1828 while (icv_off > 0) {
1829 SG_CONSUME(sg, data32, data_i, data_l);
1830 icv_off -= 4;
1831 }
1832 CVMX_MF_HSH_IV(tmp1, 0);
1833 *data32 = (uint32_t) (tmp1 >> 32);
1834 SG_CONSUME(sg, data32, data_i, data_l);
1835 *data32 = (uint32_t) tmp1;
1836 SG_CONSUME(sg, data32, data_i, data_l);
1837 CVMX_MF_HSH_IV(tmp1, 1);
1838 *data32 = (uint32_t) (tmp1 >> 32);
1839
1840 octeon_crypto_disable(&state, flags);
1841 return 0;
1842 }
1843
1844 /****************************************************************************/
1845 /* AES SHA1 */
1846
1847 int
1848 octo_aes_cbc_sha1_encrypt(
1849 struct octo_sess *od,
1850 struct scatterlist *sg, int sg_len,
1851 int auth_off, int auth_len,
1852 int crypt_off, int crypt_len,
1853 int icv_off, uint8_t *ivp)
1854 {
1855 register int next = 0;
1856 union {
1857 uint32_t data32[2];
1858 uint64_t data64[1];
1859 } mydata[2];
1860 uint64_t *pdata = &mydata[0].data64[0];
1861 uint64_t *data = &mydata[1].data64[0];
1862 uint32_t *data32;
1863 uint64_t tmp1, tmp2, tmp3;
1864 int data_i, data_l, alen = auth_len;
1865 struct octeon_cop2_state state;
1866 unsigned long flags;
1867
1868 dprintk("%s(a_off=%d a_len=%d c_off=%d c_len=%d icv_off=%d)\n",
1869 __FUNCTION__, auth_off, auth_len, crypt_off, crypt_len, icv_off);
1870
1871 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
1872 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
1873 (crypt_len & 0x7) ||
1874 (auth_len & 0x7) ||
1875 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
1876 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
1877 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1878 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
1879 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1880 return -EINVAL;
1881 }
1882
1883 SG_INIT(sg, data32, data_i, data_l);
1884
1885 CVMX_PREFETCH0(ivp);
1886 CVMX_PREFETCH0(od->octo_enckey);
1887
1888 flags = octeon_crypto_enable(&state);
1889
1890 /* load AES Key */
1891 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1892 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1893
1894 if (od->octo_encklen == 16) {
1895 CVMX_MT_AES_KEY(0x0, 2);
1896 CVMX_MT_AES_KEY(0x0, 3);
1897 } else if (od->octo_encklen == 24) {
1898 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1899 CVMX_MT_AES_KEY(0x0, 3);
1900 } else if (od->octo_encklen == 32) {
1901 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1902 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1903 } else {
1904 octeon_crypto_disable(&state, flags);
1905 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
1906 return -EINVAL;
1907 }
1908 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1909
1910 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1911 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1912
1913 /* Load SHA IV */
1914 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1915 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1916 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1917
1918 while (crypt_off > 0 && auth_off > 0) {
1919 SG_CONSUME(sg, data32, data_i, data_l);
1920 crypt_off -= 4;
1921 auth_off -= 4;
1922 }
1923
1924 /* align auth and crypt */
1925 while (crypt_off > 0 && auth_len > 0) {
1926 mydata[0].data32[0] = *data32;
1927 SG_CONSUME(sg, data32, data_i, data_l);
1928 mydata[0].data32[1] = *data32;
1929 SG_CONSUME(sg, data32, data_i, data_l);
1930 CVM_LOAD_SHA_UNIT(*pdata, next);
1931 crypt_off -= 8;
1932 auth_len -= 8;
1933 }
1934
1935 while (crypt_len > 0) {
1936 uint32_t *pdata32[3];
1937
1938 pdata32[0] = data32;
1939 mydata[0].data32[0] = *data32;
1940 SG_CONSUME(sg, data32, data_i, data_l);
1941 pdata32[1] = data32;
1942 mydata[0].data32[1] = *data32;
1943 SG_CONSUME(sg, data32, data_i, data_l);
1944 pdata32[2] = data32;
1945 mydata[1].data32[0] = *data32;
1946 SG_CONSUME(sg, data32, data_i, data_l);
1947 mydata[1].data32[1] = *data32;
1948
1949 CVMX_MT_AES_ENC_CBC0(*pdata);
1950 CVMX_MT_AES_ENC_CBC1(*data);
1951 CVMX_MF_AES_RESULT(*pdata, 0);
1952 CVMX_MF_AES_RESULT(*data, 1);
1953 crypt_len -= 16;
1954
1955 if (auth_len > 0) {
1956 CVM_LOAD_SHA_UNIT(*pdata, next);
1957 auth_len -= 8;
1958 }
1959 if (auth_len > 0) {
1960 CVM_LOAD_SHA_UNIT(*data, next);
1961 auth_len -= 8;
1962 }
1963
1964 *pdata32[0] = mydata[0].data32[0];
1965 *pdata32[1] = mydata[0].data32[1];
1966 *pdata32[2] = mydata[1].data32[0];
1967 *data32 = mydata[1].data32[1];
1968
1969 SG_CONSUME(sg, data32, data_i, data_l);
1970 }
1971
1972 /* finish and hashing */
1973 while (auth_len > 0) {
1974 mydata[0].data32[0] = *data32;
1975 SG_CONSUME(sg, data32, data_i, data_l);
1976 mydata[0].data32[1] = *data32;
1977 SG_CONSUME(sg, data32, data_i, data_l);
1978 CVM_LOAD_SHA_UNIT(*pdata, next);
1979 auth_len -= 8;
1980 }
1981
1982 /* finish the hash */
1983 CVMX_PREFETCH0(od->octo_hmouter);
1984 #if 0
1985 if (unlikely(inplen)) {
1986 uint64_t tmp = 0;
1987 uint8_t *p = (uint8_t *) & tmp;
1988 p[inplen] = 0x80;
1989 do {
1990 inplen--;
1991 p[inplen] = ((uint8_t *) data)[inplen];
1992 } while (inplen);
1993 CVM_LOAD_SHA_UNIT(tmp, next);
1994 } else {
1995 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1996 }
1997 #else
1998 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1999 #endif
2000
2001 /* Finish Inner hash */
2002 while (next != 7) {
2003 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
2004 }
2005 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
2006
2007 /* Get the inner hash of HMAC */
2008 CVMX_MF_HSH_IV(tmp1, 0);
2009 CVMX_MF_HSH_IV(tmp2, 1);
2010 tmp3 = 0;
2011 CVMX_MF_HSH_IV(tmp3, 2);
2012
2013 /* Initialize hash unit */
2014 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
2015 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
2016 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
2017
2018 CVMX_MT_HSH_DAT(tmp1, 0);
2019 CVMX_MT_HSH_DAT(tmp2, 1);
2020 tmp3 |= 0x0000000080000000;
2021 CVMX_MT_HSH_DAT(tmp3, 2);
2022 CVMX_MT_HSH_DATZ(3);
2023 CVMX_MT_HSH_DATZ(4);
2024 CVMX_MT_HSH_DATZ(5);
2025 CVMX_MT_HSH_DATZ(6);
2026 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
2027
2028 /* finish the hash */
2029 CVMX_PREFETCH0(od->octo_hmouter);
2030 #if 0
2031 if (unlikely(inplen)) {
2032 uint64_t tmp = 0;
2033 uint8_t *p = (uint8_t *) & tmp;
2034 p[inplen] = 0x80;
2035 do {
2036 inplen--;
2037 p[inplen] = ((uint8_t *) data)[inplen];
2038 } while (inplen);
2039 CVM_LOAD_MD5_UNIT(tmp, next);
2040 } else {
2041 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2042 }
2043 #else
2044 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2045 #endif
2046
2047 /* save the HMAC */
2048 SG_INIT(sg, data32, data_i, data_l);
2049 while (icv_off > 0) {
2050 SG_CONSUME(sg, data32, data_i, data_l);
2051 icv_off -= 4;
2052 }
2053 CVMX_MF_HSH_IV(tmp1, 0);
2054 *data32 = (uint32_t) (tmp1 >> 32);
2055 SG_CONSUME(sg, data32, data_i, data_l);
2056 *data32 = (uint32_t) tmp1;
2057 SG_CONSUME(sg, data32, data_i, data_l);
2058 CVMX_MF_HSH_IV(tmp1, 1);
2059 *data32 = (uint32_t) (tmp1 >> 32);
2060
2061 octeon_crypto_disable(&state, flags);
2062 return 0;
2063 }
2064
2065 int
2066 octo_aes_cbc_sha1_decrypt(
2067 struct octo_sess *od,
2068 struct scatterlist *sg, int sg_len,
2069 int auth_off, int auth_len,
2070 int crypt_off, int crypt_len,
2071 int icv_off, uint8_t *ivp)
2072 {
2073 register int next = 0;
2074 union {
2075 uint32_t data32[2];
2076 uint64_t data64[1];
2077 } mydata[2];
2078 uint64_t *pdata = &mydata[0].data64[0];
2079 uint64_t *data = &mydata[1].data64[0];
2080 uint32_t *data32;
2081 uint64_t tmp1, tmp2, tmp3;
2082 int data_i, data_l, alen = auth_len;
2083 struct octeon_cop2_state state;
2084 unsigned long flags;
2085
2086 dprintk("%s(a_off=%d a_len=%d c_off=%d c_len=%d icv_off=%d)\n",
2087 __FUNCTION__, auth_off, auth_len, crypt_off, crypt_len, icv_off);
2088
2089 if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL ||
2090 (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) ||
2091 (crypt_len & 0x7) ||
2092 (auth_len & 0x7) ||
2093 (auth_off & 0x3) || (auth_off + auth_len > sg_len))) {
2094 dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d "
2095 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
2096 "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len,
2097 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
2098 return -EINVAL;
2099 }
2100
2101 SG_INIT(sg, data32, data_i, data_l);
2102
2103 CVMX_PREFETCH0(ivp);
2104 CVMX_PREFETCH0(od->octo_enckey);
2105
2106 flags = octeon_crypto_enable(&state);
2107
2108 /* load AES Key */
2109 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
2110 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
2111
2112 if (od->octo_encklen == 16) {
2113 CVMX_MT_AES_KEY(0x0, 2);
2114 CVMX_MT_AES_KEY(0x0, 3);
2115 } else if (od->octo_encklen == 24) {
2116 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
2117 CVMX_MT_AES_KEY(0x0, 3);
2118 } else if (od->octo_encklen == 32) {
2119 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
2120 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
2121 } else {
2122 octeon_crypto_disable(&state, flags);
2123 dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen);
2124 return -EINVAL;
2125 }
2126 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
2127
2128 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
2129 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
2130
2131 /* Load SHA1 IV */
2132 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
2133 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
2134 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
2135
2136 while (crypt_off > 0 && auth_off > 0) {
2137 SG_CONSUME(sg, data32, data_i, data_l);
2138 crypt_off -= 4;
2139 auth_off -= 4;
2140 }
2141
2142 /* align auth and crypt */
2143 while (crypt_off > 0 && auth_len > 0) {
2144 mydata[0].data32[0] = *data32;
2145 SG_CONSUME(sg, data32, data_i, data_l);
2146 mydata[0].data32[1] = *data32;
2147 SG_CONSUME(sg, data32, data_i, data_l);
2148 CVM_LOAD_SHA_UNIT(*pdata, next);
2149 crypt_off -= 8;
2150 auth_len -= 8;
2151 }
2152
2153 while (crypt_len > 0) {
2154 uint32_t *pdata32[3];
2155
2156 pdata32[0] = data32;
2157 mydata[0].data32[0] = *data32;
2158 SG_CONSUME(sg, data32, data_i, data_l);
2159 pdata32[1] = data32;
2160 mydata[0].data32[1] = *data32;
2161 SG_CONSUME(sg, data32, data_i, data_l);
2162 pdata32[2] = data32;
2163 mydata[1].data32[0] = *data32;
2164 SG_CONSUME(sg, data32, data_i, data_l);
2165 mydata[1].data32[1] = *data32;
2166
2167 if (auth_len > 0) {
2168 CVM_LOAD_SHA_UNIT(*pdata, next);
2169 auth_len -= 8;
2170 }
2171 if (auth_len > 0) {
2172 CVM_LOAD_SHA_UNIT(*data, next);
2173 auth_len -= 8;
2174 }
2175
2176 CVMX_MT_AES_DEC_CBC0(*pdata);
2177 CVMX_MT_AES_DEC_CBC1(*data);
2178 CVMX_MF_AES_RESULT(*pdata, 0);
2179 CVMX_MF_AES_RESULT(*data, 1);
2180 crypt_len -= 16;
2181
2182 *pdata32[0] = mydata[0].data32[0];
2183 *pdata32[1] = mydata[0].data32[1];
2184 *pdata32[2] = mydata[1].data32[0];
2185 *data32 = mydata[1].data32[1];
2186
2187 SG_CONSUME(sg, data32, data_i, data_l);
2188 }
2189
2190 /* finish and leftover hashing */
2191 while (auth_len > 0) {
2192 mydata[0].data32[0] = *data32;
2193 SG_CONSUME(sg, data32, data_i, data_l);
2194 mydata[0].data32[1] = *data32;
2195 SG_CONSUME(sg, data32, data_i, data_l);
2196 CVM_LOAD_SHA_UNIT(*pdata, next);
2197 auth_len -= 8;
2198 }
2199
2200 /* finish the hash */
2201 CVMX_PREFETCH0(od->octo_hmouter);
2202 #if 0
2203 if (unlikely(inplen)) {
2204 uint64_t tmp = 0;
2205 uint8_t *p = (uint8_t *) & tmp;
2206 p[inplen] = 0x80;
2207 do {
2208 inplen--;
2209 p[inplen] = ((uint8_t *) data)[inplen];
2210 } while (inplen);
2211 CVM_LOAD_SHA_UNIT(tmp, next);
2212 } else {
2213 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2214 }
2215 #else
2216 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2217 #endif
2218
2219 /* Finish Inner hash */
2220 while (next != 7) {
2221 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
2222 }
2223 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
2224
2225 /* Get the inner hash of HMAC */
2226 CVMX_MF_HSH_IV(tmp1, 0);
2227 CVMX_MF_HSH_IV(tmp2, 1);
2228 tmp3 = 0;
2229 CVMX_MF_HSH_IV(tmp3, 2);
2230
2231 /* Initialize hash unit */
2232 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
2233 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
2234 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
2235
2236 CVMX_MT_HSH_DAT(tmp1, 0);
2237 CVMX_MT_HSH_DAT(tmp2, 1);
2238 tmp3 |= 0x0000000080000000;
2239 CVMX_MT_HSH_DAT(tmp3, 2);
2240 CVMX_MT_HSH_DATZ(3);
2241 CVMX_MT_HSH_DATZ(4);
2242 CVMX_MT_HSH_DATZ(5);
2243 CVMX_MT_HSH_DATZ(6);
2244 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
2245
2246 /* finish the hash */
2247 CVMX_PREFETCH0(od->octo_hmouter);
2248 #if 0
2249 if (unlikely(inplen)) {
2250 uint64_t tmp = 0;
2251 uint8_t *p = (uint8_t *) & tmp;
2252 p[inplen] = 0x80;
2253 do {
2254 inplen--;
2255 p[inplen] = ((uint8_t *) data)[inplen];
2256 } while (inplen);
2257 CVM_LOAD_MD5_UNIT(tmp, next);
2258 } else {
2259 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2260 }
2261 #else
2262 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2263 #endif
2264
2265 /* save the HMAC */
2266 SG_INIT(sg, data32, data_i, data_l);
2267 while (icv_off > 0) {
2268 SG_CONSUME(sg, data32, data_i, data_l);
2269 icv_off -= 4;
2270 }
2271 CVMX_MF_HSH_IV(tmp1, 0);
2272 *data32 = (uint32_t) (tmp1 >> 32);
2273 SG_CONSUME(sg, data32, data_i, data_l);
2274 *data32 = (uint32_t) tmp1;
2275 SG_CONSUME(sg, data32, data_i, data_l);
2276 CVMX_MF_HSH_IV(tmp1, 1);
2277 *data32 = (uint32_t) (tmp1 >> 32);
2278
2279 octeon_crypto_disable(&state, flags);
2280 return 0;
2281 }
2282
2283 /****************************************************************************/
OpenWrt Source Repository