projects / openwrt / openwrt.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
kernel: bump 4.14 to 4.14.136
[openwrt/openwrt.git] / target / linux / layerscape / patches-4.14 / 820-sec-support-layerscape.patch
1 From ba8e92b322a3763880fdc4d19e9c7085f5504be7 Mon Sep 17 00:00:00 2001
2 From: Biwen Li <biwen.li@nxp.com>
3 Date: Tue, 23 Apr 2019 17:41:43 +0800
4 Subject: [PATCH] sec: support layerscape
5 MIME-Version: 1.0
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
8
9 This is an integrated patch of sec for layerscape
10
11 Signed-off-by: Alex Porosanu <alexandru.porosanu@nxp.com>
12 Signed-off-by: Arnd Bergmann <arnd@arndb.de>
13 Signed-off-by: Biwen Li <biwen.li@nxp.com>
14 Signed-off-by: Carmen Iorga <carmen.iorga@nxp.com>
15 Signed-off-by: Cristian Stoica <cristian.stoica@nxp.com>
16 Signed-off-by: Guanhua Gao <guanhua.gao@nxp.com>
17 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
18 Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
19 Signed-off-by: Ioana Ciornei <ioana.ciornei@nxp.com>
20 Signed-off-by: Radu Alexe <radu.alexe@nxp.com>
21 Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com>
22 Signed-off-by: Yangbo Lu <yangbo.lu@nxp.com>
23 Signed-off-by: Zhao Qiang <qiang.zhao@nxp.com>
24 ---
25 crypto/Kconfig | 20 +
26 crypto/Makefile | 1 +
27 crypto/chacha20poly1305.c | 2 -
28 crypto/tcrypt.c | 27 +-
29 crypto/testmgr.c | 244 ++
30 crypto/testmgr.h | 219 ++
31 crypto/tls.c | 607 ++++
32 drivers/crypto/Makefile | 2 +-
33 drivers/crypto/caam/Kconfig | 85 +-
34 drivers/crypto/caam/Makefile | 26 +-
35 drivers/crypto/caam/caamalg.c | 468 +++-
36 drivers/crypto/caam/caamalg_desc.c | 903 +++++-
37 drivers/crypto/caam/caamalg_desc.h | 52 +-
38 drivers/crypto/caam/caamalg_qi.c | 1060 ++++++-
39 drivers/crypto/caam/caamalg_qi2.c | 5843 +++++++++++++++++++++++++++++++++++
40 drivers/crypto/caam/caamalg_qi2.h | 276 ++
41 drivers/crypto/caam/caamhash.c | 192 +-
42 drivers/crypto/caam/caamhash_desc.c | 108 +
43 drivers/crypto/caam/caamhash_desc.h | 49 +
44 drivers/crypto/caam/caampkc.c | 52 +-
45 drivers/crypto/caam/caamrng.c | 52 +-
46 drivers/crypto/caam/compat.h | 4 +
47 drivers/crypto/caam/ctrl.c | 194 +-
48 drivers/crypto/caam/desc.h | 89 +-
49 drivers/crypto/caam/desc_constr.h | 59 +-
50 drivers/crypto/caam/dpseci.c | 865 ++++++
51 drivers/crypto/caam/dpseci.h | 433 +++
52 drivers/crypto/caam/dpseci_cmd.h | 287 ++
53 drivers/crypto/caam/error.c | 81 +-
54 drivers/crypto/caam/error.h | 6 +-
55 drivers/crypto/caam/intern.h | 102 +-
56 drivers/crypto/caam/jr.c | 84 +
57 drivers/crypto/caam/jr.h | 2 +
58 drivers/crypto/caam/key_gen.c | 30 -
59 drivers/crypto/caam/key_gen.h | 30 +
60 drivers/crypto/caam/qi.c | 134 +-
61 drivers/crypto/caam/qi.h | 2 +-
62 drivers/crypto/caam/regs.h | 76 +-
63 drivers/crypto/caam/sg_sw_qm.h | 46 +-
64 drivers/crypto/talitos.c | 8 +
65 include/crypto/chacha20.h | 1 +
66 41 files changed, 12088 insertions(+), 733 deletions(-)
67 create mode 100644 crypto/tls.c
68 create mode 100644 drivers/crypto/caam/caamalg_qi2.c
69 create mode 100644 drivers/crypto/caam/caamalg_qi2.h
70 create mode 100644 drivers/crypto/caam/caamhash_desc.c
71 create mode 100644 drivers/crypto/caam/caamhash_desc.h
72 create mode 100644 drivers/crypto/caam/dpseci.c
73 create mode 100644 drivers/crypto/caam/dpseci.h
74 create mode 100644 drivers/crypto/caam/dpseci_cmd.h
75
76 --- a/crypto/Kconfig
77 +++ b/crypto/Kconfig
78 @@ -312,6 +312,26 @@ config CRYPTO_ECHAINIV
79 a sequence number xored with a salt. This is the default
80 algorithm for CBC.
81
82 +config CRYPTO_TLS
83 + tristate "TLS support"
84 + select CRYPTO_AEAD
85 + select CRYPTO_BLKCIPHER
86 + select CRYPTO_MANAGER
87 + select CRYPTO_HASH
88 + select CRYPTO_NULL
89 + select CRYPTO_AUTHENC
90 + help
91 + Support for TLS 1.0 record encryption and decryption
92 +
93 + This module adds support for encryption/decryption of TLS 1.0 frames
94 + using blockcipher algorithms. The name of the resulting algorithm is
95 + "tls10(hmac(<digest>),cbc(<cipher>))". By default, the generic base
96 + algorithms are used (e.g. aes-generic, sha1-generic), but hardware
97 + accelerated versions will be used automatically if available.
98 +
99 + User-space applications (OpenSSL, GnuTLS) can offload TLS 1.0
100 + operations through AF_ALG or cryptodev interfaces
101 +
102 comment "Block modes"
103
104 config CRYPTO_CBC
105 --- a/crypto/Makefile
106 +++ b/crypto/Makefile
107 @@ -118,6 +118,7 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_ge
108 obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
109 obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o
110 obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
111 +obj-$(CONFIG_CRYPTO_TLS) += tls.o
112 obj-$(CONFIG_CRYPTO_LZO) += lzo.o
113 obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
114 obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
115 --- a/crypto/chacha20poly1305.c
116 +++ b/crypto/chacha20poly1305.c
117 @@ -22,8 +22,6 @@
118
119 #include "internal.h"
120
121 -#define CHACHAPOLY_IV_SIZE 12
122 -
123 struct chachapoly_instance_ctx {
124 struct crypto_skcipher_spawn chacha;
125 struct crypto_ahash_spawn poly;
126 --- a/crypto/tcrypt.c
127 +++ b/crypto/tcrypt.c
128 @@ -76,7 +76,7 @@ static char *check[] = {
129 "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt",
130 "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",
131 "lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512",
132 - NULL
133 + "rsa", NULL
134 };
135
136 struct tcrypt_result {
137 @@ -355,11 +355,13 @@ static void test_aead_speed(const char *
138 iv);
139 aead_request_set_ad(req, aad_size);
140
141 - if (secs)
142 + if (secs) {
143 ret = test_aead_jiffies(req, enc, *b_size,
144 secs);
145 - else
146 + cond_resched();
147 + } else {
148 ret = test_aead_cycles(req, enc, *b_size);
149 + }
150
151 if (ret) {
152 pr_err("%s() failed return code=%d\n", e, ret);
153 @@ -736,12 +738,14 @@ static void test_ahash_speed_common(cons
154
155 ahash_request_set_crypt(req, sg, output, speed[i].plen);
156
157 - if (secs)
158 + if (secs) {
159 ret = test_ahash_jiffies(req, speed[i].blen,
160 speed[i].plen, output, secs);
161 - else
162 + cond_resched();
163 + } else {
164 ret = test_ahash_cycles(req, speed[i].blen,
165 speed[i].plen, output);
166 + }
167
168 if (ret) {
169 pr_err("hashing failed ret=%d\n", ret);
170 @@ -959,12 +963,14 @@ static void test_skcipher_speed(const ch
171
172 skcipher_request_set_crypt(req, sg, sg, *b_size, iv);
173
174 - if (secs)
175 + if (secs) {
176 ret = test_acipher_jiffies(req, enc,
177 *b_size, secs);
178 - else
179 + cond_resched();
180 + } else {
181 ret = test_acipher_cycles(req, enc,
182 *b_size);
183 + }
184
185 if (ret) {
186 pr_err("%s() failed flags=%x\n", e,
187 @@ -1336,6 +1342,10 @@ static int do_test(const char *alg, u32
188 ret += tcrypt_test("hmac(sha3-512)");
189 break;
190
191 + case 115:
192 + ret += tcrypt_test("rsa");
193 + break;
194 +
195 case 150:
196 ret += tcrypt_test("ansi_cprng");
197 break;
198 @@ -1397,6 +1407,9 @@ static int do_test(const char *alg, u32
199 case 190:
200 ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))");
201 break;
202 + case 191:
203 + ret += tcrypt_test("tls10(hmac(sha1),cbc(aes))");
204 + break;
205 case 200:
206 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
207 speed_template_16_24_32);
208 --- a/crypto/testmgr.c
209 +++ b/crypto/testmgr.c
210 @@ -117,6 +117,13 @@ struct drbg_test_suite {
211 unsigned int count;
212 };
213
214 +struct tls_test_suite {
215 + struct {
216 + struct tls_testvec *vecs;
217 + unsigned int count;
218 + } enc, dec;
219 +};
220 +
221 struct akcipher_test_suite {
222 const struct akcipher_testvec *vecs;
223 unsigned int count;
224 @@ -140,6 +147,7 @@ struct alg_test_desc {
225 struct hash_test_suite hash;
226 struct cprng_test_suite cprng;
227 struct drbg_test_suite drbg;
228 + struct tls_test_suite tls;
229 struct akcipher_test_suite akcipher;
230 struct kpp_test_suite kpp;
231 } suite;
232 @@ -991,6 +999,233 @@ static int test_aead(struct crypto_aead
233 return 0;
234 }
235
236 +static int __test_tls(struct crypto_aead *tfm, int enc,
237 + struct tls_testvec *template, unsigned int tcount,
238 + const bool diff_dst)
239 +{
240 + const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm));
241 + unsigned int i, k, authsize;
242 + char *q;
243 + struct aead_request *req;
244 + struct scatterlist *sg;
245 + struct scatterlist *sgout;
246 + const char *e, *d;
247 + struct tcrypt_result result;
248 + void *input;
249 + void *output;
250 + void *assoc;
251 + char *iv;
252 + char *key;
253 + char *xbuf[XBUFSIZE];
254 + char *xoutbuf[XBUFSIZE];
255 + char *axbuf[XBUFSIZE];
256 + int ret = -ENOMEM;
257 +
258 + if (testmgr_alloc_buf(xbuf))
259 + goto out_noxbuf;
260 +
261 + if (diff_dst && testmgr_alloc_buf(xoutbuf))
262 + goto out_nooutbuf;
263 +
264 + if (testmgr_alloc_buf(axbuf))
265 + goto out_noaxbuf;
266 +
267 + iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
268 + if (!iv)
269 + goto out_noiv;
270 +
271 + key = kzalloc(MAX_KEYLEN, GFP_KERNEL);
272 + if (!key)
273 + goto out_nokey;
274 +
275 + sg = kmalloc(sizeof(*sg) * 8 * (diff_dst ? 2 : 1), GFP_KERNEL);
276 + if (!sg)
277 + goto out_nosg;
278 +
279 + sgout = sg + 8;
280 +
281 + d = diff_dst ? "-ddst" : "";
282 + e = enc ? "encryption" : "decryption";
283 +
284 + init_completion(&result.completion);
285 +
286 + req = aead_request_alloc(tfm, GFP_KERNEL);
287 + if (!req) {
288 + pr_err("alg: tls%s: Failed to allocate request for %s\n",
289 + d, algo);
290 + goto out;
291 + }
292 +
293 + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
294 + tcrypt_complete, &result);
295 +
296 + for (i = 0; i < tcount; i++) {
297 + input = xbuf[0];
298 + assoc = axbuf[0];
299 +
300 + ret = -EINVAL;
301 + if (WARN_ON(template[i].ilen > PAGE_SIZE ||
302 + template[i].alen > PAGE_SIZE))
303 + goto out;
304 +
305 + memcpy(assoc, template[i].assoc, template[i].alen);
306 + memcpy(input, template[i].input, template[i].ilen);
307 +
308 + if (template[i].iv)
309 + memcpy(iv, template[i].iv, MAX_IVLEN);
310 + else
311 + memset(iv, 0, MAX_IVLEN);
312 +
313 + crypto_aead_clear_flags(tfm, ~0);
314 +
315 + if (template[i].klen > MAX_KEYLEN) {
316 + pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
317 + d, i, algo, template[i].klen, MAX_KEYLEN);
318 + ret = -EINVAL;
319 + goto out;
320 + }
321 + memcpy(key, template[i].key, template[i].klen);
322 +
323 + ret = crypto_aead_setkey(tfm, key, template[i].klen);
324 + if (!ret == template[i].fail) {
325 + pr_err("alg: tls%s: setkey failed on test %d for %s: flags=%x\n",
326 + d, i, algo, crypto_aead_get_flags(tfm));
327 + goto out;
328 + } else if (ret)
329 + continue;
330 +
331 + authsize = 20;
332 + ret = crypto_aead_setauthsize(tfm, authsize);
333 + if (ret) {
334 + pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n",
335 + d, authsize, i, algo);
336 + goto out;
337 + }
338 +
339 + k = !!template[i].alen;
340 + sg_init_table(sg, k + 1);
341 + sg_set_buf(&sg[0], assoc, template[i].alen);
342 + sg_set_buf(&sg[k], input, (enc ? template[i].rlen :
343 + template[i].ilen));
344 + output = input;
345 +
346 + if (diff_dst) {
347 + sg_init_table(sgout, k + 1);
348 + sg_set_buf(&sgout[0], assoc, template[i].alen);
349 +
350 + output = xoutbuf[0];
351 + sg_set_buf(&sgout[k], output,
352 + (enc ? template[i].rlen : template[i].ilen));
353 + }
354 +
355 + aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
356 + template[i].ilen, iv);
357 +
358 + aead_request_set_ad(req, template[i].alen);
359 +
360 + ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
361 +
362 + switch (ret) {
363 + case 0:
364 + if (template[i].novrfy) {
365 + /* verification was supposed to fail */
366 + pr_err("alg: tls%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n",
367 + d, e, i, algo);
368 + /* so really, we got a bad message */
369 + ret = -EBADMSG;
370 + goto out;
371 + }
372 + break;
373 + case -EINPROGRESS:
374 + case -EBUSY:
375 + wait_for_completion(&result.completion);
376 + reinit_completion(&result.completion);
377 + ret = result.err;
378 + if (!ret)
379 + break;
380 + case -EBADMSG:
381 + /* verification failure was expected */
382 + if (template[i].novrfy)
383 + continue;
384 + /* fall through */
385 + default:
386 + pr_err("alg: tls%s: %s failed on test %d for %s: ret=%d\n",
387 + d, e, i, algo, -ret);
388 + goto out;
389 + }
390 +
391 + q = output;
392 + if (memcmp(q, template[i].result, template[i].rlen)) {
393 + pr_err("alg: tls%s: Test %d failed on %s for %s\n",
394 + d, i, e, algo);
395 + hexdump(q, template[i].rlen);
396 + pr_err("should be:\n");
397 + hexdump(template[i].result, template[i].rlen);
398 + ret = -EINVAL;
399 + goto out;
400 + }
401 + }
402 +
403 +out:
404 + aead_request_free(req);
405 +
406 + kfree(sg);
407 +out_nosg:
408 + kfree(key);
409 +out_nokey:
410 + kfree(iv);
411 +out_noiv:
412 + testmgr_free_buf(axbuf);
413 +out_noaxbuf:
414 + if (diff_dst)
415 + testmgr_free_buf(xoutbuf);
416 +out_nooutbuf:
417 + testmgr_free_buf(xbuf);
418 +out_noxbuf:
419 + return ret;
420 +}
421 +
422 +static int test_tls(struct crypto_aead *tfm, int enc,
423 + struct tls_testvec *template, unsigned int tcount)
424 +{
425 + int ret;
426 + /* test 'dst == src' case */
427 + ret = __test_tls(tfm, enc, template, tcount, false);
428 + if (ret)
429 + return ret;
430 + /* test 'dst != src' case */
431 + return __test_tls(tfm, enc, template, tcount, true);
432 +}
433 +
434 +static int alg_test_tls(const struct alg_test_desc *desc, const char *driver,
435 + u32 type, u32 mask)
436 +{
437 + struct crypto_aead *tfm;
438 + int err = 0;
439 +
440 + tfm = crypto_alloc_aead(driver, type, mask);
441 + if (IS_ERR(tfm)) {
442 + pr_err("alg: aead: Failed to load transform for %s: %ld\n",
443 + driver, PTR_ERR(tfm));
444 + return PTR_ERR(tfm);
445 + }
446 +
447 + if (desc->suite.tls.enc.vecs) {
448 + err = test_tls(tfm, ENCRYPT, desc->suite.tls.enc.vecs,
449 + desc->suite.tls.enc.count);
450 + if (err)
451 + goto out;
452 + }
453 +
454 + if (!err && desc->suite.tls.dec.vecs)
455 + err = test_tls(tfm, DECRYPT, desc->suite.tls.dec.vecs,
456 + desc->suite.tls.dec.count);
457 +
458 +out:
459 + crypto_free_aead(tfm);
460 + return err;
461 +}
462 +
463 static int test_cipher(struct crypto_cipher *tfm, int enc,
464 const struct cipher_testvec *template,
465 unsigned int tcount)
466 @@ -3524,6 +3759,15 @@ static const struct alg_test_desc alg_te
467 .hash = __VECS(tgr192_tv_template)
468 }
469 }, {
470 + .alg = "tls10(hmac(sha1),cbc(aes))",
471 + .test = alg_test_tls,
472 + .suite = {
473 + .tls = {
474 + .enc = __VECS(tls_enc_tv_template),
475 + .dec = __VECS(tls_dec_tv_template)
476 + }
477 + }
478 + }, {
479 .alg = "vmac(aes)",
480 .test = alg_test_hash,
481 .suite = {
482 --- a/crypto/testmgr.h
483 +++ b/crypto/testmgr.h
484 @@ -125,6 +125,20 @@ struct drbg_testvec {
485 size_t expectedlen;
486 };
487
488 +struct tls_testvec {
489 + char *key; /* wrapped keys for encryption and authentication */
490 + char *iv; /* initialization vector */
491 + char *input; /* input data */
492 + char *assoc; /* associated data: seq num, type, version, input len */
493 + char *result; /* result data */
494 + unsigned char fail; /* the test failure is expected */
495 + unsigned char novrfy; /* dec verification failure expected */
496 + unsigned char klen; /* key length */
497 + unsigned short ilen; /* input data length */
498 + unsigned short alen; /* associated data length */
499 + unsigned short rlen; /* result length */
500 +};
501 +
502 struct akcipher_testvec {
503 const unsigned char *key;
504 const unsigned char *m;
505 @@ -153,6 +167,211 @@ struct kpp_testvec {
506 static const char zeroed_string[48];
507
508 /*
509 + * TLS1.0 synthetic test vectors
510 + */
511 +static struct tls_testvec tls_enc_tv_template[] = {
512 + {
513 +#ifdef __LITTLE_ENDIAN
514 + .key = "\x08\x00" /* rta length */
515 + "\x01\x00" /* rta type */
516 +#else
517 + .key = "\x00\x08" /* rta length */
518 + "\x00\x01" /* rta type */
519 +#endif
520 + "\x00\x00\x00\x10" /* enc key length */
521 + "authenticationkey20benckeyis16_bytes",
522 + .klen = 8 + 20 + 16,
523 + .iv = "iv0123456789abcd",
524 + .input = "Single block msg",
525 + .ilen = 16,
526 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
527 + "\x00\x03\x01\x00\x10",
528 + .alen = 13,
529 + .result = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1"
530 + "\x59\x79\x1e\x91\x5f\x52\x14\x9c"
531 + "\xc0\x75\xd8\x4c\x97\x0f\x07\x73"
532 + "\xdc\x89\x47\x49\x49\xcb\x30\x6b"
533 + "\x1b\x45\x23\xa1\xd0\x51\xcf\x02"
534 + "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61",
535 + .rlen = 16 + 20 + 12,
536 + }, {
537 +#ifdef __LITTLE_ENDIAN
538 + .key = "\x08\x00" /* rta length */
539 + "\x01\x00" /* rta type */
540 +#else
541 + .key = "\x00\x08" /* rta length */
542 + "\x00\x01" /* rta type */
543 +#endif
544 + "\x00\x00\x00\x10" /* enc key length */
545 + "authenticationkey20benckeyis16_bytes",
546 + .klen = 8 + 20 + 16,
547 + .iv = "iv0123456789abcd",
548 + .input = "",
549 + .ilen = 0,
550 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
551 + "\x00\x03\x01\x00\x00",
552 + .alen = 13,
553 + .result = "\x58\x2a\x11\xc\x86\x8e\x4b\x67"
554 + "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a"
555 + "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45"
556 + "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a",
557 + .rlen = 20 + 12,
558 + }, {
559 +#ifdef __LITTLE_ENDIAN
560 + .key = "\x08\x00" /* rta length */
561 + "\x01\x00" /* rta type */
562 +#else
563 + .key = "\x00\x08" /* rta length */
564 + "\x00\x01" /* rta type */
565 +#endif
566 + "\x00\x00\x00\x10" /* enc key length */
567 + "authenticationkey20benckeyis16_bytes",
568 + .klen = 8 + 20 + 16,
569 + .iv = "iv0123456789abcd",
570 + .input = "285 bytes plaintext285 bytes plaintext285 bytes"
571 + " plaintext285 bytes plaintext285 bytes plaintext285"
572 + " bytes plaintext285 bytes plaintext285 bytes"
573 + " plaintext285 bytes plaintext285 bytes plaintext285"
574 + " bytes plaintext285 bytes plaintext285 bytes"
575 + " plaintext285 bytes plaintext285 bytes plaintext285"
576 + " bytes plaintext285 bytes plaintext",
577 + .ilen = 285,
578 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
579 + "\x00\x03\x01\x01\x1d",
580 + .alen = 13,
581 + .result = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd"
582 + "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90"
583 + "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a"
584 + "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94"
585 + "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51"
586 + "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31"
587 + "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8"
588 + "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50"
589 + "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac"
590 + "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14"
591 + "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d"
592 + "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8"
593 + "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c"
594 + "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74"
595 + "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e"
596 + "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b"
597 + "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d"
598 + "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13"
599 + "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d"
600 + "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5"
601 + "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2"
602 + "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20"
603 + "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1"
604 + "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e"
605 + "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7"
606 + "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e"
607 + "\x2a\x9e\x26\xf1\x3d\x21\xac\x65",
608 + .rlen = 285 + 20 + 15,
609 + }
610 +};
611 +
612 +static struct tls_testvec tls_dec_tv_template[] = {
613 + {
614 +#ifdef __LITTLE_ENDIAN
615 + .key = "\x08\x00" /* rta length */
616 + "\x01\x00" /* rta type */
617 +#else
618 + .key = "\x00\x08" /* rta length */
619 + "\x00\x01" /* rta type */
620 +#endif
621 + "\x00\x00\x00\x10" /* enc key length */
622 + "authenticationkey20benckeyis16_bytes",
623 + .klen = 8 + 20 + 16,
624 + .iv = "iv0123456789abcd",
625 + .input = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1"
626 + "\x59\x79\x1e\x91\x5f\x52\x14\x9c"
627 + "\xc0\x75\xd8\x4c\x97\x0f\x07\x73"
628 + "\xdc\x89\x47\x49\x49\xcb\x30\x6b"
629 + "\x1b\x45\x23\xa1\xd0\x51\xcf\x02"
630 + "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61",
631 + .ilen = 16 + 20 + 12,
632 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
633 + "\x00\x03\x01\x00\x30",
634 + .alen = 13,
635 + .result = "Single block msg",
636 + .rlen = 16,
637 + }, {
638 +#ifdef __LITTLE_ENDIAN
639 + .key = "\x08\x00" /* rta length */
640 + "\x01\x00" /* rta type */
641 +#else
642 + .key = "\x00\x08" /* rta length */
643 + "\x00\x01" /* rta type */
644 +#endif
645 + "\x00\x00\x00\x10" /* enc key length */
646 + "authenticationkey20benckeyis16_bytes",
647 + .klen = 8 + 20 + 16,
648 + .iv = "iv0123456789abcd",
649 + .input = "\x58\x2a\x11\xc\x86\x8e\x4b\x67"
650 + "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a"
651 + "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45"
652 + "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a",
653 + .ilen = 20 + 12,
654 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
655 + "\x00\x03\x01\x00\x20",
656 + .alen = 13,
657 + .result = "",
658 + .rlen = 0,
659 + }, {
660 +#ifdef __LITTLE_ENDIAN
661 + .key = "\x08\x00" /* rta length */
662 + "\x01\x00" /* rta type */
663 +#else
664 + .key = "\x00\x08" /* rta length */
665 + "\x00\x01" /* rta type */
666 +#endif
667 + "\x00\x00\x00\x10" /* enc key length */
668 + "authenticationkey20benckeyis16_bytes",
669 + .klen = 8 + 20 + 16,
670 + .iv = "iv0123456789abcd",
671 + .input = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd"
672 + "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90"
673 + "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a"
674 + "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94"
675 + "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51"
676 + "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31"
677 + "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8"
678 + "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50"
679 + "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac"
680 + "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14"
681 + "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d"
682 + "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8"
683 + "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c"
684 + "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74"
685 + "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e"
686 + "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b"
687 + "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d"
688 + "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13"
689 + "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d"
690 + "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5"
691 + "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2"
692 + "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20"
693 + "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1"
694 + "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e"
695 + "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7"
696 + "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e"
697 + "\x2a\x9e\x26\xf1\x3d\x21\xac\x65",
698 +
699 + .ilen = 285 + 20 + 15,
700 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07"
701 + "\x00\x03\x01\x01\x40",
702 + .alen = 13,
703 + .result = "285 bytes plaintext285 bytes plaintext285 bytes"
704 + " plaintext285 bytes plaintext285 bytes plaintext285"
705 + " bytes plaintext285 bytes plaintext285 bytes"
706 + " plaintext285 bytes plaintext285 bytes plaintext285"
707 + " bytes plaintext285 bytes plaintext285 bytes"
708 + " plaintext285 bytes plaintext285 bytes plaintext",
709 + .rlen = 285,
710 + }
711 +};
712 +
713 +/*
714 * RSA test vectors. Borrowed from openSSL.
715 */
716 static const struct akcipher_testvec rsa_tv_template[] = {
717 --- /dev/null
718 +++ b/crypto/tls.c
719 @@ -0,0 +1,607 @@
720 +/*
721 + * Copyright 2013 Freescale Semiconductor, Inc.
722 + * Copyright 2017 NXP Semiconductor, Inc.
723 + *
724 + * This program is free software; you can redistribute it and/or modify it
725 + * under the terms of the GNU General Public License as published by the Free
726 + * Software Foundation; either version 2 of the License, or (at your option)
727 + * any later version.
728 + *
729 + */
730 +
731 +#include <crypto/internal/aead.h>
732 +#include <crypto/internal/hash.h>
733 +#include <crypto/internal/skcipher.h>
734 +#include <crypto/authenc.h>
735 +#include <crypto/null.h>
736 +#include <crypto/scatterwalk.h>
737 +#include <linux/err.h>
738 +#include <linux/init.h>
739 +#include <linux/module.h>
740 +#include <linux/rtnetlink.h>
741 +
742 +struct tls_instance_ctx {
743 + struct crypto_ahash_spawn auth;
744 + struct crypto_skcipher_spawn enc;
745 +};
746 +
747 +struct crypto_tls_ctx {
748 + unsigned int reqoff;
749 + struct crypto_ahash *auth;
750 + struct crypto_skcipher *enc;
751 + struct crypto_skcipher *null;
752 +};
753 +
754 +struct tls_request_ctx {
755 + /*
756 + * cryptlen holds the payload length in the case of encryption or
757 + * payload_len + icv_len + padding_len in case of decryption
758 + */
759 + unsigned int cryptlen;
760 + /* working space for partial results */
761 + struct scatterlist tmp[2];
762 + struct scatterlist cipher[2];
763 + struct scatterlist dst[2];
764 + char tail[];
765 +};
766 +
767 +struct async_op {
768 + struct completion completion;
769 + int err;
770 +};
771 +
772 +static void tls_async_op_done(struct crypto_async_request *req, int err)
773 +{
774 + struct async_op *areq = req->data;
775 +
776 + if (err == -EINPROGRESS)
777 + return;
778 +
779 + areq->err = err;
780 + complete(&areq->completion);
781 +}
782 +
783 +static int crypto_tls_setkey(struct crypto_aead *tls, const u8 *key,
784 + unsigned int keylen)
785 +{
786 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls);
787 + struct crypto_ahash *auth = ctx->auth;
788 + struct crypto_skcipher *enc = ctx->enc;
789 + struct crypto_authenc_keys keys;
790 + int err = -EINVAL;
791 +
792 + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
793 + goto badkey;
794 +
795 + crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
796 + crypto_ahash_set_flags(auth, crypto_aead_get_flags(tls) &
797 + CRYPTO_TFM_REQ_MASK);
798 + err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
799 + crypto_aead_set_flags(tls, crypto_ahash_get_flags(auth) &
800 + CRYPTO_TFM_RES_MASK);
801 +
802 + if (err)
803 + goto out;
804 +
805 + crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
806 + crypto_skcipher_set_flags(enc, crypto_aead_get_flags(tls) &
807 + CRYPTO_TFM_REQ_MASK);
808 + err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
809 + crypto_aead_set_flags(tls, crypto_skcipher_get_flags(enc) &
810 + CRYPTO_TFM_RES_MASK);
811 +
812 +out:
813 + return err;
814 +
815 +badkey:
816 + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN);
817 + goto out;
818 +}
819 +
820 +/**
821 + * crypto_tls_genicv - Calculate hmac digest for a TLS record
822 + * @hash: (output) buffer to save the digest into
823 + * @src: (input) scatterlist with the assoc and payload data
824 + * @srclen: (input) size of the source buffer (assoclen + cryptlen)
825 + * @req: (input) aead request
826 + **/
827 +static int crypto_tls_genicv(u8 *hash, struct scatterlist *src,
828 + unsigned int srclen, struct aead_request *req)
829 +{
830 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
831 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls);
832 + struct tls_request_ctx *treq_ctx = aead_request_ctx(req);
833 + struct async_op ahash_op;
834 + struct ahash_request *ahreq = (void *)(treq_ctx->tail + ctx->reqoff);
835 + unsigned int flags = CRYPTO_TFM_REQ_MAY_SLEEP;
836 + int err = -EBADMSG;
837 +
838 + /* Bail out if the request assoc len is 0 */
839 + if (!req->assoclen)
840 + return err;
841 +
842 + init_completion(&ahash_op.completion);
843 +
844 + /* the hash transform to be executed comes from the original request */
845 + ahash_request_set_tfm(ahreq, ctx->auth);
846 + /* prepare the hash request with input data and result pointer */
847 + ahash_request_set_crypt(ahreq, src, hash, srclen);
848 + /* set the notifier for when the async hash function returns */
849 + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
850 + tls_async_op_done, &ahash_op);
851 +
852 + /* Calculate the digest on the given data. The result is put in hash */
853 + err = crypto_ahash_digest(ahreq);
854 + if (err == -EINPROGRESS) {
855 + err = wait_for_completion_interruptible(&ahash_op.completion);
856 + if (!err)
857 + err = ahash_op.err;
858 + }
859 +
860 + return err;
861 +}
862 +
863 +/**
864 + * crypto_tls_gen_padicv - Calculate and pad hmac digest for a TLS record
865 + * @hash: (output) buffer to save the digest and padding into
866 + * @phashlen: (output) the size of digest + padding
867 + * @req: (input) aead request
868 + **/
869 +static int crypto_tls_gen_padicv(u8 *hash, unsigned int *phashlen,
870 + struct aead_request *req)
871 +{
872 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
873 + unsigned int hash_size = crypto_aead_authsize(tls);
874 + unsigned int block_size = crypto_aead_blocksize(tls);
875 + unsigned int srclen = req->cryptlen + hash_size;
876 + unsigned int icvlen = req->cryptlen + req->assoclen;
877 + unsigned int padlen;
878 + int err;
879 +
880 + err = crypto_tls_genicv(hash, req->src, icvlen, req);
881 + if (err)
882 + goto out;
883 +
884 + /* add padding after digest */
885 + padlen = block_size - (srclen % block_size);
886 + memset(hash + hash_size, padlen - 1, padlen);
887 +
888 + *phashlen = hash_size + padlen;
889 +out:
890 + return err;
891 +}
892 +
893 +static int crypto_tls_copy_data(struct aead_request *req,
894 + struct scatterlist *src,
895 + struct scatterlist *dst,
896 + unsigned int len)
897 +{
898 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
899 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls);
900 + SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
901 +
902 + skcipher_request_set_tfm(skreq, ctx->null);
903 + skcipher_request_set_callback(skreq, aead_request_flags(req),
904 + NULL, NULL);
905 + skcipher_request_set_crypt(skreq, src, dst, len, NULL);
906 +
907 + return crypto_skcipher_encrypt(skreq);
908 +}
909 +
910 +static int crypto_tls_encrypt(struct aead_request *req)
911 +{
912 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
913 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls);
914 + struct tls_request_ctx *treq_ctx = aead_request_ctx(req);
915 + struct skcipher_request *skreq;
916 + struct scatterlist *cipher = treq_ctx->cipher;
917 + struct scatterlist *tmp = treq_ctx->tmp;
918 + struct scatterlist *sg, *src, *dst;
919 + unsigned int cryptlen, phashlen;
920 + u8 *hash = treq_ctx->tail;
921 + int err;
922 +
923 + /*
924 + * The hash result is saved at the beginning of the tls request ctx
925 + * and is aligned as required by the hash transform. Enough space was
926 + * allocated in crypto_tls_init_tfm to accommodate the difference. The
927 + * requests themselves start later at treq_ctx->tail + ctx->reqoff so
928 + * the result is not overwritten by the second (cipher) request.
929 + */
930 + hash = (u8 *)ALIGN((unsigned long)hash +
931 + crypto_ahash_alignmask(ctx->auth),
932 + crypto_ahash_alignmask(ctx->auth) + 1);
933 +
934 + /*
935 + * STEP 1: create ICV together with necessary padding
936 + */
937 + err = crypto_tls_gen_padicv(hash, &phashlen, req);
938 + if (err)
939 + return err;
940 +
941 + /*
942 + * STEP 2: Hash and padding are combined with the payload
943 + * depending on the form it arrives. Scatter tables must have at least
944 + * one page of data before chaining with another table and can't have
945 + * an empty data page. The following code addresses these requirements.
946 + *
947 + * If the payload is empty, only the hash is encrypted, otherwise the
948 + * payload scatterlist is merged with the hash. A special merging case
949 + * is when the payload has only one page of data. In that case the
950 + * payload page is moved to another scatterlist and prepared there for
951 + * encryption.
952 + */
953 + if (req->cryptlen) {
954 + src = scatterwalk_ffwd(tmp, req->src, req->assoclen);
955 +
956 + sg_init_table(cipher, 2);
957 + sg_set_buf(cipher + 1, hash, phashlen);
958 +
959 + if (sg_is_last(src)) {
960 + sg_set_page(cipher, sg_page(src), req->cryptlen,
961 + src->offset);
962 + src = cipher;
963 + } else {
964 + unsigned int rem_len = req->cryptlen;
965 +
966 + for (sg = src; rem_len > sg->length; sg = sg_next(sg))
967 + rem_len -= min(rem_len, sg->length);
968 +
969 + sg_set_page(cipher, sg_page(sg), rem_len, sg->offset);
970 + sg_chain(sg, 1, cipher);
971 + }
972 + } else {
973 + sg_init_one(cipher, hash, phashlen);
974 + src = cipher;
975 + }
976 +
977 + /**
978 + * If src != dst copy the associated data from source to destination.
979 + * In both cases fast-forward passed the associated data in the dest.
980 + */
981 + if (req->src != req->dst) {
982 + err = crypto_tls_copy_data(req, req->src, req->dst,
983 + req->assoclen);
984 + if (err)
985 + return err;
986 + }
987 + dst = scatterwalk_ffwd(treq_ctx->dst, req->dst, req->assoclen);
988 +
989 + /*
990 + * STEP 3: encrypt the frame and return the result
991 + */
992 + cryptlen = req->cryptlen + phashlen;
993 +
994 + /*
995 + * The hash and the cipher are applied at different times and their
996 + * requests can use the same memory space without interference
997 + */
998 + skreq = (void *)(treq_ctx->tail + ctx->reqoff);
999 + skcipher_request_set_tfm(skreq, ctx->enc);
1000 + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
1001 + skcipher_request_set_callback(skreq, aead_request_flags(req),
1002 + req->base.complete, req->base.data);
1003 + /*
1004 + * Apply the cipher transform. The result will be in req->dst when the
1005 + * asynchronuous call terminates
1006 + */
1007 + err = crypto_skcipher_encrypt(skreq);
1008 +
1009 + return err;
1010 +}
1011 +
1012 +static int crypto_tls_decrypt(struct aead_request *req)
1013 +{
1014 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
1015 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls);
1016 + struct tls_request_ctx *treq_ctx = aead_request_ctx(req);
1017 + unsigned int cryptlen = req->cryptlen;
1018 + unsigned int hash_size = crypto_aead_authsize(tls);
1019 + unsigned int block_size = crypto_aead_blocksize(tls);
1020 + struct skcipher_request *skreq = (void *)(treq_ctx->tail + ctx->reqoff);
1021 + struct scatterlist *tmp = treq_ctx->tmp;
1022 + struct scatterlist *src, *dst;
1023 +
1024 + u8 padding[255]; /* padding can be 0-255 bytes */
1025 + u8 pad_size;
1026 + u16 *len_field;
1027 + u8 *ihash, *hash = treq_ctx->tail;
1028 +
1029 + int paderr = 0;
1030 + int err = -EINVAL;
1031 + int i;
1032 + struct async_op ciph_op;
1033 +
1034 + /*
1035 + * Rule out bad packets. The input packet length must be at least one
1036 + * byte more than the hash_size
1037 + */
1038 + if (cryptlen <= hash_size || cryptlen % block_size)
1039 + goto out;
1040 +
1041 + /*
1042 + * Step 1 - Decrypt the source. Fast-forward past the associated data
1043 + * to the encrypted data. The result will be overwritten in place so
1044 + * that the decrypted data will be adjacent to the associated data. The
1045 + * last step (computing the hash) will have it's input data already
1046 + * prepared and ready to be accessed at req->src.
1047 + */
1048 + src = scatterwalk_ffwd(tmp, req->src, req->assoclen);
1049 + dst = src;
1050 +
1051 + init_completion(&ciph_op.completion);
1052 + skcipher_request_set_tfm(skreq, ctx->enc);
1053 + skcipher_request_set_callback(skreq, aead_request_flags(req),
1054 + tls_async_op_done, &ciph_op);
1055 + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
1056 + err = crypto_skcipher_decrypt(skreq);
1057 + if (err == -EINPROGRESS) {
1058 + err = wait_for_completion_interruptible(&ciph_op.completion);
1059 + if (!err)
1060 + err = ciph_op.err;
1061 + }
1062 + if (err)
1063 + goto out;
1064 +
1065 + /*
1066 + * Step 2 - Verify padding
1067 + * Retrieve the last byte of the payload; this is the padding size.
1068 + */
1069 + cryptlen -= 1;
1070 + scatterwalk_map_and_copy(&pad_size, dst, cryptlen, 1, 0);
1071 +
1072 + /* RFC recommendation for invalid padding size. */
1073 + if (cryptlen < pad_size + hash_size) {
1074 + pad_size = 0;
1075 + paderr = -EBADMSG;
1076 + }
1077 + cryptlen -= pad_size;
1078 + scatterwalk_map_and_copy(padding, dst, cryptlen, pad_size, 0);
1079 +
1080 + /* Padding content must be equal with pad_size. We verify it all */
1081 + for (i = 0; i < pad_size; i++)
1082 + if (padding[i] != pad_size)
1083 + paderr = -EBADMSG;
1084 +
1085 + /*
1086 + * Step 3 - Verify hash
1087 + * Align the digest result as required by the hash transform. Enough
1088 + * space was allocated in crypto_tls_init_tfm
1089 + */
1090 + hash = (u8 *)ALIGN((unsigned long)hash +
1091 + crypto_ahash_alignmask(ctx->auth),
1092 + crypto_ahash_alignmask(ctx->auth) + 1);
1093 + /*
1094 + * Two bytes at the end of the associated data make the length field.
1095 + * It must be updated with the length of the cleartext message before
1096 + * the hash is calculated.
1097 + */
1098 + len_field = sg_virt(req->src) + req->assoclen - 2;
1099 + cryptlen -= hash_size;
1100 + *len_field = htons(cryptlen);
1101 +
1102 + /* This is the hash from the decrypted packet. Save it for later */
1103 + ihash = hash + hash_size;
1104 + scatterwalk_map_and_copy(ihash, dst, cryptlen, hash_size, 0);
1105 +
1106 + /* Now compute and compare our ICV with the one from the packet */
1107 + err = crypto_tls_genicv(hash, req->src, cryptlen + req->assoclen, req);
1108 + if (!err)
1109 + err = memcmp(hash, ihash, hash_size) ? -EBADMSG : 0;
1110 +
1111 + if (req->src != req->dst) {
1112 + err = crypto_tls_copy_data(req, req->src, req->dst, cryptlen +
1113 + req->assoclen);
1114 + if (err)
1115 + goto out;
1116 + }
1117 +
1118 + /* return the first found error */
1119 + if (paderr)
1120 + err = paderr;
1121 +
1122 +out:
1123 + aead_request_complete(req, err);
1124 + return err;
1125 +}
1126 +
1127 +static int crypto_tls_init_tfm(struct crypto_aead *tfm)
1128 +{
1129 + struct aead_instance *inst = aead_alg_instance(tfm);
1130 + struct tls_instance_ctx *ictx = aead_instance_ctx(inst);
1131 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm);
1132 + struct crypto_ahash *auth;
1133 + struct crypto_skcipher *enc;
1134 + struct crypto_skcipher *null;
1135 + int err;
1136 +
1137 + auth = crypto_spawn_ahash(&ictx->auth);
1138 + if (IS_ERR(auth))
1139 + return PTR_ERR(auth);
1140 +
1141 + enc = crypto_spawn_skcipher(&ictx->enc);
1142 + err = PTR_ERR(enc);
1143 + if (IS_ERR(enc))
1144 + goto err_free_ahash;
1145 +
1146 + null = crypto_get_default_null_skcipher2();
1147 + err = PTR_ERR(null);
1148 + if (IS_ERR(null))
1149 + goto err_free_skcipher;
1150 +
1151 + ctx->auth = auth;
1152 + ctx->enc = enc;
1153 + ctx->null = null;
1154 +
1155 + /*
1156 + * Allow enough space for two digests. The two digests will be compared
1157 + * during the decryption phase. One will come from the decrypted packet
1158 + * and the other will be calculated. For encryption, one digest is
1159 + * padded (up to a cipher blocksize) and chained with the payload
1160 + */
1161 + ctx->reqoff = ALIGN(crypto_ahash_digestsize(auth) +
1162 + crypto_ahash_alignmask(auth),
1163 + crypto_ahash_alignmask(auth) + 1) +
1164 + max(crypto_ahash_digestsize(auth),
1165 + crypto_skcipher_blocksize(enc));
1166 +
1167 + crypto_aead_set_reqsize(tfm,
1168 + sizeof(struct tls_request_ctx) +
1169 + ctx->reqoff +
1170 + max_t(unsigned int,
1171 + crypto_ahash_reqsize(auth) +
1172 + sizeof(struct ahash_request),
1173 + crypto_skcipher_reqsize(enc) +
1174 + sizeof(struct skcipher_request)));
1175 +
1176 + return 0;
1177 +
1178 +err_free_skcipher:
1179 + crypto_free_skcipher(enc);
1180 +err_free_ahash:
1181 + crypto_free_ahash(auth);
1182 + return err;
1183 +}
1184 +
1185 +static void crypto_tls_exit_tfm(struct crypto_aead *tfm)
1186 +{
1187 + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm);
1188 +
1189 + crypto_free_ahash(ctx->auth);
1190 + crypto_free_skcipher(ctx->enc);
1191 + crypto_put_default_null_skcipher2();
1192 +}
1193 +
1194 +static void crypto_tls_free(struct aead_instance *inst)
1195 +{
1196 + struct tls_instance_ctx *ctx = aead_instance_ctx(inst);
1197 +
1198 + crypto_drop_skcipher(&ctx->enc);
1199 + crypto_drop_ahash(&ctx->auth);
1200 + kfree(inst);
1201 +}
1202 +
1203 +static int crypto_tls_create(struct crypto_template *tmpl, struct rtattr **tb)
1204 +{
1205 + struct crypto_attr_type *algt;
1206 + struct aead_instance *inst;
1207 + struct hash_alg_common *auth;
1208 + struct crypto_alg *auth_base;
1209 + struct skcipher_alg *enc;
1210 + struct tls_instance_ctx *ctx;
1211 + const char *enc_name;
1212 + int err;
1213 +
1214 + algt = crypto_get_attr_type(tb);
1215 + if (IS_ERR(algt))
1216 + return PTR_ERR(algt);
1217 +
1218 + if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1219 + return -EINVAL;
1220 +
1221 + auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH,
1222 + CRYPTO_ALG_TYPE_AHASH_MASK |
1223 + crypto_requires_sync(algt->type, algt->mask));
1224 + if (IS_ERR(auth))
1225 + return PTR_ERR(auth);
1226 +
1227 + auth_base = &auth->base;
1228 +
1229 + enc_name = crypto_attr_alg_name(tb[2]);
1230 + err = PTR_ERR(enc_name);
1231 + if (IS_ERR(enc_name))
1232 + goto out_put_auth;
1233 +
1234 + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1235 + err = -ENOMEM;
1236 + if (!inst)
1237 + goto out_put_auth;
1238 +
1239 + ctx = aead_instance_ctx(inst);
1240 +
1241 + err = crypto_init_ahash_spawn(&ctx->auth, auth,
1242 + aead_crypto_instance(inst));
1243 + if (err)
1244 + goto err_free_inst;
1245 +
1246 + crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
1247 + err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
1248 + crypto_requires_sync(algt->type,
1249 + algt->mask));
1250 + if (err)
1251 + goto err_drop_auth;
1252 +
1253 + enc = crypto_spawn_skcipher_alg(&ctx->enc);
1254 +
1255 + err = -ENAMETOOLONG;
1256 + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
1257 + "tls10(%s,%s)", auth_base->cra_name,
1258 + enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
1259 + goto err_drop_enc;
1260 +
1261 + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1262 + "tls10(%s,%s)", auth_base->cra_driver_name,
1263 + enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
1264 + goto err_drop_enc;
1265 +
1266 + inst->alg.base.cra_flags = (auth_base->cra_flags |
1267 + enc->base.cra_flags) & CRYPTO_ALG_ASYNC;
1268 + inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
1269 + auth_base->cra_priority;
1270 + inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
1271 + inst->alg.base.cra_alignmask = auth_base->cra_alignmask |
1272 + enc->base.cra_alignmask;
1273 + inst->alg.base.cra_ctxsize = sizeof(struct crypto_tls_ctx);
1274 +
1275 + inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc);
1276 + inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc);
1277 + inst->alg.maxauthsize = auth->digestsize;
1278 +
1279 + inst->alg.init = crypto_tls_init_tfm;
1280 + inst->alg.exit = crypto_tls_exit_tfm;
1281 +
1282 + inst->alg.setkey = crypto_tls_setkey;
1283 + inst->alg.encrypt = crypto_tls_encrypt;
1284 + inst->alg.decrypt = crypto_tls_decrypt;
1285 +
1286 + inst->free = crypto_tls_free;
1287 +
1288 + err = aead_register_instance(tmpl, inst);
1289 + if (err)
1290 + goto err_drop_enc;
1291 +
1292 +out:
1293 + crypto_mod_put(auth_base);
1294 + return err;
1295 +
1296 +err_drop_enc:
1297 + crypto_drop_skcipher(&ctx->enc);
1298 +err_drop_auth:
1299 + crypto_drop_ahash(&ctx->auth);
1300 +err_free_inst:
1301 + kfree(inst);
1302 +out_put_auth:
1303 + goto out;
1304 +}
1305 +
1306 +static struct crypto_template crypto_tls_tmpl = {
1307 + .name = "tls10",
1308 + .create = crypto_tls_create,
1309 + .module = THIS_MODULE,
1310 +};
1311 +
1312 +static int __init crypto_tls_module_init(void)
1313 +{
1314 + return crypto_register_template(&crypto_tls_tmpl);
1315 +}
1316 +
1317 +static void __exit crypto_tls_module_exit(void)
1318 +{
1319 + crypto_unregister_template(&crypto_tls_tmpl);
1320 +}
1321 +
1322 +module_init(crypto_tls_module_init);
1323 +module_exit(crypto_tls_module_exit);
1324 +
1325 +MODULE_LICENSE("GPL");
1326 +MODULE_DESCRIPTION("TLS 1.0 record encryption");
1327 --- a/drivers/crypto/Makefile
1328 +++ b/drivers/crypto/Makefile
1329 @@ -10,7 +10,7 @@ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chel
1330 obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
1331 obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/
1332 obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o
1333 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
1334 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/
1335 obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
1336 obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
1337 obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
1338 --- a/drivers/crypto/caam/Kconfig
1339 +++ b/drivers/crypto/caam/Kconfig
1340 @@ -1,7 +1,17 @@
1341 +config CRYPTO_DEV_FSL_CAAM_COMMON
1342 + tristate
1343 +
1344 +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
1345 + tristate
1346 +
1347 +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
1348 + tristate
1349 +
1350 config CRYPTO_DEV_FSL_CAAM
1351 - tristate "Freescale CAAM-Multicore driver backend"
1352 + tristate "Freescale CAAM-Multicore platform driver backend"
1353 depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
1354 select SOC_BUS
1355 + select CRYPTO_DEV_FSL_CAAM_COMMON
1356 help
1357 Enables the driver module for Freescale's Cryptographic Accelerator
1358 and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
1359 @@ -12,9 +22,16 @@ config CRYPTO_DEV_FSL_CAAM
1360 To compile this driver as a module, choose M here: the module
1361 will be called caam.
1362
1363 +if CRYPTO_DEV_FSL_CAAM
1364 +
1365 +config CRYPTO_DEV_FSL_CAAM_DEBUG
1366 + bool "Enable debug output in CAAM driver"
1367 + help
1368 + Selecting this will enable printing of various debug
1369 + information in the CAAM driver.
1370 +
1371 config CRYPTO_DEV_FSL_CAAM_JR
1372 tristate "Freescale CAAM Job Ring driver backend"
1373 - depends on CRYPTO_DEV_FSL_CAAM
1374 default y
1375 help
1376 Enables the driver module for Job Rings which are part of
1377 @@ -25,9 +42,10 @@ config CRYPTO_DEV_FSL_CAAM_JR
1378 To compile this driver as a module, choose M here: the module
1379 will be called caam_jr.
1380
1381 +if CRYPTO_DEV_FSL_CAAM_JR
1382 +
1383 config CRYPTO_DEV_FSL_CAAM_RINGSIZE
1384 int "Job Ring size"
1385 - depends on CRYPTO_DEV_FSL_CAAM_JR
1386 range 2 9
1387 default "9"
1388 help
1389 @@ -45,7 +63,6 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE
1390
1391 config CRYPTO_DEV_FSL_CAAM_INTC
1392 bool "Job Ring interrupt coalescing"
1393 - depends on CRYPTO_DEV_FSL_CAAM_JR
1394 help
1395 Enable the Job Ring's interrupt coalescing feature.
1396
1397 @@ -74,9 +91,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THL
1398 threshold. Range is 1-65535.
1399
1400 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
1401 - tristate "Register algorithm implementations with the Crypto API"
1402 - depends on CRYPTO_DEV_FSL_CAAM_JR
1403 + bool "Register algorithm implementations with the Crypto API"
1404 default y
1405 + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
1406 select CRYPTO_AEAD
1407 select CRYPTO_AUTHENC
1408 select CRYPTO_BLKCIPHER
1409 @@ -85,13 +102,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
1410 scatterlist crypto API (such as the linux native IPSec
1411 stack) to the SEC4 via job ring.
1412
1413 - To compile this as a module, choose M here: the module
1414 - will be called caamalg.
1415 -
1416 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
1417 - tristate "Queue Interface as Crypto API backend"
1418 - depends on CRYPTO_DEV_FSL_CAAM_JR && FSL_DPAA && NET
1419 + bool "Queue Interface as Crypto API backend"
1420 + depends on FSL_SDK_DPA && NET
1421 default y
1422 + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
1423 select CRYPTO_AUTHENC
1424 select CRYPTO_BLKCIPHER
1425 help
1426 @@ -102,36 +117,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
1427 assigned to the kernel should also be more than the number of
1428 job rings.
1429
1430 - To compile this as a module, choose M here: the module
1431 - will be called caamalg_qi.
1432 -
1433 config CRYPTO_DEV_FSL_CAAM_AHASH_API
1434 - tristate "Register hash algorithm implementations with Crypto API"
1435 - depends on CRYPTO_DEV_FSL_CAAM_JR
1436 + bool "Register hash algorithm implementations with Crypto API"
1437 default y
1438 + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
1439 select CRYPTO_HASH
1440 help
1441 Selecting this will offload ahash for users of the
1442 scatterlist crypto API to the SEC4 via job ring.
1443
1444 - To compile this as a module, choose M here: the module
1445 - will be called caamhash.
1446 -
1447 config CRYPTO_DEV_FSL_CAAM_PKC_API
1448 - tristate "Register public key cryptography implementations with Crypto API"
1449 - depends on CRYPTO_DEV_FSL_CAAM_JR
1450 + bool "Register public key cryptography implementations with Crypto API"
1451 default y
1452 select CRYPTO_RSA
1453 help
1454 Selecting this will allow SEC Public key support for RSA.
1455 Supported cryptographic primitives: encryption, decryption,
1456 signature and verification.
1457 - To compile this as a module, choose M here: the module
1458 - will be called caam_pkc.
1459
1460 config CRYPTO_DEV_FSL_CAAM_RNG_API
1461 - tristate "Register caam device for hwrng API"
1462 - depends on CRYPTO_DEV_FSL_CAAM_JR
1463 + bool "Register caam device for hwrng API"
1464 default y
1465 select CRYPTO_RNG
1466 select HW_RANDOM
1467 @@ -139,16 +144,24 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
1468 Selecting this will register the SEC4 hardware rng to
1469 the hw_random API for suppying the kernel entropy pool.
1470
1471 - To compile this as a module, choose M here: the module
1472 - will be called caamrng.
1473 +endif # CRYPTO_DEV_FSL_CAAM_JR
1474
1475 -config CRYPTO_DEV_FSL_CAAM_DEBUG
1476 - bool "Enable debug output in CAAM driver"
1477 - depends on CRYPTO_DEV_FSL_CAAM
1478 - help
1479 - Selecting this will enable printing of various debug
1480 - information in the CAAM driver.
1481 +endif # CRYPTO_DEV_FSL_CAAM
1482
1483 -config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
1484 - def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \
1485 - CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI)
1486 +config CRYPTO_DEV_FSL_DPAA2_CAAM
1487 + tristate "QorIQ DPAA2 CAAM (DPSECI) driver"
1488 + depends on FSL_MC_DPIO
1489 + select CRYPTO_DEV_FSL_CAAM_COMMON
1490 + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
1491 + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
1492 + select CRYPTO_BLKCIPHER
1493 + select CRYPTO_AUTHENC
1494 + select CRYPTO_AEAD
1495 + select CRYPTO_HASH
1496 + ---help---
1497 + CAAM driver for QorIQ Data Path Acceleration Architecture 2.
1498 + It handles DPSECI DPAA2 objects that sit on the Management Complex
1499 + (MC) fsl-mc bus.
1500 +
1501 + To compile this as a module, choose M here: the module
1502 + will be called dpaa2_caam.
1503 --- a/drivers/crypto/caam/Makefile
1504 +++ b/drivers/crypto/caam/Makefile
1505 @@ -6,19 +6,27 @@ ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG
1506 ccflags-y := -DDEBUG
1507 endif
1508
1509 +ccflags-y += -DVERSION=\"\"
1510 +
1511 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o
1512 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
1513 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
1514 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
1515 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
1516 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
1517 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
1518 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
1519 -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
1520 -
1521 -caam-objs := ctrl.o
1522 -caam_jr-objs := jr.o key_gen.o error.o
1523 -caam_pkc-y := caampkc.o pkc_desc.o
1524 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o
1525 +
1526 +caam-y := ctrl.o
1527 +caam_jr-y := jr.o key_gen.o
1528 +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
1529 +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
1530 +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
1531 +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
1532 +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o
1533 +
1534 +caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o
1535 ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
1536 ccflags-y += -DCONFIG_CAAM_QI
1537 - caam-objs += qi.o
1538 endif
1539 +
1540 +obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
1541 +
1542 +dpaa2_caam-y := caamalg_qi2.o dpseci.o
1543 --- a/drivers/crypto/caam/caamalg.c
1544 +++ b/drivers/crypto/caam/caamalg.c
1545 @@ -71,6 +71,8 @@
1546 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
1547 CAAM_CMD_SZ * 5)
1548
1549 +#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
1550 +
1551 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
1552 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
1553
1554 @@ -108,6 +110,7 @@ struct caam_ctx {
1555 dma_addr_t sh_desc_dec_dma;
1556 dma_addr_t sh_desc_givenc_dma;
1557 dma_addr_t key_dma;
1558 + enum dma_data_direction dir;
1559 struct device *jrdev;
1560 struct alginfo adata;
1561 struct alginfo cdata;
1562 @@ -118,6 +121,7 @@ static int aead_null_set_sh_desc(struct
1563 {
1564 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1565 struct device *jrdev = ctx->jrdev;
1566 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
1567 u32 *desc;
1568 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
1569 ctx->adata.keylen_pad;
1570 @@ -136,9 +140,10 @@ static int aead_null_set_sh_desc(struct
1571
1572 /* aead_encrypt shared descriptor */
1573 desc = ctx->sh_desc_enc;
1574 - cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize);
1575 + cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
1576 + ctrlpriv->era);
1577 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1578 - desc_bytes(desc), DMA_TO_DEVICE);
1579 + desc_bytes(desc), ctx->dir);
1580
1581 /*
1582 * Job Descriptor and Shared Descriptors
1583 @@ -154,9 +159,10 @@ static int aead_null_set_sh_desc(struct
1584
1585 /* aead_decrypt shared descriptor */
1586 desc = ctx->sh_desc_dec;
1587 - cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize);
1588 + cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
1589 + ctrlpriv->era);
1590 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1591 - desc_bytes(desc), DMA_TO_DEVICE);
1592 + desc_bytes(desc), ctx->dir);
1593
1594 return 0;
1595 }
1596 @@ -168,6 +174,7 @@ static int aead_set_sh_desc(struct crypt
1597 unsigned int ivsize = crypto_aead_ivsize(aead);
1598 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1599 struct device *jrdev = ctx->jrdev;
1600 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
1601 u32 ctx1_iv_off = 0;
1602 u32 *desc, *nonce = NULL;
1603 u32 inl_mask;
1604 @@ -234,9 +241,9 @@ static int aead_set_sh_desc(struct crypt
1605 desc = ctx->sh_desc_enc;
1606 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
1607 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
1608 - false);
1609 + false, ctrlpriv->era);
1610 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1611 - desc_bytes(desc), DMA_TO_DEVICE);
1612 + desc_bytes(desc), ctx->dir);
1613
1614 skip_enc:
1615 /*
1616 @@ -266,9 +273,9 @@ skip_enc:
1617 desc = ctx->sh_desc_dec;
1618 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
1619 ctx->authsize, alg->caam.geniv, is_rfc3686,
1620 - nonce, ctx1_iv_off, false);
1621 + nonce, ctx1_iv_off, false, ctrlpriv->era);
1622 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1623 - desc_bytes(desc), DMA_TO_DEVICE);
1624 + desc_bytes(desc), ctx->dir);
1625
1626 if (!alg->caam.geniv)
1627 goto skip_givenc;
1628 @@ -300,9 +307,9 @@ skip_enc:
1629 desc = ctx->sh_desc_enc;
1630 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
1631 ctx->authsize, is_rfc3686, nonce,
1632 - ctx1_iv_off, false);
1633 + ctx1_iv_off, false, ctrlpriv->era);
1634 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1635 - desc_bytes(desc), DMA_TO_DEVICE);
1636 + desc_bytes(desc), ctx->dir);
1637
1638 skip_givenc:
1639 return 0;
1640 @@ -323,6 +330,7 @@ static int gcm_set_sh_desc(struct crypto
1641 {
1642 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1643 struct device *jrdev = ctx->jrdev;
1644 + unsigned int ivsize = crypto_aead_ivsize(aead);
1645 u32 *desc;
1646 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
1647 ctx->cdata.keylen;
1648 @@ -344,9 +352,9 @@ static int gcm_set_sh_desc(struct crypto
1649 }
1650
1651 desc = ctx->sh_desc_enc;
1652 - cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
1653 + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
1654 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1655 - desc_bytes(desc), DMA_TO_DEVICE);
1656 + desc_bytes(desc), ctx->dir);
1657
1658 /*
1659 * Job Descriptor and Shared Descriptors
1660 @@ -361,9 +369,9 @@ static int gcm_set_sh_desc(struct crypto
1661 }
1662
1663 desc = ctx->sh_desc_dec;
1664 - cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
1665 + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
1666 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1667 - desc_bytes(desc), DMA_TO_DEVICE);
1668 + desc_bytes(desc), ctx->dir);
1669
1670 return 0;
1671 }
1672 @@ -382,6 +390,7 @@ static int rfc4106_set_sh_desc(struct cr
1673 {
1674 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1675 struct device *jrdev = ctx->jrdev;
1676 + unsigned int ivsize = crypto_aead_ivsize(aead);
1677 u32 *desc;
1678 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
1679 ctx->cdata.keylen;
1680 @@ -403,9 +412,10 @@ static int rfc4106_set_sh_desc(struct cr
1681 }
1682
1683 desc = ctx->sh_desc_enc;
1684 - cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
1685 + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
1686 + false);
1687 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1688 - desc_bytes(desc), DMA_TO_DEVICE);
1689 + desc_bytes(desc), ctx->dir);
1690
1691 /*
1692 * Job Descriptor and Shared Descriptors
1693 @@ -420,9 +430,10 @@ static int rfc4106_set_sh_desc(struct cr
1694 }
1695
1696 desc = ctx->sh_desc_dec;
1697 - cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
1698 + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
1699 + false);
1700 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1701 - desc_bytes(desc), DMA_TO_DEVICE);
1702 + desc_bytes(desc), ctx->dir);
1703
1704 return 0;
1705 }
1706 @@ -442,6 +453,7 @@ static int rfc4543_set_sh_desc(struct cr
1707 {
1708 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1709 struct device *jrdev = ctx->jrdev;
1710 + unsigned int ivsize = crypto_aead_ivsize(aead);
1711 u32 *desc;
1712 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
1713 ctx->cdata.keylen;
1714 @@ -463,9 +475,10 @@ static int rfc4543_set_sh_desc(struct cr
1715 }
1716
1717 desc = ctx->sh_desc_enc;
1718 - cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
1719 + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
1720 + false);
1721 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1722 - desc_bytes(desc), DMA_TO_DEVICE);
1723 + desc_bytes(desc), ctx->dir);
1724
1725 /*
1726 * Job Descriptor and Shared Descriptors
1727 @@ -480,9 +493,10 @@ static int rfc4543_set_sh_desc(struct cr
1728 }
1729
1730 desc = ctx->sh_desc_dec;
1731 - cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
1732 + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
1733 + false);
1734 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1735 - desc_bytes(desc), DMA_TO_DEVICE);
1736 + desc_bytes(desc), ctx->dir);
1737
1738 return 0;
1739 }
1740 @@ -498,11 +512,67 @@ static int rfc4543_setauthsize(struct cr
1741 return 0;
1742 }
1743
1744 +static int chachapoly_set_sh_desc(struct crypto_aead *aead)
1745 +{
1746 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
1747 + struct device *jrdev = ctx->jrdev;
1748 + unsigned int ivsize = crypto_aead_ivsize(aead);
1749 + u32 *desc;
1750 +
1751 + if (!ctx->cdata.keylen || !ctx->authsize)
1752 + return 0;
1753 +
1754 + desc = ctx->sh_desc_enc;
1755 + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
1756 + ctx->authsize, true, false);
1757 + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1758 + desc_bytes(desc), ctx->dir);
1759 +
1760 + desc = ctx->sh_desc_dec;
1761 + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
1762 + ctx->authsize, false, false);
1763 + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1764 + desc_bytes(desc), ctx->dir);
1765 +
1766 + return 0;
1767 +}
1768 +
1769 +static int chachapoly_setauthsize(struct crypto_aead *aead,
1770 + unsigned int authsize)
1771 +{
1772 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
1773 +
1774 + if (authsize != POLY1305_DIGEST_SIZE)
1775 + return -EINVAL;
1776 +
1777 + ctx->authsize = authsize;
1778 + return chachapoly_set_sh_desc(aead);
1779 +}
1780 +
1781 +static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
1782 + unsigned int keylen)
1783 +{
1784 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
1785 + unsigned int ivsize = crypto_aead_ivsize(aead);
1786 + unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
1787 +
1788 + if (keylen != CHACHA20_KEY_SIZE + saltlen) {
1789 + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1790 + return -EINVAL;
1791 + }
1792 +
1793 + ctx->cdata.key_virt = key;
1794 + ctx->cdata.keylen = keylen - saltlen;
1795 +
1796 + return chachapoly_set_sh_desc(aead);
1797 +}
1798 +
1799 static int aead_setkey(struct crypto_aead *aead,
1800 const u8 *key, unsigned int keylen)
1801 {
1802 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1803 struct device *jrdev = ctx->jrdev;
1804 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
1805 struct crypto_authenc_keys keys;
1806 int ret = 0;
1807
1808 @@ -517,6 +587,27 @@ static int aead_setkey(struct crypto_aea
1809 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1810 #endif
1811
1812 + /*
1813 + * If DKP is supported, use it in the shared descriptor to generate
1814 + * the split key.
1815 + */
1816 + if (ctrlpriv->era >= 6) {
1817 + ctx->adata.keylen = keys.authkeylen;
1818 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
1819 + OP_ALG_ALGSEL_MASK);
1820 +
1821 + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1822 + goto badkey;
1823 +
1824 + memcpy(ctx->key, keys.authkey, keys.authkeylen);
1825 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
1826 + keys.enckeylen);
1827 + dma_sync_single_for_device(jrdev, ctx->key_dma,
1828 + ctx->adata.keylen_pad +
1829 + keys.enckeylen, ctx->dir);
1830 + goto skip_split_key;
1831 + }
1832 +
1833 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
1834 keys.authkeylen, CAAM_MAX_KEY_SIZE -
1835 keys.enckeylen);
1836 @@ -527,12 +618,14 @@ static int aead_setkey(struct crypto_aea
1837 /* postpend encryption key to auth split key */
1838 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
1839 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
1840 - keys.enckeylen, DMA_TO_DEVICE);
1841 + keys.enckeylen, ctx->dir);
1842 #ifdef DEBUG
1843 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1844 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1845 ctx->adata.keylen_pad + keys.enckeylen, 1);
1846 #endif
1847 +
1848 +skip_split_key:
1849 ctx->cdata.keylen = keys.enckeylen;
1850 return aead_set_sh_desc(aead);
1851 badkey:
1852 @@ -552,7 +645,7 @@ static int gcm_setkey(struct crypto_aead
1853 #endif
1854
1855 memcpy(ctx->key, key, keylen);
1856 - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
1857 + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
1858 ctx->cdata.keylen = keylen;
1859
1860 return gcm_set_sh_desc(aead);
1861 @@ -580,7 +673,7 @@ static int rfc4106_setkey(struct crypto_
1862 */
1863 ctx->cdata.keylen = keylen - 4;
1864 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
1865 - DMA_TO_DEVICE);
1866 + ctx->dir);
1867 return rfc4106_set_sh_desc(aead);
1868 }
1869
1870 @@ -606,7 +699,7 @@ static int rfc4543_setkey(struct crypto_
1871 */
1872 ctx->cdata.keylen = keylen - 4;
1873 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
1874 - DMA_TO_DEVICE);
1875 + ctx->dir);
1876 return rfc4543_set_sh_desc(aead);
1877 }
1878
1879 @@ -658,21 +751,21 @@ static int ablkcipher_setkey(struct cryp
1880 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
1881 ctx1_iv_off);
1882 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1883 - desc_bytes(desc), DMA_TO_DEVICE);
1884 + desc_bytes(desc), ctx->dir);
1885
1886 /* ablkcipher_decrypt shared descriptor */
1887 desc = ctx->sh_desc_dec;
1888 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
1889 ctx1_iv_off);
1890 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1891 - desc_bytes(desc), DMA_TO_DEVICE);
1892 + desc_bytes(desc), ctx->dir);
1893
1894 /* ablkcipher_givencrypt shared descriptor */
1895 desc = ctx->sh_desc_givenc;
1896 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
1897 ctx1_iv_off);
1898 dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma,
1899 - desc_bytes(desc), DMA_TO_DEVICE);
1900 + desc_bytes(desc), ctx->dir);
1901
1902 return 0;
1903 }
1904 @@ -701,13 +794,13 @@ static int xts_ablkcipher_setkey(struct
1905 desc = ctx->sh_desc_enc;
1906 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
1907 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
1908 - desc_bytes(desc), DMA_TO_DEVICE);
1909 + desc_bytes(desc), ctx->dir);
1910
1911 /* xts_ablkcipher_decrypt shared descriptor */
1912 desc = ctx->sh_desc_dec;
1913 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
1914 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
1915 - desc_bytes(desc), DMA_TO_DEVICE);
1916 + desc_bytes(desc), ctx->dir);
1917
1918 return 0;
1919 }
1920 @@ -989,9 +1082,6 @@ static void init_aead_job(struct aead_re
1921 append_seq_out_ptr(desc, dst_dma,
1922 req->assoclen + req->cryptlen - authsize,
1923 out_options);
1924 -
1925 - /* REG3 = assoclen */
1926 - append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1927 }
1928
1929 static void init_gcm_job(struct aead_request *req,
1930 @@ -1006,6 +1096,7 @@ static void init_gcm_job(struct aead_req
1931 unsigned int last;
1932
1933 init_aead_job(req, edesc, all_contig, encrypt);
1934 + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1935
1936 /* BUG This should not be specific to generic GCM. */
1937 last = 0;
1938 @@ -1023,6 +1114,40 @@ static void init_gcm_job(struct aead_req
1939 /* End of blank commands */
1940 }
1941
1942 +static void init_chachapoly_job(struct aead_request *req,
1943 + struct aead_edesc *edesc, bool all_contig,
1944 + bool encrypt)
1945 +{
1946 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
1947 + unsigned int ivsize = crypto_aead_ivsize(aead);
1948 + unsigned int assoclen = req->assoclen;
1949 + u32 *desc = edesc->hw_desc;
1950 + u32 ctx_iv_off = 4;
1951 +
1952 + init_aead_job(req, edesc, all_contig, encrypt);
1953 +
1954 + if (ivsize != CHACHAPOLY_IV_SIZE) {
1955 + /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1956 + ctx_iv_off += 4;
1957 +
1958 + /*
1959 + * The associated data comes already with the IV but we need
1960 + * to skip it when we authenticate or encrypt...
1961 + */
1962 + assoclen -= ivsize;
1963 + }
1964 +
1965 + append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1966 +
1967 + /*
1968 + * For IPsec load the IV further in the same register.
1969 + * For RFC7539 simply load the 12 bytes nonce in a single operation
1970 + */
1971 + append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
1972 + LDST_SRCDST_BYTE_CONTEXT |
1973 + ctx_iv_off << LDST_OFFSET_SHIFT);
1974 +}
1975 +
1976 static void init_authenc_job(struct aead_request *req,
1977 struct aead_edesc *edesc,
1978 bool all_contig, bool encrypt)
1979 @@ -1032,6 +1157,7 @@ static void init_authenc_job(struct aead
1980 struct caam_aead_alg, aead);
1981 unsigned int ivsize = crypto_aead_ivsize(aead);
1982 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1983 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1984 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1985 OP_ALG_AAI_CTR_MOD128);
1986 const bool is_rfc3686 = alg->caam.rfc3686;
1987 @@ -1055,6 +1181,15 @@ static void init_authenc_job(struct aead
1988
1989 init_aead_job(req, edesc, all_contig, encrypt);
1990
1991 + /*
1992 + * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1993 + * having DPOVRD as destination.
1994 + */
1995 + if (ctrlpriv->era < 3)
1996 + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1997 + else
1998 + append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1999 +
2000 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
2001 append_load_as_imm(desc, req->iv, ivsize,
2002 LDST_CLASS_1_CCB |
2003 @@ -1227,8 +1362,16 @@ static struct aead_edesc *aead_edesc_all
2004 }
2005 }
2006
2007 + /*
2008 + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
2009 + * the end of the table by allocating more S/G entries.
2010 + */
2011 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
2012 - sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
2013 + if (mapped_dst_nents > 1)
2014 + sec4_sg_len += ALIGN(mapped_dst_nents, 4);
2015 + else
2016 + sec4_sg_len = ALIGN(sec4_sg_len, 4);
2017 +
2018 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2019
2020 /* allocate space for base edesc and hw desc commands, link tables */
2021 @@ -1309,6 +1452,72 @@ static int gcm_encrypt(struct aead_reque
2022 return ret;
2023 }
2024
2025 +static int chachapoly_encrypt(struct aead_request *req)
2026 +{
2027 + struct aead_edesc *edesc;
2028 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
2029 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
2030 + struct device *jrdev = ctx->jrdev;
2031 + bool all_contig;
2032 + u32 *desc;
2033 + int ret;
2034 +
2035 + edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
2036 + true);
2037 + if (IS_ERR(edesc))
2038 + return PTR_ERR(edesc);
2039 +
2040 + desc = edesc->hw_desc;
2041 +
2042 + init_chachapoly_job(req, edesc, all_contig, true);
2043 + print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
2044 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
2045 + 1);
2046 +
2047 + ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2048 + if (!ret) {
2049 + ret = -EINPROGRESS;
2050 + } else {
2051 + aead_unmap(jrdev, edesc, req);
2052 + kfree(edesc);
2053 + }
2054 +
2055 + return ret;
2056 +}
2057 +
2058 +static int chachapoly_decrypt(struct aead_request *req)
2059 +{
2060 + struct aead_edesc *edesc;
2061 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
2062 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
2063 + struct device *jrdev = ctx->jrdev;
2064 + bool all_contig;
2065 + u32 *desc;
2066 + int ret;
2067 +
2068 + edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
2069 + false);
2070 + if (IS_ERR(edesc))
2071 + return PTR_ERR(edesc);
2072 +
2073 + desc = edesc->hw_desc;
2074 +
2075 + init_chachapoly_job(req, edesc, all_contig, false);
2076 + print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
2077 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
2078 + 1);
2079 +
2080 + ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2081 + if (!ret) {
2082 + ret = -EINPROGRESS;
2083 + } else {
2084 + aead_unmap(jrdev, edesc, req);
2085 + kfree(edesc);
2086 + }
2087 +
2088 + return ret;
2089 +}
2090 +
2091 static int ipsec_gcm_encrypt(struct aead_request *req)
2092 {
2093 if (req->assoclen < 8)
2094 @@ -1496,7 +1705,25 @@ static struct ablkcipher_edesc *ablkciph
2095
2096 sec4_sg_ents = 1 + mapped_src_nents;
2097 dst_sg_idx = sec4_sg_ents;
2098 - sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
2099 +
2100 + /*
2101 + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
2102 + * the end of the table by allocating more S/G entries. Logic:
2103 + * if (src != dst && output S/G)
2104 + * pad output S/G, if needed
2105 + * else if (src == dst && S/G)
2106 + * overlapping S/Gs; pad one of them
2107 + * else if (input S/G) ...
2108 + * pad input S/G, if needed
2109 + */
2110 + if (mapped_dst_nents > 1)
2111 + sec4_sg_ents += ALIGN(mapped_dst_nents, 4);
2112 + else if ((req->src == req->dst) && (mapped_src_nents > 1))
2113 + sec4_sg_ents = max(ALIGN(sec4_sg_ents, 4),
2114 + 1 + ALIGN(mapped_src_nents, 4));
2115 + else
2116 + sec4_sg_ents = ALIGN(sec4_sg_ents, 4);
2117 +
2118 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
2119
2120 /*
2121 @@ -3199,6 +3426,50 @@ static struct caam_aead_alg driver_aeads
2122 .geniv = true,
2123 },
2124 },
2125 + {
2126 + .aead = {
2127 + .base = {
2128 + .cra_name = "rfc7539(chacha20,poly1305)",
2129 + .cra_driver_name = "rfc7539-chacha20-poly1305-"
2130 + "caam",
2131 + .cra_blocksize = 1,
2132 + },
2133 + .setkey = chachapoly_setkey,
2134 + .setauthsize = chachapoly_setauthsize,
2135 + .encrypt = chachapoly_encrypt,
2136 + .decrypt = chachapoly_decrypt,
2137 + .ivsize = CHACHAPOLY_IV_SIZE,
2138 + .maxauthsize = POLY1305_DIGEST_SIZE,
2139 + },
2140 + .caam = {
2141 + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2142 + OP_ALG_AAI_AEAD,
2143 + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2144 + OP_ALG_AAI_AEAD,
2145 + },
2146 + },
2147 + {
2148 + .aead = {
2149 + .base = {
2150 + .cra_name = "rfc7539esp(chacha20,poly1305)",
2151 + .cra_driver_name = "rfc7539esp-chacha20-"
2152 + "poly1305-caam",
2153 + .cra_blocksize = 1,
2154 + },
2155 + .setkey = chachapoly_setkey,
2156 + .setauthsize = chachapoly_setauthsize,
2157 + .encrypt = chachapoly_encrypt,
2158 + .decrypt = chachapoly_decrypt,
2159 + .ivsize = 8,
2160 + .maxauthsize = POLY1305_DIGEST_SIZE,
2161 + },
2162 + .caam = {
2163 + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2164 + OP_ALG_AAI_AEAD,
2165 + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2166 + OP_ALG_AAI_AEAD,
2167 + },
2168 + },
2169 };
2170
2171 struct caam_crypto_alg {
2172 @@ -3207,9 +3478,11 @@ struct caam_crypto_alg {
2173 struct caam_alg_entry caam;
2174 };
2175
2176 -static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
2177 +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2178 + bool uses_dkp)
2179 {
2180 dma_addr_t dma_addr;
2181 + struct caam_drv_private *priv;
2182
2183 ctx->jrdev = caam_jr_alloc();
2184 if (IS_ERR(ctx->jrdev)) {
2185 @@ -3217,10 +3490,16 @@ static int caam_init_common(struct caam_
2186 return PTR_ERR(ctx->jrdev);
2187 }
2188
2189 + priv = dev_get_drvdata(ctx->jrdev->parent);
2190 + if (priv->era >= 6 && uses_dkp)
2191 + ctx->dir = DMA_BIDIRECTIONAL;
2192 + else
2193 + ctx->dir = DMA_TO_DEVICE;
2194 +
2195 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
2196 offsetof(struct caam_ctx,
2197 sh_desc_enc_dma),
2198 - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
2199 + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
2200 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
2201 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
2202 caam_jr_free(ctx->jrdev);
2203 @@ -3248,7 +3527,7 @@ static int caam_cra_init(struct crypto_t
2204 container_of(alg, struct caam_crypto_alg, crypto_alg);
2205 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2206
2207 - return caam_init_common(ctx, &caam_alg->caam);
2208 + return caam_init_common(ctx, &caam_alg->caam, false);
2209 }
2210
2211 static int caam_aead_init(struct crypto_aead *tfm)
2212 @@ -3258,14 +3537,15 @@ static int caam_aead_init(struct crypto_
2213 container_of(alg, struct caam_aead_alg, aead);
2214 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2215
2216 - return caam_init_common(ctx, &caam_alg->caam);
2217 + return caam_init_common(ctx, &caam_alg->caam,
2218 + alg->setkey == aead_setkey);
2219 }
2220
2221 static void caam_exit_common(struct caam_ctx *ctx)
2222 {
2223 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
2224 offsetof(struct caam_ctx, sh_desc_enc_dma),
2225 - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
2226 + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
2227 caam_jr_free(ctx->jrdev);
2228 }
2229
2230 @@ -3279,7 +3559,7 @@ static void caam_aead_exit(struct crypto
2231 caam_exit_common(crypto_aead_ctx(tfm));
2232 }
2233
2234 -static void __exit caam_algapi_exit(void)
2235 +void caam_algapi_exit(void)
2236 {
2237
2238 struct caam_crypto_alg *t_alg, *n;
2239 @@ -3358,56 +3638,52 @@ static void caam_aead_alg_init(struct ca
2240 alg->exit = caam_aead_exit;
2241 }
2242
2243 -static int __init caam_algapi_init(void)
2244 +int caam_algapi_init(struct device *ctrldev)
2245 {
2246 - struct device_node *dev_node;
2247 - struct platform_device *pdev;
2248 - struct device *ctrldev;
2249 - struct caam_drv_private *priv;
2250 + struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
2251 int i = 0, err = 0;
2252 - u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
2253 + u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
2254 unsigned int md_limit = SHA512_DIGEST_SIZE;
2255 bool registered = false;
2256
2257 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
2258 - if (!dev_node) {
2259 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
2260 - if (!dev_node)
2261 - return -ENODEV;
2262 - }
2263 -
2264 - pdev = of_find_device_by_node(dev_node);
2265 - if (!pdev) {
2266 - of_node_put(dev_node);
2267 - return -ENODEV;
2268 - }
2269 -
2270 - ctrldev = &pdev->dev;
2271 - priv = dev_get_drvdata(ctrldev);
2272 - of_node_put(dev_node);
2273 -
2274 - /*
2275 - * If priv is NULL, it's probably because the caam driver wasn't
2276 - * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
2277 - */
2278 - if (!priv)
2279 - return -ENODEV;
2280 -
2281 -
2282 INIT_LIST_HEAD(&alg_list);
2283
2284 /*
2285 * Register crypto algorithms the device supports.
2286 * First, detect presence and attributes of DES, AES, and MD blocks.
2287 */
2288 - cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2289 - cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2290 - des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
2291 - aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
2292 - md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2293 + if (priv->era < 10) {
2294 + u32 cha_vid, cha_inst;
2295 +
2296 + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2297 + aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2298 + md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2299 +
2300 + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2301 + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2302 + CHA_ID_LS_DES_SHIFT;
2303 + aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2304 + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2305 + ccha_inst = 0;
2306 + ptha_inst = 0;
2307 + } else {
2308 + u32 aesa, mdha;
2309 +
2310 + aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2311 + mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2312 +
2313 + aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2314 + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2315 +
2316 + des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2317 + aes_inst = aesa & CHA_VER_NUM_MASK;
2318 + md_inst = mdha & CHA_VER_NUM_MASK;
2319 + ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
2320 + ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
2321 + }
2322
2323 /* If MD is present, limit digest size based on LP256 */
2324 - if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
2325 + if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
2326 md_limit = SHA256_DIGEST_SIZE;
2327
2328 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2329 @@ -3429,10 +3705,10 @@ static int __init caam_algapi_init(void)
2330 * Check support for AES modes not available
2331 * on LP devices.
2332 */
2333 - if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
2334 - if ((alg->class1_alg_type & OP_ALG_AAI_MASK) ==
2335 - OP_ALG_AAI_XTS)
2336 - continue;
2337 + if (aes_vid == CHA_VER_VID_AES_LP &&
2338 + (alg->class1_alg_type & OP_ALG_AAI_MASK) ==
2339 + OP_ALG_AAI_XTS)
2340 + continue;
2341
2342 t_alg = caam_alg_alloc(alg);
2343 if (IS_ERR(t_alg)) {
2344 @@ -3471,21 +3747,28 @@ static int __init caam_algapi_init(void)
2345 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2346 continue;
2347
2348 + /* Skip CHACHA20 algorithms if not supported by device */
2349 + if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
2350 + continue;
2351 +
2352 + /* Skip POLY1305 algorithms if not supported by device */
2353 + if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
2354 + continue;
2355 +
2356 /*
2357 * Check support for AES algorithms not available
2358 * on LP devices.
2359 */
2360 - if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
2361 - if (alg_aai == OP_ALG_AAI_GCM)
2362 - continue;
2363 + if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2364 + continue;
2365
2366 /*
2367 * Skip algorithms requiring message digests
2368 * if MD or MD size is not supported by device.
2369 */
2370 - if (c2_alg_sel &&
2371 - (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2372 - continue;
2373 + if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
2374 + (!md_inst || t_alg->aead.maxauthsize > md_limit))
2375 + continue;
2376
2377 caam_aead_alg_init(t_alg);
2378
2379 @@ -3505,10 +3788,3 @@ static int __init caam_algapi_init(void)
2380
2381 return err;
2382 }
2383 -
2384 -module_init(caam_algapi_init);
2385 -module_exit(caam_algapi_exit);
2386 -
2387 -MODULE_LICENSE("GPL");
2388 -MODULE_DESCRIPTION("FSL CAAM support for crypto API");
2389 -MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
2390 --- a/drivers/crypto/caam/caamalg_desc.c
2391 +++ b/drivers/crypto/caam/caamalg_desc.c
2392 @@ -45,16 +45,16 @@ static inline void append_dec_op1(u32 *d
2393 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
2394 * (non-protocol) with no (null) encryption.
2395 * @desc: pointer to buffer used for descriptor construction
2396 - * @adata: pointer to authentication transform definitions. Note that since a
2397 - * split key is to be used, the size of the split key itself is
2398 - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
2399 - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2400 + * @adata: pointer to authentication transform definitions.
2401 + * A split key is required for SEC Era < 6; the size of the split key
2402 + * is specified in this case. Valid algorithm values - one of
2403 + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
2404 + * with OP_ALG_AAI_HMAC_PRECOMP.
2405 * @icvsize: integrity check value (ICV) size (truncated or full)
2406 - *
2407 - * Note: Requires an MDHA split key.
2408 + * @era: SEC Era
2409 */
2410 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
2411 - unsigned int icvsize)
2412 + unsigned int icvsize, int era)
2413 {
2414 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
2415
2416 @@ -63,13 +63,18 @@ void cnstr_shdsc_aead_null_encap(u32 * c
2417 /* Skip if already shared */
2418 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2419 JUMP_COND_SHRD);
2420 - if (adata->key_inline)
2421 - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
2422 - adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT |
2423 - KEY_ENC);
2424 - else
2425 - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
2426 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
2427 + if (era < 6) {
2428 + if (adata->key_inline)
2429 + append_key_as_imm(desc, adata->key_virt,
2430 + adata->keylen_pad, adata->keylen,
2431 + CLASS_2 | KEY_DEST_MDHA_SPLIT |
2432 + KEY_ENC);
2433 + else
2434 + append_key(desc, adata->key_dma, adata->keylen,
2435 + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
2436 + } else {
2437 + append_proto_dkp(desc, adata);
2438 + }
2439 set_jump_tgt_here(desc, key_jump_cmd);
2440
2441 /* assoclen + cryptlen = seqinlen */
2442 @@ -121,16 +126,16 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_enca
2443 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
2444 * (non-protocol) with no (null) decryption.
2445 * @desc: pointer to buffer used for descriptor construction
2446 - * @adata: pointer to authentication transform definitions. Note that since a
2447 - * split key is to be used, the size of the split key itself is
2448 - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
2449 - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2450 + * @adata: pointer to authentication transform definitions.
2451 + * A split key is required for SEC Era < 6; the size of the split key
2452 + * is specified in this case. Valid algorithm values - one of
2453 + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
2454 + * with OP_ALG_AAI_HMAC_PRECOMP.
2455 * @icvsize: integrity check value (ICV) size (truncated or full)
2456 - *
2457 - * Note: Requires an MDHA split key.
2458 + * @era: SEC Era
2459 */
2460 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
2461 - unsigned int icvsize)
2462 + unsigned int icvsize, int era)
2463 {
2464 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
2465
2466 @@ -139,13 +144,18 @@ void cnstr_shdsc_aead_null_decap(u32 * c
2467 /* Skip if already shared */
2468 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2469 JUMP_COND_SHRD);
2470 - if (adata->key_inline)
2471 - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
2472 - adata->keylen, CLASS_2 |
2473 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
2474 - else
2475 - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
2476 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
2477 + if (era < 6) {
2478 + if (adata->key_inline)
2479 + append_key_as_imm(desc, adata->key_virt,
2480 + adata->keylen_pad, adata->keylen,
2481 + CLASS_2 | KEY_DEST_MDHA_SPLIT |
2482 + KEY_ENC);
2483 + else
2484 + append_key(desc, adata->key_dma, adata->keylen,
2485 + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
2486 + } else {
2487 + append_proto_dkp(desc, adata);
2488 + }
2489 set_jump_tgt_here(desc, key_jump_cmd);
2490
2491 /* Class 2 operation */
2492 @@ -204,7 +214,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_deca
2493 static void init_sh_desc_key_aead(u32 * const desc,
2494 struct alginfo * const cdata,
2495 struct alginfo * const adata,
2496 - const bool is_rfc3686, u32 *nonce)
2497 + const bool is_rfc3686, u32 *nonce, int era)
2498 {
2499 u32 *key_jump_cmd;
2500 unsigned int enckeylen = cdata->keylen;
2501 @@ -224,13 +234,18 @@ static void init_sh_desc_key_aead(u32 *
2502 if (is_rfc3686)
2503 enckeylen -= CTR_RFC3686_NONCE_SIZE;
2504
2505 - if (adata->key_inline)
2506 - append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
2507 - adata->keylen, CLASS_2 |
2508 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
2509 - else
2510 - append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
2511 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
2512 + if (era < 6) {
2513 + if (adata->key_inline)
2514 + append_key_as_imm(desc, adata->key_virt,
2515 + adata->keylen_pad, adata->keylen,
2516 + CLASS_2 | KEY_DEST_MDHA_SPLIT |
2517 + KEY_ENC);
2518 + else
2519 + append_key(desc, adata->key_dma, adata->keylen,
2520 + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
2521 + } else {
2522 + append_proto_dkp(desc, adata);
2523 + }
2524
2525 if (cdata->key_inline)
2526 append_key_as_imm(desc, cdata->key_virt, enckeylen,
2527 @@ -261,26 +276,27 @@ static void init_sh_desc_key_aead(u32 *
2528 * @cdata: pointer to block cipher transform definitions
2529 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
2530 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
2531 - * @adata: pointer to authentication transform definitions. Note that since a
2532 - * split key is to be used, the size of the split key itself is
2533 - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
2534 - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2535 + * @adata: pointer to authentication transform definitions.
2536 + * A split key is required for SEC Era < 6; the size of the split key
2537 + * is specified in this case. Valid algorithm values - one of
2538 + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
2539 + * with OP_ALG_AAI_HMAC_PRECOMP.
2540 * @ivsize: initialization vector size
2541 * @icvsize: integrity check value (ICV) size (truncated or full)
2542 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
2543 * @nonce: pointer to rfc3686 nonce
2544 * @ctx1_iv_off: IV offset in CONTEXT1 register
2545 * @is_qi: true when called from caam/qi
2546 - *
2547 - * Note: Requires an MDHA split key.
2548 + * @era: SEC Era
2549 */
2550 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
2551 struct alginfo *adata, unsigned int ivsize,
2552 unsigned int icvsize, const bool is_rfc3686,
2553 - u32 *nonce, const u32 ctx1_iv_off, const bool is_qi)
2554 + u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
2555 + int era)
2556 {
2557 /* Note: Context registers are saved. */
2558 - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
2559 + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
2560
2561 /* Class 2 operation */
2562 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
2563 @@ -306,8 +322,13 @@ void cnstr_shdsc_aead_encap(u32 * const
2564 }
2565
2566 /* Read and write assoclen bytes */
2567 - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2568 - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
2569 + if (is_qi || era < 3) {
2570 + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2571 + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
2572 + } else {
2573 + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
2574 + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
2575 + }
2576
2577 /* Skip assoc data */
2578 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
2579 @@ -350,27 +371,27 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
2580 * @cdata: pointer to block cipher transform definitions
2581 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
2582 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
2583 - * @adata: pointer to authentication transform definitions. Note that since a
2584 - * split key is to be used, the size of the split key itself is
2585 - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
2586 - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2587 + * @adata: pointer to authentication transform definitions.
2588 + * A split key is required for SEC Era < 6; the size of the split key
2589 + * is specified in this case. Valid algorithm values - one of
2590 + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
2591 + * with OP_ALG_AAI_HMAC_PRECOMP.
2592 * @ivsize: initialization vector size
2593 * @icvsize: integrity check value (ICV) size (truncated or full)
2594 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
2595 * @nonce: pointer to rfc3686 nonce
2596 * @ctx1_iv_off: IV offset in CONTEXT1 register
2597 * @is_qi: true when called from caam/qi
2598 - *
2599 - * Note: Requires an MDHA split key.
2600 + * @era: SEC Era
2601 */
2602 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
2603 struct alginfo *adata, unsigned int ivsize,
2604 unsigned int icvsize, const bool geniv,
2605 const bool is_rfc3686, u32 *nonce,
2606 - const u32 ctx1_iv_off, const bool is_qi)
2607 + const u32 ctx1_iv_off, const bool is_qi, int era)
2608 {
2609 /* Note: Context registers are saved. */
2610 - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
2611 + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
2612
2613 /* Class 2 operation */
2614 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
2615 @@ -397,11 +418,23 @@ void cnstr_shdsc_aead_decap(u32 * const
2616 }
2617
2618 /* Read and write assoclen bytes */
2619 - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2620 - if (geniv)
2621 - append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
2622 - else
2623 - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
2624 + if (is_qi || era < 3) {
2625 + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2626 + if (geniv)
2627 + append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
2628 + ivsize);
2629 + else
2630 + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
2631 + CAAM_CMD_SZ);
2632 + } else {
2633 + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
2634 + if (geniv)
2635 + append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
2636 + ivsize);
2637 + else
2638 + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
2639 + CAAM_CMD_SZ);
2640 + }
2641
2642 /* Skip assoc data */
2643 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
2644 @@ -456,29 +489,29 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
2645 * @cdata: pointer to block cipher transform definitions
2646 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
2647 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
2648 - * @adata: pointer to authentication transform definitions. Note that since a
2649 - * split key is to be used, the size of the split key itself is
2650 - * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
2651 - * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2652 + * @adata: pointer to authentication transform definitions.
2653 + * A split key is required for SEC Era < 6; the size of the split key
2654 + * is specified in this case. Valid algorithm values - one of
2655 + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
2656 + * with OP_ALG_AAI_HMAC_PRECOMP.
2657 * @ivsize: initialization vector size
2658 * @icvsize: integrity check value (ICV) size (truncated or full)
2659 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
2660 * @nonce: pointer to rfc3686 nonce
2661 * @ctx1_iv_off: IV offset in CONTEXT1 register
2662 * @is_qi: true when called from caam/qi
2663 - *
2664 - * Note: Requires an MDHA split key.
2665 + * @era: SEC Era
2666 */
2667 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
2668 struct alginfo *adata, unsigned int ivsize,
2669 unsigned int icvsize, const bool is_rfc3686,
2670 u32 *nonce, const u32 ctx1_iv_off,
2671 - const bool is_qi)
2672 + const bool is_qi, int era)
2673 {
2674 u32 geniv, moveiv;
2675
2676 /* Note: Context registers are saved. */
2677 - init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
2678 + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
2679
2680 if (is_qi) {
2681 u32 *wait_load_cmd;
2682 @@ -528,8 +561,13 @@ copy_iv:
2683 OP_ALG_ENCRYPT);
2684
2685 /* Read and write assoclen bytes */
2686 - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2687 - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
2688 + if (is_qi || era < 3) {
2689 + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
2690 + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
2691 + } else {
2692 + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
2693 + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
2694 + }
2695
2696 /* Skip assoc data */
2697 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
2698 @@ -583,14 +621,431 @@ copy_iv:
2699 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
2700
2701 /**
2702 + * cnstr_shdsc_tls_encap - tls encapsulation shared descriptor
2703 + * @desc: pointer to buffer used for descriptor construction
2704 + * @cdata: pointer to block cipher transform definitions
2705 + * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed
2706 + * with OP_ALG_AAI_CBC
2707 + * @adata: pointer to authentication transform definitions.
2708 + * A split key is required for SEC Era < 6; the size of the split key
2709 + * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1
2710 + * ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2711 + * @assoclen: associated data length
2712 + * @ivsize: initialization vector size
2713 + * @authsize: authentication data size
2714 + * @blocksize: block cipher size
2715 + * @era: SEC Era
2716 + */
2717 +void cnstr_shdsc_tls_encap(u32 * const desc, struct alginfo *cdata,
2718 + struct alginfo *adata, unsigned int assoclen,
2719 + unsigned int ivsize, unsigned int authsize,
2720 + unsigned int blocksize, int era)
2721 +{
2722 + u32 *key_jump_cmd, *zero_payload_jump_cmd;
2723 + u32 genpad, idx_ld_datasz, idx_ld_pad, stidx;
2724 +
2725 + /*
2726 + * Compute the index (in bytes) for the LOAD with destination of
2727 + * Class 1 Data Size Register and for the LOAD that generates padding
2728 + */
2729 + if (adata->key_inline) {
2730 + idx_ld_datasz = DESC_TLS10_ENC_LEN + adata->keylen_pad +
2731 + cdata->keylen - 4 * CAAM_CMD_SZ;
2732 + idx_ld_pad = DESC_TLS10_ENC_LEN + adata->keylen_pad +
2733 + cdata->keylen - 2 * CAAM_CMD_SZ;
2734 + } else {
2735 + idx_ld_datasz = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ -
2736 + 4 * CAAM_CMD_SZ;
2737 + idx_ld_pad = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ -
2738 + 2 * CAAM_CMD_SZ;
2739 + }
2740 +
2741 + stidx = 1 << HDR_START_IDX_SHIFT;
2742 + init_sh_desc(desc, HDR_SHARE_SERIAL | stidx);
2743 +
2744 + /* skip key loading if they are loaded due to sharing */
2745 + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2746 + JUMP_COND_SHRD);
2747 +
2748 + if (era < 6) {
2749 + if (adata->key_inline)
2750 + append_key_as_imm(desc, adata->key_virt,
2751 + adata->keylen_pad, adata->keylen,
2752 + CLASS_2 | KEY_DEST_MDHA_SPLIT |
2753 + KEY_ENC);
2754 + else
2755 + append_key(desc, adata->key_dma, adata->keylen,
2756 + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
2757 + } else {
2758 + append_proto_dkp(desc, adata);
2759 + }
2760 +
2761 + if (cdata->key_inline)
2762 + append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
2763 + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
2764 + else
2765 + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
2766 + KEY_DEST_CLASS_REG);
2767 +
2768 + set_jump_tgt_here(desc, key_jump_cmd);
2769 +
2770 + /* class 2 operation */
2771 + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
2772 + OP_ALG_ENCRYPT);
2773 + /* class 1 operation */
2774 + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
2775 + OP_ALG_ENCRYPT);
2776 +
2777 + /* payloadlen = input data length - (assoclen + ivlen) */
2778 + append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, assoclen + ivsize);
2779 +
2780 + /* math1 = payloadlen + icvlen */
2781 + append_math_add_imm_u32(desc, REG1, REG0, IMM, authsize);
2782 +
2783 + /* padlen = block_size - math1 % block_size */
2784 + append_math_and_imm_u32(desc, REG3, REG1, IMM, blocksize - 1);
2785 + append_math_sub_imm_u32(desc, REG2, IMM, REG3, blocksize);
2786 +
2787 + /* cryptlen = payloadlen + icvlen + padlen */
2788 + append_math_add(desc, VARSEQOUTLEN, REG1, REG2, 4);
2789 +
2790 + /*
2791 + * update immediate data with the padding length value
2792 + * for the LOAD in the class 1 data size register.
2793 + */
2794 + append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 |
2795 + (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 7);
2796 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF |
2797 + (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 8);
2798 +
2799 + /* overwrite PL field for the padding iNFO FIFO entry */
2800 + append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 |
2801 + (idx_ld_pad << MOVE_OFFSET_SHIFT) | 7);
2802 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF |
2803 + (idx_ld_pad << MOVE_OFFSET_SHIFT) | 8);
2804 +
2805 + /* store encrypted payload, icv and padding */
2806 + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
2807 +
2808 + /* if payload length is zero, jump to zero-payload commands */
2809 + append_math_add(desc, VARSEQINLEN, ZERO, REG0, 4);
2810 + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
2811 + JUMP_COND_MATH_Z);
2812 +
2813 + /* load iv in context1 */
2814 + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX |
2815 + LDST_CLASS_1_CCB | ivsize);
2816 +
2817 + /* read assoc for authentication */
2818 + append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 |
2819 + FIFOLD_TYPE_MSG);
2820 + /* insnoop payload */
2821 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_MSG |
2822 + FIFOLD_TYPE_LAST2 | FIFOLDST_VLF);
2823 +
2824 + /* jump the zero-payload commands */
2825 + append_jump(desc, JUMP_TEST_ALL | 3);
2826 +
2827 + /* zero-payload commands */
2828 + set_jump_tgt_here(desc, zero_payload_jump_cmd);
2829 +
2830 + /* load iv in context1 */
2831 + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX |
2832 + LDST_CLASS_1_CCB | ivsize);
2833 +
2834 + /* assoc data is the only data for authentication */
2835 + append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 |
2836 + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
2837 +
2838 + /* send icv to encryption */
2839 + append_move(desc, MOVE_SRC_CLASS2CTX | MOVE_DEST_CLASS1INFIFO |
2840 + authsize);
2841 +
2842 + /* update class 1 data size register with padding length */
2843 + append_load_imm_u32(desc, 0, LDST_CLASS_1_CCB |
2844 + LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
2845 +
2846 + /* generate padding and send it to encryption */
2847 + genpad = NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_LC1 | NFIFOENTRY_FC1 |
2848 + NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_PTYPE_N;
2849 + append_load_imm_u32(desc, genpad, LDST_CLASS_IND_CCB |
2850 + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
2851 +
2852 +#ifdef DEBUG
2853 + print_hex_dump(KERN_ERR, "tls enc shdesc@" __stringify(__LINE__) ": ",
2854 + DUMP_PREFIX_ADDRESS, 16, 4, desc,
2855 + desc_bytes(desc), 1);
2856 +#endif
2857 +}
2858 +EXPORT_SYMBOL(cnstr_shdsc_tls_encap);
2859 +
2860 +/**
2861 + * cnstr_shdsc_tls_decap - tls decapsulation shared descriptor
2862 + * @desc: pointer to buffer used for descriptor construction
2863 + * @cdata: pointer to block cipher transform definitions
2864 + * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed
2865 + * with OP_ALG_AAI_CBC
2866 + * @adata: pointer to authentication transform definitions.
2867 + * A split key is required for SEC Era < 6; the size of the split key
2868 + * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1
2869 + * ANDed with OP_ALG_AAI_HMAC_PRECOMP.
2870 + * @assoclen: associated data length
2871 + * @ivsize: initialization vector size
2872 + * @authsize: authentication data size
2873 + * @blocksize: block cipher size
2874 + * @era: SEC Era
2875 + */
2876 +void cnstr_shdsc_tls_decap(u32 * const desc, struct alginfo *cdata,
2877 + struct alginfo *adata, unsigned int assoclen,
2878 + unsigned int ivsize, unsigned int authsize,
2879 + unsigned int blocksize, int era)
2880 +{
2881 + u32 stidx, jumpback;
2882 + u32 *key_jump_cmd, *zero_payload_jump_cmd, *skip_zero_jump_cmd;
2883 + /*
2884 + * Pointer Size bool determines the size of address pointers.
2885 + * false - Pointers fit in one 32-bit word.
2886 + * true - Pointers fit in two 32-bit words.
2887 + */
2888 + static const bool ps = (CAAM_PTR_SZ != CAAM_CMD_SZ);
2889 +
2890 + stidx = 1 << HDR_START_IDX_SHIFT;
2891 + init_sh_desc(desc, HDR_SHARE_SERIAL | stidx);
2892 +
2893 + /* skip key loading if they are loaded due to sharing */
2894 + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2895 + JUMP_COND_SHRD);
2896 +
2897 + if (era < 6)
2898 + append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
2899 + KEY_DEST_MDHA_SPLIT | KEY_ENC);
2900 + else
2901 + append_proto_dkp(desc, adata);
2902 +
2903 + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
2904 + KEY_DEST_CLASS_REG);
2905 +
2906 + set_jump_tgt_here(desc, key_jump_cmd);
2907 +
2908 + /* class 2 operation */
2909 + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
2910 + OP_ALG_DECRYPT | OP_ALG_ICV_ON);
2911 + /* class 1 operation */
2912 + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
2913 + OP_ALG_DECRYPT);
2914 +
2915 + /* VSIL = input data length - 2 * block_size */
2916 + append_math_sub_imm_u32(desc, VARSEQINLEN, SEQINLEN, IMM, 2 *
2917 + blocksize);
2918 +
2919 + /*
2920 + * payloadlen + icvlen + padlen = input data length - (assoclen +
2921 + * ivsize)
2922 + */
2923 + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, assoclen + ivsize);
2924 +
2925 + /* skip data to the last but one cipher block */
2926 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | LDST_VLF);
2927 +
2928 + /* load iv for the last cipher block */
2929 + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX |
2930 + LDST_CLASS_1_CCB | ivsize);
2931 +
2932 + /* read last cipher block */
2933 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG |
2934 + FIFOLD_TYPE_LAST1 | blocksize);
2935 +
2936 + /* move decrypted block into math0 and math1 */
2937 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | MOVE_DEST_MATH0 |
2938 + blocksize);
2939 +
2940 + /* reset AES CHA */
2941 + append_load_imm_u32(desc, CCTRL_RESET_CHA_AESA, LDST_CLASS_IND_CCB |
2942 + LDST_SRCDST_WORD_CHACTRL | LDST_IMM);
2943 +
2944 + /* rewind input sequence */
2945 + append_seq_in_ptr_intlen(desc, 0, 65535, SQIN_RTO);
2946 +
2947 + /* key1 is in decryption form */
2948 + append_operation(desc, cdata->algtype | OP_ALG_AAI_DK |
2949 + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
2950 +
2951 + /* load iv in context1 */
2952 + append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_1_CCB |
2953 + LDST_SRCDST_WORD_CLASS_CTX | ivsize);
2954 +
2955 + /* read sequence number */
2956 + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG);
2957 + /* load Type, Version and Len fields in math0 */
2958 + append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_DECO |
2959 + LDST_SRCDST_WORD_DECO_MATH0 | (3 << LDST_OFFSET_SHIFT) | 5);
2960 +
2961 + /* compute (padlen - 1) */
2962 + append_math_and_imm_u64(desc, REG1, REG1, IMM, 255);
2963 +
2964 + /* math2 = icvlen + (padlen - 1) + 1 */
2965 + append_math_add_imm_u32(desc, REG2, REG1, IMM, authsize + 1);
2966 +
2967 + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1);
2968 +
2969 + /* VSOL = payloadlen + icvlen + padlen */
2970 + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, 4);
2971 +
2972 + if (caam_little_end)
2973 + append_moveb(desc, MOVE_WAITCOMP |
2974 + MOVE_SRC_MATH0 | MOVE_DEST_MATH0 | 8);
2975 +
2976 + /* update Len field */
2977 + append_math_sub(desc, REG0, REG0, REG2, 8);
2978 +
2979 + /* store decrypted payload, icv and padding */
2980 + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
2981 +
2982 + /* VSIL = (payloadlen + icvlen + padlen) - (icvlen + padlen)*/
2983 + append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4);
2984 +
2985 + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
2986 + JUMP_COND_MATH_Z);
2987 +
2988 + /* send Type, Version and Len(pre ICV) fields to authentication */
2989 + append_move(desc, MOVE_WAITCOMP |
2990 + MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO |
2991 + (3 << MOVE_OFFSET_SHIFT) | 5);
2992 +
2993 + /* outsnooping payload */
2994 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
2995 + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LAST2 |
2996 + FIFOLDST_VLF);
2997 + skip_zero_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 2);
2998 +
2999 + set_jump_tgt_here(desc, zero_payload_jump_cmd);
3000 + /* send Type, Version and Len(pre ICV) fields to authentication */
3001 + append_move(desc, MOVE_WAITCOMP | MOVE_AUX_LS |
3002 + MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO |
3003 + (3 << MOVE_OFFSET_SHIFT) | 5);
3004 +
3005 + set_jump_tgt_here(desc, skip_zero_jump_cmd);
3006 + append_math_add(desc, VARSEQINLEN, ZERO, REG2, 4);
3007 +
3008 + /* load icvlen and padlen */
3009 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG |
3010 + FIFOLD_TYPE_LAST1 | FIFOLDST_VLF);
3011 +
3012 + /* VSIL = (payloadlen + icvlen + padlen) - icvlen + padlen */
3013 + append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4);
3014 +
3015 + /*
3016 + * Start a new input sequence using the SEQ OUT PTR command options,
3017 + * pointer and length used when the current output sequence was defined.
3018 + */
3019 + if (ps) {
3020 + /*
3021 + * Move the lower 32 bits of Shared Descriptor address, the
3022 + * SEQ OUT PTR command, Output Pointer (2 words) and
3023 + * Output Length into math registers.
3024 + */
3025 + if (caam_little_end)
3026 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF |
3027 + MOVE_DEST_MATH0 |
3028 + (55 * 4 << MOVE_OFFSET_SHIFT) | 20);
3029 + else
3030 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF |
3031 + MOVE_DEST_MATH0 |
3032 + (54 * 4 << MOVE_OFFSET_SHIFT) | 20);
3033 +
3034 + /* Transform SEQ OUT PTR command in SEQ IN PTR command */
3035 + append_math_and_imm_u32(desc, REG0, REG0, IMM,
3036 + ~(CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR));
3037 + /* Append a JUMP command after the copied fields */
3038 + jumpback = CMD_JUMP | (char)-9;
3039 + append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM |
3040 + LDST_SRCDST_WORD_DECO_MATH2 |
3041 + (4 << LDST_OFFSET_SHIFT));
3042 + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1);
3043 + /* Move the updated fields back to the Job Descriptor */
3044 + if (caam_little_end)
3045 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 |
3046 + MOVE_DEST_DESCBUF |
3047 + (55 * 4 << MOVE_OFFSET_SHIFT) | 24);
3048 + else
3049 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 |
3050 + MOVE_DEST_DESCBUF |
3051 + (54 * 4 << MOVE_OFFSET_SHIFT) | 24);
3052 +
3053 + /*
3054 + * Read the new SEQ IN PTR command, Input Pointer, Input Length
3055 + * and then jump back to the next command from the
3056 + * Shared Descriptor.
3057 + */
3058 + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 6);
3059 + } else {
3060 + /*
3061 + * Move the SEQ OUT PTR command, Output Pointer (1 word) and
3062 + * Output Length into math registers.
3063 + */
3064 + if (caam_little_end)
3065 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF |
3066 + MOVE_DEST_MATH0 |
3067 + (54 * 4 << MOVE_OFFSET_SHIFT) | 12);
3068 + else
3069 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF |
3070 + MOVE_DEST_MATH0 |
3071 + (53 * 4 << MOVE_OFFSET_SHIFT) | 12);
3072 +
3073 + /* Transform SEQ OUT PTR command in SEQ IN PTR command */
3074 + append_math_and_imm_u64(desc, REG0, REG0, IMM,
3075 + ~(((u64)(CMD_SEQ_IN_PTR ^
3076 + CMD_SEQ_OUT_PTR)) << 32));
3077 + /* Append a JUMP command after the copied fields */
3078 + jumpback = CMD_JUMP | (char)-7;
3079 + append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM |
3080 + LDST_SRCDST_WORD_DECO_MATH1 |
3081 + (4 << LDST_OFFSET_SHIFT));
3082 + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1);
3083 + /* Move the updated fields back to the Job Descriptor */
3084 + if (caam_little_end)
3085 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 |
3086 + MOVE_DEST_DESCBUF |
3087 + (54 * 4 << MOVE_OFFSET_SHIFT) | 16);
3088 + else
3089 + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 |
3090 + MOVE_DEST_DESCBUF |
3091 + (53 * 4 << MOVE_OFFSET_SHIFT) | 16);
3092 +
3093 + /*
3094 + * Read the new SEQ IN PTR command, Input Pointer, Input Length
3095 + * and then jump back to the next command from the
3096 + * Shared Descriptor.
3097 + */
3098 + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 5);
3099 + }
3100 +
3101 + /* skip payload */
3102 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | FIFOLDST_VLF);
3103 + /* check icv */
3104 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_ICV |
3105 + FIFOLD_TYPE_LAST2 | authsize);
3106 +
3107 +#ifdef DEBUG
3108 + print_hex_dump(KERN_ERR, "tls dec shdesc@" __stringify(__LINE__) ": ",
3109 + DUMP_PREFIX_ADDRESS, 16, 4, desc,
3110 + desc_bytes(desc), 1);
3111 +#endif
3112 +}
3113 +EXPORT_SYMBOL(cnstr_shdsc_tls_decap);
3114 +
3115 +/**
3116 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
3117 * @desc: pointer to buffer used for descriptor construction
3118 * @cdata: pointer to block cipher transform definitions
3119 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3120 + * @ivsize: initialization vector size
3121 * @icvsize: integrity check value (ICV) size (truncated or full)
3122 + * @is_qi: true when called from caam/qi
3123 */
3124 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
3125 - unsigned int icvsize)
3126 + unsigned int ivsize, unsigned int icvsize,
3127 + const bool is_qi)
3128 {
3129 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
3130 *zero_assoc_jump_cmd2;
3131 @@ -612,11 +1067,35 @@ void cnstr_shdsc_gcm_encap(u32 * const d
3132 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3133 OP_ALG_ENCRYPT);
3134
3135 + if (is_qi) {
3136 + u32 *wait_load_cmd;
3137 +
3138 + /* REG3 = assoclen */
3139 + append_seq_load(desc, 4, LDST_CLASS_DECO |
3140 + LDST_SRCDST_WORD_DECO_MATH3 |
3141 + (4 << LDST_OFFSET_SHIFT));
3142 +
3143 + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3144 + JUMP_COND_CALM | JUMP_COND_NCP |
3145 + JUMP_COND_NOP | JUMP_COND_NIP |
3146 + JUMP_COND_NIFP);
3147 + set_jump_tgt_here(desc, wait_load_cmd);
3148 +
3149 + append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
3150 + ivsize);
3151 + } else {
3152 + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
3153 + CAAM_CMD_SZ);
3154 + }
3155 +
3156 /* if assoclen + cryptlen is ZERO, skip to ICV write */
3157 - append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
3158 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
3159 JUMP_COND_MATH_Z);
3160
3161 + if (is_qi)
3162 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3163 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3164 +
3165 /* if assoclen is ZERO, skip reading the assoc data */
3166 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
3167 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
3168 @@ -648,8 +1127,11 @@ void cnstr_shdsc_gcm_encap(u32 * const d
3169 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
3170 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
3171
3172 - /* jump the zero-payload commands */
3173 - append_jump(desc, JUMP_TEST_ALL | 2);
3174 + /* jump to ICV writing */
3175 + if (is_qi)
3176 + append_jump(desc, JUMP_TEST_ALL | 4);
3177 + else
3178 + append_jump(desc, JUMP_TEST_ALL | 2);
3179
3180 /* zero-payload commands */
3181 set_jump_tgt_here(desc, zero_payload_jump_cmd);
3182 @@ -657,10 +1139,18 @@ void cnstr_shdsc_gcm_encap(u32 * const d
3183 /* read assoc data */
3184 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
3185 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
3186 + if (is_qi)
3187 + /* jump to ICV writing */
3188 + append_jump(desc, JUMP_TEST_ALL | 2);
3189
3190 /* There is no input data */
3191 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
3192
3193 + if (is_qi)
3194 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3195 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
3196 + FIFOLD_TYPE_LAST1);
3197 +
3198 /* write ICV */
3199 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
3200 LDST_SRCDST_BYTE_CONTEXT);
3201 @@ -677,10 +1167,13 @@ EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
3202 * @desc: pointer to buffer used for descriptor construction
3203 * @cdata: pointer to block cipher transform definitions
3204 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3205 + * @ivsize: initialization vector size
3206 * @icvsize: integrity check value (ICV) size (truncated or full)
3207 + * @is_qi: true when called from caam/qi
3208 */
3209 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
3210 - unsigned int icvsize)
3211 + unsigned int ivsize, unsigned int icvsize,
3212 + const bool is_qi)
3213 {
3214 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
3215
3216 @@ -701,6 +1194,24 @@ void cnstr_shdsc_gcm_decap(u32 * const d
3217 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3218 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
3219
3220 + if (is_qi) {
3221 + u32 *wait_load_cmd;
3222 +
3223 + /* REG3 = assoclen */
3224 + append_seq_load(desc, 4, LDST_CLASS_DECO |
3225 + LDST_SRCDST_WORD_DECO_MATH3 |
3226 + (4 << LDST_OFFSET_SHIFT));
3227 +
3228 + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3229 + JUMP_COND_CALM | JUMP_COND_NCP |
3230 + JUMP_COND_NOP | JUMP_COND_NIP |
3231 + JUMP_COND_NIFP);
3232 + set_jump_tgt_here(desc, wait_load_cmd);
3233 +
3234 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3235 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3236 + }
3237 +
3238 /* if assoclen is ZERO, skip reading the assoc data */
3239 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
3240 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
3241 @@ -753,10 +1264,13 @@ EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
3242 * @desc: pointer to buffer used for descriptor construction
3243 * @cdata: pointer to block cipher transform definitions
3244 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3245 + * @ivsize: initialization vector size
3246 * @icvsize: integrity check value (ICV) size (truncated or full)
3247 + * @is_qi: true when called from caam/qi
3248 */
3249 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
3250 - unsigned int icvsize)
3251 + unsigned int ivsize, unsigned int icvsize,
3252 + const bool is_qi)
3253 {
3254 u32 *key_jump_cmd;
3255
3256 @@ -777,7 +1291,29 @@ void cnstr_shdsc_rfc4106_encap(u32 * con
3257 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3258 OP_ALG_ENCRYPT);
3259
3260 - append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
3261 + if (is_qi) {
3262 + u32 *wait_load_cmd;
3263 +
3264 + /* REG3 = assoclen */
3265 + append_seq_load(desc, 4, LDST_CLASS_DECO |
3266 + LDST_SRCDST_WORD_DECO_MATH3 |
3267 + (4 << LDST_OFFSET_SHIFT));
3268 +
3269 + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3270 + JUMP_COND_CALM | JUMP_COND_NCP |
3271 + JUMP_COND_NOP | JUMP_COND_NIP |
3272 + JUMP_COND_NIFP);
3273 + set_jump_tgt_here(desc, wait_load_cmd);
3274 +
3275 + /* Read salt and IV */
3276 + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
3277 + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
3278 + FIFOLD_TYPE_IV);
3279 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3280 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3281 + }
3282 +
3283 + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
3284 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
3285
3286 /* Read assoc data */
3287 @@ -785,7 +1321,7 @@ void cnstr_shdsc_rfc4106_encap(u32 * con
3288 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
3289
3290 /* Skip IV */
3291 - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
3292 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
3293
3294 /* Will read cryptlen bytes */
3295 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
3296 @@ -824,10 +1360,13 @@ EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap)
3297 * @desc: pointer to buffer used for descriptor construction
3298 * @cdata: pointer to block cipher transform definitions
3299 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3300 + * @ivsize: initialization vector size
3301 * @icvsize: integrity check value (ICV) size (truncated or full)
3302 + * @is_qi: true when called from caam/qi
3303 */
3304 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
3305 - unsigned int icvsize)
3306 + unsigned int ivsize, unsigned int icvsize,
3307 + const bool is_qi)
3308 {
3309 u32 *key_jump_cmd;
3310
3311 @@ -849,7 +1388,29 @@ void cnstr_shdsc_rfc4106_decap(u32 * con
3312 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3313 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
3314
3315 - append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
3316 + if (is_qi) {
3317 + u32 *wait_load_cmd;
3318 +
3319 + /* REG3 = assoclen */
3320 + append_seq_load(desc, 4, LDST_CLASS_DECO |
3321 + LDST_SRCDST_WORD_DECO_MATH3 |
3322 + (4 << LDST_OFFSET_SHIFT));
3323 +
3324 + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3325 + JUMP_COND_CALM | JUMP_COND_NCP |
3326 + JUMP_COND_NOP | JUMP_COND_NIP |
3327 + JUMP_COND_NIFP);
3328 + set_jump_tgt_here(desc, wait_load_cmd);
3329 +
3330 + /* Read salt and IV */
3331 + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
3332 + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
3333 + FIFOLD_TYPE_IV);
3334 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3335 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3336 + }
3337 +
3338 + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
3339 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
3340
3341 /* Read assoc data */
3342 @@ -857,7 +1418,7 @@ void cnstr_shdsc_rfc4106_decap(u32 * con
3343 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
3344
3345 /* Skip IV */
3346 - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
3347 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
3348
3349 /* Will read cryptlen bytes */
3350 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
3351 @@ -896,10 +1457,13 @@ EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap)
3352 * @desc: pointer to buffer used for descriptor construction
3353 * @cdata: pointer to block cipher transform definitions
3354 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3355 + * @ivsize: initialization vector size
3356 * @icvsize: integrity check value (ICV) size (truncated or full)
3357 + * @is_qi: true when called from caam/qi
3358 */
3359 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
3360 - unsigned int icvsize)
3361 + unsigned int ivsize, unsigned int icvsize,
3362 + const bool is_qi)
3363 {
3364 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
3365
3366 @@ -920,6 +1484,18 @@ void cnstr_shdsc_rfc4543_encap(u32 * con
3367 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3368 OP_ALG_ENCRYPT);
3369
3370 + if (is_qi) {
3371 + /* assoclen is not needed, skip it */
3372 + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
3373 +
3374 + /* Read salt and IV */
3375 + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
3376 + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
3377 + FIFOLD_TYPE_IV);
3378 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3379 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3380 + }
3381 +
3382 /* assoclen + cryptlen = seqinlen */
3383 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
3384
3385 @@ -931,7 +1507,7 @@ void cnstr_shdsc_rfc4543_encap(u32 * con
3386 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
3387 (0x6 << MOVE_LEN_SHIFT));
3388 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
3389 - (0x8 << MOVE_LEN_SHIFT));
3390 + (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
3391
3392 /* Will read assoclen + cryptlen bytes */
3393 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
3394 @@ -966,10 +1542,13 @@ EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap)
3395 * @desc: pointer to buffer used for descriptor construction
3396 * @cdata: pointer to block cipher transform definitions
3397 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
3398 + * @ivsize: initialization vector size
3399 * @icvsize: integrity check value (ICV) size (truncated or full)
3400 + * @is_qi: true when called from caam/qi
3401 */
3402 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
3403 - unsigned int icvsize)
3404 + unsigned int ivsize, unsigned int icvsize,
3405 + const bool is_qi)
3406 {
3407 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
3408
3409 @@ -990,6 +1569,18 @@ void cnstr_shdsc_rfc4543_decap(u32 * con
3410 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3411 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
3412
3413 + if (is_qi) {
3414 + /* assoclen is not needed, skip it */
3415 + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
3416 +
3417 + /* Read salt and IV */
3418 + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
3419 + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
3420 + FIFOLD_TYPE_IV);
3421 + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
3422 + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
3423 + }
3424 +
3425 /* assoclen + cryptlen = seqoutlen */
3426 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
3427
3428 @@ -1001,7 +1592,7 @@ void cnstr_shdsc_rfc4543_decap(u32 * con
3429 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
3430 (0x6 << MOVE_LEN_SHIFT));
3431 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
3432 - (0x8 << MOVE_LEN_SHIFT));
3433 + (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
3434
3435 /* Will read assoclen + cryptlen bytes */
3436 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
3437 @@ -1035,6 +1626,138 @@ void cnstr_shdsc_rfc4543_decap(u32 * con
3438 }
3439 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
3440
3441 +/**
3442 + * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
3443 + * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
3444 + * descriptor (non-protocol).
3445 + * @desc: pointer to buffer used for descriptor construction
3446 + * @cdata: pointer to block cipher transform definitions
3447 + * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
3448 + * OP_ALG_AAI_AEAD.
3449 + * @adata: pointer to authentication transform definitions
3450 + * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
3451 + * OP_ALG_AAI_AEAD.
3452 + * @ivsize: initialization vector size
3453 + * @icvsize: integrity check value (ICV) size (truncated or full)
3454 + * @encap: true if encapsulation, false if decapsulation
3455 + * @is_qi: true when called from caam/qi
3456 + */
3457 +void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
3458 + struct alginfo *adata, unsigned int ivsize,
3459 + unsigned int icvsize, const bool encap,
3460 + const bool is_qi)
3461 +{
3462 + u32 *key_jump_cmd, *wait_cmd;
3463 + u32 nfifo;
3464 + const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
3465 +
3466 + /* Note: Context registers are saved. */
3467 + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
3468 +
3469 + /* skip key loading if they are loaded due to sharing */
3470 + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3471 + JUMP_COND_SHRD);
3472 +
3473 + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
3474 + CLASS_1 | KEY_DEST_CLASS_REG);
3475 +
3476 + /* For IPsec load the salt from keymat in the context register */
3477 + if (is_ipsec)
3478 + append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
3479 + LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
3480 + 4 << LDST_OFFSET_SHIFT);
3481 +
3482 + set_jump_tgt_here(desc, key_jump_cmd);
3483 +
3484 + /* Class 2 and 1 operations: Poly & ChaCha */
3485 + if (encap) {
3486 + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
3487 + OP_ALG_ENCRYPT);
3488 + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3489 + OP_ALG_ENCRYPT);
3490 + } else {
3491 + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
3492 + OP_ALG_DECRYPT | OP_ALG_ICV_ON);
3493 + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
3494 + OP_ALG_DECRYPT);
3495 + }
3496 +
3497 + if (is_qi) {
3498 + u32 *wait_load_cmd;
3499 + u32 ctx1_iv_off = is_ipsec ? 8 : 4;
3500 +
3501 + /* REG3 = assoclen */
3502 + append_seq_load(desc, 4, LDST_CLASS_DECO |
3503 + LDST_SRCDST_WORD_DECO_MATH3 |
3504 + 4 << LDST_OFFSET_SHIFT);
3505 +
3506 + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
3507 + JUMP_COND_CALM | JUMP_COND_NCP |
3508 + JUMP_COND_NOP | JUMP_COND_NIP |
3509 + JUMP_COND_NIFP);
3510 + set_jump_tgt_here(desc, wait_load_cmd);
3511 +
3512 + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
3513 + LDST_SRCDST_BYTE_CONTEXT |
3514 + ctx1_iv_off << LDST_OFFSET_SHIFT);
3515 + }
3516 +
3517 + /*
3518 + * MAGIC with NFIFO
3519 + * Read associated data from the input and send them to class1 and
3520 + * class2 alignment blocks. From class1 send data to output fifo and
3521 + * then write it to memory since we don't need to encrypt AD.
3522 + */
3523 + nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
3524 + NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
3525 + append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
3526 + LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
3527 +
3528 + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
3529 + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
3530 + append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
3531 + FIFOLD_CLASS_CLASS1 | LDST_VLF);
3532 + append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
3533 + MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
3534 + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
3535 +
3536 + /* IPsec - copy IV at the output */
3537 + if (is_ipsec)
3538 + append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
3539 + 0x2 << 25);
3540 +
3541 + wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
3542 + JUMP_COND_NOP | JUMP_TEST_ALL);
3543 + set_jump_tgt_here(desc, wait_cmd);
3544 +
3545 + if (encap) {
3546 + /* Read and write cryptlen bytes */
3547 + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
3548 + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
3549 + CAAM_CMD_SZ);
3550 + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
3551 +
3552 + /* Write ICV */
3553 + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
3554 + LDST_SRCDST_BYTE_CONTEXT);
3555 + } else {
3556 + /* Read and write cryptlen bytes */
3557 + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
3558 + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
3559 + CAAM_CMD_SZ);
3560 + aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
3561 +
3562 + /* Load ICV for verification */
3563 + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
3564 + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
3565 + }
3566 +
3567 + print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
3568 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3569 + 1);
3570 +}
3571 +EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
3572 +
3573 /*
3574 * For ablkcipher encrypt and decrypt, read from req->src and
3575 * write to req->dst
3576 @@ -1053,7 +1776,8 @@ static inline void ablkcipher_append_src
3577 * @desc: pointer to buffer used for descriptor construction
3578 * @cdata: pointer to block cipher transform definitions
3579 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
3580 - * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
3581 + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
3582 + * - OP_ALG_ALGSEL_CHACHA20
3583 * @ivsize: initialization vector size
3584 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
3585 * @ctx1_iv_off: IV offset in CONTEXT1 register
3586 @@ -1075,7 +1799,7 @@ void cnstr_shdsc_ablkcipher_encap(u32 *
3587
3588 /* Load nonce into CONTEXT1 reg */
3589 if (is_rfc3686) {
3590 - u8 *nonce = cdata->key_virt + cdata->keylen;
3591 + const u8 *nonce = cdata->key_virt + cdata->keylen;
3592
3593 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
3594 LDST_CLASS_IND_CCB |
3595 @@ -1118,7 +1842,8 @@ EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_enc
3596 * @desc: pointer to buffer used for descriptor construction
3597 * @cdata: pointer to block cipher transform definitions
3598 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
3599 - * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
3600 + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
3601 + * - OP_ALG_ALGSEL_CHACHA20
3602 * @ivsize: initialization vector size
3603 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
3604 * @ctx1_iv_off: IV offset in CONTEXT1 register
3605 @@ -1140,7 +1865,7 @@ void cnstr_shdsc_ablkcipher_decap(u32 *
3606
3607 /* Load nonce into CONTEXT1 reg */
3608 if (is_rfc3686) {
3609 - u8 *nonce = cdata->key_virt + cdata->keylen;
3610 + const u8 *nonce = cdata->key_virt + cdata->keylen;
3611
3612 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
3613 LDST_CLASS_IND_CCB |
3614 @@ -1209,7 +1934,7 @@ void cnstr_shdsc_ablkcipher_givencap(u32
3615
3616 /* Load Nonce into CONTEXT1 reg */
3617 if (is_rfc3686) {
3618 - u8 *nonce = cdata->key_virt + cdata->keylen;
3619 + const u8 *nonce = cdata->key_virt + cdata->keylen;
3620
3621 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
3622 LDST_CLASS_IND_CCB |
3623 --- a/drivers/crypto/caam/caamalg_desc.h
3624 +++ b/drivers/crypto/caam/caamalg_desc.h
3625 @@ -17,6 +17,9 @@
3626 #define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ)
3627 #define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ)
3628
3629 +#define DESC_TLS_BASE (4 * CAAM_CMD_SZ)
3630 +#define DESC_TLS10_ENC_LEN (DESC_TLS_BASE + 29 * CAAM_CMD_SZ)
3631 +
3632 /* Note: Nonce is counted in cdata.keylen */
3633 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ)
3634
3635 @@ -27,14 +30,20 @@
3636 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
3637 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
3638 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
3639 +#define DESC_QI_GCM_ENC_LEN (DESC_GCM_ENC_LEN + 6 * CAAM_CMD_SZ)
3640 +#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ)
3641
3642 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
3643 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
3644 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
3645 +#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ)
3646 +#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ)
3647
3648 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
3649 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
3650 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
3651 +#define DESC_QI_RFC4543_ENC_LEN (DESC_RFC4543_ENC_LEN + 4 * CAAM_CMD_SZ)
3652 +#define DESC_QI_RFC4543_DEC_LEN (DESC_RFC4543_DEC_LEN + 4 * CAAM_CMD_SZ)
3653
3654 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
3655 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
3656 @@ -43,46 +52,67 @@
3657 15 * CAAM_CMD_SZ)
3658
3659 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
3660 - unsigned int icvsize);
3661 + unsigned int icvsize, int era);
3662
3663 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
3664 - unsigned int icvsize);
3665 + unsigned int icvsize, int era);
3666
3667 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
3668 struct alginfo *adata, unsigned int ivsize,
3669 unsigned int icvsize, const bool is_rfc3686,
3670 u32 *nonce, const u32 ctx1_iv_off,
3671 - const bool is_qi);
3672 + const bool is_qi, int era);
3673
3674 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
3675 struct alginfo *adata, unsigned int ivsize,
3676 unsigned int icvsize, const bool geniv,
3677 const bool is_rfc3686, u32 *nonce,
3678 - const u32 ctx1_iv_off, const bool is_qi);
3679 + const u32 ctx1_iv_off, const bool is_qi, int era);
3680
3681 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
3682 struct alginfo *adata, unsigned int ivsize,
3683 unsigned int icvsize, const bool is_rfc3686,
3684 u32 *nonce, const u32 ctx1_iv_off,
3685 - const bool is_qi);
3686 + const bool is_qi, int era);
3687 +
3688 +void cnstr_shdsc_tls_encap(u32 *const desc, struct alginfo *cdata,
3689 + struct alginfo *adata, unsigned int assoclen,
3690 + unsigned int ivsize, unsigned int authsize,
3691 + unsigned int blocksize, int era);
3692 +
3693 +void cnstr_shdsc_tls_decap(u32 *const desc, struct alginfo *cdata,
3694 + struct alginfo *adata, unsigned int assoclen,
3695 + unsigned int ivsize, unsigned int authsize,
3696 + unsigned int blocksize, int era);
3697
3698 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
3699 - unsigned int icvsize);
3700 + unsigned int ivsize, unsigned int icvsize,
3701 + const bool is_qi);
3702
3703 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
3704 - unsigned int icvsize);
3705 + unsigned int ivsize, unsigned int icvsize,
3706 + const bool is_qi);
3707
3708 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
3709 - unsigned int icvsize);
3710 + unsigned int ivsize, unsigned int icvsize,
3711 + const bool is_qi);
3712
3713 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
3714 - unsigned int icvsize);
3715 + unsigned int ivsize, unsigned int icvsize,
3716 + const bool is_qi);
3717
3718 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
3719 - unsigned int icvsize);
3720 + unsigned int ivsize, unsigned int icvsize,
3721 + const bool is_qi);
3722
3723 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
3724 - unsigned int icvsize);
3725 + unsigned int ivsize, unsigned int icvsize,
3726 + const bool is_qi);
3727 +
3728 +void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
3729 + struct alginfo *adata, unsigned int ivsize,
3730 + unsigned int icvsize, const bool encap,
3731 + const bool is_qi);
3732
3733 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
3734 unsigned int ivsize, const bool is_rfc3686,
3735 --- a/drivers/crypto/caam/caamalg_qi.c
3736 +++ b/drivers/crypto/caam/caamalg_qi.c
3737 @@ -7,7 +7,7 @@
3738 */
3739
3740 #include "compat.h"
3741 -
3742 +#include "ctrl.h"
3743 #include "regs.h"
3744 #include "intern.h"
3745 #include "desc_constr.h"
3746 @@ -53,6 +53,7 @@ struct caam_ctx {
3747 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
3748 u8 key[CAAM_MAX_KEY_SIZE];
3749 dma_addr_t key_dma;
3750 + enum dma_data_direction dir;
3751 struct alginfo adata;
3752 struct alginfo cdata;
3753 unsigned int authsize;
3754 @@ -74,6 +75,7 @@ static int aead_set_sh_desc(struct crypt
3755 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
3756 OP_ALG_AAI_CTR_MOD128);
3757 const bool is_rfc3686 = alg->caam.rfc3686;
3758 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
3759
3760 if (!ctx->cdata.keylen || !ctx->authsize)
3761 return 0;
3762 @@ -124,7 +126,7 @@ static int aead_set_sh_desc(struct crypt
3763
3764 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
3765 ivsize, ctx->authsize, is_rfc3686, nonce,
3766 - ctx1_iv_off, true);
3767 + ctx1_iv_off, true, ctrlpriv->era);
3768
3769 skip_enc:
3770 /* aead_decrypt shared descriptor */
3771 @@ -149,7 +151,8 @@ skip_enc:
3772
3773 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
3774 ivsize, ctx->authsize, alg->caam.geniv,
3775 - is_rfc3686, nonce, ctx1_iv_off, true);
3776 + is_rfc3686, nonce, ctx1_iv_off, true,
3777 + ctrlpriv->era);
3778
3779 if (!alg->caam.geniv)
3780 goto skip_givenc;
3781 @@ -176,7 +179,7 @@ skip_enc:
3782
3783 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
3784 ivsize, ctx->authsize, is_rfc3686, nonce,
3785 - ctx1_iv_off, true);
3786 + ctx1_iv_off, true, ctrlpriv->era);
3787
3788 skip_givenc:
3789 return 0;
3790 @@ -197,6 +200,7 @@ static int aead_setkey(struct crypto_aea
3791 {
3792 struct caam_ctx *ctx = crypto_aead_ctx(aead);
3793 struct device *jrdev = ctx->jrdev;
3794 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
3795 struct crypto_authenc_keys keys;
3796 int ret = 0;
3797
3798 @@ -211,6 +215,27 @@ static int aead_setkey(struct crypto_aea
3799 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
3800 #endif
3801
3802 + /*
3803 + * If DKP is supported, use it in the shared descriptor to generate
3804 + * the split key.
3805 + */
3806 + if (ctrlpriv->era >= 6) {
3807 + ctx->adata.keylen = keys.authkeylen;
3808 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3809 + OP_ALG_ALGSEL_MASK);
3810 +
3811 + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
3812 + goto badkey;
3813 +
3814 + memcpy(ctx->key, keys.authkey, keys.authkeylen);
3815 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
3816 + keys.enckeylen);
3817 + dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
3818 + ctx->adata.keylen_pad +
3819 + keys.enckeylen, ctx->dir);
3820 + goto skip_split_key;
3821 + }
3822 +
3823 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
3824 keys.authkeylen, CAAM_MAX_KEY_SIZE -
3825 keys.enckeylen);
3826 @@ -220,13 +245,14 @@ static int aead_setkey(struct crypto_aea
3827 /* postpend encryption key to auth split key */
3828 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
3829 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
3830 - keys.enckeylen, DMA_TO_DEVICE);
3831 + keys.enckeylen, ctx->dir);
3832 #ifdef DEBUG
3833 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
3834 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
3835 ctx->adata.keylen_pad + keys.enckeylen, 1);
3836 #endif
3837
3838 +skip_split_key:
3839 ctx->cdata.keylen = keys.enckeylen;
3840
3841 ret = aead_set_sh_desc(aead);
3842 @@ -258,6 +284,468 @@ badkey:
3843 return -EINVAL;
3844 }
3845
3846 +static int tls_set_sh_desc(struct crypto_aead *tls)
3847 +{
3848 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
3849 + unsigned int ivsize = crypto_aead_ivsize(tls);
3850 + unsigned int blocksize = crypto_aead_blocksize(tls);
3851 + unsigned int assoclen = 13; /* always 13 bytes for TLS */
3852 + unsigned int data_len[2];
3853 + u32 inl_mask;
3854 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
3855 +
3856 + if (!ctx->cdata.keylen || !ctx->authsize)
3857 + return 0;
3858 +
3859 + /*
3860 + * TLS 1.0 encrypt shared descriptor
3861 + * Job Descriptor and Shared Descriptor
3862 + * must fit into the 64-word Descriptor h/w Buffer
3863 + */
3864 + data_len[0] = ctx->adata.keylen_pad;
3865 + data_len[1] = ctx->cdata.keylen;
3866 +
3867 + if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len,
3868 + &inl_mask, ARRAY_SIZE(data_len)) < 0)
3869 + return -EINVAL;
3870 +
3871 + if (inl_mask & 1)
3872 + ctx->adata.key_virt = ctx->key;
3873 + else
3874 + ctx->adata.key_dma = ctx->key_dma;
3875 +
3876 + if (inl_mask & 2)
3877 + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
3878 + else
3879 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
3880 +
3881 + ctx->adata.key_inline = !!(inl_mask & 1);
3882 + ctx->cdata.key_inline = !!(inl_mask & 2);
3883 +
3884 + cnstr_shdsc_tls_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
3885 + assoclen, ivsize, ctx->authsize, blocksize,
3886 + ctrlpriv->era);
3887 +
3888 + /*
3889 + * TLS 1.0 decrypt shared descriptor
3890 + * Keys do not fit inline, regardless of algorithms used
3891 + */
3892 + ctx->adata.key_inline = false;
3893 + ctx->adata.key_dma = ctx->key_dma;
3894 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
3895 +
3896 + cnstr_shdsc_tls_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
3897 + assoclen, ivsize, ctx->authsize, blocksize,
3898 + ctrlpriv->era);
3899 +
3900 + return 0;
3901 +}
3902 +
3903 +static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize)
3904 +{
3905 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
3906 +
3907 + ctx->authsize = authsize;
3908 + tls_set_sh_desc(tls);
3909 +
3910 + return 0;
3911 +}
3912 +
3913 +static int tls_setkey(struct crypto_aead *tls, const u8 *key,
3914 + unsigned int keylen)
3915 +{
3916 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
3917 + struct device *jrdev = ctx->jrdev;
3918 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
3919 + struct crypto_authenc_keys keys;
3920 + int ret = 0;
3921 +
3922 + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3923 + goto badkey;
3924 +
3925 +#ifdef DEBUG
3926 + dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
3927 + keys.authkeylen + keys.enckeylen, keys.enckeylen,
3928 + keys.authkeylen);
3929 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
3930 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
3931 +#endif
3932 +
3933 + /*
3934 + * If DKP is supported, use it in the shared descriptor to generate
3935 + * the split key.
3936 + */
3937 + if (ctrlpriv->era >= 6) {
3938 + ctx->adata.keylen = keys.authkeylen;
3939 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3940 + OP_ALG_ALGSEL_MASK);
3941 +
3942 + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
3943 + goto badkey;
3944 +
3945 + memcpy(ctx->key, keys.authkey, keys.authkeylen);
3946 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
3947 + keys.enckeylen);
3948 + dma_sync_single_for_device(jrdev, ctx->key_dma,
3949 + ctx->adata.keylen_pad +
3950 + keys.enckeylen, ctx->dir);
3951 + goto skip_split_key;
3952 + }
3953 +
3954 + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
3955 + keys.authkeylen, CAAM_MAX_KEY_SIZE -
3956 + keys.enckeylen);
3957 + if (ret)
3958 + goto badkey;
3959 +
3960 + /* postpend encryption key to auth split key */
3961 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
3962 + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
3963 + keys.enckeylen, ctx->dir);
3964 +
3965 +#ifdef DEBUG
3966 + dev_err(jrdev, "split keylen %d split keylen padded %d\n",
3967 + ctx->adata.keylen, ctx->adata.keylen_pad);
3968 + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
3969 + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
3970 + ctx->adata.keylen_pad + keys.enckeylen, 1);
3971 +#endif
3972 +
3973 +skip_split_key:
3974 + ctx->cdata.keylen = keys.enckeylen;
3975 +
3976 + ret = tls_set_sh_desc(tls);
3977 + if (ret)
3978 + goto badkey;
3979 +
3980 + /* Now update the driver contexts with the new shared descriptor */
3981 + if (ctx->drv_ctx[ENCRYPT]) {
3982 + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
3983 + ctx->sh_desc_enc);
3984 + if (ret) {
3985 + dev_err(jrdev, "driver enc context update failed\n");
3986 + goto badkey;
3987 + }
3988 + }
3989 +
3990 + if (ctx->drv_ctx[DECRYPT]) {
3991 + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
3992 + ctx->sh_desc_dec);
3993 + if (ret) {
3994 + dev_err(jrdev, "driver dec context update failed\n");
3995 + goto badkey;
3996 + }
3997 + }
3998 +
3999 + return ret;
4000 +badkey:
4001 + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN);
4002 + return -EINVAL;
4003 +}
4004 +
4005 +static int gcm_set_sh_desc(struct crypto_aead *aead)
4006 +{
4007 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4008 + unsigned int ivsize = crypto_aead_ivsize(aead);
4009 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
4010 + ctx->cdata.keylen;
4011 +
4012 + if (!ctx->cdata.keylen || !ctx->authsize)
4013 + return 0;
4014 +
4015 + /*
4016 + * Job Descriptor and Shared Descriptor
4017 + * must fit into the 64-word Descriptor h/w Buffer
4018 + */
4019 + if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
4020 + ctx->cdata.key_inline = true;
4021 + ctx->cdata.key_virt = ctx->key;
4022 + } else {
4023 + ctx->cdata.key_inline = false;
4024 + ctx->cdata.key_dma = ctx->key_dma;
4025 + }
4026 +
4027 + cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
4028 + ctx->authsize, true);
4029 +
4030 + /*
4031 + * Job Descriptor and Shared Descriptor
4032 + * must fit into the 64-word Descriptor h/w Buffer
4033 + */
4034 + if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
4035 + ctx->cdata.key_inline = true;
4036 + ctx->cdata.key_virt = ctx->key;
4037 + } else {
4038 + ctx->cdata.key_inline = false;
4039 + ctx->cdata.key_dma = ctx->key_dma;
4040 + }
4041 +
4042 + cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
4043 + ctx->authsize, true);
4044 +
4045 + return 0;
4046 +}
4047 +
4048 +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
4049 +{
4050 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
4051 +
4052 + ctx->authsize = authsize;
4053 + gcm_set_sh_desc(authenc);
4054 +
4055 + return 0;
4056 +}
4057 +
4058 +static int gcm_setkey(struct crypto_aead *aead,
4059 + const u8 *key, unsigned int keylen)
4060 +{
4061 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4062 + struct device *jrdev = ctx->jrdev;
4063 + int ret;
4064 +
4065 +#ifdef DEBUG
4066 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
4067 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
4068 +#endif
4069 +
4070 + memcpy(ctx->key, key, keylen);
4071 + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
4072 + ctx->cdata.keylen = keylen;
4073 +
4074 + ret = gcm_set_sh_desc(aead);
4075 + if (ret)
4076 + return ret;
4077 +
4078 + /* Now update the driver contexts with the new shared descriptor */
4079 + if (ctx->drv_ctx[ENCRYPT]) {
4080 + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
4081 + ctx->sh_desc_enc);
4082 + if (ret) {
4083 + dev_err(jrdev, "driver enc context update failed\n");
4084 + return ret;
4085 + }
4086 + }
4087 +
4088 + if (ctx->drv_ctx[DECRYPT]) {
4089 + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
4090 + ctx->sh_desc_dec);
4091 + if (ret) {
4092 + dev_err(jrdev, "driver dec context update failed\n");
4093 + return ret;
4094 + }
4095 + }
4096 +
4097 + return 0;
4098 +}
4099 +
4100 +static int rfc4106_set_sh_desc(struct crypto_aead *aead)
4101 +{
4102 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4103 + unsigned int ivsize = crypto_aead_ivsize(aead);
4104 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
4105 + ctx->cdata.keylen;
4106 +
4107 + if (!ctx->cdata.keylen || !ctx->authsize)
4108 + return 0;
4109 +
4110 + ctx->cdata.key_virt = ctx->key;
4111 +
4112 + /*
4113 + * Job Descriptor and Shared Descriptor
4114 + * must fit into the 64-word Descriptor h/w Buffer
4115 + */
4116 + if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
4117 + ctx->cdata.key_inline = true;
4118 + } else {
4119 + ctx->cdata.key_inline = false;
4120 + ctx->cdata.key_dma = ctx->key_dma;
4121 + }
4122 +
4123 + cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
4124 + ctx->authsize, true);
4125 +
4126 + /*
4127 + * Job Descriptor and Shared Descriptor
4128 + * must fit into the 64-word Descriptor h/w Buffer
4129 + */
4130 + if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
4131 + ctx->cdata.key_inline = true;
4132 + } else {
4133 + ctx->cdata.key_inline = false;
4134 + ctx->cdata.key_dma = ctx->key_dma;
4135 + }
4136 +
4137 + cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
4138 + ctx->authsize, true);
4139 +
4140 + return 0;
4141 +}
4142 +
4143 +static int rfc4106_setauthsize(struct crypto_aead *authenc,
4144 + unsigned int authsize)
4145 +{
4146 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
4147 +
4148 + ctx->authsize = authsize;
4149 + rfc4106_set_sh_desc(authenc);
4150 +
4151 + return 0;
4152 +}
4153 +
4154 +static int rfc4106_setkey(struct crypto_aead *aead,
4155 + const u8 *key, unsigned int keylen)
4156 +{
4157 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4158 + struct device *jrdev = ctx->jrdev;
4159 + int ret;
4160 +
4161 + if (keylen < 4)
4162 + return -EINVAL;
4163 +
4164 +#ifdef DEBUG
4165 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
4166 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
4167 +#endif
4168 +
4169 + memcpy(ctx->key, key, keylen);
4170 + /*
4171 + * The last four bytes of the key material are used as the salt value
4172 + * in the nonce. Update the AES key length.
4173 + */
4174 + ctx->cdata.keylen = keylen - 4;
4175 + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
4176 + ctx->dir);
4177 +
4178 + ret = rfc4106_set_sh_desc(aead);
4179 + if (ret)
4180 + return ret;
4181 +
4182 + /* Now update the driver contexts with the new shared descriptor */
4183 + if (ctx->drv_ctx[ENCRYPT]) {
4184 + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
4185 + ctx->sh_desc_enc);
4186 + if (ret) {
4187 + dev_err(jrdev, "driver enc context update failed\n");
4188 + return ret;
4189 + }
4190 + }
4191 +
4192 + if (ctx->drv_ctx[DECRYPT]) {
4193 + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
4194 + ctx->sh_desc_dec);
4195 + if (ret) {
4196 + dev_err(jrdev, "driver dec context update failed\n");
4197 + return ret;
4198 + }
4199 + }
4200 +
4201 + return 0;
4202 +}
4203 +
4204 +static int rfc4543_set_sh_desc(struct crypto_aead *aead)
4205 +{
4206 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4207 + unsigned int ivsize = crypto_aead_ivsize(aead);
4208 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
4209 + ctx->cdata.keylen;
4210 +
4211 + if (!ctx->cdata.keylen || !ctx->authsize)
4212 + return 0;
4213 +
4214 + ctx->cdata.key_virt = ctx->key;
4215 +
4216 + /*
4217 + * Job Descriptor and Shared Descriptor
4218 + * must fit into the 64-word Descriptor h/w Buffer
4219 + */
4220 + if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
4221 + ctx->cdata.key_inline = true;
4222 + } else {
4223 + ctx->cdata.key_inline = false;
4224 + ctx->cdata.key_dma = ctx->key_dma;
4225 + }
4226 +
4227 + cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
4228 + ctx->authsize, true);
4229 +
4230 + /*
4231 + * Job Descriptor and Shared Descriptor
4232 + * must fit into the 64-word Descriptor h/w Buffer
4233 + */
4234 + if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
4235 + ctx->cdata.key_inline = true;
4236 + } else {
4237 + ctx->cdata.key_inline = false;
4238 + ctx->cdata.key_dma = ctx->key_dma;
4239 + }
4240 +
4241 + cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
4242 + ctx->authsize, true);
4243 +
4244 + return 0;
4245 +}
4246 +
4247 +static int rfc4543_setauthsize(struct crypto_aead *authenc,
4248 + unsigned int authsize)
4249 +{
4250 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
4251 +
4252 + ctx->authsize = authsize;
4253 + rfc4543_set_sh_desc(authenc);
4254 +
4255 + return 0;
4256 +}
4257 +
4258 +static int rfc4543_setkey(struct crypto_aead *aead,
4259 + const u8 *key, unsigned int keylen)
4260 +{
4261 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4262 + struct device *jrdev = ctx->jrdev;
4263 + int ret;
4264 +
4265 + if (keylen < 4)
4266 + return -EINVAL;
4267 +
4268 +#ifdef DEBUG
4269 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
4270 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
4271 +#endif
4272 +
4273 + memcpy(ctx->key, key, keylen);
4274 + /*
4275 + * The last four bytes of the key material are used as the salt value
4276 + * in the nonce. Update the AES key length.
4277 + */
4278 + ctx->cdata.keylen = keylen - 4;
4279 + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
4280 + ctx->dir);
4281 +
4282 + ret = rfc4543_set_sh_desc(aead);
4283 + if (ret)
4284 + return ret;
4285 +
4286 + /* Now update the driver contexts with the new shared descriptor */
4287 + if (ctx->drv_ctx[ENCRYPT]) {
4288 + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
4289 + ctx->sh_desc_enc);
4290 + if (ret) {
4291 + dev_err(jrdev, "driver enc context update failed\n");
4292 + return ret;
4293 + }
4294 + }
4295 +
4296 + if (ctx->drv_ctx[DECRYPT]) {
4297 + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
4298 + ctx->sh_desc_dec);
4299 + if (ret) {
4300 + dev_err(jrdev, "driver dec context update failed\n");
4301 + return ret;
4302 + }
4303 + }
4304 +
4305 + return 0;
4306 +}
4307 +
4308 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
4309 const u8 *key, unsigned int keylen)
4310 {
4311 @@ -414,6 +902,29 @@ struct aead_edesc {
4312 };
4313
4314 /*
4315 + * tls_edesc - s/w-extended tls descriptor
4316 + * @src_nents: number of segments in input scatterlist
4317 + * @dst_nents: number of segments in output scatterlist
4318 + * @iv_dma: dma address of iv for checking continuity and link table
4319 + * @qm_sg_bytes: length of dma mapped h/w link table
4320 + * @tmp: array of scatterlists used by 'scatterwalk_ffwd'
4321 + * @qm_sg_dma: bus physical mapped address of h/w link table
4322 + * @drv_req: driver-specific request structure
4323 + * @sgt: the h/w link table, followed by IV
4324 + */
4325 +struct tls_edesc {
4326 + int src_nents;
4327 + int dst_nents;
4328 + dma_addr_t iv_dma;
4329 + int qm_sg_bytes;
4330 + dma_addr_t qm_sg_dma;
4331 + struct scatterlist tmp[2];
4332 + struct scatterlist *dst;
4333 + struct caam_drv_req drv_req;
4334 + struct qm_sg_entry sgt[0];
4335 +};
4336 +
4337 +/*
4338 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
4339 * @src_nents: number of segments in input scatterlist
4340 * @dst_nents: number of segments in output scatterlist
4341 @@ -508,6 +1019,19 @@ static void aead_unmap(struct device *de
4342 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
4343 }
4344
4345 +static void tls_unmap(struct device *dev,
4346 + struct tls_edesc *edesc,
4347 + struct aead_request *req)
4348 +{
4349 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
4350 + int ivsize = crypto_aead_ivsize(aead);
4351 +
4352 + caam_unmap(dev, req->src, edesc->dst, edesc->src_nents,
4353 + edesc->dst_nents, edesc->iv_dma, ivsize,
4354 + edesc->drv_req.drv_ctx->op_type, edesc->qm_sg_dma,
4355 + edesc->qm_sg_bytes);
4356 +}
4357 +
4358 static void ablkcipher_unmap(struct device *dev,
4359 struct ablkcipher_edesc *edesc,
4360 struct ablkcipher_request *req)
4361 @@ -532,8 +1056,18 @@ static void aead_done(struct caam_drv_re
4362 qidev = caam_ctx->qidev;
4363
4364 if (unlikely(status)) {
4365 + u32 ssrc = status & JRSTA_SSRC_MASK;
4366 + u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
4367 +
4368 caam_jr_strstatus(qidev, status);
4369 - ecode = -EIO;
4370 + /*
4371 + * verify hw auth check passed else return -EBADMSG
4372 + */
4373 + if (ssrc == JRSTA_SSRC_CCB_ERROR &&
4374 + err_id == JRSTA_CCBERR_ERRID_ICVCHK)
4375 + ecode = -EBADMSG;
4376 + else
4377 + ecode = -EIO;
4378 }
4379
4380 edesc = container_of(drv_req, typeof(*edesc), drv_req);
4381 @@ -647,9 +1181,24 @@ static struct aead_edesc *aead_edesc_all
4382 /*
4383 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
4384 * Input is not contiguous.
4385 + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
4386 + * the end of the table by allocating more S/G entries. Logic:
4387 + * if (src != dst && output S/G)
4388 + * pad output S/G, if needed
4389 + * else if (src == dst && S/G)
4390 + * overlapping S/Gs; pad one of them
4391 + * else if (input S/G) ...
4392 + * pad input S/G, if needed
4393 */
4394 - qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
4395 - (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
4396 + qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
4397 + if (mapped_dst_nents > 1)
4398 + qm_sg_ents += ALIGN(mapped_dst_nents, 4);
4399 + else if ((req->src == req->dst) && (mapped_src_nents > 1))
4400 + qm_sg_ents = max(ALIGN(qm_sg_ents, 4),
4401 + 1 + !!ivsize + ALIGN(mapped_src_nents, 4));
4402 + else
4403 + qm_sg_ents = ALIGN(qm_sg_ents, 4);
4404 +
4405 sg_table = &edesc->sgt[0];
4406 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
4407 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
4408 @@ -785,6 +1334,260 @@ static int aead_decrypt(struct aead_requ
4409 return aead_crypt(req, false);
4410 }
4411
4412 +static int ipsec_gcm_encrypt(struct aead_request *req)
4413 +{
4414 + if (req->assoclen < 8)
4415 + return -EINVAL;
4416 +
4417 + return aead_crypt(req, true);
4418 +}
4419 +
4420 +static int ipsec_gcm_decrypt(struct aead_request *req)
4421 +{
4422 + if (req->assoclen < 8)
4423 + return -EINVAL;
4424 +
4425 + return aead_crypt(req, false);
4426 +}
4427 +
4428 +static void tls_done(struct caam_drv_req *drv_req, u32 status)
4429 +{
4430 + struct device *qidev;
4431 + struct tls_edesc *edesc;
4432 + struct aead_request *aead_req = drv_req->app_ctx;
4433 + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
4434 + struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
4435 + int ecode = 0;
4436 +
4437 + qidev = caam_ctx->qidev;
4438 +
4439 + if (unlikely(status)) {
4440 + caam_jr_strstatus(qidev, status);
4441 + ecode = -EIO;
4442 + }
4443 +
4444 + edesc = container_of(drv_req, typeof(*edesc), drv_req);
4445 + tls_unmap(qidev, edesc, aead_req);
4446 +
4447 + aead_request_complete(aead_req, ecode);
4448 + qi_cache_free(edesc);
4449 +}
4450 +
4451 +/*
4452 + * allocate and map the tls extended descriptor
4453 + */
4454 +static struct tls_edesc *tls_edesc_alloc(struct aead_request *req, bool encrypt)
4455 +{
4456 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
4457 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4458 + unsigned int blocksize = crypto_aead_blocksize(aead);
4459 + unsigned int padsize, authsize;
4460 + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
4461 + typeof(*alg), aead);
4462 + struct device *qidev = ctx->qidev;
4463 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4464 + GFP_KERNEL : GFP_ATOMIC;
4465 + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
4466 + struct tls_edesc *edesc;
4467 + dma_addr_t qm_sg_dma, iv_dma = 0;
4468 + int ivsize = 0;
4469 + u8 *iv;
4470 + int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes;
4471 + int in_len, out_len;
4472 + struct qm_sg_entry *sg_table, *fd_sgt;
4473 + struct caam_drv_ctx *drv_ctx;
4474 + enum optype op_type = encrypt ? ENCRYPT : DECRYPT;
4475 + struct scatterlist *dst;
4476 +
4477 + if (encrypt) {
4478 + padsize = blocksize - ((req->cryptlen + ctx->authsize) %
4479 + blocksize);
4480 + authsize = ctx->authsize + padsize;
4481 + } else {
4482 + authsize = ctx->authsize;
4483 + }
4484 +
4485 + drv_ctx = get_drv_ctx(ctx, op_type);
4486 + if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
4487 + return (struct tls_edesc *)drv_ctx;
4488 +
4489 + /* allocate space for base edesc, link tables and IV */
4490 + edesc = qi_cache_alloc(GFP_DMA | flags);
4491 + if (unlikely(!edesc)) {
4492 + dev_err(qidev, "could not allocate extended descriptor\n");
4493 + return ERR_PTR(-ENOMEM);
4494 + }
4495 +
4496 + if (likely(req->src == req->dst)) {
4497 + src_nents = sg_nents_for_len(req->src, req->assoclen +
4498 + req->cryptlen +
4499 + (encrypt ? authsize : 0));
4500 + if (unlikely(src_nents < 0)) {
4501 + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
4502 + req->assoclen + req->cryptlen +
4503 + (encrypt ? authsize : 0));
4504 + qi_cache_free(edesc);
4505 + return ERR_PTR(src_nents);
4506 + }
4507 +
4508 + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
4509 + DMA_BIDIRECTIONAL);
4510 + if (unlikely(!mapped_src_nents)) {
4511 + dev_err(qidev, "unable to map source\n");
4512 + qi_cache_free(edesc);
4513 + return ERR_PTR(-ENOMEM);
4514 + }
4515 + dst = req->dst;
4516 + } else {
4517 + src_nents = sg_nents_for_len(req->src, req->assoclen +
4518 + req->cryptlen);
4519 + if (unlikely(src_nents < 0)) {
4520 + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
4521 + req->assoclen + req->cryptlen);
4522 + qi_cache_free(edesc);
4523 + return ERR_PTR(src_nents);
4524 + }
4525 +
4526 + dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen);
4527 + dst_nents = sg_nents_for_len(dst, req->cryptlen +
4528 + (encrypt ? authsize : 0));
4529 + if (unlikely(dst_nents < 0)) {
4530 + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
4531 + req->cryptlen +
4532 + (encrypt ? authsize : 0));
4533 + qi_cache_free(edesc);
4534 + return ERR_PTR(dst_nents);
4535 + }
4536 +
4537 + if (src_nents) {
4538 + mapped_src_nents = dma_map_sg(qidev, req->src,
4539 + src_nents, DMA_TO_DEVICE);
4540 + if (unlikely(!mapped_src_nents)) {
4541 + dev_err(qidev, "unable to map source\n");
4542 + qi_cache_free(edesc);
4543 + return ERR_PTR(-ENOMEM);
4544 + }
4545 + } else {
4546 + mapped_src_nents = 0;
4547 + }
4548 +
4549 + mapped_dst_nents = dma_map_sg(qidev, dst, dst_nents,
4550 + DMA_FROM_DEVICE);
4551 + if (unlikely(!mapped_dst_nents)) {
4552 + dev_err(qidev, "unable to map destination\n");
4553 + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
4554 + qi_cache_free(edesc);
4555 + return ERR_PTR(-ENOMEM);
4556 + }
4557 + }
4558 +
4559 + /*
4560 + * Create S/G table: IV, src, dst.
4561 + * Input is not contiguous.
4562 + */
4563 + qm_sg_ents = 1 + mapped_src_nents +
4564 + (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
4565 + sg_table = &edesc->sgt[0];
4566 + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
4567 +
4568 + ivsize = crypto_aead_ivsize(aead);
4569 + iv = (u8 *)(sg_table + qm_sg_ents);
4570 + /* Make sure IV is located in a DMAable area */
4571 + memcpy(iv, req->iv, ivsize);
4572 + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
4573 + if (dma_mapping_error(qidev, iv_dma)) {
4574 + dev_err(qidev, "unable to map IV\n");
4575 + caam_unmap(qidev, req->src, dst, src_nents, dst_nents, 0, 0, 0,
4576 + 0, 0);
4577 + qi_cache_free(edesc);
4578 + return ERR_PTR(-ENOMEM);
4579 + }
4580 +
4581 + edesc->src_nents = src_nents;
4582 + edesc->dst_nents = dst_nents;
4583 + edesc->dst = dst;
4584 + edesc->iv_dma = iv_dma;
4585 + edesc->drv_req.app_ctx = req;
4586 + edesc->drv_req.cbk = tls_done;
4587 + edesc->drv_req.drv_ctx = drv_ctx;
4588 +
4589 + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
4590 + qm_sg_index = 1;
4591 +
4592 + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
4593 + qm_sg_index += mapped_src_nents;
4594 +
4595 + if (mapped_dst_nents > 1)
4596 + sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table +
4597 + qm_sg_index, 0);
4598 +
4599 + qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
4600 + if (dma_mapping_error(qidev, qm_sg_dma)) {
4601 + dev_err(qidev, "unable to map S/G table\n");
4602 + caam_unmap(qidev, req->src, dst, src_nents, dst_nents, iv_dma,
4603 + ivsize, op_type, 0, 0);
4604 + qi_cache_free(edesc);
4605 + return ERR_PTR(-ENOMEM);
4606 + }
4607 +
4608 + edesc->qm_sg_dma = qm_sg_dma;
4609 + edesc->qm_sg_bytes = qm_sg_bytes;
4610 +
4611 + out_len = req->cryptlen + (encrypt ? authsize : 0);
4612 + in_len = ivsize + req->assoclen + req->cryptlen;
4613 +
4614 + fd_sgt = &edesc->drv_req.fd_sgt[0];
4615 +
4616 + dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
4617 +
4618 + if (req->dst == req->src)
4619 + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
4620 + (sg_nents_for_len(req->src, req->assoclen) +
4621 + 1) * sizeof(*sg_table), out_len, 0);
4622 + else if (mapped_dst_nents == 1)
4623 + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(dst), out_len, 0);
4624 + else
4625 + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
4626 + qm_sg_index, out_len, 0);
4627 +
4628 + return edesc;
4629 +}
4630 +
4631 +static int tls_crypt(struct aead_request *req, bool encrypt)
4632 +{
4633 + struct tls_edesc *edesc;
4634 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
4635 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
4636 + int ret;
4637 +
4638 + if (unlikely(caam_congested))
4639 + return -EAGAIN;
4640 +
4641 + edesc = tls_edesc_alloc(req, encrypt);
4642 + if (IS_ERR_OR_NULL(edesc))
4643 + return PTR_ERR(edesc);
4644 +
4645 + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
4646 + if (!ret) {
4647 + ret = -EINPROGRESS;
4648 + } else {
4649 + tls_unmap(ctx->qidev, edesc, req);
4650 + qi_cache_free(edesc);
4651 + }
4652 +
4653 + return ret;
4654 +}
4655 +
4656 +static int tls_encrypt(struct aead_request *req)
4657 +{
4658 + return tls_crypt(req, true);
4659 +}
4660 +
4661 +static int tls_decrypt(struct aead_request *req)
4662 +{
4663 + return tls_crypt(req, false);
4664 +}
4665 +
4666 static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
4667 {
4668 struct ablkcipher_edesc *edesc;
4669 @@ -900,7 +1703,24 @@ static struct ablkcipher_edesc *ablkciph
4670 qm_sg_ents = 1 + mapped_src_nents;
4671 dst_sg_idx = qm_sg_ents;
4672
4673 - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
4674 + /*
4675 + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
4676 + * the end of the table by allocating more S/G entries. Logic:
4677 + * if (src != dst && output S/G)
4678 + * pad output S/G, if needed
4679 + * else if (src == dst && S/G)
4680 + * overlapping S/Gs; pad one of them
4681 + * else if (input S/G) ...
4682 + * pad input S/G, if needed
4683 + */
4684 + if (mapped_dst_nents > 1)
4685 + qm_sg_ents += ALIGN(mapped_dst_nents, 4);
4686 + else if ((req->src == req->dst) && (mapped_src_nents > 1))
4687 + qm_sg_ents = max(ALIGN(qm_sg_ents, 4),
4688 + 1 + ALIGN(mapped_src_nents, 4));
4689 + else
4690 + qm_sg_ents = ALIGN(qm_sg_ents, 4);
4691 +
4692 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
4693 if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes +
4694 ivsize > CAAM_QI_MEMCACHE_SIZE)) {
4695 @@ -1308,6 +2128,61 @@ static struct caam_alg_template driver_a
4696 };
4697
4698 static struct caam_aead_alg driver_aeads[] = {
4699 + {
4700 + .aead = {
4701 + .base = {
4702 + .cra_name = "rfc4106(gcm(aes))",
4703 + .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
4704 + .cra_blocksize = 1,
4705 + },
4706 + .setkey = rfc4106_setkey,
4707 + .setauthsize = rfc4106_setauthsize,
4708 + .encrypt = ipsec_gcm_encrypt,
4709 + .decrypt = ipsec_gcm_decrypt,
4710 + .ivsize = 8,
4711 + .maxauthsize = AES_BLOCK_SIZE,
4712 + },
4713 + .caam = {
4714 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4715 + },
4716 + },
4717 + {
4718 + .aead = {
4719 + .base = {
4720 + .cra_name = "rfc4543(gcm(aes))",
4721 + .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
4722 + .cra_blocksize = 1,
4723 + },
4724 + .setkey = rfc4543_setkey,
4725 + .setauthsize = rfc4543_setauthsize,
4726 + .encrypt = ipsec_gcm_encrypt,
4727 + .decrypt = ipsec_gcm_decrypt,
4728 + .ivsize = 8,
4729 + .maxauthsize = AES_BLOCK_SIZE,
4730 + },
4731 + .caam = {
4732 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4733 + },
4734 + },
4735 + /* Galois Counter Mode */
4736 + {
4737 + .aead = {
4738 + .base = {
4739 + .cra_name = "gcm(aes)",
4740 + .cra_driver_name = "gcm-aes-caam-qi",
4741 + .cra_blocksize = 1,
4742 + },
4743 + .setkey = gcm_setkey,
4744 + .setauthsize = gcm_setauthsize,
4745 + .encrypt = aead_encrypt,
4746 + .decrypt = aead_decrypt,
4747 + .ivsize = 12,
4748 + .maxauthsize = AES_BLOCK_SIZE,
4749 + },
4750 + .caam = {
4751 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4752 + }
4753 + },
4754 /* single-pass ipsec_esp descriptor */
4755 {
4756 .aead = {
4757 @@ -2118,6 +2993,26 @@ static struct caam_aead_alg driver_aeads
4758 .geniv = true,
4759 }
4760 },
4761 + {
4762 + .aead = {
4763 + .base = {
4764 + .cra_name = "tls10(hmac(sha1),cbc(aes))",
4765 + .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi",
4766 + .cra_blocksize = AES_BLOCK_SIZE,
4767 + },
4768 + .setkey = tls_setkey,
4769 + .setauthsize = tls_setauthsize,
4770 + .encrypt = tls_encrypt,
4771 + .decrypt = tls_decrypt,
4772 + .ivsize = AES_BLOCK_SIZE,
4773 + .maxauthsize = SHA1_DIGEST_SIZE,
4774 + },
4775 + .caam = {
4776 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
4777 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4778 + OP_ALG_AAI_HMAC_PRECOMP,
4779 + }
4780 + }
4781 };
4782
4783 struct caam_crypto_alg {
4784 @@ -2126,9 +3021,21 @@ struct caam_crypto_alg {
4785 struct caam_alg_entry caam;
4786 };
4787
4788 -static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
4789 +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
4790 + bool uses_dkp)
4791 {
4792 struct caam_drv_private *priv;
4793 + struct device *dev;
4794 + /* Digest sizes for MD5, SHA1, SHA-224, SHA-256, SHA-384, SHA-512 */
4795 + static const u8 digest_size[] = {
4796 + MD5_DIGEST_SIZE,
4797 + SHA1_DIGEST_SIZE,
4798 + SHA224_DIGEST_SIZE,
4799 + SHA256_DIGEST_SIZE,
4800 + SHA384_DIGEST_SIZE,
4801 + SHA512_DIGEST_SIZE
4802 + };
4803 + u8 op_id;
4804
4805 /*
4806 * distribute tfms across job rings to ensure in-order
4807 @@ -2140,10 +3047,19 @@ static int caam_init_common(struct caam_
4808 return PTR_ERR(ctx->jrdev);
4809 }
4810
4811 - ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
4812 - DMA_TO_DEVICE);
4813 - if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
4814 - dev_err(ctx->jrdev, "unable to map key\n");
4815 + priv = dev_get_drvdata(ctx->jrdev->parent);
4816 + if (priv->era >= 6 && uses_dkp) {
4817 + ctx->dir = DMA_BIDIRECTIONAL;
4818 + dev = ctx->jrdev->parent;
4819 + } else {
4820 + ctx->dir = DMA_TO_DEVICE;
4821 + dev = ctx->jrdev;
4822 + }
4823 +
4824 + ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
4825 + ctx->dir);
4826 + if (dma_mapping_error(dev, ctx->key_dma)) {
4827 + dev_err(dev, "unable to map key\n");
4828 caam_jr_free(ctx->jrdev);
4829 return -ENOMEM;
4830 }
4831 @@ -2152,8 +3068,23 @@ static int caam_init_common(struct caam_
4832 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
4833 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
4834
4835 - priv = dev_get_drvdata(ctx->jrdev->parent);
4836 - ctx->qidev = priv->qidev;
4837 + if (ctx->adata.algtype) {
4838 + op_id = (ctx->adata.algtype & OP_ALG_ALGSEL_SUBMASK)
4839 + >> OP_ALG_ALGSEL_SHIFT;
4840 + if (op_id < ARRAY_SIZE(digest_size)) {
4841 + ctx->authsize = digest_size[op_id];
4842 + } else {
4843 + dev_err(ctx->jrdev,
4844 + "incorrect op_id %d; must be less than %zu\n",
4845 + op_id, ARRAY_SIZE(digest_size));
4846 + caam_jr_free(ctx->jrdev);
4847 + return -EINVAL;
4848 + }
4849 + } else {
4850 + ctx->authsize = 0;
4851 + }
4852 +
4853 + ctx->qidev = ctx->jrdev->parent;
4854
4855 spin_lock_init(&ctx->lock);
4856 ctx->drv_ctx[ENCRYPT] = NULL;
4857 @@ -2170,7 +3101,7 @@ static int caam_cra_init(struct crypto_t
4858 crypto_alg);
4859 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4860
4861 - return caam_init_common(ctx, &caam_alg->caam);
4862 + return caam_init_common(ctx, &caam_alg->caam, false);
4863 }
4864
4865 static int caam_aead_init(struct crypto_aead *tfm)
4866 @@ -2180,17 +3111,25 @@ static int caam_aead_init(struct crypto_
4867 aead);
4868 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
4869
4870 - return caam_init_common(ctx, &caam_alg->caam);
4871 + return caam_init_common(ctx, &caam_alg->caam,
4872 + (alg->setkey == aead_setkey) ||
4873 + (alg->setkey == tls_setkey));
4874 }
4875
4876 static void caam_exit_common(struct caam_ctx *ctx)
4877 {
4878 + struct device *dev;
4879 +
4880 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
4881 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
4882 caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]);
4883
4884 - dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key),
4885 - DMA_TO_DEVICE);
4886 + if (ctx->dir == DMA_BIDIRECTIONAL)
4887 + dev = ctx->jrdev->parent;
4888 + else
4889 + dev = ctx->jrdev;
4890 +
4891 + dma_unmap_single(dev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
4892
4893 caam_jr_free(ctx->jrdev);
4894 }
4895 @@ -2206,7 +3145,7 @@ static void caam_aead_exit(struct crypto
4896 }
4897
4898 static struct list_head alg_list;
4899 -static void __exit caam_qi_algapi_exit(void)
4900 +void caam_qi_algapi_exit(void)
4901 {
4902 struct caam_crypto_alg *t_alg, *n;
4903 int i;
4904 @@ -2282,53 +3221,48 @@ static void caam_aead_alg_init(struct ca
4905 alg->exit = caam_aead_exit;
4906 }
4907
4908 -static int __init caam_qi_algapi_init(void)
4909 +int caam_qi_algapi_init(struct device *ctrldev)
4910 {
4911 - struct device_node *dev_node;
4912 - struct platform_device *pdev;
4913 - struct device *ctrldev;
4914 - struct caam_drv_private *priv;
4915 + struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
4916 int i = 0, err = 0;
4917 - u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
4918 + u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
4919 unsigned int md_limit = SHA512_DIGEST_SIZE;
4920 bool registered = false;
4921
4922 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4923 - if (!dev_node) {
4924 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4925 - if (!dev_node)
4926 - return -ENODEV;
4927 - }
4928 -
4929 - pdev = of_find_device_by_node(dev_node);
4930 - of_node_put(dev_node);
4931 - if (!pdev)
4932 - return -ENODEV;
4933 -
4934 - ctrldev = &pdev->dev;
4935 - priv = dev_get_drvdata(ctrldev);
4936 -
4937 - /*
4938 - * If priv is NULL, it's probably because the caam driver wasn't
4939 - * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4940 - */
4941 - if (!priv || !priv->qi_present)
4942 - return -ENODEV;
4943 -
4944 INIT_LIST_HEAD(&alg_list);
4945
4946 /*
4947 * Register crypto algorithms the device supports.
4948 * First, detect presence and attributes of DES, AES, and MD blocks.
4949 */
4950 - cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4951 - cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4952 - des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
4953 - aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
4954 - md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4955 + if (priv->era < 10) {
4956 + u32 cha_vid, cha_inst;
4957 +
4958 + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4959 + aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
4960 + md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4961 +
4962 + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4963 + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
4964 + CHA_ID_LS_DES_SHIFT;
4965 + aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
4966 + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4967 + } else {
4968 + u32 aesa, mdha;
4969 +
4970 + aesa = rd_reg32(&priv->ctrl->vreg.aesa);
4971 + mdha = rd_reg32(&priv->ctrl->vreg.mdha);
4972 +
4973 + aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
4974 + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
4975 +
4976 + des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
4977 + aes_inst = aesa & CHA_VER_NUM_MASK;
4978 + md_inst = mdha & CHA_VER_NUM_MASK;
4979 + }
4980
4981 /* If MD is present, limit digest size based on LP256 */
4982 - if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
4983 + if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
4984 md_limit = SHA256_DIGEST_SIZE;
4985
4986 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4987 @@ -2349,14 +3283,14 @@ static int __init caam_qi_algapi_init(vo
4988 t_alg = caam_alg_alloc(alg);
4989 if (IS_ERR(t_alg)) {
4990 err = PTR_ERR(t_alg);
4991 - dev_warn(priv->qidev, "%s alg allocation failed\n",
4992 + dev_warn(ctrldev, "%s alg allocation failed\n",
4993 alg->driver_name);
4994 continue;
4995 }
4996
4997 err = crypto_register_alg(&t_alg->crypto_alg);
4998 if (err) {
4999 - dev_warn(priv->qidev, "%s alg registration failed\n",
5000 + dev_warn(ctrldev, "%s alg registration failed\n",
5001 t_alg->crypto_alg.cra_driver_name);
5002 kfree(t_alg);
5003 continue;
5004 @@ -2388,8 +3322,7 @@ static int __init caam_qi_algapi_init(vo
5005 * Check support for AES algorithms not available
5006 * on LP devices.
5007 */
5008 - if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) &&
5009 - (alg_aai == OP_ALG_AAI_GCM))
5010 + if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
5011 continue;
5012
5013 /*
5014 @@ -2414,14 +3347,7 @@ static int __init caam_qi_algapi_init(vo
5015 }
5016
5017 if (registered)
5018 - dev_info(priv->qidev, "algorithms registered in /proc/crypto\n");
5019 + dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
5020
5021 return err;
5022 }
5023 -
5024 -module_init(caam_qi_algapi_init);
5025 -module_exit(caam_qi_algapi_exit);
5026 -
5027 -MODULE_LICENSE("GPL");
5028 -MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend");
5029 -MODULE_AUTHOR("Freescale Semiconductor");
5030 --- /dev/null
5031 +++ b/drivers/crypto/caam/caamalg_qi2.c
5032 @@ -0,0 +1,5843 @@
5033 +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
5034 +/*
5035 + * Copyright 2015-2016 Freescale Semiconductor Inc.
5036 + * Copyright 2017-2018 NXP
5037 + */
5038 +
5039 +#include <linux/fsl/mc.h>
5040 +#include "compat.h"
5041 +#include "regs.h"
5042 +#include "caamalg_qi2.h"
5043 +#include "dpseci_cmd.h"
5044 +#include "desc_constr.h"
5045 +#include "error.h"
5046 +#include "sg_sw_sec4.h"
5047 +#include "sg_sw_qm2.h"
5048 +#include "key_gen.h"
5049 +#include "caamalg_desc.h"
5050 +#include "caamhash_desc.h"
5051 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h"
5052 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
5053 +
5054 +#define CAAM_CRA_PRIORITY 2000
5055 +
5056 +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
5057 +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
5058 + SHA512_DIGEST_SIZE * 2)
5059 +
5060 +/*
5061 + * This is a a cache of buffers, from which the users of CAAM QI driver
5062 + * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
5063 + * NOTE: A more elegant solution would be to have some headroom in the frames
5064 + * being processed. This can be added by the dpaa2-eth driver. This would
5065 + * pose a problem for userspace application processing which cannot
5066 + * know of this limitation. So for now, this will work.
5067 + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
5068 + */
5069 +static struct kmem_cache *qi_cache;
5070 +
5071 +struct caam_alg_entry {
5072 + struct device *dev;
5073 + int class1_alg_type;
5074 + int class2_alg_type;
5075 + bool rfc3686;
5076 + bool geniv;
5077 +};
5078 +
5079 +struct caam_aead_alg {
5080 + struct aead_alg aead;
5081 + struct caam_alg_entry caam;
5082 + bool registered;
5083 +};
5084 +
5085 +struct caam_skcipher_alg {
5086 + struct skcipher_alg skcipher;
5087 + struct caam_alg_entry caam;
5088 + bool registered;
5089 +};
5090 +
5091 +/**
5092 + * caam_ctx - per-session context
5093 + * @flc: Flow Contexts array
5094 + * @key: virtual address of the key(s): [authentication key], encryption key
5095 + * @flc_dma: I/O virtual addresses of the Flow Contexts
5096 + * @key_dma: I/O virtual address of the key
5097 + * @dir: DMA direction for mapping key and Flow Contexts
5098 + * @dev: dpseci device
5099 + * @adata: authentication algorithm details
5100 + * @cdata: encryption algorithm details
5101 + * @authsize: authentication tag (a.k.a. ICV / MAC) size
5102 + */
5103 +struct caam_ctx {
5104 + struct caam_flc flc[NUM_OP];
5105 + u8 key[CAAM_MAX_KEY_SIZE];
5106 + dma_addr_t flc_dma[NUM_OP];
5107 + dma_addr_t key_dma;
5108 + enum dma_data_direction dir;
5109 + struct device *dev;
5110 + struct alginfo adata;
5111 + struct alginfo cdata;
5112 + unsigned int authsize;
5113 +};
5114 +
5115 +void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
5116 + dma_addr_t iova_addr)
5117 +{
5118 + phys_addr_t phys_addr;
5119 +
5120 + phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
5121 + iova_addr;
5122 +
5123 + return phys_to_virt(phys_addr);
5124 +}
5125 +
5126 +/*
5127 + * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
5128 + *
5129 + * Allocate data on the hotpath. Instead of using kzalloc, one can use the
5130 + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
5131 + * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
5132 + * hosting 16 SG entries.
5133 + *
5134 + * @flags - flags that would be used for the equivalent kmalloc(..) call
5135 + *
5136 + * Returns a pointer to a retrieved buffer on success or NULL on failure.
5137 + */
5138 +static inline void *qi_cache_zalloc(gfp_t flags)
5139 +{
5140 + return kmem_cache_zalloc(qi_cache, flags);
5141 +}
5142 +
5143 +/*
5144 + * qi_cache_free - Frees buffers allocated from CAAM-QI cache
5145 + *
5146 + * @obj - buffer previously allocated by qi_cache_zalloc
5147 + *
5148 + * No checking is being done, the call is a passthrough call to
5149 + * kmem_cache_free(...)
5150 + */
5151 +static inline void qi_cache_free(void *obj)
5152 +{
5153 + kmem_cache_free(qi_cache, obj);
5154 +}
5155 +
5156 +static struct caam_request *to_caam_req(struct crypto_async_request *areq)
5157 +{
5158 + switch (crypto_tfm_alg_type(areq->tfm)) {
5159 + case CRYPTO_ALG_TYPE_SKCIPHER:
5160 + return skcipher_request_ctx(skcipher_request_cast(areq));
5161 + case CRYPTO_ALG_TYPE_AEAD:
5162 + return aead_request_ctx(container_of(areq, struct aead_request,
5163 + base));
5164 + case CRYPTO_ALG_TYPE_AHASH:
5165 + return ahash_request_ctx(ahash_request_cast(areq));
5166 + default:
5167 + return ERR_PTR(-EINVAL);
5168 + }
5169 +}
5170 +
5171 +static void caam_unmap(struct device *dev, struct scatterlist *src,
5172 + struct scatterlist *dst, int src_nents,
5173 + int dst_nents, dma_addr_t iv_dma, int ivsize,
5174 + dma_addr_t qm_sg_dma, int qm_sg_bytes)
5175 +{
5176 + if (dst != src) {
5177 + if (src_nents)
5178 + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
5179 + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
5180 + } else {
5181 + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
5182 + }
5183 +
5184 + if (iv_dma)
5185 + dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
5186 +
5187 + if (qm_sg_bytes)
5188 + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
5189 +}
5190 +
5191 +static int aead_set_sh_desc(struct crypto_aead *aead)
5192 +{
5193 + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
5194 + typeof(*alg), aead);
5195 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5196 + unsigned int ivsize = crypto_aead_ivsize(aead);
5197 + struct device *dev = ctx->dev;
5198 + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5199 + struct caam_flc *flc;
5200 + u32 *desc;
5201 + u32 ctx1_iv_off = 0;
5202 + u32 *nonce = NULL;
5203 + unsigned int data_len[2];
5204 + u32 inl_mask;
5205 + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
5206 + OP_ALG_AAI_CTR_MOD128);
5207 + const bool is_rfc3686 = alg->caam.rfc3686;
5208 +
5209 + if (!ctx->cdata.keylen || !ctx->authsize)
5210 + return 0;
5211 +
5212 + /*
5213 + * AES-CTR needs to load IV in CONTEXT1 reg
5214 + * at an offset of 128bits (16bytes)
5215 + * CONTEXT1[255:128] = IV
5216 + */
5217 + if (ctr_mode)
5218 + ctx1_iv_off = 16;
5219 +
5220 + /*
5221 + * RFC3686 specific:
5222 + * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
5223 + */
5224 + if (is_rfc3686) {
5225 + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
5226 + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
5227 + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
5228 + }
5229 +
5230 + data_len[0] = ctx->adata.keylen_pad;
5231 + data_len[1] = ctx->cdata.keylen;
5232 +
5233 + /* aead_encrypt shared descriptor */
5234 + if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
5235 + DESC_QI_AEAD_ENC_LEN) +
5236 + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
5237 + DESC_JOB_IO_LEN, data_len, &inl_mask,
5238 + ARRAY_SIZE(data_len)) < 0)
5239 + return -EINVAL;
5240 +
5241 + if (inl_mask & 1)
5242 + ctx->adata.key_virt = ctx->key;
5243 + else
5244 + ctx->adata.key_dma = ctx->key_dma;
5245 +
5246 + if (inl_mask & 2)
5247 + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
5248 + else
5249 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
5250 +
5251 + ctx->adata.key_inline = !!(inl_mask & 1);
5252 + ctx->cdata.key_inline = !!(inl_mask & 2);
5253 +
5254 + flc = &ctx->flc[ENCRYPT];
5255 + desc = flc->sh_desc;
5256 +
5257 + if (alg->caam.geniv)
5258 + cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
5259 + ivsize, ctx->authsize, is_rfc3686,
5260 + nonce, ctx1_iv_off, true,
5261 + priv->sec_attr.era);
5262 + else
5263 + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
5264 + ivsize, ctx->authsize, is_rfc3686, nonce,
5265 + ctx1_iv_off, true, priv->sec_attr.era);
5266 +
5267 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5268 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
5269 + sizeof(flc->flc) + desc_bytes(desc),
5270 + ctx->dir);
5271 +
5272 + /* aead_decrypt shared descriptor */
5273 + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
5274 + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
5275 + DESC_JOB_IO_LEN, data_len, &inl_mask,
5276 + ARRAY_SIZE(data_len)) < 0)
5277 + return -EINVAL;
5278 +
5279 + if (inl_mask & 1)
5280 + ctx->adata.key_virt = ctx->key;
5281 + else
5282 + ctx->adata.key_dma = ctx->key_dma;
5283 +
5284 + if (inl_mask & 2)
5285 + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
5286 + else
5287 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
5288 +
5289 + ctx->adata.key_inline = !!(inl_mask & 1);
5290 + ctx->cdata.key_inline = !!(inl_mask & 2);
5291 +
5292 + flc = &ctx->flc[DECRYPT];
5293 + desc = flc->sh_desc;
5294 + cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
5295 + ivsize, ctx->authsize, alg->caam.geniv,
5296 + is_rfc3686, nonce, ctx1_iv_off, true,
5297 + priv->sec_attr.era);
5298 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5299 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
5300 + sizeof(flc->flc) + desc_bytes(desc),
5301 + ctx->dir);
5302 +
5303 + return 0;
5304 +}
5305 +
5306 +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
5307 +{
5308 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
5309 +
5310 + ctx->authsize = authsize;
5311 + aead_set_sh_desc(authenc);
5312 +
5313 + return 0;
5314 +}
5315 +
5316 +struct split_key_sh_result {
5317 + struct completion completion;
5318 + int err;
5319 + struct device *dev;
5320 +};
5321 +
5322 +static void split_key_sh_done(void *cbk_ctx, u32 err)
5323 +{
5324 + struct split_key_sh_result *res = cbk_ctx;
5325 +
5326 +#ifdef DEBUG
5327 + dev_err(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
5328 +#endif
5329 +
5330 + if (err)
5331 + caam_qi2_strstatus(res->dev, err);
5332 +
5333 + res->err = err;
5334 + complete(&res->completion);
5335 +}
5336 +
5337 +static int aead_setkey(struct crypto_aead *aead, const u8 *key,
5338 + unsigned int keylen)
5339 +{
5340 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5341 + struct device *dev = ctx->dev;
5342 + struct crypto_authenc_keys keys;
5343 +
5344 + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
5345 + goto badkey;
5346 +
5347 +#ifdef DEBUG
5348 + dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n",
5349 + keys.authkeylen + keys.enckeylen, keys.enckeylen,
5350 + keys.authkeylen);
5351 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
5352 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
5353 +#endif
5354 +
5355 + ctx->adata.keylen = keys.authkeylen;
5356 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
5357 + OP_ALG_ALGSEL_MASK);
5358 +
5359 + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
5360 + goto badkey;
5361 +
5362 + memcpy(ctx->key, keys.authkey, keys.authkeylen);
5363 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
5364 + dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
5365 + keys.enckeylen, ctx->dir);
5366 +#ifdef DEBUG
5367 + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
5368 + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
5369 + ctx->adata.keylen_pad + keys.enckeylen, 1);
5370 +#endif
5371 +
5372 + ctx->cdata.keylen = keys.enckeylen;
5373 +
5374 + return aead_set_sh_desc(aead);
5375 +badkey:
5376 + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
5377 + return -EINVAL;
5378 +}
5379 +
5380 +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
5381 + bool encrypt)
5382 +{
5383 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
5384 + struct caam_request *req_ctx = aead_request_ctx(req);
5385 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
5386 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
5387 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5388 + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
5389 + typeof(*alg), aead);
5390 + struct device *dev = ctx->dev;
5391 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
5392 + GFP_KERNEL : GFP_ATOMIC;
5393 + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
5394 + struct aead_edesc *edesc;
5395 + dma_addr_t qm_sg_dma, iv_dma = 0;
5396 + int ivsize = 0;
5397 + unsigned int authsize = ctx->authsize;
5398 + int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
5399 + int in_len, out_len;
5400 + struct dpaa2_sg_entry *sg_table;
5401 +
5402 + /* allocate space for base edesc, link tables and IV */
5403 + edesc = qi_cache_zalloc(GFP_DMA | flags);
5404 + if (unlikely(!edesc)) {
5405 + dev_err(dev, "could not allocate extended descriptor\n");
5406 + return ERR_PTR(-ENOMEM);
5407 + }
5408 +
5409 + if (unlikely(req->dst != req->src)) {
5410 + src_nents = sg_nents_for_len(req->src, req->assoclen +
5411 + req->cryptlen);
5412 + if (unlikely(src_nents < 0)) {
5413 + dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
5414 + req->assoclen + req->cryptlen);
5415 + qi_cache_free(edesc);
5416 + return ERR_PTR(src_nents);
5417 + }
5418 +
5419 + dst_nents = sg_nents_for_len(req->dst, req->assoclen +
5420 + req->cryptlen +
5421 + (encrypt ? authsize :
5422 + (-authsize)));
5423 + if (unlikely(dst_nents < 0)) {
5424 + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
5425 + req->assoclen + req->cryptlen +
5426 + (encrypt ? authsize : (-authsize)));
5427 + qi_cache_free(edesc);
5428 + return ERR_PTR(dst_nents);
5429 + }
5430 +
5431 + if (src_nents) {
5432 + mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
5433 + DMA_TO_DEVICE);
5434 + if (unlikely(!mapped_src_nents)) {
5435 + dev_err(dev, "unable to map source\n");
5436 + qi_cache_free(edesc);
5437 + return ERR_PTR(-ENOMEM);
5438 + }
5439 + } else {
5440 + mapped_src_nents = 0;
5441 + }
5442 +
5443 + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
5444 + DMA_FROM_DEVICE);
5445 + if (unlikely(!mapped_dst_nents)) {
5446 + dev_err(dev, "unable to map destination\n");
5447 + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
5448 + qi_cache_free(edesc);
5449 + return ERR_PTR(-ENOMEM);
5450 + }
5451 + } else {
5452 + src_nents = sg_nents_for_len(req->src, req->assoclen +
5453 + req->cryptlen +
5454 + (encrypt ? authsize : 0));
5455 + if (unlikely(src_nents < 0)) {
5456 + dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
5457 + req->assoclen + req->cryptlen +
5458 + (encrypt ? authsize : 0));
5459 + qi_cache_free(edesc);
5460 + return ERR_PTR(src_nents);
5461 + }
5462 +
5463 + mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
5464 + DMA_BIDIRECTIONAL);
5465 + if (unlikely(!mapped_src_nents)) {
5466 + dev_err(dev, "unable to map source\n");
5467 + qi_cache_free(edesc);
5468 + return ERR_PTR(-ENOMEM);
5469 + }
5470 + }
5471 +
5472 + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
5473 + ivsize = crypto_aead_ivsize(aead);
5474 +
5475 + /*
5476 + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
5477 + * Input is not contiguous.
5478 + */
5479 + qm_sg_nents = 1 + !!ivsize + mapped_src_nents +
5480 + (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
5481 + sg_table = &edesc->sgt[0];
5482 + qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
5483 + if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
5484 + CAAM_QI_MEMCACHE_SIZE)) {
5485 + dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
5486 + qm_sg_nents, ivsize);
5487 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
5488 + 0, 0, 0);
5489 + qi_cache_free(edesc);
5490 + return ERR_PTR(-ENOMEM);
5491 + }
5492 +
5493 + if (ivsize) {
5494 + u8 *iv = (u8 *)(sg_table + qm_sg_nents);
5495 +
5496 + /* Make sure IV is located in a DMAable area */
5497 + memcpy(iv, req->iv, ivsize);
5498 +
5499 + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
5500 + if (dma_mapping_error(dev, iv_dma)) {
5501 + dev_err(dev, "unable to map IV\n");
5502 + caam_unmap(dev, req->src, req->dst, src_nents,
5503 + dst_nents, 0, 0, 0, 0);
5504 + qi_cache_free(edesc);
5505 + return ERR_PTR(-ENOMEM);
5506 + }
5507 + }
5508 +
5509 + edesc->src_nents = src_nents;
5510 + edesc->dst_nents = dst_nents;
5511 + edesc->iv_dma = iv_dma;
5512 +
5513 + if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
5514 + OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
5515 + /*
5516 + * The associated data comes already with the IV but we need
5517 + * to skip it when we authenticate or encrypt...
5518 + */
5519 + edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
5520 + else
5521 + edesc->assoclen = cpu_to_caam32(req->assoclen);
5522 + edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
5523 + DMA_TO_DEVICE);
5524 + if (dma_mapping_error(dev, edesc->assoclen_dma)) {
5525 + dev_err(dev, "unable to map assoclen\n");
5526 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
5527 + iv_dma, ivsize, 0, 0);
5528 + qi_cache_free(edesc);
5529 + return ERR_PTR(-ENOMEM);
5530 + }
5531 +
5532 + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
5533 + qm_sg_index++;
5534 + if (ivsize) {
5535 + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
5536 + qm_sg_index++;
5537 + }
5538 + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
5539 + qm_sg_index += mapped_src_nents;
5540 +
5541 + if (mapped_dst_nents > 1)
5542 + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
5543 + qm_sg_index, 0);
5544 +
5545 + qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
5546 + if (dma_mapping_error(dev, qm_sg_dma)) {
5547 + dev_err(dev, "unable to map S/G table\n");
5548 + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
5549 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
5550 + iv_dma, ivsize, 0, 0);
5551 + qi_cache_free(edesc);
5552 + return ERR_PTR(-ENOMEM);
5553 + }
5554 +
5555 + edesc->qm_sg_dma = qm_sg_dma;
5556 + edesc->qm_sg_bytes = qm_sg_bytes;
5557 +
5558 + out_len = req->assoclen + req->cryptlen +
5559 + (encrypt ? ctx->authsize : (-ctx->authsize));
5560 + in_len = 4 + ivsize + req->assoclen + req->cryptlen;
5561 +
5562 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
5563 + dpaa2_fl_set_final(in_fle, true);
5564 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
5565 + dpaa2_fl_set_addr(in_fle, qm_sg_dma);
5566 + dpaa2_fl_set_len(in_fle, in_len);
5567 +
5568 + if (req->dst == req->src) {
5569 + if (mapped_src_nents == 1) {
5570 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
5571 + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
5572 + } else {
5573 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
5574 + dpaa2_fl_set_addr(out_fle, qm_sg_dma +
5575 + (1 + !!ivsize) * sizeof(*sg_table));
5576 + }
5577 + } else if (mapped_dst_nents == 1) {
5578 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
5579 + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
5580 + } else {
5581 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
5582 + dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
5583 + sizeof(*sg_table));
5584 + }
5585 +
5586 + dpaa2_fl_set_len(out_fle, out_len);
5587 +
5588 + return edesc;
5589 +}
5590 +
5591 +static int chachapoly_set_sh_desc(struct crypto_aead *aead)
5592 +{
5593 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5594 + unsigned int ivsize = crypto_aead_ivsize(aead);
5595 + struct device *dev = ctx->dev;
5596 + struct caam_flc *flc;
5597 + u32 *desc;
5598 +
5599 + if (!ctx->cdata.keylen || !ctx->authsize)
5600 + return 0;
5601 +
5602 + flc = &ctx->flc[ENCRYPT];
5603 + desc = flc->sh_desc;
5604 + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
5605 + ctx->authsize, true, true);
5606 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5607 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
5608 + sizeof(flc->flc) + desc_bytes(desc),
5609 + ctx->dir);
5610 +
5611 + flc = &ctx->flc[DECRYPT];
5612 + desc = flc->sh_desc;
5613 + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
5614 + ctx->authsize, false, true);
5615 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5616 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
5617 + sizeof(flc->flc) + desc_bytes(desc),
5618 + ctx->dir);
5619 +
5620 + return 0;
5621 +}
5622 +
5623 +static int chachapoly_setauthsize(struct crypto_aead *aead,
5624 + unsigned int authsize)
5625 +{
5626 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5627 +
5628 + if (authsize != POLY1305_DIGEST_SIZE)
5629 + return -EINVAL;
5630 +
5631 + ctx->authsize = authsize;
5632 + return chachapoly_set_sh_desc(aead);
5633 +}
5634 +
5635 +static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
5636 + unsigned int keylen)
5637 +{
5638 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5639 + unsigned int ivsize = crypto_aead_ivsize(aead);
5640 + unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
5641 +
5642 + if (keylen != CHACHA20_KEY_SIZE + saltlen) {
5643 + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
5644 + return -EINVAL;
5645 + }
5646 +
5647 + ctx->cdata.key_virt = key;
5648 + ctx->cdata.keylen = keylen - saltlen;
5649 +
5650 + return chachapoly_set_sh_desc(aead);
5651 +}
5652 +
5653 +static struct tls_edesc *tls_edesc_alloc(struct aead_request *req,
5654 + bool encrypt)
5655 +{
5656 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
5657 + unsigned int blocksize = crypto_aead_blocksize(tls);
5658 + unsigned int padsize, authsize;
5659 + struct caam_request *req_ctx = aead_request_ctx(req);
5660 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
5661 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
5662 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
5663 + struct caam_aead_alg *alg = container_of(crypto_aead_alg(tls),
5664 + typeof(*alg), aead);
5665 + struct device *dev = ctx->dev;
5666 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
5667 + GFP_KERNEL : GFP_ATOMIC;
5668 + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
5669 + struct tls_edesc *edesc;
5670 + dma_addr_t qm_sg_dma, iv_dma = 0;
5671 + int ivsize = 0;
5672 + u8 *iv;
5673 + int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes;
5674 + int in_len, out_len;
5675 + struct dpaa2_sg_entry *sg_table;
5676 + struct scatterlist *dst;
5677 +
5678 + if (encrypt) {
5679 + padsize = blocksize - ((req->cryptlen + ctx->authsize) %
5680 + blocksize);
5681 + authsize = ctx->authsize + padsize;
5682 + } else {
5683 + authsize = ctx->authsize;
5684 + }
5685 +
5686 + /* allocate space for base edesc, link tables and IV */
5687 + edesc = qi_cache_zalloc(GFP_DMA | flags);
5688 + if (unlikely(!edesc)) {
5689 + dev_err(dev, "could not allocate extended descriptor\n");
5690 + return ERR_PTR(-ENOMEM);
5691 + }
5692 +
5693 + if (likely(req->src == req->dst)) {
5694 + src_nents = sg_nents_for_len(req->src, req->assoclen +
5695 + req->cryptlen +
5696 + (encrypt ? authsize : 0));
5697 + if (unlikely(src_nents < 0)) {
5698 + dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
5699 + req->assoclen + req->cryptlen +
5700 + (encrypt ? authsize : 0));
5701 + qi_cache_free(edesc);
5702 + return ERR_PTR(src_nents);
5703 + }
5704 +
5705 + mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
5706 + DMA_BIDIRECTIONAL);
5707 + if (unlikely(!mapped_src_nents)) {
5708 + dev_err(dev, "unable to map source\n");
5709 + qi_cache_free(edesc);
5710 + return ERR_PTR(-ENOMEM);
5711 + }
5712 + dst = req->dst;
5713 + } else {
5714 + src_nents = sg_nents_for_len(req->src, req->assoclen +
5715 + req->cryptlen);
5716 + if (unlikely(src_nents < 0)) {
5717 + dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
5718 + req->assoclen + req->cryptlen);
5719 + qi_cache_free(edesc);
5720 + return ERR_PTR(src_nents);
5721 + }
5722 +
5723 + dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen);
5724 + dst_nents = sg_nents_for_len(dst, req->cryptlen +
5725 + (encrypt ? authsize : 0));
5726 + if (unlikely(dst_nents < 0)) {
5727 + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
5728 + req->cryptlen +
5729 + (encrypt ? authsize : 0));
5730 + qi_cache_free(edesc);
5731 + return ERR_PTR(dst_nents);
5732 + }
5733 +
5734 + if (src_nents) {
5735 + mapped_src_nents = dma_map_sg(dev, req->src,
5736 + src_nents, DMA_TO_DEVICE);
5737 + if (unlikely(!mapped_src_nents)) {
5738 + dev_err(dev, "unable to map source\n");
5739 + qi_cache_free(edesc);
5740 + return ERR_PTR(-ENOMEM);
5741 + }
5742 + } else {
5743 + mapped_src_nents = 0;
5744 + }
5745 +
5746 + mapped_dst_nents = dma_map_sg(dev, dst, dst_nents,
5747 + DMA_FROM_DEVICE);
5748 + if (unlikely(!mapped_dst_nents)) {
5749 + dev_err(dev, "unable to map destination\n");
5750 + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
5751 + qi_cache_free(edesc);
5752 + return ERR_PTR(-ENOMEM);
5753 + }
5754 + }
5755 +
5756 + /*
5757 + * Create S/G table: IV, src, dst.
5758 + * Input is not contiguous.
5759 + */
5760 + qm_sg_ents = 1 + mapped_src_nents +
5761 + (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
5762 + sg_table = &edesc->sgt[0];
5763 + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
5764 +
5765 + ivsize = crypto_aead_ivsize(tls);
5766 + iv = (u8 *)(sg_table + qm_sg_ents);
5767 + /* Make sure IV is located in a DMAable area */
5768 + memcpy(iv, req->iv, ivsize);
5769 + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
5770 + if (dma_mapping_error(dev, iv_dma)) {
5771 + dev_err(dev, "unable to map IV\n");
5772 + caam_unmap(dev, req->src, dst, src_nents, dst_nents, 0, 0, 0,
5773 + 0);
5774 + qi_cache_free(edesc);
5775 + return ERR_PTR(-ENOMEM);
5776 + }
5777 +
5778 + edesc->src_nents = src_nents;
5779 + edesc->dst_nents = dst_nents;
5780 + edesc->dst = dst;
5781 + edesc->iv_dma = iv_dma;
5782 +
5783 + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
5784 + qm_sg_index = 1;
5785 +
5786 + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
5787 + qm_sg_index += mapped_src_nents;
5788 +
5789 + if (mapped_dst_nents > 1)
5790 + sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table +
5791 + qm_sg_index, 0);
5792 +
5793 + qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
5794 + if (dma_mapping_error(dev, qm_sg_dma)) {
5795 + dev_err(dev, "unable to map S/G table\n");
5796 + caam_unmap(dev, req->src, dst, src_nents, dst_nents, iv_dma,
5797 + ivsize, 0, 0);
5798 + qi_cache_free(edesc);
5799 + return ERR_PTR(-ENOMEM);
5800 + }
5801 +
5802 + edesc->qm_sg_dma = qm_sg_dma;
5803 + edesc->qm_sg_bytes = qm_sg_bytes;
5804 +
5805 + out_len = req->cryptlen + (encrypt ? authsize : 0);
5806 + in_len = ivsize + req->assoclen + req->cryptlen;
5807 +
5808 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
5809 + dpaa2_fl_set_final(in_fle, true);
5810 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
5811 + dpaa2_fl_set_addr(in_fle, qm_sg_dma);
5812 + dpaa2_fl_set_len(in_fle, in_len);
5813 +
5814 + if (req->dst == req->src) {
5815 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
5816 + dpaa2_fl_set_addr(out_fle, qm_sg_dma +
5817 + (sg_nents_for_len(req->src, req->assoclen) +
5818 + 1) * sizeof(*sg_table));
5819 + } else if (mapped_dst_nents == 1) {
5820 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
5821 + dpaa2_fl_set_addr(out_fle, sg_dma_address(dst));
5822 + } else {
5823 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
5824 + dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
5825 + sizeof(*sg_table));
5826 + }
5827 +
5828 + dpaa2_fl_set_len(out_fle, out_len);
5829 +
5830 + return edesc;
5831 +}
5832 +
5833 +static int tls_set_sh_desc(struct crypto_aead *tls)
5834 +{
5835 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
5836 + unsigned int ivsize = crypto_aead_ivsize(tls);
5837 + unsigned int blocksize = crypto_aead_blocksize(tls);
5838 + struct device *dev = ctx->dev;
5839 + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5840 + struct caam_flc *flc;
5841 + u32 *desc;
5842 + unsigned int assoclen = 13; /* always 13 bytes for TLS */
5843 + unsigned int data_len[2];
5844 + u32 inl_mask;
5845 +
5846 + if (!ctx->cdata.keylen || !ctx->authsize)
5847 + return 0;
5848 +
5849 + /*
5850 + * TLS 1.0 encrypt shared descriptor
5851 + * Job Descriptor and Shared Descriptor
5852 + * must fit into the 64-word Descriptor h/w Buffer
5853 + */
5854 + data_len[0] = ctx->adata.keylen_pad;
5855 + data_len[1] = ctx->cdata.keylen;
5856 +
5857 + if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len,
5858 + &inl_mask, ARRAY_SIZE(data_len)) < 0)
5859 + return -EINVAL;
5860 +
5861 + if (inl_mask & 1)
5862 + ctx->adata.key_virt = ctx->key;
5863 + else
5864 + ctx->adata.key_dma = ctx->key_dma;
5865 +
5866 + if (inl_mask & 2)
5867 + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
5868 + else
5869 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
5870 +
5871 + ctx->adata.key_inline = !!(inl_mask & 1);
5872 + ctx->cdata.key_inline = !!(inl_mask & 2);
5873 +
5874 + flc = &ctx->flc[ENCRYPT];
5875 + desc = flc->sh_desc;
5876 + cnstr_shdsc_tls_encap(desc, &ctx->cdata, &ctx->adata,
5877 + assoclen, ivsize, ctx->authsize, blocksize,
5878 + priv->sec_attr.era);
5879 + flc->flc[1] = cpu_to_caam32(desc_len(desc));
5880 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
5881 + sizeof(flc->flc) + desc_bytes(desc),
5882 + ctx->dir);
5883 +
5884 + /*
5885 + * TLS 1.0 decrypt shared descriptor
5886 + * Keys do not fit inline, regardless of algorithms used
5887 + */
5888 + ctx->adata.key_inline = false;
5889 + ctx->adata.key_dma = ctx->key_dma;
5890 + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
5891 +
5892 + flc = &ctx->flc[DECRYPT];
5893 + desc = flc->sh_desc;
5894 + cnstr_shdsc_tls_decap(desc, &ctx->cdata, &ctx->adata, assoclen, ivsize,
5895 + ctx->authsize, blocksize, priv->sec_attr.era);
5896 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5897 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
5898 + sizeof(flc->flc) + desc_bytes(desc),
5899 + ctx->dir);
5900 +
5901 + return 0;
5902 +}
5903 +
5904 +static int tls_setkey(struct crypto_aead *tls, const u8 *key,
5905 + unsigned int keylen)
5906 +{
5907 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
5908 + struct device *dev = ctx->dev;
5909 + struct crypto_authenc_keys keys;
5910 +
5911 + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
5912 + goto badkey;
5913 +
5914 +#ifdef DEBUG
5915 + dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n",
5916 + keys.authkeylen + keys.enckeylen, keys.enckeylen,
5917 + keys.authkeylen);
5918 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
5919 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
5920 +#endif
5921 +
5922 + ctx->adata.keylen = keys.authkeylen;
5923 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
5924 + OP_ALG_ALGSEL_MASK);
5925 +
5926 + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
5927 + goto badkey;
5928 +
5929 + memcpy(ctx->key, keys.authkey, keys.authkeylen);
5930 + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
5931 + dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
5932 + keys.enckeylen, ctx->dir);
5933 +#ifdef DEBUG
5934 + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
5935 + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
5936 + ctx->adata.keylen_pad + keys.enckeylen, 1);
5937 +#endif
5938 +
5939 + ctx->cdata.keylen = keys.enckeylen;
5940 +
5941 + return tls_set_sh_desc(tls);
5942 +badkey:
5943 + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN);
5944 + return -EINVAL;
5945 +}
5946 +
5947 +static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize)
5948 +{
5949 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
5950 +
5951 + ctx->authsize = authsize;
5952 + tls_set_sh_desc(tls);
5953 +
5954 + return 0;
5955 +}
5956 +
5957 +static int gcm_set_sh_desc(struct crypto_aead *aead)
5958 +{
5959 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
5960 + struct device *dev = ctx->dev;
5961 + unsigned int ivsize = crypto_aead_ivsize(aead);
5962 + struct caam_flc *flc;
5963 + u32 *desc;
5964 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
5965 + ctx->cdata.keylen;
5966 +
5967 + if (!ctx->cdata.keylen || !ctx->authsize)
5968 + return 0;
5969 +
5970 + /*
5971 + * AES GCM encrypt shared descriptor
5972 + * Job Descriptor and Shared Descriptor
5973 + * must fit into the 64-word Descriptor h/w Buffer
5974 + */
5975 + if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
5976 + ctx->cdata.key_inline = true;
5977 + ctx->cdata.key_virt = ctx->key;
5978 + } else {
5979 + ctx->cdata.key_inline = false;
5980 + ctx->cdata.key_dma = ctx->key_dma;
5981 + }
5982 +
5983 + flc = &ctx->flc[ENCRYPT];
5984 + desc = flc->sh_desc;
5985 + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
5986 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
5987 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
5988 + sizeof(flc->flc) + desc_bytes(desc),
5989 + ctx->dir);
5990 +
5991 + /*
5992 + * Job Descriptor and Shared Descriptors
5993 + * must all fit into the 64-word Descriptor h/w Buffer
5994 + */
5995 + if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
5996 + ctx->cdata.key_inline = true;
5997 + ctx->cdata.key_virt = ctx->key;
5998 + } else {
5999 + ctx->cdata.key_inline = false;
6000 + ctx->cdata.key_dma = ctx->key_dma;
6001 + }
6002 +
6003 + flc = &ctx->flc[DECRYPT];
6004 + desc = flc->sh_desc;
6005 + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
6006 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6007 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
6008 + sizeof(flc->flc) + desc_bytes(desc),
6009 + ctx->dir);
6010 +
6011 + return 0;
6012 +}
6013 +
6014 +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
6015 +{
6016 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
6017 +
6018 + ctx->authsize = authsize;
6019 + gcm_set_sh_desc(authenc);
6020 +
6021 + return 0;
6022 +}
6023 +
6024 +static int gcm_setkey(struct crypto_aead *aead,
6025 + const u8 *key, unsigned int keylen)
6026 +{
6027 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6028 + struct device *dev = ctx->dev;
6029 +
6030 +#ifdef DEBUG
6031 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
6032 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
6033 +#endif
6034 +
6035 + memcpy(ctx->key, key, keylen);
6036 + dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
6037 + ctx->cdata.keylen = keylen;
6038 +
6039 + return gcm_set_sh_desc(aead);
6040 +}
6041 +
6042 +static int rfc4106_set_sh_desc(struct crypto_aead *aead)
6043 +{
6044 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6045 + struct device *dev = ctx->dev;
6046 + unsigned int ivsize = crypto_aead_ivsize(aead);
6047 + struct caam_flc *flc;
6048 + u32 *desc;
6049 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
6050 + ctx->cdata.keylen;
6051 +
6052 + if (!ctx->cdata.keylen || !ctx->authsize)
6053 + return 0;
6054 +
6055 + ctx->cdata.key_virt = ctx->key;
6056 +
6057 + /*
6058 + * RFC4106 encrypt shared descriptor
6059 + * Job Descriptor and Shared Descriptor
6060 + * must fit into the 64-word Descriptor h/w Buffer
6061 + */
6062 + if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
6063 + ctx->cdata.key_inline = true;
6064 + } else {
6065 + ctx->cdata.key_inline = false;
6066 + ctx->cdata.key_dma = ctx->key_dma;
6067 + }
6068 +
6069 + flc = &ctx->flc[ENCRYPT];
6070 + desc = flc->sh_desc;
6071 + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
6072 + true);
6073 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6074 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
6075 + sizeof(flc->flc) + desc_bytes(desc),
6076 + ctx->dir);
6077 +
6078 + /*
6079 + * Job Descriptor and Shared Descriptors
6080 + * must all fit into the 64-word Descriptor h/w Buffer
6081 + */
6082 + if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
6083 + ctx->cdata.key_inline = true;
6084 + } else {
6085 + ctx->cdata.key_inline = false;
6086 + ctx->cdata.key_dma = ctx->key_dma;
6087 + }
6088 +
6089 + flc = &ctx->flc[DECRYPT];
6090 + desc = flc->sh_desc;
6091 + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
6092 + true);
6093 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6094 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
6095 + sizeof(flc->flc) + desc_bytes(desc),
6096 + ctx->dir);
6097 +
6098 + return 0;
6099 +}
6100 +
6101 +static int rfc4106_setauthsize(struct crypto_aead *authenc,
6102 + unsigned int authsize)
6103 +{
6104 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
6105 +
6106 + ctx->authsize = authsize;
6107 + rfc4106_set_sh_desc(authenc);
6108 +
6109 + return 0;
6110 +}
6111 +
6112 +static int rfc4106_setkey(struct crypto_aead *aead,
6113 + const u8 *key, unsigned int keylen)
6114 +{
6115 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6116 + struct device *dev = ctx->dev;
6117 +
6118 + if (keylen < 4)
6119 + return -EINVAL;
6120 +
6121 +#ifdef DEBUG
6122 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
6123 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
6124 +#endif
6125 +
6126 + memcpy(ctx->key, key, keylen);
6127 + /*
6128 + * The last four bytes of the key material are used as the salt value
6129 + * in the nonce. Update the AES key length.
6130 + */
6131 + ctx->cdata.keylen = keylen - 4;
6132 + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
6133 + ctx->dir);
6134 +
6135 + return rfc4106_set_sh_desc(aead);
6136 +}
6137 +
6138 +static int rfc4543_set_sh_desc(struct crypto_aead *aead)
6139 +{
6140 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6141 + struct device *dev = ctx->dev;
6142 + unsigned int ivsize = crypto_aead_ivsize(aead);
6143 + struct caam_flc *flc;
6144 + u32 *desc;
6145 + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
6146 + ctx->cdata.keylen;
6147 +
6148 + if (!ctx->cdata.keylen || !ctx->authsize)
6149 + return 0;
6150 +
6151 + ctx->cdata.key_virt = ctx->key;
6152 +
6153 + /*
6154 + * RFC4543 encrypt shared descriptor
6155 + * Job Descriptor and Shared Descriptor
6156 + * must fit into the 64-word Descriptor h/w Buffer
6157 + */
6158 + if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
6159 + ctx->cdata.key_inline = true;
6160 + } else {
6161 + ctx->cdata.key_inline = false;
6162 + ctx->cdata.key_dma = ctx->key_dma;
6163 + }
6164 +
6165 + flc = &ctx->flc[ENCRYPT];
6166 + desc = flc->sh_desc;
6167 + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
6168 + true);
6169 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6170 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
6171 + sizeof(flc->flc) + desc_bytes(desc),
6172 + ctx->dir);
6173 +
6174 + /*
6175 + * Job Descriptor and Shared Descriptors
6176 + * must all fit into the 64-word Descriptor h/w Buffer
6177 + */
6178 + if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
6179 + ctx->cdata.key_inline = true;
6180 + } else {
6181 + ctx->cdata.key_inline = false;
6182 + ctx->cdata.key_dma = ctx->key_dma;
6183 + }
6184 +
6185 + flc = &ctx->flc[DECRYPT];
6186 + desc = flc->sh_desc;
6187 + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
6188 + true);
6189 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6190 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
6191 + sizeof(flc->flc) + desc_bytes(desc),
6192 + ctx->dir);
6193 +
6194 + return 0;
6195 +}
6196 +
6197 +static int rfc4543_setauthsize(struct crypto_aead *authenc,
6198 + unsigned int authsize)
6199 +{
6200 + struct caam_ctx *ctx = crypto_aead_ctx(authenc);
6201 +
6202 + ctx->authsize = authsize;
6203 + rfc4543_set_sh_desc(authenc);
6204 +
6205 + return 0;
6206 +}
6207 +
6208 +static int rfc4543_setkey(struct crypto_aead *aead,
6209 + const u8 *key, unsigned int keylen)
6210 +{
6211 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6212 + struct device *dev = ctx->dev;
6213 +
6214 + if (keylen < 4)
6215 + return -EINVAL;
6216 +
6217 +#ifdef DEBUG
6218 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
6219 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
6220 +#endif
6221 +
6222 + memcpy(ctx->key, key, keylen);
6223 + /*
6224 + * The last four bytes of the key material are used as the salt value
6225 + * in the nonce. Update the AES key length.
6226 + */
6227 + ctx->cdata.keylen = keylen - 4;
6228 + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
6229 + ctx->dir);
6230 +
6231 + return rfc4543_set_sh_desc(aead);
6232 +}
6233 +
6234 +static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
6235 + unsigned int keylen)
6236 +{
6237 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6238 + struct caam_skcipher_alg *alg =
6239 + container_of(crypto_skcipher_alg(skcipher),
6240 + struct caam_skcipher_alg, skcipher);
6241 + struct device *dev = ctx->dev;
6242 + struct caam_flc *flc;
6243 + unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
6244 + u32 *desc;
6245 + u32 ctx1_iv_off = 0;
6246 + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
6247 + OP_ALG_AAI_CTR_MOD128) &&
6248 + ((ctx->cdata.algtype & OP_ALG_ALGSEL_MASK) !=
6249 + OP_ALG_ALGSEL_CHACHA20);
6250 + const bool is_rfc3686 = alg->caam.rfc3686;
6251 +
6252 +#ifdef DEBUG
6253 + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
6254 + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
6255 +#endif
6256 + /*
6257 + * AES-CTR needs to load IV in CONTEXT1 reg
6258 + * at an offset of 128bits (16bytes)
6259 + * CONTEXT1[255:128] = IV
6260 + */
6261 + if (ctr_mode)
6262 + ctx1_iv_off = 16;
6263 +
6264 + /*
6265 + * RFC3686 specific:
6266 + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
6267 + * | *key = {KEY, NONCE}
6268 + */
6269 + if (is_rfc3686) {
6270 + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
6271 + keylen -= CTR_RFC3686_NONCE_SIZE;
6272 + }
6273 +
6274 + ctx->cdata.keylen = keylen;
6275 + ctx->cdata.key_virt = key;
6276 + ctx->cdata.key_inline = true;
6277 +
6278 + /* skcipher_encrypt shared descriptor */
6279 + flc = &ctx->flc[ENCRYPT];
6280 + desc = flc->sh_desc;
6281 + cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize,
6282 + is_rfc3686, ctx1_iv_off);
6283 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6284 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
6285 + sizeof(flc->flc) + desc_bytes(desc),
6286 + ctx->dir);
6287 +
6288 + /* skcipher_decrypt shared descriptor */
6289 + flc = &ctx->flc[DECRYPT];
6290 + desc = flc->sh_desc;
6291 + cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize,
6292 + is_rfc3686, ctx1_iv_off);
6293 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6294 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
6295 + sizeof(flc->flc) + desc_bytes(desc),
6296 + ctx->dir);
6297 +
6298 + return 0;
6299 +}
6300 +
6301 +static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
6302 + unsigned int keylen)
6303 +{
6304 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6305 + struct device *dev = ctx->dev;
6306 + struct caam_flc *flc;
6307 + u32 *desc;
6308 +
6309 + if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
6310 + dev_err(dev, "key size mismatch\n");
6311 + crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
6312 + return -EINVAL;
6313 + }
6314 +
6315 + ctx->cdata.keylen = keylen;
6316 + ctx->cdata.key_virt = key;
6317 + ctx->cdata.key_inline = true;
6318 +
6319 + /* xts_skcipher_encrypt shared descriptor */
6320 + flc = &ctx->flc[ENCRYPT];
6321 + desc = flc->sh_desc;
6322 + cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
6323 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6324 + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
6325 + sizeof(flc->flc) + desc_bytes(desc),
6326 + ctx->dir);
6327 +
6328 + /* xts_skcipher_decrypt shared descriptor */
6329 + flc = &ctx->flc[DECRYPT];
6330 + desc = flc->sh_desc;
6331 + cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
6332 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
6333 + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
6334 + sizeof(flc->flc) + desc_bytes(desc),
6335 + ctx->dir);
6336 +
6337 + return 0;
6338 +}
6339 +
6340 +static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
6341 +{
6342 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6343 + struct caam_request *req_ctx = skcipher_request_ctx(req);
6344 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
6345 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
6346 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6347 + struct device *dev = ctx->dev;
6348 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
6349 + GFP_KERNEL : GFP_ATOMIC;
6350 + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
6351 + struct skcipher_edesc *edesc;
6352 + dma_addr_t iv_dma;
6353 + u8 *iv;
6354 + int ivsize = crypto_skcipher_ivsize(skcipher);
6355 + int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
6356 + struct dpaa2_sg_entry *sg_table;
6357 +
6358 + src_nents = sg_nents_for_len(req->src, req->cryptlen);
6359 + if (unlikely(src_nents < 0)) {
6360 + dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
6361 + req->cryptlen);
6362 + return ERR_PTR(src_nents);
6363 + }
6364 +
6365 + if (unlikely(req->dst != req->src)) {
6366 + dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
6367 + if (unlikely(dst_nents < 0)) {
6368 + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
6369 + req->cryptlen);
6370 + return ERR_PTR(dst_nents);
6371 + }
6372 +
6373 + mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
6374 + DMA_TO_DEVICE);
6375 + if (unlikely(!mapped_src_nents)) {
6376 + dev_err(dev, "unable to map source\n");
6377 + return ERR_PTR(-ENOMEM);
6378 + }
6379 +
6380 + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
6381 + DMA_FROM_DEVICE);
6382 + if (unlikely(!mapped_dst_nents)) {
6383 + dev_err(dev, "unable to map destination\n");
6384 + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
6385 + return ERR_PTR(-ENOMEM);
6386 + }
6387 + } else {
6388 + mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
6389 + DMA_BIDIRECTIONAL);
6390 + if (unlikely(!mapped_src_nents)) {
6391 + dev_err(dev, "unable to map source\n");
6392 + return ERR_PTR(-ENOMEM);
6393 + }
6394 + }
6395 +
6396 + qm_sg_ents = 1 + mapped_src_nents;
6397 + dst_sg_idx = qm_sg_ents;
6398 +
6399 + qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
6400 + qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
6401 + if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
6402 + ivsize > CAAM_QI_MEMCACHE_SIZE)) {
6403 + dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
6404 + qm_sg_ents, ivsize);
6405 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
6406 + 0, 0, 0);
6407 + return ERR_PTR(-ENOMEM);
6408 + }
6409 +
6410 + /* allocate space for base edesc, link tables and IV */
6411 + edesc = qi_cache_zalloc(GFP_DMA | flags);
6412 + if (unlikely(!edesc)) {
6413 + dev_err(dev, "could not allocate extended descriptor\n");
6414 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
6415 + 0, 0, 0);
6416 + return ERR_PTR(-ENOMEM);
6417 + }
6418 +
6419 + /* Make sure IV is located in a DMAable area */
6420 + sg_table = &edesc->sgt[0];
6421 + iv = (u8 *)(sg_table + qm_sg_ents);
6422 + memcpy(iv, req->iv, ivsize);
6423 +
6424 + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
6425 + if (dma_mapping_error(dev, iv_dma)) {
6426 + dev_err(dev, "unable to map IV\n");
6427 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
6428 + 0, 0, 0);
6429 + qi_cache_free(edesc);
6430 + return ERR_PTR(-ENOMEM);
6431 + }
6432 +
6433 + edesc->src_nents = src_nents;
6434 + edesc->dst_nents = dst_nents;
6435 + edesc->iv_dma = iv_dma;
6436 + edesc->qm_sg_bytes = qm_sg_bytes;
6437 +
6438 + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
6439 + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
6440 +
6441 + if (mapped_dst_nents > 1)
6442 + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
6443 + dst_sg_idx, 0);
6444 +
6445 + edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
6446 + DMA_TO_DEVICE);
6447 + if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
6448 + dev_err(dev, "unable to map S/G table\n");
6449 + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
6450 + iv_dma, ivsize, 0, 0);
6451 + qi_cache_free(edesc);
6452 + return ERR_PTR(-ENOMEM);
6453 + }
6454 +
6455 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
6456 + dpaa2_fl_set_final(in_fle, true);
6457 + dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
6458 + dpaa2_fl_set_len(out_fle, req->cryptlen);
6459 +
6460 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
6461 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
6462 +
6463 + if (req->src == req->dst) {
6464 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
6465 + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
6466 + sizeof(*sg_table));
6467 + } else if (mapped_dst_nents > 1) {
6468 + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
6469 + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
6470 + sizeof(*sg_table));
6471 + } else {
6472 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
6473 + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
6474 + }
6475 +
6476 + return edesc;
6477 +}
6478 +
6479 +static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
6480 + struct aead_request *req)
6481 +{
6482 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
6483 + int ivsize = crypto_aead_ivsize(aead);
6484 +
6485 + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
6486 + edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
6487 + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
6488 +}
6489 +
6490 +static void tls_unmap(struct device *dev, struct tls_edesc *edesc,
6491 + struct aead_request *req)
6492 +{
6493 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
6494 + int ivsize = crypto_aead_ivsize(tls);
6495 +
6496 + caam_unmap(dev, req->src, edesc->dst, edesc->src_nents,
6497 + edesc->dst_nents, edesc->iv_dma, ivsize, edesc->qm_sg_dma,
6498 + edesc->qm_sg_bytes);
6499 +}
6500 +
6501 +static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
6502 + struct skcipher_request *req)
6503 +{
6504 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6505 + int ivsize = crypto_skcipher_ivsize(skcipher);
6506 +
6507 + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
6508 + edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
6509 +}
6510 +
6511 +static void aead_encrypt_done(void *cbk_ctx, u32 status)
6512 +{
6513 + struct crypto_async_request *areq = cbk_ctx;
6514 + struct aead_request *req = container_of(areq, struct aead_request,
6515 + base);
6516 + struct caam_request *req_ctx = to_caam_req(areq);
6517 + struct aead_edesc *edesc = req_ctx->edesc;
6518 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
6519 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6520 + int ecode = 0;
6521 +
6522 +#ifdef DEBUG
6523 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6524 +#endif
6525 +
6526 + if (unlikely(status)) {
6527 + caam_qi2_strstatus(ctx->dev, status);
6528 + ecode = -EIO;
6529 + }
6530 +
6531 + aead_unmap(ctx->dev, edesc, req);
6532 + qi_cache_free(edesc);
6533 + aead_request_complete(req, ecode);
6534 +}
6535 +
6536 +static void aead_decrypt_done(void *cbk_ctx, u32 status)
6537 +{
6538 + struct crypto_async_request *areq = cbk_ctx;
6539 + struct aead_request *req = container_of(areq, struct aead_request,
6540 + base);
6541 + struct caam_request *req_ctx = to_caam_req(areq);
6542 + struct aead_edesc *edesc = req_ctx->edesc;
6543 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
6544 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6545 + int ecode = 0;
6546 +
6547 +#ifdef DEBUG
6548 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6549 +#endif
6550 +
6551 + if (unlikely(status)) {
6552 + caam_qi2_strstatus(ctx->dev, status);
6553 + /*
6554 + * verify hw auth check passed else return -EBADMSG
6555 + */
6556 + if ((status & JRSTA_CCBERR_ERRID_MASK) ==
6557 + JRSTA_CCBERR_ERRID_ICVCHK)
6558 + ecode = -EBADMSG;
6559 + else
6560 + ecode = -EIO;
6561 + }
6562 +
6563 + aead_unmap(ctx->dev, edesc, req);
6564 + qi_cache_free(edesc);
6565 + aead_request_complete(req, ecode);
6566 +}
6567 +
6568 +static int aead_encrypt(struct aead_request *req)
6569 +{
6570 + struct aead_edesc *edesc;
6571 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
6572 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6573 + struct caam_request *caam_req = aead_request_ctx(req);
6574 + int ret;
6575 +
6576 + /* allocate extended descriptor */
6577 + edesc = aead_edesc_alloc(req, true);
6578 + if (IS_ERR(edesc))
6579 + return PTR_ERR(edesc);
6580 +
6581 + caam_req->flc = &ctx->flc[ENCRYPT];
6582 + caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
6583 + caam_req->cbk = aead_encrypt_done;
6584 + caam_req->ctx = &req->base;
6585 + caam_req->edesc = edesc;
6586 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6587 + if (ret != -EINPROGRESS &&
6588 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6589 + aead_unmap(ctx->dev, edesc, req);
6590 + qi_cache_free(edesc);
6591 + }
6592 +
6593 + return ret;
6594 +}
6595 +
6596 +static int aead_decrypt(struct aead_request *req)
6597 +{
6598 + struct aead_edesc *edesc;
6599 + struct crypto_aead *aead = crypto_aead_reqtfm(req);
6600 + struct caam_ctx *ctx = crypto_aead_ctx(aead);
6601 + struct caam_request *caam_req = aead_request_ctx(req);
6602 + int ret;
6603 +
6604 + /* allocate extended descriptor */
6605 + edesc = aead_edesc_alloc(req, false);
6606 + if (IS_ERR(edesc))
6607 + return PTR_ERR(edesc);
6608 +
6609 + caam_req->flc = &ctx->flc[DECRYPT];
6610 + caam_req->flc_dma = ctx->flc_dma[DECRYPT];
6611 + caam_req->cbk = aead_decrypt_done;
6612 + caam_req->ctx = &req->base;
6613 + caam_req->edesc = edesc;
6614 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6615 + if (ret != -EINPROGRESS &&
6616 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6617 + aead_unmap(ctx->dev, edesc, req);
6618 + qi_cache_free(edesc);
6619 + }
6620 +
6621 + return ret;
6622 +}
6623 +
6624 +static void tls_encrypt_done(void *cbk_ctx, u32 status)
6625 +{
6626 + struct crypto_async_request *areq = cbk_ctx;
6627 + struct aead_request *req = container_of(areq, struct aead_request,
6628 + base);
6629 + struct caam_request *req_ctx = to_caam_req(areq);
6630 + struct tls_edesc *edesc = req_ctx->edesc;
6631 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
6632 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
6633 + int ecode = 0;
6634 +
6635 +#ifdef DEBUG
6636 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6637 +#endif
6638 +
6639 + if (unlikely(status)) {
6640 + caam_qi2_strstatus(ctx->dev, status);
6641 + ecode = -EIO;
6642 + }
6643 +
6644 + tls_unmap(ctx->dev, edesc, req);
6645 + qi_cache_free(edesc);
6646 + aead_request_complete(req, ecode);
6647 +}
6648 +
6649 +static void tls_decrypt_done(void *cbk_ctx, u32 status)
6650 +{
6651 + struct crypto_async_request *areq = cbk_ctx;
6652 + struct aead_request *req = container_of(areq, struct aead_request,
6653 + base);
6654 + struct caam_request *req_ctx = to_caam_req(areq);
6655 + struct tls_edesc *edesc = req_ctx->edesc;
6656 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
6657 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
6658 + int ecode = 0;
6659 +
6660 +#ifdef DEBUG
6661 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6662 +#endif
6663 +
6664 + if (unlikely(status)) {
6665 + caam_qi2_strstatus(ctx->dev, status);
6666 + /*
6667 + * verify hw auth check passed else return -EBADMSG
6668 + */
6669 + if ((status & JRSTA_CCBERR_ERRID_MASK) ==
6670 + JRSTA_CCBERR_ERRID_ICVCHK)
6671 + ecode = -EBADMSG;
6672 + else
6673 + ecode = -EIO;
6674 + }
6675 +
6676 + tls_unmap(ctx->dev, edesc, req);
6677 + qi_cache_free(edesc);
6678 + aead_request_complete(req, ecode);
6679 +}
6680 +
6681 +static int tls_encrypt(struct aead_request *req)
6682 +{
6683 + struct tls_edesc *edesc;
6684 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
6685 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
6686 + struct caam_request *caam_req = aead_request_ctx(req);
6687 + int ret;
6688 +
6689 + /* allocate extended descriptor */
6690 + edesc = tls_edesc_alloc(req, true);
6691 + if (IS_ERR(edesc))
6692 + return PTR_ERR(edesc);
6693 +
6694 + caam_req->flc = &ctx->flc[ENCRYPT];
6695 + caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
6696 + caam_req->cbk = tls_encrypt_done;
6697 + caam_req->ctx = &req->base;
6698 + caam_req->edesc = edesc;
6699 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6700 + if (ret != -EINPROGRESS &&
6701 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6702 + tls_unmap(ctx->dev, edesc, req);
6703 + qi_cache_free(edesc);
6704 + }
6705 +
6706 + return ret;
6707 +}
6708 +
6709 +static int tls_decrypt(struct aead_request *req)
6710 +{
6711 + struct tls_edesc *edesc;
6712 + struct crypto_aead *tls = crypto_aead_reqtfm(req);
6713 + struct caam_ctx *ctx = crypto_aead_ctx(tls);
6714 + struct caam_request *caam_req = aead_request_ctx(req);
6715 + int ret;
6716 +
6717 + /* allocate extended descriptor */
6718 + edesc = tls_edesc_alloc(req, false);
6719 + if (IS_ERR(edesc))
6720 + return PTR_ERR(edesc);
6721 +
6722 + caam_req->flc = &ctx->flc[DECRYPT];
6723 + caam_req->flc_dma = ctx->flc_dma[DECRYPT];
6724 + caam_req->cbk = tls_decrypt_done;
6725 + caam_req->ctx = &req->base;
6726 + caam_req->edesc = edesc;
6727 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6728 + if (ret != -EINPROGRESS &&
6729 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6730 + tls_unmap(ctx->dev, edesc, req);
6731 + qi_cache_free(edesc);
6732 + }
6733 +
6734 + return ret;
6735 +}
6736 +
6737 +static int ipsec_gcm_encrypt(struct aead_request *req)
6738 +{
6739 + if (req->assoclen < 8)
6740 + return -EINVAL;
6741 +
6742 + return aead_encrypt(req);
6743 +}
6744 +
6745 +static int ipsec_gcm_decrypt(struct aead_request *req)
6746 +{
6747 + if (req->assoclen < 8)
6748 + return -EINVAL;
6749 +
6750 + return aead_decrypt(req);
6751 +}
6752 +
6753 +static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
6754 +{
6755 + struct crypto_async_request *areq = cbk_ctx;
6756 + struct skcipher_request *req = skcipher_request_cast(areq);
6757 + struct caam_request *req_ctx = to_caam_req(areq);
6758 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6759 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6760 + struct skcipher_edesc *edesc = req_ctx->edesc;
6761 + int ecode = 0;
6762 + int ivsize = crypto_skcipher_ivsize(skcipher);
6763 +
6764 +#ifdef DEBUG
6765 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6766 +#endif
6767 +
6768 + if (unlikely(status)) {
6769 + caam_qi2_strstatus(ctx->dev, status);
6770 + ecode = -EIO;
6771 + }
6772 +
6773 +#ifdef DEBUG
6774 + print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ",
6775 + DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
6776 + edesc->src_nents > 1 ? 100 : ivsize, 1);
6777 + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
6778 + DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
6779 + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
6780 +#endif
6781 +
6782 + skcipher_unmap(ctx->dev, edesc, req);
6783 +
6784 + /*
6785 + * The crypto API expects us to set the IV (req->iv) to the last
6786 + * ciphertext block. This is used e.g. by the CTS mode.
6787 + */
6788 + scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
6789 + ivsize, 0);
6790 +
6791 + qi_cache_free(edesc);
6792 + skcipher_request_complete(req, ecode);
6793 +}
6794 +
6795 +static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
6796 +{
6797 + struct crypto_async_request *areq = cbk_ctx;
6798 + struct skcipher_request *req = skcipher_request_cast(areq);
6799 + struct caam_request *req_ctx = to_caam_req(areq);
6800 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6801 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6802 + struct skcipher_edesc *edesc = req_ctx->edesc;
6803 + int ecode = 0;
6804 +#ifdef DEBUG
6805 + int ivsize = crypto_skcipher_ivsize(skcipher);
6806 +
6807 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
6808 +#endif
6809 +
6810 + if (unlikely(status)) {
6811 + caam_qi2_strstatus(ctx->dev, status);
6812 + ecode = -EIO;
6813 + }
6814 +
6815 +#ifdef DEBUG
6816 + print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ",
6817 + DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
6818 + edesc->src_nents > 1 ? 100 : ivsize, 1);
6819 + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
6820 + DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
6821 + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
6822 +#endif
6823 +
6824 + skcipher_unmap(ctx->dev, edesc, req);
6825 + qi_cache_free(edesc);
6826 + skcipher_request_complete(req, ecode);
6827 +}
6828 +
6829 +static int skcipher_encrypt(struct skcipher_request *req)
6830 +{
6831 + struct skcipher_edesc *edesc;
6832 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6833 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6834 + struct caam_request *caam_req = skcipher_request_ctx(req);
6835 + int ret;
6836 +
6837 + /* allocate extended descriptor */
6838 + edesc = skcipher_edesc_alloc(req);
6839 + if (IS_ERR(edesc))
6840 + return PTR_ERR(edesc);
6841 +
6842 + caam_req->flc = &ctx->flc[ENCRYPT];
6843 + caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
6844 + caam_req->cbk = skcipher_encrypt_done;
6845 + caam_req->ctx = &req->base;
6846 + caam_req->edesc = edesc;
6847 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6848 + if (ret != -EINPROGRESS &&
6849 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6850 + skcipher_unmap(ctx->dev, edesc, req);
6851 + qi_cache_free(edesc);
6852 + }
6853 +
6854 + return ret;
6855 +}
6856 +
6857 +static int skcipher_decrypt(struct skcipher_request *req)
6858 +{
6859 + struct skcipher_edesc *edesc;
6860 + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
6861 + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
6862 + struct caam_request *caam_req = skcipher_request_ctx(req);
6863 + int ivsize = crypto_skcipher_ivsize(skcipher);
6864 + int ret;
6865 +
6866 + /* allocate extended descriptor */
6867 + edesc = skcipher_edesc_alloc(req);
6868 + if (IS_ERR(edesc))
6869 + return PTR_ERR(edesc);
6870 +
6871 + /*
6872 + * The crypto API expects us to set the IV (req->iv) to the last
6873 + * ciphertext block.
6874 + */
6875 + scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
6876 + ivsize, 0);
6877 +
6878 + caam_req->flc = &ctx->flc[DECRYPT];
6879 + caam_req->flc_dma = ctx->flc_dma[DECRYPT];
6880 + caam_req->cbk = skcipher_decrypt_done;
6881 + caam_req->ctx = &req->base;
6882 + caam_req->edesc = edesc;
6883 + ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
6884 + if (ret != -EINPROGRESS &&
6885 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
6886 + skcipher_unmap(ctx->dev, edesc, req);
6887 + qi_cache_free(edesc);
6888 + }
6889 +
6890 + return ret;
6891 +}
6892 +
6893 +static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
6894 + bool uses_dkp)
6895 +{
6896 + dma_addr_t dma_addr;
6897 + int i;
6898 +
6899 + /* copy descriptor header template value */
6900 + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
6901 + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
6902 +
6903 + ctx->dev = caam->dev;
6904 + ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
6905 +
6906 + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
6907 + offsetof(struct caam_ctx, flc_dma),
6908 + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
6909 + if (dma_mapping_error(ctx->dev, dma_addr)) {
6910 + dev_err(ctx->dev, "unable to map key, shared descriptors\n");
6911 + return -ENOMEM;
6912 + }
6913 +
6914 + for (i = 0; i < NUM_OP; i++)
6915 + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
6916 + ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
6917 +
6918 + return 0;
6919 +}
6920 +
6921 +static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
6922 +{
6923 + struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
6924 + struct caam_skcipher_alg *caam_alg =
6925 + container_of(alg, typeof(*caam_alg), skcipher);
6926 +
6927 + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
6928 + return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
6929 +}
6930 +
6931 +static int caam_cra_init_aead(struct crypto_aead *tfm)
6932 +{
6933 + struct aead_alg *alg = crypto_aead_alg(tfm);
6934 + struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
6935 + aead);
6936 +
6937 + crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
6938 + return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
6939 + (alg->setkey == aead_setkey) ||
6940 + (alg->setkey == tls_setkey));
6941 +}
6942 +
6943 +static void caam_exit_common(struct caam_ctx *ctx)
6944 +{
6945 + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
6946 + offsetof(struct caam_ctx, flc_dma), ctx->dir,
6947 + DMA_ATTR_SKIP_CPU_SYNC);
6948 +}
6949 +
6950 +static void caam_cra_exit(struct crypto_skcipher *tfm)
6951 +{
6952 + caam_exit_common(crypto_skcipher_ctx(tfm));
6953 +}
6954 +
6955 +static void caam_cra_exit_aead(struct crypto_aead *tfm)
6956 +{
6957 + caam_exit_common(crypto_aead_ctx(tfm));
6958 +}
6959 +
6960 +static struct caam_skcipher_alg driver_algs[] = {
6961 + {
6962 + .skcipher = {
6963 + .base = {
6964 + .cra_name = "cbc(aes)",
6965 + .cra_driver_name = "cbc-aes-caam-qi2",
6966 + .cra_blocksize = AES_BLOCK_SIZE,
6967 + },
6968 + .setkey = skcipher_setkey,
6969 + .encrypt = skcipher_encrypt,
6970 + .decrypt = skcipher_decrypt,
6971 + .min_keysize = AES_MIN_KEY_SIZE,
6972 + .max_keysize = AES_MAX_KEY_SIZE,
6973 + .ivsize = AES_BLOCK_SIZE,
6974 + },
6975 + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
6976 + },
6977 + {
6978 + .skcipher = {
6979 + .base = {
6980 + .cra_name = "cbc(des3_ede)",
6981 + .cra_driver_name = "cbc-3des-caam-qi2",
6982 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
6983 + },
6984 + .setkey = skcipher_setkey,
6985 + .encrypt = skcipher_encrypt,
6986 + .decrypt = skcipher_decrypt,
6987 + .min_keysize = DES3_EDE_KEY_SIZE,
6988 + .max_keysize = DES3_EDE_KEY_SIZE,
6989 + .ivsize = DES3_EDE_BLOCK_SIZE,
6990 + },
6991 + .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
6992 + },
6993 + {
6994 + .skcipher = {
6995 + .base = {
6996 + .cra_name = "cbc(des)",
6997 + .cra_driver_name = "cbc-des-caam-qi2",
6998 + .cra_blocksize = DES_BLOCK_SIZE,
6999 + },
7000 + .setkey = skcipher_setkey,
7001 + .encrypt = skcipher_encrypt,
7002 + .decrypt = skcipher_decrypt,
7003 + .min_keysize = DES_KEY_SIZE,
7004 + .max_keysize = DES_KEY_SIZE,
7005 + .ivsize = DES_BLOCK_SIZE,
7006 + },
7007 + .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7008 + },
7009 + {
7010 + .skcipher = {
7011 + .base = {
7012 + .cra_name = "ctr(aes)",
7013 + .cra_driver_name = "ctr-aes-caam-qi2",
7014 + .cra_blocksize = 1,
7015 + },
7016 + .setkey = skcipher_setkey,
7017 + .encrypt = skcipher_encrypt,
7018 + .decrypt = skcipher_decrypt,
7019 + .min_keysize = AES_MIN_KEY_SIZE,
7020 + .max_keysize = AES_MAX_KEY_SIZE,
7021 + .ivsize = AES_BLOCK_SIZE,
7022 + .chunksize = AES_BLOCK_SIZE,
7023 + },
7024 + .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
7025 + OP_ALG_AAI_CTR_MOD128,
7026 + },
7027 + {
7028 + .skcipher = {
7029 + .base = {
7030 + .cra_name = "rfc3686(ctr(aes))",
7031 + .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
7032 + .cra_blocksize = 1,
7033 + },
7034 + .setkey = skcipher_setkey,
7035 + .encrypt = skcipher_encrypt,
7036 + .decrypt = skcipher_decrypt,
7037 + .min_keysize = AES_MIN_KEY_SIZE +
7038 + CTR_RFC3686_NONCE_SIZE,
7039 + .max_keysize = AES_MAX_KEY_SIZE +
7040 + CTR_RFC3686_NONCE_SIZE,
7041 + .ivsize = CTR_RFC3686_IV_SIZE,
7042 + .chunksize = AES_BLOCK_SIZE,
7043 + },
7044 + .caam = {
7045 + .class1_alg_type = OP_ALG_ALGSEL_AES |
7046 + OP_ALG_AAI_CTR_MOD128,
7047 + .rfc3686 = true,
7048 + },
7049 + },
7050 + {
7051 + .skcipher = {
7052 + .base = {
7053 + .cra_name = "xts(aes)",
7054 + .cra_driver_name = "xts-aes-caam-qi2",
7055 + .cra_blocksize = AES_BLOCK_SIZE,
7056 + },
7057 + .setkey = xts_skcipher_setkey,
7058 + .encrypt = skcipher_encrypt,
7059 + .decrypt = skcipher_decrypt,
7060 + .min_keysize = 2 * AES_MIN_KEY_SIZE,
7061 + .max_keysize = 2 * AES_MAX_KEY_SIZE,
7062 + .ivsize = AES_BLOCK_SIZE,
7063 + },
7064 + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
7065 + },
7066 + {
7067 + .skcipher = {
7068 + .base = {
7069 + .cra_name = "chacha20",
7070 + .cra_driver_name = "chacha20-caam-qi2",
7071 + .cra_blocksize = 1,
7072 + },
7073 + .setkey = skcipher_setkey,
7074 + .encrypt = skcipher_encrypt,
7075 + .decrypt = skcipher_decrypt,
7076 + .min_keysize = CHACHA20_KEY_SIZE,
7077 + .max_keysize = CHACHA20_KEY_SIZE,
7078 + .ivsize = CHACHA20_IV_SIZE,
7079 + },
7080 + .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
7081 + },
7082 +};
7083 +
7084 +static struct caam_aead_alg driver_aeads[] = {
7085 + {
7086 + .aead = {
7087 + .base = {
7088 + .cra_name = "rfc4106(gcm(aes))",
7089 + .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
7090 + .cra_blocksize = 1,
7091 + },
7092 + .setkey = rfc4106_setkey,
7093 + .setauthsize = rfc4106_setauthsize,
7094 + .encrypt = ipsec_gcm_encrypt,
7095 + .decrypt = ipsec_gcm_decrypt,
7096 + .ivsize = 8,
7097 + .maxauthsize = AES_BLOCK_SIZE,
7098 + },
7099 + .caam = {
7100 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
7101 + },
7102 + },
7103 + {
7104 + .aead = {
7105 + .base = {
7106 + .cra_name = "rfc4543(gcm(aes))",
7107 + .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
7108 + .cra_blocksize = 1,
7109 + },
7110 + .setkey = rfc4543_setkey,
7111 + .setauthsize = rfc4543_setauthsize,
7112 + .encrypt = ipsec_gcm_encrypt,
7113 + .decrypt = ipsec_gcm_decrypt,
7114 + .ivsize = 8,
7115 + .maxauthsize = AES_BLOCK_SIZE,
7116 + },
7117 + .caam = {
7118 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
7119 + },
7120 + },
7121 + /* Galois Counter Mode */
7122 + {
7123 + .aead = {
7124 + .base = {
7125 + .cra_name = "gcm(aes)",
7126 + .cra_driver_name = "gcm-aes-caam-qi2",
7127 + .cra_blocksize = 1,
7128 + },
7129 + .setkey = gcm_setkey,
7130 + .setauthsize = gcm_setauthsize,
7131 + .encrypt = aead_encrypt,
7132 + .decrypt = aead_decrypt,
7133 + .ivsize = 12,
7134 + .maxauthsize = AES_BLOCK_SIZE,
7135 + },
7136 + .caam = {
7137 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
7138 + }
7139 + },
7140 + /* single-pass ipsec_esp descriptor */
7141 + {
7142 + .aead = {
7143 + .base = {
7144 + .cra_name = "authenc(hmac(md5),cbc(aes))",
7145 + .cra_driver_name = "authenc-hmac-md5-"
7146 + "cbc-aes-caam-qi2",
7147 + .cra_blocksize = AES_BLOCK_SIZE,
7148 + },
7149 + .setkey = aead_setkey,
7150 + .setauthsize = aead_setauthsize,
7151 + .encrypt = aead_encrypt,
7152 + .decrypt = aead_decrypt,
7153 + .ivsize = AES_BLOCK_SIZE,
7154 + .maxauthsize = MD5_DIGEST_SIZE,
7155 + },
7156 + .caam = {
7157 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7158 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7159 + OP_ALG_AAI_HMAC_PRECOMP,
7160 + }
7161 + },
7162 + {
7163 + .aead = {
7164 + .base = {
7165 + .cra_name = "echainiv(authenc(hmac(md5),"
7166 + "cbc(aes)))",
7167 + .cra_driver_name = "echainiv-authenc-hmac-md5-"
7168 + "cbc-aes-caam-qi2",
7169 + .cra_blocksize = AES_BLOCK_SIZE,
7170 + },
7171 + .setkey = aead_setkey,
7172 + .setauthsize = aead_setauthsize,
7173 + .encrypt = aead_encrypt,
7174 + .decrypt = aead_decrypt,
7175 + .ivsize = AES_BLOCK_SIZE,
7176 + .maxauthsize = MD5_DIGEST_SIZE,
7177 + },
7178 + .caam = {
7179 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7180 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7181 + OP_ALG_AAI_HMAC_PRECOMP,
7182 + .geniv = true,
7183 + }
7184 + },
7185 + {
7186 + .aead = {
7187 + .base = {
7188 + .cra_name = "authenc(hmac(sha1),cbc(aes))",
7189 + .cra_driver_name = "authenc-hmac-sha1-"
7190 + "cbc-aes-caam-qi2",
7191 + .cra_blocksize = AES_BLOCK_SIZE,
7192 + },
7193 + .setkey = aead_setkey,
7194 + .setauthsize = aead_setauthsize,
7195 + .encrypt = aead_encrypt,
7196 + .decrypt = aead_decrypt,
7197 + .ivsize = AES_BLOCK_SIZE,
7198 + .maxauthsize = SHA1_DIGEST_SIZE,
7199 + },
7200 + .caam = {
7201 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7202 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7203 + OP_ALG_AAI_HMAC_PRECOMP,
7204 + }
7205 + },
7206 + {
7207 + .aead = {
7208 + .base = {
7209 + .cra_name = "echainiv(authenc(hmac(sha1),"
7210 + "cbc(aes)))",
7211 + .cra_driver_name = "echainiv-authenc-"
7212 + "hmac-sha1-cbc-aes-caam-qi2",
7213 + .cra_blocksize = AES_BLOCK_SIZE,
7214 + },
7215 + .setkey = aead_setkey,
7216 + .setauthsize = aead_setauthsize,
7217 + .encrypt = aead_encrypt,
7218 + .decrypt = aead_decrypt,
7219 + .ivsize = AES_BLOCK_SIZE,
7220 + .maxauthsize = SHA1_DIGEST_SIZE,
7221 + },
7222 + .caam = {
7223 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7224 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7225 + OP_ALG_AAI_HMAC_PRECOMP,
7226 + .geniv = true,
7227 + },
7228 + },
7229 + {
7230 + .aead = {
7231 + .base = {
7232 + .cra_name = "authenc(hmac(sha224),cbc(aes))",
7233 + .cra_driver_name = "authenc-hmac-sha224-"
7234 + "cbc-aes-caam-qi2",
7235 + .cra_blocksize = AES_BLOCK_SIZE,
7236 + },
7237 + .setkey = aead_setkey,
7238 + .setauthsize = aead_setauthsize,
7239 + .encrypt = aead_encrypt,
7240 + .decrypt = aead_decrypt,
7241 + .ivsize = AES_BLOCK_SIZE,
7242 + .maxauthsize = SHA224_DIGEST_SIZE,
7243 + },
7244 + .caam = {
7245 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7246 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7247 + OP_ALG_AAI_HMAC_PRECOMP,
7248 + }
7249 + },
7250 + {
7251 + .aead = {
7252 + .base = {
7253 + .cra_name = "echainiv(authenc(hmac(sha224),"
7254 + "cbc(aes)))",
7255 + .cra_driver_name = "echainiv-authenc-"
7256 + "hmac-sha224-cbc-aes-caam-qi2",
7257 + .cra_blocksize = AES_BLOCK_SIZE,
7258 + },
7259 + .setkey = aead_setkey,
7260 + .setauthsize = aead_setauthsize,
7261 + .encrypt = aead_encrypt,
7262 + .decrypt = aead_decrypt,
7263 + .ivsize = AES_BLOCK_SIZE,
7264 + .maxauthsize = SHA224_DIGEST_SIZE,
7265 + },
7266 + .caam = {
7267 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7268 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7269 + OP_ALG_AAI_HMAC_PRECOMP,
7270 + .geniv = true,
7271 + }
7272 + },
7273 + {
7274 + .aead = {
7275 + .base = {
7276 + .cra_name = "authenc(hmac(sha256),cbc(aes))",
7277 + .cra_driver_name = "authenc-hmac-sha256-"
7278 + "cbc-aes-caam-qi2",
7279 + .cra_blocksize = AES_BLOCK_SIZE,
7280 + },
7281 + .setkey = aead_setkey,
7282 + .setauthsize = aead_setauthsize,
7283 + .encrypt = aead_encrypt,
7284 + .decrypt = aead_decrypt,
7285 + .ivsize = AES_BLOCK_SIZE,
7286 + .maxauthsize = SHA256_DIGEST_SIZE,
7287 + },
7288 + .caam = {
7289 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7290 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7291 + OP_ALG_AAI_HMAC_PRECOMP,
7292 + }
7293 + },
7294 + {
7295 + .aead = {
7296 + .base = {
7297 + .cra_name = "echainiv(authenc(hmac(sha256),"
7298 + "cbc(aes)))",
7299 + .cra_driver_name = "echainiv-authenc-"
7300 + "hmac-sha256-cbc-aes-"
7301 + "caam-qi2",
7302 + .cra_blocksize = AES_BLOCK_SIZE,
7303 + },
7304 + .setkey = aead_setkey,
7305 + .setauthsize = aead_setauthsize,
7306 + .encrypt = aead_encrypt,
7307 + .decrypt = aead_decrypt,
7308 + .ivsize = AES_BLOCK_SIZE,
7309 + .maxauthsize = SHA256_DIGEST_SIZE,
7310 + },
7311 + .caam = {
7312 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7313 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7314 + OP_ALG_AAI_HMAC_PRECOMP,
7315 + .geniv = true,
7316 + }
7317 + },
7318 + {
7319 + .aead = {
7320 + .base = {
7321 + .cra_name = "authenc(hmac(sha384),cbc(aes))",
7322 + .cra_driver_name = "authenc-hmac-sha384-"
7323 + "cbc-aes-caam-qi2",
7324 + .cra_blocksize = AES_BLOCK_SIZE,
7325 + },
7326 + .setkey = aead_setkey,
7327 + .setauthsize = aead_setauthsize,
7328 + .encrypt = aead_encrypt,
7329 + .decrypt = aead_decrypt,
7330 + .ivsize = AES_BLOCK_SIZE,
7331 + .maxauthsize = SHA384_DIGEST_SIZE,
7332 + },
7333 + .caam = {
7334 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7335 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7336 + OP_ALG_AAI_HMAC_PRECOMP,
7337 + }
7338 + },
7339 + {
7340 + .aead = {
7341 + .base = {
7342 + .cra_name = "echainiv(authenc(hmac(sha384),"
7343 + "cbc(aes)))",
7344 + .cra_driver_name = "echainiv-authenc-"
7345 + "hmac-sha384-cbc-aes-"
7346 + "caam-qi2",
7347 + .cra_blocksize = AES_BLOCK_SIZE,
7348 + },
7349 + .setkey = aead_setkey,
7350 + .setauthsize = aead_setauthsize,
7351 + .encrypt = aead_encrypt,
7352 + .decrypt = aead_decrypt,
7353 + .ivsize = AES_BLOCK_SIZE,
7354 + .maxauthsize = SHA384_DIGEST_SIZE,
7355 + },
7356 + .caam = {
7357 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7358 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7359 + OP_ALG_AAI_HMAC_PRECOMP,
7360 + .geniv = true,
7361 + }
7362 + },
7363 + {
7364 + .aead = {
7365 + .base = {
7366 + .cra_name = "authenc(hmac(sha512),cbc(aes))",
7367 + .cra_driver_name = "authenc-hmac-sha512-"
7368 + "cbc-aes-caam-qi2",
7369 + .cra_blocksize = AES_BLOCK_SIZE,
7370 + },
7371 + .setkey = aead_setkey,
7372 + .setauthsize = aead_setauthsize,
7373 + .encrypt = aead_encrypt,
7374 + .decrypt = aead_decrypt,
7375 + .ivsize = AES_BLOCK_SIZE,
7376 + .maxauthsize = SHA512_DIGEST_SIZE,
7377 + },
7378 + .caam = {
7379 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7380 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7381 + OP_ALG_AAI_HMAC_PRECOMP,
7382 + }
7383 + },
7384 + {
7385 + .aead = {
7386 + .base = {
7387 + .cra_name = "echainiv(authenc(hmac(sha512),"
7388 + "cbc(aes)))",
7389 + .cra_driver_name = "echainiv-authenc-"
7390 + "hmac-sha512-cbc-aes-"
7391 + "caam-qi2",
7392 + .cra_blocksize = AES_BLOCK_SIZE,
7393 + },
7394 + .setkey = aead_setkey,
7395 + .setauthsize = aead_setauthsize,
7396 + .encrypt = aead_encrypt,
7397 + .decrypt = aead_decrypt,
7398 + .ivsize = AES_BLOCK_SIZE,
7399 + .maxauthsize = SHA512_DIGEST_SIZE,
7400 + },
7401 + .caam = {
7402 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
7403 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7404 + OP_ALG_AAI_HMAC_PRECOMP,
7405 + .geniv = true,
7406 + }
7407 + },
7408 + {
7409 + .aead = {
7410 + .base = {
7411 + .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
7412 + .cra_driver_name = "authenc-hmac-md5-"
7413 + "cbc-des3_ede-caam-qi2",
7414 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7415 + },
7416 + .setkey = aead_setkey,
7417 + .setauthsize = aead_setauthsize,
7418 + .encrypt = aead_encrypt,
7419 + .decrypt = aead_decrypt,
7420 + .ivsize = DES3_EDE_BLOCK_SIZE,
7421 + .maxauthsize = MD5_DIGEST_SIZE,
7422 + },
7423 + .caam = {
7424 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7425 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7426 + OP_ALG_AAI_HMAC_PRECOMP,
7427 + }
7428 + },
7429 + {
7430 + .aead = {
7431 + .base = {
7432 + .cra_name = "echainiv(authenc(hmac(md5),"
7433 + "cbc(des3_ede)))",
7434 + .cra_driver_name = "echainiv-authenc-hmac-md5-"
7435 + "cbc-des3_ede-caam-qi2",
7436 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7437 + },
7438 + .setkey = aead_setkey,
7439 + .setauthsize = aead_setauthsize,
7440 + .encrypt = aead_encrypt,
7441 + .decrypt = aead_decrypt,
7442 + .ivsize = DES3_EDE_BLOCK_SIZE,
7443 + .maxauthsize = MD5_DIGEST_SIZE,
7444 + },
7445 + .caam = {
7446 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7447 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7448 + OP_ALG_AAI_HMAC_PRECOMP,
7449 + .geniv = true,
7450 + }
7451 + },
7452 + {
7453 + .aead = {
7454 + .base = {
7455 + .cra_name = "authenc(hmac(sha1),"
7456 + "cbc(des3_ede))",
7457 + .cra_driver_name = "authenc-hmac-sha1-"
7458 + "cbc-des3_ede-caam-qi2",
7459 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7460 + },
7461 + .setkey = aead_setkey,
7462 + .setauthsize = aead_setauthsize,
7463 + .encrypt = aead_encrypt,
7464 + .decrypt = aead_decrypt,
7465 + .ivsize = DES3_EDE_BLOCK_SIZE,
7466 + .maxauthsize = SHA1_DIGEST_SIZE,
7467 + },
7468 + .caam = {
7469 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7470 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7471 + OP_ALG_AAI_HMAC_PRECOMP,
7472 + },
7473 + },
7474 + {
7475 + .aead = {
7476 + .base = {
7477 + .cra_name = "echainiv(authenc(hmac(sha1),"
7478 + "cbc(des3_ede)))",
7479 + .cra_driver_name = "echainiv-authenc-"
7480 + "hmac-sha1-"
7481 + "cbc-des3_ede-caam-qi2",
7482 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7483 + },
7484 + .setkey = aead_setkey,
7485 + .setauthsize = aead_setauthsize,
7486 + .encrypt = aead_encrypt,
7487 + .decrypt = aead_decrypt,
7488 + .ivsize = DES3_EDE_BLOCK_SIZE,
7489 + .maxauthsize = SHA1_DIGEST_SIZE,
7490 + },
7491 + .caam = {
7492 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7493 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7494 + OP_ALG_AAI_HMAC_PRECOMP,
7495 + .geniv = true,
7496 + }
7497 + },
7498 + {
7499 + .aead = {
7500 + .base = {
7501 + .cra_name = "authenc(hmac(sha224),"
7502 + "cbc(des3_ede))",
7503 + .cra_driver_name = "authenc-hmac-sha224-"
7504 + "cbc-des3_ede-caam-qi2",
7505 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7506 + },
7507 + .setkey = aead_setkey,
7508 + .setauthsize = aead_setauthsize,
7509 + .encrypt = aead_encrypt,
7510 + .decrypt = aead_decrypt,
7511 + .ivsize = DES3_EDE_BLOCK_SIZE,
7512 + .maxauthsize = SHA224_DIGEST_SIZE,
7513 + },
7514 + .caam = {
7515 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7516 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7517 + OP_ALG_AAI_HMAC_PRECOMP,
7518 + },
7519 + },
7520 + {
7521 + .aead = {
7522 + .base = {
7523 + .cra_name = "echainiv(authenc(hmac(sha224),"
7524 + "cbc(des3_ede)))",
7525 + .cra_driver_name = "echainiv-authenc-"
7526 + "hmac-sha224-"
7527 + "cbc-des3_ede-caam-qi2",
7528 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7529 + },
7530 + .setkey = aead_setkey,
7531 + .setauthsize = aead_setauthsize,
7532 + .encrypt = aead_encrypt,
7533 + .decrypt = aead_decrypt,
7534 + .ivsize = DES3_EDE_BLOCK_SIZE,
7535 + .maxauthsize = SHA224_DIGEST_SIZE,
7536 + },
7537 + .caam = {
7538 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7539 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7540 + OP_ALG_AAI_HMAC_PRECOMP,
7541 + .geniv = true,
7542 + }
7543 + },
7544 + {
7545 + .aead = {
7546 + .base = {
7547 + .cra_name = "authenc(hmac(sha256),"
7548 + "cbc(des3_ede))",
7549 + .cra_driver_name = "authenc-hmac-sha256-"
7550 + "cbc-des3_ede-caam-qi2",
7551 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7552 + },
7553 + .setkey = aead_setkey,
7554 + .setauthsize = aead_setauthsize,
7555 + .encrypt = aead_encrypt,
7556 + .decrypt = aead_decrypt,
7557 + .ivsize = DES3_EDE_BLOCK_SIZE,
7558 + .maxauthsize = SHA256_DIGEST_SIZE,
7559 + },
7560 + .caam = {
7561 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7562 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7563 + OP_ALG_AAI_HMAC_PRECOMP,
7564 + },
7565 + },
7566 + {
7567 + .aead = {
7568 + .base = {
7569 + .cra_name = "echainiv(authenc(hmac(sha256),"
7570 + "cbc(des3_ede)))",
7571 + .cra_driver_name = "echainiv-authenc-"
7572 + "hmac-sha256-"
7573 + "cbc-des3_ede-caam-qi2",
7574 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7575 + },
7576 + .setkey = aead_setkey,
7577 + .setauthsize = aead_setauthsize,
7578 + .encrypt = aead_encrypt,
7579 + .decrypt = aead_decrypt,
7580 + .ivsize = DES3_EDE_BLOCK_SIZE,
7581 + .maxauthsize = SHA256_DIGEST_SIZE,
7582 + },
7583 + .caam = {
7584 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7585 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7586 + OP_ALG_AAI_HMAC_PRECOMP,
7587 + .geniv = true,
7588 + }
7589 + },
7590 + {
7591 + .aead = {
7592 + .base = {
7593 + .cra_name = "authenc(hmac(sha384),"
7594 + "cbc(des3_ede))",
7595 + .cra_driver_name = "authenc-hmac-sha384-"
7596 + "cbc-des3_ede-caam-qi2",
7597 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7598 + },
7599 + .setkey = aead_setkey,
7600 + .setauthsize = aead_setauthsize,
7601 + .encrypt = aead_encrypt,
7602 + .decrypt = aead_decrypt,
7603 + .ivsize = DES3_EDE_BLOCK_SIZE,
7604 + .maxauthsize = SHA384_DIGEST_SIZE,
7605 + },
7606 + .caam = {
7607 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7608 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7609 + OP_ALG_AAI_HMAC_PRECOMP,
7610 + },
7611 + },
7612 + {
7613 + .aead = {
7614 + .base = {
7615 + .cra_name = "echainiv(authenc(hmac(sha384),"
7616 + "cbc(des3_ede)))",
7617 + .cra_driver_name = "echainiv-authenc-"
7618 + "hmac-sha384-"
7619 + "cbc-des3_ede-caam-qi2",
7620 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7621 + },
7622 + .setkey = aead_setkey,
7623 + .setauthsize = aead_setauthsize,
7624 + .encrypt = aead_encrypt,
7625 + .decrypt = aead_decrypt,
7626 + .ivsize = DES3_EDE_BLOCK_SIZE,
7627 + .maxauthsize = SHA384_DIGEST_SIZE,
7628 + },
7629 + .caam = {
7630 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7631 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7632 + OP_ALG_AAI_HMAC_PRECOMP,
7633 + .geniv = true,
7634 + }
7635 + },
7636 + {
7637 + .aead = {
7638 + .base = {
7639 + .cra_name = "authenc(hmac(sha512),"
7640 + "cbc(des3_ede))",
7641 + .cra_driver_name = "authenc-hmac-sha512-"
7642 + "cbc-des3_ede-caam-qi2",
7643 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7644 + },
7645 + .setkey = aead_setkey,
7646 + .setauthsize = aead_setauthsize,
7647 + .encrypt = aead_encrypt,
7648 + .decrypt = aead_decrypt,
7649 + .ivsize = DES3_EDE_BLOCK_SIZE,
7650 + .maxauthsize = SHA512_DIGEST_SIZE,
7651 + },
7652 + .caam = {
7653 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7654 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7655 + OP_ALG_AAI_HMAC_PRECOMP,
7656 + },
7657 + },
7658 + {
7659 + .aead = {
7660 + .base = {
7661 + .cra_name = "echainiv(authenc(hmac(sha512),"
7662 + "cbc(des3_ede)))",
7663 + .cra_driver_name = "echainiv-authenc-"
7664 + "hmac-sha512-"
7665 + "cbc-des3_ede-caam-qi2",
7666 + .cra_blocksize = DES3_EDE_BLOCK_SIZE,
7667 + },
7668 + .setkey = aead_setkey,
7669 + .setauthsize = aead_setauthsize,
7670 + .encrypt = aead_encrypt,
7671 + .decrypt = aead_decrypt,
7672 + .ivsize = DES3_EDE_BLOCK_SIZE,
7673 + .maxauthsize = SHA512_DIGEST_SIZE,
7674 + },
7675 + .caam = {
7676 + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
7677 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7678 + OP_ALG_AAI_HMAC_PRECOMP,
7679 + .geniv = true,
7680 + }
7681 + },
7682 + {
7683 + .aead = {
7684 + .base = {
7685 + .cra_name = "authenc(hmac(md5),cbc(des))",
7686 + .cra_driver_name = "authenc-hmac-md5-"
7687 + "cbc-des-caam-qi2",
7688 + .cra_blocksize = DES_BLOCK_SIZE,
7689 + },
7690 + .setkey = aead_setkey,
7691 + .setauthsize = aead_setauthsize,
7692 + .encrypt = aead_encrypt,
7693 + .decrypt = aead_decrypt,
7694 + .ivsize = DES_BLOCK_SIZE,
7695 + .maxauthsize = MD5_DIGEST_SIZE,
7696 + },
7697 + .caam = {
7698 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7699 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7700 + OP_ALG_AAI_HMAC_PRECOMP,
7701 + },
7702 + },
7703 + {
7704 + .aead = {
7705 + .base = {
7706 + .cra_name = "echainiv(authenc(hmac(md5),"
7707 + "cbc(des)))",
7708 + .cra_driver_name = "echainiv-authenc-hmac-md5-"
7709 + "cbc-des-caam-qi2",
7710 + .cra_blocksize = DES_BLOCK_SIZE,
7711 + },
7712 + .setkey = aead_setkey,
7713 + .setauthsize = aead_setauthsize,
7714 + .encrypt = aead_encrypt,
7715 + .decrypt = aead_decrypt,
7716 + .ivsize = DES_BLOCK_SIZE,
7717 + .maxauthsize = MD5_DIGEST_SIZE,
7718 + },
7719 + .caam = {
7720 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7721 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7722 + OP_ALG_AAI_HMAC_PRECOMP,
7723 + .geniv = true,
7724 + }
7725 + },
7726 + {
7727 + .aead = {
7728 + .base = {
7729 + .cra_name = "authenc(hmac(sha1),cbc(des))",
7730 + .cra_driver_name = "authenc-hmac-sha1-"
7731 + "cbc-des-caam-qi2",
7732 + .cra_blocksize = DES_BLOCK_SIZE,
7733 + },
7734 + .setkey = aead_setkey,
7735 + .setauthsize = aead_setauthsize,
7736 + .encrypt = aead_encrypt,
7737 + .decrypt = aead_decrypt,
7738 + .ivsize = DES_BLOCK_SIZE,
7739 + .maxauthsize = SHA1_DIGEST_SIZE,
7740 + },
7741 + .caam = {
7742 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7743 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7744 + OP_ALG_AAI_HMAC_PRECOMP,
7745 + },
7746 + },
7747 + {
7748 + .aead = {
7749 + .base = {
7750 + .cra_name = "echainiv(authenc(hmac(sha1),"
7751 + "cbc(des)))",
7752 + .cra_driver_name = "echainiv-authenc-"
7753 + "hmac-sha1-cbc-des-caam-qi2",
7754 + .cra_blocksize = DES_BLOCK_SIZE,
7755 + },
7756 + .setkey = aead_setkey,
7757 + .setauthsize = aead_setauthsize,
7758 + .encrypt = aead_encrypt,
7759 + .decrypt = aead_decrypt,
7760 + .ivsize = DES_BLOCK_SIZE,
7761 + .maxauthsize = SHA1_DIGEST_SIZE,
7762 + },
7763 + .caam = {
7764 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7765 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
7766 + OP_ALG_AAI_HMAC_PRECOMP,
7767 + .geniv = true,
7768 + }
7769 + },
7770 + {
7771 + .aead = {
7772 + .base = {
7773 + .cra_name = "authenc(hmac(sha224),cbc(des))",
7774 + .cra_driver_name = "authenc-hmac-sha224-"
7775 + "cbc-des-caam-qi2",
7776 + .cra_blocksize = DES_BLOCK_SIZE,
7777 + },
7778 + .setkey = aead_setkey,
7779 + .setauthsize = aead_setauthsize,
7780 + .encrypt = aead_encrypt,
7781 + .decrypt = aead_decrypt,
7782 + .ivsize = DES_BLOCK_SIZE,
7783 + .maxauthsize = SHA224_DIGEST_SIZE,
7784 + },
7785 + .caam = {
7786 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7787 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7788 + OP_ALG_AAI_HMAC_PRECOMP,
7789 + },
7790 + },
7791 + {
7792 + .aead = {
7793 + .base = {
7794 + .cra_name = "echainiv(authenc(hmac(sha224),"
7795 + "cbc(des)))",
7796 + .cra_driver_name = "echainiv-authenc-"
7797 + "hmac-sha224-cbc-des-"
7798 + "caam-qi2",
7799 + .cra_blocksize = DES_BLOCK_SIZE,
7800 + },
7801 + .setkey = aead_setkey,
7802 + .setauthsize = aead_setauthsize,
7803 + .encrypt = aead_encrypt,
7804 + .decrypt = aead_decrypt,
7805 + .ivsize = DES_BLOCK_SIZE,
7806 + .maxauthsize = SHA224_DIGEST_SIZE,
7807 + },
7808 + .caam = {
7809 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7810 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
7811 + OP_ALG_AAI_HMAC_PRECOMP,
7812 + .geniv = true,
7813 + }
7814 + },
7815 + {
7816 + .aead = {
7817 + .base = {
7818 + .cra_name = "authenc(hmac(sha256),cbc(des))",
7819 + .cra_driver_name = "authenc-hmac-sha256-"
7820 + "cbc-des-caam-qi2",
7821 + .cra_blocksize = DES_BLOCK_SIZE,
7822 + },
7823 + .setkey = aead_setkey,
7824 + .setauthsize = aead_setauthsize,
7825 + .encrypt = aead_encrypt,
7826 + .decrypt = aead_decrypt,
7827 + .ivsize = DES_BLOCK_SIZE,
7828 + .maxauthsize = SHA256_DIGEST_SIZE,
7829 + },
7830 + .caam = {
7831 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7832 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7833 + OP_ALG_AAI_HMAC_PRECOMP,
7834 + },
7835 + },
7836 + {
7837 + .aead = {
7838 + .base = {
7839 + .cra_name = "echainiv(authenc(hmac(sha256),"
7840 + "cbc(des)))",
7841 + .cra_driver_name = "echainiv-authenc-"
7842 + "hmac-sha256-cbc-desi-"
7843 + "caam-qi2",
7844 + .cra_blocksize = DES_BLOCK_SIZE,
7845 + },
7846 + .setkey = aead_setkey,
7847 + .setauthsize = aead_setauthsize,
7848 + .encrypt = aead_encrypt,
7849 + .decrypt = aead_decrypt,
7850 + .ivsize = DES_BLOCK_SIZE,
7851 + .maxauthsize = SHA256_DIGEST_SIZE,
7852 + },
7853 + .caam = {
7854 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7855 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
7856 + OP_ALG_AAI_HMAC_PRECOMP,
7857 + .geniv = true,
7858 + },
7859 + },
7860 + {
7861 + .aead = {
7862 + .base = {
7863 + .cra_name = "authenc(hmac(sha384),cbc(des))",
7864 + .cra_driver_name = "authenc-hmac-sha384-"
7865 + "cbc-des-caam-qi2",
7866 + .cra_blocksize = DES_BLOCK_SIZE,
7867 + },
7868 + .setkey = aead_setkey,
7869 + .setauthsize = aead_setauthsize,
7870 + .encrypt = aead_encrypt,
7871 + .decrypt = aead_decrypt,
7872 + .ivsize = DES_BLOCK_SIZE,
7873 + .maxauthsize = SHA384_DIGEST_SIZE,
7874 + },
7875 + .caam = {
7876 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7877 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7878 + OP_ALG_AAI_HMAC_PRECOMP,
7879 + },
7880 + },
7881 + {
7882 + .aead = {
7883 + .base = {
7884 + .cra_name = "echainiv(authenc(hmac(sha384),"
7885 + "cbc(des)))",
7886 + .cra_driver_name = "echainiv-authenc-"
7887 + "hmac-sha384-cbc-des-"
7888 + "caam-qi2",
7889 + .cra_blocksize = DES_BLOCK_SIZE,
7890 + },
7891 + .setkey = aead_setkey,
7892 + .setauthsize = aead_setauthsize,
7893 + .encrypt = aead_encrypt,
7894 + .decrypt = aead_decrypt,
7895 + .ivsize = DES_BLOCK_SIZE,
7896 + .maxauthsize = SHA384_DIGEST_SIZE,
7897 + },
7898 + .caam = {
7899 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7900 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
7901 + OP_ALG_AAI_HMAC_PRECOMP,
7902 + .geniv = true,
7903 + }
7904 + },
7905 + {
7906 + .aead = {
7907 + .base = {
7908 + .cra_name = "authenc(hmac(sha512),cbc(des))",
7909 + .cra_driver_name = "authenc-hmac-sha512-"
7910 + "cbc-des-caam-qi2",
7911 + .cra_blocksize = DES_BLOCK_SIZE,
7912 + },
7913 + .setkey = aead_setkey,
7914 + .setauthsize = aead_setauthsize,
7915 + .encrypt = aead_encrypt,
7916 + .decrypt = aead_decrypt,
7917 + .ivsize = DES_BLOCK_SIZE,
7918 + .maxauthsize = SHA512_DIGEST_SIZE,
7919 + },
7920 + .caam = {
7921 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7922 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7923 + OP_ALG_AAI_HMAC_PRECOMP,
7924 + }
7925 + },
7926 + {
7927 + .aead = {
7928 + .base = {
7929 + .cra_name = "echainiv(authenc(hmac(sha512),"
7930 + "cbc(des)))",
7931 + .cra_driver_name = "echainiv-authenc-"
7932 + "hmac-sha512-cbc-des-"
7933 + "caam-qi2",
7934 + .cra_blocksize = DES_BLOCK_SIZE,
7935 + },
7936 + .setkey = aead_setkey,
7937 + .setauthsize = aead_setauthsize,
7938 + .encrypt = aead_encrypt,
7939 + .decrypt = aead_decrypt,
7940 + .ivsize = DES_BLOCK_SIZE,
7941 + .maxauthsize = SHA512_DIGEST_SIZE,
7942 + },
7943 + .caam = {
7944 + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
7945 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
7946 + OP_ALG_AAI_HMAC_PRECOMP,
7947 + .geniv = true,
7948 + }
7949 + },
7950 + {
7951 + .aead = {
7952 + .base = {
7953 + .cra_name = "authenc(hmac(md5),"
7954 + "rfc3686(ctr(aes)))",
7955 + .cra_driver_name = "authenc-hmac-md5-"
7956 + "rfc3686-ctr-aes-caam-qi2",
7957 + .cra_blocksize = 1,
7958 + },
7959 + .setkey = aead_setkey,
7960 + .setauthsize = aead_setauthsize,
7961 + .encrypt = aead_encrypt,
7962 + .decrypt = aead_decrypt,
7963 + .ivsize = CTR_RFC3686_IV_SIZE,
7964 + .maxauthsize = MD5_DIGEST_SIZE,
7965 + },
7966 + .caam = {
7967 + .class1_alg_type = OP_ALG_ALGSEL_AES |
7968 + OP_ALG_AAI_CTR_MOD128,
7969 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7970 + OP_ALG_AAI_HMAC_PRECOMP,
7971 + .rfc3686 = true,
7972 + },
7973 + },
7974 + {
7975 + .aead = {
7976 + .base = {
7977 + .cra_name = "seqiv(authenc("
7978 + "hmac(md5),rfc3686(ctr(aes))))",
7979 + .cra_driver_name = "seqiv-authenc-hmac-md5-"
7980 + "rfc3686-ctr-aes-caam-qi2",
7981 + .cra_blocksize = 1,
7982 + },
7983 + .setkey = aead_setkey,
7984 + .setauthsize = aead_setauthsize,
7985 + .encrypt = aead_encrypt,
7986 + .decrypt = aead_decrypt,
7987 + .ivsize = CTR_RFC3686_IV_SIZE,
7988 + .maxauthsize = MD5_DIGEST_SIZE,
7989 + },
7990 + .caam = {
7991 + .class1_alg_type = OP_ALG_ALGSEL_AES |
7992 + OP_ALG_AAI_CTR_MOD128,
7993 + .class2_alg_type = OP_ALG_ALGSEL_MD5 |
7994 + OP_ALG_AAI_HMAC_PRECOMP,
7995 + .rfc3686 = true,
7996 + .geniv = true,
7997 + },
7998 + },
7999 + {
8000 + .aead = {
8001 + .base = {
8002 + .cra_name = "authenc(hmac(sha1),"
8003 + "rfc3686(ctr(aes)))",
8004 + .cra_driver_name = "authenc-hmac-sha1-"
8005 + "rfc3686-ctr-aes-caam-qi2",
8006 + .cra_blocksize = 1,
8007 + },
8008 + .setkey = aead_setkey,
8009 + .setauthsize = aead_setauthsize,
8010 + .encrypt = aead_encrypt,
8011 + .decrypt = aead_decrypt,
8012 + .ivsize = CTR_RFC3686_IV_SIZE,
8013 + .maxauthsize = SHA1_DIGEST_SIZE,
8014 + },
8015 + .caam = {
8016 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8017 + OP_ALG_AAI_CTR_MOD128,
8018 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
8019 + OP_ALG_AAI_HMAC_PRECOMP,
8020 + .rfc3686 = true,
8021 + },
8022 + },
8023 + {
8024 + .aead = {
8025 + .base = {
8026 + .cra_name = "seqiv(authenc("
8027 + "hmac(sha1),rfc3686(ctr(aes))))",
8028 + .cra_driver_name = "seqiv-authenc-hmac-sha1-"
8029 + "rfc3686-ctr-aes-caam-qi2",
8030 + .cra_blocksize = 1,
8031 + },
8032 + .setkey = aead_setkey,
8033 + .setauthsize = aead_setauthsize,
8034 + .encrypt = aead_encrypt,
8035 + .decrypt = aead_decrypt,
8036 + .ivsize = CTR_RFC3686_IV_SIZE,
8037 + .maxauthsize = SHA1_DIGEST_SIZE,
8038 + },
8039 + .caam = {
8040 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8041 + OP_ALG_AAI_CTR_MOD128,
8042 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
8043 + OP_ALG_AAI_HMAC_PRECOMP,
8044 + .rfc3686 = true,
8045 + .geniv = true,
8046 + },
8047 + },
8048 + {
8049 + .aead = {
8050 + .base = {
8051 + .cra_name = "authenc(hmac(sha224),"
8052 + "rfc3686(ctr(aes)))",
8053 + .cra_driver_name = "authenc-hmac-sha224-"
8054 + "rfc3686-ctr-aes-caam-qi2",
8055 + .cra_blocksize = 1,
8056 + },
8057 + .setkey = aead_setkey,
8058 + .setauthsize = aead_setauthsize,
8059 + .encrypt = aead_encrypt,
8060 + .decrypt = aead_decrypt,
8061 + .ivsize = CTR_RFC3686_IV_SIZE,
8062 + .maxauthsize = SHA224_DIGEST_SIZE,
8063 + },
8064 + .caam = {
8065 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8066 + OP_ALG_AAI_CTR_MOD128,
8067 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
8068 + OP_ALG_AAI_HMAC_PRECOMP,
8069 + .rfc3686 = true,
8070 + },
8071 + },
8072 + {
8073 + .aead = {
8074 + .base = {
8075 + .cra_name = "seqiv(authenc("
8076 + "hmac(sha224),rfc3686(ctr(aes))))",
8077 + .cra_driver_name = "seqiv-authenc-hmac-sha224-"
8078 + "rfc3686-ctr-aes-caam-qi2",
8079 + .cra_blocksize = 1,
8080 + },
8081 + .setkey = aead_setkey,
8082 + .setauthsize = aead_setauthsize,
8083 + .encrypt = aead_encrypt,
8084 + .decrypt = aead_decrypt,
8085 + .ivsize = CTR_RFC3686_IV_SIZE,
8086 + .maxauthsize = SHA224_DIGEST_SIZE,
8087 + },
8088 + .caam = {
8089 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8090 + OP_ALG_AAI_CTR_MOD128,
8091 + .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
8092 + OP_ALG_AAI_HMAC_PRECOMP,
8093 + .rfc3686 = true,
8094 + .geniv = true,
8095 + },
8096 + },
8097 + {
8098 + .aead = {
8099 + .base = {
8100 + .cra_name = "authenc(hmac(sha256),"
8101 + "rfc3686(ctr(aes)))",
8102 + .cra_driver_name = "authenc-hmac-sha256-"
8103 + "rfc3686-ctr-aes-caam-qi2",
8104 + .cra_blocksize = 1,
8105 + },
8106 + .setkey = aead_setkey,
8107 + .setauthsize = aead_setauthsize,
8108 + .encrypt = aead_encrypt,
8109 + .decrypt = aead_decrypt,
8110 + .ivsize = CTR_RFC3686_IV_SIZE,
8111 + .maxauthsize = SHA256_DIGEST_SIZE,
8112 + },
8113 + .caam = {
8114 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8115 + OP_ALG_AAI_CTR_MOD128,
8116 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
8117 + OP_ALG_AAI_HMAC_PRECOMP,
8118 + .rfc3686 = true,
8119 + },
8120 + },
8121 + {
8122 + .aead = {
8123 + .base = {
8124 + .cra_name = "seqiv(authenc(hmac(sha256),"
8125 + "rfc3686(ctr(aes))))",
8126 + .cra_driver_name = "seqiv-authenc-hmac-sha256-"
8127 + "rfc3686-ctr-aes-caam-qi2",
8128 + .cra_blocksize = 1,
8129 + },
8130 + .setkey = aead_setkey,
8131 + .setauthsize = aead_setauthsize,
8132 + .encrypt = aead_encrypt,
8133 + .decrypt = aead_decrypt,
8134 + .ivsize = CTR_RFC3686_IV_SIZE,
8135 + .maxauthsize = SHA256_DIGEST_SIZE,
8136 + },
8137 + .caam = {
8138 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8139 + OP_ALG_AAI_CTR_MOD128,
8140 + .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
8141 + OP_ALG_AAI_HMAC_PRECOMP,
8142 + .rfc3686 = true,
8143 + .geniv = true,
8144 + },
8145 + },
8146 + {
8147 + .aead = {
8148 + .base = {
8149 + .cra_name = "authenc(hmac(sha384),"
8150 + "rfc3686(ctr(aes)))",
8151 + .cra_driver_name = "authenc-hmac-sha384-"
8152 + "rfc3686-ctr-aes-caam-qi2",
8153 + .cra_blocksize = 1,
8154 + },
8155 + .setkey = aead_setkey,
8156 + .setauthsize = aead_setauthsize,
8157 + .encrypt = aead_encrypt,
8158 + .decrypt = aead_decrypt,
8159 + .ivsize = CTR_RFC3686_IV_SIZE,
8160 + .maxauthsize = SHA384_DIGEST_SIZE,
8161 + },
8162 + .caam = {
8163 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8164 + OP_ALG_AAI_CTR_MOD128,
8165 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
8166 + OP_ALG_AAI_HMAC_PRECOMP,
8167 + .rfc3686 = true,
8168 + },
8169 + },
8170 + {
8171 + .aead = {
8172 + .base = {
8173 + .cra_name = "seqiv(authenc(hmac(sha384),"
8174 + "rfc3686(ctr(aes))))",
8175 + .cra_driver_name = "seqiv-authenc-hmac-sha384-"
8176 + "rfc3686-ctr-aes-caam-qi2",
8177 + .cra_blocksize = 1,
8178 + },
8179 + .setkey = aead_setkey,
8180 + .setauthsize = aead_setauthsize,
8181 + .encrypt = aead_encrypt,
8182 + .decrypt = aead_decrypt,
8183 + .ivsize = CTR_RFC3686_IV_SIZE,
8184 + .maxauthsize = SHA384_DIGEST_SIZE,
8185 + },
8186 + .caam = {
8187 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8188 + OP_ALG_AAI_CTR_MOD128,
8189 + .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
8190 + OP_ALG_AAI_HMAC_PRECOMP,
8191 + .rfc3686 = true,
8192 + .geniv = true,
8193 + },
8194 + },
8195 + {
8196 + .aead = {
8197 + .base = {
8198 + .cra_name = "rfc7539(chacha20,poly1305)",
8199 + .cra_driver_name = "rfc7539-chacha20-poly1305-"
8200 + "caam-qi2",
8201 + .cra_blocksize = 1,
8202 + },
8203 + .setkey = chachapoly_setkey,
8204 + .setauthsize = chachapoly_setauthsize,
8205 + .encrypt = aead_encrypt,
8206 + .decrypt = aead_decrypt,
8207 + .ivsize = CHACHAPOLY_IV_SIZE,
8208 + .maxauthsize = POLY1305_DIGEST_SIZE,
8209 + },
8210 + .caam = {
8211 + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
8212 + OP_ALG_AAI_AEAD,
8213 + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
8214 + OP_ALG_AAI_AEAD,
8215 + },
8216 + },
8217 + {
8218 + .aead = {
8219 + .base = {
8220 + .cra_name = "rfc7539esp(chacha20,poly1305)",
8221 + .cra_driver_name = "rfc7539esp-chacha20-"
8222 + "poly1305-caam-qi2",
8223 + .cra_blocksize = 1,
8224 + },
8225 + .setkey = chachapoly_setkey,
8226 + .setauthsize = chachapoly_setauthsize,
8227 + .encrypt = aead_encrypt,
8228 + .decrypt = aead_decrypt,
8229 + .ivsize = 8,
8230 + .maxauthsize = POLY1305_DIGEST_SIZE,
8231 + },
8232 + .caam = {
8233 + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
8234 + OP_ALG_AAI_AEAD,
8235 + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
8236 + OP_ALG_AAI_AEAD,
8237 + },
8238 + },
8239 + {
8240 + .aead = {
8241 + .base = {
8242 + .cra_name = "authenc(hmac(sha512),"
8243 + "rfc3686(ctr(aes)))",
8244 + .cra_driver_name = "authenc-hmac-sha512-"
8245 + "rfc3686-ctr-aes-caam-qi2",
8246 + .cra_blocksize = 1,
8247 + },
8248 + .setkey = aead_setkey,
8249 + .setauthsize = aead_setauthsize,
8250 + .encrypt = aead_encrypt,
8251 + .decrypt = aead_decrypt,
8252 + .ivsize = CTR_RFC3686_IV_SIZE,
8253 + .maxauthsize = SHA512_DIGEST_SIZE,
8254 + },
8255 + .caam = {
8256 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8257 + OP_ALG_AAI_CTR_MOD128,
8258 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
8259 + OP_ALG_AAI_HMAC_PRECOMP,
8260 + .rfc3686 = true,
8261 + },
8262 + },
8263 + {
8264 + .aead = {
8265 + .base = {
8266 + .cra_name = "seqiv(authenc(hmac(sha512),"
8267 + "rfc3686(ctr(aes))))",
8268 + .cra_driver_name = "seqiv-authenc-hmac-sha512-"
8269 + "rfc3686-ctr-aes-caam-qi2",
8270 + .cra_blocksize = 1,
8271 + },
8272 + .setkey = aead_setkey,
8273 + .setauthsize = aead_setauthsize,
8274 + .encrypt = aead_encrypt,
8275 + .decrypt = aead_decrypt,
8276 + .ivsize = CTR_RFC3686_IV_SIZE,
8277 + .maxauthsize = SHA512_DIGEST_SIZE,
8278 + },
8279 + .caam = {
8280 + .class1_alg_type = OP_ALG_ALGSEL_AES |
8281 + OP_ALG_AAI_CTR_MOD128,
8282 + .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
8283 + OP_ALG_AAI_HMAC_PRECOMP,
8284 + .rfc3686 = true,
8285 + .geniv = true,
8286 + },
8287 + },
8288 + {
8289 + .aead = {
8290 + .base = {
8291 + .cra_name = "tls10(hmac(sha1),cbc(aes))",
8292 + .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi2",
8293 + .cra_blocksize = AES_BLOCK_SIZE,
8294 + },
8295 + .setkey = tls_setkey,
8296 + .setauthsize = tls_setauthsize,
8297 + .encrypt = tls_encrypt,
8298 + .decrypt = tls_decrypt,
8299 + .ivsize = AES_BLOCK_SIZE,
8300 + .maxauthsize = SHA1_DIGEST_SIZE,
8301 + },
8302 + .caam = {
8303 + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
8304 + .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
8305 + OP_ALG_AAI_HMAC_PRECOMP,
8306 + },
8307 + },
8308 +};
8309 +
8310 +static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
8311 +{
8312 + struct skcipher_alg *alg = &t_alg->skcipher;
8313 +
8314 + alg->base.cra_module = THIS_MODULE;
8315 + alg->base.cra_priority = CAAM_CRA_PRIORITY;
8316 + alg->base.cra_ctxsize = sizeof(struct caam_ctx);
8317 + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
8318 +
8319 + alg->init = caam_cra_init_skcipher;
8320 + alg->exit = caam_cra_exit;
8321 +}
8322 +
8323 +static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
8324 +{
8325 + struct aead_alg *alg = &t_alg->aead;
8326 +
8327 + alg->base.cra_module = THIS_MODULE;
8328 + alg->base.cra_priority = CAAM_CRA_PRIORITY;
8329 + alg->base.cra_ctxsize = sizeof(struct caam_ctx);
8330 + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
8331 +
8332 + alg->init = caam_cra_init_aead;
8333 + alg->exit = caam_cra_exit_aead;
8334 +}
8335 +
8336 +/* max hash key is max split key size */
8337 +#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
8338 +
8339 +#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
8340 +#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
8341 +
8342 +#define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
8343 + CAAM_MAX_HASH_KEY_SIZE)
8344 +#define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
8345 +
8346 +/* caam context sizes for hashes: running digest + 8 */
8347 +#define HASH_MSG_LEN 8
8348 +#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
8349 +
8350 +enum hash_optype {
8351 + UPDATE = 0,
8352 + UPDATE_FIRST,
8353 + FINALIZE,
8354 + DIGEST,
8355 + HASH_NUM_OP
8356 +};
8357 +
8358 +/**
8359 + * caam_hash_ctx - ahash per-session context
8360 + * @flc: Flow Contexts array
8361 + * @flc_dma: I/O virtual addresses of the Flow Contexts
8362 + * @key: virtual address of the authentication key
8363 + * @dev: dpseci device
8364 + * @ctx_len: size of Context Register
8365 + * @adata: hashing algorithm details
8366 + */
8367 +struct caam_hash_ctx {
8368 + struct caam_flc flc[HASH_NUM_OP];
8369 + dma_addr_t flc_dma[HASH_NUM_OP];
8370 + u8 key[CAAM_MAX_HASH_KEY_SIZE];
8371 + struct device *dev;
8372 + int ctx_len;
8373 + struct alginfo adata;
8374 +};
8375 +
8376 +/* ahash state */
8377 +struct caam_hash_state {
8378 + struct caam_request caam_req;
8379 + dma_addr_t buf_dma;
8380 + dma_addr_t ctx_dma;
8381 + u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
8382 + int buflen_0;
8383 + u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
8384 + int buflen_1;
8385 + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
8386 + int (*update)(struct ahash_request *req);
8387 + int (*final)(struct ahash_request *req);
8388 + int (*finup)(struct ahash_request *req);
8389 + int current_buf;
8390 +};
8391 +
8392 +struct caam_export_state {
8393 + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
8394 + u8 caam_ctx[MAX_CTX_LEN];
8395 + int buflen;
8396 + int (*update)(struct ahash_request *req);
8397 + int (*final)(struct ahash_request *req);
8398 + int (*finup)(struct ahash_request *req);
8399 +};
8400 +
8401 +static inline void switch_buf(struct caam_hash_state *state)
8402 +{
8403 + state->current_buf ^= 1;
8404 +}
8405 +
8406 +static inline u8 *current_buf(struct caam_hash_state *state)
8407 +{
8408 + return state->current_buf ? state->buf_1 : state->buf_0;
8409 +}
8410 +
8411 +static inline u8 *alt_buf(struct caam_hash_state *state)
8412 +{
8413 + return state->current_buf ? state->buf_0 : state->buf_1;
8414 +}
8415 +
8416 +static inline int *current_buflen(struct caam_hash_state *state)
8417 +{
8418 + return state->current_buf ? &state->buflen_1 : &state->buflen_0;
8419 +}
8420 +
8421 +static inline int *alt_buflen(struct caam_hash_state *state)
8422 +{
8423 + return state->current_buf ? &state->buflen_0 : &state->buflen_1;
8424 +}
8425 +
8426 +/* Map current buffer in state (if length > 0) and put it in link table */
8427 +static inline int buf_map_to_qm_sg(struct device *dev,
8428 + struct dpaa2_sg_entry *qm_sg,
8429 + struct caam_hash_state *state)
8430 +{
8431 + int buflen = *current_buflen(state);
8432 +
8433 + if (!buflen)
8434 + return 0;
8435 +
8436 + state->buf_dma = dma_map_single(dev, current_buf(state), buflen,
8437 + DMA_TO_DEVICE);
8438 + if (dma_mapping_error(dev, state->buf_dma)) {
8439 + dev_err(dev, "unable to map buf\n");
8440 + state->buf_dma = 0;
8441 + return -ENOMEM;
8442 + }
8443 +
8444 + dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
8445 +
8446 + return 0;
8447 +}
8448 +
8449 +/* Map state->caam_ctx, and add it to link table */
8450 +static inline int ctx_map_to_qm_sg(struct device *dev,
8451 + struct caam_hash_state *state, int ctx_len,
8452 + struct dpaa2_sg_entry *qm_sg, u32 flag)
8453 +{
8454 + state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
8455 + if (dma_mapping_error(dev, state->ctx_dma)) {
8456 + dev_err(dev, "unable to map ctx\n");
8457 + state->ctx_dma = 0;
8458 + return -ENOMEM;
8459 + }
8460 +
8461 + dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
8462 +
8463 + return 0;
8464 +}
8465 +
8466 +static int ahash_set_sh_desc(struct crypto_ahash *ahash)
8467 +{
8468 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8469 + int digestsize = crypto_ahash_digestsize(ahash);
8470 + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
8471 + struct caam_flc *flc;
8472 + u32 *desc;
8473 +
8474 + ctx->adata.key_virt = ctx->key;
8475 + ctx->adata.key_inline = true;
8476 +
8477 + /* ahash_update shared descriptor */
8478 + flc = &ctx->flc[UPDATE];
8479 + desc = flc->sh_desc;
8480 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
8481 + ctx->ctx_len, true, priv->sec_attr.era);
8482 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
8483 + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
8484 + desc_bytes(desc), DMA_BIDIRECTIONAL);
8485 +#ifdef DEBUG
8486 + print_hex_dump(KERN_ERR,
8487 + "ahash update shdesc@" __stringify(__LINE__)": ",
8488 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
8489 +#endif
8490 +
8491 + /* ahash_update_first shared descriptor */
8492 + flc = &ctx->flc[UPDATE_FIRST];
8493 + desc = flc->sh_desc;
8494 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
8495 + ctx->ctx_len, false, priv->sec_attr.era);
8496 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
8497 + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
8498 + desc_bytes(desc), DMA_BIDIRECTIONAL);
8499 +#ifdef DEBUG
8500 + print_hex_dump(KERN_ERR,
8501 + "ahash update first shdesc@" __stringify(__LINE__)": ",
8502 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
8503 +#endif
8504 +
8505 + /* ahash_final shared descriptor */
8506 + flc = &ctx->flc[FINALIZE];
8507 + desc = flc->sh_desc;
8508 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
8509 + ctx->ctx_len, true, priv->sec_attr.era);
8510 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
8511 + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
8512 + desc_bytes(desc), DMA_BIDIRECTIONAL);
8513 +#ifdef DEBUG
8514 + print_hex_dump(KERN_ERR,
8515 + "ahash final shdesc@" __stringify(__LINE__)": ",
8516 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
8517 +#endif
8518 +
8519 + /* ahash_digest shared descriptor */
8520 + flc = &ctx->flc[DIGEST];
8521 + desc = flc->sh_desc;
8522 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
8523 + ctx->ctx_len, false, priv->sec_attr.era);
8524 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
8525 + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
8526 + desc_bytes(desc), DMA_BIDIRECTIONAL);
8527 +#ifdef DEBUG
8528 + print_hex_dump(KERN_ERR,
8529 + "ahash digest shdesc@" __stringify(__LINE__)": ",
8530 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
8531 +#endif
8532 +
8533 + return 0;
8534 +}
8535 +
8536 +/* Digest hash size if it is too large */
8537 +static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
8538 + u32 *keylen, u8 *key_out, u32 digestsize)
8539 +{
8540 + struct caam_request *req_ctx;
8541 + u32 *desc;
8542 + struct split_key_sh_result result;
8543 + dma_addr_t src_dma, dst_dma;
8544 + struct caam_flc *flc;
8545 + dma_addr_t flc_dma;
8546 + int ret = -ENOMEM;
8547 + struct dpaa2_fl_entry *in_fle, *out_fle;
8548 +
8549 + req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
8550 + if (!req_ctx)
8551 + return -ENOMEM;
8552 +
8553 + in_fle = &req_ctx->fd_flt[1];
8554 + out_fle = &req_ctx->fd_flt[0];
8555 +
8556 + flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
8557 + if (!flc)
8558 + goto err_flc;
8559 +
8560 + src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen,
8561 + DMA_TO_DEVICE);
8562 + if (dma_mapping_error(ctx->dev, src_dma)) {
8563 + dev_err(ctx->dev, "unable to map key input memory\n");
8564 + goto err_src_dma;
8565 + }
8566 + dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize,
8567 + DMA_FROM_DEVICE);
8568 + if (dma_mapping_error(ctx->dev, dst_dma)) {
8569 + dev_err(ctx->dev, "unable to map key output memory\n");
8570 + goto err_dst_dma;
8571 + }
8572 +
8573 + desc = flc->sh_desc;
8574 +
8575 + init_sh_desc(desc, 0);
8576 +
8577 + /* descriptor to perform unkeyed hash on key_in */
8578 + append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
8579 + OP_ALG_AS_INITFINAL);
8580 + append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
8581 + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
8582 + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
8583 + LDST_SRCDST_BYTE_CONTEXT);
8584 +
8585 + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
8586 + flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
8587 + desc_bytes(desc), DMA_TO_DEVICE);
8588 + if (dma_mapping_error(ctx->dev, flc_dma)) {
8589 + dev_err(ctx->dev, "unable to map shared descriptor\n");
8590 + goto err_flc_dma;
8591 + }
8592 +
8593 + dpaa2_fl_set_final(in_fle, true);
8594 + dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
8595 + dpaa2_fl_set_addr(in_fle, src_dma);
8596 + dpaa2_fl_set_len(in_fle, *keylen);
8597 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
8598 + dpaa2_fl_set_addr(out_fle, dst_dma);
8599 + dpaa2_fl_set_len(out_fle, digestsize);
8600 +
8601 +#ifdef DEBUG
8602 + print_hex_dump(KERN_ERR, "key_in@" __stringify(__LINE__)": ",
8603 + DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
8604 + print_hex_dump(KERN_ERR, "shdesc@" __stringify(__LINE__)": ",
8605 + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
8606 +#endif
8607 +
8608 + result.err = 0;
8609 + init_completion(&result.completion);
8610 + result.dev = ctx->dev;
8611 +
8612 + req_ctx->flc = flc;
8613 + req_ctx->flc_dma = flc_dma;
8614 + req_ctx->cbk = split_key_sh_done;
8615 + req_ctx->ctx = &result;
8616 +
8617 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
8618 + if (ret == -EINPROGRESS) {
8619 + /* in progress */
8620 + wait_for_completion(&result.completion);
8621 + ret = result.err;
8622 +#ifdef DEBUG
8623 + print_hex_dump(KERN_ERR,
8624 + "digested key@" __stringify(__LINE__)": ",
8625 + DUMP_PREFIX_ADDRESS, 16, 4, key_in, digestsize,
8626 + 1);
8627 +#endif
8628 + }
8629 +
8630 + dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
8631 + DMA_TO_DEVICE);
8632 +err_flc_dma:
8633 + dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE);
8634 +err_dst_dma:
8635 + dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE);
8636 +err_src_dma:
8637 + kfree(flc);
8638 +err_flc:
8639 + kfree(req_ctx);
8640 +
8641 + *keylen = digestsize;
8642 +
8643 + return ret;
8644 +}
8645 +
8646 +static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
8647 + unsigned int keylen)
8648 +{
8649 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8650 + unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
8651 + unsigned int digestsize = crypto_ahash_digestsize(ahash);
8652 + int ret;
8653 + u8 *hashed_key = NULL;
8654 +
8655 +#ifdef DEBUG
8656 + dev_err(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
8657 +#endif
8658 +
8659 + if (keylen > blocksize) {
8660 + hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key),
8661 + GFP_KERNEL | GFP_DMA);
8662 + if (!hashed_key)
8663 + return -ENOMEM;
8664 + ret = hash_digest_key(ctx, key, &keylen, hashed_key,
8665 + digestsize);
8666 + if (ret)
8667 + goto bad_free_key;
8668 + key = hashed_key;
8669 + }
8670 +
8671 + ctx->adata.keylen = keylen;
8672 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
8673 + OP_ALG_ALGSEL_MASK);
8674 + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
8675 + goto bad_free_key;
8676 +
8677 + memcpy(ctx->key, key, keylen);
8678 +
8679 + kfree(hashed_key);
8680 + return ahash_set_sh_desc(ahash);
8681 +bad_free_key:
8682 + kfree(hashed_key);
8683 + crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
8684 + return -EINVAL;
8685 +}
8686 +
8687 +static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
8688 + struct ahash_request *req, int dst_len)
8689 +{
8690 + struct caam_hash_state *state = ahash_request_ctx(req);
8691 +
8692 + if (edesc->src_nents)
8693 + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
8694 + if (edesc->dst_dma)
8695 + dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
8696 +
8697 + if (edesc->qm_sg_bytes)
8698 + dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
8699 + DMA_TO_DEVICE);
8700 +
8701 + if (state->buf_dma) {
8702 + dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
8703 + DMA_TO_DEVICE);
8704 + state->buf_dma = 0;
8705 + }
8706 +}
8707 +
8708 +static inline void ahash_unmap_ctx(struct device *dev,
8709 + struct ahash_edesc *edesc,
8710 + struct ahash_request *req, int dst_len,
8711 + u32 flag)
8712 +{
8713 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8714 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8715 + struct caam_hash_state *state = ahash_request_ctx(req);
8716 +
8717 + if (state->ctx_dma) {
8718 + dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
8719 + state->ctx_dma = 0;
8720 + }
8721 + ahash_unmap(dev, edesc, req, dst_len);
8722 +}
8723 +
8724 +static void ahash_done(void *cbk_ctx, u32 status)
8725 +{
8726 + struct crypto_async_request *areq = cbk_ctx;
8727 + struct ahash_request *req = ahash_request_cast(areq);
8728 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8729 + struct caam_hash_state *state = ahash_request_ctx(req);
8730 + struct ahash_edesc *edesc = state->caam_req.edesc;
8731 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8732 + int digestsize = crypto_ahash_digestsize(ahash);
8733 + int ecode = 0;
8734 +
8735 +#ifdef DEBUG
8736 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
8737 +#endif
8738 +
8739 + if (unlikely(status)) {
8740 + caam_qi2_strstatus(ctx->dev, status);
8741 + ecode = -EIO;
8742 + }
8743 +
8744 + ahash_unmap(ctx->dev, edesc, req, digestsize);
8745 + qi_cache_free(edesc);
8746 +
8747 +#ifdef DEBUG
8748 + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ",
8749 + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
8750 + ctx->ctx_len, 1);
8751 + if (req->result)
8752 + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ",
8753 + DUMP_PREFIX_ADDRESS, 16, 4, req->result,
8754 + digestsize, 1);
8755 +#endif
8756 +
8757 + req->base.complete(&req->base, ecode);
8758 +}
8759 +
8760 +static void ahash_done_bi(void *cbk_ctx, u32 status)
8761 +{
8762 + struct crypto_async_request *areq = cbk_ctx;
8763 + struct ahash_request *req = ahash_request_cast(areq);
8764 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8765 + struct caam_hash_state *state = ahash_request_ctx(req);
8766 + struct ahash_edesc *edesc = state->caam_req.edesc;
8767 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8768 + int ecode = 0;
8769 +#ifdef DEBUG
8770 + int digestsize = crypto_ahash_digestsize(ahash);
8771 +
8772 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
8773 +#endif
8774 +
8775 + if (unlikely(status)) {
8776 + caam_qi2_strstatus(ctx->dev, status);
8777 + ecode = -EIO;
8778 + }
8779 +
8780 + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
8781 + switch_buf(state);
8782 + qi_cache_free(edesc);
8783 +
8784 +#ifdef DEBUG
8785 + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ",
8786 + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
8787 + ctx->ctx_len, 1);
8788 + if (req->result)
8789 + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ",
8790 + DUMP_PREFIX_ADDRESS, 16, 4, req->result,
8791 + digestsize, 1);
8792 +#endif
8793 +
8794 + req->base.complete(&req->base, ecode);
8795 +}
8796 +
8797 +static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
8798 +{
8799 + struct crypto_async_request *areq = cbk_ctx;
8800 + struct ahash_request *req = ahash_request_cast(areq);
8801 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8802 + struct caam_hash_state *state = ahash_request_ctx(req);
8803 + struct ahash_edesc *edesc = state->caam_req.edesc;
8804 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8805 + int digestsize = crypto_ahash_digestsize(ahash);
8806 + int ecode = 0;
8807 +
8808 +#ifdef DEBUG
8809 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
8810 +#endif
8811 +
8812 + if (unlikely(status)) {
8813 + caam_qi2_strstatus(ctx->dev, status);
8814 + ecode = -EIO;
8815 + }
8816 +
8817 + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE);
8818 + qi_cache_free(edesc);
8819 +
8820 +#ifdef DEBUG
8821 + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ",
8822 + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
8823 + ctx->ctx_len, 1);
8824 + if (req->result)
8825 + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ",
8826 + DUMP_PREFIX_ADDRESS, 16, 4, req->result,
8827 + digestsize, 1);
8828 +#endif
8829 +
8830 + req->base.complete(&req->base, ecode);
8831 +}
8832 +
8833 +static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
8834 +{
8835 + struct crypto_async_request *areq = cbk_ctx;
8836 + struct ahash_request *req = ahash_request_cast(areq);
8837 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8838 + struct caam_hash_state *state = ahash_request_ctx(req);
8839 + struct ahash_edesc *edesc = state->caam_req.edesc;
8840 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8841 + int ecode = 0;
8842 +#ifdef DEBUG
8843 + int digestsize = crypto_ahash_digestsize(ahash);
8844 +
8845 + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
8846 +#endif
8847 +
8848 + if (unlikely(status)) {
8849 + caam_qi2_strstatus(ctx->dev, status);
8850 + ecode = -EIO;
8851 + }
8852 +
8853 + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
8854 + switch_buf(state);
8855 + qi_cache_free(edesc);
8856 +
8857 +#ifdef DEBUG
8858 + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ",
8859 + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
8860 + ctx->ctx_len, 1);
8861 + if (req->result)
8862 + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ",
8863 + DUMP_PREFIX_ADDRESS, 16, 4, req->result,
8864 + digestsize, 1);
8865 +#endif
8866 +
8867 + req->base.complete(&req->base, ecode);
8868 +}
8869 +
8870 +static int ahash_update_ctx(struct ahash_request *req)
8871 +{
8872 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
8873 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
8874 + struct caam_hash_state *state = ahash_request_ctx(req);
8875 + struct caam_request *req_ctx = &state->caam_req;
8876 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
8877 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
8878 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
8879 + GFP_KERNEL : GFP_ATOMIC;
8880 + u8 *buf = current_buf(state);
8881 + int *buflen = current_buflen(state);
8882 + u8 *next_buf = alt_buf(state);
8883 + int *next_buflen = alt_buflen(state), last_buflen;
8884 + int in_len = *buflen + req->nbytes, to_hash;
8885 + int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
8886 + struct ahash_edesc *edesc;
8887 + int ret = 0;
8888 +
8889 + last_buflen = *next_buflen;
8890 + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
8891 + to_hash = in_len - *next_buflen;
8892 +
8893 + if (to_hash) {
8894 + struct dpaa2_sg_entry *sg_table;
8895 +
8896 + src_nents = sg_nents_for_len(req->src,
8897 + req->nbytes - (*next_buflen));
8898 + if (src_nents < 0) {
8899 + dev_err(ctx->dev, "Invalid number of src SG.\n");
8900 + return src_nents;
8901 + }
8902 +
8903 + if (src_nents) {
8904 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
8905 + DMA_TO_DEVICE);
8906 + if (!mapped_nents) {
8907 + dev_err(ctx->dev, "unable to DMA map source\n");
8908 + return -ENOMEM;
8909 + }
8910 + } else {
8911 + mapped_nents = 0;
8912 + }
8913 +
8914 + /* allocate space for base edesc and link tables */
8915 + edesc = qi_cache_zalloc(GFP_DMA | flags);
8916 + if (!edesc) {
8917 + dma_unmap_sg(ctx->dev, req->src, src_nents,
8918 + DMA_TO_DEVICE);
8919 + return -ENOMEM;
8920 + }
8921 +
8922 + edesc->src_nents = src_nents;
8923 + qm_sg_src_index = 1 + (*buflen ? 1 : 0);
8924 + qm_sg_bytes = (qm_sg_src_index + mapped_nents) *
8925 + sizeof(*sg_table);
8926 + sg_table = &edesc->sgt[0];
8927 +
8928 + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
8929 + DMA_BIDIRECTIONAL);
8930 + if (ret)
8931 + goto unmap_ctx;
8932 +
8933 + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
8934 + if (ret)
8935 + goto unmap_ctx;
8936 +
8937 + if (mapped_nents) {
8938 + sg_to_qm_sg_last(req->src, mapped_nents,
8939 + sg_table + qm_sg_src_index, 0);
8940 + if (*next_buflen)
8941 + scatterwalk_map_and_copy(next_buf, req->src,
8942 + to_hash - *buflen,
8943 + *next_buflen, 0);
8944 + } else {
8945 + dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
8946 + true);
8947 + }
8948 +
8949 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
8950 + qm_sg_bytes, DMA_TO_DEVICE);
8951 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
8952 + dev_err(ctx->dev, "unable to map S/G table\n");
8953 + ret = -ENOMEM;
8954 + goto unmap_ctx;
8955 + }
8956 + edesc->qm_sg_bytes = qm_sg_bytes;
8957 +
8958 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
8959 + dpaa2_fl_set_final(in_fle, true);
8960 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
8961 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
8962 + dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
8963 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
8964 + dpaa2_fl_set_addr(out_fle, state->ctx_dma);
8965 + dpaa2_fl_set_len(out_fle, ctx->ctx_len);
8966 +
8967 + req_ctx->flc = &ctx->flc[UPDATE];
8968 + req_ctx->flc_dma = ctx->flc_dma[UPDATE];
8969 + req_ctx->cbk = ahash_done_bi;
8970 + req_ctx->ctx = &req->base;
8971 + req_ctx->edesc = edesc;
8972 +
8973 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
8974 + if (ret != -EINPROGRESS &&
8975 + !(ret == -EBUSY &&
8976 + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
8977 + goto unmap_ctx;
8978 + } else if (*next_buflen) {
8979 + scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
8980 + req->nbytes, 0);
8981 + *buflen = *next_buflen;
8982 + *next_buflen = last_buflen;
8983 + }
8984 +#ifdef DEBUG
8985 + print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ",
8986 + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
8987 + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ",
8988 + DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
8989 + *next_buflen, 1);
8990 +#endif
8991 +
8992 + return ret;
8993 +unmap_ctx:
8994 + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
8995 + qi_cache_free(edesc);
8996 + return ret;
8997 +}
8998 +
8999 +static int ahash_final_ctx(struct ahash_request *req)
9000 +{
9001 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9002 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9003 + struct caam_hash_state *state = ahash_request_ctx(req);
9004 + struct caam_request *req_ctx = &state->caam_req;
9005 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9006 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9007 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9008 + GFP_KERNEL : GFP_ATOMIC;
9009 + int buflen = *current_buflen(state);
9010 + int qm_sg_bytes, qm_sg_src_index;
9011 + int digestsize = crypto_ahash_digestsize(ahash);
9012 + struct ahash_edesc *edesc;
9013 + struct dpaa2_sg_entry *sg_table;
9014 + int ret;
9015 +
9016 + /* allocate space for base edesc and link tables */
9017 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9018 + if (!edesc)
9019 + return -ENOMEM;
9020 +
9021 + qm_sg_src_index = 1 + (buflen ? 1 : 0);
9022 + qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table);
9023 + sg_table = &edesc->sgt[0];
9024 +
9025 + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
9026 + DMA_TO_DEVICE);
9027 + if (ret)
9028 + goto unmap_ctx;
9029 +
9030 + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
9031 + if (ret)
9032 + goto unmap_ctx;
9033 +
9034 + dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true);
9035 +
9036 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
9037 + DMA_TO_DEVICE);
9038 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9039 + dev_err(ctx->dev, "unable to map S/G table\n");
9040 + ret = -ENOMEM;
9041 + goto unmap_ctx;
9042 + }
9043 + edesc->qm_sg_bytes = qm_sg_bytes;
9044 +
9045 + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
9046 + DMA_FROM_DEVICE);
9047 + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
9048 + dev_err(ctx->dev, "unable to map dst\n");
9049 + edesc->dst_dma = 0;
9050 + ret = -ENOMEM;
9051 + goto unmap_ctx;
9052 + }
9053 +
9054 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9055 + dpaa2_fl_set_final(in_fle, true);
9056 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9057 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9058 + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
9059 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9060 + dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
9061 + dpaa2_fl_set_len(out_fle, digestsize);
9062 +
9063 + req_ctx->flc = &ctx->flc[FINALIZE];
9064 + req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
9065 + req_ctx->cbk = ahash_done_ctx_src;
9066 + req_ctx->ctx = &req->base;
9067 + req_ctx->edesc = edesc;
9068 +
9069 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9070 + if (ret == -EINPROGRESS ||
9071 + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9072 + return ret;
9073 +
9074 +unmap_ctx:
9075 + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE);
9076 + qi_cache_free(edesc);
9077 + return ret;
9078 +}
9079 +
9080 +static int ahash_finup_ctx(struct ahash_request *req)
9081 +{
9082 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9083 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9084 + struct caam_hash_state *state = ahash_request_ctx(req);
9085 + struct caam_request *req_ctx = &state->caam_req;
9086 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9087 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9088 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9089 + GFP_KERNEL : GFP_ATOMIC;
9090 + int buflen = *current_buflen(state);
9091 + int qm_sg_bytes, qm_sg_src_index;
9092 + int src_nents, mapped_nents;
9093 + int digestsize = crypto_ahash_digestsize(ahash);
9094 + struct ahash_edesc *edesc;
9095 + struct dpaa2_sg_entry *sg_table;
9096 + int ret;
9097 +
9098 + src_nents = sg_nents_for_len(req->src, req->nbytes);
9099 + if (src_nents < 0) {
9100 + dev_err(ctx->dev, "Invalid number of src SG.\n");
9101 + return src_nents;
9102 + }
9103 +
9104 + if (src_nents) {
9105 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
9106 + DMA_TO_DEVICE);
9107 + if (!mapped_nents) {
9108 + dev_err(ctx->dev, "unable to DMA map source\n");
9109 + return -ENOMEM;
9110 + }
9111 + } else {
9112 + mapped_nents = 0;
9113 + }
9114 +
9115 + /* allocate space for base edesc and link tables */
9116 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9117 + if (!edesc) {
9118 + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
9119 + return -ENOMEM;
9120 + }
9121 +
9122 + edesc->src_nents = src_nents;
9123 + qm_sg_src_index = 1 + (buflen ? 1 : 0);
9124 + qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table);
9125 + sg_table = &edesc->sgt[0];
9126 +
9127 + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
9128 + DMA_TO_DEVICE);
9129 + if (ret)
9130 + goto unmap_ctx;
9131 +
9132 + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
9133 + if (ret)
9134 + goto unmap_ctx;
9135 +
9136 + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0);
9137 +
9138 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
9139 + DMA_TO_DEVICE);
9140 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9141 + dev_err(ctx->dev, "unable to map S/G table\n");
9142 + ret = -ENOMEM;
9143 + goto unmap_ctx;
9144 + }
9145 + edesc->qm_sg_bytes = qm_sg_bytes;
9146 +
9147 + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
9148 + DMA_FROM_DEVICE);
9149 + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
9150 + dev_err(ctx->dev, "unable to map dst\n");
9151 + edesc->dst_dma = 0;
9152 + ret = -ENOMEM;
9153 + goto unmap_ctx;
9154 + }
9155 +
9156 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9157 + dpaa2_fl_set_final(in_fle, true);
9158 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9159 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9160 + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
9161 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9162 + dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
9163 + dpaa2_fl_set_len(out_fle, digestsize);
9164 +
9165 + req_ctx->flc = &ctx->flc[FINALIZE];
9166 + req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
9167 + req_ctx->cbk = ahash_done_ctx_src;
9168 + req_ctx->ctx = &req->base;
9169 + req_ctx->edesc = edesc;
9170 +
9171 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9172 + if (ret == -EINPROGRESS ||
9173 + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9174 + return ret;
9175 +
9176 +unmap_ctx:
9177 + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE);
9178 + qi_cache_free(edesc);
9179 + return ret;
9180 +}
9181 +
9182 +static int ahash_digest(struct ahash_request *req)
9183 +{
9184 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9185 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9186 + struct caam_hash_state *state = ahash_request_ctx(req);
9187 + struct caam_request *req_ctx = &state->caam_req;
9188 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9189 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9190 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9191 + GFP_KERNEL : GFP_ATOMIC;
9192 + int digestsize = crypto_ahash_digestsize(ahash);
9193 + int src_nents, mapped_nents;
9194 + struct ahash_edesc *edesc;
9195 + int ret = -ENOMEM;
9196 +
9197 + state->buf_dma = 0;
9198 +
9199 + src_nents = sg_nents_for_len(req->src, req->nbytes);
9200 + if (src_nents < 0) {
9201 + dev_err(ctx->dev, "Invalid number of src SG.\n");
9202 + return src_nents;
9203 + }
9204 +
9205 + if (src_nents) {
9206 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
9207 + DMA_TO_DEVICE);
9208 + if (!mapped_nents) {
9209 + dev_err(ctx->dev, "unable to map source for DMA\n");
9210 + return ret;
9211 + }
9212 + } else {
9213 + mapped_nents = 0;
9214 + }
9215 +
9216 + /* allocate space for base edesc and link tables */
9217 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9218 + if (!edesc) {
9219 + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
9220 + return ret;
9221 + }
9222 +
9223 + edesc->src_nents = src_nents;
9224 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9225 +
9226 + if (mapped_nents > 1) {
9227 + int qm_sg_bytes;
9228 + struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
9229 +
9230 + qm_sg_bytes = mapped_nents * sizeof(*sg_table);
9231 + sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
9232 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
9233 + qm_sg_bytes, DMA_TO_DEVICE);
9234 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9235 + dev_err(ctx->dev, "unable to map S/G table\n");
9236 + goto unmap;
9237 + }
9238 + edesc->qm_sg_bytes = qm_sg_bytes;
9239 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9240 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9241 + } else {
9242 + dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
9243 + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
9244 + }
9245 +
9246 + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
9247 + DMA_FROM_DEVICE);
9248 + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
9249 + dev_err(ctx->dev, "unable to map dst\n");
9250 + edesc->dst_dma = 0;
9251 + goto unmap;
9252 + }
9253 +
9254 + dpaa2_fl_set_final(in_fle, true);
9255 + dpaa2_fl_set_len(in_fle, req->nbytes);
9256 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9257 + dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
9258 + dpaa2_fl_set_len(out_fle, digestsize);
9259 +
9260 + req_ctx->flc = &ctx->flc[DIGEST];
9261 + req_ctx->flc_dma = ctx->flc_dma[DIGEST];
9262 + req_ctx->cbk = ahash_done;
9263 + req_ctx->ctx = &req->base;
9264 + req_ctx->edesc = edesc;
9265 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9266 + if (ret == -EINPROGRESS ||
9267 + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9268 + return ret;
9269 +
9270 +unmap:
9271 + ahash_unmap(ctx->dev, edesc, req, digestsize);
9272 + qi_cache_free(edesc);
9273 + return ret;
9274 +}
9275 +
9276 +static int ahash_final_no_ctx(struct ahash_request *req)
9277 +{
9278 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9279 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9280 + struct caam_hash_state *state = ahash_request_ctx(req);
9281 + struct caam_request *req_ctx = &state->caam_req;
9282 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9283 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9284 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9285 + GFP_KERNEL : GFP_ATOMIC;
9286 + u8 *buf = current_buf(state);
9287 + int buflen = *current_buflen(state);
9288 + int digestsize = crypto_ahash_digestsize(ahash);
9289 + struct ahash_edesc *edesc;
9290 + int ret = -ENOMEM;
9291 +
9292 + /* allocate space for base edesc and link tables */
9293 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9294 + if (!edesc)
9295 + return ret;
9296 +
9297 + state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE);
9298 + if (dma_mapping_error(ctx->dev, state->buf_dma)) {
9299 + dev_err(ctx->dev, "unable to map src\n");
9300 + goto unmap;
9301 + }
9302 +
9303 + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
9304 + DMA_FROM_DEVICE);
9305 + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
9306 + dev_err(ctx->dev, "unable to map dst\n");
9307 + edesc->dst_dma = 0;
9308 + goto unmap;
9309 + }
9310 +
9311 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9312 + dpaa2_fl_set_final(in_fle, true);
9313 + dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
9314 + dpaa2_fl_set_addr(in_fle, state->buf_dma);
9315 + dpaa2_fl_set_len(in_fle, buflen);
9316 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9317 + dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
9318 + dpaa2_fl_set_len(out_fle, digestsize);
9319 +
9320 + req_ctx->flc = &ctx->flc[DIGEST];
9321 + req_ctx->flc_dma = ctx->flc_dma[DIGEST];
9322 + req_ctx->cbk = ahash_done;
9323 + req_ctx->ctx = &req->base;
9324 + req_ctx->edesc = edesc;
9325 +
9326 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9327 + if (ret == -EINPROGRESS ||
9328 + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9329 + return ret;
9330 +
9331 +unmap:
9332 + ahash_unmap(ctx->dev, edesc, req, digestsize);
9333 + qi_cache_free(edesc);
9334 + return ret;
9335 +}
9336 +
9337 +static int ahash_update_no_ctx(struct ahash_request *req)
9338 +{
9339 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9340 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9341 + struct caam_hash_state *state = ahash_request_ctx(req);
9342 + struct caam_request *req_ctx = &state->caam_req;
9343 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9344 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9345 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9346 + GFP_KERNEL : GFP_ATOMIC;
9347 + u8 *buf = current_buf(state);
9348 + int *buflen = current_buflen(state);
9349 + u8 *next_buf = alt_buf(state);
9350 + int *next_buflen = alt_buflen(state);
9351 + int in_len = *buflen + req->nbytes, to_hash;
9352 + int qm_sg_bytes, src_nents, mapped_nents;
9353 + struct ahash_edesc *edesc;
9354 + int ret = 0;
9355 +
9356 + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
9357 + to_hash = in_len - *next_buflen;
9358 +
9359 + if (to_hash) {
9360 + struct dpaa2_sg_entry *sg_table;
9361 +
9362 + src_nents = sg_nents_for_len(req->src,
9363 + req->nbytes - *next_buflen);
9364 + if (src_nents < 0) {
9365 + dev_err(ctx->dev, "Invalid number of src SG.\n");
9366 + return src_nents;
9367 + }
9368 +
9369 + if (src_nents) {
9370 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
9371 + DMA_TO_DEVICE);
9372 + if (!mapped_nents) {
9373 + dev_err(ctx->dev, "unable to DMA map source\n");
9374 + return -ENOMEM;
9375 + }
9376 + } else {
9377 + mapped_nents = 0;
9378 + }
9379 +
9380 + /* allocate space for base edesc and link tables */
9381 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9382 + if (!edesc) {
9383 + dma_unmap_sg(ctx->dev, req->src, src_nents,
9384 + DMA_TO_DEVICE);
9385 + return -ENOMEM;
9386 + }
9387 +
9388 + edesc->src_nents = src_nents;
9389 + qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table);
9390 + sg_table = &edesc->sgt[0];
9391 +
9392 + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
9393 + if (ret)
9394 + goto unmap_ctx;
9395 +
9396 + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
9397 +
9398 + if (*next_buflen)
9399 + scatterwalk_map_and_copy(next_buf, req->src,
9400 + to_hash - *buflen,
9401 + *next_buflen, 0);
9402 +
9403 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
9404 + qm_sg_bytes, DMA_TO_DEVICE);
9405 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9406 + dev_err(ctx->dev, "unable to map S/G table\n");
9407 + ret = -ENOMEM;
9408 + goto unmap_ctx;
9409 + }
9410 + edesc->qm_sg_bytes = qm_sg_bytes;
9411 +
9412 + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
9413 + ctx->ctx_len, DMA_FROM_DEVICE);
9414 + if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
9415 + dev_err(ctx->dev, "unable to map ctx\n");
9416 + state->ctx_dma = 0;
9417 + ret = -ENOMEM;
9418 + goto unmap_ctx;
9419 + }
9420 +
9421 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9422 + dpaa2_fl_set_final(in_fle, true);
9423 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9424 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9425 + dpaa2_fl_set_len(in_fle, to_hash);
9426 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9427 + dpaa2_fl_set_addr(out_fle, state->ctx_dma);
9428 + dpaa2_fl_set_len(out_fle, ctx->ctx_len);
9429 +
9430 + req_ctx->flc = &ctx->flc[UPDATE_FIRST];
9431 + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
9432 + req_ctx->cbk = ahash_done_ctx_dst;
9433 + req_ctx->ctx = &req->base;
9434 + req_ctx->edesc = edesc;
9435 +
9436 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9437 + if (ret != -EINPROGRESS &&
9438 + !(ret == -EBUSY &&
9439 + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9440 + goto unmap_ctx;
9441 +
9442 + state->update = ahash_update_ctx;
9443 + state->finup = ahash_finup_ctx;
9444 + state->final = ahash_final_ctx;
9445 + } else if (*next_buflen) {
9446 + scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
9447 + req->nbytes, 0);
9448 + *buflen = *next_buflen;
9449 + *next_buflen = 0;
9450 + }
9451 +#ifdef DEBUG
9452 + print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ",
9453 + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
9454 + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ",
9455 + DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
9456 + *next_buflen, 1);
9457 +#endif
9458 +
9459 + return ret;
9460 +unmap_ctx:
9461 + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
9462 + qi_cache_free(edesc);
9463 + return ret;
9464 +}
9465 +
9466 +static int ahash_finup_no_ctx(struct ahash_request *req)
9467 +{
9468 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9469 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9470 + struct caam_hash_state *state = ahash_request_ctx(req);
9471 + struct caam_request *req_ctx = &state->caam_req;
9472 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9473 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9474 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9475 + GFP_KERNEL : GFP_ATOMIC;
9476 + int buflen = *current_buflen(state);
9477 + int qm_sg_bytes, src_nents, mapped_nents;
9478 + int digestsize = crypto_ahash_digestsize(ahash);
9479 + struct ahash_edesc *edesc;
9480 + struct dpaa2_sg_entry *sg_table;
9481 + int ret;
9482 +
9483 + src_nents = sg_nents_for_len(req->src, req->nbytes);
9484 + if (src_nents < 0) {
9485 + dev_err(ctx->dev, "Invalid number of src SG.\n");
9486 + return src_nents;
9487 + }
9488 +
9489 + if (src_nents) {
9490 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
9491 + DMA_TO_DEVICE);
9492 + if (!mapped_nents) {
9493 + dev_err(ctx->dev, "unable to DMA map source\n");
9494 + return -ENOMEM;
9495 + }
9496 + } else {
9497 + mapped_nents = 0;
9498 + }
9499 +
9500 + /* allocate space for base edesc and link tables */
9501 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9502 + if (!edesc) {
9503 + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
9504 + return -ENOMEM;
9505 + }
9506 +
9507 + edesc->src_nents = src_nents;
9508 + qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table);
9509 + sg_table = &edesc->sgt[0];
9510 +
9511 + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
9512 + if (ret)
9513 + goto unmap;
9514 +
9515 + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
9516 +
9517 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
9518 + DMA_TO_DEVICE);
9519 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9520 + dev_err(ctx->dev, "unable to map S/G table\n");
9521 + ret = -ENOMEM;
9522 + goto unmap;
9523 + }
9524 + edesc->qm_sg_bytes = qm_sg_bytes;
9525 +
9526 + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize,
9527 + DMA_FROM_DEVICE);
9528 + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) {
9529 + dev_err(ctx->dev, "unable to map dst\n");
9530 + edesc->dst_dma = 0;
9531 + ret = -ENOMEM;
9532 + goto unmap;
9533 + }
9534 +
9535 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9536 + dpaa2_fl_set_final(in_fle, true);
9537 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9538 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9539 + dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
9540 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9541 + dpaa2_fl_set_addr(out_fle, edesc->dst_dma);
9542 + dpaa2_fl_set_len(out_fle, digestsize);
9543 +
9544 + req_ctx->flc = &ctx->flc[DIGEST];
9545 + req_ctx->flc_dma = ctx->flc_dma[DIGEST];
9546 + req_ctx->cbk = ahash_done;
9547 + req_ctx->ctx = &req->base;
9548 + req_ctx->edesc = edesc;
9549 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9550 + if (ret != -EINPROGRESS &&
9551 + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
9552 + goto unmap;
9553 +
9554 + return ret;
9555 +unmap:
9556 + ahash_unmap(ctx->dev, edesc, req, digestsize);
9557 + qi_cache_free(edesc);
9558 + return -ENOMEM;
9559 +}
9560 +
9561 +static int ahash_update_first(struct ahash_request *req)
9562 +{
9563 + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
9564 + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
9565 + struct caam_hash_state *state = ahash_request_ctx(req);
9566 + struct caam_request *req_ctx = &state->caam_req;
9567 + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
9568 + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
9569 + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
9570 + GFP_KERNEL : GFP_ATOMIC;
9571 + u8 *next_buf = alt_buf(state);
9572 + int *next_buflen = alt_buflen(state);
9573 + int to_hash;
9574 + int src_nents, mapped_nents;
9575 + struct ahash_edesc *edesc;
9576 + int ret = 0;
9577 +
9578 + *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
9579 + 1);
9580 + to_hash = req->nbytes - *next_buflen;
9581 +
9582 + if (to_hash) {
9583 + struct dpaa2_sg_entry *sg_table;
9584 +
9585 + src_nents = sg_nents_for_len(req->src,
9586 + req->nbytes - (*next_buflen));
9587 + if (src_nents < 0) {
9588 + dev_err(ctx->dev, "Invalid number of src SG.\n");
9589 + return src_nents;
9590 + }
9591 +
9592 + if (src_nents) {
9593 + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
9594 + DMA_TO_DEVICE);
9595 + if (!mapped_nents) {
9596 + dev_err(ctx->dev, "unable to map source for DMA\n");
9597 + return -ENOMEM;
9598 + }
9599 + } else {
9600 + mapped_nents = 0;
9601 + }
9602 +
9603 + /* allocate space for base edesc and link tables */
9604 + edesc = qi_cache_zalloc(GFP_DMA | flags);
9605 + if (!edesc) {
9606 + dma_unmap_sg(ctx->dev, req->src, src_nents,
9607 + DMA_TO_DEVICE);
9608 + return -ENOMEM;
9609 + }
9610 +
9611 + edesc->src_nents = src_nents;
9612 + sg_table = &edesc->sgt[0];
9613 +
9614 + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
9615 + dpaa2_fl_set_final(in_fle, true);
9616 + dpaa2_fl_set_len(in_fle, to_hash);
9617 +
9618 + if (mapped_nents > 1) {
9619 + int qm_sg_bytes;
9620 +
9621 + sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
9622 + qm_sg_bytes = mapped_nents * sizeof(*sg_table);
9623 + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
9624 + qm_sg_bytes,
9625 + DMA_TO_DEVICE);
9626 + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
9627 + dev_err(ctx->dev, "unable to map S/G table\n");
9628 + ret = -ENOMEM;
9629 + goto unmap_ctx;
9630 + }
9631 + edesc->qm_sg_bytes = qm_sg_bytes;
9632 + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
9633 + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
9634 + } else {
9635 + dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
9636 + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
9637 + }
9638 +
9639 + if (*next_buflen)
9640 + scatterwalk_map_and_copy(next_buf, req->src, to_hash,
9641 + *next_buflen, 0);
9642 +
9643 + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
9644 + ctx->ctx_len, DMA_FROM_DEVICE);
9645 + if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
9646 + dev_err(ctx->dev, "unable to map ctx\n");
9647 + state->ctx_dma = 0;
9648 + ret = -ENOMEM;
9649 + goto unmap_ctx;
9650 + }
9651 +
9652 + dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
9653 + dpaa2_fl_set_addr(out_fle, state->ctx_dma);
9654 + dpaa2_fl_set_len(out_fle, ctx->ctx_len);
9655 +
9656 + req_ctx->flc = &ctx->flc[UPDATE_FIRST];
9657 + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
9658 + req_ctx->cbk = ahash_done_ctx_dst;
9659 + req_ctx->ctx = &req->base;
9660 + req_ctx->edesc = edesc;
9661 +
9662 + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
9663 + if (ret != -EINPROGRESS &&
9664 + !(ret == -EBUSY && req->base.flags &
9665 + CRYPTO_TFM_REQ_MAY_BACKLOG))
9666 + goto unmap_ctx;
9667 +
9668 + state->update = ahash_update_ctx;
9669 + state->finup = ahash_finup_ctx;
9670 + state->final = ahash_final_ctx;
9671 + } else if (*next_buflen) {
9672 + state->update = ahash_update_no_ctx;
9673 + state->finup = ahash_finup_no_ctx;
9674 + state->final = ahash_final_no_ctx;
9675 + scatterwalk_map_and_copy(next_buf, req->src, 0,
9676 + req->nbytes, 0);
9677 + switch_buf(state);
9678 + }
9679 +#ifdef DEBUG
9680 + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ",
9681 + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 1);
9682 +#endif
9683 +
9684 + return ret;
9685 +unmap_ctx:
9686 + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
9687 + qi_cache_free(edesc);
9688 + return ret;
9689 +}
9690 +
9691 +static int ahash_finup_first(struct ahash_request *req)
9692 +{
9693 + return ahash_digest(req);
9694 +}
9695 +
9696 +static int ahash_init(struct ahash_request *req)
9697 +{
9698 + struct caam_hash_state *state = ahash_request_ctx(req);
9699 +
9700 + state->update = ahash_update_first;
9701 + state->finup = ahash_finup_first;
9702 + state->final = ahash_final_no_ctx;
9703 +
9704 + state->ctx_dma = 0;
9705 + state->current_buf = 0;
9706 + state->buf_dma = 0;
9707 + state->buflen_0 = 0;
9708 + state->buflen_1 = 0;
9709 +
9710 + return 0;
9711 +}
9712 +
9713 +static int ahash_update(struct ahash_request *req)
9714 +{
9715 + struct caam_hash_state *state = ahash_request_ctx(req);
9716 +
9717 + return state->update(req);
9718 +}
9719 +
9720 +static int ahash_finup(struct ahash_request *req)
9721 +{
9722 + struct caam_hash_state *state = ahash_request_ctx(req);
9723 +
9724 + return state->finup(req);
9725 +}
9726 +
9727 +static int ahash_final(struct ahash_request *req)
9728 +{
9729 + struct caam_hash_state *state = ahash_request_ctx(req);
9730 +
9731 + return state->final(req);
9732 +}
9733 +
9734 +static int ahash_export(struct ahash_request *req, void *out)
9735 +{
9736 + struct caam_hash_state *state = ahash_request_ctx(req);
9737 + struct caam_export_state *export = out;
9738 + int len;
9739 + u8 *buf;
9740 +
9741 + if (state->current_buf) {
9742 + buf = state->buf_1;
9743 + len = state->buflen_1;
9744 + } else {
9745 + buf = state->buf_0;
9746 + len = state->buflen_0;
9747 + }
9748 +
9749 + memcpy(export->buf, buf, len);
9750 + memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
9751 + export->buflen = len;
9752 + export->update = state->update;
9753 + export->final = state->final;
9754 + export->finup = state->finup;
9755 +
9756 + return 0;
9757 +}
9758 +
9759 +static int ahash_import(struct ahash_request *req, const void *in)
9760 +{
9761 + struct caam_hash_state *state = ahash_request_ctx(req);
9762 + const struct caam_export_state *export = in;
9763 +
9764 + memset(state, 0, sizeof(*state));
9765 + memcpy(state->buf_0, export->buf, export->buflen);
9766 + memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
9767 + state->buflen_0 = export->buflen;
9768 + state->update = export->update;
9769 + state->final = export->final;
9770 + state->finup = export->finup;
9771 +
9772 + return 0;
9773 +}
9774 +
9775 +struct caam_hash_template {
9776 + char name[CRYPTO_MAX_ALG_NAME];
9777 + char driver_name[CRYPTO_MAX_ALG_NAME];
9778 + char hmac_name[CRYPTO_MAX_ALG_NAME];
9779 + char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
9780 + unsigned int blocksize;
9781 + struct ahash_alg template_ahash;
9782 + u32 alg_type;
9783 +};
9784 +
9785 +/* ahash descriptors */
9786 +static struct caam_hash_template driver_hash[] = {
9787 + {
9788 + .name = "sha1",
9789 + .driver_name = "sha1-caam-qi2",
9790 + .hmac_name = "hmac(sha1)",
9791 + .hmac_driver_name = "hmac-sha1-caam-qi2",
9792 + .blocksize = SHA1_BLOCK_SIZE,
9793 + .template_ahash = {
9794 + .init = ahash_init,
9795 + .update = ahash_update,
9796 + .final = ahash_final,
9797 + .finup = ahash_finup,
9798 + .digest = ahash_digest,
9799 + .export = ahash_export,
9800 + .import = ahash_import,
9801 + .setkey = ahash_setkey,
9802 + .halg = {
9803 + .digestsize = SHA1_DIGEST_SIZE,
9804 + .statesize = sizeof(struct caam_export_state),
9805 + },
9806 + },
9807 + .alg_type = OP_ALG_ALGSEL_SHA1,
9808 + }, {
9809 + .name = "sha224",
9810 + .driver_name = "sha224-caam-qi2",
9811 + .hmac_name = "hmac(sha224)",
9812 + .hmac_driver_name = "hmac-sha224-caam-qi2",
9813 + .blocksize = SHA224_BLOCK_SIZE,
9814 + .template_ahash = {
9815 + .init = ahash_init,
9816 + .update = ahash_update,
9817 + .final = ahash_final,
9818 + .finup = ahash_finup,
9819 + .digest = ahash_digest,
9820 + .export = ahash_export,
9821 + .import = ahash_import,
9822 + .setkey = ahash_setkey,
9823 + .halg = {
9824 + .digestsize = SHA224_DIGEST_SIZE,
9825 + .statesize = sizeof(struct caam_export_state),
9826 + },
9827 + },
9828 + .alg_type = OP_ALG_ALGSEL_SHA224,
9829 + }, {
9830 + .name = "sha256",
9831 + .driver_name = "sha256-caam-qi2",
9832 + .hmac_name = "hmac(sha256)",
9833 + .hmac_driver_name = "hmac-sha256-caam-qi2",
9834 + .blocksize = SHA256_BLOCK_SIZE,
9835 + .template_ahash = {
9836 + .init = ahash_init,
9837 + .update = ahash_update,
9838 + .final = ahash_final,
9839 + .finup = ahash_finup,
9840 + .digest = ahash_digest,
9841 + .export = ahash_export,
9842 + .import = ahash_import,
9843 + .setkey = ahash_setkey,
9844 + .halg = {
9845 + .digestsize = SHA256_DIGEST_SIZE,
9846 + .statesize = sizeof(struct caam_export_state),
9847 + },
9848 + },
9849 + .alg_type = OP_ALG_ALGSEL_SHA256,
9850 + }, {
9851 + .name = "sha384",
9852 + .driver_name = "sha384-caam-qi2",
9853 + .hmac_name = "hmac(sha384)",
9854 + .hmac_driver_name = "hmac-sha384-caam-qi2",
9855 + .blocksize = SHA384_BLOCK_SIZE,
9856 + .template_ahash = {
9857 + .init = ahash_init,
9858 + .update = ahash_update,
9859 + .final = ahash_final,
9860 + .finup = ahash_finup,
9861 + .digest = ahash_digest,
9862 + .export = ahash_export,
9863 + .import = ahash_import,
9864 + .setkey = ahash_setkey,
9865 + .halg = {
9866 + .digestsize = SHA384_DIGEST_SIZE,
9867 + .statesize = sizeof(struct caam_export_state),
9868 + },
9869 + },
9870 + .alg_type = OP_ALG_ALGSEL_SHA384,
9871 + }, {
9872 + .name = "sha512",
9873 + .driver_name = "sha512-caam-qi2",
9874 + .hmac_name = "hmac(sha512)",
9875 + .hmac_driver_name = "hmac-sha512-caam-qi2",
9876 + .blocksize = SHA512_BLOCK_SIZE,
9877 + .template_ahash = {
9878 + .init = ahash_init,
9879 + .update = ahash_update,
9880 + .final = ahash_final,
9881 + .finup = ahash_finup,
9882 + .digest = ahash_digest,
9883 + .export = ahash_export,
9884 + .import = ahash_import,
9885 + .setkey = ahash_setkey,
9886 + .halg = {
9887 + .digestsize = SHA512_DIGEST_SIZE,
9888 + .statesize = sizeof(struct caam_export_state),
9889 + },
9890 + },
9891 + .alg_type = OP_ALG_ALGSEL_SHA512,
9892 + }, {
9893 + .name = "md5",
9894 + .driver_name = "md5-caam-qi2",
9895 + .hmac_name = "hmac(md5)",
9896 + .hmac_driver_name = "hmac-md5-caam-qi2",
9897 + .blocksize = MD5_BLOCK_WORDS * 4,
9898 + .template_ahash = {
9899 + .init = ahash_init,
9900 + .update = ahash_update,
9901 + .final = ahash_final,
9902 + .finup = ahash_finup,
9903 + .digest = ahash_digest,
9904 + .export = ahash_export,
9905 + .import = ahash_import,
9906 + .setkey = ahash_setkey,
9907 + .halg = {
9908 + .digestsize = MD5_DIGEST_SIZE,
9909 + .statesize = sizeof(struct caam_export_state),
9910 + },
9911 + },
9912 + .alg_type = OP_ALG_ALGSEL_MD5,
9913 + }
9914 +};
9915 +
9916 +struct caam_hash_alg {
9917 + struct list_head entry;
9918 + struct device *dev;
9919 + int alg_type;
9920 + struct ahash_alg ahash_alg;
9921 +};
9922 +
9923 +static int caam_hash_cra_init(struct crypto_tfm *tfm)
9924 +{
9925 + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
9926 + struct crypto_alg *base = tfm->__crt_alg;
9927 + struct hash_alg_common *halg =
9928 + container_of(base, struct hash_alg_common, base);
9929 + struct ahash_alg *alg =
9930 + container_of(halg, struct ahash_alg, halg);
9931 + struct caam_hash_alg *caam_hash =
9932 + container_of(alg, struct caam_hash_alg, ahash_alg);
9933 + struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
9934 + /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
9935 + static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
9936 + HASH_MSG_LEN + SHA1_DIGEST_SIZE,
9937 + HASH_MSG_LEN + 32,
9938 + HASH_MSG_LEN + SHA256_DIGEST_SIZE,
9939 + HASH_MSG_LEN + 64,
9940 + HASH_MSG_LEN + SHA512_DIGEST_SIZE };
9941 + dma_addr_t dma_addr;
9942 + int i;
9943 +
9944 + ctx->dev = caam_hash->dev;
9945 +
9946 + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
9947 + DMA_BIDIRECTIONAL,
9948 + DMA_ATTR_SKIP_CPU_SYNC);
9949 + if (dma_mapping_error(ctx->dev, dma_addr)) {
9950 + dev_err(ctx->dev, "unable to map shared descriptors\n");
9951 + return -ENOMEM;
9952 + }
9953 +
9954 + for (i = 0; i < HASH_NUM_OP; i++)
9955 + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
9956 +
9957 + /* copy descriptor header template value */
9958 + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
9959 +
9960 + ctx->ctx_len = runninglen[(ctx->adata.algtype &
9961 + OP_ALG_ALGSEL_SUBMASK) >>
9962 + OP_ALG_ALGSEL_SHIFT];
9963 +
9964 + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
9965 + sizeof(struct caam_hash_state));
9966 +
9967 + return ahash_set_sh_desc(ahash);
9968 +}
9969 +
9970 +static void caam_hash_cra_exit(struct crypto_tfm *tfm)
9971 +{
9972 + struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
9973 +
9974 + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
9975 + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
9976 +}
9977 +
9978 +static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
9979 + struct caam_hash_template *template, bool keyed)
9980 +{
9981 + struct caam_hash_alg *t_alg;
9982 + struct ahash_alg *halg;
9983 + struct crypto_alg *alg;
9984 +
9985 + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
9986 + if (!t_alg)
9987 + return ERR_PTR(-ENOMEM);
9988 +
9989 + t_alg->ahash_alg = template->template_ahash;
9990 + halg = &t_alg->ahash_alg;
9991 + alg = &halg->halg.base;
9992 +
9993 + if (keyed) {
9994 + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
9995 + template->hmac_name);
9996 + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
9997 + template->hmac_driver_name);
9998 + } else {
9999 + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
10000 + template->name);
10001 + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
10002 + template->driver_name);
10003 + t_alg->ahash_alg.setkey = NULL;
10004 + }
10005 + alg->cra_module = THIS_MODULE;
10006 + alg->cra_init = caam_hash_cra_init;
10007 + alg->cra_exit = caam_hash_cra_exit;
10008 + alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
10009 + alg->cra_priority = CAAM_CRA_PRIORITY;
10010 + alg->cra_blocksize = template->blocksize;
10011 + alg->cra_alignmask = 0;
10012 + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
10013 + alg->cra_type = &crypto_ahash_type;
10014 +
10015 + t_alg->alg_type = template->alg_type;
10016 + t_alg->dev = dev;
10017 +
10018 + return t_alg;
10019 +}
10020 +
10021 +static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
10022 +{
10023 + struct dpaa2_caam_priv_per_cpu *ppriv;
10024 +
10025 + ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
10026 + napi_schedule_irqoff(&ppriv->napi);
10027 +}
10028 +
10029 +static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
10030 +{
10031 + struct device *dev = priv->dev;
10032 + struct dpaa2_io_notification_ctx *nctx;
10033 + struct dpaa2_caam_priv_per_cpu *ppriv;
10034 + int err, i = 0, cpu;
10035 +
10036 + for_each_online_cpu(cpu) {
10037 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10038 + ppriv->priv = priv;
10039 + nctx = &ppriv->nctx;
10040 + nctx->is_cdan = 0;
10041 + nctx->id = ppriv->rsp_fqid;
10042 + nctx->desired_cpu = cpu;
10043 + nctx->cb = dpaa2_caam_fqdan_cb;
10044 +
10045 + /* Register notification callbacks */
10046 + ppriv->dpio = dpaa2_io_service_select(cpu);
10047 + err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
10048 + if (unlikely(err)) {
10049 + dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
10050 + nctx->cb = NULL;
10051 + /*
10052 + * If no affine DPIO for this core, there's probably
10053 + * none available for next cores either. Signal we want
10054 + * to retry later, in case the DPIO devices weren't
10055 + * probed yet.
10056 + */
10057 + err = -EPROBE_DEFER;
10058 + goto err;
10059 + }
10060 +
10061 + ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
10062 + dev);
10063 + if (unlikely(!ppriv->store)) {
10064 + dev_err(dev, "dpaa2_io_store_create() failed\n");
10065 + err = -ENOMEM;
10066 + goto err;
10067 + }
10068 +
10069 + if (++i == priv->num_pairs)
10070 + break;
10071 + }
10072 +
10073 + return 0;
10074 +
10075 +err:
10076 + for_each_online_cpu(cpu) {
10077 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10078 + if (!ppriv->nctx.cb)
10079 + break;
10080 + dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
10081 + }
10082 +
10083 + for_each_online_cpu(cpu) {
10084 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10085 + if (!ppriv->store)
10086 + break;
10087 + dpaa2_io_store_destroy(ppriv->store);
10088 + }
10089 +
10090 + return err;
10091 +}
10092 +
10093 +static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
10094 +{
10095 + struct dpaa2_caam_priv_per_cpu *ppriv;
10096 + struct device *dev = priv->dev;
10097 + int i = 0, cpu;
10098 +
10099 + for_each_online_cpu(cpu) {
10100 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10101 + dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
10102 + dpaa2_io_store_destroy(ppriv->store);
10103 +
10104 + if (++i == priv->num_pairs)
10105 + return;
10106 + }
10107 +}
10108 +
10109 +static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
10110 +{
10111 + struct dpseci_rx_queue_cfg rx_queue_cfg;
10112 + struct device *dev = priv->dev;
10113 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
10114 + struct dpaa2_caam_priv_per_cpu *ppriv;
10115 + int err = 0, i = 0, cpu;
10116 +
10117 + /* Configure Rx queues */
10118 + for_each_online_cpu(cpu) {
10119 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10120 +
10121 + rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
10122 + DPSECI_QUEUE_OPT_USER_CTX;
10123 + rx_queue_cfg.order_preservation_en = 0;
10124 + rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
10125 + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
10126 + /*
10127 + * Rx priority (WQ) doesn't really matter, since we use
10128 + * pull mode, i.e. volatile dequeues from specific FQs
10129 + */
10130 + rx_queue_cfg.dest_cfg.priority = 0;
10131 + rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
10132 +
10133 + err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
10134 + &rx_queue_cfg);
10135 + if (err) {
10136 + dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
10137 + err);
10138 + return err;
10139 + }
10140 +
10141 + if (++i == priv->num_pairs)
10142 + break;
10143 + }
10144 +
10145 + return err;
10146 +}
10147 +
10148 +static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
10149 +{
10150 + struct device *dev = priv->dev;
10151 +
10152 + if (!priv->cscn_mem)
10153 + return;
10154 +
10155 + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
10156 + kfree(priv->cscn_mem);
10157 +}
10158 +
10159 +static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
10160 +{
10161 + struct device *dev = priv->dev;
10162 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
10163 +
10164 + dpaa2_dpseci_congestion_free(priv);
10165 + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
10166 +}
10167 +
10168 +static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
10169 + const struct dpaa2_fd *fd)
10170 +{
10171 + struct caam_request *req;
10172 + u32 fd_err;
10173 +
10174 + if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
10175 + dev_err(priv->dev, "Only Frame List FD format is supported!\n");
10176 + return;
10177 + }
10178 +
10179 + fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
10180 + if (unlikely(fd_err))
10181 + dev_err(priv->dev, "FD error: %08x\n", fd_err);
10182 +
10183 + /*
10184 + * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
10185 + * in FD[ERR] or FD[FRC].
10186 + */
10187 + req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
10188 + dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
10189 + DMA_BIDIRECTIONAL);
10190 + req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
10191 +}
10192 +
10193 +static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
10194 +{
10195 + int err;
10196 +
10197 + /* Retry while portal is busy */
10198 + do {
10199 + err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
10200 + ppriv->store);
10201 + } while (err == -EBUSY);
10202 +
10203 + if (unlikely(err))
10204 + dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
10205 +
10206 + return err;
10207 +}
10208 +
10209 +static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
10210 +{
10211 + struct dpaa2_dq *dq;
10212 + int cleaned = 0, is_last;
10213 +
10214 + do {
10215 + dq = dpaa2_io_store_next(ppriv->store, &is_last);
10216 + if (unlikely(!dq)) {
10217 + if (unlikely(!is_last)) {
10218 + dev_dbg(ppriv->priv->dev,
10219 + "FQ %d returned no valid frames\n",
10220 + ppriv->rsp_fqid);
10221 + /*
10222 + * MUST retry until we get some sort of
10223 + * valid response token (be it "empty dequeue"
10224 + * or a valid frame).
10225 + */
10226 + continue;
10227 + }
10228 + break;
10229 + }
10230 +
10231 + /* Process FD */
10232 + dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
10233 + cleaned++;
10234 + } while (!is_last);
10235 +
10236 + return cleaned;
10237 +}
10238 +
10239 +static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
10240 +{
10241 + struct dpaa2_caam_priv_per_cpu *ppriv;
10242 + struct dpaa2_caam_priv *priv;
10243 + int err, cleaned = 0, store_cleaned;
10244 +
10245 + ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
10246 + priv = ppriv->priv;
10247 +
10248 + if (unlikely(dpaa2_caam_pull_fq(ppriv)))
10249 + return 0;
10250 +
10251 + do {
10252 + store_cleaned = dpaa2_caam_store_consume(ppriv);
10253 + cleaned += store_cleaned;
10254 +
10255 + if (store_cleaned == 0 ||
10256 + cleaned > budget - DPAA2_CAAM_STORE_SIZE)
10257 + break;
10258 +
10259 + /* Try to dequeue some more */
10260 + err = dpaa2_caam_pull_fq(ppriv);
10261 + if (unlikely(err))
10262 + break;
10263 + } while (1);
10264 +
10265 + if (cleaned < budget) {
10266 + napi_complete_done(napi, cleaned);
10267 + err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
10268 + if (unlikely(err))
10269 + dev_err(priv->dev, "Notification rearm failed: %d\n",
10270 + err);
10271 + }
10272 +
10273 + return cleaned;
10274 +}
10275 +
10276 +static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
10277 + u16 token)
10278 +{
10279 + struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
10280 + struct device *dev = priv->dev;
10281 + int err;
10282 +
10283 + /*
10284 + * Congestion group feature supported starting with DPSECI API v5.1
10285 + * and only when object has been created with this capability.
10286 + */
10287 + if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
10288 + !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
10289 + return 0;
10290 +
10291 + priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
10292 + GFP_KERNEL | GFP_DMA);
10293 + if (!priv->cscn_mem)
10294 + return -ENOMEM;
10295 +
10296 + priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
10297 + priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
10298 + DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
10299 + if (dma_mapping_error(dev, priv->cscn_dma)) {
10300 + dev_err(dev, "Error mapping CSCN memory area\n");
10301 + err = -ENOMEM;
10302 + goto err_dma_map;
10303 + }
10304 +
10305 + cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
10306 + cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
10307 + cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
10308 + cong_notif_cfg.message_ctx = (u64)priv;
10309 + cong_notif_cfg.message_iova = priv->cscn_dma;
10310 + cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
10311 + DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
10312 + DPSECI_CGN_MODE_COHERENT_WRITE;
10313 +
10314 + err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
10315 + &cong_notif_cfg);
10316 + if (err) {
10317 + dev_err(dev, "dpseci_set_congestion_notification failed\n");
10318 + goto err_set_cong;
10319 + }
10320 +
10321 + return 0;
10322 +
10323 +err_set_cong:
10324 + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
10325 +err_dma_map:
10326 + kfree(priv->cscn_mem);
10327 +
10328 + return err;
10329 +}
10330 +
10331 +static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
10332 +{
10333 + struct device *dev = &ls_dev->dev;
10334 + struct dpaa2_caam_priv *priv;
10335 + struct dpaa2_caam_priv_per_cpu *ppriv;
10336 + int err, cpu;
10337 + u8 i;
10338 +
10339 + priv = dev_get_drvdata(dev);
10340 +
10341 + priv->dev = dev;
10342 + priv->dpsec_id = ls_dev->obj_desc.id;
10343 +
10344 + /* Get a handle for the DPSECI this interface is associate with */
10345 + err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
10346 + if (err) {
10347 + dev_err(dev, "dpsec_open() failed: %d\n", err);
10348 + goto err_open;
10349 + }
10350 +
10351 + dev_info(dev, "Opened dpseci object successfully\n");
10352 +
10353 + err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
10354 + &priv->minor_ver);
10355 + if (err) {
10356 + dev_err(dev, "dpseci_get_api_version() failed\n");
10357 + goto err_get_vers;
10358 + }
10359 +
10360 + err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
10361 + &priv->dpseci_attr);
10362 + if (err) {
10363 + dev_err(dev, "dpseci_get_attributes() failed\n");
10364 + goto err_get_vers;
10365 + }
10366 +
10367 + err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
10368 + &priv->sec_attr);
10369 + if (err) {
10370 + dev_err(dev, "dpseci_get_sec_attr() failed\n");
10371 + goto err_get_vers;
10372 + }
10373 +
10374 + err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
10375 + if (err) {
10376 + dev_err(dev, "setup_congestion() failed\n");
10377 + goto err_get_vers;
10378 + }
10379 +
10380 + priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
10381 + priv->dpseci_attr.num_tx_queues);
10382 + if (priv->num_pairs > num_online_cpus()) {
10383 + dev_warn(dev, "%d queues won't be used\n",
10384 + priv->num_pairs - num_online_cpus());
10385 + priv->num_pairs = num_online_cpus();
10386 + }
10387 +
10388 + for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
10389 + err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
10390 + &priv->rx_queue_attr[i]);
10391 + if (err) {
10392 + dev_err(dev, "dpseci_get_rx_queue() failed\n");
10393 + goto err_get_rx_queue;
10394 + }
10395 + }
10396 +
10397 + for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
10398 + err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
10399 + &priv->tx_queue_attr[i]);
10400 + if (err) {
10401 + dev_err(dev, "dpseci_get_tx_queue() failed\n");
10402 + goto err_get_rx_queue;
10403 + }
10404 + }
10405 +
10406 + i = 0;
10407 + for_each_online_cpu(cpu) {
10408 + u8 j;
10409 +
10410 + j = i % priv->num_pairs;
10411 +
10412 + ppriv = per_cpu_ptr(priv->ppriv, cpu);
10413 + ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
10414 +
10415 + /*
10416 + * Allow all cores to enqueue, while only some of them
10417 + * will take part in dequeuing.
10418 + */
10419 + if (++i > priv->num_pairs)
10420 + continue;
10421 +
10422 + ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
10423 + ppriv->prio = j;
10424 +
10425 + dev_info(dev, "pair %d: rx queue %d, tx queue %d\n", j,
10426 + priv->rx_queue_attr[j].fqid,
10427 + priv->tx_queue_attr[j].fqid);
10428 +
10429 + ppriv->net_dev.dev = *dev;
10430 + INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
10431 + netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
10432 + DPAA2_CAAM_NAPI_WEIGHT);
10433 + }
10434 +
10435 + return 0;
10436 +
10437 +err_get_rx_queue:
10438 + dpaa2_dpseci_congestion_free(priv);
10439 +err_get_vers:
10440 + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
10441 +err_open:
10442 + return err;
10443 +}
10444 +
10445 +static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
10446 +{
10447 + struct device *dev = priv->dev;
10448 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
10449 + struct dpaa2_caam_priv_per_cpu *ppriv;
10450 + int err, i;
10451 +
10452 + for (i = 0; i < priv->num_pairs; i++) {
10453 + ppriv = per_cpu_ptr(priv->ppriv, i);
10454 + napi_enable(&ppriv->napi);
10455 + }
10456 +
10457 + err = dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
10458 + if (err) {
10459 + dev_err(dev, "dpseci_enable() failed\n");
10460 + return err;
10461 + }
10462 +
10463 + dev_info(dev, "DPSECI version %d.%d\n",
10464 + priv->major_ver,
10465 + priv->minor_ver);
10466 +
10467 + return 0;
10468 +}
10469 +
10470 +static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
10471 +{
10472 + struct device *dev = priv->dev;
10473 + struct dpaa2_caam_priv_per_cpu *ppriv;
10474 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
10475 + int i, err = 0, enabled;
10476 +
10477 + err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
10478 + if (err) {
10479 + dev_err(dev, "dpseci_disable() failed\n");
10480 + return err;
10481 + }
10482 +
10483 + err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
10484 + if (err) {
10485 + dev_err(dev, "dpseci_is_enabled() failed\n");
10486 + return err;
10487 + }
10488 +
10489 + dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
10490 +
10491 + for (i = 0; i < priv->num_pairs; i++) {
10492 + ppriv = per_cpu_ptr(priv->ppriv, i);
10493 + napi_disable(&ppriv->napi);
10494 + netif_napi_del(&ppriv->napi);
10495 + }
10496 +
10497 + return 0;
10498 +}
10499 +
10500 +static struct list_head hash_list;
10501 +
10502 +static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
10503 +{
10504 + struct device *dev;
10505 + struct dpaa2_caam_priv *priv;
10506 + int i, err = 0;
10507 + bool registered = false;
10508 +
10509 + /*
10510 + * There is no way to get CAAM endianness - there is no direct register
10511 + * space access and MC f/w does not provide this attribute.
10512 + * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
10513 + * property.
10514 + */
10515 + caam_little_end = true;
10516 +
10517 + caam_imx = false;
10518 +
10519 + dev = &dpseci_dev->dev;
10520 +
10521 + priv = kzalloc(sizeof(*priv), GFP_KERNEL);
10522 + if (!priv)
10523 + return -ENOMEM;
10524 +
10525 + dev_set_drvdata(dev, priv);
10526 +
10527 + priv->domain = iommu_get_domain_for_dev(dev);
10528 +
10529 + qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
10530 + 0, SLAB_CACHE_DMA, NULL);
10531 + if (!qi_cache) {
10532 + dev_err(dev, "Can't allocate SEC cache\n");
10533 + err = -ENOMEM;
10534 + goto err_qicache;
10535 + }
10536 +
10537 + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
10538 + if (err) {
10539 + dev_err(dev, "dma_set_mask_and_coherent() failed\n");
10540 + goto err_dma_mask;
10541 + }
10542 +
10543 + /* Obtain a MC portal */
10544 + err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
10545 + if (err) {
10546 + if (err == -ENXIO)
10547 + err = -EPROBE_DEFER;
10548 + else
10549 + dev_err(dev, "MC portal allocation failed\n");
10550 +
10551 + goto err_dma_mask;
10552 + }
10553 +
10554 + priv->ppriv = alloc_percpu(*priv->ppriv);
10555 + if (!priv->ppriv) {
10556 + dev_err(dev, "alloc_percpu() failed\n");
10557 + err = -ENOMEM;
10558 + goto err_alloc_ppriv;
10559 + }
10560 +
10561 + /* DPSECI initialization */
10562 + err = dpaa2_dpseci_setup(dpseci_dev);
10563 + if (err) {
10564 + dev_err(dev, "dpaa2_dpseci_setup() failed\n");
10565 + goto err_dpseci_setup;
10566 + }
10567 +
10568 + /* DPIO */
10569 + err = dpaa2_dpseci_dpio_setup(priv);
10570 + if (err) {
10571 + if (err != -EPROBE_DEFER)
10572 + dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
10573 + goto err_dpio_setup;
10574 + }
10575 +
10576 + /* DPSECI binding to DPIO */
10577 + err = dpaa2_dpseci_bind(priv);
10578 + if (err) {
10579 + dev_err(dev, "dpaa2_dpseci_bind() failed\n");
10580 + goto err_bind;
10581 + }
10582 +
10583 + /* DPSECI enable */
10584 + err = dpaa2_dpseci_enable(priv);
10585 + if (err) {
10586 + dev_err(dev, "dpaa2_dpseci_enable() failed");
10587 + goto err_bind;
10588 + }
10589 +
10590 + /* register crypto algorithms the device supports */
10591 + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
10592 + struct caam_skcipher_alg *t_alg = driver_algs + i;
10593 + u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
10594 +
10595 + /* Skip DES algorithms if not supported by device */
10596 + if (!priv->sec_attr.des_acc_num &&
10597 + ((alg_sel == OP_ALG_ALGSEL_3DES) ||
10598 + (alg_sel == OP_ALG_ALGSEL_DES)))
10599 + continue;
10600 +
10601 + /* Skip AES algorithms if not supported by device */
10602 + if (!priv->sec_attr.aes_acc_num &&
10603 + (alg_sel == OP_ALG_ALGSEL_AES))
10604 + continue;
10605 +
10606 + /* Skip CHACHA20 algorithms if not supported by device */
10607 + if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
10608 + !priv->sec_attr.ccha_acc_num)
10609 + continue;
10610 +
10611 + t_alg->caam.dev = dev;
10612 + caam_skcipher_alg_init(t_alg);
10613 +
10614 + err = crypto_register_skcipher(&t_alg->skcipher);
10615 + if (err) {
10616 + dev_warn(dev, "%s alg registration failed: %d\n",
10617 + t_alg->skcipher.base.cra_driver_name, err);
10618 + continue;
10619 + }
10620 +
10621 + t_alg->registered = true;
10622 + registered = true;
10623 + }
10624 +
10625 + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
10626 + struct caam_aead_alg *t_alg = driver_aeads + i;
10627 + u32 c1_alg_sel = t_alg->caam.class1_alg_type &
10628 + OP_ALG_ALGSEL_MASK;
10629 + u32 c2_alg_sel = t_alg->caam.class2_alg_type &
10630 + OP_ALG_ALGSEL_MASK;
10631 +
10632 + /* Skip DES algorithms if not supported by device */
10633 + if (!priv->sec_attr.des_acc_num &&
10634 + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
10635 + (c1_alg_sel == OP_ALG_ALGSEL_DES)))
10636 + continue;
10637 +
10638 + /* Skip AES algorithms if not supported by device */
10639 + if (!priv->sec_attr.aes_acc_num &&
10640 + (c1_alg_sel == OP_ALG_ALGSEL_AES))
10641 + continue;
10642 +
10643 + /* Skip CHACHA20 algorithms if not supported by device */
10644 + if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
10645 + !priv->sec_attr.ccha_acc_num)
10646 + continue;
10647 +
10648 + /* Skip POLY1305 algorithms if not supported by device */
10649 + if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
10650 + !priv->sec_attr.ptha_acc_num)
10651 + continue;
10652 +
10653 + /*
10654 + * Skip algorithms requiring message digests
10655 + * if MD not supported by device.
10656 + */
10657 + if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
10658 + !priv->sec_attr.md_acc_num)
10659 + continue;
10660 +
10661 + t_alg->caam.dev = dev;
10662 + caam_aead_alg_init(t_alg);
10663 +
10664 + err = crypto_register_aead(&t_alg->aead);
10665 + if (err) {
10666 + dev_warn(dev, "%s alg registration failed: %d\n",
10667 + t_alg->aead.base.cra_driver_name, err);
10668 + continue;
10669 + }
10670 +
10671 + t_alg->registered = true;
10672 + registered = true;
10673 + }
10674 + if (registered)
10675 + dev_info(dev, "algorithms registered in /proc/crypto\n");
10676 +
10677 + /* register hash algorithms the device supports */
10678 + INIT_LIST_HEAD(&hash_list);
10679 +
10680 + /*
10681 + * Skip registration of any hashing algorithms if MD block
10682 + * is not present.
10683 + */
10684 + if (!priv->sec_attr.md_acc_num)
10685 + return 0;
10686 +
10687 + for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
10688 + struct caam_hash_alg *t_alg;
10689 + struct caam_hash_template *alg = driver_hash + i;
10690 +
10691 + /* register hmac version */
10692 + t_alg = caam_hash_alloc(dev, alg, true);
10693 + if (IS_ERR(t_alg)) {
10694 + err = PTR_ERR(t_alg);
10695 + dev_warn(dev, "%s hash alg allocation failed: %d\n",
10696 + alg->driver_name, err);
10697 + continue;
10698 + }
10699 +
10700 + err = crypto_register_ahash(&t_alg->ahash_alg);
10701 + if (err) {
10702 + dev_warn(dev, "%s alg registration failed: %d\n",
10703 + t_alg->ahash_alg.halg.base.cra_driver_name,
10704 + err);
10705 + kfree(t_alg);
10706 + } else {
10707 + list_add_tail(&t_alg->entry, &hash_list);
10708 + }
10709 +
10710 + /* register unkeyed version */
10711 + t_alg = caam_hash_alloc(dev, alg, false);
10712 + if (IS_ERR(t_alg)) {
10713 + err = PTR_ERR(t_alg);
10714 + dev_warn(dev, "%s alg allocation failed: %d\n",
10715 + alg->driver_name, err);
10716 + continue;
10717 + }
10718 +
10719 + err = crypto_register_ahash(&t_alg->ahash_alg);
10720 + if (err) {
10721 + dev_warn(dev, "%s alg registration failed: %d\n",
10722 + t_alg->ahash_alg.halg.base.cra_driver_name,
10723 + err);
10724 + kfree(t_alg);
10725 + } else {
10726 + list_add_tail(&t_alg->entry, &hash_list);
10727 + }
10728 + }
10729 + if (!list_empty(&hash_list))
10730 + dev_info(dev, "hash algorithms registered in /proc/crypto\n");
10731 +
10732 + return err;
10733 +
10734 +err_bind:
10735 + dpaa2_dpseci_dpio_free(priv);
10736 +err_dpio_setup:
10737 + dpaa2_dpseci_free(priv);
10738 +err_dpseci_setup:
10739 + free_percpu(priv->ppriv);
10740 +err_alloc_ppriv:
10741 + fsl_mc_portal_free(priv->mc_io);
10742 +err_dma_mask:
10743 + kmem_cache_destroy(qi_cache);
10744 +err_qicache:
10745 + dev_set_drvdata(dev, NULL);
10746 +
10747 + return err;
10748 +}
10749 +
10750 +static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
10751 +{
10752 + struct device *dev;
10753 + struct dpaa2_caam_priv *priv;
10754 + int i;
10755 +
10756 + dev = &ls_dev->dev;
10757 + priv = dev_get_drvdata(dev);
10758 +
10759 + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
10760 + struct caam_aead_alg *t_alg = driver_aeads + i;
10761 +
10762 + if (t_alg->registered)
10763 + crypto_unregister_aead(&t_alg->aead);
10764 + }
10765 +
10766 + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
10767 + struct caam_skcipher_alg *t_alg = driver_algs + i;
10768 +
10769 + if (t_alg->registered)
10770 + crypto_unregister_skcipher(&t_alg->skcipher);
10771 + }
10772 +
10773 + if (hash_list.next) {
10774 + struct caam_hash_alg *t_hash_alg, *p;
10775 +
10776 + list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
10777 + crypto_unregister_ahash(&t_hash_alg->ahash_alg);
10778 + list_del(&t_hash_alg->entry);
10779 + kfree(t_hash_alg);
10780 + }
10781 + }
10782 +
10783 + dpaa2_dpseci_disable(priv);
10784 + dpaa2_dpseci_dpio_free(priv);
10785 + dpaa2_dpseci_free(priv);
10786 + free_percpu(priv->ppriv);
10787 + fsl_mc_portal_free(priv->mc_io);
10788 + dev_set_drvdata(dev, NULL);
10789 + kmem_cache_destroy(qi_cache);
10790 +
10791 + return 0;
10792 +}
10793 +
10794 +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
10795 +{
10796 + struct dpaa2_fd fd;
10797 + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
10798 + struct dpaa2_caam_priv_per_cpu *ppriv;
10799 + int err = 0, i;
10800 +
10801 + if (IS_ERR(req))
10802 + return PTR_ERR(req);
10803 +
10804 + if (priv->cscn_mem) {
10805 + dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
10806 + DPAA2_CSCN_SIZE,
10807 + DMA_FROM_DEVICE);
10808 + if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
10809 + dev_dbg_ratelimited(dev, "Dropping request\n");
10810 + return -EBUSY;
10811 + }
10812 + }
10813 +
10814 + dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
10815 +
10816 + req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
10817 + DMA_BIDIRECTIONAL);
10818 + if (dma_mapping_error(dev, req->fd_flt_dma)) {
10819 + dev_err(dev, "DMA mapping error for QI enqueue request\n");
10820 + goto err_out;
10821 + }
10822 +
10823 + memset(&fd, 0, sizeof(fd));
10824 + dpaa2_fd_set_format(&fd, dpaa2_fd_list);
10825 + dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
10826 + dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
10827 + dpaa2_fd_set_flc(&fd, req->flc_dma);
10828 +
10829 + ppriv = this_cpu_ptr(priv->ppriv);
10830 + for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
10831 + err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
10832 + &fd);
10833 + if (err != -EBUSY)
10834 + break;
10835 +
10836 + cpu_relax();
10837 + }
10838 +
10839 + if (unlikely(err)) {
10840 + dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
10841 + goto err_out;
10842 + }
10843 +
10844 + return -EINPROGRESS;
10845 +
10846 +err_out:
10847 + dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
10848 + DMA_BIDIRECTIONAL);
10849 + return -EIO;
10850 +}
10851 +EXPORT_SYMBOL(dpaa2_caam_enqueue);
10852 +
10853 +const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
10854 + {
10855 + .vendor = FSL_MC_VENDOR_FREESCALE,
10856 + .obj_type = "dpseci",
10857 + },
10858 + { .vendor = 0x0 }
10859 +};
10860 +
10861 +static struct fsl_mc_driver dpaa2_caam_driver = {
10862 + .driver = {
10863 + .name = KBUILD_MODNAME,
10864 + .owner = THIS_MODULE,
10865 + },
10866 + .probe = dpaa2_caam_probe,
10867 + .remove = dpaa2_caam_remove,
10868 + .match_id_table = dpaa2_caam_match_id_table
10869 +};
10870 +
10871 +MODULE_LICENSE("Dual BSD/GPL");
10872 +MODULE_AUTHOR("Freescale Semiconductor, Inc");
10873 +MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
10874 +
10875 +module_fsl_mc_driver(dpaa2_caam_driver);
10876 --- /dev/null
10877 +++ b/drivers/crypto/caam/caamalg_qi2.h
10878 @@ -0,0 +1,276 @@
10879 +/*
10880 + * Copyright 2015-2016 Freescale Semiconductor Inc.
10881 + * Copyright 2017 NXP
10882 + *
10883 + * Redistribution and use in source and binary forms, with or without
10884 + * modification, are permitted provided that the following conditions are met:
10885 + * * Redistributions of source code must retain the above copyright
10886 + * notice, this list of conditions and the following disclaimer.
10887 + * * Redistributions in binary form must reproduce the above copyright
10888 + * notice, this list of conditions and the following disclaimer in the
10889 + * documentation and/or other materials provided with the distribution.
10890 + * * Neither the names of the above-listed copyright holders nor the
10891 + * names of any contributors may be used to endorse or promote products
10892 + * derived from this software without specific prior written permission.
10893 + *
10894 + *
10895 + * ALTERNATIVELY, this software may be distributed under the terms of the
10896 + * GNU General Public License ("GPL") as published by the Free Software
10897 + * Foundation, either version 2 of that License or (at your option) any
10898 + * later version.
10899 + *
10900 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
10901 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
10902 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
10903 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
10904 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
10905 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
10906 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
10907 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
10908 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
10909 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
10910 + * POSSIBILITY OF SUCH DAMAGE.
10911 + */
10912 +
10913 +#ifndef _CAAMALG_QI2_H_
10914 +#define _CAAMALG_QI2_H_
10915 +
10916 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h"
10917 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
10918 +#include <linux/threads.h>
10919 +#include "dpseci.h"
10920 +#include "desc_constr.h"
10921 +
10922 +#define DPAA2_CAAM_STORE_SIZE 16
10923 +/* NAPI weight *must* be a multiple of the store size. */
10924 +#define DPAA2_CAAM_NAPI_WEIGHT 64
10925 +
10926 +/* The congestion entrance threshold was chosen so that on LS2088
10927 + * we support the maximum throughput for the available memory
10928 + */
10929 +#define DPAA2_SEC_CONG_ENTRY_THRESH (128 * 1024 * 1024)
10930 +#define DPAA2_SEC_CONG_EXIT_THRESH (DPAA2_SEC_CONG_ENTRY_THRESH * 9 / 10)
10931 +
10932 +/**
10933 + * dpaa2_caam_priv - driver private data
10934 + * @dpseci_id: DPSECI object unique ID
10935 + * @major_ver: DPSECI major version
10936 + * @minor_ver: DPSECI minor version
10937 + * @dpseci_attr: DPSECI attributes
10938 + * @sec_attr: SEC engine attributes
10939 + * @rx_queue_attr: array of Rx queue attributes
10940 + * @tx_queue_attr: array of Tx queue attributes
10941 + * @cscn_mem: pointer to memory region containing the
10942 + * dpaa2_cscn struct; it's size is larger than
10943 + * sizeof(struct dpaa2_cscn) to accommodate alignment
10944 + * @cscn_mem_aligned: pointer to struct dpaa2_cscn; it is computed
10945 + * as PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN)
10946 + * @cscn_dma: dma address used by the QMAN to write CSCN messages
10947 + * @dev: device associated with the DPSECI object
10948 + * @mc_io: pointer to MC portal's I/O object
10949 + * @domain: IOMMU domain
10950 + * @ppriv: per CPU pointers to privata data
10951 + */
10952 +struct dpaa2_caam_priv {
10953 + int dpsec_id;
10954 +
10955 + u16 major_ver;
10956 + u16 minor_ver;
10957 +
10958 + struct dpseci_attr dpseci_attr;
10959 + struct dpseci_sec_attr sec_attr;
10960 + struct dpseci_rx_queue_attr rx_queue_attr[DPSECI_MAX_QUEUE_NUM];
10961 + struct dpseci_tx_queue_attr tx_queue_attr[DPSECI_MAX_QUEUE_NUM];
10962 + int num_pairs;
10963 +
10964 + /* congestion */
10965 + void *cscn_mem;
10966 + void *cscn_mem_aligned;
10967 + dma_addr_t cscn_dma;
10968 +
10969 + struct device *dev;
10970 + struct fsl_mc_io *mc_io;
10971 + struct iommu_domain *domain;
10972 +
10973 + struct dpaa2_caam_priv_per_cpu __percpu *ppriv;
10974 +};
10975 +
10976 +/**
10977 + * dpaa2_caam_priv_per_cpu - per CPU private data
10978 + * @napi: napi structure
10979 + * @net_dev: netdev used by napi
10980 + * @req_fqid: (virtual) request (Tx / enqueue) FQID
10981 + * @rsp_fqid: (virtual) response (Rx / dequeue) FQID
10982 + * @prio: internal queue number - index for dpaa2_caam_priv.*_queue_attr
10983 + * @nctx: notification context of response FQ
10984 + * @store: where dequeued frames are stored
10985 + * @priv: backpointer to dpaa2_caam_priv
10986 + * @dpio: portal used for data path operations
10987 + */
10988 +struct dpaa2_caam_priv_per_cpu {
10989 + struct napi_struct napi;
10990 + struct net_device net_dev;
10991 + int req_fqid;
10992 + int rsp_fqid;
10993 + int prio;
10994 + struct dpaa2_io_notification_ctx nctx;
10995 + struct dpaa2_io_store *store;
10996 + struct dpaa2_caam_priv *priv;
10997 + struct dpaa2_io *dpio;
10998 +};
10999 +
11000 +/*
11001 + * The CAAM QI hardware constructs a job descriptor which points
11002 + * to shared descriptor (as pointed by context_a of FQ to CAAM).
11003 + * When the job descriptor is executed by deco, the whole job
11004 + * descriptor together with shared descriptor gets loaded in
11005 + * deco buffer which is 64 words long (each 32-bit).
11006 + *
11007 + * The job descriptor constructed by QI hardware has layout:
11008 + *
11009 + * HEADER (1 word)
11010 + * Shdesc ptr (1 or 2 words)
11011 + * SEQ_OUT_PTR (1 word)
11012 + * Out ptr (1 or 2 words)
11013 + * Out length (1 word)
11014 + * SEQ_IN_PTR (1 word)
11015 + * In ptr (1 or 2 words)
11016 + * In length (1 word)
11017 + *
11018 + * The shdesc ptr is used to fetch shared descriptor contents
11019 + * into deco buffer.
11020 + *
11021 + * Apart from shdesc contents, the total number of words that
11022 + * get loaded in deco buffer are '8' or '11'. The remaining words
11023 + * in deco buffer can be used for storing shared descriptor.
11024 + */
11025 +#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
11026 +
11027 +/* Length of a single buffer in the QI driver memory cache */
11028 +#define CAAM_QI_MEMCACHE_SIZE 512
11029 +
11030 +/*
11031 + * aead_edesc - s/w-extended aead descriptor
11032 + * @src_nents: number of segments in input scatterlist
11033 + * @dst_nents: number of segments in output scatterlist
11034 + * @iv_dma: dma address of iv for checking continuity and link table
11035 + * @qm_sg_bytes: length of dma mapped h/w link table
11036 + * @qm_sg_dma: bus physical mapped address of h/w link table
11037 + * @assoclen: associated data length, in CAAM endianness
11038 + * @assoclen_dma: bus physical mapped address of req->assoclen
11039 + * @sgt: the h/w link table, followed by IV
11040 + */
11041 +struct aead_edesc {
11042 + int src_nents;
11043 + int dst_nents;
11044 + dma_addr_t iv_dma;
11045 + int qm_sg_bytes;
11046 + dma_addr_t qm_sg_dma;
11047 + unsigned int assoclen;
11048 + dma_addr_t assoclen_dma;
11049 + struct dpaa2_sg_entry sgt[0];
11050 +};
11051 +
11052 +/*
11053 + * tls_edesc - s/w-extended tls descriptor
11054 + * @src_nents: number of segments in input scatterlist
11055 + * @dst_nents: number of segments in output scatterlist
11056 + * @iv_dma: dma address of iv for checking continuity and link table
11057 + * @qm_sg_bytes: length of dma mapped h/w link table
11058 + * @qm_sg_dma: bus physical mapped address of h/w link table
11059 + * @tmp: array of scatterlists used by 'scatterwalk_ffwd'
11060 + * @dst: pointer to output scatterlist, usefull for unmapping
11061 + * @sgt: the h/w link table, followed by IV
11062 + */
11063 +struct tls_edesc {
11064 + int src_nents;
11065 + int dst_nents;
11066 + dma_addr_t iv_dma;
11067 + int qm_sg_bytes;
11068 + dma_addr_t qm_sg_dma;
11069 + struct scatterlist tmp[2];
11070 + struct scatterlist *dst;
11071 + struct dpaa2_sg_entry sgt[0];
11072 +};
11073 +
11074 +/*
11075 + * skcipher_edesc - s/w-extended skcipher descriptor
11076 + * @src_nents: number of segments in input scatterlist
11077 + * @dst_nents: number of segments in output scatterlist
11078 + * @iv_dma: dma address of iv for checking continuity and link table
11079 + * @qm_sg_bytes: length of dma mapped qm_sg space
11080 + * @qm_sg_dma: I/O virtual address of h/w link table
11081 + * @sgt: the h/w link table, followed by IV
11082 + */
11083 +struct skcipher_edesc {
11084 + int src_nents;
11085 + int dst_nents;
11086 + dma_addr_t iv_dma;
11087 + int qm_sg_bytes;
11088 + dma_addr_t qm_sg_dma;
11089 + struct dpaa2_sg_entry sgt[0];
11090 +};
11091 +
11092 +/*
11093 + * ahash_edesc - s/w-extended ahash descriptor
11094 + * @dst_dma: I/O virtual address of req->result
11095 + * @qm_sg_dma: I/O virtual address of h/w link table
11096 + * @src_nents: number of segments in input scatterlist
11097 + * @qm_sg_bytes: length of dma mapped qm_sg space
11098 + * @sgt: pointer to h/w link table
11099 + */
11100 +struct ahash_edesc {
11101 + dma_addr_t dst_dma;
11102 + dma_addr_t qm_sg_dma;
11103 + int src_nents;
11104 + int qm_sg_bytes;
11105 + struct dpaa2_sg_entry sgt[0];
11106 +};
11107 +
11108 +/**
11109 + * caam_flc - Flow Context (FLC)
11110 + * @flc: Flow Context options
11111 + * @sh_desc: Shared Descriptor
11112 + */
11113 +struct caam_flc {
11114 + u32 flc[16];
11115 + u32 sh_desc[MAX_SDLEN];
11116 +} ____cacheline_aligned;
11117 +
11118 +enum optype {
11119 + ENCRYPT = 0,
11120 + DECRYPT,
11121 + NUM_OP
11122 +};
11123 +
11124 +/**
11125 + * caam_request - the request structure the driver application should fill while
11126 + * submitting a job to driver.
11127 + * @fd_flt: Frame list table defining input and output
11128 + * fd_flt[0] - FLE pointing to output buffer
11129 + * fd_flt[1] - FLE pointing to input buffer
11130 + * @fd_flt_dma: DMA address for the frame list table
11131 + * @flc: Flow Context
11132 + * @flc_dma: I/O virtual address of Flow Context
11133 + * @cbk: Callback function to invoke when job is completed
11134 + * @ctx: arbit context attached with request by the application
11135 + * @edesc: extended descriptor; points to one of {skcipher,aead}_edesc
11136 + */
11137 +struct caam_request {
11138 + struct dpaa2_fl_entry fd_flt[2];
11139 + dma_addr_t fd_flt_dma;
11140 + struct caam_flc *flc;
11141 + dma_addr_t flc_dma;
11142 + void (*cbk)(void *ctx, u32 err);
11143 + void *ctx;
11144 + void *edesc;
11145 +};
11146 +
11147 +/**
11148 + * dpaa2_caam_enqueue() - enqueue a crypto request
11149 + * @dev: device associated with the DPSECI object
11150 + * @req: pointer to caam_request
11151 + */
11152 +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req);
11153 +
11154 +#endif /* _CAAMALG_QI2_H_ */
11155 --- a/drivers/crypto/caam/caamhash.c
11156 +++ b/drivers/crypto/caam/caamhash.c
11157 @@ -2,6 +2,7 @@
11158 * caam - Freescale FSL CAAM support for ahash functions of crypto API
11159 *
11160 * Copyright 2011 Freescale Semiconductor, Inc.
11161 + * Copyright 2018 NXP
11162 *
11163 * Based on caamalg.c crypto API driver.
11164 *
11165 @@ -62,6 +63,7 @@
11166 #include "error.h"
11167 #include "sg_sw_sec4.h"
11168 #include "key_gen.h"
11169 +#include "caamhash_desc.h"
11170
11171 #define CAAM_CRA_PRIORITY 3000
11172
11173 @@ -71,14 +73,6 @@
11174 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
11175 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
11176
11177 -/* length of descriptors text */
11178 -#define DESC_AHASH_BASE (3 * CAAM_CMD_SZ)
11179 -#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
11180 -#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
11181 -#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
11182 -#define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
11183 -#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
11184 -
11185 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
11186 CAAM_MAX_HASH_KEY_SIZE)
11187 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
11188 @@ -107,6 +101,7 @@ struct caam_hash_ctx {
11189 dma_addr_t sh_desc_update_first_dma;
11190 dma_addr_t sh_desc_fin_dma;
11191 dma_addr_t sh_desc_digest_dma;
11192 + enum dma_data_direction dir;
11193 struct device *jrdev;
11194 u8 key[CAAM_MAX_HASH_KEY_SIZE];
11195 int ctx_len;
11196 @@ -218,7 +213,7 @@ static inline int buf_map_to_sec4_sg(str
11197 }
11198
11199 /* Map state->caam_ctx, and add it to link table */
11200 -static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
11201 +static inline int ctx_map_to_sec4_sg(struct device *jrdev,
11202 struct caam_hash_state *state, int ctx_len,
11203 struct sec4_sg_entry *sec4_sg, u32 flag)
11204 {
11205 @@ -234,68 +229,22 @@ static inline int ctx_map_to_sec4_sg(u32
11206 return 0;
11207 }
11208
11209 -/*
11210 - * For ahash update, final and finup (import_ctx = true)
11211 - * import context, read and write to seqout
11212 - * For ahash firsts and digest (import_ctx = false)
11213 - * read and write to seqout
11214 - */
11215 -static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
11216 - struct caam_hash_ctx *ctx, bool import_ctx)
11217 -{
11218 - u32 op = ctx->adata.algtype;
11219 - u32 *skip_key_load;
11220 -
11221 - init_sh_desc(desc, HDR_SHARE_SERIAL);
11222 -
11223 - /* Append key if it has been set; ahash update excluded */
11224 - if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
11225 - /* Skip key loading if already shared */
11226 - skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
11227 - JUMP_COND_SHRD);
11228 -
11229 - append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
11230 - ctx->adata.keylen, CLASS_2 |
11231 - KEY_DEST_MDHA_SPLIT | KEY_ENC);
11232 -
11233 - set_jump_tgt_here(desc, skip_key_load);
11234 -
11235 - op |= OP_ALG_AAI_HMAC_PRECOMP;
11236 - }
11237 -
11238 - /* If needed, import context from software */
11239 - if (import_ctx)
11240 - append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
11241 - LDST_SRCDST_BYTE_CONTEXT);
11242 -
11243 - /* Class 2 operation */
11244 - append_operation(desc, op | state | OP_ALG_ENCRYPT);
11245 -
11246 - /*
11247 - * Load from buf and/or src and write to req->result or state->context
11248 - * Calculate remaining bytes to read
11249 - */
11250 - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
11251 - /* Read remaining bytes */
11252 - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
11253 - FIFOLD_TYPE_MSG | KEY_VLF);
11254 - /* Store class2 context bytes */
11255 - append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
11256 - LDST_SRCDST_BYTE_CONTEXT);
11257 -}
11258 -
11259 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
11260 {
11261 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
11262 int digestsize = crypto_ahash_digestsize(ahash);
11263 struct device *jrdev = ctx->jrdev;
11264 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
11265 u32 *desc;
11266
11267 + ctx->adata.key_virt = ctx->key;
11268 +
11269 /* ahash_update shared descriptor */
11270 desc = ctx->sh_desc_update;
11271 - ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
11272 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
11273 + ctx->ctx_len, true, ctrlpriv->era);
11274 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
11275 - desc_bytes(desc), DMA_TO_DEVICE);
11276 + desc_bytes(desc), ctx->dir);
11277 #ifdef DEBUG
11278 print_hex_dump(KERN_ERR,
11279 "ahash update shdesc@"__stringify(__LINE__)": ",
11280 @@ -304,9 +253,10 @@ static int ahash_set_sh_desc(struct cryp
11281
11282 /* ahash_update_first shared descriptor */
11283 desc = ctx->sh_desc_update_first;
11284 - ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
11285 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
11286 + ctx->ctx_len, false, ctrlpriv->era);
11287 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
11288 - desc_bytes(desc), DMA_TO_DEVICE);
11289 + desc_bytes(desc), ctx->dir);
11290 #ifdef DEBUG
11291 print_hex_dump(KERN_ERR,
11292 "ahash update first shdesc@"__stringify(__LINE__)": ",
11293 @@ -315,9 +265,10 @@ static int ahash_set_sh_desc(struct cryp
11294
11295 /* ahash_final shared descriptor */
11296 desc = ctx->sh_desc_fin;
11297 - ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
11298 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
11299 + ctx->ctx_len, true, ctrlpriv->era);
11300 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
11301 - desc_bytes(desc), DMA_TO_DEVICE);
11302 + desc_bytes(desc), ctx->dir);
11303 #ifdef DEBUG
11304 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
11305 DUMP_PREFIX_ADDRESS, 16, 4, desc,
11306 @@ -326,9 +277,10 @@ static int ahash_set_sh_desc(struct cryp
11307
11308 /* ahash_digest shared descriptor */
11309 desc = ctx->sh_desc_digest;
11310 - ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
11311 + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
11312 + ctx->ctx_len, false, ctrlpriv->era);
11313 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
11314 - desc_bytes(desc), DMA_TO_DEVICE);
11315 + desc_bytes(desc), ctx->dir);
11316 #ifdef DEBUG
11317 print_hex_dump(KERN_ERR,
11318 "ahash digest shdesc@"__stringify(__LINE__)": ",
11319 @@ -421,6 +373,7 @@ static int ahash_setkey(struct crypto_ah
11320 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
11321 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
11322 int digestsize = crypto_ahash_digestsize(ahash);
11323 + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
11324 int ret;
11325 u8 *hashed_key = NULL;
11326
11327 @@ -441,16 +394,26 @@ static int ahash_setkey(struct crypto_ah
11328 key = hashed_key;
11329 }
11330
11331 - ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
11332 - CAAM_MAX_HASH_KEY_SIZE);
11333 - if (ret)
11334 - goto bad_free_key;
11335 + /*
11336 + * If DKP is supported, use it in the shared descriptor to generate
11337 + * the split key.
11338 + */
11339 + if (ctrlpriv->era >= 6) {
11340 + ctx->adata.key_inline = true;
11341 + ctx->adata.keylen = keylen;
11342 + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
11343 + OP_ALG_ALGSEL_MASK);
11344
11345 -#ifdef DEBUG
11346 - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
11347 - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
11348 - ctx->adata.keylen_pad, 1);
11349 -#endif
11350 + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
11351 + goto bad_free_key;
11352 +
11353 + memcpy(ctx->key, key, keylen);
11354 + } else {
11355 + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
11356 + keylen, CAAM_MAX_HASH_KEY_SIZE);
11357 + if (ret)
11358 + goto bad_free_key;
11359 + }
11360
11361 kfree(hashed_key);
11362 return ahash_set_sh_desc(ahash);
11363 @@ -773,7 +736,7 @@ static int ahash_update_ctx(struct ahash
11364 edesc->src_nents = src_nents;
11365 edesc->sec4_sg_bytes = sec4_sg_bytes;
11366
11367 - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
11368 + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
11369 edesc->sec4_sg, DMA_BIDIRECTIONAL);
11370 if (ret)
11371 goto unmap_ctx;
11372 @@ -871,9 +834,8 @@ static int ahash_final_ctx(struct ahash_
11373 desc = edesc->hw_desc;
11374
11375 edesc->sec4_sg_bytes = sec4_sg_bytes;
11376 - edesc->src_nents = 0;
11377
11378 - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
11379 + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
11380 edesc->sec4_sg, DMA_TO_DEVICE);
11381 if (ret)
11382 goto unmap_ctx;
11383 @@ -967,7 +929,7 @@ static int ahash_finup_ctx(struct ahash_
11384
11385 edesc->src_nents = src_nents;
11386
11387 - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
11388 + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
11389 edesc->sec4_sg, DMA_TO_DEVICE);
11390 if (ret)
11391 goto unmap_ctx;
11392 @@ -1126,7 +1088,6 @@ static int ahash_final_no_ctx(struct aha
11393 dev_err(jrdev, "unable to map dst\n");
11394 goto unmap;
11395 }
11396 - edesc->src_nents = 0;
11397
11398 #ifdef DEBUG
11399 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
11400 @@ -1208,7 +1169,6 @@ static int ahash_update_no_ctx(struct ah
11401
11402 edesc->src_nents = src_nents;
11403 edesc->sec4_sg_bytes = sec4_sg_bytes;
11404 - edesc->dst_dma = 0;
11405
11406 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
11407 if (ret)
11408 @@ -1420,7 +1380,6 @@ static int ahash_update_first(struct aha
11409 }
11410
11411 edesc->src_nents = src_nents;
11412 - edesc->dst_dma = 0;
11413
11414 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
11415 to_hash);
11416 @@ -1722,6 +1681,7 @@ static int caam_hash_cra_init(struct cry
11417 HASH_MSG_LEN + 64,
11418 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
11419 dma_addr_t dma_addr;
11420 + struct caam_drv_private *priv;
11421
11422 /*
11423 * Get a Job ring from Job Ring driver to ensure in-order
11424 @@ -1733,10 +1693,13 @@ static int caam_hash_cra_init(struct cry
11425 return PTR_ERR(ctx->jrdev);
11426 }
11427
11428 + priv = dev_get_drvdata(ctx->jrdev->parent);
11429 + ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
11430 +
11431 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
11432 offsetof(struct caam_hash_ctx,
11433 sh_desc_update_dma),
11434 - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
11435 + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
11436 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
11437 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
11438 caam_jr_free(ctx->jrdev);
11439 @@ -1771,11 +1734,11 @@ static void caam_hash_cra_exit(struct cr
11440 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
11441 offsetof(struct caam_hash_ctx,
11442 sh_desc_update_dma),
11443 - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
11444 + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
11445 caam_jr_free(ctx->jrdev);
11446 }
11447
11448 -static void __exit caam_algapi_hash_exit(void)
11449 +void caam_algapi_hash_exit(void)
11450 {
11451 struct caam_hash_alg *t_alg, *n;
11452
11453 @@ -1834,56 +1797,38 @@ caam_hash_alloc(struct caam_hash_templat
11454 return t_alg;
11455 }
11456
11457 -static int __init caam_algapi_hash_init(void)
11458 +int caam_algapi_hash_init(struct device *ctrldev)
11459 {
11460 - struct device_node *dev_node;
11461 - struct platform_device *pdev;
11462 - struct device *ctrldev;
11463 int i = 0, err = 0;
11464 - struct caam_drv_private *priv;
11465 + struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
11466 unsigned int md_limit = SHA512_DIGEST_SIZE;
11467 - u32 cha_inst, cha_vid;
11468 -
11469 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
11470 - if (!dev_node) {
11471 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
11472 - if (!dev_node)
11473 - return -ENODEV;
11474 - }
11475 -
11476 - pdev = of_find_device_by_node(dev_node);
11477 - if (!pdev) {
11478 - of_node_put(dev_node);
11479 - return -ENODEV;
11480 - }
11481 -
11482 - ctrldev = &pdev->dev;
11483 - priv = dev_get_drvdata(ctrldev);
11484 - of_node_put(dev_node);
11485 -
11486 - /*
11487 - * If priv is NULL, it's probably because the caam driver wasn't
11488 - * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
11489 - */
11490 - if (!priv)
11491 - return -ENODEV;
11492 + u32 md_inst, md_vid;
11493
11494 /*
11495 * Register crypto algorithms the device supports. First, identify
11496 * presence and attributes of MD block.
11497 */
11498 - cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
11499 - cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
11500 + if (priv->era < 10) {
11501 + md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) &
11502 + CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
11503 + md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
11504 + CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
11505 + } else {
11506 + u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
11507 +
11508 + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
11509 + md_inst = mdha & CHA_VER_NUM_MASK;
11510 + }
11511
11512 /*
11513 * Skip registration of any hashing algorithms if MD block
11514 * is not present.
11515 */
11516 - if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
11517 + if (!md_inst)
11518 return -ENODEV;
11519
11520 /* Limit digest size based on LP256 */
11521 - if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
11522 + if (md_vid == CHA_VER_VID_MD_LP256)
11523 md_limit = SHA256_DIGEST_SIZE;
11524
11525 INIT_LIST_HEAD(&hash_list);
11526 @@ -1934,10 +1879,3 @@ static int __init caam_algapi_hash_init(
11527
11528 return err;
11529 }
11530 -
11531 -module_init(caam_algapi_hash_init);
11532 -module_exit(caam_algapi_hash_exit);
11533 -
11534 -MODULE_LICENSE("GPL");
11535 -MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
11536 -MODULE_AUTHOR("Freescale Semiconductor - NMG");
11537 --- /dev/null
11538 +++ b/drivers/crypto/caam/caamhash_desc.c
11539 @@ -0,0 +1,108 @@
11540 +/*
11541 + * Shared descriptors for ahash algorithms
11542 + *
11543 + * Copyright 2017 NXP
11544 + *
11545 + * Redistribution and use in source and binary forms, with or without
11546 + * modification, are permitted provided that the following conditions are met:
11547 + * * Redistributions of source code must retain the above copyright
11548 + * notice, this list of conditions and the following disclaimer.
11549 + * * Redistributions in binary form must reproduce the above copyright
11550 + * notice, this list of conditions and the following disclaimer in the
11551 + * documentation and/or other materials provided with the distribution.
11552 + * * Neither the names of the above-listed copyright holders nor the
11553 + * names of any contributors may be used to endorse or promote products
11554 + * derived from this software without specific prior written permission.
11555 + *
11556 + *
11557 + * ALTERNATIVELY, this software may be distributed under the terms of the
11558 + * GNU General Public License ("GPL") as published by the Free Software
11559 + * Foundation, either version 2 of that License or (at your option) any
11560 + * later version.
11561 + *
11562 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
11563 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
11564 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
11565 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
11566 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
11567 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
11568 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
11569 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
11570 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
11571 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
11572 + * POSSIBILITY OF SUCH DAMAGE.
11573 + */
11574 +
11575 +#include "compat.h"
11576 +#include "desc_constr.h"
11577 +#include "caamhash_desc.h"
11578 +
11579 +/**
11580 + * cnstr_shdsc_ahash - ahash shared descriptor
11581 + * @desc: pointer to buffer used for descriptor construction
11582 + * @adata: pointer to authentication transform definitions.
11583 + * A split key is required for SEC Era < 6; the size of the split key
11584 + * is specified in this case.
11585 + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224,
11586 + * SHA256, SHA384, SHA512}.
11587 + * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE}
11588 + * @digestsize: algorithm's digest size
11589 + * @ctx_len: size of Context Register
11590 + * @import_ctx: true if previous Context Register needs to be restored
11591 + * must be true for ahash update and final
11592 + * must be false for for ahash first and digest
11593 + * @era: SEC Era
11594 + */
11595 +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
11596 + int digestsize, int ctx_len, bool import_ctx, int era)
11597 +{
11598 + u32 op = adata->algtype;
11599 +
11600 + init_sh_desc(desc, HDR_SHARE_SERIAL);
11601 +
11602 + /* Append key if it has been set; ahash update excluded */
11603 + if (state != OP_ALG_AS_UPDATE && adata->keylen) {
11604 + u32 *skip_key_load;
11605 +
11606 + /* Skip key loading if already shared */
11607 + skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
11608 + JUMP_COND_SHRD);
11609 +
11610 + if (era < 6)
11611 + append_key_as_imm(desc, adata->key_virt,
11612 + adata->keylen_pad,
11613 + adata->keylen, CLASS_2 |
11614 + KEY_DEST_MDHA_SPLIT | KEY_ENC);
11615 + else
11616 + append_proto_dkp(desc, adata);
11617 +
11618 + set_jump_tgt_here(desc, skip_key_load);
11619 +
11620 + op |= OP_ALG_AAI_HMAC_PRECOMP;
11621 + }
11622 +
11623 + /* If needed, import context from software */
11624 + if (import_ctx)
11625 + append_seq_load(desc, ctx_len, LDST_CLASS_2_CCB |
11626 + LDST_SRCDST_BYTE_CONTEXT);
11627 +
11628 + /* Class 2 operation */
11629 + append_operation(desc, op | state | OP_ALG_ENCRYPT);
11630 +
11631 + /*
11632 + * Load from buf and/or src and write to req->result or state->context
11633 + * Calculate remaining bytes to read
11634 + */
11635 + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
11636 + /* Read remaining bytes */
11637 + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
11638 + FIFOLD_TYPE_MSG | KEY_VLF);
11639 + /* Store class2 context bytes */
11640 + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
11641 + LDST_SRCDST_BYTE_CONTEXT);
11642 +}
11643 +EXPORT_SYMBOL(cnstr_shdsc_ahash);
11644 +
11645 +MODULE_LICENSE("Dual BSD/GPL");
11646 +MODULE_DESCRIPTION("FSL CAAM ahash descriptors support");
11647 +MODULE_AUTHOR("NXP Semiconductors");
11648 --- /dev/null
11649 +++ b/drivers/crypto/caam/caamhash_desc.h
11650 @@ -0,0 +1,49 @@
11651 +/*
11652 + * Shared descriptors for ahash algorithms
11653 + *
11654 + * Copyright 2017 NXP
11655 + *
11656 + * Redistribution and use in source and binary forms, with or without
11657 + * modification, are permitted provided that the following conditions are met:
11658 + * * Redistributions of source code must retain the above copyright
11659 + * notice, this list of conditions and the following disclaimer.
11660 + * * Redistributions in binary form must reproduce the above copyright
11661 + * notice, this list of conditions and the following disclaimer in the
11662 + * documentation and/or other materials provided with the distribution.
11663 + * * Neither the names of the above-listed copyright holders nor the
11664 + * names of any contributors may be used to endorse or promote products
11665 + * derived from this software without specific prior written permission.
11666 + *
11667 + *
11668 + * ALTERNATIVELY, this software may be distributed under the terms of the
11669 + * GNU General Public License ("GPL") as published by the Free Software
11670 + * Foundation, either version 2 of that License or (at your option) any
11671 + * later version.
11672 + *
11673 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
11674 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
11675 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
11676 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
11677 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
11678 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
11679 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
11680 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
11681 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
11682 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
11683 + * POSSIBILITY OF SUCH DAMAGE.
11684 + */
11685 +
11686 +#ifndef _CAAMHASH_DESC_H_
11687 +#define _CAAMHASH_DESC_H_
11688 +
11689 +/* length of descriptors text */
11690 +#define DESC_AHASH_BASE (3 * CAAM_CMD_SZ)
11691 +#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
11692 +#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
11693 +#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
11694 +#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
11695 +
11696 +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
11697 + int digestsize, int ctx_len, bool import_ctx, int era);
11698 +
11699 +#endif /* _CAAMHASH_DESC_H_ */
11700 --- a/drivers/crypto/caam/caampkc.c
11701 +++ b/drivers/crypto/caam/caampkc.c
11702 @@ -2,6 +2,7 @@
11703 * caam - Freescale FSL CAAM support for Public Key Cryptography
11704 *
11705 * Copyright 2016 Freescale Semiconductor, Inc.
11706 + * Copyright 2018 NXP
11707 *
11708 * There is no Shared Descriptor for PKC so that the Job Descriptor must carry
11709 * all the desired key parameters, input and output pointers.
11710 @@ -1017,46 +1018,22 @@ static struct akcipher_alg caam_rsa = {
11711 };
11712
11713 /* Public Key Cryptography module initialization handler */
11714 -static int __init caam_pkc_init(void)
11715 +int caam_pkc_init(struct device *ctrldev)
11716 {
11717 - struct device_node *dev_node;
11718 - struct platform_device *pdev;
11719 - struct device *ctrldev;
11720 - struct caam_drv_private *priv;
11721 - u32 cha_inst, pk_inst;
11722 + struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
11723 + u32 pk_inst;
11724 int err;
11725
11726 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
11727 - if (!dev_node) {
11728 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
11729 - if (!dev_node)
11730 - return -ENODEV;
11731 - }
11732 -
11733 - pdev = of_find_device_by_node(dev_node);
11734 - if (!pdev) {
11735 - of_node_put(dev_node);
11736 - return -ENODEV;
11737 - }
11738 -
11739 - ctrldev = &pdev->dev;
11740 - priv = dev_get_drvdata(ctrldev);
11741 - of_node_put(dev_node);
11742 -
11743 - /*
11744 - * If priv is NULL, it's probably because the caam driver wasn't
11745 - * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
11746 - */
11747 - if (!priv)
11748 - return -ENODEV;
11749 -
11750 /* Determine public key hardware accelerator presence. */
11751 - cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
11752 - pk_inst = (cha_inst & CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
11753 + if (priv->era < 10)
11754 + pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
11755 + CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
11756 + else
11757 + pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK;
11758
11759 /* Do not register algorithms if PKHA is not present. */
11760 if (!pk_inst)
11761 - return -ENODEV;
11762 + return 0;
11763
11764 err = crypto_register_akcipher(&caam_rsa);
11765 if (err)
11766 @@ -1068,14 +1045,7 @@ static int __init caam_pkc_init(void)
11767 return err;
11768 }
11769
11770 -static void __exit caam_pkc_exit(void)
11771 +void caam_pkc_exit(void)
11772 {
11773 crypto_unregister_akcipher(&caam_rsa);
11774 }
11775 -
11776 -module_init(caam_pkc_init);
11777 -module_exit(caam_pkc_exit);
11778 -
11779 -MODULE_LICENSE("Dual BSD/GPL");
11780 -MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API");
11781 -MODULE_AUTHOR("Freescale Semiconductor");
11782 --- a/drivers/crypto/caam/caamrng.c
11783 +++ b/drivers/crypto/caam/caamrng.c
11784 @@ -2,6 +2,7 @@
11785 * caam - Freescale FSL CAAM support for hw_random
11786 *
11787 * Copyright 2011 Freescale Semiconductor, Inc.
11788 + * Copyright 2018 NXP
11789 *
11790 * Based on caamalg.c crypto API driver.
11791 *
11792 @@ -294,49 +295,29 @@ static struct hwrng caam_rng = {
11793 .read = caam_read,
11794 };
11795
11796 -static void __exit caam_rng_exit(void)
11797 +void caam_rng_exit(void)
11798 {
11799 caam_jr_free(rng_ctx->jrdev);
11800 hwrng_unregister(&caam_rng);
11801 kfree(rng_ctx);
11802 }
11803
11804 -static int __init caam_rng_init(void)
11805 +int caam_rng_init(struct device *ctrldev)
11806 {
11807 struct device *dev;
11808 - struct device_node *dev_node;
11809 - struct platform_device *pdev;
11810 - struct device *ctrldev;
11811 - struct caam_drv_private *priv;
11812 + u32 rng_inst;
11813 + struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
11814 int err;
11815
11816 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
11817 - if (!dev_node) {
11818 - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
11819 - if (!dev_node)
11820 - return -ENODEV;
11821 - }
11822 -
11823 - pdev = of_find_device_by_node(dev_node);
11824 - if (!pdev) {
11825 - of_node_put(dev_node);
11826 - return -ENODEV;
11827 - }
11828 -
11829 - ctrldev = &pdev->dev;
11830 - priv = dev_get_drvdata(ctrldev);
11831 - of_node_put(dev_node);
11832 -
11833 - /*
11834 - * If priv is NULL, it's probably because the caam driver wasn't
11835 - * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
11836 - */
11837 - if (!priv)
11838 - return -ENODEV;
11839 -
11840 /* Check for an instantiated RNG before registration */
11841 - if (!(rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & CHA_ID_LS_RNG_MASK))
11842 - return -ENODEV;
11843 + if (priv->era < 10)
11844 + rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
11845 + CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT;
11846 + else
11847 + rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK;
11848 +
11849 + if (!rng_inst)
11850 + return 0;
11851
11852 dev = caam_jr_alloc();
11853 if (IS_ERR(dev)) {
11854 @@ -361,10 +342,3 @@ free_caam_alloc:
11855 caam_jr_free(dev);
11856 return err;
11857 }
11858 -
11859 -module_init(caam_rng_init);
11860 -module_exit(caam_rng_exit);
11861 -
11862 -MODULE_LICENSE("GPL");
11863 -MODULE_DESCRIPTION("FSL CAAM support for hw_random API");
11864 -MODULE_AUTHOR("Freescale Semiconductor - NMG");
11865 --- a/drivers/crypto/caam/compat.h
11866 +++ b/drivers/crypto/caam/compat.h
11867 @@ -17,6 +17,7 @@
11868 #include <linux/of_platform.h>
11869 #include <linux/dma-mapping.h>
11870 #include <linux/io.h>
11871 +#include <linux/iommu.h>
11872 #include <linux/spinlock.h>
11873 #include <linux/rtnetlink.h>
11874 #include <linux/in.h>
11875 @@ -34,10 +35,13 @@
11876 #include <crypto/des.h>
11877 #include <crypto/sha.h>
11878 #include <crypto/md5.h>
11879 +#include <crypto/chacha20.h>
11880 +#include <crypto/poly1305.h>
11881 #include <crypto/internal/aead.h>
11882 #include <crypto/authenc.h>
11883 #include <crypto/akcipher.h>
11884 #include <crypto/scatterwalk.h>
11885 +#include <crypto/skcipher.h>
11886 #include <crypto/internal/skcipher.h>
11887 #include <crypto/internal/hash.h>
11888 #include <crypto/internal/rsa.h>
11889 --- a/drivers/crypto/caam/ctrl.c
11890 +++ b/drivers/crypto/caam/ctrl.c
11891 @@ -2,6 +2,7 @@
11892 * Controller-level driver, kernel property detection, initialization
11893 *
11894 * Copyright 2008-2012 Freescale Semiconductor, Inc.
11895 + * Copyright 2018 NXP
11896 */
11897
11898 #include <linux/device.h>
11899 @@ -16,17 +17,15 @@
11900 #include "desc_constr.h"
11901 #include "ctrl.h"
11902
11903 -bool caam_little_end;
11904 -EXPORT_SYMBOL(caam_little_end);
11905 bool caam_dpaa2;
11906 EXPORT_SYMBOL(caam_dpaa2);
11907 -bool caam_imx;
11908 -EXPORT_SYMBOL(caam_imx);
11909
11910 #ifdef CONFIG_CAAM_QI
11911 #include "qi.h"
11912 #endif
11913
11914 +static struct platform_device *caam_dma_dev;
11915 +
11916 /*
11917 * i.MX targets tend to have clock control subsystems that can
11918 * enable/disable clocking to our device.
11919 @@ -105,7 +104,7 @@ static inline int run_descriptor_deco0(s
11920 struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
11921 struct caam_deco __iomem *deco = ctrlpriv->deco;
11922 unsigned int timeout = 100000;
11923 - u32 deco_dbg_reg, flags;
11924 + u32 deco_dbg_reg, deco_state, flags;
11925 int i;
11926
11927
11928 @@ -148,13 +147,22 @@ static inline int run_descriptor_deco0(s
11929 timeout = 10000000;
11930 do {
11931 deco_dbg_reg = rd_reg32(&deco->desc_dbg);
11932 +
11933 + if (ctrlpriv->era < 10)
11934 + deco_state = (deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) >>
11935 + DESC_DBG_DECO_STAT_SHIFT;
11936 + else
11937 + deco_state = (rd_reg32(&deco->dbg_exec) &
11938 + DESC_DER_DECO_STAT_MASK) >>
11939 + DESC_DER_DECO_STAT_SHIFT;
11940 +
11941 /*
11942 * If an error occured in the descriptor, then
11943 * the DECO status field will be set to 0x0D
11944 */
11945 - if ((deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) ==
11946 - DESC_DBG_DECO_STAT_HOST_ERR)
11947 + if (deco_state == DECO_STAT_HOST_ERR)
11948 break;
11949 +
11950 cpu_relax();
11951 } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout);
11952
11953 @@ -316,15 +324,15 @@ static int caam_remove(struct platform_d
11954 of_platform_depopulate(ctrldev);
11955
11956 #ifdef CONFIG_CAAM_QI
11957 - if (ctrlpriv->qidev)
11958 - caam_qi_shutdown(ctrlpriv->qidev);
11959 + if (ctrlpriv->qi_init)
11960 + caam_qi_shutdown(ctrldev);
11961 #endif
11962
11963 /*
11964 * De-initialize RNG state handles initialized by this driver.
11965 - * In case of DPAA 2.x, RNG is managed by MC firmware.
11966 + * In case of SoCs with Management Complex, RNG is managed by MC f/w.
11967 */
11968 - if (!caam_dpaa2 && ctrlpriv->rng4_sh_init)
11969 + if (!ctrlpriv->mc_en && ctrlpriv->rng4_sh_init)
11970 deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init);
11971
11972 /* Shut down debug views */
11973 @@ -332,6 +340,9 @@ static int caam_remove(struct platform_d
11974 debugfs_remove_recursive(ctrlpriv->dfs_root);
11975 #endif
11976
11977 + if (caam_dma_dev)
11978 + platform_device_unregister(caam_dma_dev);
11979 +
11980 /* Unmap controller region */
11981 iounmap(ctrl);
11982
11983 @@ -433,6 +444,10 @@ static int caam_probe(struct platform_de
11984 {.family = "Freescale i.MX"},
11985 {},
11986 };
11987 + static struct platform_device_info caam_dma_pdev_info = {
11988 + .name = "caam-dma",
11989 + .id = PLATFORM_DEVID_NONE
11990 + };
11991 struct device *dev;
11992 struct device_node *nprop, *np;
11993 struct caam_ctrl __iomem *ctrl;
11994 @@ -442,7 +457,7 @@ static int caam_probe(struct platform_de
11995 struct caam_perfmon *perfmon;
11996 #endif
11997 u32 scfgr, comp_params;
11998 - u32 cha_vid_ls;
11999 + u8 rng_vid;
12000 int pg_size;
12001 int BLOCK_OFFSET = 0;
12002
12003 @@ -454,15 +469,54 @@ static int caam_probe(struct platform_de
12004 dev_set_drvdata(dev, ctrlpriv);
12005 nprop = pdev->dev.of_node;
12006
12007 + /* Get configuration properties from device tree */
12008 + /* First, get register page */
12009 + ctrl = of_iomap(nprop, 0);
12010 + if (!ctrl) {
12011 + dev_err(dev, "caam: of_iomap() failed\n");
12012 + return -ENOMEM;
12013 + }
12014 +
12015 + caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) &
12016 + (CSTA_PLEND | CSTA_ALT_PLEND));
12017 caam_imx = (bool)soc_device_match(imx_soc);
12018
12019 + comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
12020 + caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
12021 + ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
12022 +
12023 +#ifdef CONFIG_CAAM_QI
12024 + /* If (DPAA 1.x) QI present, check whether dependencies are available */
12025 + if (ctrlpriv->qi_present && !caam_dpaa2) {
12026 + ret = qman_is_probed();
12027 + if (!ret) {
12028 + ret = -EPROBE_DEFER;
12029 + goto iounmap_ctrl;
12030 + } else if (ret < 0) {
12031 + dev_err(dev, "failing probe due to qman probe error\n");
12032 + ret = -ENODEV;
12033 + goto iounmap_ctrl;
12034 + }
12035 +
12036 + ret = qman_portals_probed();
12037 + if (!ret) {
12038 + ret = -EPROBE_DEFER;
12039 + goto iounmap_ctrl;
12040 + } else if (ret < 0) {
12041 + dev_err(dev, "failing probe due to qman portals probe error\n");
12042 + ret = -ENODEV;
12043 + goto iounmap_ctrl;
12044 + }
12045 + }
12046 +#endif
12047 +
12048 /* Enable clocking */
12049 clk = caam_drv_identify_clk(&pdev->dev, "ipg");
12050 if (IS_ERR(clk)) {
12051 ret = PTR_ERR(clk);
12052 dev_err(&pdev->dev,
12053 "can't identify CAAM ipg clk: %d\n", ret);
12054 - return ret;
12055 + goto iounmap_ctrl;
12056 }
12057 ctrlpriv->caam_ipg = clk;
12058
12059 @@ -471,7 +525,7 @@ static int caam_probe(struct platform_de
12060 ret = PTR_ERR(clk);
12061 dev_err(&pdev->dev,
12062 "can't identify CAAM mem clk: %d\n", ret);
12063 - return ret;
12064 + goto iounmap_ctrl;
12065 }
12066 ctrlpriv->caam_mem = clk;
12067
12068 @@ -480,7 +534,7 @@ static int caam_probe(struct platform_de
12069 ret = PTR_ERR(clk);
12070 dev_err(&pdev->dev,
12071 "can't identify CAAM aclk clk: %d\n", ret);
12072 - return ret;
12073 + goto iounmap_ctrl;
12074 }
12075 ctrlpriv->caam_aclk = clk;
12076
12077 @@ -490,7 +544,7 @@ static int caam_probe(struct platform_de
12078 ret = PTR_ERR(clk);
12079 dev_err(&pdev->dev,
12080 "can't identify CAAM emi_slow clk: %d\n", ret);
12081 - return ret;
12082 + goto iounmap_ctrl;
12083 }
12084 ctrlpriv->caam_emi_slow = clk;
12085 }
12086 @@ -498,7 +552,7 @@ static int caam_probe(struct platform_de
12087 ret = clk_prepare_enable(ctrlpriv->caam_ipg);
12088 if (ret < 0) {
12089 dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret);
12090 - return ret;
12091 + goto iounmap_ctrl;
12092 }
12093
12094 ret = clk_prepare_enable(ctrlpriv->caam_mem);
12095 @@ -523,25 +577,10 @@ static int caam_probe(struct platform_de
12096 }
12097 }
12098
12099 - /* Get configuration properties from device tree */
12100 - /* First, get register page */
12101 - ctrl = of_iomap(nprop, 0);
12102 - if (ctrl == NULL) {
12103 - dev_err(dev, "caam: of_iomap() failed\n");
12104 - ret = -ENOMEM;
12105 - goto disable_caam_emi_slow;
12106 - }
12107 -
12108 - caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) &
12109 - (CSTA_PLEND | CSTA_ALT_PLEND));
12110 -
12111 - /* Finding the page size for using the CTPR_MS register */
12112 - comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
12113 - pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
12114 -
12115 /* Allocating the BLOCK_OFFSET based on the supported page size on
12116 * the platform
12117 */
12118 + pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
12119 if (pg_size == 0)
12120 BLOCK_OFFSET = PG_SIZE_4K;
12121 else
12122 @@ -563,11 +602,14 @@ static int caam_probe(struct platform_de
12123 /*
12124 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
12125 * long pointers in master configuration register.
12126 - * In case of DPAA 2.x, Management Complex firmware performs
12127 + * In case of SoCs with Management Complex, MC f/w performs
12128 * the configuration.
12129 */
12130 - caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
12131 - if (!caam_dpaa2)
12132 + np = of_find_compatible_node(NULL, NULL, "fsl,qoriq-mc");
12133 + ctrlpriv->mc_en = !!np;
12134 + of_node_put(np);
12135 +
12136 + if (!ctrlpriv->mc_en)
12137 clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR,
12138 MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF |
12139 MCFGR_WDENABLE | MCFGR_LARGE_BURST |
12140 @@ -612,14 +654,11 @@ static int caam_probe(struct platform_de
12141 }
12142 if (ret) {
12143 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
12144 - goto iounmap_ctrl;
12145 + goto disable_caam_emi_slow;
12146 }
12147
12148 - ret = of_platform_populate(nprop, caam_match, NULL, dev);
12149 - if (ret) {
12150 - dev_err(dev, "JR platform devices creation error\n");
12151 - goto iounmap_ctrl;
12152 - }
12153 + ctrlpriv->era = caam_get_era();
12154 + ctrlpriv->domain = iommu_get_domain_for_dev(dev);
12155
12156 #ifdef CONFIG_DEBUG_FS
12157 /*
12158 @@ -633,21 +672,7 @@ static int caam_probe(struct platform_de
12159 ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
12160 #endif
12161
12162 - ring = 0;
12163 - for_each_available_child_of_node(nprop, np)
12164 - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
12165 - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
12166 - ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
12167 - ((__force uint8_t *)ctrl +
12168 - (ring + JR_BLOCK_NUMBER) *
12169 - BLOCK_OFFSET
12170 - );
12171 - ctrlpriv->total_jobrs++;
12172 - ring++;
12173 - }
12174 -
12175 /* Check to see if (DPAA 1.x) QI present. If so, enable */
12176 - ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
12177 if (ctrlpriv->qi_present && !caam_dpaa2) {
12178 ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
12179 ((__force uint8_t *)ctrl +
12180 @@ -664,6 +689,25 @@ static int caam_probe(struct platform_de
12181 #endif
12182 }
12183
12184 + ret = of_platform_populate(nprop, caam_match, NULL, dev);
12185 + if (ret) {
12186 + dev_err(dev, "JR platform devices creation error\n");
12187 + goto shutdown_qi;
12188 + }
12189 +
12190 + ring = 0;
12191 + for_each_available_child_of_node(nprop, np)
12192 + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
12193 + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
12194 + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
12195 + ((__force uint8_t *)ctrl +
12196 + (ring + JR_BLOCK_NUMBER) *
12197 + BLOCK_OFFSET
12198 + );
12199 + ctrlpriv->total_jobrs++;
12200 + ring++;
12201 + }
12202 +
12203 /* If no QI and no rings specified, quit and go home */
12204 if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
12205 dev_err(dev, "no queues configured, terminating\n");
12206 @@ -671,15 +715,29 @@ static int caam_probe(struct platform_de
12207 goto caam_remove;
12208 }
12209
12210 - cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls);
12211 + caam_dma_pdev_info.parent = dev;
12212 + caam_dma_pdev_info.dma_mask = dma_get_mask(dev);
12213 + caam_dma_dev = platform_device_register_full(&caam_dma_pdev_info);
12214 + if (IS_ERR(caam_dma_dev)) {
12215 + dev_err(dev, "Unable to create and register caam-dma dev\n");
12216 + caam_dma_dev = 0;
12217 + } else {
12218 + set_dma_ops(&caam_dma_dev->dev, get_dma_ops(dev));
12219 + }
12220 +
12221 + if (ctrlpriv->era < 10)
12222 + rng_vid = (rd_reg32(&ctrl->perfmon.cha_id_ls) &
12223 + CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT;
12224 + else
12225 + rng_vid = (rd_reg32(&ctrl->vreg.rng) & CHA_VER_VID_MASK) >>
12226 + CHA_VER_VID_SHIFT;
12227
12228 /*
12229 * If SEC has RNG version >= 4 and RNG state handle has not been
12230 * already instantiated, do RNG instantiation
12231 - * In case of DPAA 2.x, RNG is managed by MC firmware.
12232 + * In case of SoCs with Management Complex, RNG is managed by MC f/w.
12233 */
12234 - if (!caam_dpaa2 &&
12235 - (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
12236 + if (!ctrlpriv->mc_en && rng_vid >= 4) {
12237 ctrlpriv->rng4_sh_init =
12238 rd_reg32(&ctrl->r4tst[0].rdsta);
12239 /*
12240 @@ -746,10 +804,9 @@ static int caam_probe(struct platform_de
12241
12242 /* Report "alive" for developer to see */
12243 dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
12244 - caam_get_era());
12245 - dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n",
12246 - ctrlpriv->total_jobrs, ctrlpriv->qi_present,
12247 - caam_dpaa2 ? "yes" : "no");
12248 + ctrlpriv->era);
12249 + dev_info(dev, "job rings = %d, qi = %d\n",
12250 + ctrlpriv->total_jobrs, ctrlpriv->qi_present);
12251
12252 #ifdef CONFIG_DEBUG_FS
12253 debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH,
12254 @@ -816,8 +873,11 @@ caam_remove:
12255 caam_remove(pdev);
12256 return ret;
12257
12258 -iounmap_ctrl:
12259 - iounmap(ctrl);
12260 +shutdown_qi:
12261 +#ifdef CONFIG_CAAM_QI
12262 + if (ctrlpriv->qi_init)
12263 + caam_qi_shutdown(dev);
12264 +#endif
12265 disable_caam_emi_slow:
12266 if (ctrlpriv->caam_emi_slow)
12267 clk_disable_unprepare(ctrlpriv->caam_emi_slow);
12268 @@ -827,6 +887,8 @@ disable_caam_mem:
12269 clk_disable_unprepare(ctrlpriv->caam_mem);
12270 disable_caam_ipg:
12271 clk_disable_unprepare(ctrlpriv->caam_ipg);
12272 +iounmap_ctrl:
12273 + iounmap(ctrl);
12274 return ret;
12275 }
12276
12277 --- a/drivers/crypto/caam/desc.h
12278 +++ b/drivers/crypto/caam/desc.h
12279 @@ -4,6 +4,7 @@
12280 * Definitions to support CAAM descriptor instruction generation
12281 *
12282 * Copyright 2008-2011 Freescale Semiconductor, Inc.
12283 + * Copyright 2018 NXP
12284 */
12285
12286 #ifndef DESC_H
12287 @@ -42,6 +43,7 @@
12288 #define CMD_SEQ_LOAD (0x03 << CMD_SHIFT)
12289 #define CMD_FIFO_LOAD (0x04 << CMD_SHIFT)
12290 #define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT)
12291 +#define CMD_MOVEB (0x07 << CMD_SHIFT)
12292 #define CMD_STORE (0x0a << CMD_SHIFT)
12293 #define CMD_SEQ_STORE (0x0b << CMD_SHIFT)
12294 #define CMD_FIFO_STORE (0x0c << CMD_SHIFT)
12295 @@ -242,6 +244,7 @@
12296 #define LDST_SRCDST_WORD_DESCBUF_SHARED (0x42 << LDST_SRCDST_SHIFT)
12297 #define LDST_SRCDST_WORD_DESCBUF_JOB_WE (0x45 << LDST_SRCDST_SHIFT)
12298 #define LDST_SRCDST_WORD_DESCBUF_SHARED_WE (0x46 << LDST_SRCDST_SHIFT)
12299 +#define LDST_SRCDST_WORD_INFO_FIFO_SM (0x71 << LDST_SRCDST_SHIFT)
12300 #define LDST_SRCDST_WORD_INFO_FIFO (0x7a << LDST_SRCDST_SHIFT)
12301
12302 /* Offset in source/destination */
12303 @@ -284,6 +287,12 @@
12304 #define LDLEN_SET_OFIFO_OFFSET_SHIFT 0
12305 #define LDLEN_SET_OFIFO_OFFSET_MASK (3 << LDLEN_SET_OFIFO_OFFSET_SHIFT)
12306
12307 +/* Special Length definitions when dst=sm, nfifo-{sm,m} */
12308 +#define LDLEN_MATH0 0
12309 +#define LDLEN_MATH1 1
12310 +#define LDLEN_MATH2 2
12311 +#define LDLEN_MATH3 3
12312 +
12313 /*
12314 * FIFO_LOAD/FIFO_STORE/SEQ_FIFO_LOAD/SEQ_FIFO_STORE
12315 * Command Constructs
12316 @@ -355,6 +364,7 @@
12317 #define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT)
12318 #define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT)
12319 #define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT)
12320 +#define FIFOLD_TYPE_IFIFO (0x0f << FIFOLD_TYPE_SHIFT)
12321
12322 /* Other types. Need to OR in last/flush bits as desired */
12323 #define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT)
12324 @@ -408,6 +418,7 @@
12325 #define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT)
12326 #define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT)
12327 #define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT)
12328 +#define FIFOST_TYPE_METADATA (0x3e << FIFOST_TYPE_SHIFT)
12329 #define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT)
12330
12331 /*
12332 @@ -444,6 +455,18 @@
12333 #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT)
12334 #define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT)
12335 #define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT)
12336 +#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT)
12337 +#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT)
12338 +#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT)
12339 +#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT)
12340 +#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT)
12341 +#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT)
12342 +#define OP_PCLID_DKP_RIF_MD5 (0x60 << OP_PCLID_SHIFT)
12343 +#define OP_PCLID_DKP_RIF_SHA1 (0x61 << OP_PCLID_SHIFT)
12344 +#define OP_PCLID_DKP_RIF_SHA224 (0x62 << OP_PCLID_SHIFT)
12345 +#define OP_PCLID_DKP_RIF_SHA256 (0x63 << OP_PCLID_SHIFT)
12346 +#define OP_PCLID_DKP_RIF_SHA384 (0x64 << OP_PCLID_SHIFT)
12347 +#define OP_PCLID_DKP_RIF_SHA512 (0x65 << OP_PCLID_SHIFT)
12348
12349 /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */
12350 #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT)
12351 @@ -1093,6 +1116,22 @@
12352 /* MacSec protinfos */
12353 #define OP_PCL_MACSEC 0x0001
12354
12355 +/* Derived Key Protocol (DKP) Protinfo */
12356 +#define OP_PCL_DKP_SRC_SHIFT 14
12357 +#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT)
12358 +#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT)
12359 +#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT)
12360 +#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT)
12361 +#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT)
12362 +#define OP_PCL_DKP_DST_SHIFT 12
12363 +#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT)
12364 +#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT)
12365 +#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT)
12366 +#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT)
12367 +#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT)
12368 +#define OP_PCL_DKP_KEY_SHIFT 0
12369 +#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT)
12370 +
12371 /* PKI unidirectional protocol protinfo bits */
12372 #define OP_PCL_PKPROT_TEST 0x0008
12373 #define OP_PCL_PKPROT_DECRYPT 0x0004
12374 @@ -1105,6 +1144,12 @@
12375 #define OP_ALG_TYPE_CLASS1 (2 << OP_ALG_TYPE_SHIFT)
12376 #define OP_ALG_TYPE_CLASS2 (4 << OP_ALG_TYPE_SHIFT)
12377
12378 +/* version register fields */
12379 +#define OP_VER_CCHA_NUM 0x000000ff /* Number CCHAs instantiated */
12380 +#define OP_VER_CCHA_MISC 0x0000ff00 /* CCHA Miscellaneous Information */
12381 +#define OP_VER_CCHA_REV 0x00ff0000 /* CCHA Revision Number */
12382 +#define OP_VER_CCHA_VID 0xff000000 /* CCHA Version ID */
12383 +
12384 #define OP_ALG_ALGSEL_SHIFT 16
12385 #define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT)
12386 #define OP_ALG_ALGSEL_SUBMASK (0x0f << OP_ALG_ALGSEL_SHIFT)
12387 @@ -1124,6 +1169,8 @@
12388 #define OP_ALG_ALGSEL_KASUMI (0x70 << OP_ALG_ALGSEL_SHIFT)
12389 #define OP_ALG_ALGSEL_CRC (0x90 << OP_ALG_ALGSEL_SHIFT)
12390 #define OP_ALG_ALGSEL_SNOW_F9 (0xA0 << OP_ALG_ALGSEL_SHIFT)
12391 +#define OP_ALG_ALGSEL_CHACHA20 (0xD0 << OP_ALG_ALGSEL_SHIFT)
12392 +#define OP_ALG_ALGSEL_POLY1305 (0xE0 << OP_ALG_ALGSEL_SHIFT)
12393
12394 #define OP_ALG_AAI_SHIFT 4
12395 #define OP_ALG_AAI_MASK (0x1ff << OP_ALG_AAI_SHIFT)
12396 @@ -1171,6 +1218,11 @@
12397 #define OP_ALG_AAI_RNG4_AI (0x80 << OP_ALG_AAI_SHIFT)
12398 #define OP_ALG_AAI_RNG4_SK (0x100 << OP_ALG_AAI_SHIFT)
12399
12400 +/* Chacha20 AAI set */
12401 +#define OP_ALG_AAI_AEAD (0x002 << OP_ALG_AAI_SHIFT)
12402 +#define OP_ALG_AAI_KEYSTREAM (0x001 << OP_ALG_AAI_SHIFT)
12403 +#define OP_ALG_AAI_BC8 (0x008 << OP_ALG_AAI_SHIFT)
12404 +
12405 /* hmac/smac AAI set */
12406 #define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT)
12407 #define OP_ALG_AAI_HMAC (0x01 << OP_ALG_AAI_SHIFT)
12408 @@ -1359,6 +1411,7 @@
12409 #define MOVE_SRC_MATH3 (0x07 << MOVE_SRC_SHIFT)
12410 #define MOVE_SRC_INFIFO (0x08 << MOVE_SRC_SHIFT)
12411 #define MOVE_SRC_INFIFO_CL (0x09 << MOVE_SRC_SHIFT)
12412 +#define MOVE_SRC_AUX_ABLK (0x0a << MOVE_SRC_SHIFT)
12413
12414 #define MOVE_DEST_SHIFT 16
12415 #define MOVE_DEST_MASK (0x0f << MOVE_DEST_SHIFT)
12416 @@ -1385,6 +1438,10 @@
12417
12418 #define MOVELEN_MRSEL_SHIFT 0
12419 #define MOVELEN_MRSEL_MASK (0x3 << MOVE_LEN_SHIFT)
12420 +#define MOVELEN_MRSEL_MATH0 (0 << MOVELEN_MRSEL_SHIFT)
12421 +#define MOVELEN_MRSEL_MATH1 (1 << MOVELEN_MRSEL_SHIFT)
12422 +#define MOVELEN_MRSEL_MATH2 (2 << MOVELEN_MRSEL_SHIFT)
12423 +#define MOVELEN_MRSEL_MATH3 (3 << MOVELEN_MRSEL_SHIFT)
12424
12425 /*
12426 * MATH Command Constructs
12427 @@ -1440,10 +1497,11 @@
12428 #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT)
12429 #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT)
12430 #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT)
12431 -#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT)
12432 +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT)
12433 #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT)
12434 #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT)
12435 #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT)
12436 +#define MATH_SRC1_ZERO (0x0f << MATH_SRC1_SHIFT)
12437
12438 /* Destination selectors */
12439 #define MATH_DEST_SHIFT 8
12440 @@ -1452,6 +1510,7 @@
12441 #define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT)
12442 #define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT)
12443 #define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT)
12444 +#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT)
12445 #define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT)
12446 #define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT)
12447 #define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT)
12448 @@ -1560,6 +1619,7 @@
12449 #define NFIFOENTRY_DTYPE_IV (0x2 << NFIFOENTRY_DTYPE_SHIFT)
12450 #define NFIFOENTRY_DTYPE_SAD (0x3 << NFIFOENTRY_DTYPE_SHIFT)
12451 #define NFIFOENTRY_DTYPE_ICV (0xA << NFIFOENTRY_DTYPE_SHIFT)
12452 +#define NFIFOENTRY_DTYPE_POLY (0xB << NFIFOENTRY_DTYPE_SHIFT)
12453 #define NFIFOENTRY_DTYPE_SKIP (0xE << NFIFOENTRY_DTYPE_SHIFT)
12454 #define NFIFOENTRY_DTYPE_MSG (0xF << NFIFOENTRY_DTYPE_SHIFT)
12455
12456 @@ -1624,4 +1684,31 @@
12457 /* Frame Descriptor Command for Replacement Job Descriptor */
12458 #define FD_CMD_REPLACE_JOB_DESC 0x20000000
12459
12460 +/* CHA Control Register bits */
12461 +#define CCTRL_RESET_CHA_ALL 0x1
12462 +#define CCTRL_RESET_CHA_AESA 0x2
12463 +#define CCTRL_RESET_CHA_DESA 0x4
12464 +#define CCTRL_RESET_CHA_AFHA 0x8
12465 +#define CCTRL_RESET_CHA_KFHA 0x10
12466 +#define CCTRL_RESET_CHA_SF8A 0x20
12467 +#define CCTRL_RESET_CHA_PKHA 0x40
12468 +#define CCTRL_RESET_CHA_MDHA 0x80
12469 +#define CCTRL_RESET_CHA_CRCA 0x100
12470 +#define CCTRL_RESET_CHA_RNG 0x200
12471 +#define CCTRL_RESET_CHA_SF9A 0x400
12472 +#define CCTRL_RESET_CHA_ZUCE 0x800
12473 +#define CCTRL_RESET_CHA_ZUCA 0x1000
12474 +#define CCTRL_UNLOAD_PK_A0 0x10000
12475 +#define CCTRL_UNLOAD_PK_A1 0x20000
12476 +#define CCTRL_UNLOAD_PK_A2 0x40000
12477 +#define CCTRL_UNLOAD_PK_A3 0x80000
12478 +#define CCTRL_UNLOAD_PK_B0 0x100000
12479 +#define CCTRL_UNLOAD_PK_B1 0x200000
12480 +#define CCTRL_UNLOAD_PK_B2 0x400000
12481 +#define CCTRL_UNLOAD_PK_B3 0x800000
12482 +#define CCTRL_UNLOAD_PK_N 0x1000000
12483 +#define CCTRL_UNLOAD_PK_A 0x4000000
12484 +#define CCTRL_UNLOAD_PK_B 0x8000000
12485 +#define CCTRL_UNLOAD_SBOX 0x10000000
12486 +
12487 #endif /* DESC_H */
12488 --- a/drivers/crypto/caam/desc_constr.h
12489 +++ b/drivers/crypto/caam/desc_constr.h
12490 @@ -109,7 +109,7 @@ static inline void init_job_desc_shared(
12491 append_ptr(desc, ptr);
12492 }
12493
12494 -static inline void append_data(u32 * const desc, void *data, int len)
12495 +static inline void append_data(u32 * const desc, const void *data, int len)
12496 {
12497 u32 *offset = desc_end(desc);
12498
12499 @@ -172,7 +172,7 @@ static inline void append_cmd_ptr_extlen
12500 append_cmd(desc, len);
12501 }
12502
12503 -static inline void append_cmd_data(u32 * const desc, void *data, int len,
12504 +static inline void append_cmd_data(u32 * const desc, const void *data, int len,
12505 u32 command)
12506 {
12507 append_cmd(desc, command | IMMEDIATE | len);
12508 @@ -189,6 +189,8 @@ static inline u32 *append_##cmd(u32 * co
12509 }
12510 APPEND_CMD_RET(jump, JUMP)
12511 APPEND_CMD_RET(move, MOVE)
12512 +APPEND_CMD_RET(moveb, MOVEB)
12513 +APPEND_CMD_RET(move_len, MOVE_LEN)
12514
12515 static inline void set_jump_tgt_here(u32 * const desc, u32 *jump_cmd)
12516 {
12517 @@ -271,7 +273,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN)
12518 APPEND_SEQ_PTR_INTLEN(out, OUT)
12519
12520 #define APPEND_CMD_PTR_TO_IMM(cmd, op) \
12521 -static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
12522 +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \
12523 unsigned int len, u32 options) \
12524 { \
12525 PRINT_POS; \
12526 @@ -312,7 +314,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_
12527 * from length of immediate data provided, e.g., split keys
12528 */
12529 #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \
12530 -static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
12531 +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \
12532 unsigned int data_len, \
12533 unsigned int len, u32 options) \
12534 { \
12535 @@ -327,7 +329,11 @@ static inline void append_##cmd##_imm_##
12536 u32 options) \
12537 { \
12538 PRINT_POS; \
12539 - append_cmd(desc, CMD_##op | IMMEDIATE | options | sizeof(type)); \
12540 + if (options & LDST_LEN_MASK) \
12541 + append_cmd(desc, CMD_##op | IMMEDIATE | options); \
12542 + else \
12543 + append_cmd(desc, CMD_##op | IMMEDIATE | options | \
12544 + sizeof(type)); \
12545 append_cmd(desc, immediate); \
12546 }
12547 APPEND_CMD_RAW_IMM(load, LOAD, u32);
12548 @@ -452,7 +458,7 @@ struct alginfo {
12549 unsigned int keylen_pad;
12550 union {
12551 dma_addr_t key_dma;
12552 - void *key_virt;
12553 + const void *key_virt;
12554 };
12555 bool key_inline;
12556 };
12557 @@ -496,4 +502,45 @@ static inline int desc_inline_query(unsi
12558 return (rem_bytes >= 0) ? 0 : -1;
12559 }
12560
12561 +/**
12562 + * append_proto_dkp - Derived Key Protocol (DKP): key -> split key
12563 + * @desc: pointer to buffer used for descriptor construction
12564 + * @adata: pointer to authentication transform definitions.
12565 + * keylen should be the length of initial key, while keylen_pad
12566 + * the length of the derived (split) key.
12567 + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224,
12568 + * SHA256, SHA384, SHA512}.
12569 + */
12570 +static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata)
12571 +{
12572 + u32 protid;
12573 +
12574 + /*
12575 + * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*}
12576 + * to OP_PCLID_DKP_{MD5, SHA*}
12577 + */
12578 + protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) |
12579 + (0x20 << OP_ALG_ALGSEL_SHIFT);
12580 +
12581 + if (adata->key_inline) {
12582 + int words;
12583 +
12584 + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
12585 + OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM |
12586 + adata->keylen);
12587 + append_data(desc, adata->key_virt, adata->keylen);
12588 +
12589 + /* Reserve space in descriptor buffer for the derived key */
12590 + words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
12591 + ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ;
12592 + if (words)
12593 + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words);
12594 + } else {
12595 + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
12596 + OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR |
12597 + adata->keylen);
12598 + append_ptr(desc, adata->key_dma);
12599 + }
12600 +}
12601 +
12602 #endif /* DESC_CONSTR_H */
12603 --- /dev/null
12604 +++ b/drivers/crypto/caam/dpseci.c
12605 @@ -0,0 +1,865 @@
12606 +/*
12607 + * Copyright 2013-2016 Freescale Semiconductor Inc.
12608 + * Copyright 2017 NXP
12609 + *
12610 + * Redistribution and use in source and binary forms, with or without
12611 + * modification, are permitted provided that the following conditions are met:
12612 + * * Redistributions of source code must retain the above copyright
12613 + * notice, this list of conditions and the following disclaimer.
12614 + * * Redistributions in binary form must reproduce the above copyright
12615 + * notice, this list of conditions and the following disclaimer in the
12616 + * documentation and/or other materials provided with the distribution.
12617 + * * Neither the names of the above-listed copyright holders nor the
12618 + * names of any contributors may be used to endorse or promote products
12619 + * derived from this software without specific prior written permission.
12620 + *
12621 + *
12622 + * ALTERNATIVELY, this software may be distributed under the terms of the
12623 + * GNU General Public License ("GPL") as published by the Free Software
12624 + * Foundation, either version 2 of that License or (at your option) any
12625 + * later version.
12626 + *
12627 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
12628 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
12629 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
12630 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
12631 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
12632 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
12633 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
12634 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
12635 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
12636 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
12637 + * POSSIBILITY OF SUCH DAMAGE.
12638 + */
12639 +
12640 +#include <linux/fsl/mc.h>
12641 +#include "../../../drivers/staging/fsl-mc/include/dpopr.h"
12642 +#include "dpseci.h"
12643 +#include "dpseci_cmd.h"
12644 +
12645 +/**
12646 + * dpseci_open() - Open a control session for the specified object
12647 + * @mc_io: Pointer to MC portal's I/O object
12648 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12649 + * @dpseci_id: DPSECI unique ID
12650 + * @token: Returned token; use in subsequent API calls
12651 + *
12652 + * This function can be used to open a control session for an already created
12653 + * object; an object may have been declared in the DPL or by calling the
12654 + * dpseci_create() function.
12655 + * This function returns a unique authentication token, associated with the
12656 + * specific object ID and the specific MC portal; this token must be used in all
12657 + * subsequent commands for this specific object.
12658 + *
12659 + * Return: '0' on success, error code otherwise
12660 + */
12661 +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id,
12662 + u16 *token)
12663 +{
12664 + struct fsl_mc_command cmd = { 0 };
12665 + struct dpseci_cmd_open *cmd_params;
12666 + int err;
12667 +
12668 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN,
12669 + cmd_flags,
12670 + 0);
12671 + cmd_params = (struct dpseci_cmd_open *)cmd.params;
12672 + cmd_params->dpseci_id = cpu_to_le32(dpseci_id);
12673 + err = mc_send_command(mc_io, &cmd);
12674 + if (err)
12675 + return err;
12676 +
12677 + *token = mc_cmd_hdr_read_token(&cmd);
12678 +
12679 + return 0;
12680 +}
12681 +
12682 +/**
12683 + * dpseci_close() - Close the control session of the object
12684 + * @mc_io: Pointer to MC portal's I/O object
12685 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12686 + * @token: Token of DPSECI object
12687 + *
12688 + * After this function is called, no further operations are allowed on the
12689 + * object without opening a new control session.
12690 + *
12691 + * Return: '0' on success, error code otherwise
12692 + */
12693 +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
12694 +{
12695 + struct fsl_mc_command cmd = { 0 };
12696 +
12697 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE,
12698 + cmd_flags,
12699 + token);
12700 + return mc_send_command(mc_io, &cmd);
12701 +}
12702 +
12703 +/**
12704 + * dpseci_create() - Create the DPSECI object
12705 + * @mc_io: Pointer to MC portal's I/O object
12706 + * @dprc_token: Parent container token; '0' for default container
12707 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12708 + * @cfg: Configuration structure
12709 + * @obj_id: returned object id
12710 + *
12711 + * Create the DPSECI object, allocate required resources and perform required
12712 + * initialization.
12713 + *
12714 + * The object can be created either by declaring it in the DPL file, or by
12715 + * calling this function.
12716 + *
12717 + * The function accepts an authentication token of a parent container that this
12718 + * object should be assigned to. The token can be '0' so the object will be
12719 + * assigned to the default container.
12720 + * The newly created object can be opened with the returned object id and using
12721 + * the container's associated tokens and MC portals.
12722 + *
12723 + * Return: '0' on success, error code otherwise
12724 + */
12725 +int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags,
12726 + const struct dpseci_cfg *cfg, u32 *obj_id)
12727 +{
12728 + struct fsl_mc_command cmd = { 0 };
12729 + struct dpseci_cmd_create *cmd_params;
12730 + int i, err;
12731 +
12732 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CREATE,
12733 + cmd_flags,
12734 + dprc_token);
12735 + cmd_params = (struct dpseci_cmd_create *)cmd.params;
12736 + for (i = 0; i < 8; i++)
12737 + cmd_params->priorities[i] = cfg->priorities[i];
12738 + for (i = 0; i < 8; i++)
12739 + cmd_params->priorities2[i] = cfg->priorities[8 + i];
12740 + cmd_params->num_tx_queues = cfg->num_tx_queues;
12741 + cmd_params->num_rx_queues = cfg->num_rx_queues;
12742 + cmd_params->options = cpu_to_le32(cfg->options);
12743 + err = mc_send_command(mc_io, &cmd);
12744 + if (err)
12745 + return err;
12746 +
12747 + *obj_id = mc_cmd_read_object_id(&cmd);
12748 +
12749 + return 0;
12750 +}
12751 +
12752 +/**
12753 + * dpseci_destroy() - Destroy the DPSECI object and release all its resources
12754 + * @mc_io: Pointer to MC portal's I/O object
12755 + * @dprc_token: Parent container token; '0' for default container
12756 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12757 + * @object_id: The object id; it must be a valid id within the container that
12758 + * created this object
12759 + *
12760 + * The function accepts the authentication token of the parent container that
12761 + * created the object (not the one that currently owns the object). The object
12762 + * is searched within parent using the provided 'object_id'.
12763 + * All tokens to the object must be closed before calling destroy.
12764 + *
12765 + * Return: '0' on success, error code otherwise
12766 + */
12767 +int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags,
12768 + u32 object_id)
12769 +{
12770 + struct fsl_mc_command cmd = { 0 };
12771 + struct dpseci_cmd_destroy *cmd_params;
12772 +
12773 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DESTROY,
12774 + cmd_flags,
12775 + dprc_token);
12776 + cmd_params = (struct dpseci_cmd_destroy *)cmd.params;
12777 + cmd_params->object_id = cpu_to_le32(object_id);
12778 +
12779 + return mc_send_command(mc_io, &cmd);
12780 +}
12781 +
12782 +/**
12783 + * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames
12784 + * @mc_io: Pointer to MC portal's I/O object
12785 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12786 + * @token: Token of DPSECI object
12787 + *
12788 + * Return: '0' on success, error code otherwise
12789 + */
12790 +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
12791 +{
12792 + struct fsl_mc_command cmd = { 0 };
12793 +
12794 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE,
12795 + cmd_flags,
12796 + token);
12797 + return mc_send_command(mc_io, &cmd);
12798 +}
12799 +
12800 +/**
12801 + * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames
12802 + * @mc_io: Pointer to MC portal's I/O object
12803 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12804 + * @token: Token of DPSECI object
12805 + *
12806 + * Return: '0' on success, error code otherwise
12807 + */
12808 +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
12809 +{
12810 + struct fsl_mc_command cmd = { 0 };
12811 +
12812 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE,
12813 + cmd_flags,
12814 + token);
12815 +
12816 + return mc_send_command(mc_io, &cmd);
12817 +}
12818 +
12819 +/**
12820 + * dpseci_is_enabled() - Check if the DPSECI is enabled.
12821 + * @mc_io: Pointer to MC portal's I/O object
12822 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12823 + * @token: Token of DPSECI object
12824 + * @en: Returns '1' if object is enabled; '0' otherwise
12825 + *
12826 + * Return: '0' on success, error code otherwise
12827 + */
12828 +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
12829 + int *en)
12830 +{
12831 + struct fsl_mc_command cmd = { 0 };
12832 + struct dpseci_rsp_is_enabled *rsp_params;
12833 + int err;
12834 +
12835 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED,
12836 + cmd_flags,
12837 + token);
12838 + err = mc_send_command(mc_io, &cmd);
12839 + if (err)
12840 + return err;
12841 +
12842 + rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params;
12843 + *en = dpseci_get_field(rsp_params->is_enabled, ENABLE);
12844 +
12845 + return 0;
12846 +}
12847 +
12848 +/**
12849 + * dpseci_reset() - Reset the DPSECI, returns the object to initial state.
12850 + * @mc_io: Pointer to MC portal's I/O object
12851 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12852 + * @token: Token of DPSECI object
12853 + *
12854 + * Return: '0' on success, error code otherwise
12855 + */
12856 +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token)
12857 +{
12858 + struct fsl_mc_command cmd = { 0 };
12859 +
12860 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET,
12861 + cmd_flags,
12862 + token);
12863 +
12864 + return mc_send_command(mc_io, &cmd);
12865 +}
12866 +
12867 +/**
12868 + * dpseci_get_irq_enable() - Get overall interrupt state
12869 + * @mc_io: Pointer to MC portal's I/O object
12870 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12871 + * @token: Token of DPSECI object
12872 + * @irq_index: The interrupt index to configure
12873 + * @en: Returned Interrupt state - enable = 1, disable = 0
12874 + *
12875 + * Return: '0' on success, error code otherwise
12876 + */
12877 +int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
12878 + u8 irq_index, u8 *en)
12879 +{
12880 + struct fsl_mc_command cmd = { 0 };
12881 + struct dpseci_cmd_irq_enable *cmd_params;
12882 + struct dpseci_rsp_get_irq_enable *rsp_params;
12883 + int err;
12884 +
12885 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_ENABLE,
12886 + cmd_flags,
12887 + token);
12888 + cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params;
12889 + cmd_params->irq_index = irq_index;
12890 + err = mc_send_command(mc_io, &cmd);
12891 + if (err)
12892 + return err;
12893 +
12894 + rsp_params = (struct dpseci_rsp_get_irq_enable *)cmd.params;
12895 + *en = rsp_params->enable_state;
12896 +
12897 + return 0;
12898 +}
12899 +
12900 +/**
12901 + * dpseci_set_irq_enable() - Set overall interrupt state.
12902 + * @mc_io: Pointer to MC portal's I/O object
12903 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12904 + * @token: Token of DPSECI object
12905 + * @irq_index: The interrupt index to configure
12906 + * @en: Interrupt state - enable = 1, disable = 0
12907 + *
12908 + * Allows GPP software to control when interrupts are generated.
12909 + * Each interrupt can have up to 32 causes. The enable/disable control's the
12910 + * overall interrupt state. If the interrupt is disabled no causes will cause
12911 + * an interrupt.
12912 + *
12913 + * Return: '0' on success, error code otherwise
12914 + */
12915 +int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
12916 + u8 irq_index, u8 en)
12917 +{
12918 + struct fsl_mc_command cmd = { 0 };
12919 + struct dpseci_cmd_irq_enable *cmd_params;
12920 +
12921 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_ENABLE,
12922 + cmd_flags,
12923 + token);
12924 + cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params;
12925 + cmd_params->irq_index = irq_index;
12926 + cmd_params->enable_state = en;
12927 +
12928 + return mc_send_command(mc_io, &cmd);
12929 +}
12930 +
12931 +/**
12932 + * dpseci_get_irq_mask() - Get interrupt mask.
12933 + * @mc_io: Pointer to MC portal's I/O object
12934 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12935 + * @token: Token of DPSECI object
12936 + * @irq_index: The interrupt index to configure
12937 + * @mask: Returned event mask to trigger interrupt
12938 + *
12939 + * Every interrupt can have up to 32 causes and the interrupt model supports
12940 + * masking/unmasking each cause independently.
12941 + *
12942 + * Return: '0' on success, error code otherwise
12943 + */
12944 +int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
12945 + u8 irq_index, u32 *mask)
12946 +{
12947 + struct fsl_mc_command cmd = { 0 };
12948 + struct dpseci_cmd_irq_mask *cmd_params;
12949 + int err;
12950 +
12951 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_MASK,
12952 + cmd_flags,
12953 + token);
12954 + cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params;
12955 + cmd_params->irq_index = irq_index;
12956 + err = mc_send_command(mc_io, &cmd);
12957 + if (err)
12958 + return err;
12959 +
12960 + *mask = le32_to_cpu(cmd_params->mask);
12961 +
12962 + return 0;
12963 +}
12964 +
12965 +/**
12966 + * dpseci_set_irq_mask() - Set interrupt mask.
12967 + * @mc_io: Pointer to MC portal's I/O object
12968 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
12969 + * @token: Token of DPSECI object
12970 + * @irq_index: The interrupt index to configure
12971 + * @mask: event mask to trigger interrupt;
12972 + * each bit:
12973 + * 0 = ignore event
12974 + * 1 = consider event for asserting IRQ
12975 + *
12976 + * Every interrupt can have up to 32 causes and the interrupt model supports
12977 + * masking/unmasking each cause independently
12978 + *
12979 + * Return: '0' on success, error code otherwise
12980 + */
12981 +int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
12982 + u8 irq_index, u32 mask)
12983 +{
12984 + struct fsl_mc_command cmd = { 0 };
12985 + struct dpseci_cmd_irq_mask *cmd_params;
12986 +
12987 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_MASK,
12988 + cmd_flags,
12989 + token);
12990 + cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params;
12991 + cmd_params->mask = cpu_to_le32(mask);
12992 + cmd_params->irq_index = irq_index;
12993 +
12994 + return mc_send_command(mc_io, &cmd);
12995 +}
12996 +
12997 +/**
12998 + * dpseci_get_irq_status() - Get the current status of any pending interrupts
12999 + * @mc_io: Pointer to MC portal's I/O object
13000 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13001 + * @token: Token of DPSECI object
13002 + * @irq_index: The interrupt index to configure
13003 + * @status: Returned interrupts status - one bit per cause:
13004 + * 0 = no interrupt pending
13005 + * 1 = interrupt pending
13006 + *
13007 + * Return: '0' on success, error code otherwise
13008 + */
13009 +int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13010 + u8 irq_index, u32 *status)
13011 +{
13012 + struct fsl_mc_command cmd = { 0 };
13013 + struct dpseci_cmd_irq_status *cmd_params;
13014 + int err;
13015 +
13016 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_STATUS,
13017 + cmd_flags,
13018 + token);
13019 + cmd_params = (struct dpseci_cmd_irq_status *)cmd.params;
13020 + cmd_params->status = cpu_to_le32(*status);
13021 + cmd_params->irq_index = irq_index;
13022 + err = mc_send_command(mc_io, &cmd);
13023 + if (err)
13024 + return err;
13025 +
13026 + *status = le32_to_cpu(cmd_params->status);
13027 +
13028 + return 0;
13029 +}
13030 +
13031 +/**
13032 + * dpseci_clear_irq_status() - Clear a pending interrupt's status
13033 + * @mc_io: Pointer to MC portal's I/O object
13034 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13035 + * @token: Token of DPSECI object
13036 + * @irq_index: The interrupt index to configure
13037 + * @status: bits to clear (W1C) - one bit per cause:
13038 + * 0 = don't change
13039 + * 1 = clear status bit
13040 + *
13041 + * Return: '0' on success, error code otherwise
13042 + */
13043 +int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13044 + u8 irq_index, u32 status)
13045 +{
13046 + struct fsl_mc_command cmd = { 0 };
13047 + struct dpseci_cmd_irq_status *cmd_params;
13048 +
13049 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLEAR_IRQ_STATUS,
13050 + cmd_flags,
13051 + token);
13052 + cmd_params = (struct dpseci_cmd_irq_status *)cmd.params;
13053 + cmd_params->status = cpu_to_le32(status);
13054 + cmd_params->irq_index = irq_index;
13055 +
13056 + return mc_send_command(mc_io, &cmd);
13057 +}
13058 +
13059 +/**
13060 + * dpseci_get_attributes() - Retrieve DPSECI attributes
13061 + * @mc_io: Pointer to MC portal's I/O object
13062 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13063 + * @token: Token of DPSECI object
13064 + * @attr: Returned object's attributes
13065 + *
13066 + * Return: '0' on success, error code otherwise
13067 + */
13068 +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13069 + struct dpseci_attr *attr)
13070 +{
13071 + struct fsl_mc_command cmd = { 0 };
13072 + struct dpseci_rsp_get_attributes *rsp_params;
13073 + int err;
13074 +
13075 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR,
13076 + cmd_flags,
13077 + token);
13078 + err = mc_send_command(mc_io, &cmd);
13079 + if (err)
13080 + return err;
13081 +
13082 + rsp_params = (struct dpseci_rsp_get_attributes *)cmd.params;
13083 + attr->id = le32_to_cpu(rsp_params->id);
13084 + attr->num_tx_queues = rsp_params->num_tx_queues;
13085 + attr->num_rx_queues = rsp_params->num_rx_queues;
13086 + attr->options = le32_to_cpu(rsp_params->options);
13087 +
13088 + return 0;
13089 +}
13090 +
13091 +/**
13092 + * dpseci_set_rx_queue() - Set Rx queue configuration
13093 + * @mc_io: Pointer to MC portal's I/O object
13094 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13095 + * @token: Token of DPSECI object
13096 + * @queue: Select the queue relative to number of priorities configured at
13097 + * DPSECI creation; use DPSECI_ALL_QUEUES to configure all
13098 + * Rx queues identically.
13099 + * @cfg: Rx queue configuration
13100 + *
13101 + * Return: '0' on success, error code otherwise
13102 + */
13103 +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13104 + u8 queue, const struct dpseci_rx_queue_cfg *cfg)
13105 +{
13106 + struct fsl_mc_command cmd = { 0 };
13107 + struct dpseci_cmd_queue *cmd_params;
13108 +
13109 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE,
13110 + cmd_flags,
13111 + token);
13112 + cmd_params = (struct dpseci_cmd_queue *)cmd.params;
13113 + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
13114 + cmd_params->priority = cfg->dest_cfg.priority;
13115 + cmd_params->queue = queue;
13116 + dpseci_set_field(cmd_params->dest_type, DEST_TYPE,
13117 + cfg->dest_cfg.dest_type);
13118 + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
13119 + cmd_params->options = cpu_to_le32(cfg->options);
13120 + dpseci_set_field(cmd_params->order_preservation_en, ORDER_PRESERVATION,
13121 + cfg->order_preservation_en);
13122 +
13123 + return mc_send_command(mc_io, &cmd);
13124 +}
13125 +
13126 +/**
13127 + * dpseci_get_rx_queue() - Retrieve Rx queue attributes
13128 + * @mc_io: Pointer to MC portal's I/O object
13129 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13130 + * @token: Token of DPSECI object
13131 + * @queue: Select the queue relative to number of priorities configured at
13132 + * DPSECI creation
13133 + * @attr: Returned Rx queue attributes
13134 + *
13135 + * Return: '0' on success, error code otherwise
13136 + */
13137 +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13138 + u8 queue, struct dpseci_rx_queue_attr *attr)
13139 +{
13140 + struct fsl_mc_command cmd = { 0 };
13141 + struct dpseci_cmd_queue *cmd_params;
13142 + int err;
13143 +
13144 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE,
13145 + cmd_flags,
13146 + token);
13147 + cmd_params = (struct dpseci_cmd_queue *)cmd.params;
13148 + cmd_params->queue = queue;
13149 + err = mc_send_command(mc_io, &cmd);
13150 + if (err)
13151 + return err;
13152 +
13153 + attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id);
13154 + attr->dest_cfg.priority = cmd_params->priority;
13155 + attr->dest_cfg.dest_type = dpseci_get_field(cmd_params->dest_type,
13156 + DEST_TYPE);
13157 + attr->user_ctx = le64_to_cpu(cmd_params->user_ctx);
13158 + attr->fqid = le32_to_cpu(cmd_params->fqid);
13159 + attr->order_preservation_en =
13160 + dpseci_get_field(cmd_params->order_preservation_en,
13161 + ORDER_PRESERVATION);
13162 +
13163 + return 0;
13164 +}
13165 +
13166 +/**
13167 + * dpseci_get_tx_queue() - Retrieve Tx queue attributes
13168 + * @mc_io: Pointer to MC portal's I/O object
13169 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13170 + * @token: Token of DPSECI object
13171 + * @queue: Select the queue relative to number of priorities configured at
13172 + * DPSECI creation
13173 + * @attr: Returned Tx queue attributes
13174 + *
13175 + * Return: '0' on success, error code otherwise
13176 + */
13177 +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13178 + u8 queue, struct dpseci_tx_queue_attr *attr)
13179 +{
13180 + struct fsl_mc_command cmd = { 0 };
13181 + struct dpseci_cmd_queue *cmd_params;
13182 + struct dpseci_rsp_get_tx_queue *rsp_params;
13183 + int err;
13184 +
13185 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE,
13186 + cmd_flags,
13187 + token);
13188 + cmd_params = (struct dpseci_cmd_queue *)cmd.params;
13189 + cmd_params->queue = queue;
13190 + err = mc_send_command(mc_io, &cmd);
13191 + if (err)
13192 + return err;
13193 +
13194 + rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params;
13195 + attr->fqid = le32_to_cpu(rsp_params->fqid);
13196 + attr->priority = rsp_params->priority;
13197 +
13198 + return 0;
13199 +}
13200 +
13201 +/**
13202 + * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes
13203 + * @mc_io: Pointer to MC portal's I/O object
13204 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13205 + * @token: Token of DPSECI object
13206 + * @attr: Returned SEC attributes
13207 + *
13208 + * Return: '0' on success, error code otherwise
13209 + */
13210 +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13211 + struct dpseci_sec_attr *attr)
13212 +{
13213 + struct fsl_mc_command cmd = { 0 };
13214 + struct dpseci_rsp_get_sec_attr *rsp_params;
13215 + int err;
13216 +
13217 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR,
13218 + cmd_flags,
13219 + token);
13220 + err = mc_send_command(mc_io, &cmd);
13221 + if (err)
13222 + return err;
13223 +
13224 + rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params;
13225 + attr->ip_id = le16_to_cpu(rsp_params->ip_id);
13226 + attr->major_rev = rsp_params->major_rev;
13227 + attr->minor_rev = rsp_params->minor_rev;
13228 + attr->era = rsp_params->era;
13229 + attr->deco_num = rsp_params->deco_num;
13230 + attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num;
13231 + attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num;
13232 + attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num;
13233 + attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num;
13234 + attr->crc_acc_num = rsp_params->crc_acc_num;
13235 + attr->pk_acc_num = rsp_params->pk_acc_num;
13236 + attr->kasumi_acc_num = rsp_params->kasumi_acc_num;
13237 + attr->rng_acc_num = rsp_params->rng_acc_num;
13238 + attr->md_acc_num = rsp_params->md_acc_num;
13239 + attr->arc4_acc_num = rsp_params->arc4_acc_num;
13240 + attr->des_acc_num = rsp_params->des_acc_num;
13241 + attr->aes_acc_num = rsp_params->aes_acc_num;
13242 + attr->ccha_acc_num = rsp_params->ccha_acc_num;
13243 + attr->ptha_acc_num = rsp_params->ptha_acc_num;
13244 +
13245 + return 0;
13246 +}
13247 +
13248 +/**
13249 + * dpseci_get_sec_counters() - Retrieve SEC accelerator counters
13250 + * @mc_io: Pointer to MC portal's I/O object
13251 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13252 + * @token: Token of DPSECI object
13253 + * @counters: Returned SEC counters
13254 + *
13255 + * Return: '0' on success, error code otherwise
13256 + */
13257 +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13258 + struct dpseci_sec_counters *counters)
13259 +{
13260 + struct fsl_mc_command cmd = { 0 };
13261 + struct dpseci_rsp_get_sec_counters *rsp_params;
13262 + int err;
13263 +
13264 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS,
13265 + cmd_flags,
13266 + token);
13267 + err = mc_send_command(mc_io, &cmd);
13268 + if (err)
13269 + return err;
13270 +
13271 + rsp_params = (struct dpseci_rsp_get_sec_counters *)cmd.params;
13272 + counters->dequeued_requests =
13273 + le64_to_cpu(rsp_params->dequeued_requests);
13274 + counters->ob_enc_requests = le64_to_cpu(rsp_params->ob_enc_requests);
13275 + counters->ib_dec_requests = le64_to_cpu(rsp_params->ib_dec_requests);
13276 + counters->ob_enc_bytes = le64_to_cpu(rsp_params->ob_enc_bytes);
13277 + counters->ob_prot_bytes = le64_to_cpu(rsp_params->ob_prot_bytes);
13278 + counters->ib_dec_bytes = le64_to_cpu(rsp_params->ib_dec_bytes);
13279 + counters->ib_valid_bytes = le64_to_cpu(rsp_params->ib_valid_bytes);
13280 +
13281 + return 0;
13282 +}
13283 +
13284 +/**
13285 + * dpseci_get_api_version() - Get Data Path SEC Interface API version
13286 + * @mc_io: Pointer to MC portal's I/O object
13287 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13288 + * @major_ver: Major version of data path sec API
13289 + * @minor_ver: Minor version of data path sec API
13290 + *
13291 + * Return: '0' on success, error code otherwise
13292 + */
13293 +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags,
13294 + u16 *major_ver, u16 *minor_ver)
13295 +{
13296 + struct fsl_mc_command cmd = { 0 };
13297 + struct dpseci_rsp_get_api_version *rsp_params;
13298 + int err;
13299 +
13300 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION,
13301 + cmd_flags, 0);
13302 + err = mc_send_command(mc_io, &cmd);
13303 + if (err)
13304 + return err;
13305 +
13306 + rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params;
13307 + *major_ver = le16_to_cpu(rsp_params->major);
13308 + *minor_ver = le16_to_cpu(rsp_params->minor);
13309 +
13310 + return 0;
13311 +}
13312 +
13313 +/**
13314 + * dpseci_set_opr() - Set Order Restoration configuration
13315 + * @mc_io: Pointer to MC portal's I/O object
13316 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13317 + * @token: Token of DPSECI object
13318 + * @index: The queue index
13319 + * @options: Configuration mode options; can be OPR_OPT_CREATE or
13320 + * OPR_OPT_RETIRE
13321 + * @cfg: Configuration options for the OPR
13322 + *
13323 + * Return: '0' on success, error code otherwise
13324 + */
13325 +int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index,
13326 + u8 options, struct opr_cfg *cfg)
13327 +{
13328 + struct fsl_mc_command cmd = { 0 };
13329 + struct dpseci_cmd_opr *cmd_params;
13330 +
13331 + cmd.header = mc_encode_cmd_header(
13332 + DPSECI_CMDID_SET_OPR,
13333 + cmd_flags,
13334 + token);
13335 + cmd_params = (struct dpseci_cmd_opr *)cmd.params;
13336 + cmd_params->index = index;
13337 + cmd_params->options = options;
13338 + cmd_params->oloe = cfg->oloe;
13339 + cmd_params->oeane = cfg->oeane;
13340 + cmd_params->olws = cfg->olws;
13341 + cmd_params->oa = cfg->oa;
13342 + cmd_params->oprrws = cfg->oprrws;
13343 +
13344 + return mc_send_command(mc_io, &cmd);
13345 +}
13346 +
13347 +/**
13348 + * dpseci_get_opr() - Retrieve Order Restoration config and query
13349 + * @mc_io: Pointer to MC portal's I/O object
13350 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13351 + * @token: Token of DPSECI object
13352 + * @index: The queue index
13353 + * @cfg: Returned OPR configuration
13354 + * @qry: Returned OPR query
13355 + *
13356 + * Return: '0' on success, error code otherwise
13357 + */
13358 +int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index,
13359 + struct opr_cfg *cfg, struct opr_qry *qry)
13360 +{
13361 + struct fsl_mc_command cmd = { 0 };
13362 + struct dpseci_cmd_opr *cmd_params;
13363 + struct dpseci_rsp_get_opr *rsp_params;
13364 + int err;
13365 +
13366 + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_OPR,
13367 + cmd_flags,
13368 + token);
13369 + cmd_params = (struct dpseci_cmd_opr *)cmd.params;
13370 + cmd_params->index = index;
13371 + err = mc_send_command(mc_io, &cmd);
13372 + if (err)
13373 + return err;
13374 +
13375 + rsp_params = (struct dpseci_rsp_get_opr *)cmd.params;
13376 + qry->rip = dpseci_get_field(rsp_params->flags, OPR_RIP);
13377 + qry->enable = dpseci_get_field(rsp_params->flags, OPR_ENABLE);
13378 + cfg->oloe = rsp_params->oloe;
13379 + cfg->oeane = rsp_params->oeane;
13380 + cfg->olws = rsp_params->olws;
13381 + cfg->oa = rsp_params->oa;
13382 + cfg->oprrws = rsp_params->oprrws;
13383 + qry->nesn = le16_to_cpu(rsp_params->nesn);
13384 + qry->ndsn = le16_to_cpu(rsp_params->ndsn);
13385 + qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq);
13386 + qry->tseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, OPR_TSEQ_NLIS);
13387 + qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq);
13388 + qry->hseq_nlis = dpseci_get_field(rsp_params->hseq_nlis, OPR_HSEQ_NLIS);
13389 + qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr);
13390 + qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr);
13391 + qry->opr_vid = le16_to_cpu(rsp_params->opr_vid);
13392 + qry->opr_id = le16_to_cpu(rsp_params->opr_id);
13393 +
13394 + return 0;
13395 +}
13396 +
13397 +/**
13398 + * dpseci_set_congestion_notification() - Set congestion group
13399 + * notification configuration
13400 + * @mc_io: Pointer to MC portal's I/O object
13401 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13402 + * @token: Token of DPSECI object
13403 + * @cfg: congestion notification configuration
13404 + *
13405 + * Return: '0' on success, error code otherwise
13406 + */
13407 +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
13408 + u16 token, const struct dpseci_congestion_notification_cfg *cfg)
13409 +{
13410 + struct fsl_mc_command cmd = { 0 };
13411 + struct dpseci_cmd_congestion_notification *cmd_params;
13412 +
13413 + cmd.header = mc_encode_cmd_header(
13414 + DPSECI_CMDID_SET_CONGESTION_NOTIFICATION,
13415 + cmd_flags,
13416 + token);
13417 + cmd_params = (struct dpseci_cmd_congestion_notification *)cmd.params;
13418 + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
13419 + cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode);
13420 + cmd_params->priority = cfg->dest_cfg.priority;
13421 + dpseci_set_field(cmd_params->options, CGN_DEST_TYPE,
13422 + cfg->dest_cfg.dest_type);
13423 + dpseci_set_field(cmd_params->options, CGN_UNITS, cfg->units);
13424 + cmd_params->message_iova = cpu_to_le64(cfg->message_iova);
13425 + cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx);
13426 + cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry);
13427 + cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit);
13428 +
13429 + return mc_send_command(mc_io, &cmd);
13430 +}
13431 +
13432 +/**
13433 + * dpseci_get_congestion_notification() - Get congestion group notification
13434 + * configuration
13435 + * @mc_io: Pointer to MC portal's I/O object
13436 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
13437 + * @token: Token of DPSECI object
13438 + * @cfg: congestion notification configuration
13439 + *
13440 + * Return: '0' on success, error code otherwise
13441 + */
13442 +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
13443 + u16 token, struct dpseci_congestion_notification_cfg *cfg)
13444 +{
13445 + struct fsl_mc_command cmd = { 0 };
13446 + struct dpseci_cmd_congestion_notification *rsp_params;
13447 + int err;
13448 +
13449 + cmd.header = mc_encode_cmd_header(
13450 + DPSECI_CMDID_GET_CONGESTION_NOTIFICATION,
13451 + cmd_flags,
13452 + token);
13453 + err = mc_send_command(mc_io, &cmd);
13454 + if (err)
13455 + return err;
13456 +
13457 + rsp_params = (struct dpseci_cmd_congestion_notification *)cmd.params;
13458 + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id);
13459 + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode);
13460 + cfg->dest_cfg.priority = rsp_params->priority;
13461 + cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->options,
13462 + CGN_DEST_TYPE);
13463 + cfg->units = dpseci_get_field(rsp_params->options, CGN_UNITS);
13464 + cfg->message_iova = le64_to_cpu(rsp_params->message_iova);
13465 + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx);
13466 + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry);
13467 + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit);
13468 +
13469 + return 0;
13470 +}
13471 --- /dev/null
13472 +++ b/drivers/crypto/caam/dpseci.h
13473 @@ -0,0 +1,433 @@
13474 +/*
13475 + * Copyright 2013-2016 Freescale Semiconductor Inc.
13476 + * Copyright 2017 NXP
13477 + *
13478 + * Redistribution and use in source and binary forms, with or without
13479 + * modification, are permitted provided that the following conditions are met:
13480 + * * Redistributions of source code must retain the above copyright
13481 + * notice, this list of conditions and the following disclaimer.
13482 + * * Redistributions in binary form must reproduce the above copyright
13483 + * notice, this list of conditions and the following disclaimer in the
13484 + * documentation and/or other materials provided with the distribution.
13485 + * * Neither the names of the above-listed copyright holders nor the
13486 + * names of any contributors may be used to endorse or promote products
13487 + * derived from this software without specific prior written permission.
13488 + *
13489 + *
13490 + * ALTERNATIVELY, this software may be distributed under the terms of the
13491 + * GNU General Public License ("GPL") as published by the Free Software
13492 + * Foundation, either version 2 of that License or (at your option) any
13493 + * later version.
13494 + *
13495 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
13496 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
13497 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
13498 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
13499 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
13500 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
13501 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
13502 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
13503 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
13504 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
13505 + * POSSIBILITY OF SUCH DAMAGE.
13506 + */
13507 +#ifndef _DPSECI_H_
13508 +#define _DPSECI_H_
13509 +
13510 +/*
13511 + * Data Path SEC Interface API
13512 + * Contains initialization APIs and runtime control APIs for DPSECI
13513 + */
13514 +
13515 +struct fsl_mc_io;
13516 +struct opr_cfg;
13517 +struct opr_qry;
13518 +
13519 +/**
13520 + * General DPSECI macros
13521 + */
13522 +
13523 +/**
13524 + * Maximum number of Tx/Rx queues per DPSECI object
13525 + */
13526 +#define DPSECI_MAX_QUEUE_NUM 16
13527 +
13528 +/**
13529 + * All queues considered; see dpseci_set_rx_queue()
13530 + */
13531 +#define DPSECI_ALL_QUEUES (u8)(-1)
13532 +
13533 +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id,
13534 + u16 *token);
13535 +
13536 +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
13537 +
13538 +/**
13539 + * Enable the Congestion Group support
13540 + */
13541 +#define DPSECI_OPT_HAS_CG 0x000020
13542 +
13543 +/**
13544 + * Enable the Order Restoration support
13545 + */
13546 +#define DPSECI_OPT_HAS_OPR 0x000040
13547 +
13548 +/**
13549 + * Order Point Records are shared for the entire DPSECI
13550 + */
13551 +#define DPSECI_OPT_OPR_SHARED 0x000080
13552 +
13553 +/**
13554 + * struct dpseci_cfg - Structure representing DPSECI configuration
13555 + * @options: Any combination of the following options:
13556 + * DPSECI_OPT_HAS_CG
13557 + * DPSECI_OPT_HAS_OPR
13558 + * DPSECI_OPT_OPR_SHARED
13559 + * @num_tx_queues: num of queues towards the SEC
13560 + * @num_rx_queues: num of queues back from the SEC
13561 + * @priorities: Priorities for the SEC hardware processing;
13562 + * each place in the array is the priority of the tx queue
13563 + * towards the SEC;
13564 + * valid priorities are configured with values 1-8;
13565 + */
13566 +struct dpseci_cfg {
13567 + u32 options;
13568 + u8 num_tx_queues;
13569 + u8 num_rx_queues;
13570 + u8 priorities[DPSECI_MAX_QUEUE_NUM];
13571 +};
13572 +
13573 +int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags,
13574 + const struct dpseci_cfg *cfg, u32 *obj_id);
13575 +
13576 +int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags,
13577 + u32 object_id);
13578 +
13579 +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
13580 +
13581 +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
13582 +
13583 +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13584 + int *en);
13585 +
13586 +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
13587 +
13588 +int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13589 + u8 irq_index, u8 *en);
13590 +
13591 +int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13592 + u8 irq_index, u8 en);
13593 +
13594 +int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13595 + u8 irq_index, u32 *mask);
13596 +
13597 +int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13598 + u8 irq_index, u32 mask);
13599 +
13600 +int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13601 + u8 irq_index, u32 *status);
13602 +
13603 +int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13604 + u8 irq_index, u32 status);
13605 +
13606 +/**
13607 + * struct dpseci_attr - Structure representing DPSECI attributes
13608 + * @id: DPSECI object ID
13609 + * @num_tx_queues: number of queues towards the SEC
13610 + * @num_rx_queues: number of queues back from the SEC
13611 + * @options: any combination of the following options:
13612 + * DPSECI_OPT_HAS_CG
13613 + * DPSECI_OPT_HAS_OPR
13614 + * DPSECI_OPT_OPR_SHARED
13615 + */
13616 +struct dpseci_attr {
13617 + int id;
13618 + u8 num_tx_queues;
13619 + u8 num_rx_queues;
13620 + u32 options;
13621 +};
13622 +
13623 +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13624 + struct dpseci_attr *attr);
13625 +
13626 +/**
13627 + * enum dpseci_dest - DPSECI destination types
13628 + * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode
13629 + * and does not generate FQDAN notifications; user is expected to dequeue
13630 + * from the queue based on polling or other user-defined method
13631 + * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
13632 + * notifications to the specified DPIO; user is expected to dequeue from
13633 + * the queue only after notification is received
13634 + * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate
13635 + * FQDAN notifications, but is connected to the specified DPCON object;
13636 + * user is expected to dequeue from the DPCON channel
13637 + */
13638 +enum dpseci_dest {
13639 + DPSECI_DEST_NONE = 0,
13640 + DPSECI_DEST_DPIO,
13641 + DPSECI_DEST_DPCON
13642 +};
13643 +
13644 +/**
13645 + * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters
13646 + * @dest_type: Destination type
13647 + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
13648 + * @priority: Priority selection within the DPIO or DPCON channel; valid values
13649 + * are 0-1 or 0-7, depending on the number of priorities in that channel;
13650 + * not relevant for 'DPSECI_DEST_NONE' option
13651 + */
13652 +struct dpseci_dest_cfg {
13653 + enum dpseci_dest dest_type;
13654 + int dest_id;
13655 + u8 priority;
13656 +};
13657 +
13658 +/**
13659 + * DPSECI queue modification options
13660 + */
13661 +
13662 +/**
13663 + * Select to modify the user's context associated with the queue
13664 + */
13665 +#define DPSECI_QUEUE_OPT_USER_CTX 0x00000001
13666 +
13667 +/**
13668 + * Select to modify the queue's destination
13669 + */
13670 +#define DPSECI_QUEUE_OPT_DEST 0x00000002
13671 +
13672 +/**
13673 + * Select to modify the queue's order preservation
13674 + */
13675 +#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION 0x00000004
13676 +
13677 +/**
13678 + * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration
13679 + * @options: Flags representing the suggested modifications to the queue;
13680 + * Use any combination of 'DPSECI_QUEUE_OPT_<X>' flags
13681 + * @order_preservation_en: order preservation configuration for the rx queue
13682 + * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options'
13683 + * @user_ctx: User context value provided in the frame descriptor of each
13684 + * dequeued frame; valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained
13685 + * in 'options'
13686 + * @dest_cfg: Queue destination parameters; valid only if
13687 + * 'DPSECI_QUEUE_OPT_DEST' is contained in 'options'
13688 + */
13689 +struct dpseci_rx_queue_cfg {
13690 + u32 options;
13691 + int order_preservation_en;
13692 + u64 user_ctx;
13693 + struct dpseci_dest_cfg dest_cfg;
13694 +};
13695 +
13696 +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13697 + u8 queue, const struct dpseci_rx_queue_cfg *cfg);
13698 +
13699 +/**
13700 + * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues
13701 + * @user_ctx: User context value provided in the frame descriptor of each
13702 + * dequeued frame
13703 + * @order_preservation_en: Status of the order preservation configuration on the
13704 + * queue
13705 + * @dest_cfg: Queue destination configuration
13706 + * @fqid: Virtual FQID value to be used for dequeue operations
13707 + */
13708 +struct dpseci_rx_queue_attr {
13709 + u64 user_ctx;
13710 + int order_preservation_en;
13711 + struct dpseci_dest_cfg dest_cfg;
13712 + u32 fqid;
13713 +};
13714 +
13715 +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13716 + u8 queue, struct dpseci_rx_queue_attr *attr);
13717 +
13718 +/**
13719 + * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues
13720 + * @fqid: Virtual FQID to be used for sending frames to SEC hardware
13721 + * @priority: SEC hardware processing priority for the queue
13722 + */
13723 +struct dpseci_tx_queue_attr {
13724 + u32 fqid;
13725 + u8 priority;
13726 +};
13727 +
13728 +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13729 + u8 queue, struct dpseci_tx_queue_attr *attr);
13730 +
13731 +/**
13732 + * struct dpseci_sec_attr - Structure representing attributes of the SEC
13733 + * hardware accelerator
13734 + * @ip_id: ID for SEC
13735 + * @major_rev: Major revision number for SEC
13736 + * @minor_rev: Minor revision number for SEC
13737 + * @era: SEC Era
13738 + * @deco_num: The number of copies of the DECO that are implemented in this
13739 + * version of SEC
13740 + * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented in this
13741 + * version of SEC
13742 + * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented in this
13743 + * version of SEC
13744 + * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are
13745 + * implemented in this version of SEC
13746 + * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are
13747 + * implemented in this version of SEC
13748 + * @crc_acc_num: The number of copies of the CRC module that are implemented in
13749 + * this version of SEC
13750 + * @pk_acc_num: The number of copies of the Public Key module that are
13751 + * implemented in this version of SEC
13752 + * @kasumi_acc_num: The number of copies of the Kasumi module that are
13753 + * implemented in this version of SEC
13754 + * @rng_acc_num: The number of copies of the Random Number Generator that are
13755 + * implemented in this version of SEC
13756 + * @md_acc_num: The number of copies of the MDHA (Hashing module) that are
13757 + * implemented in this version of SEC
13758 + * @arc4_acc_num: The number of copies of the ARC4 module that are implemented
13759 + * in this version of SEC
13760 + * @des_acc_num: The number of copies of the DES module that are implemented in
13761 + * this version of SEC
13762 + * @aes_acc_num: The number of copies of the AES module that are implemented in
13763 + * this version of SEC
13764 + * @ccha_acc_num: The number of copies of the ChaCha20 module that are
13765 + * implemented in this version of SEC.
13766 + * @ptha_acc_num: The number of copies of the Poly1305 module that are
13767 + * implemented in this version of SEC.
13768 + **/
13769 +struct dpseci_sec_attr {
13770 + u16 ip_id;
13771 + u8 major_rev;
13772 + u8 minor_rev;
13773 + u8 era;
13774 + u8 deco_num;
13775 + u8 zuc_auth_acc_num;
13776 + u8 zuc_enc_acc_num;
13777 + u8 snow_f8_acc_num;
13778 + u8 snow_f9_acc_num;
13779 + u8 crc_acc_num;
13780 + u8 pk_acc_num;
13781 + u8 kasumi_acc_num;
13782 + u8 rng_acc_num;
13783 + u8 md_acc_num;
13784 + u8 arc4_acc_num;
13785 + u8 des_acc_num;
13786 + u8 aes_acc_num;
13787 + u8 ccha_acc_num;
13788 + u8 ptha_acc_num;
13789 +};
13790 +
13791 +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13792 + struct dpseci_sec_attr *attr);
13793 +
13794 +/**
13795 + * struct dpseci_sec_counters - Structure representing global SEC counters and
13796 + * not per dpseci counters
13797 + * @dequeued_requests: Number of Requests Dequeued
13798 + * @ob_enc_requests: Number of Outbound Encrypt Requests
13799 + * @ib_dec_requests: Number of Inbound Decrypt Requests
13800 + * @ob_enc_bytes: Number of Outbound Bytes Encrypted
13801 + * @ob_prot_bytes: Number of Outbound Bytes Protected
13802 + * @ib_dec_bytes: Number of Inbound Bytes Decrypted
13803 + * @ib_valid_bytes: Number of Inbound Bytes Validated
13804 + */
13805 +struct dpseci_sec_counters {
13806 + u64 dequeued_requests;
13807 + u64 ob_enc_requests;
13808 + u64 ib_dec_requests;
13809 + u64 ob_enc_bytes;
13810 + u64 ob_prot_bytes;
13811 + u64 ib_dec_bytes;
13812 + u64 ib_valid_bytes;
13813 +};
13814 +
13815 +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
13816 + struct dpseci_sec_counters *counters);
13817 +
13818 +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags,
13819 + u16 *major_ver, u16 *minor_ver);
13820 +
13821 +int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index,
13822 + u8 options, struct opr_cfg *cfg);
13823 +
13824 +int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index,
13825 + struct opr_cfg *cfg, struct opr_qry *qry);
13826 +
13827 +/**
13828 + * enum dpseci_congestion_unit - DPSECI congestion units
13829 + * @DPSECI_CONGESTION_UNIT_BYTES: bytes units
13830 + * @DPSECI_CONGESTION_UNIT_FRAMES: frames units
13831 + */
13832 +enum dpseci_congestion_unit {
13833 + DPSECI_CONGESTION_UNIT_BYTES = 0,
13834 + DPSECI_CONGESTION_UNIT_FRAMES
13835 +};
13836 +
13837 +/**
13838 + * CSCN message is written to message_iova once entering a
13839 + * congestion state (see 'threshold_entry')
13840 + */
13841 +#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001
13842 +
13843 +/**
13844 + * CSCN message is written to message_iova once exiting a
13845 + * congestion state (see 'threshold_exit')
13846 + */
13847 +#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002
13848 +
13849 +/**
13850 + * CSCN write will attempt to allocate into a cache (coherent write);
13851 + * valid only if 'DPSECI_CGN_MODE_WRITE_MEM_<X>' is selected
13852 + */
13853 +#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004
13854 +
13855 +/**
13856 + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to
13857 + * DPIO/DPCON's WQ channel once entering a congestion state
13858 + * (see 'threshold_entry')
13859 + */
13860 +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008
13861 +
13862 +/**
13863 + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to
13864 + * DPIO/DPCON's WQ channel once exiting a congestion state
13865 + * (see 'threshold_exit')
13866 + */
13867 +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010
13868 +
13869 +/**
13870 + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' when the CSCN is written
13871 + * to the sw-portal's DQRR, the DQRI interrupt is asserted immediately
13872 + * (if enabled)
13873 + */
13874 +#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020
13875 +
13876 +/**
13877 + * struct dpseci_congestion_notification_cfg - congestion notification
13878 + * configuration
13879 + * @units: units type
13880 + * @threshold_entry: above this threshold we enter a congestion state.
13881 + * set it to '0' to disable it
13882 + * @threshold_exit: below this threshold we exit the congestion state.
13883 + * @message_ctx: The context that will be part of the CSCN message
13884 + * @message_iova: I/O virtual address (must be in DMA-able memory),
13885 + * must be 16B aligned;
13886 + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel
13887 + * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_<X>'
13888 + * values
13889 + */
13890 +struct dpseci_congestion_notification_cfg {
13891 + enum dpseci_congestion_unit units;
13892 + u32 threshold_entry;
13893 + u32 threshold_exit;
13894 + u64 message_ctx;
13895 + u64 message_iova;
13896 + struct dpseci_dest_cfg dest_cfg;
13897 + u16 notification_mode;
13898 +};
13899 +
13900 +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
13901 + u16 token, const struct dpseci_congestion_notification_cfg *cfg);
13902 +
13903 +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags,
13904 + u16 token, struct dpseci_congestion_notification_cfg *cfg);
13905 +
13906 +#endif /* _DPSECI_H_ */
13907 --- /dev/null
13908 +++ b/drivers/crypto/caam/dpseci_cmd.h
13909 @@ -0,0 +1,287 @@
13910 +/*
13911 + * Copyright 2013-2016 Freescale Semiconductor Inc.
13912 + * Copyright 2017 NXP
13913 + *
13914 + * Redistribution and use in source and binary forms, with or without
13915 + * modification, are permitted provided that the following conditions are met:
13916 + * * Redistributions of source code must retain the above copyright
13917 + * notice, this list of conditions and the following disclaimer.
13918 + * * Redistributions in binary form must reproduce the above copyright
13919 + * notice, this list of conditions and the following disclaimer in the
13920 + * documentation and/or other materials provided with the distribution.
13921 + * * Neither the names of the above-listed copyright holders nor the
13922 + * names of any contributors may be used to endorse or promote products
13923 + * derived from this software without specific prior written permission.
13924 + *
13925 + *
13926 + * ALTERNATIVELY, this software may be distributed under the terms of the
13927 + * GNU General Public License ("GPL") as published by the Free Software
13928 + * Foundation, either version 2 of that License or (at your option) any
13929 + * later version.
13930 + *
13931 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
13932 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
13933 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
13934 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
13935 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
13936 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
13937 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
13938 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
13939 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
13940 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
13941 + * POSSIBILITY OF SUCH DAMAGE.
13942 + */
13943 +
13944 +#ifndef _DPSECI_CMD_H_
13945 +#define _DPSECI_CMD_H_
13946 +
13947 +/* DPSECI Version */
13948 +#define DPSECI_VER_MAJOR 5
13949 +#define DPSECI_VER_MINOR 3
13950 +
13951 +#define DPSECI_VER(maj, min) (((maj) << 16) | (min))
13952 +#define DPSECI_VERSION DPSECI_VER(DPSECI_VER_MAJOR, DPSECI_VER_MINOR)
13953 +
13954 +/* Command versioning */
13955 +#define DPSECI_CMD_BASE_VERSION 1
13956 +#define DPSECI_CMD_BASE_VERSION_V2 2
13957 +#define DPSECI_CMD_BASE_VERSION_V3 3
13958 +#define DPSECI_CMD_ID_OFFSET 4
13959 +
13960 +#define DPSECI_CMD_V1(id) (((id) << DPSECI_CMD_ID_OFFSET) | \
13961 + DPSECI_CMD_BASE_VERSION)
13962 +
13963 +#define DPSECI_CMD_V2(id) (((id) << DPSECI_CMD_ID_OFFSET) | \
13964 + DPSECI_CMD_BASE_VERSION_V2)
13965 +
13966 +#define DPSECI_CMD_V3(id) (((id) << DPSECI_CMD_ID_OFFSET) | \
13967 + DPSECI_CMD_BASE_VERSION_V3)
13968 +
13969 +/* Command IDs */
13970 +#define DPSECI_CMDID_CLOSE DPSECI_CMD_V1(0x800)
13971 +#define DPSECI_CMDID_OPEN DPSECI_CMD_V1(0x809)
13972 +#define DPSECI_CMDID_CREATE DPSECI_CMD_V3(0x909)
13973 +#define DPSECI_CMDID_DESTROY DPSECI_CMD_V1(0x989)
13974 +#define DPSECI_CMDID_GET_API_VERSION DPSECI_CMD_V1(0xa09)
13975 +
13976 +#define DPSECI_CMDID_ENABLE DPSECI_CMD_V1(0x002)
13977 +#define DPSECI_CMDID_DISABLE DPSECI_CMD_V1(0x003)
13978 +#define DPSECI_CMDID_GET_ATTR DPSECI_CMD_V1(0x004)
13979 +#define DPSECI_CMDID_RESET DPSECI_CMD_V1(0x005)
13980 +#define DPSECI_CMDID_IS_ENABLED DPSECI_CMD_V1(0x006)
13981 +
13982 +#define DPSECI_CMDID_SET_IRQ_ENABLE DPSECI_CMD_V1(0x012)
13983 +#define DPSECI_CMDID_GET_IRQ_ENABLE DPSECI_CMD_V1(0x013)
13984 +#define DPSECI_CMDID_SET_IRQ_MASK DPSECI_CMD_V1(0x014)
13985 +#define DPSECI_CMDID_GET_IRQ_MASK DPSECI_CMD_V1(0x015)
13986 +#define DPSECI_CMDID_GET_IRQ_STATUS DPSECI_CMD_V1(0x016)
13987 +#define DPSECI_CMDID_CLEAR_IRQ_STATUS DPSECI_CMD_V1(0x017)
13988 +
13989 +#define DPSECI_CMDID_SET_RX_QUEUE DPSECI_CMD_V1(0x194)
13990 +#define DPSECI_CMDID_GET_RX_QUEUE DPSECI_CMD_V1(0x196)
13991 +#define DPSECI_CMDID_GET_TX_QUEUE DPSECI_CMD_V1(0x197)
13992 +#define DPSECI_CMDID_GET_SEC_ATTR DPSECI_CMD_V2(0x198)
13993 +#define DPSECI_CMDID_GET_SEC_COUNTERS DPSECI_CMD_V1(0x199)
13994 +#define DPSECI_CMDID_SET_OPR DPSECI_CMD_V1(0x19A)
13995 +#define DPSECI_CMDID_GET_OPR DPSECI_CMD_V1(0x19B)
13996 +#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x170)
13997 +#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x171)
13998 +
13999 +/* Macros for accessing command fields smaller than 1 byte */
14000 +#define DPSECI_MASK(field) \
14001 + GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \
14002 + DPSECI_##field##_SHIFT)
14003 +
14004 +#define dpseci_set_field(var, field, val) \
14005 + ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field)))
14006 +
14007 +#define dpseci_get_field(var, field) \
14008 + (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT)
14009 +
14010 +struct dpseci_cmd_open {
14011 + __le32 dpseci_id;
14012 +};
14013 +
14014 +struct dpseci_cmd_create {
14015 + u8 priorities[8];
14016 + u8 num_tx_queues;
14017 + u8 num_rx_queues;
14018 + u8 pad0[6];
14019 + __le32 options;
14020 + __le32 pad1;
14021 + u8 priorities2[8];
14022 +};
14023 +
14024 +struct dpseci_cmd_destroy {
14025 + __le32 object_id;
14026 +};
14027 +
14028 +#define DPSECI_ENABLE_SHIFT 0
14029 +#define DPSECI_ENABLE_SIZE 1
14030 +
14031 +struct dpseci_rsp_is_enabled {
14032 + u8 is_enabled;
14033 +};
14034 +
14035 +struct dpseci_cmd_irq_enable {
14036 + u8 enable_state;
14037 + u8 pad[3];
14038 + u8 irq_index;
14039 +};
14040 +
14041 +struct dpseci_rsp_get_irq_enable {
14042 + u8 enable_state;
14043 +};
14044 +
14045 +struct dpseci_cmd_irq_mask {
14046 + __le32 mask;
14047 + u8 irq_index;
14048 +};
14049 +
14050 +struct dpseci_cmd_irq_status {
14051 + __le32 status;
14052 + u8 irq_index;
14053 +};
14054 +
14055 +struct dpseci_rsp_get_attributes {
14056 + __le32 id;
14057 + __le32 pad0;
14058 + u8 num_tx_queues;
14059 + u8 num_rx_queues;
14060 + u8 pad1[6];
14061 + __le32 options;
14062 +};
14063 +
14064 +#define DPSECI_DEST_TYPE_SHIFT 0
14065 +#define DPSECI_DEST_TYPE_SIZE 4
14066 +
14067 +#define DPSECI_ORDER_PRESERVATION_SHIFT 0
14068 +#define DPSECI_ORDER_PRESERVATION_SIZE 1
14069 +
14070 +struct dpseci_cmd_queue {
14071 + __le32 dest_id;
14072 + u8 priority;
14073 + u8 queue;
14074 + u8 dest_type;
14075 + u8 pad;
14076 + __le64 user_ctx;
14077 + union {
14078 + __le32 options;
14079 + __le32 fqid;
14080 + };
14081 + u8 order_preservation_en;
14082 +};
14083 +
14084 +struct dpseci_rsp_get_tx_queue {
14085 + __le32 pad;
14086 + __le32 fqid;
14087 + u8 priority;
14088 +};
14089 +
14090 +struct dpseci_rsp_get_sec_attr {
14091 + __le16 ip_id;
14092 + u8 major_rev;
14093 + u8 minor_rev;
14094 + u8 era;
14095 + u8 pad0[3];
14096 + u8 deco_num;
14097 + u8 zuc_auth_acc_num;
14098 + u8 zuc_enc_acc_num;
14099 + u8 pad1;
14100 + u8 snow_f8_acc_num;
14101 + u8 snow_f9_acc_num;
14102 + u8 crc_acc_num;
14103 + u8 pad2;
14104 + u8 pk_acc_num;
14105 + u8 kasumi_acc_num;
14106 + u8 rng_acc_num;
14107 + u8 pad3;
14108 + u8 md_acc_num;
14109 + u8 arc4_acc_num;
14110 + u8 des_acc_num;
14111 + u8 aes_acc_num;
14112 + u8 ccha_acc_num;
14113 + u8 ptha_acc_num;
14114 +};
14115 +
14116 +struct dpseci_rsp_get_sec_counters {
14117 + __le64 dequeued_requests;
14118 + __le64 ob_enc_requests;
14119 + __le64 ib_dec_requests;
14120 + __le64 ob_enc_bytes;
14121 + __le64 ob_prot_bytes;
14122 + __le64 ib_dec_bytes;
14123 + __le64 ib_valid_bytes;
14124 +};
14125 +
14126 +struct dpseci_rsp_get_api_version {
14127 + __le16 major;
14128 + __le16 minor;
14129 +};
14130 +
14131 +struct dpseci_cmd_opr {
14132 + __le16 pad;
14133 + u8 index;
14134 + u8 options;
14135 + u8 pad1[7];
14136 + u8 oloe;
14137 + u8 oeane;
14138 + u8 olws;
14139 + u8 oa;
14140 + u8 oprrws;
14141 +};
14142 +
14143 +#define DPSECI_OPR_RIP_SHIFT 0
14144 +#define DPSECI_OPR_RIP_SIZE 1
14145 +#define DPSECI_OPR_ENABLE_SHIFT 1
14146 +#define DPSECI_OPR_ENABLE_SIZE 1
14147 +#define DPSECI_OPR_TSEQ_NLIS_SHIFT 0
14148 +#define DPSECI_OPR_TSEQ_NLIS_SIZE 1
14149 +#define DPSECI_OPR_HSEQ_NLIS_SHIFT 0
14150 +#define DPSECI_OPR_HSEQ_NLIS_SIZE 1
14151 +
14152 +struct dpseci_rsp_get_opr {
14153 + __le64 pad;
14154 + u8 flags;
14155 + u8 pad0[2];
14156 + u8 oloe;
14157 + u8 oeane;
14158 + u8 olws;
14159 + u8 oa;
14160 + u8 oprrws;
14161 + __le16 nesn;
14162 + __le16 pad1;
14163 + __le16 ndsn;
14164 + __le16 pad2;
14165 + __le16 ea_tseq;
14166 + u8 tseq_nlis;
14167 + u8 pad3;
14168 + __le16 ea_hseq;
14169 + u8 hseq_nlis;
14170 + u8 pad4;
14171 + __le16 ea_hptr;
14172 + __le16 pad5;
14173 + __le16 ea_tptr;
14174 + __le16 pad6;
14175 + __le16 opr_vid;
14176 + __le16 pad7;
14177 + __le16 opr_id;
14178 +};
14179 +
14180 +#define DPSECI_CGN_DEST_TYPE_SHIFT 0
14181 +#define DPSECI_CGN_DEST_TYPE_SIZE 4
14182 +#define DPSECI_CGN_UNITS_SHIFT 4
14183 +#define DPSECI_CGN_UNITS_SIZE 2
14184 +
14185 +struct dpseci_cmd_congestion_notification {
14186 + __le32 dest_id;
14187 + __le16 notification_mode;
14188 + u8 priority;
14189 + u8 options;
14190 + __le64 message_iova;
14191 + __le64 message_ctx;
14192 + __le32 threshold_entry;
14193 + __le32 threshold_exit;
14194 +};
14195 +
14196 +#endif /* _DPSECI_CMD_H_ */
14197 --- a/drivers/crypto/caam/error.c
14198 +++ b/drivers/crypto/caam/error.c
14199 @@ -50,6 +50,12 @@ void caam_dump_sg(const char *level, con
14200 #endif /* DEBUG */
14201 EXPORT_SYMBOL(caam_dump_sg);
14202
14203 +bool caam_little_end;
14204 +EXPORT_SYMBOL(caam_little_end);
14205 +
14206 +bool caam_imx;
14207 +EXPORT_SYMBOL(caam_imx);
14208 +
14209 static const struct {
14210 u8 value;
14211 const char *error_text;
14212 @@ -108,6 +114,54 @@ static const struct {
14213 { 0xF1, "3GPP HFN matches or exceeds the Threshold" },
14214 };
14215
14216 +static const struct {
14217 + u8 value;
14218 + const char *error_text;
14219 +} qi_error_list[] = {
14220 + { 0x1F, "Job terminated by FQ or ICID flush" },
14221 + { 0x20, "FD format error"},
14222 + { 0x21, "FD command format error"},
14223 + { 0x23, "FL format error"},
14224 + { 0x25, "CRJD specified in FD, but not enabled in FLC"},
14225 + { 0x30, "Max. buffer size too small"},
14226 + { 0x31, "DHR exceeds max. buffer size (allocate mode, S/G format)"},
14227 + { 0x32, "SGT exceeds max. buffer size (allocate mode, S/G format"},
14228 + { 0x33, "Size over/underflow (allocate mode)"},
14229 + { 0x34, "Size over/underflow (reuse mode)"},
14230 + { 0x35, "Length exceeds max. short length (allocate mode, S/G/ format)"},
14231 + { 0x36, "Memory footprint exceeds max. value (allocate mode, S/G/ format)"},
14232 + { 0x41, "SBC frame format not supported (allocate mode)"},
14233 + { 0x42, "Pool 0 invalid / pool 1 size < pool 0 size (allocate mode)"},
14234 + { 0x43, "Annotation output enabled but ASAR = 0 (allocate mode)"},
14235 + { 0x44, "Unsupported or reserved frame format or SGHR = 1 (reuse mode)"},
14236 + { 0x45, "DHR correction underflow (reuse mode, single buffer format)"},
14237 + { 0x46, "Annotation length exceeds offset (reuse mode)"},
14238 + { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"},
14239 + { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"},
14240 + { 0x4B, "Annotation output enabled but ASA cannote be expanded (frame list)"},
14241 + { 0x51, "Unsupported IF reuse mode"},
14242 + { 0x52, "Unsupported FL use mode"},
14243 + { 0x53, "Unsupported RJD use mode"},
14244 + { 0x54, "Unsupported inline descriptor use mode"},
14245 + { 0xC0, "Table buffer pool 0 depletion"},
14246 + { 0xC1, "Table buffer pool 1 depletion"},
14247 + { 0xC2, "Data buffer pool 0 depletion, no OF allocated"},
14248 + { 0xC3, "Data buffer pool 1 depletion, no OF allocated"},
14249 + { 0xC4, "Data buffer pool 0 depletion, partial OF allocated"},
14250 + { 0xC5, "Data buffer pool 1 depletion, partial OF allocated"},
14251 + { 0xD0, "FLC read error"},
14252 + { 0xD1, "FL read error"},
14253 + { 0xD2, "FL write error"},
14254 + { 0xD3, "OF SGT write error"},
14255 + { 0xD4, "PTA read error"},
14256 + { 0xD5, "PTA write error"},
14257 + { 0xD6, "OF SGT F-bit write error"},
14258 + { 0xD7, "ASA write error"},
14259 + { 0xE1, "FLC[ICR]=0 ICID error"},
14260 + { 0xE2, "FLC[ICR]=1 ICID error"},
14261 + { 0xE4, "source of ICID flush not trusted (BDI = 0)"},
14262 +};
14263 +
14264 static const char * const cha_id_list[] = {
14265 "",
14266 "AES",
14267 @@ -236,6 +290,27 @@ static void report_deco_status(struct de
14268 status, error, idx_str, idx, err_str, err_err_code);
14269 }
14270
14271 +static void report_qi_status(struct device *qidev, const u32 status,
14272 + const char *error)
14273 +{
14274 + u8 err_id = status & JRSTA_QIERR_ERROR_MASK;
14275 + const char *err_str = "unidentified error value 0x";
14276 + char err_err_code[3] = { 0 };
14277 + int i;
14278 +
14279 + for (i = 0; i < ARRAY_SIZE(qi_error_list); i++)
14280 + if (qi_error_list[i].value == err_id)
14281 + break;
14282 +
14283 + if (i != ARRAY_SIZE(qi_error_list) && qi_error_list[i].error_text)
14284 + err_str = qi_error_list[i].error_text;
14285 + else
14286 + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
14287 +
14288 + dev_err(qidev, "%08x: %s: %s%s\n",
14289 + status, error, err_str, err_err_code);
14290 +}
14291 +
14292 static void report_jr_status(struct device *jrdev, const u32 status,
14293 const char *error)
14294 {
14295 @@ -250,7 +325,7 @@ static void report_cond_code_status(stru
14296 status, error, __func__);
14297 }
14298
14299 -void caam_jr_strstatus(struct device *jrdev, u32 status)
14300 +void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
14301 {
14302 static const struct stat_src {
14303 void (*report_ssed)(struct device *jrdev, const u32 status,
14304 @@ -262,7 +337,7 @@ void caam_jr_strstatus(struct device *jr
14305 { report_ccb_status, "CCB" },
14306 { report_jump_status, "Jump" },
14307 { report_deco_status, "DECO" },
14308 - { NULL, "Queue Manager Interface" },
14309 + { report_qi_status, "Queue Manager Interface" },
14310 { report_jr_status, "Job Ring" },
14311 { report_cond_code_status, "Condition Code" },
14312 { NULL, NULL },
14313 @@ -288,4 +363,4 @@ void caam_jr_strstatus(struct device *jr
14314 else
14315 dev_err(jrdev, "%d: unknown error source\n", ssrc);
14316 }
14317 -EXPORT_SYMBOL(caam_jr_strstatus);
14318 +EXPORT_SYMBOL(caam_strstatus);
14319 --- a/drivers/crypto/caam/error.h
14320 +++ b/drivers/crypto/caam/error.h
14321 @@ -8,7 +8,11 @@
14322 #ifndef CAAM_ERROR_H
14323 #define CAAM_ERROR_H
14324 #define CAAM_ERROR_STR_MAX 302
14325 -void caam_jr_strstatus(struct device *jrdev, u32 status);
14326 +
14327 +void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
14328 +
14329 +#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
14330 +#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)
14331
14332 void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
14333 int rowsize, int groupsize, struct scatterlist *sg,
14334 --- a/drivers/crypto/caam/intern.h
14335 +++ b/drivers/crypto/caam/intern.h
14336 @@ -65,10 +65,6 @@ struct caam_drv_private_jr {
14337 * Driver-private storage for a single CAAM block instance
14338 */
14339 struct caam_drv_private {
14340 -#ifdef CONFIG_CAAM_QI
14341 - struct device *qidev;
14342 -#endif
14343 -
14344 /* Physical-presence section */
14345 struct caam_ctrl __iomem *ctrl; /* controller region */
14346 struct caam_deco __iomem *deco; /* DECO/CCB views */
14347 @@ -76,14 +72,21 @@ struct caam_drv_private {
14348 struct caam_queue_if __iomem *qi; /* QI control region */
14349 struct caam_job_ring __iomem *jr[4]; /* JobR's register space */
14350
14351 + struct iommu_domain *domain;
14352 +
14353 /*
14354 * Detected geometry block. Filled in from device tree if powerpc,
14355 * or from register-based version detection code
14356 */
14357 u8 total_jobrs; /* Total Job Rings in device */
14358 u8 qi_present; /* Nonzero if QI present in device */
14359 +#ifdef CONFIG_CAAM_QI
14360 + u8 qi_init; /* Nonzero if QI has been initialized */
14361 +#endif
14362 + u8 mc_en; /* Nonzero if MC f/w is active */
14363 int secvio_irq; /* Security violation interrupt number */
14364 int virt_en; /* Virtualization enabled in CAAM */
14365 + int era; /* CAAM Era (internal HW revision) */
14366
14367 #define RNG4_MAX_HANDLES 2
14368 /* RNG4 block */
14369 @@ -108,8 +111,95 @@ struct caam_drv_private {
14370 #endif
14371 };
14372
14373 -void caam_jr_algapi_init(struct device *dev);
14374 -void caam_jr_algapi_remove(struct device *dev);
14375 +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
14376 +
14377 +int caam_algapi_init(struct device *dev);
14378 +void caam_algapi_exit(void);
14379 +
14380 +#else
14381 +
14382 +static inline int caam_algapi_init(struct device *dev)
14383 +{
14384 + return 0;
14385 +}
14386 +
14387 +static inline void caam_algapi_exit(void)
14388 +{
14389 +}
14390 +
14391 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */
14392 +
14393 +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API
14394 +
14395 +int caam_algapi_hash_init(struct device *dev);
14396 +void caam_algapi_hash_exit(void);
14397 +
14398 +#else
14399 +
14400 +static inline int caam_algapi_hash_init(struct device *dev)
14401 +{
14402 + return 0;
14403 +}
14404 +
14405 +static inline void caam_algapi_hash_exit(void)
14406 +{
14407 +}
14408 +
14409 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */
14410 +
14411 +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API
14412 +
14413 +int caam_pkc_init(struct device *dev);
14414 +void caam_pkc_exit(void);
14415 +
14416 +#else
14417 +
14418 +static inline int caam_pkc_init(struct device *dev)
14419 +{
14420 + return 0;
14421 +}
14422 +
14423 +static inline void caam_pkc_exit(void)
14424 +{
14425 +}
14426 +
14427 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */
14428 +
14429 +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API
14430 +
14431 +int caam_rng_init(struct device *dev);
14432 +void caam_rng_exit(void);
14433 +
14434 +#else
14435 +
14436 +static inline int caam_rng_init(struct device *dev)
14437 +{
14438 + return 0;
14439 +}
14440 +
14441 +static inline void caam_rng_exit(void)
14442 +{
14443 +}
14444 +
14445 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */
14446 +
14447 +#ifdef CONFIG_CAAM_QI
14448 +
14449 +int caam_qi_algapi_init(struct device *dev);
14450 +void caam_qi_algapi_exit(void);
14451 +
14452 +#else
14453 +
14454 +static inline int caam_qi_algapi_init(struct device *dev)
14455 +{
14456 + return 0;
14457 +}
14458 +
14459 +static inline void caam_qi_algapi_exit(void)
14460 +{
14461 +}
14462 +
14463 +#endif /* CONFIG_CAAM_QI */
14464
14465 #ifdef CONFIG_DEBUG_FS
14466 static int caam_debugfs_u64_get(void *data, u64 *val)
14467 --- a/drivers/crypto/caam/jr.c
14468 +++ b/drivers/crypto/caam/jr.c
14469 @@ -23,6 +23,52 @@ struct jr_driver_data {
14470
14471 static struct jr_driver_data driver_data;
14472
14473 +static int jr_driver_probed;
14474 +
14475 +int caam_jr_driver_probed(void)
14476 +{
14477 + return jr_driver_probed;
14478 +}
14479 +EXPORT_SYMBOL(caam_jr_driver_probed);
14480 +
14481 +static DEFINE_MUTEX(algs_lock);
14482 +static unsigned int active_devs;
14483 +
14484 +static void register_algs(struct device *dev)
14485 +{
14486 + mutex_lock(&algs_lock);
14487 +
14488 + if (++active_devs != 1)
14489 + goto algs_unlock;
14490 +
14491 + caam_algapi_init(dev);
14492 + caam_algapi_hash_init(dev);
14493 + caam_pkc_init(dev);
14494 + caam_rng_init(dev);
14495 + caam_qi_algapi_init(dev);
14496 +
14497 +algs_unlock:
14498 + mutex_unlock(&algs_lock);
14499 +}
14500 +
14501 +static void unregister_algs(void)
14502 +{
14503 + mutex_lock(&algs_lock);
14504 +
14505 + if (--active_devs != 0)
14506 + goto algs_unlock;
14507 +
14508 + caam_qi_algapi_exit();
14509 +
14510 + caam_rng_exit();
14511 + caam_pkc_exit();
14512 + caam_algapi_hash_exit();
14513 + caam_algapi_exit();
14514 +
14515 +algs_unlock:
14516 + mutex_unlock(&algs_lock);
14517 +}
14518 +
14519 static int caam_reset_hw_jr(struct device *dev)
14520 {
14521 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
14522 @@ -108,6 +154,9 @@ static int caam_jr_remove(struct platfor
14523 return -EBUSY;
14524 }
14525
14526 + /* Unregister JR-based RNG & crypto algorithms */
14527 + unregister_algs();
14528 +
14529 /* Remove the node from Physical JobR list maintained by driver */
14530 spin_lock(&driver_data.jr_alloc_lock);
14531 list_del(&jrpriv->list_node);
14532 @@ -119,6 +168,8 @@ static int caam_jr_remove(struct platfor
14533 dev_err(jrdev, "Failed to shut down job ring\n");
14534 irq_dispose_mapping(jrpriv->irq);
14535
14536 + jr_driver_probed--;
14537 +
14538 return ret;
14539 }
14540
14541 @@ -282,6 +333,36 @@ struct device *caam_jr_alloc(void)
14542 EXPORT_SYMBOL(caam_jr_alloc);
14543
14544 /**
14545 + * caam_jridx_alloc() - Alloc a specific job ring based on its index.
14546 + *
14547 + * returns : pointer to the newly allocated physical
14548 + * JobR dev can be written to if successful.
14549 + **/
14550 +struct device *caam_jridx_alloc(int idx)
14551 +{
14552 + struct caam_drv_private_jr *jrpriv;
14553 + struct device *dev = ERR_PTR(-ENODEV);
14554 +
14555 + spin_lock(&driver_data.jr_alloc_lock);
14556 +
14557 + if (list_empty(&driver_data.jr_list))
14558 + goto end;
14559 +
14560 + list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
14561 + if (jrpriv->ridx == idx) {
14562 + atomic_inc(&jrpriv->tfm_count);
14563 + dev = jrpriv->dev;
14564 + break;
14565 + }
14566 + }
14567 +
14568 +end:
14569 + spin_unlock(&driver_data.jr_alloc_lock);
14570 + return dev;
14571 +}
14572 +EXPORT_SYMBOL(caam_jridx_alloc);
14573 +
14574 +/**
14575 * caam_jr_free() - Free the Job Ring
14576 * @rdev - points to the dev that identifies the Job ring to
14577 * be released.
14578 @@ -539,6 +620,9 @@ static int caam_jr_probe(struct platform
14579
14580 atomic_set(&jrpriv->tfm_count, 0);
14581
14582 + register_algs(jrdev->parent);
14583 + jr_driver_probed++;
14584 +
14585 return 0;
14586 }
14587
14588 --- a/drivers/crypto/caam/jr.h
14589 +++ b/drivers/crypto/caam/jr.h
14590 @@ -9,7 +9,9 @@
14591 #define JR_H
14592
14593 /* Prototypes for backend-level services exposed to APIs */
14594 +int caam_jr_driver_probed(void);
14595 struct device *caam_jr_alloc(void);
14596 +struct device *caam_jridx_alloc(int idx);
14597 void caam_jr_free(struct device *rdev);
14598 int caam_jr_enqueue(struct device *dev, u32 *desc,
14599 void (*cbk)(struct device *dev, u32 *desc, u32 status,
14600 --- a/drivers/crypto/caam/key_gen.c
14601 +++ b/drivers/crypto/caam/key_gen.c
14602 @@ -11,36 +11,6 @@
14603 #include "desc_constr.h"
14604 #include "key_gen.h"
14605
14606 -/**
14607 - * split_key_len - Compute MDHA split key length for a given algorithm
14608 - * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
14609 - * SHA224, SHA384, SHA512.
14610 - *
14611 - * Return: MDHA split key length
14612 - */
14613 -static inline u32 split_key_len(u32 hash)
14614 -{
14615 - /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
14616 - static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
14617 - u32 idx;
14618 -
14619 - idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
14620 -
14621 - return (u32)(mdpadlen[idx] * 2);
14622 -}
14623 -
14624 -/**
14625 - * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
14626 - * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
14627 - * SHA224, SHA384, SHA512.
14628 - *
14629 - * Return: MDHA split key pad length
14630 - */
14631 -static inline u32 split_key_pad_len(u32 hash)
14632 -{
14633 - return ALIGN(split_key_len(hash), 16);
14634 -}
14635 -
14636 void split_key_done(struct device *dev, u32 *desc, u32 err,
14637 void *context)
14638 {
14639 --- a/drivers/crypto/caam/key_gen.h
14640 +++ b/drivers/crypto/caam/key_gen.h
14641 @@ -6,6 +6,36 @@
14642 *
14643 */
14644
14645 +/**
14646 + * split_key_len - Compute MDHA split key length for a given algorithm
14647 + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
14648 + * SHA224, SHA384, SHA512.
14649 + *
14650 + * Return: MDHA split key length
14651 + */
14652 +static inline u32 split_key_len(u32 hash)
14653 +{
14654 + /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
14655 + static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
14656 + u32 idx;
14657 +
14658 + idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
14659 +
14660 + return (u32)(mdpadlen[idx] * 2);
14661 +}
14662 +
14663 +/**
14664 + * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
14665 + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
14666 + * SHA224, SHA384, SHA512.
14667 + *
14668 + * Return: MDHA split key pad length
14669 + */
14670 +static inline u32 split_key_pad_len(u32 hash)
14671 +{
14672 + return ALIGN(split_key_len(hash), 16);
14673 +}
14674 +
14675 struct split_key_result {
14676 struct completion completion;
14677 int err;
14678 --- a/drivers/crypto/caam/qi.c
14679 +++ b/drivers/crypto/caam/qi.c
14680 @@ -9,7 +9,7 @@
14681
14682 #include <linux/cpumask.h>
14683 #include <linux/kthread.h>
14684 -#include <soc/fsl/qman.h>
14685 +#include <linux/fsl_qman.h>
14686
14687 #include "regs.h"
14688 #include "qi.h"
14689 @@ -58,11 +58,9 @@ static DEFINE_PER_CPU(int, last_cpu);
14690 /*
14691 * caam_qi_priv - CAAM QI backend private params
14692 * @cgr: QMan congestion group
14693 - * @qi_pdev: platform device for QI backend
14694 */
14695 struct caam_qi_priv {
14696 struct qman_cgr cgr;
14697 - struct platform_device *qi_pdev;
14698 };
14699
14700 static struct caam_qi_priv qipriv ____cacheline_aligned;
14701 @@ -102,26 +100,34 @@ static int mod_init_cpu;
14702 */
14703 static struct kmem_cache *qi_cache;
14704
14705 +static void *caam_iova_to_virt(struct iommu_domain *domain,
14706 + dma_addr_t iova_addr)
14707 +{
14708 + phys_addr_t phys_addr;
14709 +
14710 + phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
14711 +
14712 + return phys_to_virt(phys_addr);
14713 +}
14714 +
14715 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
14716 {
14717 struct qm_fd fd;
14718 - dma_addr_t addr;
14719 int ret;
14720 int num_retries = 0;
14721
14722 - qm_fd_clear_fd(&fd);
14723 - qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
14724 -
14725 - addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
14726 + fd.cmd = 0;
14727 + fd.format = qm_fd_compound;
14728 + fd.cong_weight = caam32_to_cpu(req->fd_sgt[1].length);
14729 + fd.addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
14730 DMA_BIDIRECTIONAL);
14731 - if (dma_mapping_error(qidev, addr)) {
14732 + if (dma_mapping_error(qidev, fd.addr)) {
14733 dev_err(qidev, "DMA mapping error for QI enqueue request\n");
14734 return -EIO;
14735 }
14736 - qm_fd_addr_set64(&fd, addr);
14737
14738 do {
14739 - ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
14740 + ret = qman_enqueue(req->drv_ctx->req_fq, &fd, 0);
14741 if (likely(!ret))
14742 return 0;
14743
14744 @@ -137,20 +143,21 @@ int caam_qi_enqueue(struct device *qidev
14745 EXPORT_SYMBOL(caam_qi_enqueue);
14746
14747 static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
14748 - const union qm_mr_entry *msg)
14749 + const struct qm_mr_entry *msg)
14750 {
14751 const struct qm_fd *fd;
14752 struct caam_drv_req *drv_req;
14753 struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
14754 + struct caam_drv_private *priv = dev_get_drvdata(qidev);
14755
14756 fd = &msg->ern.fd;
14757
14758 - if (qm_fd_get_format(fd) != qm_fd_compound) {
14759 + if (fd->format != qm_fd_compound) {
14760 dev_err(qidev, "Non-compound FD from CAAM\n");
14761 return;
14762 }
14763
14764 - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
14765 + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
14766 if (!drv_req) {
14767 dev_err(qidev,
14768 "Can't find original request for CAAM response\n");
14769 @@ -180,20 +187,22 @@ static struct qman_fq *create_caam_req_f
14770 req_fq->cb.fqs = NULL;
14771
14772 ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
14773 - QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
14774 + QMAN_FQ_FLAG_TO_DCPORTAL | QMAN_FQ_FLAG_LOCKED,
14775 + req_fq);
14776 if (ret) {
14777 dev_err(qidev, "Failed to create session req FQ\n");
14778 goto create_req_fq_fail;
14779 }
14780
14781 - memset(&opts, 0, sizeof(opts));
14782 - opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
14783 - QM_INITFQ_WE_CONTEXTB |
14784 - QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
14785 - opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
14786 - qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
14787 - opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
14788 - qm_fqd_context_a_set64(&opts.fqd, hwdesc);
14789 + opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
14790 + QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA |
14791 + QM_INITFQ_WE_CGID;
14792 + opts.fqd.fq_ctrl = QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE;
14793 + opts.fqd.dest.channel = qm_channel_caam;
14794 + opts.fqd.dest.wq = 2;
14795 + opts.fqd.context_b = qman_fq_fqid(rsp_fq);
14796 + opts.fqd.context_a.hi = upper_32_bits(hwdesc);
14797 + opts.fqd.context_a.lo = lower_32_bits(hwdesc);
14798 opts.fqd.cgid = qipriv.cgr.cgrid;
14799
14800 ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
14801 @@ -207,7 +216,7 @@ static struct qman_fq *create_caam_req_f
14802 return req_fq;
14803
14804 init_req_fq_fail:
14805 - qman_destroy_fq(req_fq);
14806 + qman_destroy_fq(req_fq, 0);
14807 create_req_fq_fail:
14808 kfree(req_fq);
14809 return ERR_PTR(ret);
14810 @@ -275,7 +284,7 @@ empty_fq:
14811 if (ret)
14812 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
14813
14814 - qman_destroy_fq(fq);
14815 + qman_destroy_fq(fq, 0);
14816 kfree(fq);
14817
14818 return ret;
14819 @@ -292,7 +301,7 @@ static int empty_caam_fq(struct qman_fq
14820 if (ret)
14821 return ret;
14822
14823 - if (!qm_mcr_np_get(&np, frm_cnt))
14824 + if (!np.frm_cnt)
14825 break;
14826
14827 msleep(20);
14828 @@ -495,7 +504,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel);
14829 int caam_qi_shutdown(struct device *qidev)
14830 {
14831 int i, ret;
14832 - struct caam_qi_priv *priv = dev_get_drvdata(qidev);
14833 + struct caam_qi_priv *priv = &qipriv;
14834 const cpumask_t *cpus = qman_affine_cpus();
14835 struct cpumask old_cpumask = current->cpus_allowed;
14836
14837 @@ -528,7 +537,6 @@ int caam_qi_shutdown(struct device *qide
14838 /* Now that we're done with the CGRs, restore the cpus allowed mask */
14839 set_cpus_allowed_ptr(current, &old_cpumask);
14840
14841 - platform_device_unregister(priv->qi_pdev);
14842 return ret;
14843 }
14844
14845 @@ -572,22 +580,28 @@ static enum qman_cb_dqrr_result caam_rsp
14846 struct caam_drv_req *drv_req;
14847 const struct qm_fd *fd;
14848 struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
14849 - u32 status;
14850 + struct caam_drv_private *priv = dev_get_drvdata(qidev);
14851
14852 if (caam_qi_napi_schedule(p, caam_napi))
14853 return qman_cb_dqrr_stop;
14854
14855 fd = &dqrr->fd;
14856 - status = be32_to_cpu(fd->status);
14857 - if (unlikely(status))
14858 - dev_err(qidev, "Error: %#x in CAAM response FD\n", status);
14859 + if (unlikely(fd->status)) {
14860 + u32 ssrc = fd->status & JRSTA_SSRC_MASK;
14861 + u8 err_id = fd->status & JRSTA_CCBERR_ERRID_MASK;
14862
14863 - if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
14864 + if (ssrc != JRSTA_SSRC_CCB_ERROR ||
14865 + err_id != JRSTA_CCBERR_ERRID_ICVCHK)
14866 + dev_err(qidev, "Error: %#x in CAAM response FD\n",
14867 + fd->status);
14868 + }
14869 +
14870 + if (unlikely(fd->format != qm_fd_compound)) {
14871 dev_err(qidev, "Non-compound FD from CAAM\n");
14872 return qman_cb_dqrr_consume;
14873 }
14874
14875 - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
14876 + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
14877 if (unlikely(!drv_req)) {
14878 dev_err(qidev,
14879 "Can't find original request for caam response\n");
14880 @@ -597,7 +611,7 @@ static enum qman_cb_dqrr_result caam_rsp
14881 dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
14882 sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
14883
14884 - drv_req->cbk(drv_req, status);
14885 + drv_req->cbk(drv_req, fd->status);
14886 return qman_cb_dqrr_consume;
14887 }
14888
14889 @@ -621,17 +635,18 @@ static int alloc_rsp_fq_cpu(struct devic
14890 return -ENODEV;
14891 }
14892
14893 - memset(&opts, 0, sizeof(opts));
14894 - opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
14895 - QM_INITFQ_WE_CONTEXTB |
14896 - QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
14897 - opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
14898 - QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
14899 - qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
14900 + opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
14901 + QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA |
14902 + QM_INITFQ_WE_CGID;
14903 + opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING | QM_FQCTRL_CPCSTASH |
14904 + QM_FQCTRL_CGE;
14905 + opts.fqd.dest.channel = qman_affine_channel(cpu);
14906 + opts.fqd.dest.wq = 3;
14907 opts.fqd.cgid = qipriv.cgr.cgrid;
14908 opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
14909 QM_STASHING_EXCL_DATA;
14910 - qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
14911 + opts.fqd.context_a.stashing.data_cl = 1;
14912 + opts.fqd.context_a.stashing.context_cl = 1;
14913
14914 ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
14915 if (ret) {
14916 @@ -650,9 +665,8 @@ static int init_cgr(struct device *qidev
14917 {
14918 int ret;
14919 struct qm_mcc_initcgr opts;
14920 - const u64 cpus = *(u64 *)qman_affine_cpus();
14921 - const int num_cpus = hweight64(cpus);
14922 - const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU;
14923 + const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
14924 + MAX_RSP_FQ_BACKLOG_PER_CPU;
14925
14926 ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
14927 if (ret) {
14928 @@ -662,8 +676,7 @@ static int init_cgr(struct device *qidev
14929
14930 qipriv.cgr.cb = cgr_cb;
14931 memset(&opts, 0, sizeof(opts));
14932 - opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
14933 - QM_CGR_WE_MODE);
14934 + opts.we_mask = QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | QM_CGR_WE_MODE;
14935 opts.cgr.cscn_en = QM_CGR_EN;
14936 opts.cgr.mode = QMAN_CGR_MODE_FRAME;
14937 qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
14938 @@ -708,15 +721,10 @@ static void free_rsp_fqs(void)
14939 int caam_qi_init(struct platform_device *caam_pdev)
14940 {
14941 int err, i;
14942 - struct platform_device *qi_pdev;
14943 struct device *ctrldev = &caam_pdev->dev, *qidev;
14944 struct caam_drv_private *ctrlpriv;
14945 const cpumask_t *cpus = qman_affine_cpus();
14946 struct cpumask old_cpumask = current->cpus_allowed;
14947 - static struct platform_device_info qi_pdev_info = {
14948 - .name = "caam_qi",
14949 - .id = PLATFORM_DEVID_NONE
14950 - };
14951
14952 /*
14953 * QMAN requires CGRs to be removed from same CPU+portal from where it
14954 @@ -728,24 +736,13 @@ int caam_qi_init(struct platform_device
14955 mod_init_cpu = cpumask_first(cpus);
14956 set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu));
14957
14958 - qi_pdev_info.parent = ctrldev;
14959 - qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
14960 - qi_pdev = platform_device_register_full(&qi_pdev_info);
14961 - if (IS_ERR(qi_pdev))
14962 - return PTR_ERR(qi_pdev);
14963 - set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
14964 -
14965 ctrlpriv = dev_get_drvdata(ctrldev);
14966 - qidev = &qi_pdev->dev;
14967 -
14968 - qipriv.qi_pdev = qi_pdev;
14969 - dev_set_drvdata(qidev, &qipriv);
14970 + qidev = ctrldev;
14971
14972 /* Initialize the congestion detection */
14973 err = init_cgr(qidev);
14974 if (err) {
14975 dev_err(qidev, "CGR initialization failed: %d\n", err);
14976 - platform_device_unregister(qi_pdev);
14977 return err;
14978 }
14979
14980 @@ -754,7 +751,6 @@ int caam_qi_init(struct platform_device
14981 if (err) {
14982 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
14983 free_rsp_fqs();
14984 - platform_device_unregister(qi_pdev);
14985 return err;
14986 }
14987
14988 @@ -777,15 +773,11 @@ int caam_qi_init(struct platform_device
14989 napi_enable(irqtask);
14990 }
14991
14992 - /* Hook up QI device to parent controlling caam device */
14993 - ctrlpriv->qidev = qidev;
14994 -
14995 qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
14996 SLAB_CACHE_DMA, NULL);
14997 if (!qi_cache) {
14998 dev_err(qidev, "Can't allocate CAAM cache\n");
14999 free_rsp_fqs();
15000 - platform_device_unregister(qi_pdev);
15001 return -ENOMEM;
15002 }
15003
15004 @@ -795,6 +787,8 @@ int caam_qi_init(struct platform_device
15005 debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
15006 &times_congested, &caam_fops_u64_ro);
15007 #endif
15008 +
15009 + ctrlpriv->qi_init = 1;
15010 dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
15011 return 0;
15012 }
15013 --- a/drivers/crypto/caam/qi.h
15014 +++ b/drivers/crypto/caam/qi.h
15015 @@ -9,7 +9,7 @@
15016 #ifndef __QI_H__
15017 #define __QI_H__
15018
15019 -#include <soc/fsl/qman.h>
15020 +#include <linux/fsl_qman.h>
15021 #include "compat.h"
15022 #include "desc.h"
15023 #include "desc_constr.h"
15024 --- a/drivers/crypto/caam/regs.h
15025 +++ b/drivers/crypto/caam/regs.h
15026 @@ -3,6 +3,7 @@
15027 * CAAM hardware register-level view
15028 *
15029 * Copyright 2008-2011 Freescale Semiconductor, Inc.
15030 + * Copyright 2018 NXP
15031 */
15032
15033 #ifndef REGS_H
15034 @@ -211,6 +212,47 @@ struct jr_outentry {
15035 u32 jrstatus; /* Status for completed descriptor */
15036 } __packed;
15037
15038 +/* Version registers (Era 10+) e80-eff */
15039 +struct version_regs {
15040 + u32 crca; /* CRCA_VERSION */
15041 + u32 afha; /* AFHA_VERSION */
15042 + u32 kfha; /* KFHA_VERSION */
15043 + u32 pkha; /* PKHA_VERSION */
15044 + u32 aesa; /* AESA_VERSION */
15045 + u32 mdha; /* MDHA_VERSION */
15046 + u32 desa; /* DESA_VERSION */
15047 + u32 snw8a; /* SNW8A_VERSION */
15048 + u32 snw9a; /* SNW9A_VERSION */
15049 + u32 zuce; /* ZUCE_VERSION */
15050 + u32 zuca; /* ZUCA_VERSION */
15051 + u32 ccha; /* CCHA_VERSION */
15052 + u32 ptha; /* PTHA_VERSION */
15053 + u32 rng; /* RNG_VERSION */
15054 + u32 trng; /* TRNG_VERSION */
15055 + u32 aaha; /* AAHA_VERSION */
15056 + u32 rsvd[10];
15057 + u32 sr; /* SR_VERSION */
15058 + u32 dma; /* DMA_VERSION */
15059 + u32 ai; /* AI_VERSION */
15060 + u32 qi; /* QI_VERSION */
15061 + u32 jr; /* JR_VERSION */
15062 + u32 deco; /* DECO_VERSION */
15063 +};
15064 +
15065 +/* Version registers bitfields */
15066 +
15067 +/* Number of CHAs instantiated */
15068 +#define CHA_VER_NUM_MASK 0xffull
15069 +/* CHA Miscellaneous Information */
15070 +#define CHA_VER_MISC_SHIFT 8
15071 +#define CHA_VER_MISC_MASK (0xffull << CHA_VER_MISC_SHIFT)
15072 +/* CHA Revision Number */
15073 +#define CHA_VER_REV_SHIFT 16
15074 +#define CHA_VER_REV_MASK (0xffull << CHA_VER_REV_SHIFT)
15075 +/* CHA Version ID */
15076 +#define CHA_VER_VID_SHIFT 24
15077 +#define CHA_VER_VID_MASK (0xffull << CHA_VER_VID_SHIFT)
15078 +
15079 /*
15080 * caam_perfmon - Performance Monitor/Secure Memory Status/
15081 * CAAM Global Status/Component Version IDs
15082 @@ -223,15 +265,13 @@ struct jr_outentry {
15083 #define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT)
15084
15085 /*
15086 - * CHA version IDs / instantiation bitfields
15087 + * CHA version IDs / instantiation bitfields (< Era 10)
15088 * Defined for use with the cha_id fields in perfmon, but the same shift/mask
15089 * selectors can be used to pull out the number of instantiated blocks within
15090 * cha_num fields in perfmon because the locations are the same.
15091 */
15092 #define CHA_ID_LS_AES_SHIFT 0
15093 #define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT)
15094 -#define CHA_ID_LS_AES_LP (0x3ull << CHA_ID_LS_AES_SHIFT)
15095 -#define CHA_ID_LS_AES_HP (0x4ull << CHA_ID_LS_AES_SHIFT)
15096
15097 #define CHA_ID_LS_DES_SHIFT 4
15098 #define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT)
15099 @@ -241,9 +281,6 @@ struct jr_outentry {
15100
15101 #define CHA_ID_LS_MD_SHIFT 12
15102 #define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT)
15103 -#define CHA_ID_LS_MD_LP256 (0x0ull << CHA_ID_LS_MD_SHIFT)
15104 -#define CHA_ID_LS_MD_LP512 (0x1ull << CHA_ID_LS_MD_SHIFT)
15105 -#define CHA_ID_LS_MD_HP (0x2ull << CHA_ID_LS_MD_SHIFT)
15106
15107 #define CHA_ID_LS_RNG_SHIFT 16
15108 #define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT)
15109 @@ -269,6 +306,13 @@ struct jr_outentry {
15110 #define CHA_ID_MS_JR_SHIFT 28
15111 #define CHA_ID_MS_JR_MASK (0xfull << CHA_ID_MS_JR_SHIFT)
15112
15113 +/* Specific CHA version IDs */
15114 +#define CHA_VER_VID_AES_LP 0x3ull
15115 +#define CHA_VER_VID_AES_HP 0x4ull
15116 +#define CHA_VER_VID_MD_LP256 0x0ull
15117 +#define CHA_VER_VID_MD_LP512 0x1ull
15118 +#define CHA_VER_VID_MD_HP 0x2ull
15119 +
15120 struct sec_vid {
15121 u16 ip_id;
15122 u8 maj_rev;
15123 @@ -473,8 +517,10 @@ struct caam_ctrl {
15124 struct rng4tst r4tst[2];
15125 };
15126
15127 - u32 rsvd9[448];
15128 + u32 rsvd9[416];
15129
15130 + /* Version registers - introduced with era 10 e80-eff */
15131 + struct version_regs vreg;
15132 /* Performance Monitor f00-fff */
15133 struct caam_perfmon perfmon;
15134 };
15135 @@ -564,8 +610,10 @@ struct caam_job_ring {
15136 u32 rsvd11;
15137 u32 jrcommand; /* JRCRx - JobR command */
15138
15139 - u32 rsvd12[932];
15140 + u32 rsvd12[900];
15141
15142 + /* Version registers - introduced with era 10 e80-eff */
15143 + struct version_regs vreg;
15144 /* Performance Monitor f00-fff */
15145 struct caam_perfmon perfmon;
15146 };
15147 @@ -627,6 +675,8 @@ struct caam_job_ring {
15148 #define JRSTA_DECOERR_INVSIGN 0x86
15149 #define JRSTA_DECOERR_DSASIGN 0x87
15150
15151 +#define JRSTA_QIERR_ERROR_MASK 0x00ff
15152 +
15153 #define JRSTA_CCBERR_JUMP 0x08000000
15154 #define JRSTA_CCBERR_INDEX_MASK 0xff00
15155 #define JRSTA_CCBERR_INDEX_SHIFT 8
15156 @@ -870,13 +920,19 @@ struct caam_deco {
15157 u32 rsvd29[48];
15158 u32 descbuf[64]; /* DxDESB - Descriptor buffer */
15159 u32 rscvd30[193];
15160 -#define DESC_DBG_DECO_STAT_HOST_ERR 0x00D00000
15161 #define DESC_DBG_DECO_STAT_VALID 0x80000000
15162 #define DESC_DBG_DECO_STAT_MASK 0x00F00000
15163 +#define DESC_DBG_DECO_STAT_SHIFT 20
15164 u32 desc_dbg; /* DxDDR - DECO Debug Register */
15165 - u32 rsvd31[126];
15166 + u32 rsvd31[13];
15167 +#define DESC_DER_DECO_STAT_MASK 0x000F0000
15168 +#define DESC_DER_DECO_STAT_SHIFT 16
15169 + u32 dbg_exec; /* DxDER - DECO Debug Exec Register */
15170 + u32 rsvd32[112];
15171 };
15172
15173 +#define DECO_STAT_HOST_ERR 0xD
15174 +
15175 #define DECO_JQCR_WHL 0x20000000
15176 #define DECO_JQCR_FOUR 0x10000000
15177
15178 --- a/drivers/crypto/caam/sg_sw_qm.h
15179 +++ b/drivers/crypto/caam/sg_sw_qm.h
15180 @@ -34,46 +34,61 @@
15181 #ifndef __SG_SW_QM_H
15182 #define __SG_SW_QM_H
15183
15184 -#include <soc/fsl/qman.h>
15185 +#include <linux/fsl_qman.h>
15186 #include "regs.h"
15187
15188 +static inline void cpu_to_hw_sg(struct qm_sg_entry *qm_sg_ptr)
15189 +{
15190 + dma_addr_t addr = qm_sg_ptr->opaque;
15191 +
15192 + qm_sg_ptr->opaque = cpu_to_caam64(addr);
15193 + qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl);
15194 +}
15195 +
15196 static inline void __dma_to_qm_sg(struct qm_sg_entry *qm_sg_ptr, dma_addr_t dma,
15197 - u16 offset)
15198 + u32 len, u16 offset)
15199 {
15200 - qm_sg_entry_set64(qm_sg_ptr, dma);
15201 + qm_sg_ptr->addr = dma;
15202 + qm_sg_ptr->length = len;
15203 qm_sg_ptr->__reserved2 = 0;
15204 qm_sg_ptr->bpid = 0;
15205 - qm_sg_ptr->offset = cpu_to_be16(offset & QM_SG_OFF_MASK);
15206 + qm_sg_ptr->__reserved3 = 0;
15207 + qm_sg_ptr->offset = offset & QM_SG_OFFSET_MASK;
15208 +
15209 + cpu_to_hw_sg(qm_sg_ptr);
15210 }
15211
15212 static inline void dma_to_qm_sg_one(struct qm_sg_entry *qm_sg_ptr,
15213 dma_addr_t dma, u32 len, u16 offset)
15214 {
15215 - __dma_to_qm_sg(qm_sg_ptr, dma, offset);
15216 - qm_sg_entry_set_len(qm_sg_ptr, len);
15217 + qm_sg_ptr->extension = 0;
15218 + qm_sg_ptr->final = 0;
15219 + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset);
15220 }
15221
15222 static inline void dma_to_qm_sg_one_last(struct qm_sg_entry *qm_sg_ptr,
15223 dma_addr_t dma, u32 len, u16 offset)
15224 {
15225 - __dma_to_qm_sg(qm_sg_ptr, dma, offset);
15226 - qm_sg_entry_set_f(qm_sg_ptr, len);
15227 + qm_sg_ptr->extension = 0;
15228 + qm_sg_ptr->final = 1;
15229 + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset);
15230 }
15231
15232 static inline void dma_to_qm_sg_one_ext(struct qm_sg_entry *qm_sg_ptr,
15233 dma_addr_t dma, u32 len, u16 offset)
15234 {
15235 - __dma_to_qm_sg(qm_sg_ptr, dma, offset);
15236 - qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | (len & QM_SG_LEN_MASK));
15237 + qm_sg_ptr->extension = 1;
15238 + qm_sg_ptr->final = 0;
15239 + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset);
15240 }
15241
15242 static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr,
15243 dma_addr_t dma, u32 len,
15244 u16 offset)
15245 {
15246 - __dma_to_qm_sg(qm_sg_ptr, dma, offset);
15247 - qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | QM_SG_FIN |
15248 - (len & QM_SG_LEN_MASK));
15249 + qm_sg_ptr->extension = 1;
15250 + qm_sg_ptr->final = 1;
15251 + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset);
15252 }
15253
15254 /*
15255 @@ -102,7 +117,10 @@ static inline void sg_to_qm_sg_last(stru
15256 struct qm_sg_entry *qm_sg_ptr, u16 offset)
15257 {
15258 qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
15259 - qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr));
15260 +
15261 + qm_sg_ptr->sgt_efl = caam32_to_cpu(qm_sg_ptr->sgt_efl);
15262 + qm_sg_ptr->final = 1;
15263 + qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl);
15264 }
15265
15266 #endif /* __SG_SW_QM_H */
15267 --- a/drivers/crypto/talitos.c
15268 +++ b/drivers/crypto/talitos.c
15269 @@ -1247,6 +1247,14 @@ static int ipsec_esp(struct talitos_edes
15270 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
15271 sg_count, areq->assoclen, tbl_off, elen);
15272
15273 + /*
15274 + * In case of SEC 2.x+, cipher in len must include only the ciphertext,
15275 + * while extent is used for ICV len.
15276 + */
15277 + if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
15278 + (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
15279 + desc->ptr[4].len = cpu_to_be16(cryptlen);
15280 +
15281 if (ret > 1) {
15282 tbl_off += ret;
15283 sync_needed = true;
15284 --- a/include/crypto/chacha20.h
15285 +++ b/include/crypto/chacha20.h
15286 @@ -13,6 +13,7 @@
15287 #define CHACHA20_IV_SIZE 16
15288 #define CHACHA20_KEY_SIZE 32
15289 #define CHACHA20_BLOCK_SIZE 64
15290 +#define CHACHAPOLY_IV_SIZE 12
15291
15292 struct chacha20_ctx {
15293 u32 key[8];
OpenWrt Source Repository