+++ /dev/null
-/*
- * Cryptographic API.
- *
- * Driver for EIP97 AES acceleration.
- *
- * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Some ideas are from atmel-aes.c drivers.
- */
-
-#include <crypto/aes.h>
-#include "mtk-platform.h"
-
-#define AES_QUEUE_SIZE 512
-#define AES_BUF_ORDER 2
-#define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \
- & ~(AES_BLOCK_SIZE - 1))
-#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \
- AES_BLOCK_SIZE * 2)
-#define AES_MAX_CT_SIZE 6
-
-#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000)
-
-/* AES-CBC/ECB/CTR command token */
-#define AES_CMD0 cpu_to_le32(0x05000000)
-#define AES_CMD1 cpu_to_le32(0x2d060000)
-#define AES_CMD2 cpu_to_le32(0xe4a63806)
-/* AES-GCM command token */
-#define AES_GCM_CMD0 cpu_to_le32(0x0b000000)
-#define AES_GCM_CMD1 cpu_to_le32(0xa0800000)
-#define AES_GCM_CMD2 cpu_to_le32(0x25000010)
-#define AES_GCM_CMD3 cpu_to_le32(0x0f020000)
-#define AES_GCM_CMD4 cpu_to_le32(0x21e60000)
-#define AES_GCM_CMD5 cpu_to_le32(0x40e60000)
-#define AES_GCM_CMD6 cpu_to_le32(0xd0070000)
-
-/* AES transform information word 0 fields */
-#define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0)
-#define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0)
-#define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0)
-#define AES_TFM_GCM_IN cpu_to_le32(0xf << 0)
-#define AES_TFM_SIZE(x) cpu_to_le32((x) << 8)
-#define AES_TFM_128BITS cpu_to_le32(0xb << 16)
-#define AES_TFM_192BITS cpu_to_le32(0xd << 16)
-#define AES_TFM_256BITS cpu_to_le32(0xf << 16)
-#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21)
-#define AES_TFM_GHASH cpu_to_le32(0x4 << 23)
-/* AES transform information word 1 fields */
-#define AES_TFM_ECB cpu_to_le32(0x0 << 0)
-#define AES_TFM_CBC cpu_to_le32(0x1 << 0)
-#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */
-#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */
-#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */
-#define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */
-#define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10)
-#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17)
-
-/* AES flags */
-#define AES_FLAGS_CIPHER_MSK GENMASK(2, 0)
-#define AES_FLAGS_ECB BIT(0)
-#define AES_FLAGS_CBC BIT(1)
-#define AES_FLAGS_CTR BIT(2)
-#define AES_FLAGS_GCM BIT(3)
-#define AES_FLAGS_ENCRYPT BIT(4)
-#define AES_FLAGS_BUSY BIT(5)
-
-#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26))
-
-/**
- * mtk_aes_info - hardware information of AES
- * @cmd: command token, hardware instruction
- * @tfm: transform state of cipher algorithm.
- * @state: contains keys and initial vectors.
- *
- * Memory layout of GCM buffer:
- * /-----------\
- * | AES KEY | 128/196/256 bits
- * |-----------|
- * | HASH KEY | a string 128 zero bits encrypted using the block cipher
- * |-----------|
- * | IVs | 4 * 4 bytes
- * \-----------/
- *
- * The engine requires all these info to do:
- * - Commands decoding and control of the engine's data path.
- * - Coordinating hardware data fetch and store operations.
- * - Result token construction and output.
- */
-struct mtk_aes_info {
- __le32 cmd[AES_MAX_CT_SIZE];
- __le32 tfm[2];
- __le32 state[AES_MAX_STATE_BUF_SIZE];
-};
-
-struct mtk_aes_reqctx {
- u64 mode;
-};
-
-struct mtk_aes_base_ctx {
- struct mtk_cryp *cryp;
- u32 keylen;
- __le32 keymode;
-
- mtk_aes_fn start;
-
- struct mtk_aes_info info;
- dma_addr_t ct_dma;
- dma_addr_t tfm_dma;
-
- __le32 ct_hdr;
- u32 ct_size;
-};
-
-struct mtk_aes_ctx {
- struct mtk_aes_base_ctx base;
-};
-
-struct mtk_aes_ctr_ctx {
- struct mtk_aes_base_ctx base;
-
- u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
- size_t offset;
- struct scatterlist src[2];
- struct scatterlist dst[2];
-};
-
-struct mtk_aes_gcm_ctx {
- struct mtk_aes_base_ctx base;
-
- u32 authsize;
- size_t textlen;
-
- struct crypto_skcipher *ctr;
-};
-
-struct mtk_aes_gcm_setkey_result {
- int err;
- struct completion completion;
-};
-
-struct mtk_aes_drv {
- struct list_head dev_list;
- /* Device list lock */
- spinlock_t lock;
-};
-
-static struct mtk_aes_drv mtk_aes = {
- .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list),
- .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock),
-};
-
-static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset)
-{
- return readl_relaxed(cryp->base + offset);
-}
-
-static inline void mtk_aes_write(struct mtk_cryp *cryp,
- u32 offset, u32 value)
-{
- writel_relaxed(value, cryp->base + offset);
-}
-
-static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx)
-{
- struct mtk_cryp *cryp = NULL;
- struct mtk_cryp *tmp;
-
- spin_lock_bh(&mtk_aes.lock);
- if (!ctx->cryp) {
- list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) {
- cryp = tmp;
- break;
- }
- ctx->cryp = cryp;
- } else {
- cryp = ctx->cryp;
- }
- spin_unlock_bh(&mtk_aes.lock);
-
- return cryp;
-}
-
-static inline size_t mtk_aes_padlen(size_t len)
-{
- len &= AES_BLOCK_SIZE - 1;
- return len ? AES_BLOCK_SIZE - len : 0;
-}
-
-static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len,
- struct mtk_aes_dma *dma)
-{
- int nents;
-
- if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
- return false;
-
- for (nents = 0; sg; sg = sg_next(sg), ++nents) {
- if (!IS_ALIGNED(sg->offset, sizeof(u32)))
- return false;
-
- if (len <= sg->length) {
- if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
- return false;
-
- dma->nents = nents + 1;
- dma->remainder = sg->length - len;
- sg->length = len;
- return true;
- }
-
- if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
- return false;
-
- len -= sg->length;
- }
-
- return false;
-}
-
-static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,
- const struct mtk_aes_reqctx *rctx)
-{
- /* Clear all but persistent flags and set request flags. */
- aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;
-}
-
-static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
-{
- struct scatterlist *sg = dma->sg;
- int nents = dma->nents;
-
- if (!dma->remainder)
- return;
-
- while (--nents > 0 && sg)
- sg = sg_next(sg);
-
- if (!sg)
- return;
-
- sg->length += dma->remainder;
-}
-
-static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size)
-{
- int i;
-
- for (i = 0; i < SIZE_IN_WORDS(size); i++)
- dst[i] = cpu_to_le32(src[i]);
-}
-
-static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size)
-{
- int i;
-
- for (i = 0; i < SIZE_IN_WORDS(size); i++)
- dst[i] = cpu_to_be32(src[i]);
-}
-
-static inline int mtk_aes_complete(struct mtk_cryp *cryp,
- struct mtk_aes_rec *aes,
- int err)
-{
- aes->flags &= ~AES_FLAGS_BUSY;
- aes->areq->complete(aes->areq, err);
- /* Handle new request */
- tasklet_schedule(&aes->queue_task);
- return err;
-}
-
-/*
- * Write descriptors for processing. This will configure the engine, load
- * the transform information and then start the packet processing.
- */
-static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_ring *ring = cryp->ring[aes->id];
- struct mtk_desc *cmd = NULL, *res = NULL;
- struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;
- u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;
- int nents;
-
- /* Write command descriptors */
- for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) {
- cmd = ring->cmd_next;
- cmd->hdr = MTK_DESC_BUF_LEN(ssg->length);
- cmd->buf = cpu_to_le32(sg_dma_address(ssg));
-
- if (nents == 0) {
- cmd->hdr |= MTK_DESC_FIRST |
- MTK_DESC_CT_LEN(aes->ctx->ct_size);
- cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
- cmd->ct_hdr = aes->ctx->ct_hdr;
- cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
- }
-
- /* Shift ring buffer and check boundary */
- if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM)
- ring->cmd_next = ring->cmd_base;
- }
- cmd->hdr |= MTK_DESC_LAST;
-
- /* Prepare result descriptors */
- for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) {
- res = ring->res_next;
- res->hdr = MTK_DESC_BUF_LEN(dsg->length);
- res->buf = cpu_to_le32(sg_dma_address(dsg));
-
- if (nents == 0)
- res->hdr |= MTK_DESC_FIRST;
-
- /* Shift ring buffer and check boundary */
- if (++ring->res_next == ring->res_base + MTK_DESC_NUM)
- ring->res_next = ring->res_base;
- }
- res->hdr |= MTK_DESC_LAST;
-
- /* Pointer to current result descriptor */
- ring->res_prev = res;
-
- /* Prepare enough space for authenticated tag */
- if (aes->flags & AES_FLAGS_GCM)
- res->hdr += AES_BLOCK_SIZE;
-
- /*
- * Make sure that all changes to the DMA ring are done before we
- * start engine.
- */
- wmb();
- /* Start DMA transfer */
- mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));
- mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));
-
- return -EINPROGRESS;
-}
-
-static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_aes_base_ctx *ctx = aes->ctx;
-
- dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
- DMA_TO_DEVICE);
-
- if (aes->src.sg == aes->dst.sg) {
- dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
- DMA_BIDIRECTIONAL);
-
- if (aes->src.sg != &aes->aligned_sg)
- mtk_aes_restore_sg(&aes->src);
- } else {
- dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,
- DMA_FROM_DEVICE);
-
- if (aes->dst.sg != &aes->aligned_sg)
- mtk_aes_restore_sg(&aes->dst);
-
- dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
- DMA_TO_DEVICE);
-
- if (aes->src.sg != &aes->aligned_sg)
- mtk_aes_restore_sg(&aes->src);
- }
-
- if (aes->dst.sg == &aes->aligned_sg)
- sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),
- aes->buf, aes->total);
-}
-
-static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_aes_base_ctx *ctx = aes->ctx;
- struct mtk_aes_info *info = &ctx->info;
-
- ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
- DMA_TO_DEVICE);
- if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
- goto exit;
-
- ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd);
-
- if (aes->src.sg == aes->dst.sg) {
- aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
- aes->src.nents,
- DMA_BIDIRECTIONAL);
- aes->dst.sg_len = aes->src.sg_len;
- if (unlikely(!aes->src.sg_len))
- goto sg_map_err;
- } else {
- aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
- aes->src.nents, DMA_TO_DEVICE);
- if (unlikely(!aes->src.sg_len))
- goto sg_map_err;
-
- aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
- aes->dst.nents, DMA_FROM_DEVICE);
- if (unlikely(!aes->dst.sg_len)) {
- dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
- DMA_TO_DEVICE);
- goto sg_map_err;
- }
- }
-
- return mtk_aes_xmit(cryp, aes);
-
-sg_map_err:
- dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE);
-exit:
- return mtk_aes_complete(cryp, aes, -EINVAL);
-}
-
-/* Initialize transform information of CBC/ECB/CTR mode */
-static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
- size_t len)
-{
- struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
- struct mtk_aes_base_ctx *ctx = aes->ctx;
- struct mtk_aes_info *info = &ctx->info;
- u32 cnt = 0;
-
- ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
- info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len);
- info->cmd[cnt++] = AES_CMD1;
-
- info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode;
- if (aes->flags & AES_FLAGS_ENCRYPT)
- info->tfm[0] |= AES_TFM_BASIC_OUT;
- else
- info->tfm[0] |= AES_TFM_BASIC_IN;
-
- switch (aes->flags & AES_FLAGS_CIPHER_MSK) {
- case AES_FLAGS_CBC:
- info->tfm[1] = AES_TFM_CBC;
- break;
- case AES_FLAGS_ECB:
- info->tfm[1] = AES_TFM_ECB;
- goto ecb;
- case AES_FLAGS_CTR:
- info->tfm[1] = AES_TFM_CTR_LOAD;
- goto ctr;
-
- default:
- /* Should not happen... */
- return;
- }
-
- mtk_aes_write_state_le(info->state + ctx->keylen, req->info,
- AES_BLOCK_SIZE);
-ctr:
- info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE));
- info->tfm[1] |= AES_TFM_FULL_IV;
- info->cmd[cnt++] = AES_CMD2;
-ecb:
- ctx->ct_size = cnt;
-}
-
-static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
- struct scatterlist *src, struct scatterlist *dst,
- size_t len)
-{
- size_t padlen = 0;
- bool src_aligned, dst_aligned;
-
- aes->total = len;
- aes->src.sg = src;
- aes->dst.sg = dst;
- aes->real_dst = dst;
-
- src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
- if (src == dst)
- dst_aligned = src_aligned;
- else
- dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
-
- if (!src_aligned || !dst_aligned) {
- padlen = mtk_aes_padlen(len);
-
- if (len + padlen > AES_BUF_SIZE)
- return mtk_aes_complete(cryp, aes, -ENOMEM);
-
- if (!src_aligned) {
- sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
- aes->src.sg = &aes->aligned_sg;
- aes->src.nents = 1;
- aes->src.remainder = 0;
- }
-
- if (!dst_aligned) {
- aes->dst.sg = &aes->aligned_sg;
- aes->dst.nents = 1;
- aes->dst.remainder = 0;
- }
-
- sg_init_table(&aes->aligned_sg, 1);
- sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);
- }
-
- mtk_aes_info_init(cryp, aes, len + padlen);
-
- return mtk_aes_map(cryp, aes);
-}
-
-static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
- struct crypto_async_request *new_areq)
-{
- struct mtk_aes_rec *aes = cryp->aes[id];
- struct crypto_async_request *areq, *backlog;
- struct mtk_aes_base_ctx *ctx;
- unsigned long flags;
- int ret = 0;
-
- spin_lock_irqsave(&aes->lock, flags);
- if (new_areq)
- ret = crypto_enqueue_request(&aes->queue, new_areq);
- if (aes->flags & AES_FLAGS_BUSY) {
- spin_unlock_irqrestore(&aes->lock, flags);
- return ret;
- }
- backlog = crypto_get_backlog(&aes->queue);
- areq = crypto_dequeue_request(&aes->queue);
- if (areq)
- aes->flags |= AES_FLAGS_BUSY;
- spin_unlock_irqrestore(&aes->lock, flags);
-
- if (!areq)
- return ret;
-
- if (backlog)
- backlog->complete(backlog, -EINPROGRESS);
-
- ctx = crypto_tfm_ctx(areq->tfm);
-
- aes->areq = areq;
- aes->ctx = ctx;
-
- return ctx->start(cryp, aes);
-}
-
-static int mtk_aes_transfer_complete(struct mtk_cryp *cryp,
- struct mtk_aes_rec *aes)
-{
- return mtk_aes_complete(cryp, aes, 0);
-}
-
-static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
- struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
-
- mtk_aes_set_mode(aes, rctx);
- aes->resume = mtk_aes_transfer_complete;
-
- return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);
-}
-
-static inline struct mtk_aes_ctr_ctx *
-mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx)
-{
- return container_of(ctx, struct mtk_aes_ctr_ctx, base);
-}
-
-static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_aes_base_ctx *ctx = aes->ctx;
- struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx);
- struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
- struct scatterlist *src, *dst;
- u32 start, end, ctr, blocks;
- size_t datalen;
- bool fragmented = false;
-
- /* Check for transfer completion. */
- cctx->offset += aes->total;
- if (cctx->offset >= req->nbytes)
- return mtk_aes_transfer_complete(cryp, aes);
-
- /* Compute data length. */
- datalen = req->nbytes - cctx->offset;
- blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
- ctr = be32_to_cpu(cctx->iv[3]);
-
- /* Check 32bit counter overflow. */
- start = ctr;
- end = start + blocks - 1;
- if (end < start) {
- ctr |= 0xffffffff;
- datalen = AES_BLOCK_SIZE * -start;
- fragmented = true;
- }
-
- /* Jump to offset. */
- src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset);
- dst = ((req->src == req->dst) ? src :
- scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
-
- /* Write IVs into transform state buffer. */
- mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv,
- AES_BLOCK_SIZE);
-
- if (unlikely(fragmented)) {
- /*
- * Increment the counter manually to cope with the hardware
- * counter overflow.
- */
- cctx->iv[3] = cpu_to_be32(ctr);
- crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE);
- }
-
- return mtk_aes_dma(cryp, aes, src, dst, datalen);
-}
-
-static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);
- struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
- struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
-
- mtk_aes_set_mode(aes, rctx);
-
- memcpy(cctx->iv, req->info, AES_BLOCK_SIZE);
- cctx->offset = 0;
- aes->total = 0;
- aes->resume = mtk_aes_ctr_transfer;
-
- return mtk_aes_ctr_transfer(cryp, aes);
-}
-
-/* Check and set the AES key to transform state buffer */
-static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
- const u8 *key, u32 keylen)
-{
- struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
- switch (keylen) {
- case AES_KEYSIZE_128:
- ctx->keymode = AES_TFM_128BITS;
- break;
- case AES_KEYSIZE_192:
- ctx->keymode = AES_TFM_192BITS;
- break;
- case AES_KEYSIZE_256:
- ctx->keymode = AES_TFM_256BITS;
- break;
-
- default:
- crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
-
- ctx->keylen = SIZE_IN_WORDS(keylen);
- mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
-
- return 0;
-}
-
-static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
-{
- struct mtk_aes_base_ctx *ctx;
- struct mtk_aes_reqctx *rctx;
-
- ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
- rctx = ablkcipher_request_ctx(req);
- rctx->mode = mode;
-
- return mtk_aes_handle_queue(ctx->cryp, !(mode & AES_FLAGS_ENCRYPT),
- &req->base);
-}
-
-static int mtk_aes_ecb_encrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB);
-}
-
-static int mtk_aes_ecb_decrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_ECB);
-}
-
-static int mtk_aes_cbc_encrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC);
-}
-
-static int mtk_aes_cbc_decrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_CBC);
-}
-
-static int mtk_aes_ctr_encrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR);
-}
-
-static int mtk_aes_ctr_decrypt(struct ablkcipher_request *req)
-{
- return mtk_aes_crypt(req, AES_FLAGS_CTR);
-}
-
-static int mtk_aes_cra_init(struct crypto_tfm *tfm)
-{
- struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
-
- tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
- ctx->base.start = mtk_aes_start;
- return 0;
-}
-
-static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
-{
- struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
-
- tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
- ctx->base.start = mtk_aes_ctr_start;
- return 0;
-}
-
-static struct crypto_alg aes_algs[] = {
-{
- .cra_name = "cbc(aes)",
- .cra_driver_name = "cbc-aes-mtk",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
- CRYPTO_ALG_ASYNC,
- .cra_init = mtk_aes_cra_init,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct mtk_aes_ctx),
- .cra_alignmask = 0xf,
- .cra_type = &crypto_ablkcipher_type,
- .cra_module = THIS_MODULE,
- .cra_u.ablkcipher = {
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = mtk_aes_setkey,
- .encrypt = mtk_aes_cbc_encrypt,
- .decrypt = mtk_aes_cbc_decrypt,
- .ivsize = AES_BLOCK_SIZE,
- }
-},
-{
- .cra_name = "ecb(aes)",
- .cra_driver_name = "ecb-aes-mtk",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
- CRYPTO_ALG_ASYNC,
- .cra_init = mtk_aes_cra_init,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct mtk_aes_ctx),
- .cra_alignmask = 0xf,
- .cra_type = &crypto_ablkcipher_type,
- .cra_module = THIS_MODULE,
- .cra_u.ablkcipher = {
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = mtk_aes_setkey,
- .encrypt = mtk_aes_ecb_encrypt,
- .decrypt = mtk_aes_ecb_decrypt,
- }
-},
-{
- .cra_name = "ctr(aes)",
- .cra_driver_name = "ctr-aes-mtk",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
- CRYPTO_ALG_ASYNC,
- .cra_init = mtk_aes_ctr_cra_init,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct mtk_aes_ctr_ctx),
- .cra_alignmask = 0xf,
- .cra_type = &crypto_ablkcipher_type,
- .cra_module = THIS_MODULE,
- .cra_u.ablkcipher = {
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = mtk_aes_setkey,
- .encrypt = mtk_aes_ctr_encrypt,
- .decrypt = mtk_aes_ctr_decrypt,
- }
-},
-};
-
-static inline struct mtk_aes_gcm_ctx *
-mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx)
-{
- return container_of(ctx, struct mtk_aes_gcm_ctx, base);
-}
-
-/*
- * Engine will verify and compare tag automatically, so we just need
- * to check returned status which stored in the result descriptor.
- */
-static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp,
- struct mtk_aes_rec *aes)
-{
- u32 status = cryp->ring[aes->id]->res_prev->ct;
-
- return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?
- -EBADMSG : 0);
-}
-
-/* Initialize transform information of GCM mode */
-static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,
- struct mtk_aes_rec *aes,
- size_t len)
-{
- struct aead_request *req = aead_request_cast(aes->areq);
- struct mtk_aes_base_ctx *ctx = aes->ctx;
- struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
- struct mtk_aes_info *info = &ctx->info;
- u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
- u32 cnt = 0;
-
- ctx->ct_hdr = AES_CT_CTRL_HDR | len;
-
- info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
- info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
- info->cmd[cnt++] = AES_GCM_CMD2;
- info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen);
-
- if (aes->flags & AES_FLAGS_ENCRYPT) {
- info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize);
- info->tfm[0] = AES_TFM_GCM_OUT;
- } else {
- info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize);
- info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize);
- info->tfm[0] = AES_TFM_GCM_IN;
- }
- ctx->ct_size = cnt;
-
- info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE(
- ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) |
- ctx->keymode;
- info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV |
- AES_TFM_ENC_HASH;
-
- mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS(
- AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize);
-}
-
-static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
- struct scatterlist *src, struct scatterlist *dst,
- size_t len)
-{
- bool src_aligned, dst_aligned;
-
- aes->src.sg = src;
- aes->dst.sg = dst;
- aes->real_dst = dst;
-
- src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
- if (src == dst)
- dst_aligned = src_aligned;
- else
- dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
-
- if (!src_aligned || !dst_aligned) {
- if (aes->total > AES_BUF_SIZE)
- return mtk_aes_complete(cryp, aes, -ENOMEM);
-
- if (!src_aligned) {
- sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
- aes->src.sg = &aes->aligned_sg;
- aes->src.nents = 1;
- aes->src.remainder = 0;
- }
-
- if (!dst_aligned) {
- aes->dst.sg = &aes->aligned_sg;
- aes->dst.nents = 1;
- aes->dst.remainder = 0;
- }
-
- sg_init_table(&aes->aligned_sg, 1);
- sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);
- }
-
- mtk_aes_gcm_info_init(cryp, aes, len);
-
- return mtk_aes_map(cryp, aes);
-}
-
-/* Todo: GMAC */
-static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
-{
- struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);
- struct aead_request *req = aead_request_cast(aes->areq);
- struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
- u32 len = req->assoclen + req->cryptlen;
-
- mtk_aes_set_mode(aes, rctx);
-
- if (aes->flags & AES_FLAGS_ENCRYPT) {
- u32 tag[4];
-
- aes->resume = mtk_aes_transfer_complete;
- /* Compute total process length. */
- aes->total = len + gctx->authsize;
- /* Compute text length. */
- gctx->textlen = req->cryptlen;
- /* Hardware will append authenticated tag to output buffer */
- scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
- } else {
- aes->resume = mtk_aes_gcm_tag_verify;
- aes->total = len;
- gctx->textlen = req->cryptlen - gctx->authsize;
- }
-
- return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
-}
-
-static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
-{
- struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
-
- rctx->mode = AES_FLAGS_GCM | mode;
-
- return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT),
- &req->base);
-}
-
-static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err)
-{
- struct mtk_aes_gcm_setkey_result *result = req->data;
-
- if (err == -EINPROGRESS)
- return;
-
- result->err = err;
- complete(&result->completion);
-}
-
-/*
- * Because of the hardware limitation, we need to pre-calculate key(H)
- * for the GHASH operation. The result of the encryption operation
- * need to be stored in the transform state buffer.
- */
-static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
- u32 keylen)
-{
- struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
- struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
- struct crypto_skcipher *ctr = gctx->ctr;
- struct {
- u32 hash[4];
- u8 iv[8];
-
- struct mtk_aes_gcm_setkey_result result;
-
- struct scatterlist sg[1];
- struct skcipher_request req;
- } *data;
- int err;
-
- switch (keylen) {
- case AES_KEYSIZE_128:
- ctx->keymode = AES_TFM_128BITS;
- break;
- case AES_KEYSIZE_192:
- ctx->keymode = AES_TFM_192BITS;
- break;
- case AES_KEYSIZE_256:
- ctx->keymode = AES_TFM_256BITS;
- break;
-
- default:
- crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
-
- ctx->keylen = SIZE_IN_WORDS(keylen);
-
- /* Same as crypto_gcm_setkey() from crypto/gcm.c */
- crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
- crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
- CRYPTO_TFM_REQ_MASK);
- err = crypto_skcipher_setkey(ctr, key, keylen);
- crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
- CRYPTO_TFM_RES_MASK);
- if (err)
- return err;
-
- data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
- GFP_KERNEL);
- if (!data)
- return -ENOMEM;
-
- init_completion(&data->result.completion);
- sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE);
- skcipher_request_set_tfm(&data->req, ctr);
- skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
- CRYPTO_TFM_REQ_MAY_BACKLOG,
- mtk_gcm_setkey_done, &data->result);
- skcipher_request_set_crypt(&data->req, data->sg, data->sg,
- AES_BLOCK_SIZE, data->iv);
-
- err = crypto_skcipher_encrypt(&data->req);
- if (err == -EINPROGRESS || err == -EBUSY) {
- err = wait_for_completion_interruptible(
- &data->result.completion);
- if (!err)
- err = data->result.err;
- }
- if (err)
- goto out;
-
- /* Write key into state buffer */
- mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
- /* Write key(H) into state buffer */
- mtk_aes_write_state_be(ctx->info.state + ctx->keylen, data->hash,
- AES_BLOCK_SIZE);
-out:
- kzfree(data);
- return err;
-}
-
-static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead,
- u32 authsize)
-{
- struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
- struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
-
- /* Same as crypto_gcm_authsize() from crypto/gcm.c */
- switch (authsize) {
- case 8:
- case 12:
- case 16:
- break;
- default:
- return -EINVAL;
- }
-
- gctx->authsize = authsize;
- return 0;
-}
-
-static int mtk_aes_gcm_encrypt(struct aead_request *req)
-{
- return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
-}
-
-static int mtk_aes_gcm_decrypt(struct aead_request *req)
-{
- return mtk_aes_gcm_crypt(req, 0);
-}
-
-static int mtk_aes_gcm_init(struct crypto_aead *aead)
-{
- struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
-
- ctx->ctr = crypto_alloc_skcipher("ctr(aes)", 0,
- CRYPTO_ALG_ASYNC);
- if (IS_ERR(ctx->ctr)) {
- pr_err("Error allocating ctr(aes)\n");
- return PTR_ERR(ctx->ctr);
- }
-
- crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx));
- ctx->base.start = mtk_aes_gcm_start;
- return 0;
-}
-
-static void mtk_aes_gcm_exit(struct crypto_aead *aead)
-{
- struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
-
- crypto_free_skcipher(ctx->ctr);
-}
-
-static struct aead_alg aes_gcm_alg = {
- .setkey = mtk_aes_gcm_setkey,
- .setauthsize = mtk_aes_gcm_setauthsize,
- .encrypt = mtk_aes_gcm_encrypt,
- .decrypt = mtk_aes_gcm_decrypt,
- .init = mtk_aes_gcm_init,
- .exit = mtk_aes_gcm_exit,
- .ivsize = 12,
- .maxauthsize = AES_BLOCK_SIZE,
-
- .base = {
- .cra_name = "gcm(aes)",
- .cra_driver_name = "gcm-aes-mtk",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx),
- .cra_alignmask = 0xf,
- .cra_module = THIS_MODULE,
- },
-};
-
-static void mtk_aes_queue_task(unsigned long data)
-{
- struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
-
- mtk_aes_handle_queue(aes->cryp, aes->id, NULL);
-}
-
-static void mtk_aes_done_task(unsigned long data)
-{
- struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
- struct mtk_cryp *cryp = aes->cryp;
-
- mtk_aes_unmap(cryp, aes);
- aes->resume(cryp, aes);
-}
-
-static irqreturn_t mtk_aes_irq(int irq, void *dev_id)
-{
- struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id;
- struct mtk_cryp *cryp = aes->cryp;
- u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));
-
- mtk_aes_write(cryp, RDR_STAT(aes->id), val);
-
- if (likely(AES_FLAGS_BUSY & aes->flags)) {
- mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);
- mtk_aes_write(cryp, RDR_THRESH(aes->id),
- MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
-
- tasklet_schedule(&aes->done_task);
- } else {
- dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
- }
- return IRQ_HANDLED;
-}
-
-/*
- * The purpose of creating encryption and decryption records is
- * to process outbound/inbound data in parallel, it can improve
- * performance in most use cases, such as IPSec VPN, especially
- * under heavy network traffic.
- */
-static int mtk_aes_record_init(struct mtk_cryp *cryp)
-{
- struct mtk_aes_rec **aes = cryp->aes;
- int i, err = -ENOMEM;
-
- for (i = 0; i < MTK_REC_NUM; i++) {
- aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);
- if (!aes[i])
- goto err_cleanup;
-
- aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,
- AES_BUF_ORDER);
- if (!aes[i]->buf)
- goto err_cleanup;
-
- aes[i]->cryp = cryp;
-
- spin_lock_init(&aes[i]->lock);
- crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);
-
- tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task,
- (unsigned long)aes[i]);
- tasklet_init(&aes[i]->done_task, mtk_aes_done_task,
- (unsigned long)aes[i]);
- }
-
- /* Link to ring0 and ring1 respectively */
- aes[0]->id = MTK_RING0;
- aes[1]->id = MTK_RING1;
-
- return 0;
-
-err_cleanup:
- for (; i--; ) {
- free_page((unsigned long)aes[i]->buf);
- kfree(aes[i]);
- }
-
- return err;
-}
-
-static void mtk_aes_record_free(struct mtk_cryp *cryp)
-{
- int i;
-
- for (i = 0; i < MTK_REC_NUM; i++) {
- tasklet_kill(&cryp->aes[i]->done_task);
- tasklet_kill(&cryp->aes[i]->queue_task);
-
- free_page((unsigned long)cryp->aes[i]->buf);
- kfree(cryp->aes[i]);
- }
-}
-
-static void mtk_aes_unregister_algs(void)
-{
- int i;
-
- crypto_unregister_aead(&aes_gcm_alg);
-
- for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
- crypto_unregister_alg(&aes_algs[i]);
-}
-
-static int mtk_aes_register_algs(void)
-{
- int err, i;
-
- for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
- err = crypto_register_alg(&aes_algs[i]);
- if (err)
- goto err_aes_algs;
- }
-
- err = crypto_register_aead(&aes_gcm_alg);
- if (err)
- goto err_aes_algs;
-
- return 0;
-
-err_aes_algs:
- for (; i--; )
- crypto_unregister_alg(&aes_algs[i]);
-
- return err;
-}
-
-int mtk_cipher_alg_register(struct mtk_cryp *cryp)
-{
- int ret;
-
- INIT_LIST_HEAD(&cryp->aes_list);
-
- /* Initialize two cipher records */
- ret = mtk_aes_record_init(cryp);
- if (ret)
- goto err_record;
-
- ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq,
- 0, "mtk-aes", cryp->aes[0]);
- if (ret) {
- dev_err(cryp->dev, "unable to request AES irq.\n");
- goto err_res;
- }
-
- ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq,
- 0, "mtk-aes", cryp->aes[1]);
- if (ret) {
- dev_err(cryp->dev, "unable to request AES irq.\n");
- goto err_res;
- }
-
- /* Enable ring0 and ring1 interrupt */
- mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0);
- mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1);
-
- spin_lock(&mtk_aes.lock);
- list_add_tail(&cryp->aes_list, &mtk_aes.dev_list);
- spin_unlock(&mtk_aes.lock);
-
- ret = mtk_aes_register_algs();
- if (ret)
- goto err_algs;
-
- return 0;
-
-err_algs:
- spin_lock(&mtk_aes.lock);
- list_del(&cryp->aes_list);
- spin_unlock(&mtk_aes.lock);
-err_res:
- mtk_aes_record_free(cryp);
-err_record:
-
- dev_err(cryp->dev, "mtk-aes initialization failed.\n");
- return ret;
-}
-
-void mtk_cipher_alg_release(struct mtk_cryp *cryp)
-{
- spin_lock(&mtk_aes.lock);
- list_del(&cryp->aes_list);
- spin_unlock(&mtk_aes.lock);
-
- mtk_aes_unregister_algs();
- mtk_aes_record_free(cryp);
-}
projects / openwrt / openwrt.git / blobdiff