--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * StarFive AES acceleration driver
+ *
+ * Copyright (c) 2022 StarFive Technology
+ */
+
+#include <linux/iopoll.h>
+#include <crypto/gcm.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include "jh7110-cryp.h"
+
+#define STARFIVE_AES_REGS_OFFSET 0x100
+#define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0)
+#define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4)
+#define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8)
+#define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC)
+#define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10)
+#define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14)
+#define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18)
+#define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C)
+#define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20)
+#define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24)
+#define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28)
+#define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C)
+#define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30)
+#define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34)
+#define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C)
+#define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40)
+#define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44)
+#define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48)
+#define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C)
+#define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50)
+#define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54)
+#define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58)
+#define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C)
+
+#define FLG_MODE_MASK GENMASK(2, 0)
+#define FLG_ENCRYPT BIT(4)
+
+/* Misc */
+#define CCM_B0_ADATA 0x40
+#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
+
+static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)
+{
+ u32 status;
+
+ return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
+ !(status & STARFIVE_AES_BUSY), 10, 100000);
+}
+
+static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)
+{
+ u32 status;
+
+ return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
+ (status & STARFIVE_AES_KEY_DONE), 10, 100000);
+}
+
+static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)
+{
+ u32 status;
+
+ return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
+ (status & STARFIVE_AES_GCM_DONE), 10, 100000);
+}
+
+static inline int is_gcm(struct starfive_cryp_dev *cryp)
+{
+ return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;
+}
+
+static inline int is_encrypt(struct starfive_cryp_dev *cryp)
+{
+ return cryp->flags & FLG_ENCRYPT;
+}
+
+static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ unsigned int value;
+
+ switch (hw_mode) {
+ case STARFIVE_AES_MODE_GCM:
+ value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
+ value |= STARFIVE_AES_GCM_START;
+ writel(value, cryp->base + STARFIVE_AES_CSR);
+ starfive_aes_wait_gcmdone(cryp);
+ break;
+ case STARFIVE_AES_MODE_CCM:
+ value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
+ value |= STARFIVE_AES_CCM_START;
+ writel(value, cryp->base + STARFIVE_AES_CSR);
+ break;
+ }
+}
+
+static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ if (is_gcm(cryp))
+ writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);
+ else
+ writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN);
+}
+
+static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);
+ writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);
+}
+
+static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);
+ writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);
+}
+
+static inline int starfive_aes_ccm_check_iv(const u8 *iv)
+{
+ /* 2 <= L <= 8, so 1 <= L' <= 7. */
+ if (iv[0] < 1 || iv[0] > 7)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ writel(iv[0], cryp->base + STARFIVE_AES_IV0);
+ writel(iv[1], cryp->base + STARFIVE_AES_IV1);
+ writel(iv[2], cryp->base + STARFIVE_AES_IV2);
+
+ if (is_gcm(cryp)) {
+ if (starfive_aes_wait_gcmdone(cryp))
+ return -ETIMEDOUT;
+
+ return 0;
+ }
+
+ writel(iv[3], cryp->base + STARFIVE_AES_IV3);
+
+ return 0;
+}
+
+static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)
+{
+ iv[0] = readl(cryp->base + STARFIVE_AES_IV0);
+ iv[1] = readl(cryp->base + STARFIVE_AES_IV1);
+ iv[2] = readl(cryp->base + STARFIVE_AES_IV2);
+ iv[3] = readl(cryp->base + STARFIVE_AES_IV3);
+}
+
+static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);
+ writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);
+ writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);
+ writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);
+}
+
+static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ u32 *key = (u32 *)ctx->key;
+
+ if (ctx->keylen >= AES_KEYSIZE_128) {
+ writel(key[0], cryp->base + STARFIVE_AES_KEY0);
+ writel(key[1], cryp->base + STARFIVE_AES_KEY1);
+ writel(key[2], cryp->base + STARFIVE_AES_KEY2);
+ writel(key[3], cryp->base + STARFIVE_AES_KEY3);
+ }
+
+ if (ctx->keylen >= AES_KEYSIZE_192) {
+ writel(key[4], cryp->base + STARFIVE_AES_KEY4);
+ writel(key[5], cryp->base + STARFIVE_AES_KEY5);
+ }
+
+ if (ctx->keylen >= AES_KEYSIZE_256) {
+ writel(key[6], cryp->base + STARFIVE_AES_KEY6);
+ writel(key[7], cryp->base + STARFIVE_AES_KEY7);
+ }
+
+ if (starfive_aes_wait_keydone(cryp))
+ return -ETIMEDOUT;
+
+ return 0;
+}
+
+static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
+ unsigned int textlen;
+
+ memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);
+ memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
+
+ /* Build B0 */
+ memcpy(b0, iv, AES_BLOCK_SIZE);
+
+ b0[0] |= (8 * ((cryp->authsize - 2) / 2));
+
+ if (cryp->assoclen)
+ b0[0] |= CCM_B0_ADATA;
+
+ textlen = cryp->total_in;
+
+ b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
+ b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
+
+ starfive_aes_write_nonce(ctx, (u32 *)b0);
+
+ return 0;
+}
+
+static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_request_ctx *rctx = ctx->rctx;
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ u32 hw_mode;
+
+ /* reset */
+ rctx->csr.aes.v = 0;
+ rctx->csr.aes.aesrst = 1;
+ writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
+
+ /* csr setup */
+ hw_mode = cryp->flags & FLG_MODE_MASK;
+
+ rctx->csr.aes.v = 0;
+
+ switch (ctx->keylen) {
+ case AES_KEYSIZE_128:
+ rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128;
+ break;
+ case AES_KEYSIZE_192:
+ rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192;
+ break;
+ case AES_KEYSIZE_256:
+ rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256;
+ break;
+ }
+
+ rctx->csr.aes.mode = hw_mode;
+ rctx->csr.aes.cmode = !is_encrypt(cryp);
+ rctx->csr.aes.ie = 1;
+
+ if (hw_mode == STARFIVE_AES_MODE_CFB ||
+ hw_mode == STARFIVE_AES_MODE_OFB)
+ rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_128;
+ else
+ rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1;
+
+ if (cryp->side_chan) {
+ rctx->csr.aes.delay_aes = 1;
+ rctx->csr.aes.vaes_start = 1;
+ }
+
+ writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
+
+ cryp->err = starfive_aes_write_key(ctx);
+ if (cryp->err)
+ return cryp->err;
+
+ switch (hw_mode) {
+ case STARFIVE_AES_MODE_GCM:
+ starfive_aes_set_alen(ctx);
+ starfive_aes_set_mlen(ctx);
+ starfive_aes_set_ivlen(ctx);
+ starfive_aes_aead_hw_start(ctx, hw_mode);
+ starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);
+ break;
+ case STARFIVE_AES_MODE_CCM:
+ starfive_aes_set_alen(ctx);
+ starfive_aes_set_mlen(ctx);
+ starfive_aes_ccm_init(ctx);
+ starfive_aes_aead_hw_start(ctx, hw_mode);
+ break;
+ case STARFIVE_AES_MODE_OFB:
+ case STARFIVE_AES_MODE_CFB:
+ case STARFIVE_AES_MODE_CBC:
+ case STARFIVE_AES_MODE_CTR:
+ starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);
+ break;
+ default:
+ break;
+ }
+
+ return cryp->err;
+}
+
+static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp)
+{
+ int i, start_addr;
+
+ if (starfive_aes_wait_busy(cryp))
+ return dev_err_probe(cryp->dev, -ETIMEDOUT,
+ "Timeout waiting for tag generation.");
+
+ start_addr = STARFIVE_AES_NONCE0;
+
+ if (is_gcm(cryp))
+ for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4)
+ cryp->tag_out[i] = readl(cryp->base + start_addr);
+ else
+ for (i = 0; i < AES_BLOCK_32; i++)
+ cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
+
+ if (is_encrypt(cryp)) {
+ scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1);
+ } else {
+ scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0);
+
+ if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))
+ return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n");
+ }
+
+ return 0;
+}
+
+static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp)
+{
+ union starfive_aes_csr csr;
+ int err = cryp->err;
+
+ if (!err && cryp->authsize)
+ err = starfive_aes_read_authtag(cryp);
+
+ if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||
+ (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))
+ starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);
+
+ /* reset irq flags*/
+ csr.v = 0;
+ csr.aesrst = 1;
+ writel(csr.v, cryp->base + STARFIVE_AES_CSR);
+
+ if (cryp->authsize)
+ crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);
+ else
+ crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,
+ err);
+}
+
+void starfive_aes_done_task(unsigned long param)
+{
+ struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
+ u32 block[AES_BLOCK_32];
+ u32 stat;
+ int i;
+
+ for (i = 0; i < AES_BLOCK_32; i++)
+ block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
+
+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE,
+ cryp->total_out), 1);
+
+ cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out);
+
+ if (!cryp->total_out) {
+ starfive_aes_finish_req(cryp);
+ return;
+ }
+
+ memset(block, 0, AES_BLOCK_SIZE);
+ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
+ cryp->total_in), 0);
+ cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
+
+ for (i = 0; i < AES_BLOCK_32; i++)
+ writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
+
+ stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
+ stat &= ~STARFIVE_IE_MASK_AES_DONE;
+ writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
+}
+
+static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ struct starfive_cryp_request_ctx *rctx = ctx->rctx;
+ u32 *buffer;
+ int total_len, loop;
+
+ total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);
+ buffer = (u32 *)rctx->adata;
+
+ for (loop = 0; loop < total_len; loop += 4) {
+ writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);
+ buffer++;
+ writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);
+ buffer++;
+ writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);
+ buffer++;
+ writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);
+ buffer++;
+ }
+
+ if (starfive_aes_wait_gcmdone(cryp))
+ return dev_err_probe(cryp->dev, -ETIMEDOUT,
+ "Timeout processing gcm aad block");
+
+ return 0;
+}
+
+static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx)
+{
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ struct starfive_cryp_request_ctx *rctx = ctx->rctx;
+ u32 *buffer;
+ u8 *ci;
+ int total_len, loop;
+
+ total_len = cryp->assoclen;
+
+ ci = rctx->adata;
+ writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
+ ci++;
+ writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
+ ci++;
+ total_len -= 2;
+ buffer = (u32 *)ci;
+
+ for (loop = 0; loop < 3; loop++, buffer++)
+ writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
+
+ total_len -= 12;
+
+ while (total_len > 0) {
+ for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++)
+ writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
+
+ total_len -= AES_BLOCK_SIZE;
+ }
+
+ if (starfive_aes_wait_busy(cryp))
+ return dev_err_probe(cryp->dev, -ETIMEDOUT,
+ "Timeout processing ccm aad block");
+
+ return 0;
+}
+
+static int starfive_aes_prepare_req(struct skcipher_request *req,
+ struct aead_request *areq)
+{
+ struct starfive_cryp_ctx *ctx;
+ struct starfive_cryp_request_ctx *rctx;
+ struct starfive_cryp_dev *cryp;
+
+ if (!req && !areq)
+ return -EINVAL;
+
+ ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
+ crypto_aead_ctx(crypto_aead_reqtfm(areq));
+
+ cryp = ctx->cryp;
+ rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
+
+ if (req) {
+ cryp->req.sreq = req;
+ cryp->total_in = req->cryptlen;
+ cryp->total_out = req->cryptlen;
+ cryp->assoclen = 0;
+ cryp->authsize = 0;
+ } else {
+ cryp->req.areq = areq;
+ cryp->assoclen = areq->assoclen;
+ cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
+ if (is_encrypt(cryp)) {
+ cryp->total_in = areq->cryptlen;
+ cryp->total_out = areq->cryptlen;
+ } else {
+ cryp->total_in = areq->cryptlen - cryp->authsize;
+ cryp->total_out = cryp->total_in;
+ }
+ }
+
+ rctx->in_sg = req ? req->src : areq->src;
+ scatterwalk_start(&cryp->in_walk, rctx->in_sg);
+
+ rctx->out_sg = req ? req->dst : areq->dst;
+ scatterwalk_start(&cryp->out_walk, rctx->out_sg);
+
+ if (cryp->assoclen) {
+ rctx->adata = kzalloc(ALIGN(cryp->assoclen, AES_BLOCK_SIZE), GFP_KERNEL);
+ if (IS_ERR(rctx->adata))
+ return dev_err_probe(cryp->dev, PTR_ERR(rctx->adata),
+ "Failed to alloc memory for adata");
+
+ scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0);
+ scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2);
+ }
+
+ ctx->rctx = rctx;
+
+ return starfive_aes_hw_init(ctx);
+}
+
+static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct skcipher_request *req =
+ container_of(areq, struct skcipher_request, base);
+ struct starfive_cryp_ctx *ctx =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ u32 block[AES_BLOCK_32];
+ u32 stat;
+ int i;
+
+ /*
+ * Write first plain/ciphertext block to start the module
+ * then let irq tasklet handle the rest of the data blocks.
+ */
+ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
+ cryp->total_in), 0);
+ cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
+
+ for (i = 0; i < AES_BLOCK_32; i++)
+ writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
+
+ stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
+ stat &= ~STARFIVE_IE_MASK_AES_DONE;
+ writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
+
+ return 0;
+}
+
+static int starfive_aes_skcipher_prepare_req(struct crypto_engine *engine,
+ void *areq)
+{
+ struct skcipher_request *req =
+ container_of(areq, struct skcipher_request, base);
+
+ return starfive_aes_prepare_req(req, NULL);
+}
+
+static int starfive_aes_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ ctx->cryp = starfive_cryp_find_dev(ctx);
+ if (!ctx->cryp)
+ return -ENODEV;
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) +
+ sizeof(struct skcipher_request));
+
+ ctx->enginectx.op.do_one_request = starfive_aes_do_one_req;
+ ctx->enginectx.op.prepare_request = starfive_aes_skcipher_prepare_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void starfive_aes_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req =
+ container_of(areq, struct aead_request, base);
+ struct starfive_cryp_ctx *ctx =
+ crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ struct starfive_cryp_request_ctx *rctx = ctx->rctx;
+ u32 block[AES_BLOCK_32];
+ u32 stat;
+ int i;
+
+ if (!cryp->assoclen)
+ goto write_text;
+
+ if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)
+ cryp->err = starfive_aes_ccm_write_adata(ctx);
+ else
+ cryp->err = starfive_aes_gcm_write_adata(ctx);
+
+ kfree(rctx->adata);
+
+ if (cryp->err)
+ return cryp->err;
+
+write_text:
+ if (!cryp->total_in)
+ goto finish_req;
+
+ /*
+ * Write first plain/ciphertext block to start the module
+ * then let irq tasklet handle the rest of the data blocks.
+ */
+ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
+ cryp->total_in), 0);
+ cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
+
+ for (i = 0; i < AES_BLOCK_32; i++)
+ writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
+
+ stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
+ stat &= ~STARFIVE_IE_MASK_AES_DONE;
+ writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
+
+ return 0;
+
+finish_req:
+ starfive_aes_finish_req(cryp);
+ return 0;
+}
+
+static int starfive_aes_aead_prepare_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req =
+ container_of(areq, struct aead_request, base);
+
+ return starfive_aes_prepare_req(NULL, req);
+}
+
+static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm)
+{
+ struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ struct crypto_tfm *aead = crypto_aead_tfm(tfm);
+ struct crypto_alg *alg = aead->__crt_alg;
+
+ ctx->cryp = starfive_cryp_find_dev(ctx);
+ if (!ctx->cryp)
+ return -ENODEV;
+
+ if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
+ ctx->aead_fbk = crypto_alloc_aead(alg->cra_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(ctx->aead_fbk))
+ return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk),
+ "%s() failed to allocate fallback for %s\n",
+ __func__, alg->cra_name);
+ }
+
+ crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_ctx) +
+ sizeof(struct aead_request));
+
+ ctx->enginectx.op.do_one_request = starfive_aes_aead_do_one_req;
+ ctx->enginectx.op.prepare_request = starfive_aes_aead_prepare_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm)
+{
+ struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
+
+ if (ctx->aead_fbk) {
+ crypto_free_aead(ctx->aead_fbk);
+ ctx->aead_fbk = NULL;
+ }
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+ unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1;
+
+ cryp->flags = flags;
+
+ if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||
+ (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)
+ if (req->cryptlen & blocksize_align)
+ return -EINVAL;
+
+ return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
+}
+
+static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags)
+{
+ struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct starfive_cryp_dev *cryp = ctx->cryp;
+
+ cryp->flags = flags;
+
+ /*
+ * HW engine could not perform CCM tag verification on
+ * non-blocksize aligned text, use fallback algo instead
+ */
+ if (ctx->aead_fbk && !is_encrypt(cryp)) {
+ struct aead_request *subreq = aead_request_ctx(req);
+
+ aead_request_set_tfm(subreq, ctx->aead_fbk);
+ aead_request_set_callback(subreq, req->base.flags,
+ req->base.complete, req->base.data);
+ aead_request_set_crypt(subreq, req->src,
+ req->dst, req->cryptlen, req->iv);
+ aead_request_set_ad(subreq, req->assoclen);
+
+ return crypto_aead_decrypt(subreq);
+ }
+
+ return crypto_transfer_aead_request_to_engine(cryp->engine, req);
+}
+
+static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ if (!key || !keylen)
+ return -EINVAL;
+
+ if (keylen != AES_KEYSIZE_128 &&
+ keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
+
+ if (!key || !keylen)
+ return -EINVAL;
+
+ if (keylen != AES_KEYSIZE_128 &&
+ keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ if (ctx->aead_fbk)
+ return crypto_aead_setkey(ctx->aead_fbk, key, keylen);
+
+ return 0;
+}
+
+static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ return crypto_gcm_check_authsize(authsize);
+}
+
+static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
+
+ switch (authsize) {
+ case 4:
+ case 6:
+ case 8:
+ case 10:
+ case 12:
+ case 14:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return crypto_aead_setauthsize(ctx->aead_fbk, authsize);
+}
+
+static int starfive_aes_ecb_encrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT);
+}
+
+static int starfive_aes_ecb_decrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB);
+}
+
+static int starfive_aes_cbc_encrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT);
+}
+
+static int starfive_aes_cbc_decrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC);
+}
+
+static int starfive_aes_cfb_encrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB | FLG_ENCRYPT);
+}
+
+static int starfive_aes_cfb_decrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB);
+}
+
+static int starfive_aes_ofb_encrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB | FLG_ENCRYPT);
+}
+
+static int starfive_aes_ofb_decrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB);
+}
+
+static int starfive_aes_ctr_encrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT);
+}
+
+static int starfive_aes_ctr_decrypt(struct skcipher_request *req)
+{
+ return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR);
+}
+
+static int starfive_aes_gcm_encrypt(struct aead_request *req)
+{
+ return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT);
+}
+
+static int starfive_aes_gcm_decrypt(struct aead_request *req)
+{
+ return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM);
+}
+
+static int starfive_aes_ccm_encrypt(struct aead_request *req)
+{
+ int ret;
+
+ ret = starfive_aes_ccm_check_iv(req->iv);
+ if (ret)
+ return ret;
+
+ return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT);
+}
+
+static int starfive_aes_ccm_decrypt(struct aead_request *req)
+{
+ int ret;
+
+ ret = starfive_aes_ccm_check_iv(req->iv);
+ if (ret)
+ return ret;
+
+ return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM);
+}
+
+static struct skcipher_alg skcipher_algs[] = {
+{
+ .init = starfive_aes_init_tfm,
+ .exit = starfive_aes_exit_tfm,
+ .setkey = starfive_aes_setkey,
+ .encrypt = starfive_aes_ecb_encrypt,
+ .decrypt = starfive_aes_ecb_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .base = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "starfive-ecb-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .init = starfive_aes_init_tfm,
+ .exit = starfive_aes_exit_tfm,
+ .setkey = starfive_aes_setkey,
+ .encrypt = starfive_aes_cbc_encrypt,
+ .decrypt = starfive_aes_cbc_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "starfive-cbc-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .init = starfive_aes_init_tfm,
+ .exit = starfive_aes_exit_tfm,
+ .setkey = starfive_aes_setkey,
+ .encrypt = starfive_aes_ctr_encrypt,
+ .decrypt = starfive_aes_ctr_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "starfive-ctr-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .init = starfive_aes_init_tfm,
+ .exit = starfive_aes_exit_tfm,
+ .setkey = starfive_aes_setkey,
+ .encrypt = starfive_aes_cfb_encrypt,
+ .decrypt = starfive_aes_cfb_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "cfb(aes)",
+ .cra_driver_name = "starfive-cfb-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .init = starfive_aes_init_tfm,
+ .exit = starfive_aes_exit_tfm,
+ .setkey = starfive_aes_setkey,
+ .encrypt = starfive_aes_ofb_encrypt,
+ .decrypt = starfive_aes_ofb_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "ofb(aes)",
+ .cra_driver_name = "starfive-ofb-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+};
+
+static struct aead_alg aead_algs[] = {
+{
+ .setkey = starfive_aes_aead_setkey,
+ .setauthsize = starfive_aes_gcm_setauthsize,
+ .encrypt = starfive_aes_gcm_encrypt,
+ .decrypt = starfive_aes_gcm_decrypt,
+ .init = starfive_aes_aead_init_tfm,
+ .exit = starfive_aes_aead_exit_tfm,
+ .ivsize = GCM_AES_IV_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "starfive-gcm-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .setkey = starfive_aes_aead_setkey,
+ .setauthsize = starfive_aes_ccm_setauthsize,
+ .encrypt = starfive_aes_ccm_encrypt,
+ .decrypt = starfive_aes_ccm_decrypt,
+ .init = starfive_aes_aead_init_tfm,
+ .exit = starfive_aes_aead_exit_tfm,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "starfive-ccm-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+};
+
+int starfive_aes_register_algs(void)
+{
+ int ret;
+
+ ret = crypto_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
+ if (ret)
+ return ret;
+
+ ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
+ if (ret)
+ crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
+
+ return ret;
+}
+
+void starfive_aes_unregister_algs(void)
+{
+ crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
+ crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
+}
projects / platform / kernel / linux-rpi.git / commitdiff