From: william.qiu Date: Wed, 27 Apr 2022 11:18:11 +0000 (+0800) Subject: crypto:starfive:jh7110: add crypto driver for starfive jh7110 soc. X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=98d93a48993218ae0d9184028d54c5932c3b0207;p=platform%2Fkernel%2Flinux-starfive.git crypto:starfive:jh7110: add crypto driver for starfive jh7110 soc. add crypto driver for starfive jh7110 soc. Signed-off-by: william.qiu --- diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 4c2082d..53b0541 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -923,6 +923,8 @@ config CRYPTO_DEV_CCREE source "drivers/crypto/hisilicon/Kconfig" +source "drivers/crypto/starfive/Kconfig" + source "drivers/crypto/amlogic/Kconfig" config CRYPTO_DEV_SA2UL diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 1fe5120..d67b5bf 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += xilinx/ +obj-$(CONFIG_SOC_STARFIVE) += starfive/ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ obj-y += keembay/ diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig new file mode 100644 index 0000000..24c791b --- /dev/null +++ b/drivers/crypto/starfive/Kconfig @@ -0,0 +1,15 @@ +# select ARM_AMBA + +config CRYPTO_DEV_JH7110_ENCRYPT + tristate "Encryption acceleration support on jh7110" + depends on RISCV + select CRYPTO_ENGINE + select CRYPTO_RSA + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select AMBA_PL08X + default m + help + Support for SIFIVE JH7110 cryptographic acceleration instructions on riscv64 CPU. + This module supports acceleration for AES and GHASH in hardware. If you + choose 'M' here, this module will be called jh7110-crypto. diff --git a/drivers/crypto/starfive/Makefile b/drivers/crypto/starfive/Makefile new file mode 100644 index 0000000..9bdfd41 --- /dev/null +++ b/drivers/crypto/starfive/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += jh7110/ diff --git a/drivers/crypto/starfive/jh7110/Makefile b/drivers/crypto/starfive/jh7110/Makefile new file mode 100644 index 0000000..342384c --- /dev/null +++ b/drivers/crypto/starfive/jh7110/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_JH7110_ENCRYPT) += jh7110-crypto.o +jh7110-crypto-objs := jh7110-sec.o jh7110-sha.o jh7110-aes.o jh7110-des.o jh7110-pka.o diff --git a/drivers/crypto/starfive/jh7110/jh7110-aes.c b/drivers/crypto/starfive/jh7110/jh7110-aes.c new file mode 100644 index 0000000..4534060 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-aes.c @@ -0,0 +1,1669 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "jh7110-pl080.h" +#include "jh7110-str.h" + +/* Mode mask = bits [3..0] */ +#define FLG_MODE_MASK GENMASK(2, 0) + +/* Bit [4] encrypt / decrypt */ +#define FLG_ENCRYPT BIT(4) + +/* Bit [31..16] status */ +#define FLG_CCM_PADDED_WA BIT(5) + +#define SR_BUSY 0x00000010 +#define SR_OFNE 0x00000004 + +#define IMSCR_IN BIT(0) +#define IMSCR_OUT BIT(1) + +#define MISR_IN BIT(0) +#define MISR_OUT BIT(1) + +/* Misc */ +#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) +#define GCM_CTR_INIT 1 +#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset) +#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset) +#define CRYP_AUTOSUSPEND_DELAY 50 + +static inline int jh7110_aes_wait_busy(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status, + !(status & JH7110_AES_BUSY), 10, 100000); +} + +static inline int jh7110_aes_wait_keydone(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status, + (status & JH7110_AES_KEY_DONE), 10, 100000); +} + +static inline int jh7110_aes_wait_gcmdone(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status, + (status & JH7110_AES_GCM_DONE), 10, 100000); +} + +static inline int is_ecb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_ECB; +} + +static inline int is_cbc(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CBC; +} + +static inline int is_ofb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_OFB; +} + +static inline int is_cfb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CFB; +} + +static inline int is_ctr(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CTR; +} + +static inline int is_gcm(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_GCM; +} + +static inline int is_ccm(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CCM; +} + +static inline int get_aes_mode(struct jh7110_sec_request_ctx *rctx) +{ + return rctx->flags & FLG_MODE_MASK; +} + +static inline int is_encrypt(struct jh7110_sec_request_ctx *rctx) +{ + return !!(rctx->flags & FLG_ENCRYPT); +} + +static inline int is_decrypt(struct jh7110_sec_request_ctx *rctx) +{ + return !is_encrypt(rctx); +} + +static int jh7110_cryp_read_auth_tag(struct jh7110_sec_ctx *ctx); + +static inline void jh7110_aes_reset(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + rctx->csr.aes_csr.v = 0; + rctx->csr.aes_csr.aesrst = 1; + jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, rctx->csr.aes_csr.v); + +} + +static inline void jh7110_aes_xcm_start(struct jh7110_sec_ctx *ctx, u32 hw_mode) +{ + unsigned int value; + + switch (hw_mode) { + case JH7110_AES_MODE_GCM: + value = jh7110_sec_read(ctx->sdev, JH7110_AES_CSR); + value |= JH7110_AES_GCM_START; + jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, value); + break; + case JH7110_AES_MODE_CCM: + value = jh7110_sec_read(ctx->sdev, JH7110_AES_CSR); + value |= JH7110_AES_CCM_START; + jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, value); + break; + } +} + +static inline void jh7110_aes_csr_setup(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + rctx->csr.aes_csr.v = 0; + switch (ctx->keylen) { + case AES_KEYSIZE_128: + rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_128; + break; + case AES_KEYSIZE_192: + rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_192; + break; + case AES_KEYSIZE_256: + rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_256; + break; + default: + return; + } + rctx->csr.aes_csr.mode = rctx->flags & FLG_MODE_MASK; + rctx->csr.aes_csr.cmode = is_decrypt(rctx); + if (!ctx->sdev->use_dma || is_ccm(rctx) || is_gcm(rctx)) + rctx->csr.aes_csr.aesie = 1; + rctx->csr.aes_csr.stream_mode = rctx->stmode; + + if (jh7110_aes_wait_busy(ctx)) { + dev_err(ctx->sdev->dev, "reset error\n"); + return; + } + + jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, rctx->csr.aes_csr.v); +} + +static inline void jh7110_aes_set_ivlen(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + + jh7110_sec_write(sdev, JH7110_AES_IVLEN, 0x40); +} + +static inline void jh7110_aes_set_alen(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + + jh7110_sec_write(sdev, JH7110_AES_ALEN0, (ctx->rctx->assoclen >> 32) & 0xffffffff); + jh7110_sec_write(sdev, JH7110_AES_ALEN1, ctx->rctx->assoclen & 0xffffffff); +} + +static inline void jh7110_aes_set_mlen(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + size_t data_len = rctx->total_in - rctx->assoclen; + + jh7110_sec_write(sdev, JH7110_AES_MLEN0, (data_len >> 32) & 0xffffffff); + jh7110_sec_write(sdev, JH7110_AES_MLEN1, data_len & 0xffffffff); +} + +static void jh7110_cryp_hw_write_iv(struct jh7110_sec_ctx *ctx, u32 *iv) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + int loop; + + if (!iv) + return; + + for (loop = 0; loop < 4; loop++) + dev_dbg(sdev->dev, "iv[%d] = %x\n", loop, iv[loop]); + + jh7110_sec_write(sdev, JH7110_AES_IV0, iv[0]); + jh7110_sec_write(sdev, JH7110_AES_IV1, iv[1]); + jh7110_sec_write(sdev, JH7110_AES_IV2, iv[2]); + jh7110_sec_write(sdev, JH7110_AES_IV3, iv[3]); +} + +static void jh7110_cryp_hw_write_ctr(struct jh7110_sec_ctx *ctx, u32 *ctr) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + int loop; + + if (!ctr) + return; + + for (loop = 0; loop < 4; loop++) + dev_dbg(sdev->dev, "ctr[%d] = %x\n", loop, ctr[loop]); + + jh7110_sec_write(sdev, JH7110_AES_NONCE0, ctr[0]); + jh7110_sec_write(sdev, JH7110_AES_NONCE1, ctr[1]); + jh7110_sec_write(sdev, JH7110_AES_NONCE2, ctr[2]); + jh7110_sec_write(sdev, JH7110_AES_NONCE3, ctr[3]); +} + +static int jh7110_cryp_hw_write_key(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 *key = (u32 *)ctx->key; + + switch (ctx->keylen) { + case AES_KEYSIZE_256: + case AES_KEYSIZE_192: + case AES_KEYSIZE_128: + break; + default: + return -EINVAL; + } + + if (ctx->keylen >= AES_KEYSIZE_128) { + jh7110_sec_write(sdev, JH7110_AES_KEY0, key[0]); + jh7110_sec_write(sdev, JH7110_AES_KEY1, key[1]); + jh7110_sec_write(sdev, JH7110_AES_KEY2, key[2]); + jh7110_sec_write(sdev, JH7110_AES_KEY3, key[3]); + } + + if (ctx->keylen >= AES_KEYSIZE_192) { + jh7110_sec_write(sdev, JH7110_AES_KEY4, key[4]); + jh7110_sec_write(sdev, JH7110_AES_KEY5, key[5]); + } + + if (ctx->keylen >= AES_KEYSIZE_256) { + jh7110_sec_write(sdev, JH7110_AES_KEY6, key[6]); + jh7110_sec_write(sdev, JH7110_AES_KEY7, key[7]); + } + + if (jh7110_aes_wait_keydone(ctx)) + return -ETIMEDOUT; + + return 0; +} + +static unsigned int jh7110_cryp_get_input_text_len(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + return is_encrypt(rctx) ? rctx->req.areq->cryptlen : + rctx->req.areq->cryptlen - rctx->authsize; +} + +static int jh7110_cryp_gcm_init(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + /* Phase 1 : init */ + memcpy(rctx->last_ctr, rctx->req.areq->iv, 12); + rctx->last_ctr[3] = cpu_to_be32(GCM_CTR_INIT); + + jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->last_ctr); + return 0; +} + +static int jh7110_cryp_ccm_init(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + u8 iv[AES_BLOCK_SIZE], *b0; + unsigned int textlen; + + /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ + memcpy(iv, rctx->req.areq->iv, AES_BLOCK_SIZE); + memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); + + /* Build B0 */ + b0 = (u8 *)sdev->aes_data; + memcpy(b0, iv, AES_BLOCK_SIZE); + + b0[0] |= (8 * ((rctx->authsize - 2) / 2)); + + if (rctx->req.areq->assoclen) + b0[0] |= 0x40; + + textlen = jh7110_cryp_get_input_text_len(ctx); + + b0[AES_BLOCK_SIZE - 2] = textlen >> 8; + b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; + + memcpy((void *)rctx->last_ctr, sdev->aes_data, AES_BLOCK_SIZE); + jh7110_cryp_hw_write_ctr(ctx, (u32 *)b0); + + return 0; +} + +static int jh7110_cryp_hw_init(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + int ret; + u32 hw_mode; + int loop; + + jh7110_aes_reset(ctx); + + hw_mode = get_aes_mode(ctx->rctx); + if (hw_mode == JH7110_AES_MODE_CFB || + hw_mode == JH7110_AES_MODE_OFB) + rctx->stmode = JH7110_AES_MODE_XFB_128; + else + rctx->stmode = JH7110_AES_MODE_XFB_1; + + jh7110_aes_csr_setup(ctx); + + switch (hw_mode) { + case JH7110_AES_MODE_GCM: + memset(ctx->sdev->aes_data, 0, JH7110_MSG_BUFFER_SIZE); + jh7110_aes_set_alen(ctx); + jh7110_aes_set_mlen(ctx); + jh7110_aes_set_ivlen(ctx); + ret = jh7110_cryp_hw_write_key(ctx); + + jh7110_aes_xcm_start(ctx, hw_mode); + + if (jh7110_aes_wait_gcmdone(ctx)) + return -ETIMEDOUT; + + memset((void *)rctx->last_ctr, 0, sizeof(rctx->last_ctr)); + jh7110_cryp_gcm_init(ctx); + if (jh7110_aes_wait_gcmdone(ctx)) + return -ETIMEDOUT; + + break; + case JH7110_AES_MODE_CCM: + memset(ctx->sdev->aes_data, 0, JH7110_MSG_BUFFER_SIZE); + memset((void *)rctx->last_ctr, 0, sizeof(rctx->last_ctr)); + jh7110_aes_set_alen(ctx); + jh7110_aes_set_mlen(ctx); + jh7110_aes_set_ivlen(ctx); + /* Phase 1 : init */ + jh7110_cryp_ccm_init(ctx); + + ret = jh7110_cryp_hw_write_key(ctx); + + jh7110_aes_xcm_start(ctx, hw_mode); + + break; + case JH7110_AES_MODE_ECB: + ret = jh7110_cryp_hw_write_key(ctx); + break; + case JH7110_AES_MODE_OFB: + if (ctx->sec_init) { + for (loop = 0; loop < JH7110_AES_IV_LEN / sizeof(unsigned int); loop++) + rctx->aes_iv[loop] = (rctx->msg_end[loop]) ^ (rctx->dec_end[loop]); + } else + memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN); + jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv); + + ret = jh7110_cryp_hw_write_key(ctx); + ctx->sec_init = 1; + break; + case JH7110_AES_MODE_CFB: + case JH7110_AES_MODE_CBC: + if (ctx->sec_init) + memcpy((void *)rctx->aes_iv, (void *)rctx->dec_end, JH7110_AES_IV_LEN); + else + memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN); + jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv); + + ret = jh7110_cryp_hw_write_key(ctx); + ctx->sec_init = 1; + break; + case JH7110_AES_MODE_CTR: + if (ctx->sec_init) + memcpy((void *)rctx->aes_iv, (void *)rctx->dec_end, JH7110_AES_IV_LEN); + else + memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN); + jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv); + memcpy((void *)rctx->last_ctr, (void *)rctx->aes_iv, JH7110_AES_IV_LEN); + ret = jh7110_cryp_hw_write_key(ctx); + ctx->sec_init = 1; + break; + default: + break; + } + + return ret; +} + +static int jh7110_cryp_get_from_sg(struct jh7110_sec_request_ctx *rctx, size_t offset, + size_t count, size_t data_offset) +{ + size_t of, ct, index; + struct scatterlist *sg = rctx->sg; + + of = offset; + ct = count; + while (sg->length <= of) { + of -= sg->length; + + if (!sg_is_last(sg)) { + sg = sg_next(sg); + continue; + } else { + return -EBADE; + } + } + + index = data_offset; + while (ct > 0) { + if (sg->length - of >= ct) { + scatterwalk_map_and_copy(rctx->sdev->aes_data + index, sg, + of, ct, 0); + index = index + ct; + return index - data_offset; + } + scatterwalk_map_and_copy(rctx->sdev->aes_data + index, sg, + of, sg->length - of, 0); + index += sg->length - of; + ct = ct - (sg->length - of); + + of = 0; + + if (!sg_is_last(sg)) + sg = sg_next(sg); + else + return -EBADE; + } + return index - data_offset; +} + +static int jh7110_cryp_read_auth_tag(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + int loop; + int err = 0; + + if (jh7110_aes_wait_busy(ctx)) + return -EBUSY; + + for (loop = 0; loop < 3 + rctx->authsize / 4; loop++) { + rctx->aes_nonce[loop] = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_auth[%d] = %x\n", loop, rctx->aes_nonce[loop]); + } + + if (is_encrypt(rctx)) { + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), rctx->aes_nonce, + rctx->authsize, rctx->offset, 0); + } + + return err; +} + +static int jh7110_gcm_zero_message_data(struct jh7110_sec_ctx *ctx); + +static int jh7110_cryp_finish_req(struct jh7110_sec_ctx *ctx, int err) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + if (!err && (is_gcm(rctx) || is_ccm(rctx))) { + /* Phase 4 : output tag */ + err = jh7110_cryp_read_auth_tag(ctx); + } + + if (is_gcm(rctx) || is_ccm(rctx)) + crypto_finalize_aead_request(ctx->sdev->engine, rctx->req.areq, err); + else + crypto_finalize_skcipher_request(ctx->sdev->engine, rctx->req.sreq, + err); + + return err; +} + +static bool jh7110_check_counter_overflow(struct jh7110_sec_ctx *ctx, size_t count) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + bool ret = false; + u32 start, end, ctr, blocks; + + if (count) { + blocks = DIV_ROUND_UP(count, AES_BLOCK_SIZE); + rctx->last_ctr[3] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[3]) + blocks); + + if (rctx->last_ctr[3] == 0) { + rctx->last_ctr[2] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[2]) + 1); + if (rctx->last_ctr[2] == 0) { + rctx->last_ctr[1] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[1]) + 1); + if (rctx->last_ctr[1] == 0) { + rctx->last_ctr[0] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[0]) + 1); + if (rctx->last_ctr[1] == 0) + jh7110_cryp_hw_write_ctr(ctx, (u32 *)rctx->last_ctr); + } + } + } + } + + /* ctr counter overflow. */ + ctr = rctx->total_in - rctx->assoclen - rctx->authsize; + blocks = DIV_ROUND_UP(ctr, AES_BLOCK_SIZE); + start = be32_to_cpu(rctx->last_ctr[3]); + + end = start + blocks - 1; + if (end < start) { + rctx->ctr_over_count = AES_BLOCK_SIZE * -start; + ret = true; + } + + return ret; +} + +static void jh7110_aes_dma_callback(void *param) +{ + struct jh7110_sec_dev *sdev = param; + + complete(&sdev->sec_comp_p); +} + +static void vic_debug_dma_info(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + int loop; + + for (loop = 0; loop <= 0x34; loop += 4) + dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop)); + for (loop = 0x100; loop <= 0x110; loop += 4) + dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop)); + for (loop = 0x120; loop <= 0x130; loop += 4) + dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop)); +} + +static int jh7110_cryp_write_out_dma(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + struct dma_async_tx_descriptor *in_desc, *out_desc; + union jh7110_alg_cr alg_cr; + dma_cookie_t cookie; + unsigned int *out; + int total_len; + int err; + int loop; + + total_len = rctx->bufcnt; + + jh7110_sec_write(sdev, JH7110_DMA_IN_LEN_OFFSET, total_len); + jh7110_sec_write(sdev, JH7110_DMA_OUT_LEN_OFFSET, total_len); + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.aes_dma_en = 1; + jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v); + + total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len; + + sg_init_table(&ctx->sg[0], 1); + sg_set_buf(&ctx->sg[0], sdev->aes_data, total_len); + sg_dma_address(&ctx->sg[0]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->aes_data)); + sg_dma_len(&ctx->sg[0]) = total_len; + + sg_init_table(&ctx->sg[1], 1); + sg_set_buf(&ctx->sg[1], sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1), total_len); + sg_dma_address(&ctx->sg[1]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1))); + sg_dma_len(&ctx->sg[1]) = total_len; + + err = dma_map_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE); + if (!err) { + dev_err(sdev->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(sdev->dev, &ctx->sg[1], 1, DMA_FROM_DEVICE); + if (!err) { + dev_err(sdev->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + sdev->cfg_in.direction = DMA_MEM_TO_DEV; + sdev->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_in.src_maxburst = sdev->dma_maxburst; + sdev->cfg_in.dst_maxburst = sdev->dma_maxburst; + sdev->cfg_in.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET; + + dmaengine_slave_config(sdev->sec_xm_m, &sdev->cfg_in); + + sdev->cfg_out.direction = DMA_DEV_TO_MEM; + sdev->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_out.src_maxburst = 4; + sdev->cfg_out.dst_maxburst = 4; + sdev->cfg_out.src_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET; + + dmaengine_slave_config(sdev->sec_xm_p, &sdev->cfg_out); + + in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &ctx->sg[0], + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) + return -EINVAL; + + cookie = dmaengine_submit(in_desc); + dma_async_issue_pending(sdev->sec_xm_m); + + out_desc = dmaengine_prep_slave_sg(sdev->sec_xm_p, &ctx->sg[1], + 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) + return -EINVAL; + + reinit_completion(&sdev->sec_comp_p); + + out_desc->callback = jh7110_aes_dma_callback; + out_desc->callback_param = sdev; + + dmaengine_submit(out_desc); + dma_async_issue_pending(sdev->sec_xm_p); + + err = 0; + + if (!wait_for_completion_timeout(&sdev->sec_comp_p, + msecs_to_jiffies(10000))) { + vic_debug_dma_info(ctx); + dev_err(sdev->dev, "wait_for_completion_timeout out error cookie = %x\n", + dma_async_is_tx_complete(sdev->sec_xm_p, cookie, + NULL, NULL)); + } + + out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)); + + for (loop = 0; loop < total_len / 4; loop++) + dev_dbg(sdev->dev, "this is debug [%d] = %x\n", loop, out[loop]); + + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), out, + total_len, rctx->offset, 0); + + if (get_aes_mode(rctx) == JH7110_AES_MODE_CTR) { + rctx->dec_end[0] = jh7110_sec_read(sdev, JH7110_AES_IV0); + rctx->dec_end[1] = jh7110_sec_read(sdev, JH7110_AES_IV1); + rctx->dec_end[2] = jh7110_sec_read(sdev, JH7110_AES_IV2); + rctx->dec_end[3] = jh7110_sec_read(sdev, JH7110_AES_IV3); + } else + memcpy((void *)rctx->dec_end, ((void *)out) + total_len - JH7110_AES_IV_LEN, + JH7110_AES_IV_LEN); + + dma_unmap_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE); + dma_unmap_sg(sdev->dev, &ctx->sg[1], 1, DMA_FROM_DEVICE); + + alg_cr.v = 0; + alg_cr.clear = 1; + jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v); + + return 0; +} + +static int jh7110_cryp_write_out_cpu(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + unsigned int *buffer, *out; + unsigned char *ci, *co; + int total_len, mlen, loop, count; + + total_len = rctx->bufcnt; + buffer = (unsigned int *)sdev->aes_data; + out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)); + + while (total_len >= 16) { + for (loop = 0; loop < 4; loop++, buffer++) + jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer); + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + for (loop = 0; loop < 4; loop++, out++) { + *out = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_sec_read[%d] = %x\n", loop, *out); + } + total_len -= 16; + } + + if (total_len > 0) { + mlen = total_len; + for (; total_len >= 4; total_len -= 4, buffer++) + jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer); + + ci = (unsigned char *)buffer; + for (; total_len > 0; total_len--, ci++) + jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci); + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + for (; mlen >= 4; mlen -= 4, out++) { + *out = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_sec_read(mlen)[%d] = %x\n", loop, *out); + } + co = (unsigned char *)out; + for (; mlen > 0; mlen--, co++) { + *co = jh7110_sec_readb(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_sec_read(co)[%d] = %x\n", loop, *co); + } + } + + if (is_ccm(rctx)) { + if (jh7110_aes_wait_busy(ctx)) + return -EBUSY; + + for (loop = 0; loop < 3 + rctx->authsize / 4; loop++) { + rctx->aes_nonce[loop] = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_auth[%d] = %x\n", loop, rctx->aes_nonce[loop]); + } + } + + if (rctx->bufcnt >= rctx->total_out) + count = rctx->total_out; + else + count = rctx->bufcnt; + + out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)); + + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), out, + count, rctx->offset, 0); + + if (get_aes_mode(rctx) == JH7110_AES_MODE_CTR) { + rctx->dec_end[0] = jh7110_sec_read(sdev, JH7110_AES_IV0); + rctx->dec_end[1] = jh7110_sec_read(sdev, JH7110_AES_IV1); + rctx->dec_end[2] = jh7110_sec_read(sdev, JH7110_AES_IV2); + rctx->dec_end[3] = jh7110_sec_read(sdev, JH7110_AES_IV3); + } else + memcpy((void *)rctx->dec_end, ((void *)out) + count - JH7110_AES_IV_LEN, + JH7110_AES_IV_LEN); + + out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)); + + for (loop = 0; loop < rctx->bufcnt >> 2; loop++) + dev_dbg(sdev->dev, "aes_out[%d] = %x\n", loop, out[loop]); + + return 0; +} + +static int jh7110_cryp_write_data(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + size_t data_len, total, count, data_buf_len, data_offset; + int ret; + bool fragmented = false; + + if (unlikely(!rctx->total_in)) { + dev_warn(sdev->dev, "No more data to process\n"); + return -EINVAL; + } + + sdev->cry_type = JH7110_AES_TYPE; + + /* ctr counter overflow. */ + fragmented = jh7110_check_counter_overflow(ctx, 0); + + total = 0; + rctx->offset = 0; + rctx->data_offset = 0; + + if (is_ccm(rctx)) { + int index = AES_BLOCK_SIZE; + + if (rctx->assoclen <= 65280) { + ((u8 *)sdev->aes_data)[index] = (rctx->assoclen >> 8) & 0xff; + ((u8 *)sdev->aes_data)[index + 1] = rctx->assoclen & 0xff; + rctx->data_offset = index + 2; + } else { + ((u8 *)sdev->aes_data)[index] = 0xff; + ((u8 *)sdev->aes_data)[index + 1] = 0xfe; + ((u8 *)sdev->aes_data)[index + 2] = rctx->assoclen & 0xFF000000; + ((u8 *)sdev->aes_data)[index + 3] = rctx->assoclen & 0x00FF0000; + ((u8 *)sdev->aes_data)[index + 4] = rctx->assoclen & 0x0000FF00; + ((u8 *)sdev->aes_data)[index + 5] = rctx->assoclen & 0x000000FF; + rctx->data_offset = index + 6; + } + } + data_offset = rctx->data_offset; + while (total < rctx->total_in) { + data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen) - data_offset; + count = min(rctx->total_in - rctx->offset, data_buf_len); + + /* ctr counter overflow. */ + if (fragmented && rctx->ctr_over_count != 0) { + if (count >= rctx->ctr_over_count) + count = rctx->ctr_over_count; + } + data_len = jh7110_cryp_get_from_sg(rctx, rctx->offset, count, data_offset); + + if (data_len < 0) + return data_len; + if (data_len != count) + return -EINVAL; + + rctx->bufcnt = data_len + data_offset; + total += data_len; + + if (!is_encrypt(rctx) && (total + rctx->assoclen >= rctx->total_in)) + rctx->bufcnt = rctx->bufcnt - rctx->authsize; + + if (rctx->bufcnt) { + memcpy((void *)rctx->msg_end, (void *)sdev->aes_data + rctx->bufcnt - JH7110_AES_IV_LEN, + JH7110_AES_IV_LEN); + if (sdev->use_dma) + ret = jh7110_cryp_write_out_dma(ctx); + else + ret = jh7110_cryp_write_out_cpu(ctx); + + if (ret) + return ret; + } + data_offset = 0; + rctx->offset += data_len; + fragmented = jh7110_check_counter_overflow(ctx, data_len); + } + + return ret; +} + +static int jh7110_cryp_write_aad(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + unsigned int *buffer, *out; + unsigned char *ci; + int total_len, mlen, loop; + + total_len = rctx->bufcnt; + buffer = (unsigned int *)sdev->aes_data; + out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)); + + ci = (unsigned char *)buffer; + jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci); + ci++; + jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci); + ci++; + total_len -= 2; + buffer = (unsigned int *)ci; + for (loop = 0; loop < 3; loop++, buffer++) + jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer); + + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + total_len -= 12; + for (loop = 0; loop < 4; loop++, out++) { + unsigned int tmp; + + tmp = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_aad_read[%d] = %x\n", loop, tmp); + } + + while (total_len >= 16) { + for (loop = 0; loop < 4; loop++, buffer++) + jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer); + + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + for (loop = 0; loop < 4; loop++, out++) { + unsigned int tmp; + + tmp = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R); + dev_dbg(sdev->dev, "aes_aad_read[%d] = %x\n", loop, tmp); + } + total_len -= 16; + } + + if (total_len > 0) { + mlen = total_len; + for (; total_len >= 4; total_len -= 4, buffer++) + jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer); + + ci = (unsigned char *)buffer; + for (; total_len > 0; total_len--, ci++) + jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci); + + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + } + + if (jh7110_aes_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static int jh7110_cryp_xcm_write_data(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + size_t data_len, total, count, data_buf_len, offset, auths; + unsigned int *out; + int loop; + int ret; + bool fragmented = false; + + if (unlikely(!rctx->total_in)) { + dev_warn(sdev->dev, "No more data to process\n"); + return -EINVAL; + } + + sdev->cry_type = JH7110_AES_TYPE; + + /* ctr counter overflow. */ + fragmented = jh7110_check_counter_overflow(ctx, 0); + + total = 0; + rctx->offset = 0; + rctx->data_offset = 0; + offset = 0; + + while (total < rctx->assoclen) { + data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen); + count = min(rctx->assoclen - offset, data_buf_len); + count = min(count, rctx->assoclen - total); + data_len = jh7110_cryp_get_from_sg(rctx, offset, count, 0); + if (data_len < 0) + return data_len; + if (data_len != count) + return -EINVAL; + + offset += data_len; + rctx->offset += data_len; + if ((data_len + 2) & 0xF) { + memset(sdev->aes_data + data_len, 0, 16 - ((data_len + 2) & 0xf)); + data_len += 16 - ((data_len + 2) & 0xf); + } + + rctx->bufcnt = data_len; + + total += data_len; + ret = jh7110_cryp_write_aad(ctx); + } + + total = 0; + auths = 0; + + while (total + auths < rctx->total_in - rctx->assoclen) { + data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen); + count = min(rctx->total_in - rctx->offset, data_buf_len); + + if (is_encrypt(rctx)) + count = min(count, rctx->total_in - rctx->assoclen - total); + else { + //count = min (count, rctx->total_in - rctx->assoclen - total); + count = min(count, rctx->total_in - rctx->assoclen - total - rctx->authsize); + auths = rctx->authsize; + } + + /* ctr counter overflow. */ + if (fragmented && rctx->ctr_over_count != 0) { + if (count >= rctx->ctr_over_count) + count = rctx->ctr_over_count; + } + + data_len = jh7110_cryp_get_from_sg(rctx, rctx->offset, count, 0); + + if (data_len < 0) + return data_len; + if (data_len != count) + return -EINVAL; + + if (data_len % JH7110_AES_IV_LEN) { + memset(sdev->aes_data + data_len, 0, JH7110_AES_IV_LEN - (data_len % JH7110_AES_IV_LEN)); + data_len = data_len + (JH7110_AES_IV_LEN - (data_len % JH7110_AES_IV_LEN)); + } + + rctx->bufcnt = data_len; + total += data_len; + + if (!is_encrypt(rctx) && (total + rctx->assoclen >= rctx->total_in)) + rctx->bufcnt = rctx->bufcnt - rctx->authsize; + + if (rctx->bufcnt) { + memcpy((void *)rctx->msg_end, (void *)sdev->aes_data + rctx->bufcnt - JH7110_AES_IV_LEN, + JH7110_AES_IV_LEN); + out = (unsigned int *)sdev->aes_data; + for (loop = 0; loop < rctx->bufcnt / 4; loop++) + dev_dbg(sdev->dev, "aes_data[%d] = %x\n", loop, out[loop]); + ret = jh7110_cryp_write_out_cpu(ctx); + + if (ret) + return ret; + } + rctx->offset += count; + offset += count; + + fragmented = jh7110_check_counter_overflow(ctx, data_len); + } + + return ret; +} + +static int jh7110_gcm_zero_message_data(struct jh7110_sec_ctx *ctx) +{ + int ret; + + ctx->rctx->bufcnt = 0; + ctx->rctx->offset = 0; + if (ctx->sdev->use_dma) + ret = jh7110_cryp_write_out_dma(ctx); + else + ret = jh7110_cryp_write_out_cpu(ctx); + + return ret; +} + +static int jh7110_cryp_cpu_start(struct jh7110_sec_ctx *ctx, struct jh7110_sec_request_ctx *rctx) +{ + int ret; + + ret = jh7110_cryp_write_data(ctx); + if (ret) + return ret; + + ret = jh7110_cryp_finish_req(ctx, ret); + + return ret; +} + +static int jh7110_cryp_xcm_start(struct jh7110_sec_ctx *ctx, struct jh7110_sec_request_ctx *rctx) +{ + int ret; + + ret = jh7110_cryp_xcm_write_data(ctx); + if (ret) + return ret; + + ret = jh7110_cryp_finish_req(ctx, ret); + + return ret; +} + +static int jh7110_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); +static int jh7110_cryp_prepare_cipher_req(struct crypto_engine *engine, + void *areq); + +static int jh7110_cryp_cra_init(struct crypto_skcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->sdev = jh7110_sec_find_dev(ctx); + if (!ctx->sdev) + return -ENODEV; + + ctx->sec_init = 0; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx)); + + ctx->enginectx.op.do_one_request = jh7110_cryp_cipher_one_req; + ctx->enginectx.op.prepare_request = jh7110_cryp_prepare_cipher_req; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void jh7110_cryp_cra_exit(struct crypto_skcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->enginectx.op.do_one_request = NULL; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; +} + +static int jh7110_cryp_aead_one_req(struct crypto_engine *engine, void *areq); +static int jh7110_cryp_prepare_aead_req(struct crypto_engine *engine, + void *areq); + +static int jh7110_cryp_aes_aead_init(struct crypto_aead *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->sdev = jh7110_sec_find_dev(ctx); + + if (!ctx->sdev) + return -ENODEV; + + crypto_aead_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx)); + + ctx->enginectx.op.do_one_request = jh7110_cryp_aead_one_req; + ctx->enginectx.op.prepare_request = jh7110_cryp_prepare_aead_req; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void jh7110_cryp_aes_aead_exit(struct crypto_aead *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->enginectx.op.do_one_request = NULL; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; +} + +static int jh7110_cryp_crypt(struct skcipher_request *req, unsigned long flags) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req); + struct jh7110_sec_dev *sdev = ctx->sdev; + + if (!sdev) + return -ENODEV; + + rctx->flags = flags; + rctx->req_type = JH7110_ABLK_REQ; + + return crypto_transfer_skcipher_request_to_engine(sdev->engine, req); +} + +static int jh7110_cryp_aead_crypt(struct aead_request *req, unsigned long flags) +{ + struct jh7110_sec_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct jh7110_sec_request_ctx *rctx = aead_request_ctx(req); + struct jh7110_sec_dev *sdev = ctx->sdev; + + if (!sdev) + return -ENODEV; + + rctx->flags = flags; + rctx->req_type = JH7110_AEAD_REQ; + + return crypto_transfer_aead_request_to_engine(sdev->engine, req); +} + +static int jh7110_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + { + int loop; + + for (loop = 0; loop < keylen; loop++) + pr_debug("key[%d] = %x ctx->key[%d] = %x\n", loop, key[loop], loop, ctx->key[loop]); + } + return 0; +} + +static int jh7110_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + else + return jh7110_cryp_setkey(tfm, key, keylen); +} + +static int jh7110_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) { + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + { + int loop; + + for (loop = 0; loop < keylen; loop++) + pr_debug("key[%d] = %x ctx->key[%d] = %x\n", loop, key[loop], loop, ctx->key[loop]); + } + return 0; +} + +static int jh7110_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL; +} + +static int jh7110_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int jh7110_cryp_aes_ecb_encrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_ECB | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_ecb_decrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_ECB); +} + +static int jh7110_cryp_aes_cbc_encrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CBC | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_cbc_decrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CBC); +} + +static int jh7110_cryp_aes_cfb_encrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CFB | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_cfb_decrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CFB); +} + +static int jh7110_cryp_aes_ofb_encrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_OFB | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_ofb_decrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_OFB); +} + +static int jh7110_cryp_aes_ctr_encrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CTR | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_ctr_decrypt(struct skcipher_request *req) +{ + return jh7110_cryp_crypt(req, JH7110_AES_MODE_CTR); +} + +static int jh7110_cryp_aes_gcm_encrypt(struct aead_request *req) +{ + return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_GCM | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_gcm_decrypt(struct aead_request *req) +{ + return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_GCM); +} + +static int jh7110_cryp_aes_ccm_encrypt(struct aead_request *req) +{ + return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_CCM | FLG_ENCRYPT); +} + +static int jh7110_cryp_aes_ccm_decrypt(struct aead_request *req) +{ + return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_CCM); +} + +static int jh7110_cryp_prepare_req(struct skcipher_request *req, + struct aead_request *areq) +{ + struct jh7110_sec_ctx *ctx; + struct jh7110_sec_dev *sdev; + struct jh7110_sec_request_ctx *rctx; + int ret; + + if (!req && !areq) + return -EINVAL; + + ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : + crypto_aead_ctx(crypto_aead_reqtfm(areq)); + + sdev = ctx->sdev; + + if (!sdev) + return -ENODEV; + + rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); + + mutex_lock(&ctx->sdev->lock); + + rctx->sdev = sdev; + ctx->rctx = rctx; + + if (req) { + rctx->req.sreq = req; + rctx->req_type = JH7110_ABLK_REQ; + rctx->total_in = req->cryptlen; + rctx->total_out = rctx->total_in; + rctx->authsize = 0; + rctx->assoclen = 0; + } else { + /* + * Length of input and output data: + * Encryption case: + * INPUT = AssocData || PlainText + * <- assoclen -> <- cryptlen -> + * <------- total_in -----------> + * + * OUTPUT = AssocData || CipherText || AuthTag + * <- assoclen -> <- cryptlen -> <- authsize -> + * <---------------- total_out -----------------> + * + * Decryption case: + * INPUT = AssocData || CipherText || AuthTag + * <- assoclen -> <--------- cryptlen ---------> + * <- authsize -> + * <---------------- total_in ------------------> + * + * OUTPUT = AssocData || PlainText + * <- assoclen -> <- crypten - authsize -> + * <---------- total_out -----------------> + */ + rctx->req.areq = areq; + rctx->req_type = JH7110_AEAD_REQ; + rctx->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); + rctx->total_in = areq->assoclen + areq->cryptlen; + rctx->assoclen = areq->assoclen; + if (is_encrypt(rctx)) + /* Append auth tag to output */ + rctx->total_out = rctx->total_in + rctx->authsize; + else + /* No auth tag in output */ + rctx->total_out = rctx->total_in - rctx->authsize; + } + + rctx->sg = req ? req->src : areq->src; + rctx->out_sg = req ? req->dst : areq->dst; + + rctx->in_sg_len = sg_nents_for_len(rctx->sg, rctx->total_in); + if (rctx->in_sg_len < 0) { + dev_err(sdev->dev, "Cannot get in_sg_len\n"); + ret = rctx->in_sg_len; + goto out; + } + + rctx->out_sg_len = sg_nents_for_len(rctx->out_sg, rctx->total_out); + if (rctx->out_sg_len < 0) { + dev_err(sdev->dev, "Cannot get out_sg_len\n"); + ret = rctx->out_sg_len; + goto out; + } + if (1) { + int loop; + u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)]; + + scatterwalk_map_and_copy(idigest, rctx->req.areq->src, rctx->total_in - rctx->authsize, + rctx->authsize, 0); + for (loop = 0; loop < rctx->authsize / 4; loop++) { + dev_dbg(sdev->dev, "idigest[%d] = %x\n", + loop, idigest[loop]); + } + } + + ret = jh7110_cryp_hw_init(ctx); + if (ret) + goto out; + + return ret; +out: + mutex_unlock(&sdev->doing); + + return ret; +} + +static int jh7110_cryp_prepare_cipher_req(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = container_of(areq, + struct skcipher_request, + base); + return jh7110_cryp_prepare_req(req, NULL); +} + +static int jh7110_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, + struct skcipher_request, + base); + struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req); + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret; + + if (!sdev) + return -ENODEV; + + ret = jh7110_cryp_cpu_start(ctx, rctx); + + mutex_unlock(&sdev->lock); + + return ret; +} + +static int jh7110_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, + base); + return jh7110_cryp_prepare_req(NULL, req); +} + +static int jh7110_cryp_aead_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct jh7110_sec_request_ctx *rctx = aead_request_ctx(req); + struct jh7110_sec_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret; + + if (!sdev) + return -ENODEV; + + if (unlikely(!rctx->req.areq->assoclen && + !jh7110_cryp_get_input_text_len(ctx))) { + /* No input data to process: get tag and finish */ + jh7110_gcm_zero_message_data(ctx); + jh7110_cryp_finish_req(ctx, 0); + ret = 0; + goto out; + } + + ret = jh7110_cryp_xcm_start(ctx, rctx); + + out: + mutex_unlock(&ctx->sdev->lock); + + return ret; +} + +static struct skcipher_alg crypto_algs[] = { +{ + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "jh7110-ecb-aes", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = jh7110_cryp_cra_init, + .exit = jh7110_cryp_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = jh7110_cryp_aes_setkey, + .encrypt = jh7110_cryp_aes_ecb_encrypt, + .decrypt = jh7110_cryp_aes_ecb_decrypt, +}, +{ + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "jh7110-cbc-aes", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = jh7110_cryp_cra_init, + .exit = jh7110_cryp_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = jh7110_cryp_aes_setkey, + .encrypt = jh7110_cryp_aes_cbc_encrypt, + .decrypt = jh7110_cryp_aes_cbc_decrypt, +}, +{ + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "jh7110-ctr-aes", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = jh7110_cryp_cra_init, + .exit = jh7110_cryp_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = jh7110_cryp_aes_setkey, + .encrypt = jh7110_cryp_aes_ctr_encrypt, + .decrypt = jh7110_cryp_aes_ctr_decrypt, +}, +{ + .base.cra_name = "cfb(aes)", + .base.cra_driver_name = "jh7110-cfb-aes", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = jh7110_cryp_cra_init, + .exit = jh7110_cryp_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = jh7110_cryp_aes_setkey, + .encrypt = jh7110_cryp_aes_cfb_encrypt, + .decrypt = jh7110_cryp_aes_cfb_decrypt, +}, +{ + .base.cra_name = "ofb(aes)", + .base.cra_driver_name = "jh7110-ofb-aes", + .base.cra_priority = 200, + .base.cra_flags = CRYPTO_ALG_ASYNC, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = jh7110_cryp_cra_init, + .exit = jh7110_cryp_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = jh7110_cryp_aes_setkey, + .encrypt = jh7110_cryp_aes_ofb_encrypt, + .decrypt = jh7110_cryp_aes_ofb_decrypt, +}, +}; + +static struct aead_alg aead_algs[] = { +{ + .setkey = jh7110_cryp_aes_aead_setkey, + .setauthsize = jh7110_cryp_aes_gcm_setauthsize, + .encrypt = jh7110_cryp_aes_gcm_encrypt, + .decrypt = jh7110_cryp_aes_gcm_decrypt, + .init = jh7110_cryp_aes_aead_init, + .exit = jh7110_cryp_aes_aead_exit, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "jh7110-gcm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, +}, + +{ + .setkey = jh7110_cryp_aes_aead_setkey, + .setauthsize = jh7110_cryp_aes_ccm_setauthsize, + .encrypt = jh7110_cryp_aes_ccm_encrypt, + .decrypt = jh7110_cryp_aes_ccm_decrypt, + .init = jh7110_cryp_aes_aead_init, + .exit = jh7110_cryp_aes_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "jh7110-ccm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, +}, +}; + +int jh7110_aes_register_algs(void) +{ + int ret; + + ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); + if (ret) { + pr_err("Could not register algs\n"); + goto err_algs; + } + + ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ret) + goto err_aead_algs; + + return 0; + +err_aead_algs: + crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); +err_algs: + return ret; +} + +void jh7110_aes_unregister_algs(void) +{ + crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); +} diff --git a/drivers/crypto/starfive/jh7110/jh7110-des.c b/drivers/crypto/starfive/jh7110/jh7110-des.c new file mode 100644 index 0000000..a833e11 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-des.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "jh7110-pl080.h" +#include "jh7110-str.h" + +#define FLG_DES_MODE_MASK GENMASK(1, 0) + +#define FLAGS_DES_ENCRYPT BIT(4) + +enum { + JH7110_CRYPTO_NOT_ALIGNED = 1, + JH7110_CRYPTO_BAD_DATA_LENGTH, +}; + +#define JH7110_CRYPTO_DATA_COPIED BIT(0) +#define JH7110_CRYPTO_SG_COPIED BIT(1) + +#define JH7110_CRYPTO_COPY_MASK 0x3 + +#define JH7110_CRYPTO_COPY_DATA BIT(0) +#define JH7110_CRYPTO_FORCE_COPY BIT(1) +#define JH7110_CRYPTO_ZERO_BUF BIT(2) +#define JH7110_CRYPTO_FORCE_SINGLE_ENTRY BIT(3) + +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 + + +static inline int jh7110_des_wait_busy(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_DES_DAECSR_OFFSET, status, + !(status & JH7110_DES_BUSY), 10, 100000); +} + +static inline int jh7110_des_wait_done(struct jh7110_sec_dev *sdev) +{ + int ret = -1; + + mutex_lock(&sdev->doing); + + if (sdev->done_flags & JH7110_DES_DONE) + ret = 0; + + mutex_unlock(&sdev->doing); + + return ret; +} + +static inline int is_des_ecb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_ECB; +} + +static inline int is_des_cbc(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_CBC; +} + +static inline int is_des_ofb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_OFB; +} + +static inline int is_des_cfb(struct jh7110_sec_request_ctx *rctx) +{ + return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_CFB; +} + +static inline int get_des_mode(struct jh7110_sec_request_ctx *rctx) +{ + return rctx->flags & FLG_DES_MODE_MASK; +} + +static inline int is_des_encrypt(struct jh7110_sec_request_ctx *rctx) +{ + return !!(rctx->flags & FLAGS_DES_ENCRYPT); +} + +static inline int is_des_decrypt(struct jh7110_sec_request_ctx *rctx) +{ + return !is_des_encrypt(rctx); +} + +static inline void jh7110_des_reset(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + + rctx->csr.des_csr.v = 0; + rctx->csr.des_csr.reset = 1; + jh7110_sec_write(ctx->sdev, JH7110_DES_DAECSR_OFFSET, rctx->csr.des_csr.v); + +} + +static int jh7110_des_hw_write_key(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + u64 key64, *k; + int loop; + + key64 = ctx->keylen / sizeof(u64); + + switch (key64) { + case 1: + rctx->csr.des_csr.ks = 0x0; + break; + case 2: + rctx->csr.des_csr.ks = 0x2; + break; + case 3: + rctx->csr.des_csr.ks = 0x3; + break; + } + + jh7110_sec_write(ctx->sdev, JH7110_DES_DAECSR_OFFSET, rctx->csr.des_csr.v); + + k = (u64 *)ctx->key; + for (loop = 0; loop < key64; loop++) { + jh7110_sec_writeq(sdev, JH7110_DES_DAEKIN1R_HI_OFFSET + loop * 8, *k); + k++; + } + + return 0; +} + + + +static int jh7110_des_hw_init(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret; + u32 hw_mode; + + jh7110_des_reset(ctx); + + hw_mode = get_des_mode(ctx->rctx); + + rctx->csr.des_csr.v = 0; + rctx->csr.des_csr.mode = rctx->flags & FLG_DES_MODE_MASK; + rctx->csr.des_csr.encryt = !!(rctx->flags & FLAGS_DES_ENCRYPT); + rctx->csr.des_csr.ie = 1; + rctx->csr.des_csr.bitmode = JH7110_DES_BITMODE_64; + rctx->csr.des_csr.en = 1; + rctx->csr.des_csr.disturb = 1; + rctx->csr.des_csr.vdes_en = 1; + + switch (hw_mode) { + case JH7110_DES_MODE_ECB: + ret = jh7110_des_hw_write_key(ctx); + break; + case JH7110_DES_MODE_OFB: + case JH7110_DES_MODE_CFB: + case JH7110_DES_MODE_CBC: + jh7110_sec_writeq(sdev, JH7110_DES_DAEIVINR_HI_OFFSET, *(u64 *)rctx->req.sreq->iv); + break; + default: + break; + } + + return ret; +} + +static void jh7110_des_dma_callback(void *param) +{ + struct jh7110_sec_dev *sdev = param; + + complete(&sdev->sec_comp_p); +} + +static int jh7110_des_dma_start(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + struct dma_async_tx_descriptor *in_desc, *out_desc; + struct dma_slave_config cfg; + int err; + int ret; + + rctx->total = (rctx->total & 0x3) ? 1 : 0; + sg_init_table(&sdev->in_sg, 1); + sg_set_buf(&sdev->in_sg, sdev->des_data, rctx->total); + sg_dma_address(&sdev->in_sg) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->des_data)); + sg_dma_len(&sdev->in_sg) = rctx->total; + + sg_init_table(&sdev->out_sg, 1); + sg_set_buf(&sdev->out_sg, sdev->des_data, rctx->total); + sg_dma_address(&sdev->out_sg) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->des_data)); + sg_dma_len(&sdev->out_sg) = rctx->total; + + err = dma_map_sg(sdev->dev, &sdev->in_sg, 1, DMA_TO_DEVICE); + if (!err) { + dev_err(sdev->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(sdev->dev, &sdev->out_sg, 1, DMA_FROM_DEVICE); + if (!err) { + dev_err(sdev->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + + dma_sync_sg_for_device(sdev->dev, &sdev->in_sg, sdev->in_sg_len, DMA_TO_DEVICE); + + cfg.src_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET; + cfg.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET; + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 4; + cfg.dst_maxburst = 4; + + ret = dmaengine_slave_config(sdev->sec_xm_m, &cfg); + if (ret) { + dev_err(sdev->dev, "can't configure IN dmaengine slave: %d\n", + ret); + return ret; + } + + in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &sdev->in_sg, + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + + if (!in_desc) { + dev_err(sdev->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + in_desc->callback_param = sdev; + + ret = dmaengine_slave_config(sdev->sec_xm_p, &cfg); + if (ret) { + dev_err(sdev->dev, "can't configure OUT dmaengine slave: %d\n", + ret); + return ret; + } + + out_desc = dmaengine_prep_slave_sg(sdev->sec_xm_p, &sdev->out_sg, + 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) { + dev_err(sdev->dev, "OUT prep_slave_sg() failed\n"); + return -EINVAL; + } + + out_desc->callback = jh7110_des_dma_callback; + out_desc->callback_param = sdev; + + dmaengine_submit(in_desc); + dmaengine_submit(out_desc); + + dma_async_issue_pending(sdev->sec_xm_m); + dma_async_issue_pending(sdev->sec_xm_p); + + dma_unmap_sg(sdev->dev, &sdev->in_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(sdev->dev, &sdev->out_sg, 1, DMA_FROM_DEVICE); + + return 0; +} + +static int jh7110_des_cpu_start(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct sg_mapping_iter mi; + void *buffer; + int total_len, mlen, bs, offset; + u64 value; + u8 bvalue; + + total_len = rctx->total; + + sg_miter_start(&mi, rctx->in_sg, rctx->sg_len, + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + + bs = 8; + mlen = 0; + + while (total_len) { + switch (!(total_len >> 3)) { + case 0: + bs = 8; + break; + case 1: + bs = 1; + break; + } + if (!mlen) { + sg_miter_next(&mi); + mlen = mi.length; + if (!mlen) { + pr_err("sg miter failure.\n"); + return -EINVAL; + } + buffer = mi.addr; + } + + bs = min(bs, mlen); + + switch (bs) { + case 8: + jh7110_sec_writeq(sdev, JH7110_DES_DAEDINR_HI_OFFSET, + *(u64 *)buffer); + break; + case 1: + jh7110_sec_writeb(sdev, JH7110_DES_DAEDINR_HI_OFFSET, + *(u8 *)buffer); + break; + } + mlen -= bs; + total_len -= bs; + buffer += bs; + } + + sg_miter_stop(&mi); + + if (jh7110_des_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + if (jh7110_des_wait_done(sdev)) { + dev_err(sdev->dev, "jh7110_hash_wait_done error\n"); + return -ETIMEDOUT; + } + + total_len = rctx->total; + offset = 0; + + while (total_len) { + switch (!(total_len >> 3)) { + case 0: + value = jh7110_sec_readq(sdev, JH7110_DES_DAEDINR_HI_OFFSET); + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), (void *)&value, + sizeof(u64), offset, 0); + offset += sizeof(u64); + total_len -= 8; + break; + case 1: + bvalue = jh7110_sec_readb(sdev, JH7110_DES_DAEDINR_HI_OFFSET); + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), (void *)&bvalue, + sizeof(u8), offset, 0); + offset += sizeof(u8); + total_len -= 1; + break; + } + } + + return 0; +} + +static int jh7110_des_crypt_start(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret; + + if (sdev->use_dma) + ret = jh7110_des_dma_start(ctx); + else + ret = jh7110_des_cpu_start(ctx); + + return ret; +} + +static int jh7110_des_prepare_req(struct crypto_engine *engine, + void *areq); +static int jh7110_des_one_req(struct crypto_engine *engine, void *areq); + + +static int jh7110_des_cra_init(struct crypto_skcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->sdev = jh7110_sec_find_dev(ctx); + if (!ctx->sdev) + return -ENODEV; + + ctx->sec_init = 0; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx)); + + ctx->enginectx.op.do_one_request = jh7110_des_one_req; + ctx->enginectx.op.prepare_request = jh7110_des_prepare_req; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void jh7110_des_cra_exit(struct crypto_skcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + + ctx->enginectx.op.do_one_request = NULL; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; +} + +static int jh7110_des_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req); + struct jh7110_sec_dev *sdev; + + if (!req->cryptlen) + return 0; + + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) + return -EINVAL; + + sdev = jh7110_sec_find_dev(ctx); + if (!sdev) + return -ENODEV; + + rctx->flags = mode; + + return crypto_transfer_skcipher_request_to_engine(sdev->engine, req); +} + +static int jh7110_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des_key(tfm, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int jh7110_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des3_key(tfm, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int jh7110_des_ecb_encrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_ECB); +} + +static int jh7110_des_ecb_decrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, JH7110_DES_MODE_ECB); +} + +static int jh7110_des_cbc_encrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_CBC); +} + +static int jh7110_des_cbc_decrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, JH7110_DES_MODE_CBC); +} + +static int jh7110_des_cfb_encrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_CFB); +} + +static int jh7110_des_cfb_decrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, JH7110_DES_MODE_CFB); +} + +static int jh7110_des_ofb_encrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_OFB); +} + +static int jh7110_des_ofb_decrypt(struct skcipher_request *req) +{ + return jh7110_des_crypt(req, JH7110_DES_MODE_OFB); +} + + +static int jh7110_des_prepare_req(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct jh7110_sec_dev *sdev = jh7110_sec_find_dev(ctx); + struct jh7110_sec_request_ctx *rctx; + + if (!sdev) + return -ENODEV; + + rctx = skcipher_request_ctx(req); + + /* assign new request to device */ + rctx->req.sreq = req; + rctx->req_type = JH7110_ABLK_REQ; + rctx->total_in = req->cryptlen; + rctx->total_out = rctx->total_in; + rctx->authsize = 0; + rctx->assoclen = 0; + + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + + rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (rctx->in_sg_len < 0) + return rctx->in_sg_len; + + rctx->out_sg_len = sg_nents_for_len(rctx->out_sg, rctx->total); + if (rctx->out_sg_len < 0) + return rctx->out_sg_len; + + rctx = skcipher_request_ctx(req); + ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + rctx->ctx = ctx; + rctx->sdev = sdev; + ctx->sdev = sdev; + ctx->rctx = rctx; + + return jh7110_des_hw_init(ctx); +} + +static int jh7110_des_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, + struct skcipher_request, + base); + struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret; + + if (!sdev) + return -ENODEV; + + ret = jh7110_des_crypt_start(ctx); + + mutex_unlock(&sdev->lock); + + return ret; +} + +static struct skcipher_alg algs_des_jh7110[] = { + { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = jh7110_des_setkey, + .encrypt = jh7110_des_ecb_encrypt, + .decrypt = jh7110_des_ecb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des_setkey, + .encrypt = jh7110_des_cbc_encrypt, + .decrypt = jh7110_des_cbc_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "cfb(des)", + .base.cra_driver_name = "cfb-des-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des_setkey, + .encrypt = jh7110_des_cfb_encrypt, + .decrypt = jh7110_des_cfb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "ofb(des)", + .base.cra_driver_name = "ofb-des-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des_setkey, + .encrypt = jh7110_des_ofb_encrypt, + .decrypt = jh7110_des_ofb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "ecb(des3)", + .base.cra_driver_name = "ecb-des3-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = jh7110_des3_setkey, + .encrypt = jh7110_des_ecb_encrypt, + .decrypt = jh7110_des_ecb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "cbc(des3)", + .base.cra_driver_name = "cbc-des3-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des3_setkey, + .encrypt = jh7110_des_cbc_encrypt, + .decrypt = jh7110_des_cbc_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "cfb(des3)", + .base.cra_driver_name = "cfb-des3-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des3_setkey, + .encrypt = jh7110_des_cfb_encrypt, + .decrypt = jh7110_des_cfb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, + { + .base.cra_name = "ofb(des3)", + .base.cra_driver_name = "ofb-des3-jh7110", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = jh7110_des3_setkey, + .encrypt = jh7110_des_ofb_encrypt, + .decrypt = jh7110_des_ofb_decrypt, + .init = jh7110_des_cra_init, + .exit = jh7110_des_cra_exit, + }, +}; + +int jh7110_des_register_algs(void) +{ + int ret; + + ret = crypto_register_skciphers(algs_des_jh7110, ARRAY_SIZE(algs_des_jh7110)); + if (ret) { + pr_debug("Could not register algs\n"); + return ret; + } + return ret; +} + +void jh7110_des_unregister_algs(void) +{ + crypto_unregister_skciphers(algs_des_jh7110, ARRAY_SIZE(algs_des_jh7110)); +} + diff --git a/drivers/crypto/starfive/jh7110/jh7110-pka.c b/drivers/crypto/starfive/jh7110/jh7110-pka.c new file mode 100644 index 0000000..7c4b3eb --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-pka.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "jh7110-str.h" + +#define JH7110_RSA_KEYSZ_LEN (2048 >> 2) +#define JH7110_RSA_KEY_SIZE (JH7110_RSA_KEYSZ_LEN * 3) + +#define swap32(val) ( \ + (((u32)(val) << 24) & (u32)0xFF000000) | \ + (((u32)(val) << 8) & (u32)0x00FF0000) | \ + (((u32)(val) >> 8) & (u32)0x0000FF00) | \ + (((u32)(val) >> 24) & (u32)0x000000FF)) + +static inline int jh7110_pka_wait_pre(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_CRYPTO_CASR_OFFSET, status, + (status & JH7110_PKA_DONE_FLAGS), 10, 100000); +} + +static int jh7110_pka_wait_done(struct jh7110_sec_dev *sdev) +{ + int ret = -1; + + mutex_lock(&sdev->doing); + if (sdev->done_flags & JH7110_PKA_DONE) + ret = 0; + mutex_unlock(&sdev->doing); + + return ret; +} + +static void jh7110_rsa_free_key(struct jh7110_rsa_key *key) +{ + if (key->d) + kfree_sensitive(key->d); + if (key->e) + kfree_sensitive(key->e); + if (key->n) + kfree_sensitive(key->n); + memset(key, 0, sizeof(*key)); +} + +static unsigned int jh7110_rsa_get_nbit(u8 *pa, u32 snum, int key_sz) +{ + u32 i; + u8 value; + + i = snum >> 3; + + value = pa[key_sz - i - 1]; + value >>= snum & 0x7; + value &= 0x1; + + return value; +} + +static int jh7110_rsa_domain_transfer(struct jh7110_sec_ctx *ctx, u32 *result, u32 *opa, u8 domain, u32 *mod, int bit_len) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + unsigned int *info; + int loop; + u8 opsize; + u32 temp; + + mutex_lock(&sdev->doing); + + opsize = (bit_len - 1) >> 5; + rctx->csr.pka_csr.v = 0; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + info = (unsigned int *)mod; + for (loop = 0; loop <= opsize; loop++) + jh7110_sec_write(sdev, JH7110_CRYPTO_CANR_OFFSET + loop * 4, info[opsize - loop]); + + + if (domain != 0) { + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.start = 0x1; + rctx->csr.pka_csr.not_r2 = 0x1; + //rctx->csr.pka_csr.bigendian = 1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + + mutex_lock(&sdev->doing); + + info = (unsigned int *)opa; + for (loop = 0; loop <= opsize; loop++) + jh7110_sec_write(sdev, JH7110_CRYPTO_CAAR_OFFSET + loop * 4, info[opsize - loop]); + + for (loop = 0; loop <= opsize; loop++) { + if (loop == 0) + jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, 0x1000000); + else + jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, 0); + } + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka_csr.start = 0x1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + } else { + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.start = 0x1; + rctx->csr.pka_csr.pre_expf = 0x1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + + mutex_lock(&sdev->doing); + + info = (unsigned int *)opa; + for (loop = 0; loop <= opsize; loop++) + jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, info[opsize - loop]); + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_ARN; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.start = 0x1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + } + + mutex_lock(&sdev->doing); + for (loop = 0; loop <= opsize; loop++) { + temp = jh7110_sec_read(sdev, JH7110_CRYPTO_CAAR_OFFSET + 0x4 * loop); + result[opsize - loop] = temp; + } + mutex_unlock(&sdev->doing); + + return 0; +} + +static int jh7110_rsa_cpu_powm(struct jh7110_sec_ctx *ctx, u32 *result, u8 *de, u32 *n, int key_sz) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_rsa_key *key = &ctx->rsa_key; + u32 initial; + int opsize, mlen, bs, loop; + unsigned int *mta; + + opsize = (key_sz - 1) >> 2; + initial = 1; + + mta = kmalloc(key_sz, GFP_KERNEL); + if (!mta) + return -ENOMEM; + + jh7110_rsa_domain_transfer(ctx, mta, sdev->pka_data, 0, n, key_sz << 3); + + for (loop = 0; loop <= opsize; loop++) + jh7110_sec_write(sdev, JH7110_CRYPTO_CANR_OFFSET + loop * 4, n[opsize - loop]); + + mutex_lock(&sdev->doing); + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.not_r2 = 0x1; + rctx->csr.pka_csr.start = 0x1; + + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + + for (loop = 0; loop <= opsize; loop++) + jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, mta[opsize - loop]); + + for (loop = key->bitlen; loop > 0; loop--) { + if (initial) { + for (bs = 0; bs <= opsize; bs++) + result[bs] = mta[bs]; + + initial = 0; + } else { + mlen = jh7110_rsa_get_nbit(de, loop - 1, key_sz); + + mutex_lock(&sdev->doing); + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_AARN; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.start = 0x1; + + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + + if (mlen) { + mutex_lock(&sdev->doing); + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.cln_done = 1; + rctx->csr.pka_csr.opsize = opsize; + rctx->csr.pka_csr.exposize = opsize; + rctx->csr.pka_csr.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka_csr.ie = 1; + rctx->csr.pka_csr.start = 0x1; + + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + jh7110_pka_wait_done(sdev); + } + } + } + + mutex_lock(&sdev->doing); + for (loop = 0; loop <= opsize; loop++) { + unsigned int temp; + + temp = jh7110_sec_read(sdev, JH7110_CRYPTO_CAAR_OFFSET + 0x4 * loop); + result[opsize - loop] = temp; + } + mutex_unlock(&sdev->doing); + + jh7110_rsa_domain_transfer(ctx, result, result, 1, n, key_sz << 3); + + return 0; +} + +static int jh7110_rsa_powm(struct jh7110_sec_ctx *ctx, u8 *result, u8 *de, u8 *n, int key_sz) +{ + return jh7110_rsa_cpu_powm(ctx, (u32 *)result, de, (u32 *)n, key_sz); +} + +static int jh7110_rsa_get_from_sg(struct jh7110_sec_request_ctx *rctx, size_t offset, + size_t count, size_t data_offset) +{ + size_t of, ct, index; + struct scatterlist *sg = rctx->sg; + + of = offset; + ct = count; + + while (sg->length <= of) { + of -= sg->length; + + if (!sg_is_last(sg)) { + sg = sg_next(sg); + continue; + } else { + return -EBADE; + } + } + + index = data_offset; + while (ct > 0) { + if (sg->length - of >= ct) { + scatterwalk_map_and_copy(rctx->sdev->pka_data + index, sg, + of, ct, 0); + index = index + ct; + return index - data_offset; + } + scatterwalk_map_and_copy(rctx->sdev->pka_data + index, sg, + of, sg->length - of, 0); + index += sg->length - of; + ct = ct - (sg->length - of); + + of = 0; + + if (!sg_is_last(sg)) + sg = sg_next(sg); + else + return -EBADE; + } + return index - data_offset; +} + +static int jh7110_rsa_enc_core(struct jh7110_sec_ctx *ctx, int enc) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_rsa_key *key = &ctx->rsa_key; + size_t data_len, total, count, data_offset; + int ret = 0; + unsigned int *info; + int loop; + + sdev->cry_type = JH7110_PKA_TYPE; + + rctx->csr.pka_csr.v = 0; + rctx->csr.pka_csr.reset = 1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v); + + if (jh7110_pka_wait_pre(ctx)) + dev_dbg(sdev->dev, "this is debug for lophyel pka_casr = %x %s %s %d\n", + jh7110_sec_read(sdev, JH7110_CRYPTO_CASR_OFFSET), __FILE__, __func__, __LINE__); + + rctx->offset = 0; + total = 0; + + while (total < rctx->total_in) { + count = min(sdev->data_buf_len, rctx->total_in); + count = min(count, key->key_sz); + memset(sdev->pka_data, 0, key->key_sz); + data_offset = key->key_sz - count; + + data_len = jh7110_rsa_get_from_sg(rctx, rctx->offset, count, data_offset); + if (data_len < 0) + return data_len; + if (data_len != count) + return -EINVAL; + + if (enc) { + key->bitlen = key->e_bitlen; + ret = jh7110_rsa_powm(ctx, sdev->pka_data + JH7110_RSA_KEYSZ_LEN, key->e, key->n, key->key_sz); + } else { + key->bitlen = key->d_bitlen; + ret = jh7110_rsa_powm(ctx, sdev->pka_data + JH7110_RSA_KEYSZ_LEN, key->d, key->n, key->key_sz); + } + + + if (ret) + return ret; + + info = (unsigned int *)(sdev->pka_data + JH7110_RSA_KEYSZ_LEN); + for (loop = 0; loop < key->key_sz >> 2; loop++) + dev_dbg(sdev->dev, "result[%d] = %x\n", loop, info[loop]); + + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), sdev->pka_data + JH7110_RSA_KEYSZ_LEN, + key->key_sz, rctx->offset, 0); + + rctx->offset += data_len; + total += data_len; + } + + return ret; +} + +static int jh7110_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + struct jh7110_rsa_key *key = &ctx->rsa_key; + struct jh7110_sec_request_ctx *rctx = akcipher_request_ctx(req); + int ret = 0; + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->key_sz) { + req->dst_len = key->key_sz; + dev_err(ctx->sdev->dev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + mutex_lock(&ctx->sdev->rsa_lock); + + rctx->sg = req->src; + rctx->out_sg = req->dst; + rctx->sdev = ctx->sdev; + ctx->rctx = rctx; + rctx->total_in = req->src_len; + rctx->total_out = req->dst_len; + + ret = jh7110_rsa_enc_core(ctx, 1); + + mutex_unlock(&ctx->sdev->rsa_lock); + + return ret; +} + +static int jh7110_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + struct jh7110_rsa_key *key = &ctx->rsa_key; + struct jh7110_sec_request_ctx *rctx = akcipher_request_ctx(req); + int ret = 0; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->key_sz) { + req->dst_len = key->key_sz; + dev_err(ctx->sdev->dev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + mutex_lock(&ctx->sdev->rsa_lock); + + rctx->sg = req->src; + rctx->out_sg = req->dst; + rctx->sdev = ctx->sdev; + ctx->rctx = rctx; + rctx->total_in = req->src_len; + rctx->total_out = req->dst_len; + + ret = jh7110_rsa_enc_core(ctx, 0); + + mutex_unlock(&ctx->sdev->rsa_lock); + + return ret; +} + +static unsigned long jh7110_rsa_enc_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + if (bitslen & 0x1f) + return -EINVAL; + + if (bitslen < 32 || bitslen > 2048) + return -EINVAL; + + return 0; +} + +static int jh7110_rsa_set_n(struct jh7110_rsa_key *rsa_key, const char *value, + size_t vlen) +{ + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + rsa_key->key_sz = vlen; + ret = -EINVAL; + /* invalid key size provided */ + if (jh7110_rsa_enc_fn_id(rsa_key->key_sz)) + goto err; + + ret = -ENOMEM; + rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->n) + goto err; + + return 0; + err: + rsa_key->key_sz = 0; + rsa_key->n = NULL; + return ret; +} + +static int jh7110_rsa_set_e(struct jh7110_rsa_key *rsa_key, const char *value, + size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) { + rsa_key->e = NULL; + return -EINVAL; + } + + rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->e) + return -ENOMEM; + + for (loop = 8; loop > 0; loop--) { + if (pt >> (loop - 1)) + break; + } + + rsa_key->e_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; +} + +static int jh7110_rsa_set_d(struct jh7110_rsa_key *rsa_key, const char *value, + size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + ret = -EINVAL; + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) + goto err; + + ret = -ENOMEM; + rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->d) + goto err; + + for (loop = 8; loop > 0; loop--) { + pr_debug("this is debug for lophyel loop = %d pt >> (loop - 1) = %x value[%d] = %x %s %s %d\n", + loop, pt >> (loop - 1), loop, value[loop], __FILE__, __func__, __LINE__); + if (pt >> (loop - 1)) + break; + } + + rsa_key->d_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; + err: + rsa_key->d = NULL; + return ret; +} + +static int jh7110_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {NULL}; + struct jh7110_rsa_key *rsa_key = &ctx->rsa_key; + int ret; + + jh7110_rsa_free_key(rsa_key); + + if (private) + ret = rsa_parse_priv_key(&raw_key, key, keylen); + else + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret < 0) + goto err; + + ret = jh7110_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz); + if (ret < 0) + goto err; + ret = jh7110_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz); + if (ret < 0) + goto err; + + if (private) { + ret = jh7110_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz); + if (ret < 0) + goto err; + } + + if (!rsa_key->n || !rsa_key->e) { + /* invalid key provided */ + ret = -EINVAL; + goto err; + } + if (private && !rsa_key->d) { + /* invalid private key provided */ + ret = -EINVAL; + goto err; + } + + return 0; + err: + jh7110_rsa_free_key(rsa_key); + return ret; +} + +static int jh7110_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return jh7110_rsa_setkey(tfm, key, keylen, false); +} + +static int jh7110_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return jh7110_rsa_setkey(tfm, key, keylen, true); +} + +static unsigned int jh7110_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + + return ctx->rsa_key.key_sz; +} + +/* Per session pkc's driver context creation function */ +static int jh7110_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->sdev = jh7110_sec_find_dev(ctx); + if (!ctx->sdev) + return -ENODEV; + + akcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx)); + + return 0; +} + +/* Per session pkc's driver context cleanup function */ +static void jh7110_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm); + struct jh7110_rsa_key *key = (struct jh7110_rsa_key *)&ctx->rsa_key; + + jh7110_rsa_free_key(key); +} + +static struct akcipher_alg jh7110_rsa = { + .encrypt = jh7110_rsa_enc, + .decrypt = jh7110_rsa_dec, + .sign = jh7110_rsa_dec, + .verify = jh7110_rsa_enc, + .set_pub_key = jh7110_rsa_set_pub_key, + .set_priv_key = jh7110_rsa_set_priv_key, + .max_size = jh7110_rsa_max_size, + .init = jh7110_rsa_init_tfm, + .exit = jh7110_rsa_exit_tfm, + .reqsize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-jh7110", + .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + }, +}; + +int jh7110_pka_register_algs(void) +{ + int ret = 0; + + ret = crypto_register_akcipher(&jh7110_rsa); + if (ret) + pr_err("JH7110 RSA registration failed\n"); + + return ret; +} + +void jh7110_pka_unregister_algs(void) +{ + crypto_unregister_akcipher(&jh7110_rsa); +} diff --git a/drivers/crypto/starfive/jh7110/jh7110-pl080.c b/drivers/crypto/starfive/jh7110/jh7110-pl080.c new file mode 100644 index 0000000..4af53f6 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-pl080.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jh7110-pl080.h" +#include "jh7110-str.h" + +static volatile u32 g_dmac_done; +static volatile u32 g_dmac_err; +struct jh7110_pl080_lli_build_data *bd; + +/* Maximum times we call dma_pool_alloc on this pool without freeing */ +#define MAX_NUM_TSFR_LLIS 512 + +static inline int jh7110_dma_wait_done(struct jh7110_sec_dev *sdev, int chan) +{ + int ret = -1; + + mutex_lock(&sdev->pl080_doing); + if (g_dmac_done & BIT(chan)) + ret = 0; + mutex_unlock(&sdev->pl080_doing); + + return ret; +} + +static inline int jh7110_dmac_channel_wait_busy(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 status; + + return readl_relaxed_poll_timeout(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG, status, + !(status & PL080_CONFIG_ACTIVE), 10, 100000); +} + +static irqreturn_t jh7110_pl080_irq_thread(int irq, void *arg) +{ + struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg; + + pr_debug("this is debug mutex_unlock doing ---------------- %s %s %d\n", __FILE__, __func__, __LINE__); + mutex_unlock(&sdev->pl080_doing); + + return IRQ_HANDLED; +} + +static irqreturn_t jh7110_pl080_irq(int irq, void *dev) +{ + struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) dev; + u32 err, tc, i; + + /* check & clear - ERR & TC interrupts */ + err = readl_relaxed(sdev->dma_base + PL080_ERR_STATUS); + if (err) { + pr_err("%s error interrupt, register value 0x%08x\n", + __func__, err); + writel_relaxed(err, sdev->dma_base + PL080_ERR_CLEAR); + } + tc = readl_relaxed(sdev->dma_base + PL080_TC_STATUS); + if (tc) + writel_relaxed(tc, sdev->dma_base + PL080_TC_CLEAR); + + if (!err && !tc) + return IRQ_NONE; + + for (i = 0; i < PL080_CHANNELS_NUM; i++) { + if (BIT(i) & err) + g_dmac_err |= BIT(i); + if (BIT(i) & tc) + g_dmac_done |= BIT(i); + } + + return IRQ_WAKE_THREAD; +} + +static int jh7110_dmac_enable(struct jh7110_sec_dev *sdev) +{ + writel_relaxed(PL080_CONFIG_ENABLE, sdev->dma_base + PL080_CONFIG); + + return 0; +} + +static int jh7110_dmac_disable(struct jh7110_sec_dev *sdev) +{ + writel_relaxed(0, sdev->dma_base + PL080_CONFIG); + return 0; +} + +static int jh7110_dmac_channel_enable(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 control; + + control = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + control |= PL080_CONFIG_ENABLE; + writel_relaxed(control, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + + return 0; +} + +static int jh7110_dmac_channel_disable(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 control; + + control = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + control &= (~PL080_CONFIG_ENABLE); + writel_relaxed(control, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + + return 0; +} + +static int jh7110_dmac_channel_halt(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 val; + + val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + val |= PL080_CONFIG_HALT; + writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + + return 0; +} + +static int jh7110_dmac_channel_resume(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 val; + + val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + val &= (~PL080_CONFIG_HALT); + writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + + return 0; +} + +static void jh7110_dmac_interrupt_enable(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 val; + + val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + val |= PL080_CONFIG_TC_IRQ_MASK | PL080_CONFIG_ERR_IRQ_MASK; + writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); +} + +static void jh7110_dmac_interrupt_disable(struct jh7110_sec_dev *sdev, u8 chan) +{ + u32 val; + + val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + val &= ~(PL080_CONFIG_TC_IRQ_MASK | PL080_CONFIG_ERR_IRQ_MASK); + writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); +} + +static u32 jh7110_dmac_cctl_bits(struct jh7110_pl080_chan_config *config, u32 tsize) +{ + u32 cctl = 0; + + if (config->src_width == PL08X_BUS_WIDTH_16_BITS) + tsize >>= 1; + else if (config->src_width == PL08X_BUS_WIDTH_32_BITS) + tsize >>= 2; + + cctl = (1 << PL080_CONTROL_PROT_SYS) | (config->di<<27) | (config->si<<26) | (config->dst_ahb << 25) + | (config->src_ahb << 24) | (config->dst_width << 21) | (config->src_width << 18) | (config->dst_bsize<<15) + | (config->src_bsize<<12) | tsize; + return cctl; +} + +static int jh7110_dmac_fill_llis(struct jh7110_pl080_lli_build_data *bd, int num_llis, + struct jh7110_pl080_chan_config *config) +{ + u32 cctl; + u32 llis_addr = bd->llis_phy_addr; + struct jh7110_pl080_lli *llis = bd->llis; + + cctl = jh7110_dmac_cctl_bits(config, bd->tsize); + + llis[num_llis].control0 = cctl; + llis[num_llis].src_addr = bd->src_addr; + llis[num_llis].dst_addr = bd->dst_addr; + llis[num_llis].next_lli = llis_addr + (num_llis + 1) * sizeof(struct jh7110_pl080_lli); + + bd->remainder -= bd->tsize; + if (bd->remainder == 0) { + llis[num_llis].next_lli = 0; + llis[num_llis].control0 |= PL080_CONTROL_TC_IRQ_EN; + } + + return 0; +} + +static int jh7110_dmac_llis_prep(struct jh7110_pl080_lli_build_data *bd, struct jh7110_pl080_chan_config *config) +{ + u32 llisize; + u32 buswidth, width; + u32 max_bytes_per_lli, tsize; + u32 src_tsize, dst_tsize; + u32 num_llis = 0; + + bd->src_addr = config->src_addr; + bd->dst_addr = config->dst_addr; + bd->remainder = config->xfer_size; + + buswidth = min(config->src_width, config->dst_width); + switch (buswidth) { + case PL08X_BUS_WIDTH_8_BITS: + width = 1; + break; + case PL08X_BUS_WIDTH_16_BITS: + width = 2; + break; + case PL08X_BUS_WIDTH_32_BITS: + width = 4; + break; + default: + break; + } + max_bytes_per_lli = width * PL080_CONTROL_TRANSFER_SIZE_MASK; + tsize = max_bytes_per_lli; + llisize = bd->remainder % tsize; + if (llisize == 0) + llisize = bd->remainder / tsize; + else + llisize = (bd->remainder / tsize) + 1; + + if (llisize > MAX_NUM_TSFR_LLIS) + return -1; + + llisize += 1; + + bd->llis = kmalloc(llisize * sizeof(struct jh7110_pl080_lli), GFP_KERNEL); + if (bd->llis == NULL) + return -1; + memset(bd->llis, 0, llisize * sizeof(struct jh7110_pl080_lli)); + bd->llis_phy_addr = (u32)bd->llis; + + while (1) { + if (!bd->remainder) + break; + + if (bd->remainder > tsize) + bd->tsize = tsize; + else + bd->tsize = bd->remainder; + + jh7110_dmac_fill_llis(bd, num_llis++, config); + + if (config->si) + bd->src_addr += bd->tsize; + if (config->di) + bd->dst_addr += bd->tsize; + } + config->src_addr = bd->src_addr; + config->dst_addr = bd->dst_addr; + + return 0; +} + +/*dmac_setup_xfer support dma link item*/ +int jh7110_dmac_setup_xfer(struct jh7110_sec_dev *sdev, u8 chan, struct jh7110_pl080_chan_config *config) +{ + u32 val; + struct jh7110_pl080_lli *lli = NULL; + + bd = kmalloc(sizeof(struct jh7110_pl080_lli_build_data), GFP_KERNEL); + if (bd == NULL) + return -1; + + memset(bd, 0, sizeof(struct jh7110_pl080_lli_build_data)); + jh7110_dmac_llis_prep(bd, config); + lli = &bd->llis[0]; + + writel_relaxed(lli->src_addr, sdev->pl080->phy_chans[chan].base + PL080_CH_SRC_ADDR); + writel_relaxed(lli->dst_addr, sdev->pl080->phy_chans[chan].base + PL080_CH_DST_ADDR); + writel_relaxed(lli->next_lli, sdev->pl080->phy_chans[chan].base + PL080_CH_LLI); + writel_relaxed(lli->control0, sdev->pl080->phy_chans[chan].base + PL080_CH_CONTROL); + + val = (config->flow << 11)|(config->src_peri<<1)|(config->dst_peri<<6); + + writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG); + + jh7110_dmac_interrupt_enable(sdev, chan); + + g_dmac_done &= ~BIT(chan); + g_dmac_err &= ~BIT(chan); + + + if (chan) + mutex_lock(&sdev->pl080_doing); + + return 0; +} + +void jh7110_dmac_free_llis(void) +{ + if (bd) { + kfree(bd->llis); + kfree(bd); + bd = NULL; + } +} + +int jh7110_dmac_wait_done(struct jh7110_sec_dev *sdev, u8 chan) +{ + if (jh7110_dma_wait_done(sdev, chan)) { + pr_debug("this is debug for lophyel status = %x err = %x control0 = %x control1 = %x %s %s %d\n", + readl_relaxed(sdev->dma_base + PL080_TC_STATUS), readl_relaxed(sdev->dma_base + PL080_ERR_STATUS), + readl_relaxed(sdev->dma_base + 0x10c), readl_relaxed(sdev->dma_base + 0x12c), + __FILE__, __func__, __LINE__); + + return -1; + } + + g_dmac_done &= ~BIT(chan); + jh7110_dmac_free_llis(); + + return 0; +} + +int jh7110_dmac_secdata_in(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size) +{ + struct jh7110_pl080_chan_config config; + int ret; + + config.si = PL08X_INCREMENT; + config.di = PL08X_INCREMENT_FIX; + config.src_ahb = PL08X_AHB1; + config.dst_ahb = PL08X_AHB2; + config.src_width = PL08X_BUS_WIDTH_32_BITS; + config.dst_width = PL08X_BUS_WIDTH_32_BITS; + config.src_bsize = PL08X_BURST_SZ_32; + config.dst_bsize = PL08X_BURST_SZ_32; + config.src_peri = 1; + config.dst_peri = 1; + config.src_addr = src; + config.dst_addr = dst; + config.xfer_size = size; + config.flow = PL080_FLOW_MEM2PER; + + if (jh7110_dmac_channel_wait_busy(sdev, chan)) + return -ETIMEDOUT; + + ret = jh7110_dmac_setup_xfer(sdev, chan, &config); + if (ret != 0) + return ret; + + jh7110_dmac_channel_enable(sdev, chan); + + return 0; +} + +int jh7110_dmac_secdata_out(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size) +{ + struct jh7110_pl080_chan_config config; + int ret; + + config.si = PL08X_INCREMENT_FIX; + config.di = PL08X_INCREMENT; + config.src_ahb = PL08X_AHB2; + config.dst_ahb = PL08X_AHB1; + config.src_width = PL08X_BUS_WIDTH_32_BITS; + config.dst_width = PL08X_BUS_WIDTH_32_BITS; + config.src_bsize = PL08X_BURST_SZ_4;//follow hardware limit + config.dst_bsize = PL08X_BURST_SZ_4; + config.src_peri = 0; + config.dst_peri = 0; + config.src_addr = src; + config.dst_addr = dst; + config.xfer_size = size; + config.flow = PL080_FLOW_PER2MEM; + + if (jh7110_dmac_channel_wait_busy(sdev, chan)) + return -ETIMEDOUT; + + ret = jh7110_dmac_setup_xfer(sdev, chan, &config); + if (ret != 0) + return ret; + + jh7110_dmac_channel_enable(sdev, chan); + + return 0; +} + +int jh7110_dmac_init(struct jh7110_sec_dev *sdev, int irq) +{ + int chan; + int ret; + + ret = devm_request_threaded_irq(sdev->dev, irq, jh7110_pl080_irq, + jh7110_pl080_irq_thread, 0, + dev_name(sdev->dev), sdev); + if (ret) { + dev_err(sdev->dev, "Can't get interrupt working.\n"); + return ret; + } + + sdev->pl080 = kmalloc(sizeof(struct jh7110_pl08x_device), GFP_KERNEL); + if (!sdev->pl080) + return -ENOMEM; + + memset(sdev->pl080, 0, sizeof(struct jh7110_pl08x_device)); + + for (chan = 0; chan < PL080_CHANNELS_NUM; chan++) { + struct jh7110_pl08x_phy_chan *ch = &sdev->pl080->phy_chans[chan]; + + ch->id = chan; + ch->base = sdev->dma_base + PL080_Cx_BASE(chan); + ch->reg_config = ch->base + PL080_CH_CONFIG; + ch->reg_control = ch->base + PL080_CH_CONTROL; + ch->reg_src = ch->base + PL080_CH_SRC_ADDR; + ch->reg_dst = ch->base + PL080_CH_DST_ADDR; + ch->reg_lli = ch->base + PL080_CH_LLI; + } + + jh7110_dmac_enable(sdev); + g_dmac_done = 0; + g_dmac_err = 0; + + return 0; +} diff --git a/drivers/crypto/starfive/jh7110/jh7110-pl080.h b/drivers/crypto/starfive/jh7110/jh7110-pl080.h new file mode 100644 index 0000000..78a6f9f --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-pl080.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2021 StarFive, Inc + */ + + +#ifndef __JH7110_PL080_H__ +#define __JH7110_PL080_H__ + +#define PL080_INT_STATUS (0x00) +#define PL080_TC_STATUS (0x04) +#define PL080_TC_CLEAR (0x08) +#define PL080_ERR_STATUS (0x0C) +#define PL080_ERR_CLEAR (0x10) +#define PL080_RAW_TC_STATUS (0x14) +#define PL080_RAW_ERR_STATUS (0x18) +#define PL080_EN_CHAN (0x1c) +#define PL080_SOFT_BREQ (0x20) +#define PL080_SOFT_SREQ (0x24) +#define PL080_SOFT_LBREQ (0x28) +#define PL080_SOFT_LSREQ (0x2C) +#define PL080_CONFIG (0x30) +#define PL080_CONFIG_M2_BE BIT(2) +#define PL080_CONFIG_M1_BE BIT(1) +#define PL080_CONFIG_ENABLE BIT(0) +#define PL080_SYNC (0x34) + +/* Per channel configuration registers */ +#define PL080_Cx_BASE(x) ((0x100 + (x * 0x20))) +#define PL080_CH_SRC_ADDR (0x00) +#define PL080_CH_DST_ADDR (0x04) +#define PL080_CH_LLI (0x08) +#define PL080_CH_CONTROL (0x0C) +#define PL080_CH_CONFIG (0x10) +#define PL080S_CH_CONTROL2 (0x10) +#define PL080S_CH_CONFIG (0x14) + +#define PL080_LLI_ADDR_SHIFT (2) +#define PL080_CONTROL_PROT_SHIFT (28) +#define PL080_CONTROL_DWIDTH_SHIFT (21) +#define PL080_CONTROL_SWIDTH_SHIFT (18) +#define PL080_CONTROL_DB_SIZE_SHIFT (15) +#define PL080_CONTROL_SB_SIZE_SHIFT (12) +#define PL080_CONTROL_TRANSFER_SIZE_SHIFT (0) + +#define PL080_LLI_LM_AHB2 BIT(0) +#define PL080_CONTROL_TC_IRQ_EN BIT(31) +#define PL080_CONTROL_PROT_CACHE BIT(30) +#define PL080_CONTROL_PROT_BUFF BIT(29) +#define PL080_CONTROL_PROT_SYS BIT(28) +#define PL080_CONTROL_DST_INCR BIT(27) +#define PL080_CONTROL_SRC_INCR BIT(26) +#define PL080_CONTROL_DST_AHB2 BIT(25) +#define PL080_CONTROL_SRC_AHB2 BIT(24) + +#define PL080_CONTROL_TRANSFER_SIZE_MASK (0xfff<<0) +#define PL080N_CONFIG_ITPROT BIT(20) +#define PL080N_CONFIG_SECPROT BIT(19) +#define PL080_CONFIG_HALT BIT(18) +#define PL080_CONFIG_ACTIVE BIT(17) /* RO */ +#define PL080_CONFIG_LOCK BIT(16) +#define PL080_CONFIG_TC_IRQ_MASK BIT(15) +#define PL080_CONFIG_ERR_IRQ_MASK BIT(14) +#define PL080_CONFIG_ENABLE BIT(0) + +#define PL080_CONFIG_FLOW_CONTROL_SHIFT (11) +#define PL080_CONFIG_DST_SEL_SHIFT (6) +#define PL080_CONFIG_SRC_SEL_SHIFT (1) + +#define PL080_CONFIG_FLOW_CONTROL_MASK GENMASK(13, 11) +#define PL080_CONFIG_DST_SEL_MASK GENMASK(9, 6) +#define PL080_CONFIG_SRC_SEL_MASK GENMASK(4, 1) + + +#define PL080_CHANNELS_NUM (8) +#define PL080_SIGNAL_NUM (16) + +/* Bitmasks for selecting AHB ports for DMA transfers */ +enum jh7110_pl08x_ahb_master { + PL08X_AHB1 = (0 << 0), + PL08X_AHB2 = (1 << 0) +}; + +enum jh7110_pl08x_burst_size { + PL08X_BURST_SZ_1, + PL08X_BURST_SZ_4, + PL08X_BURST_SZ_8, + PL08X_BURST_SZ_16, + PL08X_BURST_SZ_32, + PL08X_BURST_SZ_64, + PL08X_BURST_SZ_128, + PL08X_BURST_SZ_256, +}; + +enum jh7110_pl08x_bus_width { + PL08X_BUS_WIDTH_8_BITS, + PL08X_BUS_WIDTH_16_BITS, + PL08X_BUS_WIDTH_32_BITS, +}; + +enum jh7110_pl08x_flow_control { + PL080_FLOW_MEM2MEM, + PL080_FLOW_MEM2PER, + PL080_FLOW_PER2MEM, + PL080_FLOW_SRC2DST, + PL080_FLOW_SRC2DST_DST, + PL080_FLOW_MEM2PER_PER, + PL080_FLOW_PER2MEM_PER, + PL080_FLOW_SRC2DST_SRC +}; + +enum jh7110_pl08x_increment { + PL08X_INCREMENT_FIX, + PL08X_INCREMENT, +}; + +struct jh7110_pl080_chan_config { + u8 src_peri; + u8 dst_peri; + u32 src_addr; + u32 dst_addr; + u32 xfer_size; + u32 data_scattered; + u32 src_distance; + u32 dst_distance; + enum jh7110_pl08x_increment si; + enum jh7110_pl08x_increment di; + enum jh7110_pl08x_ahb_master src_ahb; + enum jh7110_pl08x_ahb_master dst_ahb; + enum jh7110_pl08x_bus_width src_width; + enum jh7110_pl08x_bus_width dst_width; + enum jh7110_pl08x_burst_size src_bsize; + enum jh7110_pl08x_burst_size dst_bsize; + enum jh7110_pl08x_flow_control flow; +}; + +/* DMA linked list chain structure */ +struct jh7110_pl080_lli { + u32 src_addr; + u32 dst_addr; + u32 next_lli; + u32 control0; +}; + +struct jh7110_pl080_lli_build_data { + u32 src_addr; + u32 dst_addr; + struct jh7110_pl080_lli *llis; + u32 llis_phy_addr; + u32 tsize; + u32 remainder; +}; + +struct jh7110_pl08x_phy_chan { + unsigned int id; + void *base; + void *reg_config; + void *reg_control; + void *reg_src; + void *reg_dst; + void *reg_lli; +}; + +struct jh7110_pl08x_device { + struct jh7110_pl08x_phy_chan phy_chans[PL080_CHANNELS_NUM]; +}; +#endif diff --git a/drivers/crypto/starfive/jh7110/jh7110-regs.h b/drivers/crypto/starfive/jh7110/jh7110-regs.h new file mode 100644 index 0000000..574d071 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-regs.h @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2021 StarFive, Inc + */ +#ifndef __JH7110_REGS_H__ +#define __JH7110_REGS_H__ + +#include +#include +#include + +#define JH7110_ALG_CR_OFFSET 0x0 +#define JH7110_ALG_FIFO_OFFSET 0x4 +#define JH7110_IE_MASK_OFFSET 0x8 +#define JH7110_IE_FLAG_OFFSET 0xc +#define JH7110_DMA_IN_LEN_OFFSET 0x10 +#define JH7110_DMA_OUT_LEN_OFFSET 0x14 + +#define JH7110_AES_REGS_OFFSET 0x100 +#define JH7110_DES_REGS_OFFSET 0x200 +#define JH7110_SHA_REGS_OFFSET 0x300 +#define JH7110_CRYPTO_REGS_OFFSET 0x400 + +union jh7110_alg_cr { + u32 v; + struct { + u32 start :1; + u32 aes_dma_en :1; + u32 des_dma_en :1; + u32 sha_dma_en :1; + u32 alg_done :1; + u32 rsvd_0 :3; + u32 clear :1; + u32 rsvd_1 :23; + }; +}; + +union jh7110_ie_mask { + u32 v; + struct { + u32 aes_ie_mask :1; + u32 des_ie_mask :1; + u32 sha_ie_mask :1; + u32 crypto_ie_mask :1; + u32 rsvd_0 :28; + }; +}; + +union jh7110_ie_flag { + u32 v; + struct { + u32 aes_ie_done :1; + u32 des_ie_done :1; + u32 sha_ie_done :1; + u32 crypto_ie_done :1; + u32 rsvd_0 :28; + }; +}; + +#define JH7110_CRYPTO_CACR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x0) +#define JH7110_CRYPTO_CASR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x4) +#define JH7110_CRYPTO_CAAR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x8) +#define JH7110_CRYPTO_CAER_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x108) +#define JH7110_CRYPTO_CANR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x208) +#define JH7110_CRYPTO_CAAFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x308) +#define JH7110_CRYPTO_CAEFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x30c) +#define JH7110_CRYPTO_CANFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x310) +#define JH7110_FIFO_COUNTER_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x314) + +// R^2 mod N and N0' +#define CRYPTO_CMD_PRE 0x0 +// (A + A) mod N, ==> A +#define CRYPTO_CMD_AAN 0x1 +// A ^ E mod N ==> A +#define CRYPTO_CMD_AMEN 0x2 +// A + E mod N ==> A +#define CRYPTO_CMD_AAEN 0x3 +// A - E mod N ==> A +#define CRYPTO_CMD_ADEN 0x4 +// A * R mod N ==> A +#define CRYPTO_CMD_ARN 0x5 +// A * E * R mod N ==> A +#define CRYPTO_CMD_AERN 0x6 +// A * A * R mod N ==> A +#define CRYPTO_CMD_AARN 0x7 +// ECC2P ==> A +#define CRYPTO_CMD_ECC2P 0x8 +// ECCPQ ==> A +#define CRYPTO_CMD_ECCPQ 0x9 + +union jh7110_crypto_cacr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 rsvd_0 :1; + u32 fifo_mode :1; + u32 not_r2 :1; + u32 ecc_sub :1; + u32 pre_expf :1; + + u32 cmd :4; + u32 rsvd_1 :1; + u32 ctrl_dummy :1; + u32 ctrl_false :1; + u32 cln_done :1; + + u32 opsize :6; + u32 rsvd_2 :2; + + u32 exposize :6; + u32 rsvd_3 :1; + u32 bigendian :1; + }; +}; + +union jh7110_crypto_casr { + u32 v; + struct { +#define JH7110_PKA_DONE_FLAGS BIT(0) + u32 done :1; + u32 rsvd_0 :31; + }; +}; + +#define JH7110_DES_DAEDINR_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x0) +#define JH7110_DES_DAEDINR_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x4) +#define JH7110_DES_DAEKIN1R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x8) +#define JH7110_DES_DAEKIN1R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0xC) +#define JH7110_DES_DAEKIN2R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x10) +#define JH7110_DES_DAEKIN2R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x14) +#define JH7110_DES_DAEKIN3R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x18) +#define JH7110_DES_DAEKIN3R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x1C) +#define JH7110_DES_DAEIVINR_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x20) +#define JH7110_DES_DAEIVINR_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x24) +#define JH7110_DES_DAECSR_OFFSET (JH7110_DES_REGS_OFFSET + 0x28) +#define JH7110_DES_KEY0 (JH7110_DES_REGS_OFFSET + 0X2C) +#define JH7110_DES_KEY1 (JH7110_DES_REGS_OFFSET + 0X30) +#define JH7110_DES_IV0 (JH7110_DES_REGS_OFFSET + 0X34) +#define JH7110_DES_IV1 (JH7110_DES_REGS_OFFSET + 0X38) + +union jh7110_des_daecsr { + u32 v; + struct { +#define JH7110_DES_BUSY BIT(0) + u32 busy :1; + u32 en :1; + u32 encryt :1; + u32 reset :1; + u32 disturb :1; +#define JH7110_DES_KS_64 0x0 +#define JH7110_DES_KS_128 0x2 +#define JH7110_DES_KS_192 0x3 + u32 ks :2; + u32 vdes_en :1; + +#define JH7110_DES_MODE_ECB 0x0 +#define JH7110_DES_MODE_CBC 0x1 +#define JH7110_DES_MODE_CFB 0x2 +#define JH7110_DES_MODE_OFB 0x3 + u32 mode :2; + u32 ie :1; +#define JH7110_DES_CSR_DONE BIT(11) + u32 done :1; +#define JH7110_DES_BITMODE_1 0x0 +#define JH7110_DES_BITMODE_64 0x4 + u32 bitmode :3; + u32 rsvd_0 :17; + }; +}; + +#define JH7110_AES_AESDIO0R (JH7110_AES_REGS_OFFSET + 0x0) +#define JH7110_AES_KEY0 (JH7110_AES_REGS_OFFSET + 0x4) +#define JH7110_AES_KEY1 (JH7110_AES_REGS_OFFSET + 0x8) +#define JH7110_AES_KEY2 (JH7110_AES_REGS_OFFSET + 0xC) +#define JH7110_AES_KEY3 (JH7110_AES_REGS_OFFSET + 0x10) +#define JH7110_AES_KEY4 (JH7110_AES_REGS_OFFSET + 0x14) +#define JH7110_AES_KEY5 (JH7110_AES_REGS_OFFSET + 0x18) +#define JH7110_AES_KEY6 (JH7110_AES_REGS_OFFSET + 0x1C) +#define JH7110_AES_KEY7 (JH7110_AES_REGS_OFFSET + 0x20) +#define JH7110_AES_CSR (JH7110_AES_REGS_OFFSET + 0x24) +#define JH7110_AES_IV0 (JH7110_AES_REGS_OFFSET + 0x28) +#define JH7110_AES_IV1 (JH7110_AES_REGS_OFFSET + 0x2C) +#define JH7110_AES_IV2 (JH7110_AES_REGS_OFFSET + 0x30) +#define JH7110_AES_IV3 (JH7110_AES_REGS_OFFSET + 0x34) +#define JH7110_AES_NONCE0 (JH7110_AES_REGS_OFFSET + 0x3C) +#define JH7110_AES_NONCE1 (JH7110_AES_REGS_OFFSET + 0x40) +#define JH7110_AES_NONCE2 (JH7110_AES_REGS_OFFSET + 0x44) +#define JH7110_AES_NONCE3 (JH7110_AES_REGS_OFFSET + 0x48) +#define JH7110_AES_ALEN0 (JH7110_AES_REGS_OFFSET + 0x4C) +#define JH7110_AES_ALEN1 (JH7110_AES_REGS_OFFSET + 0x50) +#define JH7110_AES_MLEN0 (JH7110_AES_REGS_OFFSET + 0x54) +#define JH7110_AES_MLEN1 (JH7110_AES_REGS_OFFSET + 0x58) +#define JH7110_AES_IVLEN (JH7110_AES_REGS_OFFSET + 0x5C) + +union jh7110_aes_csr { + u32 v; + struct { + u32 cmode :1; +#define JH7110_AES_KEYMODE_128 0x0 +#define JH7110_AES_KEYMODE_192 0x1 +#define JH7110_AES_KEYMODE_256 0x2 + u32 keymode :2; +#define JH7110_AES_BUSY BIT(3) + u32 busy :1; + u32 done :1; +#define JH7110_AES_KEY_DONE BIT(5) + u32 krdy :1; + u32 aesrst :1; + u32 aesie :1; + +#define JH7110_AES_CCM_START BIT(8) + u32 ccm_start :1; +#define JH7110_AES_MODE_ECB 0x0 +#define JH7110_AES_MODE_CBC 0x1 +#define JH7110_AES_MODE_CFB 0x2 +#define JH7110_AES_MODE_OFB 0x3 +#define JH7110_AES_MODE_CTR 0x4 +#define JH7110_AES_MODE_CCM 0x5 +#define JH7110_AES_MODE_GCM 0x6 + u32 mode :3; +#define JH7110_AES_GCM_START BIT(12) + u32 gcm_start :1; +#define JH7110_AES_GCM_DONE BIT(13) + u32 gcm_done :1; + u32 delay_aes :1; + u32 vaes_start :1; + + u32 rsvd_0 :8; + +#define JH7110_AES_MODE_XFB_1 0x0 +#define JH7110_AES_MODE_XFB_128 0x5 + u32 stream_mode :3; + u32 rsvd_1 :5; + }; +}; + +#define JH7110_SHA_SHACSR (JH7110_SHA_REGS_OFFSET + 0x0) +#define JH7110_SHA_SHAWDR (JH7110_SHA_REGS_OFFSET + 0x4) +#define JH7110_SHA_SHARDR (JH7110_SHA_REGS_OFFSET + 0x8) +#define JH7110_SHA_SHAWSR (JH7110_SHA_REGS_OFFSET + 0xC) +#define JH7110_SHA_SHAWLEN3 (JH7110_SHA_REGS_OFFSET + 0x10) +#define JH7110_SHA_SHAWLEN2 (JH7110_SHA_REGS_OFFSET + 0x14) +#define JH7110_SHA_SHAWLEN1 (JH7110_SHA_REGS_OFFSET + 0x18) +#define JH7110_SHA_SHAWLEN0 (JH7110_SHA_REGS_OFFSET + 0x1C) +#define JH7110_SHA_SHAWKR (JH7110_SHA_REGS_OFFSET + 0x20) +#define JH7110_SHA_SHAWKLEN (JH7110_SHA_REGS_OFFSET + 0x24) + +union jh7110_sha_shacsr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 firstb :1; +#define JH7110_SHA_SM3 0x0 +#define JH7110_SHA_SHA0 0x1 +#define JH7110_SHA_SHA1 0x2 +#define JH7110_SHA_SHA224 0x3 +#define JH7110_SHA_SHA256 0x4 +#define JH7110_SHA_SHA384 0x5 +#define JH7110_SHA_SHA512 0x6 +#define JH7110_SHA_MODE_MASK 0x7 + u32 mode :3; + u32 rsvd_0 :1; + + u32 final :1; + u32 rsvd_1 :2; +#define JH7110_SHA_HMAC_FLAGS 0x800 + u32 hmac :1; + u32 rsvd_2 :1; + u32 key_done :1; + u32 key_flag :1; + u32 hmac_done :1; + +#define JH7110_SHA_BUSY BIT(16) + u32 busy :1; + u32 shadone :1; + u32 rsvd_3 :14; + }; +}; + +#endif diff --git a/drivers/crypto/starfive/jh7110/jh7110-sec.c b/drivers/crypto/starfive/jh7110/jh7110-sec.c new file mode 100644 index 0000000..7b10289 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-sec.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jh7110-str.h" + +#define DRIVER_NAME "jh7110-sec" + +struct jh7110_dev_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct jh7110_dev_list dev_list = { + .dev_list = LIST_HEAD_INIT(dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dev_list.lock), +}; + +struct jh7110_sec_dev *jh7110_sec_find_dev(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = NULL, *tmp; + + spin_lock_bh(&dev_list.lock); + if (!ctx->sdev) { + list_for_each_entry(tmp, &dev_list.dev_list, list) { + sdev = tmp; + break; + } + ctx->sdev = sdev; + } else { + sdev = ctx->sdev; + } + + spin_unlock_bh(&dev_list.lock); + + return sdev; +} + +static irqreturn_t jh7110_cryp_irq_thread(int irq, void *arg) +{ + struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg; + + if (sdev->use_dma) + if (sdev->cry_type != JH7110_PKA_TYPE) + return IRQ_HANDLED; + + mutex_unlock(&sdev->doing); + + return IRQ_HANDLED; +} + +static irqreturn_t jh7110_cryp_irq(int irq, void *arg) +{ + struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg; + union jh7110_sha_shacsr sha_csr; + union jh7110_aes_csr aes_csr; + union jh7110_des_daecsr des_csr; + union jh7110_crypto_cacr cry_cacr; + union jh7110_crypto_casr cry_casr; + irqreturn_t ret = IRQ_WAKE_THREAD; + + switch (sdev->cry_type) { + case JH7110_SHA_TYPE: + sha_csr.v = jh7110_sec_read(sdev, JH7110_SHA_SHACSR); + if (sha_csr.key_done) + sdev->done_flags |= JH7110_SHA_KEY_DONE; + if (sha_csr.hmac_done) + sdev->done_flags |= JH7110_SHA_HMAC_DONE; + if (sha_csr.shadone) + sdev->done_flags |= JH7110_SHA_SHA_DONE; + + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, sha_csr.v | BIT(17)); + break; + case JH7110_AES_TYPE: + aes_csr.v = jh7110_sec_read(sdev, JH7110_AES_CSR); + if (aes_csr.done) { + sdev->done_flags |= JH7110_AES_DONE; + jh7110_sec_write(sdev, JH7110_AES_CSR, aes_csr.v); + } + + break; + case JH7110_DES_TYPE: + des_csr.v = jh7110_sec_read(sdev, JH7110_DES_DAECSR_OFFSET); + if (des_csr.done) { + sdev->done_flags |= JH7110_DES_DONE; + jh7110_sec_write(sdev, JH7110_DES_DAECSR_OFFSET, des_csr.v); + } + + break; + case JH7110_PKA_TYPE: + cry_casr.v = jh7110_sec_read(sdev, JH7110_CRYPTO_CASR_OFFSET); + if (cry_casr.done) + sdev->done_flags |= JH7110_PKA_DONE_FLAGS; + cry_cacr.v = jh7110_sec_read(sdev, JH7110_CRYPTO_CACR_OFFSET); + cry_cacr.cln_done = 1; + jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, cry_cacr.v); + break; + default: + break; + } + return ret; +} + +static const struct of_device_id jh7110_dt_ids[] = { + { .compatible = "starfive,jh7110-sec", .data = NULL}, + {}, +}; +MODULE_DEVICE_TABLE(of, jh7110_dt_ids); + +static int jh7110_dma_init(struct jh7110_sec_dev *sdev) +{ + dma_cap_mask_t mask; + int err; + + sdev->sec_xm_m = NULL; + sdev->sec_xm_p = NULL; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + sdev->sec_xm_m = dma_request_chan(sdev->dev, "sec_m"); + if (IS_ERR(sdev->sec_xm_m)) { + dev_err(sdev->dev, "Unable to request sec_m dma channel in DMA channel\n"); + return PTR_ERR(sdev->sec_xm_m); + } + + sdev->sec_xm_p = dma_request_chan(sdev->dev, "sec_p"); + if (IS_ERR(sdev->sec_xm_p)) { + dev_err(sdev->dev, "Unable to request sec_p dma channel in DMA channel\n"); + goto err_sha_out; + } + + init_completion(&sdev->sec_comp_m); + init_completion(&sdev->sec_comp_p); + + return 0; + +err_sha_out: + dma_release_channel(sdev->sec_xm_m); + + return err; +} + +static void jh7110_dma_cleanup(struct jh7110_sec_dev *sdev) +{ + dma_release_channel(sdev->sec_xm_p); + dma_release_channel(sdev->sec_xm_m); +} +struct gpio_desc *gpio; + +static int jh7110_cryp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct jh7110_sec_dev *sdev; + struct resource *res; + int pages = 0; + int ret; + + sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + sdev->dev = dev; + + mutex_init(&sdev->lock); + mutex_init(&sdev->doing); + mutex_init(&sdev->pl080_doing); + mutex_init(&sdev->sha_lock); + mutex_init(&sdev->aes_lock); + mutex_init(&sdev->des_lock); + mutex_init(&sdev->rsa_lock); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "secreg"); + sdev->io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(sdev->io_base)) + return PTR_ERR(sdev->io_base); + + sdev->io_phys_base = res->start; + + sdev->dma_base = ioremap(0x16008000, 0x4000); + if (IS_ERR(sdev->dma_base)) + return PTR_ERR(sdev->dma_base); + + sdev->use_dma = device_property_read_bool(dev, "enable-dma"); + sdev->dma_maxburst = 32; + + sdev->secirq = platform_get_irq_byname(pdev, "secirq"); + sdev->secirq = platform_get_irq(pdev, 0); + if (sdev->secirq < 0) { + dev_err(dev, "Cannot get IRQ resource\n"); + return sdev->secirq; + } + + ret = devm_request_threaded_irq(dev, sdev->secirq, jh7110_cryp_irq, + jh7110_cryp_irq_thread, IRQF_SHARED, + dev_name(dev), sdev); + if (ret) { + dev_err(&pdev->dev, "Can't get interrupt working.\n"); + return ret; + } + + sdev->sec_hclk = devm_clk_get(dev, "sec_hclk"); + if (IS_ERR(sdev->sec_hclk)) { + dev_err(dev, "failed to get sec clock\n"); + return PTR_ERR(sdev->sec_hclk); + } + + sdev->sec_ahb = devm_clk_get(dev, "sec_ahb"); + if (IS_ERR(sdev->sec_ahb)) { + dev_err(dev, "failed to get sec clock\n"); + return PTR_ERR(sdev->sec_ahb); + } + + sdev->rst_hresetn = devm_reset_control_get_exclusive(sdev->dev, "sec_hre"); + if (IS_ERR(sdev->rst_hresetn)) { + dev_err(sdev->dev, "failed to get sec reset\n"); + return PTR_ERR(sdev->rst_hresetn); + } + + clk_prepare_enable(sdev->sec_hclk); + clk_prepare_enable(sdev->sec_ahb); + reset_control_deassert(sdev->rst_hresetn); + + platform_set_drvdata(pdev, sdev); + + spin_lock(&dev_list.lock); + list_add(&sdev->list, &dev_list.dev_list); + spin_unlock(&dev_list.lock); + + if (sdev->use_dma) { + ret = jh7110_dma_init(sdev); + if (ret) { + dev_err(dev, "Cannot initial dma chan\n"); + goto err_dma_init; + } + } + + pages = get_order(JH7110_MSG_BUFFER_SIZE); + + sdev->sha_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages); + if (!sdev->sha_data) { + dev_err(sdev->dev, "Can't allocate aes buffer pages when unaligned\n"); + goto err_sha_data; + } + + sdev->aes_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages); + if (!sdev->aes_data) { + dev_err(sdev->dev, "Can't allocate aes buffer pages when unaligned\n"); + goto err_aes_data; + } + + sdev->des_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages); + if (!sdev->des_data) { + dev_err(sdev->dev, "Can't allocate des buffer pages when unaligned\n"); + goto err_des_data; + } + + sdev->pka_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages); + if (!sdev->pka_data) { + dev_err(sdev->dev, "Can't allocate pka buffer pages when unaligned\n"); + goto err_pka_data; + } + + sdev->pages_count = pages >> 1; + sdev->data_buf_len = JH7110_MSG_BUFFER_SIZE >> 1; + + /* Initialize crypto engine */ + sdev->engine = crypto_engine_alloc_init(dev, 1); + if (!sdev->engine) { + ret = -ENOMEM; + goto err_engine; + } + + ret = crypto_engine_start(sdev->engine); + if (ret) + goto err_engine_start; + + ret = jh7110_hash_register_algs(); + if (ret) + goto err_algs_sha; + + ret = jh7110_aes_register_algs(); + if (ret) + goto err_algs_aes; + + ret = jh7110_des_register_algs(); + if (ret) + goto err_algs_des; + + ret = jh7110_pka_register_algs(); + if (ret) + goto err_algs_pka; + + dev_info(dev, "Initialized\n"); + + return 0; + err_algs_pka: + jh7110_des_unregister_algs(); + err_algs_des: + jh7110_aes_unregister_algs(); + err_algs_aes: + jh7110_hash_unregister_algs(); + err_algs_sha: + crypto_engine_stop(sdev->engine); + err_engine_start: + crypto_engine_exit(sdev->engine); + err_engine: + free_pages((unsigned long)sdev->pka_data, pages); + err_pka_data: + free_pages((unsigned long)sdev->des_data, pages); + err_des_data: + free_pages((unsigned long)sdev->aes_data, pages); + err_aes_data: + free_pages((unsigned long)sdev->sha_data, pages); + err_sha_data: + jh7110_dma_cleanup(sdev); + err_dma_init: + spin_lock(&dev_list.lock); + list_del(&sdev->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(sdev->sec_hclk); + clk_disable_unprepare(sdev->sec_ahb); + + return ret; +} + +static int jh7110_cryp_remove(struct platform_device *pdev) +{ + struct jh7110_sec_dev *sdev = platform_get_drvdata(pdev); + + if (!sdev) + return -ENODEV; + + jh7110_pka_unregister_algs(); + jh7110_des_unregister_algs(); + jh7110_aes_unregister_algs(); + jh7110_hash_unregister_algs(); + + crypto_engine_stop(sdev->engine); + crypto_engine_exit(sdev->engine); + + jh7110_dma_cleanup(sdev); + + free_pages((unsigned long)sdev->pka_data, sdev->pages_count); + free_pages((unsigned long)sdev->des_data, sdev->pages_count); + free_pages((unsigned long)sdev->aes_data, sdev->pages_count); + free_pages((unsigned long)sdev->sha_data, sdev->pages_count); + sdev->pka_data = NULL; + sdev->des_data = NULL; + sdev->aes_data = NULL; + sdev->sha_data = NULL; + + spin_lock(&dev_list.lock); + list_del(&sdev->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(sdev->sec_hclk); + clk_disable_unprepare(sdev->sec_ahb); + + return 0; +} + +#ifdef CONFIG_PM +static int jh7110_cryp_runtime_suspend(struct device *dev) +{ + struct jh7110_sec_dev *sdev = dev_get_drvdata(dev); + + clk_disable_unprepare(sdev->clk); + + return 0; +} + +static int jh7110_cryp_runtime_resume(struct device *dev) +{ + struct jh7110_sec_dev *sdev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(sdev->clk); + if (ret) { + dev_err(sdev->dev, "Failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops jh7110_cryp_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(jh7110_cryp_runtime_suspend, + jh7110_cryp_runtime_resume, NULL) +}; + +static struct platform_driver jh7110_cryp_driver = { + .probe = jh7110_cryp_probe, + .remove = jh7110_cryp_remove, + .driver = { + .name = DRIVER_NAME, + .pm = &jh7110_cryp_pm_ops, + .of_match_table = jh7110_dt_ids, + }, +}; + +module_platform_driver(jh7110_cryp_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Huan Feng "); +MODULE_DESCRIPTION("Starfive JH7110 CRYP DES SHA and AES driver"); diff --git a/drivers/crypto/starfive/jh7110/jh7110-sha.c b/drivers/crypto/starfive/jh7110/jh7110-sha.c new file mode 100644 index 0000000..6d0792c --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-sha.c @@ -0,0 +1,1260 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2021 StarFive, Inc + * + * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING + * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER + * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE + * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY + * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE + * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION + * WITH THEIR PRODUCTS. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "jh7110-pl080.h" +#include "jh7110-str.h" + +#define HASH_OP_UPDATE 1 +#define HASH_OP_FINAL 2 + +#define HASH_FLAGS_INIT BIT(0) +#define HASH_FLAGS_FINAL BIT(1) +#define HASH_FLAGS_FINUP BIT(2) + +#define JH7110_MAX_ALIGN_SIZE SHA512_BLOCK_SIZE + +#define JH7110_HASH_BUFLEN 8192 + +static inline int jh7110_hash_wait_done(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + int ret = -1; + + if (sdev->done_flags & (JH7110_SHA_KEY_DONE | JH7110_SHA_HMAC_DONE | JH7110_SHA_SHA_DONE)) + ret = 0; + return ret; +} + +static inline int jh7110_hash_wait_busy(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_dev *sdev = ctx->sdev; + u32 status; + + return readl_relaxed_poll_timeout(sdev->io_base + JH7110_SHA_SHACSR, status, + !(status & JH7110_SHA_BUSY), 10, 100000); +} + +static unsigned int jh7110_hash_reverse(unsigned int data) +{ + unsigned int ret; + + ret = (((data & 0x000000ff) << 24) | ((data & 0x0000ff00) << 8) | + ((data & 0x00ff0000) >> 8) | ((data & 0xff000000) >> 24)); + + return ret; +} + +static int jh7110_get_hash_size(struct jh7110_sec_ctx *ctx) +{ + unsigned int hashsize; + + switch (ctx->sha_mode & JH7110_SHA_MODE_MASK) { + case JH7110_SHA_SHA1: + hashsize = SHA1_DIGEST_SIZE; + break; + case JH7110_SHA_SHA224: + hashsize = SHA224_DIGEST_SIZE; + break; + case JH7110_SHA_SHA256: + hashsize = SHA256_DIGEST_SIZE; + break; + case JH7110_SHA_SHA384: + hashsize = SHA384_DIGEST_SIZE; + break; + case JH7110_SHA_SHA512: + hashsize = SHA512_DIGEST_SIZE; + break; + case JH7110_SHA_SM3: + hashsize = SM3_DIGEST_SIZE; + break; + default: + return 0; + } + return hashsize; +} + +static void jh7110_hash_start(struct jh7110_sec_ctx *ctx, int flags) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + + rctx->csr.sha_csr.v = jh7110_sec_read(sdev, JH7110_SHA_SHACSR); + rctx->csr.sha_csr.firstb = 0; + + if (flags) + rctx->csr.sha_csr.final = 1; + + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v); +} + +static void jh7110_sha_hmac_key(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + int klen = ctx->keylen, loop; + unsigned int *key_tmp; + + sdev->done_flags = 0; + sdev->cry_type = JH7110_SHA_TYPE; + + jh7110_sec_write(sdev, JH7110_SHA_SHAWKLEN, ctx->keylen); + rctx->csr.sha_csr.key_flag = 1; + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v); + + key_tmp = (unsigned int *)ctx->key; + + for (loop = 0; loop < klen / sizeof(unsigned int); loop++) + jh7110_sec_write(sdev, JH7110_SHA_SHAWKR, key_tmp[loop]); + + jh7110_hash_wait_done(ctx); +} + +static void jh7110_hash_write_back(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + unsigned int *data = (unsigned int *)rctx->sha_digest_mid; + int loop; + + for (loop = 0; loop < jh7110_get_hash_size(ctx) / sizeof(unsigned int); loop++) + jh7110_sec_write(sdev, JH7110_SHA_SHAWSR, data[loop]); +} + +static void jh7110_sha_dma_callback(void *param) +{ + struct jh7110_sec_dev *sdev = param; + + complete(&sdev->sec_comp_m); +} + +static int jh7110_hash_xmit_dma(struct jh7110_sec_ctx *ctx, int flags) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + struct dma_async_tx_descriptor *in_desc; + dma_cookie_t cookie; + union jh7110_alg_cr alg_cr; + int total_len; + int ret; + + if (!rctx->bufcnt) + return 0; + + ctx->sha_len_total += rctx->bufcnt; + + total_len = rctx->bufcnt; + + jh7110_sec_write(sdev, JH7110_DMA_IN_LEN_OFFSET, rctx->bufcnt); + + total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len; + + memset(sdev->sha_data + rctx->bufcnt, 0, total_len - rctx->bufcnt); + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.sha_dma_en = 1; + jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v); + + sg_init_table(&ctx->sg[0], 1); + sg_set_buf(&ctx->sg[0], sdev->sha_data, total_len); + sg_dma_address(&ctx->sg[0]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->sha_data)); + sg_dma_len(&ctx->sg[0]) = total_len; + + ret = dma_map_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE); + if (!ret) { + dev_err(sdev->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + sdev->cfg_in.direction = DMA_MEM_TO_DEV; + sdev->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sdev->cfg_in.src_maxburst = sdev->dma_maxburst; + sdev->cfg_in.dst_maxburst = sdev->dma_maxburst; + sdev->cfg_in.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET; + + dmaengine_slave_config(sdev->sec_xm_m, &sdev->cfg_in); + + in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &ctx->sg[0], + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) + return -EINVAL; + + reinit_completion(&sdev->sec_comp_m); + + in_desc->callback = jh7110_sha_dma_callback; + in_desc->callback_param = sdev; + + cookie = dmaengine_submit(in_desc); + dma_async_issue_pending(sdev->sec_xm_m); + + if (!wait_for_completion_timeout(&sdev->sec_comp_m, + msecs_to_jiffies(10000))) { + dev_dbg(sdev->dev, "this is debug for lophyel status = %x err = %x control0 = %x control1 = %x %s %s %d\n", + readl_relaxed(sdev->dma_base + PL080_TC_STATUS), readl_relaxed(sdev->dma_base + PL080_ERR_STATUS), + readl_relaxed(sdev->dma_base + 0x10c), readl_relaxed(sdev->dma_base + 0x12c), + __FILE__, __func__, __LINE__); + dev_err(sdev->dev, "wait_for_completion_timeout out error cookie = %x\n", + dma_async_is_tx_complete(sdev->sec_xm_p, cookie, + NULL, NULL)); + } + + dma_unmap_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE); + + alg_cr.v = 0; + alg_cr.clear = 1; + jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v); + + return 0; +} + +static int jh7110_hash_xmit_cpu(struct jh7110_sec_ctx *ctx, int flags) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + int total_len, mlen, loop; + unsigned int *buffer; + unsigned char *cl; + + if (!rctx->bufcnt) + return 0; + + ctx->sha_len_total += rctx->bufcnt; + + total_len = rctx->bufcnt; + mlen = total_len / sizeof(u32);// DIV_ROUND_UP(total_len, sizeof(u32)); + buffer = (unsigned int *)ctx->buffer; + for (loop = 0; loop < mlen; loop++, buffer++) + jh7110_sec_write(sdev, JH7110_SHA_SHAWDR, *buffer); + + if (total_len & 0x3) { + cl = (unsigned char *)buffer; + for (loop = 0; loop < (total_len & 0x3); loop++, cl++) + jh7110_sec_writeb(sdev, JH7110_SHA_SHAWDR, *cl); + } + + return 0; +} + +static void jh7110_hash_append_sg(struct jh7110_sec_request_ctx *rctx) +{ + struct jh7110_sec_ctx *ctx = rctx->ctx; + size_t count; + + while ((rctx->bufcnt < rctx->buflen) && rctx->total) { + count = min(rctx->in_sg->length - rctx->offset, rctx->total); + count = min(count, rctx->buflen - rctx->bufcnt); + + if (count <= 0) { + if ((rctx->in_sg->length == 0) && !sg_is_last(rctx->in_sg)) { + rctx->in_sg = sg_next(rctx->in_sg); + continue; + } else { + break; + } + } + + scatterwalk_map_and_copy(ctx->buffer + rctx->bufcnt, rctx->in_sg, + rctx->offset, count, 0); + + rctx->bufcnt += count; + rctx->offset += count; + rctx->total -= count; + + if (rctx->offset == rctx->in_sg->length) { + rctx->in_sg = sg_next(rctx->in_sg); + if (rctx->in_sg) + rctx->offset = 0; + else + rctx->total = 0; + } + } +} + +static int jh7110_hash_xmit(struct jh7110_sec_ctx *ctx, int flags) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + struct jh7110_sec_dev *sdev = ctx->sdev; + int count, *data, mlen; + int ret; + + mlen = jh7110_get_hash_size(ctx) / sizeof(unsigned int); + + sdev->cry_type = JH7110_SHA_TYPE; + + rctx->csr.sha_csr.v = 0; + rctx->csr.sha_csr.reset = 1; + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v); + + if (jh7110_hash_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + if (ctx->sha_mode & JH7110_SHA_HMAC_FLAGS) + jh7110_sha_hmac_key(ctx); + + rctx->csr.sha_csr.v = 0; + rctx->csr.sha_csr.start = 1; + rctx->csr.sha_csr.mode = ctx->sha_mode & JH7110_SHA_MODE_MASK; + + if (ctx->sec_init) { + rctx->csr.sha_csr.firstb = 1; + ctx->sec_init = 0; + } else { + jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN3, 0X00000000); + jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN2, 0X00000000); + jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN1, + jh7110_hash_reverse(ctx->sha_len_total >> 32)); + jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN0, + jh7110_hash_reverse(ctx->sha_len_total & 0xffffffff)); + jh7110_hash_write_back(ctx); + } + + rctx->csr.sha_csr.hmac = !!(ctx->sha_mode & JH7110_SHA_HMAC_FLAGS); + if (!ctx->sdev->use_dma) + rctx->csr.sha_csr.ie = 1; + + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v); + + if (jh7110_hash_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + if (ctx->sdev->use_dma) { + ret = jh7110_hash_xmit_dma(ctx, flags); + if (flags) + rctx->flags |= HASH_FLAGS_FINAL; + } else { + ret = jh7110_hash_xmit_cpu(ctx, flags); + if (flags) + rctx->flags |= HASH_FLAGS_FINAL; + } + + if (ret) + return ret; + + if (jh7110_hash_wait_busy(ctx)) { + dev_err(sdev->dev, "jh7110_hash_wait_busy error\n"); + return -ETIMEDOUT; + } + + jh7110_hash_start(ctx, flags); + + if (ctx->sha_mode & JH7110_SHA_HMAC_FLAGS) { + rctx->csr.sha_csr.final = 1; + jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v); + } + + if (jh7110_hash_wait_busy(ctx)) + dev_dbg(sdev->dev, "this is debug %s %s %d\n", __FILE__, __func__, __LINE__); + + if (!flags) { + data = (unsigned int *)rctx->sha_digest_mid; + for (count = 0; count < mlen; count++) + data[count] = jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR); + } + + return 0; +} + +static int jh7110_hash_update_req(struct jh7110_sec_ctx *ctx) +{ + struct jh7110_sec_request_ctx *rctx = ctx->rctx; + int err = 0, final; + + final = (rctx->flags & HASH_FLAGS_FINUP); + + while ((rctx->total >= rctx->buflen) || + (rctx->bufcnt + rctx->total >= rctx->buflen)) { + jh7110_hash_append_sg(rctx); + err = jh7110_hash_xmit(ctx, 0); + rctx->bufcnt = 0; + } + + jh7110_hash_append_sg(rctx); + + if (final) { + err = jh7110_hash_xmit(ctx, + (rctx->flags & HASH_FLAGS_FINUP)); + rctx->bufcnt = 0; + } + + return err; +} + +static int jh7110_hash_final_req(struct jh7110_sec_ctx *ctx) +{ + struct ahash_request *req = ctx->rctx->req.hreq; + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + int err = 0; + + err = jh7110_hash_xmit(ctx, 1); + rctx->bufcnt = 0; + + return err; +} + + +static int jh7110_hash_out_cpu(struct ahash_request *req) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + struct jh7110_sec_ctx *ctx = rctx->ctx; + int count, *data; + int mlen; + + if (!req->result) + return 0; + + mlen = jh7110_get_hash_size(ctx) / sizeof(u32); + + if (mlen == 28) + jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR); + + if (mlen == 48) { + for (count = 0; count < 4; count++) + jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR); + } + data = (u32 *)req->result; + for (count = 0; count < mlen; count++) + data[count] = jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR); + + return 0; +} + +static int jh7110_hash_copy_hash(struct ahash_request *req) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + struct jh7110_sec_ctx *ctx = rctx->ctx; + int hashsize; + int ret; + + hashsize = jh7110_get_hash_size(ctx); + + ret = jh7110_hash_out_cpu(req); + + if (ret) + return ret; + + memcpy(rctx->sha_digest_mid, req->result, hashsize); + rctx->sha_digest_len = hashsize; + + return ret; +} + +static void jh7110_hash_finish_req(struct ahash_request *req, int err) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + struct jh7110_sec_dev *sdev = rctx->sdev; + + if (!err && (HASH_FLAGS_FINAL & rctx->flags)) { + err = jh7110_hash_copy_hash(req); + rctx->flags &= ~(HASH_FLAGS_FINAL | + HASH_FLAGS_INIT); + } + + crypto_finalize_hash_request(sdev->engine, req, err); +} + +static int jh7110_hash_prepare_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx; + + if (!sdev) + return -ENODEV; + + mutex_lock(&ctx->sdev->lock); + + rctx = ahash_request_ctx(req); + + rctx->req.hreq = req; + + return 0; +} + +static int jh7110_hash_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct jh7110_sec_dev *sdev = ctx->sdev; + struct jh7110_sec_request_ctx *rctx; + int err = 0; + + if (!sdev) + return -ENODEV; + + rctx = ahash_request_ctx(req); + + if (rctx->op == HASH_OP_UPDATE) + err = jh7110_hash_update_req(ctx); + else if (rctx->op == HASH_OP_FINAL) + err = jh7110_hash_final_req(ctx); + + if (err != -EINPROGRESS) + /* done task will not finish it, so do it here */ + jh7110_hash_finish_req(req, err); + + mutex_unlock(&ctx->sdev->lock); + + return 0; +} + +static int jh7110_hash_handle_queue(struct jh7110_sec_dev *sdev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine(sdev->engine, req); +} + +static int jh7110_hash_enqueue(struct ahash_request *req, unsigned int op) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct jh7110_sec_dev *sdev = ctx->sdev; + + rctx->op = op; + + return jh7110_hash_handle_queue(sdev, req); +} + +static int jh7110_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + struct jh7110_sec_dev *sdev = ctx->sdev; + + memset(rctx, 0, sizeof(struct jh7110_sec_request_ctx)); + + rctx->sdev = sdev; + rctx->ctx = ctx; + rctx->req.hreq = req; + rctx->bufcnt = 0; + + rctx->total = 0; + rctx->offset = 0; + rctx->bufcnt = 0; + rctx->buflen = JH7110_HASH_BUFLEN; + + memset(ctx->buffer, 0, JH7110_HASH_BUFLEN); + + ctx->rctx = rctx; + + dev_dbg(sdev->dev, "%s Flags %lx\n", __func__, rctx->flags); + + return 0; +} + +static int jh7110_hash_update(struct ahash_request *req) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + rctx->total = req->nbytes; + rctx->in_sg = req->src; + rctx->offset = 0; + + if ((rctx->bufcnt + rctx->total < rctx->buflen)) { + jh7110_hash_append_sg(rctx); + return 0; + } + + return jh7110_hash_enqueue(req, HASH_OP_UPDATE); +} + +static int jh7110_hash_final(struct ahash_request *req) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + + rctx->flags |= HASH_FLAGS_FINUP; + + return jh7110_hash_enqueue(req, HASH_OP_FINAL); +} + +static int jh7110_hash_finup(struct ahash_request *req) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + int err1, err2; + int nents; + + nents = sg_nents_for_len(req->src, req->nbytes); + + rctx->flags |= HASH_FLAGS_FINUP; + + err1 = jh7110_hash_update(req); + + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources + * even if update() failed, except EINPROGRESS + */ + err2 = jh7110_hash_final(req); + + return err1 ?: err2; +} + +static int jh7110_hash_digest(struct ahash_request *req) +{ + return jh7110_hash_init(req) ?: jh7110_hash_finup(req); +} + +static int jh7110_hash_export(struct ahash_request *req, void *out) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int jh7110_hash_import(struct ahash_request *req, const void *in) +{ + struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static int jh7110_hash_cra_init_algs(struct crypto_tfm *tfm, + const char *algs_hmac_name, + unsigned int mode) +{ + struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->sdev = jh7110_sec_find_dev(ctx); + + if (!ctx->sdev) + return -ENODEV; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct jh7110_sec_request_ctx)); + + ctx->sec_init = 1; + ctx->keylen = 0; + ctx->sha_mode = mode; + ctx->sha_len_total = 0; + ctx->buffer = ctx->sdev->sha_data; + + if (algs_hmac_name) + ctx->sha_mode |= JH7110_SHA_HMAC_FLAGS; + + ctx->enginectx.op.do_one_request = jh7110_hash_one_request; + ctx->enginectx.op.prepare_request = jh7110_hash_prepare_req; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void jh7110_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->enginectx.op.do_one_request = NULL; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; +} + +static int jh7110_hash_long_setkey(struct jh7110_sec_ctx *ctx, + const u8 *key, unsigned int keylen, + const char *alg_name) +{ + struct crypto_wait wait; + struct ahash_request *req; + struct scatterlist sg; + struct crypto_ahash *ahash_tfm; + u8 *buf; + int ret; + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + buf = kzalloc(keylen + JH7110_MAX_ALIGN_SIZE, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_req; + } + + memcpy(buf, key, keylen); + sg_init_one(&sg, buf, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + +err_free_req: + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + return ret; +} + +static int jh7110_hash1_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha1"); + } + + return ret; +} + +static int jh7110_hash224_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha224"); + } + + return ret; +} + +static int jh7110_hash256_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha256"); + } + + return ret; +} + +static int jh7110_hash384_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha384"); + } + + return ret; +} + +static int jh7110_hash512_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha512"); + } + + return ret; +} + +static int jh7110_sm3_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int blocksize; + int ret = 0; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + ctx->keylen = digestsize; + ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sm3"); + } + + return ret; +} + +static int jh7110_hash_cra_sha1_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA1); +} + +static int jh7110_hash_cra_sha224_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA224); +} + +static int jh7110_hash_cra_sha256_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA256); +} + +static int jh7110_hash_cra_sha384_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA384); +} + +static int jh7110_hash_cra_sha512_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA512); +} + +static int jh7110_hash_cra_sm3_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SM3); +} + +static int jh7110_hash_cra_hmac_sha1_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sha1", JH7110_SHA_SHA1); +} + +static int jh7110_hash_cra_hmac_sha224_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sha224", JH7110_SHA_SHA224); +} + +static int jh7110_hash_cra_hmac_sha256_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sha256", JH7110_SHA_SHA256); +} + +static int jh7110_hash_cra_hmac_sha384_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sha384", JH7110_SHA_SHA384); +} + +static int jh7110_hash_cra_hmac_sha512_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sha512", JH7110_SHA_SHA512); +} + +static int jh7110_hash_cra_hmac_sm3_init(struct crypto_tfm *tfm) +{ + return jh7110_hash_cra_init_algs(tfm, "sm3", JH7110_SHA_SM3); +} + +static struct ahash_alg algs_sha0_sha512_sm3[] = { + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "jh7110-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sha1_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .setkey = jh7110_hash1_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "jh7110-hmac-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sha1_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "jh7110-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sha224_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .setkey = jh7110_hash224_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "jh7110-hmac-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sha224_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "jh7110-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sha256_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .setkey = jh7110_hash256_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "jh7110-hmac-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sha256_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "jh7110-sha384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sha384_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .setkey = jh7110_hash384_setkey, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "jh7110-hmac-sha384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sha384_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "jh7110-sha512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sha512_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .setkey = jh7110_hash512_setkey, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "jh7110-hmac-sha512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sha512_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "jh7110-sm3", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_sm3_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = jh7110_hash_init, + .update = jh7110_hash_update, + .final = jh7110_hash_final, + .finup = jh7110_hash_finup, + .digest = jh7110_hash_digest, + .setkey = jh7110_sm3_setkey, + .export = jh7110_hash_export, + .import = jh7110_hash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct jh7110_sec_request_ctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "jh7110-hmac-sm3", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct jh7110_sec_ctx), + .cra_alignmask = 3, + .cra_init = jh7110_hash_cra_hmac_sm3_init, + .cra_exit = jh7110_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, +}; + +int jh7110_hash_register_algs(void) +{ + int ret = 0; + + ret = crypto_register_ahashes(algs_sha0_sha512_sm3, ARRAY_SIZE(algs_sha0_sha512_sm3)); + + return ret; +} + +void jh7110_hash_unregister_algs(void) +{ + crypto_unregister_ahashes(algs_sha0_sha512_sm3, ARRAY_SIZE(algs_sha0_sha512_sm3)); +} + diff --git a/drivers/crypto/starfive/jh7110/jh7110-str.h b/drivers/crypto/starfive/jh7110/jh7110-str.h new file mode 100644 index 0000000..757665b6 --- /dev/null +++ b/drivers/crypto/starfive/jh7110/jh7110-str.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2021 StarFive, Inc + */ +#ifndef __JH7110_STR_H__ +#define __JH7110_STR_H__ + +#include +#include +#include +#include +#include +#include + +#include "jh7110-regs.h" + +#define JH7110_MSG_BUFFER_SIZE (16 * 1024) +#define MAX_KEY_SIZE SHA512_BLOCK_SIZE + +#define JH7110_AES_IV_LEN AES_BLOCK_SIZE +#define JH7110_AES_CTR_LEN AES_BLOCK_SIZE + + +struct jh7110_rsa_key { + u8 *n; + u8 *e; + u8 *d; + u8 *p; + u8 *q; + u8 *dp; + u8 *dq; + u8 *qinv; + u8 *rinv; + u8 *rinv_p; + u8 *rinv_q; + u8 *mp; + u8 *rsqr; + u8 *rsqr_p; + u8 *rsqr_q; + u8 *pmp; + u8 *qmp; + int e_bitlen; + int d_bitlen; + int bitlen; + size_t key_sz; + bool crt_mode; +}; + +struct jh7110_sec_ctx { + struct crypto_engine_ctx enginectx; + + struct jh7110_sec_request_ctx *rctx; + struct jh7110_sec_dev *sdev; + + unsigned int sha_mode; + + u8 key[MAX_KEY_SIZE]; + int keylen; + int sec_init; + struct scatterlist sg[2]; + struct jh7110_rsa_key rsa_key; + size_t sha_len_total; + u8 *buffer; +}; + +struct jh7110_sec_dev { + struct list_head list; + struct device *dev; + + struct clk *sec_hclk; + struct clk *sec_ahb; + struct reset_control *rst_hresetn; + + struct jh7110_pl08x_device *pl080; + + void __iomem *io_base; + void __iomem *dma_base; + phys_addr_t io_phys_base; + void *sha_data; + void *aes_data; + void *des_data; + void *pka_data; + unsigned int secirq; + unsigned int irq; + + size_t data_buf_len; + int pages_count; + u32 use_dma; + u32 dma_maxburst; + struct dma_chan *sec_xm_m; + struct dma_chan *sec_xm_p; + struct dma_slave_config cfg_in; + struct dma_slave_config cfg_out; + struct completion sec_comp_m; + struct completion sec_comp_p; + struct scatterlist in_sg; + struct scatterlist out_sg; + unsigned long in_sg_len; + unsigned long out_sg_len; + + + struct mutex doing; + struct mutex pl080_doing; + struct mutex lock; /* protects req / areq */ + struct mutex sha_lock; + struct mutex des_lock; + struct mutex aes_lock; + struct mutex rsa_lock; + +#define JH7110_SHA_KEY_DONE BIT(0) +#define JH7110_SHA_HMAC_DONE BIT(1) +#define JH7110_SHA_SHA_DONE BIT(2) +#define JH7110_AES_DONE BIT(3) +#define JH7110_DES_DONE BIT(4) +#define JH7110_PKA_DONE BIT(5) + u32 done_flags; +#define JH7110_SHA_TYPE 0x1 +#define JH7110_AES_TYPE 0x2 +#define JH7110_DES_TYPE 0x3 +#define JH7110_PKA_TYPE 0x4 + u32 cry_type; + + struct crypto_engine *engine; + + union jh7110_alg_cr alg_cr; + union jh7110_ie_mask ie_mask; + union jh7110_ie_flag ie_flag; +}; + +struct jh7110_sec_request_ctx { + struct jh7110_sec_ctx *ctx; + struct jh7110_sec_dev *sdev; + + union { + struct ahash_request *hreq; + struct skcipher_request *sreq; + struct aead_request *areq; + } req; + +#define JH7110_AHASH_REQ 0 +#define JH7110_ABLK_REQ 1 +#define JH7110_AEAD_REQ 2 + unsigned int req_type; + + union { + union jh7110_crypto_cacr pka_csr; + union jh7110_des_daecsr des_csr; + union jh7110_aes_csr aes_csr; + union jh7110_sha_shacsr sha_csr; + } csr; + + struct scatterlist *sg; + struct scatterlist *in_sg; + struct scatterlist *out_sg; + struct scatterlist in_sgl; + struct scatterlist out_sgl; + + unsigned long sg_len; + unsigned long in_sg_len; + unsigned long out_sg_len; + + unsigned long flags; + unsigned long op; + unsigned long stmode; + unsigned long long jiffies_hw; + unsigned long long jiffies_cp; + + size_t bufcnt; + size_t buflen; + size_t total; + size_t offset; + size_t data_offset; + size_t authsize; + size_t total_in; + size_t total_out; + size_t assoclen; + size_t ctr_over_count; + + u32 msg_end[4]; + u32 dec_end[4]; + u32 last_ctr[4]; + u32 aes_nonce[4]; + u32 aes_iv[4]; + u8 sha_digest_mid[SHA512_DIGEST_SIZE]__aligned(sizeof(u32)); + unsigned int sha_digest_len; +}; + +struct jh7110_sec_dma { + struct dma_slave_config cfg; + union jh7110_alg_cr alg_cr; + struct dma_chan *chan; + struct completion *dma_comp; + struct scatterlist *sg; + struct jh7110_sec_ctx *ctx; + void *data; + size_t total; +}; + +static inline u64 jh7110_sec_readq(struct jh7110_sec_dev *sdev, u32 offset) +{ +#ifdef CONFIG_64BIT + return __raw_readq(sdev->io_base + offset); +#else + return ((u64)__raw_readl(sdev->io_base + offset) << 32) | (u64)__raw_readl(sdev->io_base + offset + 4); +#endif +} + +static inline u32 jh7110_sec_read(struct jh7110_sec_dev *sdev, u32 offset) +{ + return __raw_readl(sdev->io_base + offset); +} + +static inline u16 jh7110_sec_readw(struct jh7110_sec_dev *sdev, u32 offset) +{ + return __raw_readw(sdev->io_base + offset); +} + +static inline u8 jh7110_sec_readb(struct jh7110_sec_dev *sdev, u32 offset) +{ + return __raw_readb(sdev->io_base + offset); +} + +static inline void jh7110_sec_writeq(struct jh7110_sec_dev *sdev, + u32 offset, u64 value) +{ +#ifdef CONFIG_64BIT + __raw_writeq(value, sdev->io_base + offset); +#else + __raw_writel((value >> 32), sdev->io_base + offset); + __raw_writel(value & 0xffffffff, sdev->io_base + offset + 4); +#endif +} + +static inline void jh7110_sec_write(struct jh7110_sec_dev *sdev, + u32 offset, u32 value) +{ + __raw_writel(value, sdev->io_base + offset); +} + +static inline void jh7110_sec_writew(struct jh7110_sec_dev *sdev, + u32 offset, u16 value) +{ + __raw_writew(value, sdev->io_base + offset); +} + +static inline void jh7110_sec_writeb(struct jh7110_sec_dev *sdev, + u32 offset, u8 value) +{ + __raw_writeb(value, sdev->io_base + offset); +} + +extern struct jh7110_sec_dev *jh7110_sec_find_dev(struct jh7110_sec_ctx *ctx); + +extern int jh7110_hash_register_algs(void); +extern void jh7110_hash_unregister_algs(void); + +extern int jh7110_aes_register_algs(void); +extern void jh7110_aes_unregister_algs(void); + +extern int jh7110_des_register_algs(void); +extern void jh7110_des_unregister_algs(void); + +extern int jh7110_pka_register_algs(void); +extern void jh7110_pka_unregister_algs(void); + +extern int jh7110_dma_sg_to_device(struct jh7110_sec_dma *sdma); +extern int jh7110_dma_mem_to_device(struct jh7110_sec_dma *sdma); +extern int jh7110_dma_sg_from_device(struct jh7110_sec_dma *sdma); +extern int jh7110_dma_mem_from_device(struct jh7110_sec_dma *sdma); +extern int jh7110_mem_to_mem_test(struct jh7110_sec_ctx *ctx); + +extern int jh7110_dmac_init(struct jh7110_sec_dev *sdev, int irq); +extern int jh7110_dmac_secdata_out(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size); +extern int jh7110_dmac_secdata_in(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size); +extern int jh7110_dmac_wait_done(struct jh7110_sec_dev *sdev, u8 chan); + +#endif diff --git a/drivers/dma/jh7110-pl08x.c b/drivers/dma/jh7110-pl08x.c new file mode 100755 index 0000000..76df9cf --- /dev/null +++ b/drivers/dma/jh7110-pl08x.c @@ -0,0 +1,3200 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2006 ARM Ltd. + * Copyright (c) 2010 ST-Ericsson SA + * Copyirght (c) 2017 Linaro Ltd. + * + * Author: Peter Pearse + * Author: Linus Walleij + * + * Documentation: ARM DDI 0196G == PL080 + * Documentation: ARM DDI 0218E == PL081 + * Documentation: S3C6410 User's Manual == PL080S + * + * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any + * channel. + * + * The PL080 has 8 channels available for simultaneous use, and the PL081 + * has only two channels. So on these DMA controllers the number of channels + * and the number of incoming DMA signals are two totally different things. + * It is usually not possible to theoretically handle all physical signals, + * so a multiplexing scheme with possible denial of use is necessary. + * + * The PL080 has a dual bus master, PL081 has a single master. + * + * PL080S is a version modified by Samsung and used in S3C64xx SoCs. + * It differs in following aspects: + * - CH_CONFIG register at different offset, + * - separate CH_CONTROL2 register for transfer size, + * - bigger maximum transfer size, + * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, + * - no support for peripheral flow control. + * + * Memory to peripheral transfer may be visualized as + * Get data from memory to DMAC + * Until no data left + * On burst request from peripheral + * Destination burst from DMAC to peripheral + * Clear burst request + * Raise terminal count interrupt + * + * For peripherals with a FIFO: + * Source burst size == half the depth of the peripheral FIFO + * Destination burst size == the depth of the peripheral FIFO + * + * (Bursts are irrelevant for mem to mem transfers - there are no burst + * signals, the DMA controller will simply facilitate its AHB master.) + * + * ASSUMES default (little) endianness for DMA transfers + * + * The PL08x has two flow control settings: + * - DMAC flow control: the transfer size defines the number of transfers + * which occur for the current LLI entry, and the DMAC raises TC at the + * end of every LLI entry. Observed behaviour shows the DMAC listening + * to both the BREQ and SREQ signals (contrary to documented), + * transferring data if either is active. The LBREQ and LSREQ signals + * are ignored. + * + * - Peripheral flow control: the transfer size is ignored (and should be + * zero). The data is transferred from the current LLI entry, until + * after the final transfer signalled by LBREQ or LSREQ. The DMAC + * will then move to the next LLI entry. Unsupported by PL080S. + */ +//#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define DRIVER_NAME "pl08xdmac" + +#define PL80X_DMA_BUSWIDTHS \ + BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) + +//static struct amba_driver pl08x_amba_driver; +struct pl08x_driver_data; + +/** + * struct vendor_data - vendor-specific config parameters for PL08x derivatives + * @config_offset: offset to the configuration register + * @channels: the number of channels available in this variant + * @signals: the number of request signals available from the hardware + * @dualmaster: whether this version supports dual AHB masters or not. + * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the + * channels have Nomadik security extension bits that need to be checked + * for permission before use and some registers are missing + * @pl080s: whether this variant is a Samsung PL080S, which has separate + * register and LLI word for transfer size. + * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020 + * @max_transfer_size: the maximum single element transfer size for this + * PL08x variant. + */ +struct vendor_data { + u8 config_offset; + u8 channels; + u8 signals; + bool dualmaster; + bool nomadik; + bool pl080s; + bool ftdmac020; + u32 max_transfer_size; +}; + +/** + * struct pl08x_bus_data - information of source or destination + * busses for a transfer + * @addr: current address + * @maxwidth: the maximum width of a transfer on this bus + * @buswidth: the width of this bus in bytes: 1, 2 or 4 + */ +struct pl08x_bus_data { + dma_addr_t addr; + u8 maxwidth; + u8 buswidth; +}; + +#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) + +/** + * struct pl08x_phy_chan - holder for the physical channels + * @id: physical index to this channel + * @base: memory base address for this physical channel + * @reg_config: configuration address for this physical channel + * @reg_control: control address for this physical channel + * @reg_src: transfer source address register + * @reg_dst: transfer destination address register + * @reg_lli: transfer LLI address register + * @reg_busy: if the variant has a special per-channel busy register, + * this contains a pointer to it + * @lock: a lock to use when altering an instance of this struct + * @serving: the virtual channel currently being served by this physical + * channel + * @locked: channel unavailable for the system, e.g. dedicated to secure + * world + * @ftdmac020: channel is on a FTDMAC020 + * @pl080s: channel is on a PL08s + */ +struct pl08x_phy_chan { + unsigned int id; + void __iomem *base; + void __iomem *reg_config; + void __iomem *reg_control; + void __iomem *reg_src; + void __iomem *reg_dst; + void __iomem *reg_lli; + void __iomem *reg_busy; + spinlock_t lock; + struct pl08x_dma_chan *serving; + bool locked; + bool ftdmac020; + bool pl080s; +}; + +/** + * struct pl08x_sg - structure containing data per sg + * @src_addr: src address of sg + * @dst_addr: dst address of sg + * @len: transfer len in bytes + * @node: node for txd's dsg_list + */ +struct pl08x_sg { + dma_addr_t src_addr; + dma_addr_t dst_addr; + size_t len; + struct list_head node; +}; + +/** + * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor + * @vd: virtual DMA descriptor + * @dsg_list: list of children sg's + * @llis_bus: DMA memory address (physical) start for the LLIs + * @llis_va: virtual memory address start for the LLIs + * @cctl: control reg values for current txd + * @ccfg: config reg values for current txd + * @done: this marks completed descriptors, which should not have their + * mux released. + * @cyclic: indicate cyclic transfers + */ +struct pl08x_txd { + struct virt_dma_desc vd; + struct list_head dsg_list; + dma_addr_t llis_bus; + u32 *llis_va; + /* Default cctl value for LLIs */ + u32 cctl; + /* + * Settings to be put into the physical channel when we + * trigger this txd. Other registers are in llis_va[0]. + */ + u32 ccfg; + bool done; + bool cyclic; +}; + +/** + * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel + * states + * @PL08X_CHAN_IDLE: the channel is idle + * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport + * channel and is running a transfer on it + * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport + * channel, but the transfer is currently paused + * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport + * channel to become available (only pertains to memcpy channels) + */ +enum pl08x_dma_chan_state { + PL08X_CHAN_IDLE, + PL08X_CHAN_RUNNING, + PL08X_CHAN_PAUSED, + PL08X_CHAN_WAITING, +}; + +/** + * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel + * @vc: wrappped virtual channel + * @phychan: the physical channel utilized by this channel, if there is one + * @name: name of channel + * @cd: channel platform data + * @cfg: slave configuration + * @at: active transaction on this channel + * @host: a pointer to the host (internal use) + * @state: whether the channel is idle, paused, running etc + * @slave: whether this channel is a device (slave) or for memcpy + * @signal: the physical DMA request signal which this channel is using + * @mux_use: count of descriptors using this DMA request signal setting + * @waiting_at: time in jiffies when this channel moved to waiting state + */ +struct pl08x_dma_chan { + struct virt_dma_chan vc; + struct pl08x_phy_chan *phychan; + const char *name; + struct pl08x_channel_data *cd; + struct dma_slave_config cfg; + struct pl08x_txd *at; + struct pl08x_driver_data *host; + enum pl08x_dma_chan_state state; + int chan_id; + bool slave; + int signal; + unsigned mux_use; + unsigned long waiting_at; +}; + +/** + * struct pl08x_driver_data - the local state holder for the PL08x + * @slave: optional slave engine for this instance + * @memcpy: memcpy engine for this instance + * @has_slave: the PL08x has a slave engine (routed signals) + * @base: virtual memory base (remapped) for the PL08x + * @adev: the corresponding AMBA (PrimeCell) bus entry + * @vd: vendor data for this PL08x variant + * @pd: platform data passed in from the platform/machine + * @phy_chans: array of data for the physical channels + * @pool: a pool for the LLI descriptors + * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI + * fetches + * @mem_buses: set to indicate memory transfers on AHB2. + * @lli_words: how many words are used in each LLI item for this variant + */ +struct pl08x_driver_data { + struct dma_device slave; + struct dma_device memcpy; + bool has_slave; + void __iomem *base; + struct platform_device *adev; + const struct vendor_data *vd; + struct pl08x_platform_data *pd; + struct pl08x_phy_chan *phy_chans; + struct dma_pool *pool; + u8 lli_buses; + u8 mem_buses; + u8 lli_words; +}; + +/* + * PL08X specific defines + */ + +/* The order of words in an LLI. */ +#define PL080_LLI_SRC 0 +#define PL080_LLI_DST 1 +#define PL080_LLI_LLI 2 +#define PL080_LLI_CCTL 3 +#define PL080S_LLI_CCTL2 4 + +/* Total words in an LLI. */ +#define PL080_LLI_WORDS 4 +#define PL080S_LLI_WORDS 8 + +/* + * Number of LLIs in each LLI buffer allocated for one transfer + * (maximum times we call dma_pool_alloc on this pool without freeing) + */ +#define MAX_NUM_TSFR_LLIS 512 +#define PL08X_ALIGN 8 + +static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) +{ + return container_of(chan, struct pl08x_dma_chan, vc.chan); +} + +static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct pl08x_txd, vd.tx); +} + +/* + * Mux handling. + * + * This gives us the DMA request input to the PL08x primecell which the + * peripheral described by the channel data will be routed to, possibly + * via a board/SoC specific external MUX. One important point to note + * here is that this does not depend on the physical channel. + */ +static int pl08x_request_mux(struct pl08x_dma_chan *plchan) +{ + const struct pl08x_platform_data *pd = plchan->host->pd; + int ret; + + if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { + ret = pd->get_xfer_signal(plchan->cd); + if (ret < 0) { + plchan->mux_use = 0; + return ret; + } + + plchan->signal = ret; + } + return 0; +} + +static void pl08x_release_mux(struct pl08x_dma_chan *plchan) +{ + const struct pl08x_platform_data *pd = plchan->host->pd; + + if (plchan->signal >= 0) { + WARN_ON(plchan->mux_use == 0); + + if (--plchan->mux_use == 0 && pd->put_xfer_signal) { + pd->put_xfer_signal(plchan->cd, plchan->signal); + plchan->signal = -1; + } + } +} + +/* + * Physical channel handling + */ + +/* Whether a certain channel is busy or not */ +static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) +{ + unsigned int val; + + /* If we have a special busy register, take a shortcut */ + if (ch->reg_busy) { + val = readl(ch->reg_busy); + return !!(val & BIT(ch->id)); + } + val = readl(ch->reg_config); + return val & PL080_CONFIG_ACTIVE; +} + +/* + * pl08x_write_lli() - Write an LLI into the DMA controller. + * + * The PL08x derivatives support linked lists, but the first item of the + * list containing the source, destination, control word and next LLI is + * ignored. Instead the driver has to write those values directly into the + * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE + * register need to be set up for the first transfer. + */ +static void pl08x_write_lli(struct pl08x_driver_data *pl08x, + struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) +{ + if (pl08x->vd->pl080s) + dev_vdbg(&pl08x->adev->dev, + "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " + "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", + phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], + lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], + lli[PL080S_LLI_CCTL2], ccfg); + else + //dev_vdbg(&pl08x->adev->dev, + dev_info(&pl08x->adev->dev, + "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " + "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", + phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], + lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); + + writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); + writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); + writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); + + /* + * The FTMAC020 has a different layout in the CCTL word of the LLI + * and the CCTL register which is split in CSR and SIZE registers. + * Convert the LLI item CCTL into the proper values to write into + * the CSR and SIZE registers. + */ + if (phychan->ftdmac020) { + u32 llictl = lli[PL080_LLI_CCTL]; + u32 val = 0; + + /* Write the transfer size (12 bits) to the size register */ + writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK, + phychan->base + FTDMAC020_CH_SIZE); + /* + * Then write the control bits 28..16 to the control register + * by shuffleing the bits around to where they are in the + * main register. The mapping is as follows: + * Bit 28: TC_MSK - mask on all except last LLI + * Bit 27..25: SRC_WIDTH + * Bit 24..22: DST_WIDTH + * Bit 21..20: SRCAD_CTRL + * Bit 19..17: DSTAD_CTRL + * Bit 17: SRC_SEL + * Bit 16: DST_SEL + */ + if (llictl & FTDMAC020_LLI_TC_MSK) + val |= FTDMAC020_CH_CSR_TC_MSK; + val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> + (FTDMAC020_LLI_SRC_WIDTH_SHIFT - + FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT)); + val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >> + (FTDMAC020_LLI_DST_WIDTH_SHIFT - + FTDMAC020_CH_CSR_DST_WIDTH_SHIFT)); + val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >> + (FTDMAC020_LLI_SRCAD_CTL_SHIFT - + FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT)); + val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >> + (FTDMAC020_LLI_DSTAD_CTL_SHIFT - + FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT)); + if (llictl & FTDMAC020_LLI_SRC_SEL) + val |= FTDMAC020_CH_CSR_SRC_SEL; + if (llictl & FTDMAC020_LLI_DST_SEL) + val |= FTDMAC020_CH_CSR_DST_SEL; + + /* + * Set up the bits that exist in the CSR but are not + * part the LLI, i.e. only gets written to the control + * register right here. + * + * FIXME: do not just handle memcpy, also handle slave DMA. + */ + switch (pl08x->pd->memcpy_burst_size) { + default: + case PL08X_BURST_SZ_1: + val |= PL080_BSIZE_1 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_4: + val |= PL080_BSIZE_4 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_8: + val |= PL080_BSIZE_8 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_16: + val |= PL080_BSIZE_16 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_32: + val |= PL080_BSIZE_32 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_64: + val |= PL080_BSIZE_64 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_128: + val |= PL080_BSIZE_128 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_256: + val |= PL080_BSIZE_256 << + FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; + break; + } + + /* Protection flags */ + if (pl08x->pd->memcpy_prot_buff) + val |= FTDMAC020_CH_CSR_PROT2; + if (pl08x->pd->memcpy_prot_cache) + val |= FTDMAC020_CH_CSR_PROT3; + /* We are the kernel, so we are in privileged mode */ + val |= FTDMAC020_CH_CSR_PROT1; + + writel_relaxed(val, phychan->reg_control); + } else { + /* printk("this is debug lli[PL080_LLI_CCTL] = %x reg_control = %x %s %s %d\n", + lli[PL080_LLI_CCTL],phychan->reg_control,__FILE__,__func__,__LINE__); + */ + /* Bits are just identical */ + writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); + } + + /* Second control word on the PL080s */ + if (pl08x->vd->pl080s) + writel_relaxed(lli[PL080S_LLI_CCTL2], + phychan->base + PL080S_CH_CONTROL2); + + /* + printk("this is debug ccfg = %x reg_config = %x %s %s %d\n", + ccfg,phychan->reg_config,__FILE__,__func__,__LINE__); + */ + writel(ccfg, phychan->reg_config); +} + +/* + * Set the initial DMA register values i.e. those for the first LLI + * The next LLI pointer and the configuration interrupt bit have + * been set when the LLIs were constructed. Poke them into the hardware + * and start the transfer. + */ +static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_phy_chan *phychan = plchan->phychan; + struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); + struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); + u32 val; + + list_del(&txd->vd.node); + + plchan->at = txd; + + /* Wait for channel inactive */ + while (pl08x_phy_channel_busy(phychan)) + cpu_relax(); + //printk("this is debug txd->ccfg = %x %s %s %d\n",txd->ccfg,__FILE__,__func__,__LINE__); + pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); + + /* Enable the DMA channel */ + /* Do not access config register until channel shows as disabled */ + //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__); + while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) + cpu_relax(); + + //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__); + /* Do not access config register until channel shows as inactive */ + if (phychan->ftdmac020) { + val = readl(phychan->reg_config); + while (val & FTDMAC020_CH_CFG_BUSY) + val = readl(phychan->reg_config); + + val = readl(phychan->reg_control); + while (val & FTDMAC020_CH_CSR_EN) + val = readl(phychan->reg_control); + + writel(val | FTDMAC020_CH_CSR_EN, + phychan->reg_control); + } else { + val = readl(phychan->reg_config); + while ((val & PL080_CONFIG_ACTIVE) || + (val & PL080_CONFIG_ENABLE)) + val = readl(phychan->reg_config); + //printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__); + writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); +#if 0 + while(!(readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))){ + printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__); + writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); + } +#endif + } + //printk("this is debug reg_config = %x en_chan = %x id = %d %s %s %d\n",readl(phychan->reg_config), + // readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__); +} + +/* + * Pause the channel by setting the HALT bit. + * + * For M->P transfers, pause the DMAC first and then stop the peripheral - + * the FIFO can only drain if the peripheral is still requesting data. + * (note: this can still timeout if the DMAC FIFO never drains of data.) + * + * For P->M transfers, disable the peripheral first to stop it filling + * the DMAC FIFO, and then pause the DMAC. + */ +static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) +{ + u32 val; + int timeout; + + if (ch->ftdmac020) { + /* Use the enable bit on the FTDMAC020 */ + val = readl(ch->reg_control); + val &= ~FTDMAC020_CH_CSR_EN; + writel(val, ch->reg_control); + return; + } + + /* Set the HALT bit and wait for the FIFO to drain */ + val = readl(ch->reg_config); + val |= PL080_CONFIG_HALT; + writel(val, ch->reg_config); + + /* Wait for channel inactive */ + for (timeout = 1000; timeout; timeout--) { + if (!pl08x_phy_channel_busy(ch)) + break; + udelay(1); + } + if (pl08x_phy_channel_busy(ch)) + pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); +} + +static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) +{ + u32 val; + + /* Use the enable bit on the FTDMAC020 */ + if (ch->ftdmac020) { + val = readl(ch->reg_control); + val |= FTDMAC020_CH_CSR_EN; + writel(val, ch->reg_control); + return; + } + + /* Clear the HALT bit */ + val = readl(ch->reg_config); + val &= ~PL080_CONFIG_HALT; + writel(val, ch->reg_config); +} + +/* + * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and + * clears any pending interrupt status. This should not be used for + * an on-going transfer, but as a method of shutting down a channel + * (eg, when it's no longer used) or terminating a transfer. + */ +static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, + struct pl08x_phy_chan *ch) +{ + u32 val; + + /* The layout for the FTDMAC020 is different */ + if (ch->ftdmac020) { + /* Disable all interrupts */ + val = readl(ch->reg_config); + val |= (FTDMAC020_CH_CFG_INT_ABT_MASK | + FTDMAC020_CH_CFG_INT_ERR_MASK | + FTDMAC020_CH_CFG_INT_TC_MASK); + writel(val, ch->reg_config); + + /* Abort and disable channel */ + val = readl(ch->reg_control); + val &= ~FTDMAC020_CH_CSR_EN; + val |= FTDMAC020_CH_CSR_ABT; + writel(val, ch->reg_control); + + /* Clear ABT and ERR interrupt flags */ + writel(BIT(ch->id) | BIT(ch->id + 16), + pl08x->base + PL080_ERR_CLEAR); + writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); + + return; + } + + val = readl(ch->reg_config); + val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | + PL080_CONFIG_TC_IRQ_MASK); + writel(val, ch->reg_config); + + writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); + writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); +} + +static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch) +{ + u32 val; + u32 bytes; + + if (ch->ftdmac020) { + bytes = readl(ch->base + FTDMAC020_CH_SIZE); + + val = readl(ch->reg_control); + val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK; + val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT; + } else if (ch->pl080s) { + val = readl(ch->base + PL080S_CH_CONTROL2); + bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; + + val = readl(ch->reg_control); + val &= PL080_CONTROL_SWIDTH_MASK; + val >>= PL080_CONTROL_SWIDTH_SHIFT; + } else { + /* Plain PL08x */ + val = readl(ch->reg_control); + bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; + + val &= PL080_CONTROL_SWIDTH_MASK; + val >>= PL080_CONTROL_SWIDTH_SHIFT; + } + + switch (val) { + case PL080_WIDTH_8BIT: + break; + case PL080_WIDTH_16BIT: + bytes *= 2; + break; + case PL080_WIDTH_32BIT: + bytes *= 4; + break; + } + return bytes; +} + +static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va) +{ + u32 val; + u32 bytes; + + if (ch->ftdmac020) { + val = llis_va[PL080_LLI_CCTL]; + bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK; + + val = llis_va[PL080_LLI_CCTL]; + val &= FTDMAC020_LLI_SRC_WIDTH_MSK; + val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT; + } else if (ch->pl080s) { + val = llis_va[PL080S_LLI_CCTL2]; + bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; + + val = llis_va[PL080_LLI_CCTL]; + val &= PL080_CONTROL_SWIDTH_MASK; + val >>= PL080_CONTROL_SWIDTH_SHIFT; + } else { + /* Plain PL08x */ + val = llis_va[PL080_LLI_CCTL]; + bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; + + val &= PL080_CONTROL_SWIDTH_MASK; + val >>= PL080_CONTROL_SWIDTH_SHIFT; + } + + switch (val) { + case PL080_WIDTH_8BIT: + break; + case PL080_WIDTH_16BIT: + bytes *= 2; + break; + case PL080_WIDTH_32BIT: + bytes *= 4; + break; + } + return bytes; +} + +/* The channel should be paused when calling this */ +static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + const u32 *llis_va, *llis_va_limit; + struct pl08x_phy_chan *ch; + dma_addr_t llis_bus; + struct pl08x_txd *txd; + u32 llis_max_words; + size_t bytes; + u32 clli; + + ch = plchan->phychan; + txd = plchan->at; + + if (!ch || !txd) + return 0; + + /* + * Follow the LLIs to get the number of remaining + * bytes in the currently active transaction. + */ + clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; + + /* First get the remaining bytes in the active transfer */ + bytes = get_bytes_in_phy_channel(ch); + + if (!clli) + return bytes; + + llis_va = txd->llis_va; + llis_bus = txd->llis_bus; + + llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; + BUG_ON(clli < llis_bus || clli >= llis_bus + + sizeof(u32) * llis_max_words); + + /* + * Locate the next LLI - as this is an array, + * it's simple maths to find. + */ + llis_va += (clli - llis_bus) / sizeof(u32); + + llis_va_limit = llis_va + llis_max_words; + + for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { + bytes += get_bytes_in_lli(ch, llis_va); + + /* + * A LLI pointer going backward terminates the LLI list + */ + if (llis_va[PL080_LLI_LLI] <= clli) + break; + } + + return bytes; +} + +/* + * Allocate a physical channel for a virtual channel + * + * Try to locate a physical channel to be used for this transfer. If all + * are taken return NULL and the requester will have to cope by using + * some fallback PIO mode or retrying later. + */ +static struct pl08x_phy_chan * +pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, + struct pl08x_dma_chan *virt_chan) +{ + struct pl08x_phy_chan *ch = NULL; + unsigned long flags; + int i; + + //printk("this is debug virt_chan->id = %d %s %s %d\n",virt_chan->chan_id,__FILE__,__func__,__LINE__); +#if 1 + ch = &pl08x->phy_chans[virt_chan->chan_id]; + + spin_lock_irqsave(&ch->lock, flags); + + if (!ch->locked && !ch->serving) { + ch->serving = virt_chan; + spin_unlock_irqrestore(&ch->lock, flags); + return ch; + } + + spin_unlock_irqrestore(&ch->lock, flags); +#endif + for (i = 0; i < pl08x->vd->channels; i++) { + ch = &pl08x->phy_chans[i]; + + spin_lock_irqsave(&ch->lock, flags); + + if (!ch->locked && !ch->serving) { + ch->serving = virt_chan; + spin_unlock_irqrestore(&ch->lock, flags); + break; + } + + spin_unlock_irqrestore(&ch->lock, flags); + } + + if (i == pl08x->vd->channels) { + /* No physical channel available, cope with it */ + return NULL; + } + + return ch; +} + +/* Mark the physical channel as free. Note, this write is atomic. */ +static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, + struct pl08x_phy_chan *ch) +{ + ch->serving = NULL; +} + +/* + * Try to allocate a physical channel. When successful, assign it to + * this virtual channel, and initiate the next descriptor. The + * virtual channel lock must be held at this point. + */ +static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_phy_chan *ch; + + ch = pl08x_get_phy_channel(pl08x, plchan); + if (!ch) { + dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); + plchan->state = PL08X_CHAN_WAITING; + plchan->waiting_at = jiffies; + return; + } + + dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", + ch->id, plchan->name); + + plchan->phychan = ch; + plchan->state = PL08X_CHAN_RUNNING; + pl08x_start_next_txd(plchan); +} + +static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, + struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + + dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", + ch->id, plchan->name); + + /* + * We do this without taking the lock; we're really only concerned + * about whether this pointer is NULL or not, and we're guaranteed + * that this will only be called when it _already_ is non-NULL. + */ + ch->serving = plchan; + plchan->phychan = ch; + plchan->state = PL08X_CHAN_RUNNING; + pl08x_start_next_txd(plchan); +} + +/* + * Free a physical DMA channel, potentially reallocating it to another + * virtual channel if we have any pending. + */ +static void pl08x_phy_free(struct pl08x_dma_chan *plchan) +{ + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_dma_chan *p, *next; + unsigned long waiting_at; + retry: + next = NULL; + waiting_at = jiffies; + + /* + * Find a waiting virtual channel for the next transfer. + * To be fair, time when each channel reached waiting state is compared + * to select channel that is waiting for the longest time. + */ + list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) + if (p->state == PL08X_CHAN_WAITING && + p->waiting_at <= waiting_at) { + next = p; + waiting_at = p->waiting_at; + } + + if (!next && pl08x->has_slave) { + list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) + if (p->state == PL08X_CHAN_WAITING && + p->waiting_at <= waiting_at) { + next = p; + waiting_at = p->waiting_at; + } + } + + /* Ensure that the physical channel is stopped */ + pl08x_terminate_phy_chan(pl08x, plchan->phychan); + + if (next) { + bool success; + + /* + * Eww. We know this isn't going to deadlock + * but lockdep probably doesn't. + */ + spin_lock(&next->vc.lock); + /* Re-check the state now that we have the lock */ + success = next->state == PL08X_CHAN_WAITING; + if (success) + pl08x_phy_reassign_start(plchan->phychan, next); + spin_unlock(&next->vc.lock); + + /* If the state changed, try to find another channel */ + if (!success) + goto retry; + } else { + /* No more jobs, so free up the physical channel */ + pl08x_put_phy_channel(pl08x, plchan->phychan); + } + + plchan->phychan = NULL; + plchan->state = PL08X_CHAN_IDLE; +} + +/* + * LLI handling + */ + +static inline unsigned int +pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x, + u32 cctl, + bool source) +{ + u32 val; + + if (pl08x->vd->ftdmac020) { + if (source) + val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> + FTDMAC020_LLI_SRC_WIDTH_SHIFT; + else + val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >> + FTDMAC020_LLI_DST_WIDTH_SHIFT; + } else { + if (source) + val = (cctl & PL080_CONTROL_SWIDTH_MASK) >> + PL080_CONTROL_SWIDTH_SHIFT; + else + val = (cctl & PL080_CONTROL_DWIDTH_MASK) >> + PL080_CONTROL_DWIDTH_SHIFT; + } + + switch (val) { + case PL080_WIDTH_8BIT: + return 1; + case PL080_WIDTH_16BIT: + return 2; + case PL080_WIDTH_32BIT: + return 4; + default: + break; + } + BUG(); + return 0; +} + +static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x, + u32 cctl, + u8 srcwidth, u8 dstwidth, + size_t tsize) +{ + u32 retbits = cctl; + + /* + * Remove all src, dst and transfer size bits, then set the + * width and size according to the parameters. The bit offsets + * are different in the FTDMAC020 so we need to accound for this. + */ + if (pl08x->vd->ftdmac020) { + retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK; + retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK; + retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK; + + switch (srcwidth) { + case 1: + retbits |= PL080_WIDTH_8BIT << + FTDMAC020_LLI_SRC_WIDTH_SHIFT; + break; + case 2: + retbits |= PL080_WIDTH_16BIT << + FTDMAC020_LLI_SRC_WIDTH_SHIFT; + break; + case 4: + retbits |= PL080_WIDTH_32BIT << + FTDMAC020_LLI_SRC_WIDTH_SHIFT; + break; + default: + BUG(); + break; + } + + switch (dstwidth) { + case 1: + retbits |= PL080_WIDTH_8BIT << + FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + case 2: + retbits |= PL080_WIDTH_16BIT << + FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + case 4: + retbits |= PL080_WIDTH_32BIT << + FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + default: + BUG(); + break; + } + + tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK; + retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT; + } else { + retbits &= ~PL080_CONTROL_DWIDTH_MASK; + retbits &= ~PL080_CONTROL_SWIDTH_MASK; + retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; + + switch (srcwidth) { + case 1: + retbits |= PL080_WIDTH_8BIT << + PL080_CONTROL_SWIDTH_SHIFT; + break; + case 2: + retbits |= PL080_WIDTH_16BIT << + PL080_CONTROL_SWIDTH_SHIFT; + break; + case 4: + retbits |= PL080_WIDTH_32BIT << + PL080_CONTROL_SWIDTH_SHIFT; + break; + default: + BUG(); + break; + } + + switch (dstwidth) { + case 1: + retbits |= PL080_WIDTH_8BIT << + PL080_CONTROL_DWIDTH_SHIFT; + break; + case 2: + retbits |= PL080_WIDTH_16BIT << + PL080_CONTROL_DWIDTH_SHIFT; + break; + case 4: + retbits |= PL080_WIDTH_32BIT << + PL080_CONTROL_DWIDTH_SHIFT; + break; + default: + BUG(); + break; + } + + tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; + retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; + } + + return retbits; +} + +struct pl08x_lli_build_data { + struct pl08x_txd *txd; + struct pl08x_bus_data srcbus; + struct pl08x_bus_data dstbus; + size_t remainder; + u32 lli_bus; +}; + +/* + * Autoselect a master bus to use for the transfer. Slave will be the chosen as + * victim in case src & dest are not similarly aligned. i.e. If after aligning + * masters address with width requirements of transfer (by sending few byte by + * byte data), slave is still not aligned, then its width will be reduced to + * BYTE. + * - prefers the destination bus if both available + * - prefers bus with fixed address (i.e. peripheral) + */ +static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x, + struct pl08x_lli_build_data *bd, + struct pl08x_bus_data **mbus, + struct pl08x_bus_data **sbus, + u32 cctl) +{ + bool dst_incr; + bool src_incr; + + /* + * The FTDMAC020 only supports memory-to-memory transfer, so + * source and destination always increase. + */ + if (pl08x->vd->ftdmac020) { + dst_incr = true; + src_incr = true; + } else { + dst_incr = !!(cctl & PL080_CONTROL_DST_INCR); + src_incr = !!(cctl & PL080_CONTROL_SRC_INCR); + } + + /* + * If either bus is not advancing, i.e. it is a peripheral, that + * one becomes master + */ + if (!dst_incr) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else if (!src_incr) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; + } else { + if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { + *mbus = &bd->dstbus; + *sbus = &bd->srcbus; + } else { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; + } + } +} + +/* + * Fills in one LLI for a certain transfer descriptor and advance the counter + */ +static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, + struct pl08x_lli_build_data *bd, + int num_llis, int len, u32 cctl, u32 cctl2) +{ + u32 offset = num_llis * pl08x->lli_words; + u32 *llis_va = bd->txd->llis_va + offset; + dma_addr_t llis_bus = bd->txd->llis_bus; + + BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); + + /* + printk("this is debug num_llis = %x lli_words = %x %s %s %d\n", + num_llis,pl08x->lli_words,__FILE__,__func__,__LINE__); + */ + /* Advance the offset to next LLI. */ + offset += pl08x->lli_words; + + llis_va[PL080_LLI_SRC] = bd->srcbus.addr; + llis_va[PL080_LLI_DST] = bd->dstbus.addr; + llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); + llis_va[PL080_LLI_LLI] |= bd->lli_bus; + llis_va[PL080_LLI_CCTL] = cctl; + if (pl08x->vd->pl080s) + llis_va[PL080S_LLI_CCTL2] = cctl2; + + if (pl08x->vd->ftdmac020) { + /* FIXME: only memcpy so far so both increase */ + bd->srcbus.addr += len; + bd->dstbus.addr += len; + } else { + if (cctl & PL080_CONTROL_SRC_INCR) + bd->srcbus.addr += len; + if (cctl & PL080_CONTROL_DST_INCR) + bd->dstbus.addr += len; + } + + BUG_ON(bd->remainder < len); + + bd->remainder -= len; +} + +static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, + struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, + int num_llis, size_t *total_bytes) +{ + *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len); + pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); + (*total_bytes) += len; +} + +#if 1 +static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, + const u32 *llis_va, int num_llis) +{ + int i; + + if (pl08x->vd->pl080s) { + dev_vdbg(&pl08x->adev->dev, + "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", + "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); + for (i = 0; i < num_llis; i++) { + dev_vdbg(&pl08x->adev->dev, + "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", + i, llis_va, llis_va[PL080_LLI_SRC], + llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], + llis_va[PL080_LLI_CCTL], + llis_va[PL080S_LLI_CCTL2]); + llis_va += pl08x->lli_words; + } + } else { + //dev_vdbg(&pl08x->adev->dev, + dev_info(&pl08x->adev->dev, + "%-3s %-9s %-10s %-10s %-10s %s\n", + "lli", "", "csrc", "cdst", "clli", "cctl"); + for (i = 0; i < num_llis; i++) { + //dev_vdbg(&pl08x->adev->dev, + dev_info(&pl08x->adev->dev, + "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", + i, llis_va, llis_va[PL080_LLI_SRC], + llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], + llis_va[PL080_LLI_CCTL]); + llis_va += pl08x->lli_words; + } + } +} +#else +static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, + const u32 *llis_va, int num_llis) {} +#endif + +extern u64 dw_virt_to_phys(void *vaddr); +/* + * This fills in the table of LLIs for the transfer descriptor + * Note that we assume we never have to change the burst sizes + * Return 0 for error + */ +static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, + struct pl08x_txd *txd) +{ + struct pl08x_bus_data *mbus, *sbus; + struct pl08x_lli_build_data bd; + int num_llis = 0; + u32 cctl, early_bytes = 0; + size_t max_bytes_per_lli, total_bytes; + u32 *llis_va, *last_lli; + struct pl08x_sg *dsg; + + txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); + if (!txd->llis_va) { + dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); + return 0; + } + + /* + printk("this is debug txd->llis_bus = %llx pl08x->lli_buses = %x llis_va = %llx %s %s %d\n", + txd->llis_bus,pl08x->lli_buses,dw_virt_to_phys(txd->llis_va),__FILE__,__func__,__LINE__); + */ + bd.txd = txd; + bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; + cctl = txd->cctl; + + /* Find maximum width of the source bus */ + bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true); + + /* Find maximum width of the destination bus */ + bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false); + + list_for_each_entry(dsg, &txd->dsg_list, node) { + total_bytes = 0; + cctl = txd->cctl; + + bd.srcbus.addr = dsg->src_addr; + bd.dstbus.addr = dsg->dst_addr; + bd.remainder = dsg->len; + bd.srcbus.buswidth = bd.srcbus.maxwidth; + bd.dstbus.buswidth = bd.dstbus.maxwidth; + + pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl); + + dev_vdbg(&pl08x->adev->dev, + "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", + (u64)bd.srcbus.addr, + cctl & PL080_CONTROL_SRC_INCR ? "+" : "", + bd.srcbus.buswidth, + (u64)bd.dstbus.addr, + cctl & PL080_CONTROL_DST_INCR ? "+" : "", + bd.dstbus.buswidth, + bd.remainder); + dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", + mbus == &bd.srcbus ? "src" : "dst", + sbus == &bd.srcbus ? "src" : "dst"); + + /* + * Zero length is only allowed if all these requirements are + * met: + * - flow controller is peripheral. + * - src.addr is aligned to src.width + * - dst.addr is aligned to dst.width + * + * sg_len == 1 should be true, as there can be two cases here: + * + * - Memory addresses are contiguous and are not scattered. + * Here, Only one sg will be passed by user driver, with + * memory address and zero length. We pass this to controller + * and after the transfer it will receive the last burst + * request from peripheral and so transfer finishes. + * + * - Memory addresses are scattered and are not contiguous. + * Here, Obviously as DMA controller doesn't know when a lli's + * transfer gets over, it can't load next lli. So in this + * case, there has to be an assumption that only one lli is + * supported. Thus, we can't have scattered addresses. + */ + if (!bd.remainder) { + u32 fc; + + /* FTDMAC020 only does memory-to-memory */ + if (pl08x->vd->ftdmac020) + fc = PL080_FLOW_MEM2MEM; + else + fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> + PL080_CONFIG_FLOW_CONTROL_SHIFT; + if (!((fc >= PL080_FLOW_SRC2DST_DST) && + (fc <= PL080_FLOW_SRC2DST_SRC))) { + dev_err(&pl08x->adev->dev, "%s sg len can't be zero", + __func__); + return 0; + } + + if (!IS_BUS_ALIGNED(&bd.srcbus) || + !IS_BUS_ALIGNED(&bd.dstbus)) { + dev_err(&pl08x->adev->dev, + "%s src & dst address must be aligned to src" + " & dst width if peripheral is flow controller", + __func__); + return 0; + } + + cctl = pl08x_lli_control_bits(pl08x, cctl, + bd.srcbus.buswidth, bd.dstbus.buswidth, + 0); + pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, + 0, cctl, 0); + break; + } + + /* + * Send byte by byte for following cases + * - Less than a bus width available + * - until master bus is aligned + */ + if (bd.remainder < mbus->buswidth) + early_bytes = bd.remainder; + else if (!IS_BUS_ALIGNED(mbus)) { + early_bytes = mbus->buswidth - + (mbus->addr & (mbus->buswidth - 1)); + if ((bd.remainder - early_bytes) < mbus->buswidth) + early_bytes = bd.remainder; + } + + if (early_bytes) { + dev_vdbg(&pl08x->adev->dev, + "%s byte width LLIs (remain 0x%08zx)\n", + __func__, bd.remainder); + prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, + num_llis++, &total_bytes); + } + + if (bd.remainder) { + /* + * Master now aligned + * - if slave is not then we must set its width down + */ + if (!IS_BUS_ALIGNED(sbus)) { + dev_dbg(&pl08x->adev->dev, + "%s set down bus width to one byte\n", + __func__); + + sbus->buswidth = 1; + } + + /* + * Bytes transferred = tsize * src width, not + * MIN(buswidths) + */ + max_bytes_per_lli = bd.srcbus.buswidth * + pl08x->vd->max_transfer_size; + dev_vdbg(&pl08x->adev->dev, + "%s max bytes per lli = %zu\n", + __func__, max_bytes_per_lli); + + /* + * Make largest possible LLIs until less than one bus + * width left + */ + while (bd.remainder > (mbus->buswidth - 1)) { + size_t lli_len, tsize, width; + + /* + * If enough left try to send max possible, + * otherwise try to send the remainder + */ + lli_len = min(bd.remainder, max_bytes_per_lli); + + /* + * Check against maximum bus alignment: + * Calculate actual transfer size in relation to + * bus width an get a maximum remainder of the + * highest bus width - 1 + */ + width = max(mbus->buswidth, sbus->buswidth); + lli_len = (lli_len / width) * width; + tsize = lli_len / bd.srcbus.buswidth; + + dev_vdbg(&pl08x->adev->dev, + "%s fill lli with single lli chunk of " + "size 0x%08zx (remainder 0x%08zx)\n", + __func__, lli_len, bd.remainder); + + cctl = pl08x_lli_control_bits(pl08x, cctl, + bd.srcbus.buswidth, bd.dstbus.buswidth, + tsize); + pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, + lli_len, cctl, tsize); + total_bytes += lli_len; + } + + /* + * Send any odd bytes + */ + if (bd.remainder) { + dev_vdbg(&pl08x->adev->dev, + "%s align with boundary, send odd bytes (remain %zu)\n", + __func__, bd.remainder); + prep_byte_width_lli(pl08x, &bd, &cctl, + bd.remainder, num_llis++, &total_bytes); + } + } + + if (total_bytes != dsg->len) { + dev_err(&pl08x->adev->dev, + "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", + __func__, total_bytes, dsg->len); + return 0; + } + + if (num_llis >= MAX_NUM_TSFR_LLIS) { + dev_err(&pl08x->adev->dev, + "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", + __func__, MAX_NUM_TSFR_LLIS); + return 0; + } + } + + llis_va = txd->llis_va; + last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; + + if (txd->cyclic) { + /* Link back to the first LLI. */ + last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; + } else { + /* The final LLI terminates the LLI. */ + last_lli[PL080_LLI_LLI] = 0; + /* The final LLI element shall also fire an interrupt. */ + if (pl08x->vd->ftdmac020) + last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK; + else + last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; + } + + pl08x_dump_lli(pl08x, llis_va, num_llis); + + return num_llis; +} + +static void pl08x_free_txd(struct pl08x_driver_data *pl08x, + struct pl08x_txd *txd) +{ + struct pl08x_sg *dsg, *_dsg; + + if (txd->llis_va) + dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); + + list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { + list_del(&dsg->node); + kfree(dsg); + } + + kfree(txd); +} + +static void pl08x_desc_free(struct virt_dma_desc *vd) +{ + struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); + struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); + + dma_descriptor_unmap(&vd->tx); + if (!txd->done) + pl08x_release_mux(plchan); + + pl08x_free_txd(plchan->host, txd); +} + +static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, + struct pl08x_dma_chan *plchan) +{ + LIST_HEAD(head); + + vchan_get_all_descriptors(&plchan->vc, &head); + vchan_dma_desc_free_list(&plchan->vc, &head); +} + +/* + * The DMA ENGINE API + */ +static void pl08x_free_chan_resources(struct dma_chan *chan) +{ + /* Ensure all queued descriptors are freed */ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( + struct dma_chan *chan, unsigned long flags) +{ + struct dma_async_tx_descriptor *retval = NULL; + + return retval; +} + +/* + * Code accessing dma_async_is_complete() in a tight loop may give problems. + * If slaves are relying on interrupts to signal completion this function + * must not be called with interrupts disabled. + */ +static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct virt_dma_desc *vd; + unsigned long flags; + enum dma_status ret; + size_t bytes = 0; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + /* + * There's no point calculating the residue if there's + * no txstate to store the value. + */ + if (!txstate) { + if (plchan->state == PL08X_CHAN_PAUSED) + ret = DMA_PAUSED; + return ret; + } + + spin_lock_irqsave(&plchan->vc.lock, flags); + ret = dma_cookie_status(chan, cookie, txstate); + if (ret != DMA_COMPLETE) { + vd = vchan_find_desc(&plchan->vc, cookie); + if (vd) { + /* On the issued list, so hasn't been processed yet */ + struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); + struct pl08x_sg *dsg; + + list_for_each_entry(dsg, &txd->dsg_list, node) + bytes += dsg->len; + } else { + bytes = pl08x_getbytes_chan(plchan); + } + } + spin_unlock_irqrestore(&plchan->vc.lock, flags); + + /* + * This cookie not complete yet + * Get number of bytes left in the active transactions and queue + */ + dma_set_residue(txstate, bytes); + + if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) + ret = DMA_PAUSED; + + /* Whether waiting or running, we're in progress */ + return ret; +} + +/* PrimeCell DMA extension */ +struct burst_table { + u32 burstwords; + u32 reg; +}; + +static const struct burst_table burst_sizes[] = { + { + .burstwords = 256, + .reg = PL080_BSIZE_256, + }, + { + .burstwords = 128, + .reg = PL080_BSIZE_128, + }, + { + .burstwords = 64, + .reg = PL080_BSIZE_64, + }, + { + .burstwords = 32, + .reg = PL080_BSIZE_32, + }, + { + .burstwords = 16, + .reg = PL080_BSIZE_16, + }, + { + .burstwords = 8, + .reg = PL080_BSIZE_8, + }, + { + .burstwords = 4, + .reg = PL080_BSIZE_4, + }, + { + .burstwords = 0, + .reg = PL080_BSIZE_1, + }, +}; + +/* + * Given the source and destination available bus masks, select which + * will be routed to each port. We try to have source and destination + * on separate ports, but always respect the allowable settings. + */ +static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst) +{ + u32 cctl = 0; + u32 dst_ahb2; + u32 src_ahb2; + + /* The FTDMAC020 use different bits to indicate src/dst bus */ + if (ftdmac020) { + dst_ahb2 = FTDMAC020_LLI_DST_SEL; + src_ahb2 = FTDMAC020_LLI_SRC_SEL; + } else { + dst_ahb2 = PL080_CONTROL_DST_AHB2; + src_ahb2 = PL080_CONTROL_SRC_AHB2; + } + + if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) + cctl |= dst_ahb2; + if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) + cctl |= src_ahb2; + + //printk("this is debug dst = %x src = %x cctl = %x %s %s %d\n",dst,src,cctl,__FILE__,__func__,__LINE__); + + return cctl; +} + +static u32 pl08x_cctl(u32 cctl) +{ + cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | + PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | + PL080_CONTROL_PROT_MASK); + + /* Access the cell in privileged mode, non-bufferable, non-cacheable */ + return cctl | PL080_CONTROL_PROT_SYS; +} + +static u32 pl08x_width(enum dma_slave_buswidth width) +{ + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + return PL080_WIDTH_8BIT; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + return PL080_WIDTH_16BIT; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + return PL080_WIDTH_32BIT; + default: + return ~0; + } +} + +static u32 pl08x_burst(u32 maxburst) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) + if (burst_sizes[i].burstwords <= maxburst) + break; + + return burst_sizes[i].reg; +} + +static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, + enum dma_slave_buswidth addr_width, u32 maxburst) +{ + u32 width, burst, cctl = 0; + + width = pl08x_width(addr_width); + if (width == ~0) + return ~0; + + cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; + cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; + + /* + * If this channel will only request single transfers, set this + * down to ONE element. Also select one element if no maxburst + * is specified. + */ + if (plchan->cd->single) + maxburst = 1; + + burst = pl08x_burst(maxburst); + cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; + cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; + + return pl08x_cctl(cctl); +} + +/* + * Slave transactions callback to the slave device to allow + * synchronization of slave DMA signals with the DMAC enable + */ +static void pl08x_issue_pending(struct dma_chan *chan) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + unsigned long flags; + + plchan->chan_id = chan->chan_id; + + spin_lock_irqsave(&plchan->vc.lock, flags); + if (vchan_issue_pending(&plchan->vc)) { + if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) + pl08x_phy_alloc_and_start(plchan); + } + spin_unlock_irqrestore(&plchan->vc.lock, flags); +} + +static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) +{ + struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); + + if (txd) + INIT_LIST_HEAD(&txd->dsg_list); + return txd; +} + +static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x) +{ + u32 cctl = 0; + + /* Conjure cctl */ + switch (pl08x->pd->memcpy_burst_size) { + default: + dev_err(&pl08x->adev->dev, + "illegal burst size for memcpy, set to 1\n"); + fallthrough; + case PL08X_BURST_SZ_1: + cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_4: + cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_8: + cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_16: + cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_32: + cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_64: + cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_128: + cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case PL08X_BURST_SZ_256: + cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + } + + switch (pl08x->pd->memcpy_bus_width) { + default: + dev_err(&pl08x->adev->dev, + "illegal bus width for memcpy, set to 8 bits\n"); + fallthrough; + case PL08X_BUS_WIDTH_8_BITS: + cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + case PL08X_BUS_WIDTH_16_BITS: + cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + case PL08X_BUS_WIDTH_32_BITS: + cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + } + + /* Protection flags */ + if (pl08x->pd->memcpy_prot_buff) + cctl |= PL080_CONTROL_PROT_BUFF; + if (pl08x->pd->memcpy_prot_cache) + cctl |= PL080_CONTROL_PROT_CACHE; + + /* We are the kernel, so we are in privileged mode */ + cctl |= PL080_CONTROL_PROT_SYS; + + /* Both to be incremented or the code will break */ + cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; + + if (pl08x->vd->dualmaster) + cctl |= pl08x_select_bus(false, + pl08x->mem_buses, + pl08x->mem_buses); + + return cctl; +} + +static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x) +{ + u32 cctl = 0; + + /* Conjure cctl */ + switch (pl08x->pd->memcpy_bus_width) { + default: + dev_err(&pl08x->adev->dev, + "illegal bus width for memcpy, set to 8 bits\n"); + fallthrough; + case PL08X_BUS_WIDTH_8_BITS: + cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | + PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + case PL08X_BUS_WIDTH_16_BITS: + cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | + PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + case PL08X_BUS_WIDTH_32_BITS: + cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | + PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; + break; + } + + /* + * By default mask the TC IRQ on all LLIs, it will be unmasked on + * the last LLI item by other code. + */ + cctl |= FTDMAC020_LLI_TC_MSK; + + /* + * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL + * and FTDMAC020_LLI_DSTAD_CTL as zero + */ + if (pl08x->vd->dualmaster) + cctl |= pl08x_select_bus(true, + pl08x->mem_buses, + pl08x->mem_buses); + + return cctl; +} + +/* + * Initialize a descriptor to be used by memcpy submit + */ +static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + struct pl08x_sg *dsg; + int ret; + //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__); + txd = pl08x_get_txd(plchan); + if (!txd) { + dev_err(&pl08x->adev->dev, + "%s no memory for descriptor\n", __func__); + return NULL; + } + //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__); + dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); + if (!dsg) { + pl08x_free_txd(pl08x, txd); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__); + dsg->src_addr = src; + dsg->dst_addr = dest; + dsg->len = len; + if (pl08x->vd->ftdmac020) { + /* Writing CCFG zero ENABLES all interrupts */ + txd->ccfg = 0; + txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); + } else { + txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | + PL080_CONFIG_TC_IRQ_MASK | + PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; + txd->cctl = pl08x_memcpy_cctl(pl08x); + } + //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__); + ret = pl08x_fill_llis_for_desc(plchan->host, txd); + if (!ret) { + pl08x_free_txd(pl08x, txd); + return NULL; + } + //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__); + return vchan_tx_prep(&plchan->vc, &txd->vd, flags); +} + +static struct pl08x_txd *pl08x_init_txd( + struct dma_chan *chan, + enum dma_transfer_direction direction, + dma_addr_t *slave_addr) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + enum dma_slave_buswidth addr_width; + int ret, tmp; + u8 src_buses, dst_buses; + u32 maxburst, cctl; + + txd = pl08x_get_txd(plchan); + if (!txd) { + dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); + return NULL; + } + + /* + * Set up addresses, the PrimeCell configured address + * will take precedence since this may configure the + * channel target address dynamically at runtime. + */ + if (direction == DMA_MEM_TO_DEV) { + //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + cctl = PL080_CONTROL_SRC_INCR; + *slave_addr = plchan->cfg.dst_addr; + addr_width = plchan->cfg.dst_addr_width; + maxburst = plchan->cfg.dst_maxburst; + src_buses = pl08x->mem_buses; + dst_buses = plchan->cd->periph_buses; + } else if (direction == DMA_DEV_TO_MEM) { + //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + cctl = PL080_CONTROL_DST_INCR; + *slave_addr = plchan->cfg.src_addr; + addr_width = plchan->cfg.src_addr_width; + maxburst = plchan->cfg.src_maxburst; + src_buses = plchan->cd->periph_buses; + dst_buses = pl08x->mem_buses; + } else { + pl08x_free_txd(pl08x, txd); + dev_err(&pl08x->adev->dev, + "%s direction unsupported\n", __func__); + return NULL; + } + + cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); + if (cctl == ~0) { + pl08x_free_txd(pl08x, txd); + dev_err(&pl08x->adev->dev, + "DMA slave configuration botched?\n"); + return NULL; + } + + txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); + + if (plchan->cfg.device_fc) + tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : + PL080_FLOW_PER2MEM_PER; + else + tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : + PL080_FLOW_PER2MEM; + + txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | + PL080_CONFIG_TC_IRQ_MASK | + tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; + //printk("this is debug cctl = %x ccfg = %x %s %s %d\n",txd->cctl,txd->ccfg,__FILE__,__func__,__LINE__); + + ret = pl08x_request_mux(plchan); + if (ret < 0) { + pl08x_free_txd(pl08x, txd); + dev_dbg(&pl08x->adev->dev, + "unable to mux for transfer on %s due to platform restrictions\n", + plchan->name); + return NULL; + } + + dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", + plchan->signal, plchan->name); + + /* Assign the flow control signal to this channel */ + if (direction == DMA_MEM_TO_DEV) + txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; + else + txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; + + return txd; +} + +static int pl08x_tx_add_sg(struct pl08x_txd *txd, + enum dma_transfer_direction direction, + dma_addr_t slave_addr, + dma_addr_t buf_addr, + unsigned int len) +{ + struct pl08x_sg *dsg; + + dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); + if (!dsg) + return -ENOMEM; + + list_add_tail(&dsg->node, &txd->dsg_list); + + dsg->len = len; + if (direction == DMA_MEM_TO_DEV) { + dsg->src_addr = buf_addr; + dsg->dst_addr = slave_addr; + } else { + dsg->src_addr = slave_addr; + dsg->dst_addr = buf_addr; + } + + return 0; +} + +static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + struct scatterlist *sg; + int ret, tmp; + dma_addr_t slave_addr; + + dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", + __func__, sg_dma_len(sgl), plchan->name); + + txd = pl08x_init_txd(chan, direction, &slave_addr); + if (!txd) + return NULL; + + for_each_sg(sgl, sg, sg_len, tmp) { + ret = pl08x_tx_add_sg(txd, direction, slave_addr, + sg_dma_address(sg), + sg_dma_len(sg)); + /* + printk("this is debug direction = %x slave_addr = %x addr = %x len = %x %s %s %d\n", + direction,slave_addr,sg_dma_address(sg),sg_dma_len(sg), + __FILE__,__func__,__LINE__); + */ + if (ret) { + pl08x_release_mux(plchan); + pl08x_free_txd(pl08x, txd); + dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", + __func__); + return NULL; + } + } + + ret = pl08x_fill_llis_for_desc(plchan->host, txd); + if (!ret) { + pl08x_release_mux(plchan); + pl08x_free_txd(pl08x, txd); + return NULL; + } + + return vchan_tx_prep(&plchan->vc, &txd->vd, flags); +} + +static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + struct pl08x_txd *txd; + int ret, tmp; + dma_addr_t slave_addr; + + dev_dbg(&pl08x->adev->dev, + "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", + __func__, period_len, buf_len, + direction == DMA_MEM_TO_DEV ? "to" : "from", + plchan->name); + + txd = pl08x_init_txd(chan, direction, &slave_addr); + if (!txd) + return NULL; + + txd->cyclic = true; + txd->cctl |= PL080_CONTROL_TC_IRQ_EN; + for (tmp = 0; tmp < buf_len; tmp += period_len) { + ret = pl08x_tx_add_sg(txd, direction, slave_addr, + buf_addr + tmp, period_len); + if (ret) { + pl08x_release_mux(plchan); + pl08x_free_txd(pl08x, txd); + return NULL; + } + } + + ret = pl08x_fill_llis_for_desc(plchan->host, txd); + if (!ret) { + pl08x_release_mux(plchan); + pl08x_free_txd(pl08x, txd); + return NULL; + } + + return vchan_tx_prep(&plchan->vc, &txd->vd, flags); +} + +static int pl08x_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + + if (!plchan->slave) + return -EINVAL; + + /* Reject definitely invalid configurations */ + if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || + config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) + return -EINVAL; + + if (config->device_fc && pl08x->vd->pl080s) { + dev_err(&pl08x->adev->dev, + "%s: PL080S does not support peripheral flow control\n", + __func__); + return -EINVAL; + } + + /* + printk("this is debug chan = %x chan-id = %d plchan = %x plchan->signal = %d %s %s %d\n", + chan,chan->chan_id,plchan,plchan->signal,__FILE__,__func__,__LINE__); + */ + plchan->cfg = *config; + + return 0; +} + +static int pl08x_terminate_all(struct dma_chan *chan) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_driver_data *pl08x = plchan->host; + unsigned long flags; + + spin_lock_irqsave(&plchan->vc.lock, flags); + if (!plchan->phychan && !plchan->at) { + spin_unlock_irqrestore(&plchan->vc.lock, flags); + return 0; + } + + plchan->state = PL08X_CHAN_IDLE; + + if (plchan->phychan) { + /* + * Mark physical channel as free and free any slave + * signal + */ + pl08x_phy_free(plchan); + } + /* Dequeue jobs and free LLIs */ + if (plchan->at) { + vchan_terminate_vdesc(&plchan->at->vd); + plchan->at = NULL; + } + /* Dequeue jobs not yet fired as well */ + pl08x_free_txd_list(pl08x, plchan); + + spin_unlock_irqrestore(&plchan->vc.lock, flags); + + return 0; +} + +static void pl08x_synchronize(struct dma_chan *chan) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + + printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__); + vchan_synchronize(&plchan->vc); +} + +static int pl08x_pause(struct dma_chan *chan) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + unsigned long flags; + + /* + * Anything succeeds on channels with no physical allocation and + * no queued transfers. + */ + printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__); + spin_lock_irqsave(&plchan->vc.lock, flags); + if (!plchan->phychan && !plchan->at) { + spin_unlock_irqrestore(&plchan->vc.lock, flags); + return 0; + } + + pl08x_pause_phy_chan(plchan->phychan); + plchan->state = PL08X_CHAN_PAUSED; + + spin_unlock_irqrestore(&plchan->vc.lock, flags); + + return 0; +} + +static int pl08x_resume(struct dma_chan *chan) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + unsigned long flags; + + /* + * Anything succeeds on channels with no physical allocation and + * no queued transfers. + */ + printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__); + spin_lock_irqsave(&plchan->vc.lock, flags); + if (!plchan->phychan && !plchan->at) { + spin_unlock_irqrestore(&plchan->vc.lock, flags); + return 0; + } + + pl08x_resume_phy_chan(plchan->phychan); + plchan->state = PL08X_CHAN_RUNNING; + + spin_unlock_irqrestore(&plchan->vc.lock, flags); + + return 0; +} +#if 0 +bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) +{ + struct pl08x_dma_chan *plchan; + char *name = chan_id; + + /* Reject channels for devices not bound to this driver */ + if (chan->device->dev->driver != &pl08x_amba_driver.drv) + return false; + + plchan = to_pl08x_chan(chan); + + /* Check that the channel is not taken! */ + if (!strcmp(plchan->name, name)) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(pl08x_filter_id); +#endif +static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + + return plchan->cd == chan_id; +} + +/* + * Just check that the device is there and active + * TODO: turn this bit on/off depending on the number of physical channels + * actually used, if it is zero... well shut it off. That will save some + * power. Cut the clock at the same time. + */ +static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) +{ + /* The Nomadik variant does not have the config register */ + if (pl08x->vd->nomadik) + return; + /* The FTDMAC020 variant does this in another register */ + if (pl08x->vd->ftdmac020) { + writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); + return; + } + writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); +} + +static irqreturn_t pl08x_irq(int irq, void *dev) +{ + struct pl08x_driver_data *pl08x = dev; + u32 mask = 0, err, tc, i; + + /* check & clear - ERR & TC interrupts */ + err = readl(pl08x->base + PL080_ERR_STATUS); + if (err) { + dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", + __func__, err); + writel(err, pl08x->base + PL080_ERR_CLEAR); + } + tc = readl(pl08x->base + PL080_TC_STATUS); + if (tc) { + writel(tc, pl08x->base + PL080_TC_CLEAR); + } + + if (!err && !tc) { + return IRQ_NONE; + } + + for (i = 0; i < pl08x->vd->channels; i++) { + if ((BIT(i) & err) || (BIT(i) & tc)) { + /* Locate physical channel */ + struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; + struct pl08x_dma_chan *plchan = phychan->serving; + struct pl08x_txd *tx; + + if (!plchan) { + dev_err(&pl08x->adev->dev, + "%s Error TC interrupt on unused channel: 0x%08x\n", + __func__, i); + continue; + } + + spin_lock(&plchan->vc.lock); + tx = plchan->at; + if (tx && tx->cyclic) { + vchan_cyclic_callback(&tx->vd); + } else if (tx) { + plchan->at = NULL; + /* + * This descriptor is done, release its mux + * reservation. + */ + pl08x_release_mux(plchan); + tx->done = true; + vchan_cookie_complete(&tx->vd); + + /* + * And start the next descriptor (if any), + * otherwise free this channel. + */ + if (vchan_next_desc(&plchan->vc)) + pl08x_start_next_txd(plchan); + else + pl08x_phy_free(plchan); + } + spin_unlock(&plchan->vc.lock); + + mask |= BIT(i); + } + } + + return mask ? IRQ_HANDLED : IRQ_NONE; +} + +static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) +{ + chan->slave = true; + chan->name = chan->cd->bus_id; + chan->cfg.src_addr = chan->cd->addr; + chan->cfg.dst_addr = chan->cd->addr; +} + +/* + * Initialise the DMAC memcpy/slave channels. + * Make a local wrapper to hold required data + */ +static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, + struct dma_device *dmadev, unsigned int channels, bool slave) +{ + struct pl08x_dma_chan *chan; + int i; + + INIT_LIST_HEAD(&dmadev->channels); + + /* + * Register as many many memcpy as we have physical channels, + * we won't always be able to use all but the code will have + * to cope with that situation. + */ + for (i = 0; i < channels; i++) { + chan = kzalloc(sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + + chan->host = pl08x; + chan->state = PL08X_CHAN_IDLE; + chan->signal = -1; + + if (slave) { + chan->cd = &pl08x->pd->slave_channels[i]; + /* + * Some implementations have muxed signals, whereas some + * use a mux in front of the signals and need dynamic + * assignment of signals. + */ + chan->signal = i; + pl08x_dma_slave_init(chan); + } else { + chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); + if (!chan->cd) { + kfree(chan); + return -ENOMEM; + } + chan->cd->bus_id = "memcpy"; + chan->cd->periph_buses = pl08x->pd->mem_buses; + chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); + if (!chan->name) { + kfree(chan->cd); + kfree(chan); + return -ENOMEM; + } + } + dev_dbg(&pl08x->adev->dev, + "initialize virtual channel \"%s\"\n", + chan->name); + + chan->vc.desc_free = pl08x_desc_free; + vchan_init(&chan->vc, dmadev); + } + dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", + i, slave ? "slave" : "memcpy"); + return i; +} + +static void pl08x_free_virtual_channels(struct dma_device *dmadev) +{ + struct pl08x_dma_chan *chan = NULL; + struct pl08x_dma_chan *next; + + list_for_each_entry_safe(chan, + next, &dmadev->channels, vc.chan.device_node) { + list_del(&chan->vc.chan.device_node); + kfree(chan); + } +} + +#ifdef CONFIG_DEBUG_FS +static const char *pl08x_state_str(enum pl08x_dma_chan_state state) +{ + switch (state) { + case PL08X_CHAN_IDLE: + return "idle"; + case PL08X_CHAN_RUNNING: + return "running"; + case PL08X_CHAN_PAUSED: + return "paused"; + case PL08X_CHAN_WAITING: + return "waiting"; + default: + break; + } + return "UNKNOWN STATE"; +} + +static int pl08x_debugfs_show(struct seq_file *s, void *data) +{ + struct pl08x_driver_data *pl08x = s->private; + struct pl08x_dma_chan *chan; + struct pl08x_phy_chan *ch; + unsigned long flags; + int i; + + seq_printf(s, "PL08x physical channels:\n"); + seq_printf(s, "CHANNEL:\tUSER:\n"); + seq_printf(s, "--------\t-----\n"); + for (i = 0; i < pl08x->vd->channels; i++) { + struct pl08x_dma_chan *virt_chan; + + ch = &pl08x->phy_chans[i]; + + spin_lock_irqsave(&ch->lock, flags); + virt_chan = ch->serving; + + seq_printf(s, "%d\t\t%s%s\n", + ch->id, + virt_chan ? virt_chan->name : "(none)", + ch->locked ? " LOCKED" : ""); + + spin_unlock_irqrestore(&ch->lock, flags); + } + + seq_printf(s, "\nPL08x virtual memcpy channels:\n"); + seq_printf(s, "CHANNEL:\tSTATE:\n"); + seq_printf(s, "--------\t------\n"); + list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { + seq_printf(s, "%s\t\t%s\n", chan->name, + pl08x_state_str(chan->state)); + } + + if (pl08x->has_slave) { + seq_printf(s, "\nPL08x virtual slave channels:\n"); + seq_printf(s, "CHANNEL:\tSTATE:\n"); + seq_printf(s, "--------\t------\n"); + list_for_each_entry(chan, &pl08x->slave.channels, + vc.chan.device_node) { + seq_printf(s, "%s\t\t%s\n", chan->name, + pl08x_state_str(chan->state)); + } + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs); + +static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) +{ + /* Expose a simple debugfs interface to view all clocks */ + debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, + NULL, pl08x, &pl08x_debugfs_fops); +} + +#else +static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) +{ +} +#endif + +#ifdef CONFIG_OF +static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, + u32 id) +{ + struct pl08x_dma_chan *chan; + + /* Trying to get a slave channel from something with no slave support */ + if (!pl08x->has_slave) + return NULL; + + list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { + if (chan->signal == id) + return &chan->vc.chan; + } + + return NULL; +} + +static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct pl08x_driver_data *pl08x = ofdma->of_dma_data; + struct dma_chan *dma_chan; + struct pl08x_dma_chan *plchan; + + if (!pl08x) + return NULL; + + if (dma_spec->args_count != 2) { + dev_err(&pl08x->adev->dev, + "DMA channel translation requires two cells\n"); + return NULL; + } + + dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); + if (!dma_chan) { + dev_err(&pl08x->adev->dev, + "DMA slave channel not found\n"); + return NULL; + } + + plchan = to_pl08x_chan(dma_chan); + dev_dbg(&pl08x->adev->dev, + "translated channel for signal %d\n", + dma_spec->args[0]); + + /* Augment channel data for applicable AHB buses */ + plchan->cd->periph_buses = dma_spec->args[1]; + return dma_get_slave_channel(dma_chan); +} + +static int pl08x_of_probe(struct platform_device *adev, + struct pl08x_driver_data *pl08x, + struct device_node *np) +{ + struct pl08x_platform_data *pd; + struct pl08x_channel_data *chanp = NULL; + u32 val; + int ret; + int i; + + pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + /* Eligible bus masters for fetching LLIs */ + if (of_property_read_bool(np, "lli-bus-interface-ahb1")) + pd->lli_buses |= PL08X_AHB1; + if (of_property_read_bool(np, "lli-bus-interface-ahb2")) + pd->lli_buses |= PL08X_AHB2; + if (!pd->lli_buses) { + dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); + pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; + } + + /* Eligible bus masters for memory access */ + if (of_property_read_bool(np, "mem-bus-interface-ahb1")) + pd->mem_buses |= PL08X_AHB1; + if (of_property_read_bool(np, "mem-bus-interface-ahb2")) + pd->mem_buses |= PL08X_AHB2; + if (!pd->mem_buses) { + dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); + pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; + } + + /* Parse the memcpy channel properties */ + ret = of_property_read_u32(np, "memcpy-burst-size", &val); + if (ret) { + dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); + val = 1; + } + switch (val) { + default: + dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); + fallthrough; + case 1: + pd->memcpy_burst_size = PL08X_BURST_SZ_1; + break; + case 4: + pd->memcpy_burst_size = PL08X_BURST_SZ_4; + break; + case 8: + pd->memcpy_burst_size = PL08X_BURST_SZ_8; + break; + case 16: + pd->memcpy_burst_size = PL08X_BURST_SZ_16; + break; + case 32: + pd->memcpy_burst_size = PL08X_BURST_SZ_32; + break; + case 64: + pd->memcpy_burst_size = PL08X_BURST_SZ_64; + break; + case 128: + pd->memcpy_burst_size = PL08X_BURST_SZ_128; + break; + case 256: + pd->memcpy_burst_size = PL08X_BURST_SZ_256; + break; + } + + ret = of_property_read_u32(np, "memcpy-bus-width", &val); + if (ret) { + dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); + val = 8; + } + switch (val) { + default: + dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); + fallthrough; + case 8: + pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; + break; + case 16: + pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; + break; + case 32: + pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; + break; + } + + /* + * Allocate channel data for all possible slave channels (one + * for each possible signal), channels will then be allocated + * for a device and have it's AHB interfaces set up at + * translation time. + */ + if (pl08x->vd->signals) { + chanp = devm_kcalloc(&adev->dev, + pl08x->vd->signals, + sizeof(struct pl08x_channel_data), + GFP_KERNEL); + if (!chanp) + return -ENOMEM; + + pd->slave_channels = chanp; + for (i = 0; i < pl08x->vd->signals; i++) { + /* + * chanp->periph_buses will be assigned at translation + */ + chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); + chanp++; + } + pd->num_slave_channels = pl08x->vd->signals; + } + + pl08x->pd = pd; + + return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, + pl08x); +} +#else +static inline int pl08x_of_probe(struct platform_device *adev, + struct pl08x_driver_data *pl08x, + struct device_node *np) +{ + return -EINVAL; +} +#endif + +static int pl08x_probe(struct platform_device *adev) //, const struct amba_id *id) +{ + struct pl08x_driver_data *pl08x; + struct vendor_data *vd; + struct device_node *np = adev->dev.of_node; + struct resource *res; + u32 tsfr_size; + int irq, ret = 0; + int i; + + //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); +#if 0 + ret = amba_request_regions(adev, NULL); + if (ret){ + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + return ret; + } +#endif + /* Ensure that we can do DMA */ + ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); + if (ret){ + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + return ret; + } + + /* Create the driver state holder */ + pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); + if (!pl08x) { + ret = -ENOMEM; + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + return ret; + } + + /* Assign useful pointers to the driver state */ + pl08x->adev = adev; + vd = of_device_get_match_data(&adev->dev); + if(!vd) + return -ENODEV; + pl08x->vd = vd; + + res = platform_get_resource_byname(adev, IORESOURCE_MEM, "sec_dma"); + pl08x->base = devm_ioremap_resource(&adev->dev, res); + if (!pl08x->base) { + ret = -ENOMEM; + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + goto out_no_ioremap; + } + + if (vd->ftdmac020) { + u32 val; + + val = readl(pl08x->base + FTDMAC020_REVISION); + dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", + (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff); + val = readl(pl08x->base + FTDMAC020_FEATURE); + dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d channels, " + "%s built-in bridge, %s, %s linked lists\n", + (val >> 12) & 0x0f, + (val & BIT(10)) ? "no" : "has", + (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0", + (val & BIT(8)) ? "supports" : "does not support"); + + /* Vendor data from feature register */ + if (!(val & BIT(8))) + dev_warn(&pl08x->adev->dev, + "linked lists not supported, required\n"); + vd->channels = (val >> 12) & 0x0f; + vd->dualmaster = !!(val & BIT(9)); + } + + /* Initialize memcpy engine */ + dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); + pl08x->memcpy.dev = &adev->dev; + pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; + pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; + pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; + pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; + pl08x->memcpy.device_issue_pending = pl08x_issue_pending; + pl08x->memcpy.device_config = pl08x_config; + pl08x->memcpy.device_pause = pl08x_pause; + pl08x->memcpy.device_resume = pl08x_resume; + pl08x->memcpy.device_terminate_all = pl08x_terminate_all; + pl08x->memcpy.device_synchronize = pl08x_synchronize; + pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; + pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; + pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); + pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + if (vd->ftdmac020) + pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; + + + /* + * Initialize slave engine, if the block has no signals, that means + * we have no slave support. + */ + if (vd->signals) { + pl08x->has_slave = true; + dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); + dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); + pl08x->slave.dev = &adev->dev; + pl08x->slave.device_free_chan_resources = + pl08x_free_chan_resources; + pl08x->slave.device_prep_dma_interrupt = + pl08x_prep_dma_interrupt; + pl08x->slave.device_tx_status = pl08x_dma_tx_status; + pl08x->slave.device_issue_pending = pl08x_issue_pending; + pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; + pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; + pl08x->slave.device_config = pl08x_config; + pl08x->slave.device_pause = pl08x_pause; + pl08x->slave.device_resume = pl08x_resume; + pl08x->slave.device_terminate_all = pl08x_terminate_all; + pl08x->slave.device_synchronize = pl08x_synchronize; + pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; + pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; + pl08x->slave.directions = + BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + pl08x->slave.residue_granularity = + DMA_RESIDUE_GRANULARITY_SEGMENT; + } + + /* Get the platform data */ + pl08x->pd = dev_get_platdata(&adev->dev); + if (!pl08x->pd) { + if (np) { + ret = pl08x_of_probe(adev, pl08x, np); + if (ret) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + goto out_no_platdata; + } + } else { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_err(&adev->dev, "no platform data supplied\n"); + ret = -EINVAL; + goto out_no_platdata; + } + } else { + pl08x->slave.filter.map = pl08x->pd->slave_map; + pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; + pl08x->slave.filter.fn = pl08x_filter_fn; + } + + /* By default, AHB1 only. If dualmaster, from platform */ + pl08x->lli_buses = PL08X_AHB1; + pl08x->mem_buses = PL08X_AHB1; + if (pl08x->vd->dualmaster) { + pl08x->lli_buses = pl08x->pd->lli_buses; + pl08x->mem_buses = pl08x->pd->mem_buses; + } + + if (vd->pl080s) + pl08x->lli_words = PL080S_LLI_WORDS; + else + pl08x->lli_words = PL080_LLI_WORDS; + tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); + + /* A DMA memory pool for LLIs, align on 1-byte boundary */ + pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, + tsfr_size, PL08X_ALIGN, 0); + if (!pl08x->pool) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + ret = -ENOMEM; + goto out_no_lli_pool; + } + + /* Turn on the PL08x */ + pl08x_ensure_on(pl08x); + + /* Clear any pending interrupts */ + if (vd->ftdmac020) + /* This variant has error IRQs in bits 16-19 */ + writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); + else + writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); + writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); + + /* Attach the interrupt handler */ + irq = platform_get_irq(adev, 0); + if (irq < 0) { + dev_err(&adev->dev, "Cannot get IRQ resource\n"); + return irq; + } + + ret = request_irq(irq, pl08x_irq, 0, DRIVER_NAME, pl08x); + if (ret) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_err(&adev->dev, "%s failed to request interrupt %d\n", + __func__, irq); + goto out_no_irq; + } + + /* Initialize physical channels */ + pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), + GFP_KERNEL); + if (!pl08x->phy_chans) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + ret = -ENOMEM; + goto out_no_phychans; + } + + for (i = 0; i < vd->channels; i++) { + struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; + + ch->id = i; + ch->base = pl08x->base + PL080_Cx_BASE(i); + if (vd->ftdmac020) { + /* FTDMA020 has a special channel busy register */ + ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; + ch->reg_config = ch->base + FTDMAC020_CH_CFG; + ch->reg_control = ch->base + FTDMAC020_CH_CSR; + ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; + ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; + ch->reg_lli = ch->base + FTDMAC020_CH_LLP; + ch->ftdmac020 = true; + } else { + printk("this is debug i = %d ch->base = %x %s %s %d\n",i,ch->base,__FILE__,__func__,__LINE__); + ch->reg_config = ch->base + vd->config_offset; + ch->reg_control = ch->base + PL080_CH_CONTROL; + ch->reg_src = ch->base + PL080_CH_SRC_ADDR; + ch->reg_dst = ch->base + PL080_CH_DST_ADDR; + ch->reg_lli = ch->base + PL080_CH_LLI; + } + if (vd->pl080s) + ch->pl080s = true; + + spin_lock_init(&ch->lock); + + /* + * Nomadik variants can have channels that are locked + * down for the secure world only. Lock up these channels + * by perpetually serving a dummy virtual channel. + */ + if (vd->nomadik) { + u32 val; + + val = readl(ch->reg_config); + if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { + dev_dbg(&adev->dev, "physical channel %d reserved for secure access only\n", i); + ch->locked = true; + } + } + + //dev_dbg(&adev->dev, "physical channel %d is %s\n", + dev_dbg(&adev->dev, "physical channel %d is %s\n", + i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); + } + + /* Register as many memcpy channels as there are physical channels */ + ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, + pl08x->vd->channels, false); + if (ret <= 0) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_warn(&pl08x->adev->dev, + "%s failed to enumerate memcpy channels - %d\n", + __func__, ret); + goto out_no_memcpy; + } + + /* Register slave channels */ + if (pl08x->has_slave) { + ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, + pl08x->pd->num_slave_channels, true); + if (ret < 0) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_warn(&pl08x->adev->dev, + "%s failed to enumerate slave channels - %d\n", + __func__, ret); + goto out_no_slave; + } + } + + ret = dma_async_device_register(&pl08x->memcpy); + if (ret) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_warn(&pl08x->adev->dev, + "%s failed to register memcpy as an async device - %d\n", + __func__, ret); + goto out_no_memcpy_reg; + } + + if (pl08x->has_slave) { + ret = dma_async_device_register(&pl08x->slave); + if (ret) { + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_warn(&pl08x->adev->dev, + "%s failed to register slave as an async device - %d\n", + __func__, ret); + goto out_no_slave_reg; + } + } + + platform_set_drvdata(adev, pl08x); + init_pl08x_debugfs(pl08x); + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + { + int loop; + + for (loop = 0xfe0;loop <= 0xffc; loop += 4) { + dev_dbg(&pl08x->adev->dev, "periphid[0x%x] = %x ", + loop,readl(pl08x->base + loop)); + } + } + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + dev_dbg(&pl08x->adev->dev, "DMA: PL080 at 0x%08llx irq %d\n", + (unsigned long long)res->start, irq); + + return 0; + +out_no_slave_reg: + dma_async_device_unregister(&pl08x->memcpy); +out_no_memcpy_reg: + if (pl08x->has_slave) + pl08x_free_virtual_channels(&pl08x->slave); +out_no_slave: + pl08x_free_virtual_channels(&pl08x->memcpy); +out_no_memcpy: + kfree(pl08x->phy_chans); +out_no_phychans: + free_irq(irq, pl08x); +out_no_irq: + dma_pool_destroy(pl08x->pool); +out_no_lli_pool: +out_no_platdata: + iounmap(pl08x->base); +out_no_ioremap: + kfree(pl08x); + //out_no_pl08x: + //amba_release_regions(adev); + printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__); + return ret; +} + +/* PL080 has 8 channels and the PL080 have just 2 */ +static struct vendor_data vendor_pl080 = { + .config_offset = PL080_CH_CONFIG, + .channels = 8, + .signals = 16, + .dualmaster = true, + .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, +}; +#if 0 +static struct vendor_data vendor_nomadik = { + .config_offset = PL080_CH_CONFIG, + .channels = 8, + .signals = 32, + .dualmaster = true, + .nomadik = true, + .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, +}; + +static struct vendor_data vendor_pl080s = { + .config_offset = PL080S_CH_CONFIG, + .channels = 8, + .signals = 32, + .pl080s = true, + .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, +}; + +static struct vendor_data vendor_pl081 = { + .config_offset = PL080_CH_CONFIG, + .channels = 2, + .signals = 16, + .dualmaster = false, + .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, +}; + +static struct vendor_data vendor_ftdmac020 = { + .config_offset = PL080_CH_CONFIG, + .ftdmac020 = true, + .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, +}; + +static const struct amba_id pl08x_ids[] = { + /* Samsung PL080S variant */ + { + .id = 0x0a141080, + .mask = 0xffffffff, + .data = &vendor_pl080s, + }, + /* PL080 */ + { + .id = 0x00041080, + .mask = 0x000fffff, + .data = &vendor_pl080, + }, + /* PL081 */ + { + .id = 0x00041081, + .mask = 0x000fffff, + .data = &vendor_pl081, + }, + /* Nomadik 8815 PL080 variant */ + { + .id = 0x00280080, + .mask = 0x00ffffff, + .data = &vendor_nomadik, + }, + /* Faraday Technology FTDMAC020 */ + { + .id = 0x0003b080, + .mask = 0x000fffff, + .data = &vendor_ftdmac020, + }, + { 0, 0 }, +}; + +MODULE_DEVICE_TABLE(amba, pl08x_ids); + +static struct amba_driver pl08x_amba_driver = { + .drv.name = DRIVER_NAME, + .id_table = pl08x_ids, + .probe = pl08x_probe, +}; +#endif +static const struct of_device_id vic7110_dma_ids[] = { + { .compatible = "starfive,pl080", .data = &vendor_pl080}, + {}, +}; +MODULE_DEVICE_TABLE(of, vic7110_dma_ids); + +static struct platform_driver vic7110_pl08x_driver = { + .probe = pl08x_probe, + .driver = { + .name = DRIVER_NAME, + .of_match_table = vic7110_dma_ids, + }, +}; + +module_platform_driver(vic7110_pl08x_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Huan Feng "); +MODULE_DESCRIPTION("Starfive VIC7110 CRYP DMA driver");