add crypto driver for starfive jh7110 soc.
Signed-off-by: william.qiu <william.qiu@starfivetech.com>
source "drivers/crypto/hisilicon/Kconfig"
+source "drivers/crypto/starfive/Kconfig"
+
source "drivers/crypto/amlogic/Kconfig"
config CRYPTO_DEV_SA2UL
obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += xilinx/
+obj-$(CONFIG_SOC_STARFIVE) += starfive/
obj-y += hisilicon/
obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/
obj-y += keembay/
--- /dev/null
+# select ARM_AMBA
+
+config CRYPTO_DEV_JH7110_ENCRYPT
+ tristate "Encryption acceleration support on jh7110"
+ depends on RISCV
+ select CRYPTO_ENGINE
+ select CRYPTO_RSA
+ select CRYPTO_LIB_DES
+ select CRYPTO_SKCIPHER
+ select AMBA_PL08X
+ default m
+ help
+ Support for SIFIVE JH7110 cryptographic acceleration instructions on riscv64 CPU.
+ This module supports acceleration for AES and GHASH in hardware. If you
+ choose 'M' here, this module will be called jh7110-crypto.
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+obj-y += jh7110/
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_JH7110_ENCRYPT) += jh7110-crypto.o
+jh7110-crypto-objs := jh7110-sec.o jh7110-sha.o jh7110-aes.o jh7110-des.o jh7110-pka.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <william.qiu@starfivetech.com>
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <crypto/hash.h>
+
+#include <linux/dma-direct.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+
+#include "jh7110-pl080.h"
+#include "jh7110-str.h"
+
+/* Mode mask = bits [3..0] */
+#define FLG_MODE_MASK GENMASK(2, 0)
+
+/* Bit [4] encrypt / decrypt */
+#define FLG_ENCRYPT BIT(4)
+
+/* Bit [31..16] status */
+#define FLG_CCM_PADDED_WA BIT(5)
+
+#define SR_BUSY 0x00000010
+#define SR_OFNE 0x00000004
+
+#define IMSCR_IN BIT(0)
+#define IMSCR_OUT BIT(1)
+
+#define MISR_IN BIT(0)
+#define MISR_OUT BIT(1)
+
+/* Misc */
+#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
+#define GCM_CTR_INIT 1
+#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset)
+#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset)
+#define CRYP_AUTOSUSPEND_DELAY 50
+
+static inline int jh7110_aes_wait_busy(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status,
+ !(status & JH7110_AES_BUSY), 10, 100000);
+}
+
+static inline int jh7110_aes_wait_keydone(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status,
+ (status & JH7110_AES_KEY_DONE), 10, 100000);
+}
+
+static inline int jh7110_aes_wait_gcmdone(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_AES_CSR, status,
+ (status & JH7110_AES_GCM_DONE), 10, 100000);
+}
+
+static inline int is_ecb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_ECB;
+}
+
+static inline int is_cbc(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CBC;
+}
+
+static inline int is_ofb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_OFB;
+}
+
+static inline int is_cfb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CFB;
+}
+
+static inline int is_ctr(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CTR;
+}
+
+static inline int is_gcm(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_GCM;
+}
+
+static inline int is_ccm(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_MODE_MASK) == JH7110_AES_MODE_CCM;
+}
+
+static inline int get_aes_mode(struct jh7110_sec_request_ctx *rctx)
+{
+ return rctx->flags & FLG_MODE_MASK;
+}
+
+static inline int is_encrypt(struct jh7110_sec_request_ctx *rctx)
+{
+ return !!(rctx->flags & FLG_ENCRYPT);
+}
+
+static inline int is_decrypt(struct jh7110_sec_request_ctx *rctx)
+{
+ return !is_encrypt(rctx);
+}
+
+static int jh7110_cryp_read_auth_tag(struct jh7110_sec_ctx *ctx);
+
+static inline void jh7110_aes_reset(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ rctx->csr.aes_csr.v = 0;
+ rctx->csr.aes_csr.aesrst = 1;
+ jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, rctx->csr.aes_csr.v);
+
+}
+
+static inline void jh7110_aes_xcm_start(struct jh7110_sec_ctx *ctx, u32 hw_mode)
+{
+ unsigned int value;
+
+ switch (hw_mode) {
+ case JH7110_AES_MODE_GCM:
+ value = jh7110_sec_read(ctx->sdev, JH7110_AES_CSR);
+ value |= JH7110_AES_GCM_START;
+ jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, value);
+ break;
+ case JH7110_AES_MODE_CCM:
+ value = jh7110_sec_read(ctx->sdev, JH7110_AES_CSR);
+ value |= JH7110_AES_CCM_START;
+ jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, value);
+ break;
+ }
+}
+
+static inline void jh7110_aes_csr_setup(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ rctx->csr.aes_csr.v = 0;
+ switch (ctx->keylen) {
+ case AES_KEYSIZE_128:
+ rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_128;
+ break;
+ case AES_KEYSIZE_192:
+ rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_192;
+ break;
+ case AES_KEYSIZE_256:
+ rctx->csr.aes_csr.keymode = JH7110_AES_KEYMODE_256;
+ break;
+ default:
+ return;
+ }
+ rctx->csr.aes_csr.mode = rctx->flags & FLG_MODE_MASK;
+ rctx->csr.aes_csr.cmode = is_decrypt(rctx);
+ if (!ctx->sdev->use_dma || is_ccm(rctx) || is_gcm(rctx))
+ rctx->csr.aes_csr.aesie = 1;
+ rctx->csr.aes_csr.stream_mode = rctx->stmode;
+
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(ctx->sdev->dev, "reset error\n");
+ return;
+ }
+
+ jh7110_sec_write(ctx->sdev, JH7110_AES_CSR, rctx->csr.aes_csr.v);
+}
+
+static inline void jh7110_aes_set_ivlen(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ jh7110_sec_write(sdev, JH7110_AES_IVLEN, 0x40);
+}
+
+static inline void jh7110_aes_set_alen(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ jh7110_sec_write(sdev, JH7110_AES_ALEN0, (ctx->rctx->assoclen >> 32) & 0xffffffff);
+ jh7110_sec_write(sdev, JH7110_AES_ALEN1, ctx->rctx->assoclen & 0xffffffff);
+}
+
+static inline void jh7110_aes_set_mlen(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ size_t data_len = rctx->total_in - rctx->assoclen;
+
+ jh7110_sec_write(sdev, JH7110_AES_MLEN0, (data_len >> 32) & 0xffffffff);
+ jh7110_sec_write(sdev, JH7110_AES_MLEN1, data_len & 0xffffffff);
+}
+
+static void jh7110_cryp_hw_write_iv(struct jh7110_sec_ctx *ctx, u32 *iv)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int loop;
+
+ if (!iv)
+ return;
+
+ for (loop = 0; loop < 4; loop++)
+ dev_dbg(sdev->dev, "iv[%d] = %x\n", loop, iv[loop]);
+
+ jh7110_sec_write(sdev, JH7110_AES_IV0, iv[0]);
+ jh7110_sec_write(sdev, JH7110_AES_IV1, iv[1]);
+ jh7110_sec_write(sdev, JH7110_AES_IV2, iv[2]);
+ jh7110_sec_write(sdev, JH7110_AES_IV3, iv[3]);
+}
+
+static void jh7110_cryp_hw_write_ctr(struct jh7110_sec_ctx *ctx, u32 *ctr)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int loop;
+
+ if (!ctr)
+ return;
+
+ for (loop = 0; loop < 4; loop++)
+ dev_dbg(sdev->dev, "ctr[%d] = %x\n", loop, ctr[loop]);
+
+ jh7110_sec_write(sdev, JH7110_AES_NONCE0, ctr[0]);
+ jh7110_sec_write(sdev, JH7110_AES_NONCE1, ctr[1]);
+ jh7110_sec_write(sdev, JH7110_AES_NONCE2, ctr[2]);
+ jh7110_sec_write(sdev, JH7110_AES_NONCE3, ctr[3]);
+}
+
+static int jh7110_cryp_hw_write_key(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 *key = (u32 *)ctx->key;
+
+ switch (ctx->keylen) {
+ case AES_KEYSIZE_256:
+ case AES_KEYSIZE_192:
+ case AES_KEYSIZE_128:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (ctx->keylen >= AES_KEYSIZE_128) {
+ jh7110_sec_write(sdev, JH7110_AES_KEY0, key[0]);
+ jh7110_sec_write(sdev, JH7110_AES_KEY1, key[1]);
+ jh7110_sec_write(sdev, JH7110_AES_KEY2, key[2]);
+ jh7110_sec_write(sdev, JH7110_AES_KEY3, key[3]);
+ }
+
+ if (ctx->keylen >= AES_KEYSIZE_192) {
+ jh7110_sec_write(sdev, JH7110_AES_KEY4, key[4]);
+ jh7110_sec_write(sdev, JH7110_AES_KEY5, key[5]);
+ }
+
+ if (ctx->keylen >= AES_KEYSIZE_256) {
+ jh7110_sec_write(sdev, JH7110_AES_KEY6, key[6]);
+ jh7110_sec_write(sdev, JH7110_AES_KEY7, key[7]);
+ }
+
+ if (jh7110_aes_wait_keydone(ctx))
+ return -ETIMEDOUT;
+
+ return 0;
+}
+
+static unsigned int jh7110_cryp_get_input_text_len(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ return is_encrypt(rctx) ? rctx->req.areq->cryptlen :
+ rctx->req.areq->cryptlen - rctx->authsize;
+}
+
+static int jh7110_cryp_gcm_init(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ /* Phase 1 : init */
+ memcpy(rctx->last_ctr, rctx->req.areq->iv, 12);
+ rctx->last_ctr[3] = cpu_to_be32(GCM_CTR_INIT);
+
+ jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->last_ctr);
+ return 0;
+}
+
+static int jh7110_cryp_ccm_init(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ u8 iv[AES_BLOCK_SIZE], *b0;
+ unsigned int textlen;
+
+ /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */
+ memcpy(iv, rctx->req.areq->iv, AES_BLOCK_SIZE);
+ memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
+
+ /* Build B0 */
+ b0 = (u8 *)sdev->aes_data;
+ memcpy(b0, iv, AES_BLOCK_SIZE);
+
+ b0[0] |= (8 * ((rctx->authsize - 2) / 2));
+
+ if (rctx->req.areq->assoclen)
+ b0[0] |= 0x40;
+
+ textlen = jh7110_cryp_get_input_text_len(ctx);
+
+ b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
+ b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
+
+ memcpy((void *)rctx->last_ctr, sdev->aes_data, AES_BLOCK_SIZE);
+ jh7110_cryp_hw_write_ctr(ctx, (u32 *)b0);
+
+ return 0;
+}
+
+static int jh7110_cryp_hw_init(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ int ret;
+ u32 hw_mode;
+ int loop;
+
+ jh7110_aes_reset(ctx);
+
+ hw_mode = get_aes_mode(ctx->rctx);
+ if (hw_mode == JH7110_AES_MODE_CFB ||
+ hw_mode == JH7110_AES_MODE_OFB)
+ rctx->stmode = JH7110_AES_MODE_XFB_128;
+ else
+ rctx->stmode = JH7110_AES_MODE_XFB_1;
+
+ jh7110_aes_csr_setup(ctx);
+
+ switch (hw_mode) {
+ case JH7110_AES_MODE_GCM:
+ memset(ctx->sdev->aes_data, 0, JH7110_MSG_BUFFER_SIZE);
+ jh7110_aes_set_alen(ctx);
+ jh7110_aes_set_mlen(ctx);
+ jh7110_aes_set_ivlen(ctx);
+ ret = jh7110_cryp_hw_write_key(ctx);
+
+ jh7110_aes_xcm_start(ctx, hw_mode);
+
+ if (jh7110_aes_wait_gcmdone(ctx))
+ return -ETIMEDOUT;
+
+ memset((void *)rctx->last_ctr, 0, sizeof(rctx->last_ctr));
+ jh7110_cryp_gcm_init(ctx);
+ if (jh7110_aes_wait_gcmdone(ctx))
+ return -ETIMEDOUT;
+
+ break;
+ case JH7110_AES_MODE_CCM:
+ memset(ctx->sdev->aes_data, 0, JH7110_MSG_BUFFER_SIZE);
+ memset((void *)rctx->last_ctr, 0, sizeof(rctx->last_ctr));
+ jh7110_aes_set_alen(ctx);
+ jh7110_aes_set_mlen(ctx);
+ jh7110_aes_set_ivlen(ctx);
+ /* Phase 1 : init */
+ jh7110_cryp_ccm_init(ctx);
+
+ ret = jh7110_cryp_hw_write_key(ctx);
+
+ jh7110_aes_xcm_start(ctx, hw_mode);
+
+ break;
+ case JH7110_AES_MODE_ECB:
+ ret = jh7110_cryp_hw_write_key(ctx);
+ break;
+ case JH7110_AES_MODE_OFB:
+ if (ctx->sec_init) {
+ for (loop = 0; loop < JH7110_AES_IV_LEN / sizeof(unsigned int); loop++)
+ rctx->aes_iv[loop] = (rctx->msg_end[loop]) ^ (rctx->dec_end[loop]);
+ } else
+ memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN);
+ jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv);
+
+ ret = jh7110_cryp_hw_write_key(ctx);
+ ctx->sec_init = 1;
+ break;
+ case JH7110_AES_MODE_CFB:
+ case JH7110_AES_MODE_CBC:
+ if (ctx->sec_init)
+ memcpy((void *)rctx->aes_iv, (void *)rctx->dec_end, JH7110_AES_IV_LEN);
+ else
+ memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN);
+ jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv);
+
+ ret = jh7110_cryp_hw_write_key(ctx);
+ ctx->sec_init = 1;
+ break;
+ case JH7110_AES_MODE_CTR:
+ if (ctx->sec_init)
+ memcpy((void *)rctx->aes_iv, (void *)rctx->dec_end, JH7110_AES_IV_LEN);
+ else
+ memcpy((void *)rctx->aes_iv, (void *)rctx->req.sreq->iv, JH7110_AES_IV_LEN);
+ jh7110_cryp_hw_write_iv(ctx, (u32 *)rctx->aes_iv);
+ memcpy((void *)rctx->last_ctr, (void *)rctx->aes_iv, JH7110_AES_IV_LEN);
+ ret = jh7110_cryp_hw_write_key(ctx);
+ ctx->sec_init = 1;
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+static int jh7110_cryp_get_from_sg(struct jh7110_sec_request_ctx *rctx, size_t offset,
+ size_t count, size_t data_offset)
+{
+ size_t of, ct, index;
+ struct scatterlist *sg = rctx->sg;
+
+ of = offset;
+ ct = count;
+ while (sg->length <= of) {
+ of -= sg->length;
+
+ if (!sg_is_last(sg)) {
+ sg = sg_next(sg);
+ continue;
+ } else {
+ return -EBADE;
+ }
+ }
+
+ index = data_offset;
+ while (ct > 0) {
+ if (sg->length - of >= ct) {
+ scatterwalk_map_and_copy(rctx->sdev->aes_data + index, sg,
+ of, ct, 0);
+ index = index + ct;
+ return index - data_offset;
+ }
+ scatterwalk_map_and_copy(rctx->sdev->aes_data + index, sg,
+ of, sg->length - of, 0);
+ index += sg->length - of;
+ ct = ct - (sg->length - of);
+
+ of = 0;
+
+ if (!sg_is_last(sg))
+ sg = sg_next(sg);
+ else
+ return -EBADE;
+ }
+ return index - data_offset;
+}
+
+static int jh7110_cryp_read_auth_tag(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ int loop;
+ int err = 0;
+
+ if (jh7110_aes_wait_busy(ctx))
+ return -EBUSY;
+
+ for (loop = 0; loop < 3 + rctx->authsize / 4; loop++) {
+ rctx->aes_nonce[loop] = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_auth[%d] = %x\n", loop, rctx->aes_nonce[loop]);
+ }
+
+ if (is_encrypt(rctx)) {
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), rctx->aes_nonce,
+ rctx->authsize, rctx->offset, 0);
+ }
+
+ return err;
+}
+
+static int jh7110_gcm_zero_message_data(struct jh7110_sec_ctx *ctx);
+
+static int jh7110_cryp_finish_req(struct jh7110_sec_ctx *ctx, int err)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ if (!err && (is_gcm(rctx) || is_ccm(rctx))) {
+ /* Phase 4 : output tag */
+ err = jh7110_cryp_read_auth_tag(ctx);
+ }
+
+ if (is_gcm(rctx) || is_ccm(rctx))
+ crypto_finalize_aead_request(ctx->sdev->engine, rctx->req.areq, err);
+ else
+ crypto_finalize_skcipher_request(ctx->sdev->engine, rctx->req.sreq,
+ err);
+
+ return err;
+}
+
+static bool jh7110_check_counter_overflow(struct jh7110_sec_ctx *ctx, size_t count)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ bool ret = false;
+ u32 start, end, ctr, blocks;
+
+ if (count) {
+ blocks = DIV_ROUND_UP(count, AES_BLOCK_SIZE);
+ rctx->last_ctr[3] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[3]) + blocks);
+
+ if (rctx->last_ctr[3] == 0) {
+ rctx->last_ctr[2] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[2]) + 1);
+ if (rctx->last_ctr[2] == 0) {
+ rctx->last_ctr[1] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[1]) + 1);
+ if (rctx->last_ctr[1] == 0) {
+ rctx->last_ctr[0] = cpu_to_be32(be32_to_cpu(rctx->last_ctr[0]) + 1);
+ if (rctx->last_ctr[1] == 0)
+ jh7110_cryp_hw_write_ctr(ctx, (u32 *)rctx->last_ctr);
+ }
+ }
+ }
+ }
+
+ /* ctr counter overflow. */
+ ctr = rctx->total_in - rctx->assoclen - rctx->authsize;
+ blocks = DIV_ROUND_UP(ctr, AES_BLOCK_SIZE);
+ start = be32_to_cpu(rctx->last_ctr[3]);
+
+ end = start + blocks - 1;
+ if (end < start) {
+ rctx->ctr_over_count = AES_BLOCK_SIZE * -start;
+ ret = true;
+ }
+
+ return ret;
+}
+
+static void jh7110_aes_dma_callback(void *param)
+{
+ struct jh7110_sec_dev *sdev = param;
+
+ complete(&sdev->sec_comp_p);
+}
+
+static void vic_debug_dma_info(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int loop;
+
+ for (loop = 0; loop <= 0x34; loop += 4)
+ dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop));
+ for (loop = 0x100; loop <= 0x110; loop += 4)
+ dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop));
+ for (loop = 0x120; loop <= 0x130; loop += 4)
+ dev_dbg(sdev->dev, "dma[%x] = %x\n", loop, readl_relaxed(sdev->dma_base + loop));
+}
+
+static int jh7110_cryp_write_out_dma(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct dma_async_tx_descriptor *in_desc, *out_desc;
+ union jh7110_alg_cr alg_cr;
+ dma_cookie_t cookie;
+ unsigned int *out;
+ int total_len;
+ int err;
+ int loop;
+
+ total_len = rctx->bufcnt;
+
+ jh7110_sec_write(sdev, JH7110_DMA_IN_LEN_OFFSET, total_len);
+ jh7110_sec_write(sdev, JH7110_DMA_OUT_LEN_OFFSET, total_len);
+
+ alg_cr.v = 0;
+ alg_cr.start = 1;
+ alg_cr.aes_dma_en = 1;
+ jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v);
+
+ total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
+
+ sg_init_table(&ctx->sg[0], 1);
+ sg_set_buf(&ctx->sg[0], sdev->aes_data, total_len);
+ sg_dma_address(&ctx->sg[0]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->aes_data));
+ sg_dma_len(&ctx->sg[0]) = total_len;
+
+ sg_init_table(&ctx->sg[1], 1);
+ sg_set_buf(&ctx->sg[1], sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1), total_len);
+ sg_dma_address(&ctx->sg[1]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1)));
+ sg_dma_len(&ctx->sg[1]) = total_len;
+
+ err = dma_map_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+ if (!err) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ err = dma_map_sg(sdev->dev, &ctx->sg[1], 1, DMA_FROM_DEVICE);
+ if (!err) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ sdev->cfg_in.direction = DMA_MEM_TO_DEV;
+ sdev->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.src_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET;
+
+ dmaengine_slave_config(sdev->sec_xm_m, &sdev->cfg_in);
+
+ sdev->cfg_out.direction = DMA_DEV_TO_MEM;
+ sdev->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_out.src_maxburst = 4;
+ sdev->cfg_out.dst_maxburst = 4;
+ sdev->cfg_out.src_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET;
+
+ dmaengine_slave_config(sdev->sec_xm_p, &sdev->cfg_out);
+
+ in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &ctx->sg[0],
+ 1, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!in_desc)
+ return -EINVAL;
+
+ cookie = dmaengine_submit(in_desc);
+ dma_async_issue_pending(sdev->sec_xm_m);
+
+ out_desc = dmaengine_prep_slave_sg(sdev->sec_xm_p, &ctx->sg[1],
+ 1, DMA_DEV_TO_MEM,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!out_desc)
+ return -EINVAL;
+
+ reinit_completion(&sdev->sec_comp_p);
+
+ out_desc->callback = jh7110_aes_dma_callback;
+ out_desc->callback_param = sdev;
+
+ dmaengine_submit(out_desc);
+ dma_async_issue_pending(sdev->sec_xm_p);
+
+ err = 0;
+
+ if (!wait_for_completion_timeout(&sdev->sec_comp_p,
+ msecs_to_jiffies(10000))) {
+ vic_debug_dma_info(ctx);
+ dev_err(sdev->dev, "wait_for_completion_timeout out error cookie = %x\n",
+ dma_async_is_tx_complete(sdev->sec_xm_p, cookie,
+ NULL, NULL));
+ }
+
+ out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1));
+
+ for (loop = 0; loop < total_len / 4; loop++)
+ dev_dbg(sdev->dev, "this is debug [%d] = %x\n", loop, out[loop]);
+
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), out,
+ total_len, rctx->offset, 0);
+
+ if (get_aes_mode(rctx) == JH7110_AES_MODE_CTR) {
+ rctx->dec_end[0] = jh7110_sec_read(sdev, JH7110_AES_IV0);
+ rctx->dec_end[1] = jh7110_sec_read(sdev, JH7110_AES_IV1);
+ rctx->dec_end[2] = jh7110_sec_read(sdev, JH7110_AES_IV2);
+ rctx->dec_end[3] = jh7110_sec_read(sdev, JH7110_AES_IV3);
+ } else
+ memcpy((void *)rctx->dec_end, ((void *)out) + total_len - JH7110_AES_IV_LEN,
+ JH7110_AES_IV_LEN);
+
+ dma_unmap_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+ dma_unmap_sg(sdev->dev, &ctx->sg[1], 1, DMA_FROM_DEVICE);
+
+ alg_cr.v = 0;
+ alg_cr.clear = 1;
+ jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v);
+
+ return 0;
+}
+
+static int jh7110_cryp_write_out_cpu(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ unsigned int *buffer, *out;
+ unsigned char *ci, *co;
+ int total_len, mlen, loop, count;
+
+ total_len = rctx->bufcnt;
+ buffer = (unsigned int *)sdev->aes_data;
+ out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1));
+
+ while (total_len >= 16) {
+ for (loop = 0; loop < 4; loop++, buffer++)
+ jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer);
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ for (loop = 0; loop < 4; loop++, out++) {
+ *out = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_sec_read[%d] = %x\n", loop, *out);
+ }
+ total_len -= 16;
+ }
+
+ if (total_len > 0) {
+ mlen = total_len;
+ for (; total_len >= 4; total_len -= 4, buffer++)
+ jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer);
+
+ ci = (unsigned char *)buffer;
+ for (; total_len > 0; total_len--, ci++)
+ jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci);
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+ for (; mlen >= 4; mlen -= 4, out++) {
+ *out = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_sec_read(mlen)[%d] = %x\n", loop, *out);
+ }
+ co = (unsigned char *)out;
+ for (; mlen > 0; mlen--, co++) {
+ *co = jh7110_sec_readb(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_sec_read(co)[%d] = %x\n", loop, *co);
+ }
+ }
+
+ if (is_ccm(rctx)) {
+ if (jh7110_aes_wait_busy(ctx))
+ return -EBUSY;
+
+ for (loop = 0; loop < 3 + rctx->authsize / 4; loop++) {
+ rctx->aes_nonce[loop] = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_auth[%d] = %x\n", loop, rctx->aes_nonce[loop]);
+ }
+ }
+
+ if (rctx->bufcnt >= rctx->total_out)
+ count = rctx->total_out;
+ else
+ count = rctx->bufcnt;
+
+ out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1));
+
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), out,
+ count, rctx->offset, 0);
+
+ if (get_aes_mode(rctx) == JH7110_AES_MODE_CTR) {
+ rctx->dec_end[0] = jh7110_sec_read(sdev, JH7110_AES_IV0);
+ rctx->dec_end[1] = jh7110_sec_read(sdev, JH7110_AES_IV1);
+ rctx->dec_end[2] = jh7110_sec_read(sdev, JH7110_AES_IV2);
+ rctx->dec_end[3] = jh7110_sec_read(sdev, JH7110_AES_IV3);
+ } else
+ memcpy((void *)rctx->dec_end, ((void *)out) + count - JH7110_AES_IV_LEN,
+ JH7110_AES_IV_LEN);
+
+ out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1));
+
+ for (loop = 0; loop < rctx->bufcnt >> 2; loop++)
+ dev_dbg(sdev->dev, "aes_out[%d] = %x\n", loop, out[loop]);
+
+ return 0;
+}
+
+static int jh7110_cryp_write_data(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ size_t data_len, total, count, data_buf_len, data_offset;
+ int ret;
+ bool fragmented = false;
+
+ if (unlikely(!rctx->total_in)) {
+ dev_warn(sdev->dev, "No more data to process\n");
+ return -EINVAL;
+ }
+
+ sdev->cry_type = JH7110_AES_TYPE;
+
+ /* ctr counter overflow. */
+ fragmented = jh7110_check_counter_overflow(ctx, 0);
+
+ total = 0;
+ rctx->offset = 0;
+ rctx->data_offset = 0;
+
+ if (is_ccm(rctx)) {
+ int index = AES_BLOCK_SIZE;
+
+ if (rctx->assoclen <= 65280) {
+ ((u8 *)sdev->aes_data)[index] = (rctx->assoclen >> 8) & 0xff;
+ ((u8 *)sdev->aes_data)[index + 1] = rctx->assoclen & 0xff;
+ rctx->data_offset = index + 2;
+ } else {
+ ((u8 *)sdev->aes_data)[index] = 0xff;
+ ((u8 *)sdev->aes_data)[index + 1] = 0xfe;
+ ((u8 *)sdev->aes_data)[index + 2] = rctx->assoclen & 0xFF000000;
+ ((u8 *)sdev->aes_data)[index + 3] = rctx->assoclen & 0x00FF0000;
+ ((u8 *)sdev->aes_data)[index + 4] = rctx->assoclen & 0x0000FF00;
+ ((u8 *)sdev->aes_data)[index + 5] = rctx->assoclen & 0x000000FF;
+ rctx->data_offset = index + 6;
+ }
+ }
+ data_offset = rctx->data_offset;
+ while (total < rctx->total_in) {
+ data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen) - data_offset;
+ count = min(rctx->total_in - rctx->offset, data_buf_len);
+
+ /* ctr counter overflow. */
+ if (fragmented && rctx->ctr_over_count != 0) {
+ if (count >= rctx->ctr_over_count)
+ count = rctx->ctr_over_count;
+ }
+ data_len = jh7110_cryp_get_from_sg(rctx, rctx->offset, count, data_offset);
+
+ if (data_len < 0)
+ return data_len;
+ if (data_len != count)
+ return -EINVAL;
+
+ rctx->bufcnt = data_len + data_offset;
+ total += data_len;
+
+ if (!is_encrypt(rctx) && (total + rctx->assoclen >= rctx->total_in))
+ rctx->bufcnt = rctx->bufcnt - rctx->authsize;
+
+ if (rctx->bufcnt) {
+ memcpy((void *)rctx->msg_end, (void *)sdev->aes_data + rctx->bufcnt - JH7110_AES_IV_LEN,
+ JH7110_AES_IV_LEN);
+ if (sdev->use_dma)
+ ret = jh7110_cryp_write_out_dma(ctx);
+ else
+ ret = jh7110_cryp_write_out_cpu(ctx);
+
+ if (ret)
+ return ret;
+ }
+ data_offset = 0;
+ rctx->offset += data_len;
+ fragmented = jh7110_check_counter_overflow(ctx, data_len);
+ }
+
+ return ret;
+}
+
+static int jh7110_cryp_write_aad(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ unsigned int *buffer, *out;
+ unsigned char *ci;
+ int total_len, mlen, loop;
+
+ total_len = rctx->bufcnt;
+ buffer = (unsigned int *)sdev->aes_data;
+ out = (unsigned int *)(sdev->aes_data + (JH7110_MSG_BUFFER_SIZE >> 1));
+
+ ci = (unsigned char *)buffer;
+ jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci);
+ ci++;
+ jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci);
+ ci++;
+ total_len -= 2;
+ buffer = (unsigned int *)ci;
+ for (loop = 0; loop < 3; loop++, buffer++)
+ jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer);
+
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+ total_len -= 12;
+ for (loop = 0; loop < 4; loop++, out++) {
+ unsigned int tmp;
+
+ tmp = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_aad_read[%d] = %x\n", loop, tmp);
+ }
+
+ while (total_len >= 16) {
+ for (loop = 0; loop < 4; loop++, buffer++)
+ jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer);
+
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ for (loop = 0; loop < 4; loop++, out++) {
+ unsigned int tmp;
+
+ tmp = jh7110_sec_read(sdev, JH7110_AES_AESDIO0R);
+ dev_dbg(sdev->dev, "aes_aad_read[%d] = %x\n", loop, tmp);
+ }
+ total_len -= 16;
+ }
+
+ if (total_len > 0) {
+ mlen = total_len;
+ for (; total_len >= 4; total_len -= 4, buffer++)
+ jh7110_sec_write(sdev, JH7110_AES_AESDIO0R, *buffer);
+
+ ci = (unsigned char *)buffer;
+ for (; total_len > 0; total_len--, ci++)
+ jh7110_sec_writeb(sdev, JH7110_AES_AESDIO0R, *ci);
+
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+ }
+
+ if (jh7110_aes_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static int jh7110_cryp_xcm_write_data(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ size_t data_len, total, count, data_buf_len, offset, auths;
+ unsigned int *out;
+ int loop;
+ int ret;
+ bool fragmented = false;
+
+ if (unlikely(!rctx->total_in)) {
+ dev_warn(sdev->dev, "No more data to process\n");
+ return -EINVAL;
+ }
+
+ sdev->cry_type = JH7110_AES_TYPE;
+
+ /* ctr counter overflow. */
+ fragmented = jh7110_check_counter_overflow(ctx, 0);
+
+ total = 0;
+ rctx->offset = 0;
+ rctx->data_offset = 0;
+ offset = 0;
+
+ while (total < rctx->assoclen) {
+ data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen);
+ count = min(rctx->assoclen - offset, data_buf_len);
+ count = min(count, rctx->assoclen - total);
+ data_len = jh7110_cryp_get_from_sg(rctx, offset, count, 0);
+ if (data_len < 0)
+ return data_len;
+ if (data_len != count)
+ return -EINVAL;
+
+ offset += data_len;
+ rctx->offset += data_len;
+ if ((data_len + 2) & 0xF) {
+ memset(sdev->aes_data + data_len, 0, 16 - ((data_len + 2) & 0xf));
+ data_len += 16 - ((data_len + 2) & 0xf);
+ }
+
+ rctx->bufcnt = data_len;
+
+ total += data_len;
+ ret = jh7110_cryp_write_aad(ctx);
+ }
+
+ total = 0;
+ auths = 0;
+
+ while (total + auths < rctx->total_in - rctx->assoclen) {
+ data_buf_len = sdev->data_buf_len - (sdev->data_buf_len % ctx->keylen);
+ count = min(rctx->total_in - rctx->offset, data_buf_len);
+
+ if (is_encrypt(rctx))
+ count = min(count, rctx->total_in - rctx->assoclen - total);
+ else {
+ //count = min (count, rctx->total_in - rctx->assoclen - total);
+ count = min(count, rctx->total_in - rctx->assoclen - total - rctx->authsize);
+ auths = rctx->authsize;
+ }
+
+ /* ctr counter overflow. */
+ if (fragmented && rctx->ctr_over_count != 0) {
+ if (count >= rctx->ctr_over_count)
+ count = rctx->ctr_over_count;
+ }
+
+ data_len = jh7110_cryp_get_from_sg(rctx, rctx->offset, count, 0);
+
+ if (data_len < 0)
+ return data_len;
+ if (data_len != count)
+ return -EINVAL;
+
+ if (data_len % JH7110_AES_IV_LEN) {
+ memset(sdev->aes_data + data_len, 0, JH7110_AES_IV_LEN - (data_len % JH7110_AES_IV_LEN));
+ data_len = data_len + (JH7110_AES_IV_LEN - (data_len % JH7110_AES_IV_LEN));
+ }
+
+ rctx->bufcnt = data_len;
+ total += data_len;
+
+ if (!is_encrypt(rctx) && (total + rctx->assoclen >= rctx->total_in))
+ rctx->bufcnt = rctx->bufcnt - rctx->authsize;
+
+ if (rctx->bufcnt) {
+ memcpy((void *)rctx->msg_end, (void *)sdev->aes_data + rctx->bufcnt - JH7110_AES_IV_LEN,
+ JH7110_AES_IV_LEN);
+ out = (unsigned int *)sdev->aes_data;
+ for (loop = 0; loop < rctx->bufcnt / 4; loop++)
+ dev_dbg(sdev->dev, "aes_data[%d] = %x\n", loop, out[loop]);
+ ret = jh7110_cryp_write_out_cpu(ctx);
+
+ if (ret)
+ return ret;
+ }
+ rctx->offset += count;
+ offset += count;
+
+ fragmented = jh7110_check_counter_overflow(ctx, data_len);
+ }
+
+ return ret;
+}
+
+static int jh7110_gcm_zero_message_data(struct jh7110_sec_ctx *ctx)
+{
+ int ret;
+
+ ctx->rctx->bufcnt = 0;
+ ctx->rctx->offset = 0;
+ if (ctx->sdev->use_dma)
+ ret = jh7110_cryp_write_out_dma(ctx);
+ else
+ ret = jh7110_cryp_write_out_cpu(ctx);
+
+ return ret;
+}
+
+static int jh7110_cryp_cpu_start(struct jh7110_sec_ctx *ctx, struct jh7110_sec_request_ctx *rctx)
+{
+ int ret;
+
+ ret = jh7110_cryp_write_data(ctx);
+ if (ret)
+ return ret;
+
+ ret = jh7110_cryp_finish_req(ctx, ret);
+
+ return ret;
+}
+
+static int jh7110_cryp_xcm_start(struct jh7110_sec_ctx *ctx, struct jh7110_sec_request_ctx *rctx)
+{
+ int ret;
+
+ ret = jh7110_cryp_xcm_write_data(ctx);
+ if (ret)
+ return ret;
+
+ ret = jh7110_cryp_finish_req(ctx, ret);
+
+ return ret;
+}
+
+static int jh7110_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);
+static int jh7110_cryp_prepare_cipher_req(struct crypto_engine *engine,
+ void *areq);
+
+static int jh7110_cryp_cra_init(struct crypto_skcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ ctx->sdev = jh7110_sec_find_dev(ctx);
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ ctx->sec_init = 0;
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx));
+
+ ctx->enginectx.op.do_one_request = jh7110_cryp_cipher_one_req;
+ ctx->enginectx.op.prepare_request = jh7110_cryp_prepare_cipher_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void jh7110_cryp_cra_exit(struct crypto_skcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int jh7110_cryp_aead_one_req(struct crypto_engine *engine, void *areq);
+static int jh7110_cryp_prepare_aead_req(struct crypto_engine *engine,
+ void *areq);
+
+static int jh7110_cryp_aes_aead_init(struct crypto_aead *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm);
+
+ ctx->sdev = jh7110_sec_find_dev(ctx);
+
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ crypto_aead_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx));
+
+ ctx->enginectx.op.do_one_request = jh7110_cryp_aead_one_req;
+ ctx->enginectx.op.prepare_request = jh7110_cryp_prepare_aead_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void jh7110_cryp_aes_aead_exit(struct crypto_aead *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm);
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int jh7110_cryp_crypt(struct skcipher_request *req, unsigned long flags)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(
+ crypto_skcipher_reqtfm(req));
+ struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req);
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx->flags = flags;
+ rctx->req_type = JH7110_ABLK_REQ;
+
+ return crypto_transfer_skcipher_request_to_engine(sdev->engine, req);
+}
+
+static int jh7110_cryp_aead_crypt(struct aead_request *req, unsigned long flags)
+{
+ struct jh7110_sec_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct jh7110_sec_request_ctx *rctx = aead_request_ctx(req);
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx->flags = flags;
+ rctx->req_type = JH7110_AEAD_REQ;
+
+ return crypto_transfer_aead_request_to_engine(sdev->engine, req);
+}
+
+static int jh7110_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ {
+ int loop;
+
+ for (loop = 0; loop < keylen; loop++)
+ pr_debug("key[%d] = %x ctx->key[%d] = %x\n", loop, key[loop], loop, ctx->key[loop]);
+ }
+ return 0;
+}
+
+static int jh7110_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+ else
+ return jh7110_cryp_setkey(tfm, key, keylen);
+}
+
+static int jh7110_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_aead_ctx(tfm);
+
+ if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256) {
+ return -EINVAL;
+ }
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ {
+ int loop;
+
+ for (loop = 0; loop < keylen; loop++)
+ pr_debug("key[%d] = %x ctx->key[%d] = %x\n", loop, key[loop], loop, ctx->key[loop]);
+ }
+ return 0;
+}
+
+static int jh7110_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
+}
+
+static int jh7110_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 4:
+ case 6:
+ case 8:
+ case 10:
+ case 12:
+ case 14:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int jh7110_cryp_aes_ecb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_ECB | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_ecb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_ECB);
+}
+
+static int jh7110_cryp_aes_cbc_encrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CBC | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_cbc_decrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CBC);
+}
+
+static int jh7110_cryp_aes_cfb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CFB | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_cfb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CFB);
+}
+
+static int jh7110_cryp_aes_ofb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_OFB | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_ofb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_OFB);
+}
+
+static int jh7110_cryp_aes_ctr_encrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CTR | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_ctr_decrypt(struct skcipher_request *req)
+{
+ return jh7110_cryp_crypt(req, JH7110_AES_MODE_CTR);
+}
+
+static int jh7110_cryp_aes_gcm_encrypt(struct aead_request *req)
+{
+ return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_GCM | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_gcm_decrypt(struct aead_request *req)
+{
+ return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_GCM);
+}
+
+static int jh7110_cryp_aes_ccm_encrypt(struct aead_request *req)
+{
+ return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_CCM | FLG_ENCRYPT);
+}
+
+static int jh7110_cryp_aes_ccm_decrypt(struct aead_request *req)
+{
+ return jh7110_cryp_aead_crypt(req, JH7110_AES_MODE_CCM);
+}
+
+static int jh7110_cryp_prepare_req(struct skcipher_request *req,
+ struct aead_request *areq)
+{
+ struct jh7110_sec_ctx *ctx;
+ struct jh7110_sec_dev *sdev;
+ struct jh7110_sec_request_ctx *rctx;
+ int ret;
+
+ if (!req && !areq)
+ return -EINVAL;
+
+ ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
+ crypto_aead_ctx(crypto_aead_reqtfm(areq));
+
+ sdev = ctx->sdev;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
+
+ mutex_lock(&ctx->sdev->lock);
+
+ rctx->sdev = sdev;
+ ctx->rctx = rctx;
+
+ if (req) {
+ rctx->req.sreq = req;
+ rctx->req_type = JH7110_ABLK_REQ;
+ rctx->total_in = req->cryptlen;
+ rctx->total_out = rctx->total_in;
+ rctx->authsize = 0;
+ rctx->assoclen = 0;
+ } else {
+ /*
+ * Length of input and output data:
+ * Encryption case:
+ * INPUT = AssocData || PlainText
+ * <- assoclen -> <- cryptlen ->
+ * <------- total_in ----------->
+ *
+ * OUTPUT = AssocData || CipherText || AuthTag
+ * <- assoclen -> <- cryptlen -> <- authsize ->
+ * <---------------- total_out ----------------->
+ *
+ * Decryption case:
+ * INPUT = AssocData || CipherText || AuthTag
+ * <- assoclen -> <--------- cryptlen --------->
+ * <- authsize ->
+ * <---------------- total_in ------------------>
+ *
+ * OUTPUT = AssocData || PlainText
+ * <- assoclen -> <- crypten - authsize ->
+ * <---------- total_out ----------------->
+ */
+ rctx->req.areq = areq;
+ rctx->req_type = JH7110_AEAD_REQ;
+ rctx->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
+ rctx->total_in = areq->assoclen + areq->cryptlen;
+ rctx->assoclen = areq->assoclen;
+ if (is_encrypt(rctx))
+ /* Append auth tag to output */
+ rctx->total_out = rctx->total_in + rctx->authsize;
+ else
+ /* No auth tag in output */
+ rctx->total_out = rctx->total_in - rctx->authsize;
+ }
+
+ rctx->sg = req ? req->src : areq->src;
+ rctx->out_sg = req ? req->dst : areq->dst;
+
+ rctx->in_sg_len = sg_nents_for_len(rctx->sg, rctx->total_in);
+ if (rctx->in_sg_len < 0) {
+ dev_err(sdev->dev, "Cannot get in_sg_len\n");
+ ret = rctx->in_sg_len;
+ goto out;
+ }
+
+ rctx->out_sg_len = sg_nents_for_len(rctx->out_sg, rctx->total_out);
+ if (rctx->out_sg_len < 0) {
+ dev_err(sdev->dev, "Cannot get out_sg_len\n");
+ ret = rctx->out_sg_len;
+ goto out;
+ }
+ if (1) {
+ int loop;
+ u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)];
+
+ scatterwalk_map_and_copy(idigest, rctx->req.areq->src, rctx->total_in - rctx->authsize,
+ rctx->authsize, 0);
+ for (loop = 0; loop < rctx->authsize / 4; loop++) {
+ dev_dbg(sdev->dev, "idigest[%d] = %x\n",
+ loop, idigest[loop]);
+ }
+ }
+
+ ret = jh7110_cryp_hw_init(ctx);
+ if (ret)
+ goto out;
+
+ return ret;
+out:
+ mutex_unlock(&sdev->doing);
+
+ return ret;
+}
+
+static int jh7110_cryp_prepare_cipher_req(struct crypto_engine *engine,
+ void *areq)
+{
+ struct skcipher_request *req = container_of(areq,
+ struct skcipher_request,
+ base);
+ return jh7110_cryp_prepare_req(req, NULL);
+}
+
+static int jh7110_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct skcipher_request *req = container_of(areq,
+ struct skcipher_request,
+ base);
+ struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req);
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(
+ crypto_skcipher_reqtfm(req));
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret;
+
+ if (!sdev)
+ return -ENODEV;
+
+ ret = jh7110_cryp_cpu_start(ctx, rctx);
+
+ mutex_unlock(&sdev->lock);
+
+ return ret;
+}
+
+static int jh7110_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req = container_of(areq, struct aead_request,
+ base);
+ return jh7110_cryp_prepare_req(NULL, req);
+}
+
+static int jh7110_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req = container_of(areq, struct aead_request,
+ base);
+ struct jh7110_sec_request_ctx *rctx = aead_request_ctx(req);
+ struct jh7110_sec_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret;
+
+ if (!sdev)
+ return -ENODEV;
+
+ if (unlikely(!rctx->req.areq->assoclen &&
+ !jh7110_cryp_get_input_text_len(ctx))) {
+ /* No input data to process: get tag and finish */
+ jh7110_gcm_zero_message_data(ctx);
+ jh7110_cryp_finish_req(ctx, 0);
+ ret = 0;
+ goto out;
+ }
+
+ ret = jh7110_cryp_xcm_start(ctx, rctx);
+
+ out:
+ mutex_unlock(&ctx->sdev->lock);
+
+ return ret;
+}
+
+static struct skcipher_alg crypto_algs[] = {
+{
+ .base.cra_name = "ecb(aes)",
+ .base.cra_driver_name = "jh7110-ecb-aes",
+ .base.cra_priority = 200,
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_alignmask = 0xf,
+ .base.cra_module = THIS_MODULE,
+ .init = jh7110_cryp_cra_init,
+ .exit = jh7110_cryp_cra_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = jh7110_cryp_aes_setkey,
+ .encrypt = jh7110_cryp_aes_ecb_encrypt,
+ .decrypt = jh7110_cryp_aes_ecb_decrypt,
+},
+{
+ .base.cra_name = "cbc(aes)",
+ .base.cra_driver_name = "jh7110-cbc-aes",
+ .base.cra_priority = 200,
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_alignmask = 0xf,
+ .base.cra_module = THIS_MODULE,
+ .init = jh7110_cryp_cra_init,
+ .exit = jh7110_cryp_cra_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = jh7110_cryp_aes_setkey,
+ .encrypt = jh7110_cryp_aes_cbc_encrypt,
+ .decrypt = jh7110_cryp_aes_cbc_decrypt,
+},
+{
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "jh7110-ctr-aes",
+ .base.cra_priority = 200,
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_alignmask = 0xf,
+ .base.cra_module = THIS_MODULE,
+ .init = jh7110_cryp_cra_init,
+ .exit = jh7110_cryp_cra_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = jh7110_cryp_aes_setkey,
+ .encrypt = jh7110_cryp_aes_ctr_encrypt,
+ .decrypt = jh7110_cryp_aes_ctr_decrypt,
+},
+{
+ .base.cra_name = "cfb(aes)",
+ .base.cra_driver_name = "jh7110-cfb-aes",
+ .base.cra_priority = 200,
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_alignmask = 0xf,
+ .base.cra_module = THIS_MODULE,
+ .init = jh7110_cryp_cra_init,
+ .exit = jh7110_cryp_cra_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = jh7110_cryp_aes_setkey,
+ .encrypt = jh7110_cryp_aes_cfb_encrypt,
+ .decrypt = jh7110_cryp_aes_cfb_decrypt,
+},
+{
+ .base.cra_name = "ofb(aes)",
+ .base.cra_driver_name = "jh7110-ofb-aes",
+ .base.cra_priority = 200,
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_alignmask = 0xf,
+ .base.cra_module = THIS_MODULE,
+ .init = jh7110_cryp_cra_init,
+ .exit = jh7110_cryp_cra_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = jh7110_cryp_aes_setkey,
+ .encrypt = jh7110_cryp_aes_ofb_encrypt,
+ .decrypt = jh7110_cryp_aes_ofb_decrypt,
+},
+};
+
+static struct aead_alg aead_algs[] = {
+{
+ .setkey = jh7110_cryp_aes_aead_setkey,
+ .setauthsize = jh7110_cryp_aes_gcm_setauthsize,
+ .encrypt = jh7110_cryp_aes_gcm_encrypt,
+ .decrypt = jh7110_cryp_aes_gcm_decrypt,
+ .init = jh7110_cryp_aes_aead_init,
+ .exit = jh7110_cryp_aes_aead_exit,
+ .ivsize = 12,
+ .maxauthsize = AES_BLOCK_SIZE,
+
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "jh7110-gcm-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+
+{
+ .setkey = jh7110_cryp_aes_aead_setkey,
+ .setauthsize = jh7110_cryp_aes_ccm_setauthsize,
+ .encrypt = jh7110_cryp_aes_ccm_encrypt,
+ .decrypt = jh7110_cryp_aes_ccm_decrypt,
+ .init = jh7110_cryp_aes_aead_init,
+ .exit = jh7110_cryp_aes_aead_exit,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+
+ .base = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "jh7110-ccm-aes",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ },
+},
+};
+
+int jh7110_aes_register_algs(void)
+{
+ int ret;
+
+ ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
+ if (ret) {
+ pr_err("Could not register algs\n");
+ goto err_algs;
+ }
+
+ ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
+ if (ret)
+ goto err_aead_algs;
+
+ return 0;
+
+err_aead_algs:
+ crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
+err_algs:
+ return ret;
+}
+
+void jh7110_aes_unregister_algs(void)
+{
+ crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
+ crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <william.qiu@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/scatterlist.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <crypto/hash.h>
+
+#include <linux/dma-direct.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/des.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+
+#include "jh7110-pl080.h"
+#include "jh7110-str.h"
+
+#define FLG_DES_MODE_MASK GENMASK(1, 0)
+
+#define FLAGS_DES_ENCRYPT BIT(4)
+
+enum {
+ JH7110_CRYPTO_NOT_ALIGNED = 1,
+ JH7110_CRYPTO_BAD_DATA_LENGTH,
+};
+
+#define JH7110_CRYPTO_DATA_COPIED BIT(0)
+#define JH7110_CRYPTO_SG_COPIED BIT(1)
+
+#define JH7110_CRYPTO_COPY_MASK 0x3
+
+#define JH7110_CRYPTO_COPY_DATA BIT(0)
+#define JH7110_CRYPTO_FORCE_COPY BIT(1)
+#define JH7110_CRYPTO_ZERO_BUF BIT(2)
+#define JH7110_CRYPTO_FORCE_SINGLE_ENTRY BIT(3)
+
+#define FLAGS_IN_DATA_ST_SHIFT 8
+#define FLAGS_OUT_DATA_ST_SHIFT 10
+
+
+static inline int jh7110_des_wait_busy(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_DES_DAECSR_OFFSET, status,
+ !(status & JH7110_DES_BUSY), 10, 100000);
+}
+
+static inline int jh7110_des_wait_done(struct jh7110_sec_dev *sdev)
+{
+ int ret = -1;
+
+ mutex_lock(&sdev->doing);
+
+ if (sdev->done_flags & JH7110_DES_DONE)
+ ret = 0;
+
+ mutex_unlock(&sdev->doing);
+
+ return ret;
+}
+
+static inline int is_des_ecb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_ECB;
+}
+
+static inline int is_des_cbc(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_CBC;
+}
+
+static inline int is_des_ofb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_OFB;
+}
+
+static inline int is_des_cfb(struct jh7110_sec_request_ctx *rctx)
+{
+ return (rctx->flags & FLG_DES_MODE_MASK) == JH7110_DES_MODE_CFB;
+}
+
+static inline int get_des_mode(struct jh7110_sec_request_ctx *rctx)
+{
+ return rctx->flags & FLG_DES_MODE_MASK;
+}
+
+static inline int is_des_encrypt(struct jh7110_sec_request_ctx *rctx)
+{
+ return !!(rctx->flags & FLAGS_DES_ENCRYPT);
+}
+
+static inline int is_des_decrypt(struct jh7110_sec_request_ctx *rctx)
+{
+ return !is_des_encrypt(rctx);
+}
+
+static inline void jh7110_des_reset(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+
+ rctx->csr.des_csr.v = 0;
+ rctx->csr.des_csr.reset = 1;
+ jh7110_sec_write(ctx->sdev, JH7110_DES_DAECSR_OFFSET, rctx->csr.des_csr.v);
+
+}
+
+static int jh7110_des_hw_write_key(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ u64 key64, *k;
+ int loop;
+
+ key64 = ctx->keylen / sizeof(u64);
+
+ switch (key64) {
+ case 1:
+ rctx->csr.des_csr.ks = 0x0;
+ break;
+ case 2:
+ rctx->csr.des_csr.ks = 0x2;
+ break;
+ case 3:
+ rctx->csr.des_csr.ks = 0x3;
+ break;
+ }
+
+ jh7110_sec_write(ctx->sdev, JH7110_DES_DAECSR_OFFSET, rctx->csr.des_csr.v);
+
+ k = (u64 *)ctx->key;
+ for (loop = 0; loop < key64; loop++) {
+ jh7110_sec_writeq(sdev, JH7110_DES_DAEKIN1R_HI_OFFSET + loop * 8, *k);
+ k++;
+ }
+
+ return 0;
+}
+
+
+
+static int jh7110_des_hw_init(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret;
+ u32 hw_mode;
+
+ jh7110_des_reset(ctx);
+
+ hw_mode = get_des_mode(ctx->rctx);
+
+ rctx->csr.des_csr.v = 0;
+ rctx->csr.des_csr.mode = rctx->flags & FLG_DES_MODE_MASK;
+ rctx->csr.des_csr.encryt = !!(rctx->flags & FLAGS_DES_ENCRYPT);
+ rctx->csr.des_csr.ie = 1;
+ rctx->csr.des_csr.bitmode = JH7110_DES_BITMODE_64;
+ rctx->csr.des_csr.en = 1;
+ rctx->csr.des_csr.disturb = 1;
+ rctx->csr.des_csr.vdes_en = 1;
+
+ switch (hw_mode) {
+ case JH7110_DES_MODE_ECB:
+ ret = jh7110_des_hw_write_key(ctx);
+ break;
+ case JH7110_DES_MODE_OFB:
+ case JH7110_DES_MODE_CFB:
+ case JH7110_DES_MODE_CBC:
+ jh7110_sec_writeq(sdev, JH7110_DES_DAEIVINR_HI_OFFSET, *(u64 *)rctx->req.sreq->iv);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+static void jh7110_des_dma_callback(void *param)
+{
+ struct jh7110_sec_dev *sdev = param;
+
+ complete(&sdev->sec_comp_p);
+}
+
+static int jh7110_des_dma_start(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct dma_async_tx_descriptor *in_desc, *out_desc;
+ struct dma_slave_config cfg;
+ int err;
+ int ret;
+
+ rctx->total = (rctx->total & 0x3) ? 1 : 0;
+ sg_init_table(&sdev->in_sg, 1);
+ sg_set_buf(&sdev->in_sg, sdev->des_data, rctx->total);
+ sg_dma_address(&sdev->in_sg) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->des_data));
+ sg_dma_len(&sdev->in_sg) = rctx->total;
+
+ sg_init_table(&sdev->out_sg, 1);
+ sg_set_buf(&sdev->out_sg, sdev->des_data, rctx->total);
+ sg_dma_address(&sdev->out_sg) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->des_data));
+ sg_dma_len(&sdev->out_sg) = rctx->total;
+
+ err = dma_map_sg(sdev->dev, &sdev->in_sg, 1, DMA_TO_DEVICE);
+ if (!err) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ err = dma_map_sg(sdev->dev, &sdev->out_sg, 1, DMA_FROM_DEVICE);
+ if (!err) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+
+ dma_sync_sg_for_device(sdev->dev, &sdev->in_sg, sdev->in_sg_len, DMA_TO_DEVICE);
+
+ cfg.src_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET;
+ cfg.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET;
+ cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ cfg.src_maxburst = 4;
+ cfg.dst_maxburst = 4;
+
+ ret = dmaengine_slave_config(sdev->sec_xm_m, &cfg);
+ if (ret) {
+ dev_err(sdev->dev, "can't configure IN dmaengine slave: %d\n",
+ ret);
+ return ret;
+ }
+
+ in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &sdev->in_sg,
+ 1, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+
+ if (!in_desc) {
+ dev_err(sdev->dev, "IN prep_slave_sg() failed\n");
+ return -EINVAL;
+ }
+
+ in_desc->callback_param = sdev;
+
+ ret = dmaengine_slave_config(sdev->sec_xm_p, &cfg);
+ if (ret) {
+ dev_err(sdev->dev, "can't configure OUT dmaengine slave: %d\n",
+ ret);
+ return ret;
+ }
+
+ out_desc = dmaengine_prep_slave_sg(sdev->sec_xm_p, &sdev->out_sg,
+ 1, DMA_DEV_TO_MEM,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!out_desc) {
+ dev_err(sdev->dev, "OUT prep_slave_sg() failed\n");
+ return -EINVAL;
+ }
+
+ out_desc->callback = jh7110_des_dma_callback;
+ out_desc->callback_param = sdev;
+
+ dmaengine_submit(in_desc);
+ dmaengine_submit(out_desc);
+
+ dma_async_issue_pending(sdev->sec_xm_m);
+ dma_async_issue_pending(sdev->sec_xm_p);
+
+ dma_unmap_sg(sdev->dev, &sdev->in_sg, 1, DMA_TO_DEVICE);
+ dma_unmap_sg(sdev->dev, &sdev->out_sg, 1, DMA_FROM_DEVICE);
+
+ return 0;
+}
+
+static int jh7110_des_cpu_start(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct sg_mapping_iter mi;
+ void *buffer;
+ int total_len, mlen, bs, offset;
+ u64 value;
+ u8 bvalue;
+
+ total_len = rctx->total;
+
+ sg_miter_start(&mi, rctx->in_sg, rctx->sg_len,
+ SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+
+ bs = 8;
+ mlen = 0;
+
+ while (total_len) {
+ switch (!(total_len >> 3)) {
+ case 0:
+ bs = 8;
+ break;
+ case 1:
+ bs = 1;
+ break;
+ }
+ if (!mlen) {
+ sg_miter_next(&mi);
+ mlen = mi.length;
+ if (!mlen) {
+ pr_err("sg miter failure.\n");
+ return -EINVAL;
+ }
+ buffer = mi.addr;
+ }
+
+ bs = min(bs, mlen);
+
+ switch (bs) {
+ case 8:
+ jh7110_sec_writeq(sdev, JH7110_DES_DAEDINR_HI_OFFSET,
+ *(u64 *)buffer);
+ break;
+ case 1:
+ jh7110_sec_writeb(sdev, JH7110_DES_DAEDINR_HI_OFFSET,
+ *(u8 *)buffer);
+ break;
+ }
+ mlen -= bs;
+ total_len -= bs;
+ buffer += bs;
+ }
+
+ sg_miter_stop(&mi);
+
+ if (jh7110_des_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ if (jh7110_des_wait_done(sdev)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_done error\n");
+ return -ETIMEDOUT;
+ }
+
+ total_len = rctx->total;
+ offset = 0;
+
+ while (total_len) {
+ switch (!(total_len >> 3)) {
+ case 0:
+ value = jh7110_sec_readq(sdev, JH7110_DES_DAEDINR_HI_OFFSET);
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), (void *)&value,
+ sizeof(u64), offset, 0);
+ offset += sizeof(u64);
+ total_len -= 8;
+ break;
+ case 1:
+ bvalue = jh7110_sec_readb(sdev, JH7110_DES_DAEDINR_HI_OFFSET);
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), (void *)&bvalue,
+ sizeof(u8), offset, 0);
+ offset += sizeof(u8);
+ total_len -= 1;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int jh7110_des_crypt_start(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret;
+
+ if (sdev->use_dma)
+ ret = jh7110_des_dma_start(ctx);
+ else
+ ret = jh7110_des_cpu_start(ctx);
+
+ return ret;
+}
+
+static int jh7110_des_prepare_req(struct crypto_engine *engine,
+ void *areq);
+static int jh7110_des_one_req(struct crypto_engine *engine, void *areq);
+
+
+static int jh7110_des_cra_init(struct crypto_skcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ ctx->sdev = jh7110_sec_find_dev(ctx);
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ ctx->sec_init = 0;
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx));
+
+ ctx->enginectx.op.do_one_request = jh7110_des_one_req;
+ ctx->enginectx.op.prepare_request = jh7110_des_prepare_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void jh7110_des_cra_exit(struct crypto_skcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int jh7110_des_crypt(struct skcipher_request *req, unsigned long mode)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(
+ crypto_skcipher_reqtfm(req));
+ struct jh7110_sec_request_ctx *rctx = skcipher_request_ctx(req);
+ struct jh7110_sec_dev *sdev;
+
+ if (!req->cryptlen)
+ return 0;
+
+ if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE))
+ return -EINVAL;
+
+ sdev = jh7110_sec_find_dev(ctx);
+ if (!sdev)
+ return -ENODEV;
+
+ rctx->flags = mode;
+
+ return crypto_transfer_skcipher_request_to_engine(sdev->engine, req);
+}
+
+static int jh7110_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+ int err;
+
+ err = verify_skcipher_des_key(tfm, key);
+ if (err)
+ return err;
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static int jh7110_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+ int err;
+
+ err = verify_skcipher_des3_key(tfm, key);
+ if (err)
+ return err;
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static int jh7110_des_ecb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_ECB);
+}
+
+static int jh7110_des_ecb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, JH7110_DES_MODE_ECB);
+}
+
+static int jh7110_des_cbc_encrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_CBC);
+}
+
+static int jh7110_des_cbc_decrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, JH7110_DES_MODE_CBC);
+}
+
+static int jh7110_des_cfb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_CFB);
+}
+
+static int jh7110_des_cfb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, JH7110_DES_MODE_CFB);
+}
+
+static int jh7110_des_ofb_encrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, FLAGS_DES_ENCRYPT | JH7110_DES_MODE_OFB);
+}
+
+static int jh7110_des_ofb_decrypt(struct skcipher_request *req)
+{
+ return jh7110_des_crypt(req, JH7110_DES_MODE_OFB);
+}
+
+
+static int jh7110_des_prepare_req(struct crypto_engine *engine,
+ void *areq)
+{
+ struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(
+ crypto_skcipher_reqtfm(req));
+ struct jh7110_sec_dev *sdev = jh7110_sec_find_dev(ctx);
+ struct jh7110_sec_request_ctx *rctx;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx = skcipher_request_ctx(req);
+
+ /* assign new request to device */
+ rctx->req.sreq = req;
+ rctx->req_type = JH7110_ABLK_REQ;
+ rctx->total_in = req->cryptlen;
+ rctx->total_out = rctx->total_in;
+ rctx->authsize = 0;
+ rctx->assoclen = 0;
+
+
+ rctx->in_sg = req->src;
+ rctx->out_sg = req->dst;
+
+ rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
+ if (rctx->in_sg_len < 0)
+ return rctx->in_sg_len;
+
+ rctx->out_sg_len = sg_nents_for_len(rctx->out_sg, rctx->total);
+ if (rctx->out_sg_len < 0)
+ return rctx->out_sg_len;
+
+ rctx = skcipher_request_ctx(req);
+ ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ rctx->ctx = ctx;
+ rctx->sdev = sdev;
+ ctx->sdev = sdev;
+ ctx->rctx = rctx;
+
+ return jh7110_des_hw_init(ctx);
+}
+
+static int jh7110_des_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct skcipher_request *req = container_of(areq,
+ struct skcipher_request,
+ base);
+ struct jh7110_sec_ctx *ctx = crypto_skcipher_ctx(
+ crypto_skcipher_reqtfm(req));
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret;
+
+ if (!sdev)
+ return -ENODEV;
+
+ ret = jh7110_des_crypt_start(ctx);
+
+ mutex_unlock(&sdev->lock);
+
+ return ret;
+}
+
+static struct skcipher_alg algs_des_jh7110[] = {
+ {
+ .base.cra_name = "ecb(des)",
+ .base.cra_driver_name = "ecb-des-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .setkey = jh7110_des_setkey,
+ .encrypt = jh7110_des_ecb_encrypt,
+ .decrypt = jh7110_des_ecb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "cbc(des)",
+ .base.cra_driver_name = "cbc-des-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des_setkey,
+ .encrypt = jh7110_des_cbc_encrypt,
+ .decrypt = jh7110_des_cbc_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "cfb(des)",
+ .base.cra_driver_name = "cfb-des-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des_setkey,
+ .encrypt = jh7110_des_cfb_encrypt,
+ .decrypt = jh7110_des_cfb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "ofb(des)",
+ .base.cra_driver_name = "ofb-des-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des_setkey,
+ .encrypt = jh7110_des_ofb_encrypt,
+ .decrypt = jh7110_des_ofb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "ecb(des3)",
+ .base.cra_driver_name = "ecb-des3-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .setkey = jh7110_des3_setkey,
+ .encrypt = jh7110_des_ecb_encrypt,
+ .decrypt = jh7110_des_ecb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "cbc(des3)",
+ .base.cra_driver_name = "cbc-des3-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des3_setkey,
+ .encrypt = jh7110_des_cbc_encrypt,
+ .decrypt = jh7110_des_cbc_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "cfb(des3)",
+ .base.cra_driver_name = "cfb-des3-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des3_setkey,
+ .encrypt = jh7110_des_cfb_encrypt,
+ .decrypt = jh7110_des_cfb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+ {
+ .base.cra_name = "ofb(des3)",
+ .base.cra_driver_name = "ofb-des3-jh7110",
+ .base.cra_priority = 100,
+ .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = jh7110_des3_setkey,
+ .encrypt = jh7110_des_ofb_encrypt,
+ .decrypt = jh7110_des_ofb_decrypt,
+ .init = jh7110_des_cra_init,
+ .exit = jh7110_des_cra_exit,
+ },
+};
+
+int jh7110_des_register_algs(void)
+{
+ int ret;
+
+ ret = crypto_register_skciphers(algs_des_jh7110, ARRAY_SIZE(algs_des_jh7110));
+ if (ret) {
+ pr_debug("Could not register algs\n");
+ return ret;
+ }
+ return ret;
+}
+
+void jh7110_des_unregister_algs(void)
+{
+ crypto_unregister_skciphers(algs_des_jh7110, ARRAY_SIZE(algs_des_jh7110));
+}
+
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+#include <linux/mod_devicetable.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/crypto.h>
+#include <linux/io.h>
+
+#include <linux/delay.h>
+#include <linux/dma-direct.h>
+#include <crypto/scatterwalk.h>
+
+#include "jh7110-str.h"
+
+#define JH7110_RSA_KEYSZ_LEN (2048 >> 2)
+#define JH7110_RSA_KEY_SIZE (JH7110_RSA_KEYSZ_LEN * 3)
+
+#define swap32(val) ( \
+ (((u32)(val) << 24) & (u32)0xFF000000) | \
+ (((u32)(val) << 8) & (u32)0x00FF0000) | \
+ (((u32)(val) >> 8) & (u32)0x0000FF00) | \
+ (((u32)(val) >> 24) & (u32)0x000000FF))
+
+static inline int jh7110_pka_wait_pre(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_CRYPTO_CASR_OFFSET, status,
+ (status & JH7110_PKA_DONE_FLAGS), 10, 100000);
+}
+
+static int jh7110_pka_wait_done(struct jh7110_sec_dev *sdev)
+{
+ int ret = -1;
+
+ mutex_lock(&sdev->doing);
+ if (sdev->done_flags & JH7110_PKA_DONE)
+ ret = 0;
+ mutex_unlock(&sdev->doing);
+
+ return ret;
+}
+
+static void jh7110_rsa_free_key(struct jh7110_rsa_key *key)
+{
+ if (key->d)
+ kfree_sensitive(key->d);
+ if (key->e)
+ kfree_sensitive(key->e);
+ if (key->n)
+ kfree_sensitive(key->n);
+ memset(key, 0, sizeof(*key));
+}
+
+static unsigned int jh7110_rsa_get_nbit(u8 *pa, u32 snum, int key_sz)
+{
+ u32 i;
+ u8 value;
+
+ i = snum >> 3;
+
+ value = pa[key_sz - i - 1];
+ value >>= snum & 0x7;
+ value &= 0x1;
+
+ return value;
+}
+
+static int jh7110_rsa_domain_transfer(struct jh7110_sec_ctx *ctx, u32 *result, u32 *opa, u8 domain, u32 *mod, int bit_len)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ unsigned int *info;
+ int loop;
+ u8 opsize;
+ u32 temp;
+
+ mutex_lock(&sdev->doing);
+
+ opsize = (bit_len - 1) >> 5;
+ rctx->csr.pka_csr.v = 0;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ info = (unsigned int *)mod;
+ for (loop = 0; loop <= opsize; loop++)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CANR_OFFSET + loop * 4, info[opsize - loop]);
+
+
+ if (domain != 0) {
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.start = 0x1;
+ rctx->csr.pka_csr.not_r2 = 0x1;
+ //rctx->csr.pka_csr.bigendian = 1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+
+ mutex_lock(&sdev->doing);
+
+ info = (unsigned int *)opa;
+ for (loop = 0; loop <= opsize; loop++)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CAAR_OFFSET + loop * 4, info[opsize - loop]);
+
+ for (loop = 0; loop <= opsize; loop++) {
+ if (loop == 0)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, 0x1000000);
+ else
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, 0);
+ }
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_AERN;
+ rctx->csr.pka_csr.start = 0x1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+ } else {
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.start = 0x1;
+ rctx->csr.pka_csr.pre_expf = 0x1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+
+ mutex_lock(&sdev->doing);
+
+ info = (unsigned int *)opa;
+ for (loop = 0; loop <= opsize; loop++)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, info[opsize - loop]);
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_ARN;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.start = 0x1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+ }
+
+ mutex_lock(&sdev->doing);
+ for (loop = 0; loop <= opsize; loop++) {
+ temp = jh7110_sec_read(sdev, JH7110_CRYPTO_CAAR_OFFSET + 0x4 * loop);
+ result[opsize - loop] = temp;
+ }
+ mutex_unlock(&sdev->doing);
+
+ return 0;
+}
+
+static int jh7110_rsa_cpu_powm(struct jh7110_sec_ctx *ctx, u32 *result, u8 *de, u32 *n, int key_sz)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_rsa_key *key = &ctx->rsa_key;
+ u32 initial;
+ int opsize, mlen, bs, loop;
+ unsigned int *mta;
+
+ opsize = (key_sz - 1) >> 2;
+ initial = 1;
+
+ mta = kmalloc(key_sz, GFP_KERNEL);
+ if (!mta)
+ return -ENOMEM;
+
+ jh7110_rsa_domain_transfer(ctx, mta, sdev->pka_data, 0, n, key_sz << 3);
+
+ for (loop = 0; loop <= opsize; loop++)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CANR_OFFSET + loop * 4, n[opsize - loop]);
+
+ mutex_lock(&sdev->doing);
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_PRE;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.not_r2 = 0x1;
+ rctx->csr.pka_csr.start = 0x1;
+
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+
+ for (loop = 0; loop <= opsize; loop++)
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CAER_OFFSET + loop * 4, mta[opsize - loop]);
+
+ for (loop = key->bitlen; loop > 0; loop--) {
+ if (initial) {
+ for (bs = 0; bs <= opsize; bs++)
+ result[bs] = mta[bs];
+
+ initial = 0;
+ } else {
+ mlen = jh7110_rsa_get_nbit(de, loop - 1, key_sz);
+
+ mutex_lock(&sdev->doing);
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_AARN;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.start = 0x1;
+
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+
+ if (mlen) {
+ mutex_lock(&sdev->doing);
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.cln_done = 1;
+ rctx->csr.pka_csr.opsize = opsize;
+ rctx->csr.pka_csr.exposize = opsize;
+ rctx->csr.pka_csr.cmd = CRYPTO_CMD_AERN;
+ rctx->csr.pka_csr.ie = 1;
+ rctx->csr.pka_csr.start = 0x1;
+
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ jh7110_pka_wait_done(sdev);
+ }
+ }
+ }
+
+ mutex_lock(&sdev->doing);
+ for (loop = 0; loop <= opsize; loop++) {
+ unsigned int temp;
+
+ temp = jh7110_sec_read(sdev, JH7110_CRYPTO_CAAR_OFFSET + 0x4 * loop);
+ result[opsize - loop] = temp;
+ }
+ mutex_unlock(&sdev->doing);
+
+ jh7110_rsa_domain_transfer(ctx, result, result, 1, n, key_sz << 3);
+
+ return 0;
+}
+
+static int jh7110_rsa_powm(struct jh7110_sec_ctx *ctx, u8 *result, u8 *de, u8 *n, int key_sz)
+{
+ return jh7110_rsa_cpu_powm(ctx, (u32 *)result, de, (u32 *)n, key_sz);
+}
+
+static int jh7110_rsa_get_from_sg(struct jh7110_sec_request_ctx *rctx, size_t offset,
+ size_t count, size_t data_offset)
+{
+ size_t of, ct, index;
+ struct scatterlist *sg = rctx->sg;
+
+ of = offset;
+ ct = count;
+
+ while (sg->length <= of) {
+ of -= sg->length;
+
+ if (!sg_is_last(sg)) {
+ sg = sg_next(sg);
+ continue;
+ } else {
+ return -EBADE;
+ }
+ }
+
+ index = data_offset;
+ while (ct > 0) {
+ if (sg->length - of >= ct) {
+ scatterwalk_map_and_copy(rctx->sdev->pka_data + index, sg,
+ of, ct, 0);
+ index = index + ct;
+ return index - data_offset;
+ }
+ scatterwalk_map_and_copy(rctx->sdev->pka_data + index, sg,
+ of, sg->length - of, 0);
+ index += sg->length - of;
+ ct = ct - (sg->length - of);
+
+ of = 0;
+
+ if (!sg_is_last(sg))
+ sg = sg_next(sg);
+ else
+ return -EBADE;
+ }
+ return index - data_offset;
+}
+
+static int jh7110_rsa_enc_core(struct jh7110_sec_ctx *ctx, int enc)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_rsa_key *key = &ctx->rsa_key;
+ size_t data_len, total, count, data_offset;
+ int ret = 0;
+ unsigned int *info;
+ int loop;
+
+ sdev->cry_type = JH7110_PKA_TYPE;
+
+ rctx->csr.pka_csr.v = 0;
+ rctx->csr.pka_csr.reset = 1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, rctx->csr.pka_csr.v);
+
+ if (jh7110_pka_wait_pre(ctx))
+ dev_dbg(sdev->dev, "this is debug for lophyel pka_casr = %x %s %s %d\n",
+ jh7110_sec_read(sdev, JH7110_CRYPTO_CASR_OFFSET), __FILE__, __func__, __LINE__);
+
+ rctx->offset = 0;
+ total = 0;
+
+ while (total < rctx->total_in) {
+ count = min(sdev->data_buf_len, rctx->total_in);
+ count = min(count, key->key_sz);
+ memset(sdev->pka_data, 0, key->key_sz);
+ data_offset = key->key_sz - count;
+
+ data_len = jh7110_rsa_get_from_sg(rctx, rctx->offset, count, data_offset);
+ if (data_len < 0)
+ return data_len;
+ if (data_len != count)
+ return -EINVAL;
+
+ if (enc) {
+ key->bitlen = key->e_bitlen;
+ ret = jh7110_rsa_powm(ctx, sdev->pka_data + JH7110_RSA_KEYSZ_LEN, key->e, key->n, key->key_sz);
+ } else {
+ key->bitlen = key->d_bitlen;
+ ret = jh7110_rsa_powm(ctx, sdev->pka_data + JH7110_RSA_KEYSZ_LEN, key->d, key->n, key->key_sz);
+ }
+
+
+ if (ret)
+ return ret;
+
+ info = (unsigned int *)(sdev->pka_data + JH7110_RSA_KEYSZ_LEN);
+ for (loop = 0; loop < key->key_sz >> 2; loop++)
+ dev_dbg(sdev->dev, "result[%d] = %x\n", loop, info[loop]);
+
+ sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), sdev->pka_data + JH7110_RSA_KEYSZ_LEN,
+ key->key_sz, rctx->offset, 0);
+
+ rctx->offset += data_len;
+ total += data_len;
+ }
+
+ return ret;
+}
+
+static int jh7110_rsa_enc(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct jh7110_rsa_key *key = &ctx->rsa_key;
+ struct jh7110_sec_request_ctx *rctx = akcipher_request_ctx(req);
+ int ret = 0;
+
+ if (unlikely(!key->n || !key->e))
+ return -EINVAL;
+
+ if (req->dst_len < key->key_sz) {
+ req->dst_len = key->key_sz;
+ dev_err(ctx->sdev->dev, "Output buffer length less than parameter n\n");
+ return -EOVERFLOW;
+ }
+
+ mutex_lock(&ctx->sdev->rsa_lock);
+
+ rctx->sg = req->src;
+ rctx->out_sg = req->dst;
+ rctx->sdev = ctx->sdev;
+ ctx->rctx = rctx;
+ rctx->total_in = req->src_len;
+ rctx->total_out = req->dst_len;
+
+ ret = jh7110_rsa_enc_core(ctx, 1);
+
+ mutex_unlock(&ctx->sdev->rsa_lock);
+
+ return ret;
+}
+
+static int jh7110_rsa_dec(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct jh7110_rsa_key *key = &ctx->rsa_key;
+ struct jh7110_sec_request_ctx *rctx = akcipher_request_ctx(req);
+ int ret = 0;
+
+ if (unlikely(!key->n || !key->d))
+ return -EINVAL;
+
+ if (req->dst_len < key->key_sz) {
+ req->dst_len = key->key_sz;
+ dev_err(ctx->sdev->dev, "Output buffer length less than parameter n\n");
+ return -EOVERFLOW;
+ }
+
+ mutex_lock(&ctx->sdev->rsa_lock);
+
+ rctx->sg = req->src;
+ rctx->out_sg = req->dst;
+ rctx->sdev = ctx->sdev;
+ ctx->rctx = rctx;
+ rctx->total_in = req->src_len;
+ rctx->total_out = req->dst_len;
+
+ ret = jh7110_rsa_enc_core(ctx, 0);
+
+ mutex_unlock(&ctx->sdev->rsa_lock);
+
+ return ret;
+}
+
+static unsigned long jh7110_rsa_enc_fn_id(unsigned int len)
+{
+ unsigned int bitslen = len << 3;
+
+ if (bitslen & 0x1f)
+ return -EINVAL;
+
+ if (bitslen < 32 || bitslen > 2048)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int jh7110_rsa_set_n(struct jh7110_rsa_key *rsa_key, const char *value,
+ size_t vlen)
+{
+ const char *ptr = value;
+ int ret;
+
+ while (!*ptr && vlen) {
+ ptr++;
+ vlen--;
+ }
+ rsa_key->key_sz = vlen;
+ ret = -EINVAL;
+ /* invalid key size provided */
+ if (jh7110_rsa_enc_fn_id(rsa_key->key_sz))
+ goto err;
+
+ ret = -ENOMEM;
+ rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL);
+ if (!rsa_key->n)
+ goto err;
+
+ return 0;
+ err:
+ rsa_key->key_sz = 0;
+ rsa_key->n = NULL;
+ return ret;
+}
+
+static int jh7110_rsa_set_e(struct jh7110_rsa_key *rsa_key, const char *value,
+ size_t vlen)
+{
+ const char *ptr = value;
+ unsigned char pt;
+ int loop;
+
+ while (!*ptr && vlen) {
+ ptr++;
+ vlen--;
+ }
+ pt = *ptr;
+
+ if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) {
+ rsa_key->e = NULL;
+ return -EINVAL;
+ }
+
+ rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL);
+ if (!rsa_key->e)
+ return -ENOMEM;
+
+ for (loop = 8; loop > 0; loop--) {
+ if (pt >> (loop - 1))
+ break;
+ }
+
+ rsa_key->e_bitlen = (vlen - 1) * 8 + loop;
+
+ memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen);
+
+ return 0;
+}
+
+static int jh7110_rsa_set_d(struct jh7110_rsa_key *rsa_key, const char *value,
+ size_t vlen)
+{
+ const char *ptr = value;
+ unsigned char pt;
+ int loop;
+ int ret;
+
+ while (!*ptr && vlen) {
+ ptr++;
+ vlen--;
+ }
+ pt = *ptr;
+
+ ret = -EINVAL;
+ if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz)
+ goto err;
+
+ ret = -ENOMEM;
+ rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL);
+ if (!rsa_key->d)
+ goto err;
+
+ for (loop = 8; loop > 0; loop--) {
+ pr_debug("this is debug for lophyel loop = %d pt >> (loop - 1) = %x value[%d] = %x %s %s %d\n",
+ loop, pt >> (loop - 1), loop, value[loop], __FILE__, __func__, __LINE__);
+ if (pt >> (loop - 1))
+ break;
+ }
+
+ rsa_key->d_bitlen = (vlen - 1) * 8 + loop;
+
+ memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen);
+
+ return 0;
+ err:
+ rsa_key->d = NULL;
+ return ret;
+}
+
+static int jh7110_rsa_setkey(struct crypto_akcipher *tfm, const void *key,
+ unsigned int keylen, bool private)
+{
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct rsa_key raw_key = {NULL};
+ struct jh7110_rsa_key *rsa_key = &ctx->rsa_key;
+ int ret;
+
+ jh7110_rsa_free_key(rsa_key);
+
+ if (private)
+ ret = rsa_parse_priv_key(&raw_key, key, keylen);
+ else
+ ret = rsa_parse_pub_key(&raw_key, key, keylen);
+ if (ret < 0)
+ goto err;
+
+ ret = jh7110_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz);
+ if (ret < 0)
+ goto err;
+ ret = jh7110_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz);
+ if (ret < 0)
+ goto err;
+
+ if (private) {
+ ret = jh7110_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz);
+ if (ret < 0)
+ goto err;
+ }
+
+ if (!rsa_key->n || !rsa_key->e) {
+ /* invalid key provided */
+ ret = -EINVAL;
+ goto err;
+ }
+ if (private && !rsa_key->d) {
+ /* invalid private key provided */
+ ret = -EINVAL;
+ goto err;
+ }
+
+ return 0;
+ err:
+ jh7110_rsa_free_key(rsa_key);
+ return ret;
+}
+
+static int jh7110_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
+ unsigned int keylen)
+{
+ return jh7110_rsa_setkey(tfm, key, keylen, false);
+}
+
+static int jh7110_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
+ unsigned int keylen)
+{
+ return jh7110_rsa_setkey(tfm, key, keylen, true);
+}
+
+static unsigned int jh7110_rsa_max_size(struct crypto_akcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+ return ctx->rsa_key.key_sz;
+}
+
+/* Per session pkc's driver context creation function */
+static int jh7110_rsa_init_tfm(struct crypto_akcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+ ctx->sdev = jh7110_sec_find_dev(ctx);
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ akcipher_set_reqsize(tfm, sizeof(struct jh7110_sec_request_ctx));
+
+ return 0;
+}
+
+/* Per session pkc's driver context cleanup function */
+static void jh7110_rsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+ struct jh7110_sec_ctx *ctx = akcipher_tfm_ctx(tfm);
+ struct jh7110_rsa_key *key = (struct jh7110_rsa_key *)&ctx->rsa_key;
+
+ jh7110_rsa_free_key(key);
+}
+
+static struct akcipher_alg jh7110_rsa = {
+ .encrypt = jh7110_rsa_enc,
+ .decrypt = jh7110_rsa_dec,
+ .sign = jh7110_rsa_dec,
+ .verify = jh7110_rsa_enc,
+ .set_pub_key = jh7110_rsa_set_pub_key,
+ .set_priv_key = jh7110_rsa_set_priv_key,
+ .max_size = jh7110_rsa_max_size,
+ .init = jh7110_rsa_init_tfm,
+ .exit = jh7110_rsa_exit_tfm,
+ .reqsize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "rsa",
+ .cra_driver_name = "rsa-jh7110",
+ .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_priority = 3000,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ },
+};
+
+int jh7110_pka_register_algs(void)
+{
+ int ret = 0;
+
+ ret = crypto_register_akcipher(&jh7110_rsa);
+ if (ret)
+ pr_err("JH7110 RSA registration failed\n");
+
+ return ret;
+}
+
+void jh7110_pka_unregister_algs(void)
+{
+ crypto_unregister_akcipher(&jh7110_rsa);
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/dma-direct.h>
+#include <crypto/scatterwalk.h>
+
+#include "jh7110-pl080.h"
+#include "jh7110-str.h"
+
+static volatile u32 g_dmac_done;
+static volatile u32 g_dmac_err;
+struct jh7110_pl080_lli_build_data *bd;
+
+/* Maximum times we call dma_pool_alloc on this pool without freeing */
+#define MAX_NUM_TSFR_LLIS 512
+
+static inline int jh7110_dma_wait_done(struct jh7110_sec_dev *sdev, int chan)
+{
+ int ret = -1;
+
+ mutex_lock(&sdev->pl080_doing);
+ if (g_dmac_done & BIT(chan))
+ ret = 0;
+ mutex_unlock(&sdev->pl080_doing);
+
+ return ret;
+}
+
+static inline int jh7110_dmac_channel_wait_busy(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG, status,
+ !(status & PL080_CONFIG_ACTIVE), 10, 100000);
+}
+
+static irqreturn_t jh7110_pl080_irq_thread(int irq, void *arg)
+{
+ struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg;
+
+ pr_debug("this is debug mutex_unlock doing ---------------- %s %s %d\n", __FILE__, __func__, __LINE__);
+ mutex_unlock(&sdev->pl080_doing);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t jh7110_pl080_irq(int irq, void *dev)
+{
+ struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) dev;
+ u32 err, tc, i;
+
+ /* check & clear - ERR & TC interrupts */
+ err = readl_relaxed(sdev->dma_base + PL080_ERR_STATUS);
+ if (err) {
+ pr_err("%s error interrupt, register value 0x%08x\n",
+ __func__, err);
+ writel_relaxed(err, sdev->dma_base + PL080_ERR_CLEAR);
+ }
+ tc = readl_relaxed(sdev->dma_base + PL080_TC_STATUS);
+ if (tc)
+ writel_relaxed(tc, sdev->dma_base + PL080_TC_CLEAR);
+
+ if (!err && !tc)
+ return IRQ_NONE;
+
+ for (i = 0; i < PL080_CHANNELS_NUM; i++) {
+ if (BIT(i) & err)
+ g_dmac_err |= BIT(i);
+ if (BIT(i) & tc)
+ g_dmac_done |= BIT(i);
+ }
+
+ return IRQ_WAKE_THREAD;
+}
+
+static int jh7110_dmac_enable(struct jh7110_sec_dev *sdev)
+{
+ writel_relaxed(PL080_CONFIG_ENABLE, sdev->dma_base + PL080_CONFIG);
+
+ return 0;
+}
+
+static int jh7110_dmac_disable(struct jh7110_sec_dev *sdev)
+{
+ writel_relaxed(0, sdev->dma_base + PL080_CONFIG);
+ return 0;
+}
+
+static int jh7110_dmac_channel_enable(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 control;
+
+ control = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ control |= PL080_CONFIG_ENABLE;
+ writel_relaxed(control, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+
+ return 0;
+}
+
+static int jh7110_dmac_channel_disable(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 control;
+
+ control = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ control &= (~PL080_CONFIG_ENABLE);
+ writel_relaxed(control, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+
+ return 0;
+}
+
+static int jh7110_dmac_channel_halt(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 val;
+
+ val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ val |= PL080_CONFIG_HALT;
+ writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+
+ return 0;
+}
+
+static int jh7110_dmac_channel_resume(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 val;
+
+ val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ val &= (~PL080_CONFIG_HALT);
+ writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+
+ return 0;
+}
+
+static void jh7110_dmac_interrupt_enable(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 val;
+
+ val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ val |= PL080_CONFIG_TC_IRQ_MASK | PL080_CONFIG_ERR_IRQ_MASK;
+ writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+}
+
+static void jh7110_dmac_interrupt_disable(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ u32 val;
+
+ val = readl_relaxed(sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+ val &= ~(PL080_CONFIG_TC_IRQ_MASK | PL080_CONFIG_ERR_IRQ_MASK);
+ writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+}
+
+static u32 jh7110_dmac_cctl_bits(struct jh7110_pl080_chan_config *config, u32 tsize)
+{
+ u32 cctl = 0;
+
+ if (config->src_width == PL08X_BUS_WIDTH_16_BITS)
+ tsize >>= 1;
+ else if (config->src_width == PL08X_BUS_WIDTH_32_BITS)
+ tsize >>= 2;
+
+ cctl = (1 << PL080_CONTROL_PROT_SYS) | (config->di<<27) | (config->si<<26) | (config->dst_ahb << 25)
+ | (config->src_ahb << 24) | (config->dst_width << 21) | (config->src_width << 18) | (config->dst_bsize<<15)
+ | (config->src_bsize<<12) | tsize;
+ return cctl;
+}
+
+static int jh7110_dmac_fill_llis(struct jh7110_pl080_lli_build_data *bd, int num_llis,
+ struct jh7110_pl080_chan_config *config)
+{
+ u32 cctl;
+ u32 llis_addr = bd->llis_phy_addr;
+ struct jh7110_pl080_lli *llis = bd->llis;
+
+ cctl = jh7110_dmac_cctl_bits(config, bd->tsize);
+
+ llis[num_llis].control0 = cctl;
+ llis[num_llis].src_addr = bd->src_addr;
+ llis[num_llis].dst_addr = bd->dst_addr;
+ llis[num_llis].next_lli = llis_addr + (num_llis + 1) * sizeof(struct jh7110_pl080_lli);
+
+ bd->remainder -= bd->tsize;
+ if (bd->remainder == 0) {
+ llis[num_llis].next_lli = 0;
+ llis[num_llis].control0 |= PL080_CONTROL_TC_IRQ_EN;
+ }
+
+ return 0;
+}
+
+static int jh7110_dmac_llis_prep(struct jh7110_pl080_lli_build_data *bd, struct jh7110_pl080_chan_config *config)
+{
+ u32 llisize;
+ u32 buswidth, width;
+ u32 max_bytes_per_lli, tsize;
+ u32 src_tsize, dst_tsize;
+ u32 num_llis = 0;
+
+ bd->src_addr = config->src_addr;
+ bd->dst_addr = config->dst_addr;
+ bd->remainder = config->xfer_size;
+
+ buswidth = min(config->src_width, config->dst_width);
+ switch (buswidth) {
+ case PL08X_BUS_WIDTH_8_BITS:
+ width = 1;
+ break;
+ case PL08X_BUS_WIDTH_16_BITS:
+ width = 2;
+ break;
+ case PL08X_BUS_WIDTH_32_BITS:
+ width = 4;
+ break;
+ default:
+ break;
+ }
+ max_bytes_per_lli = width * PL080_CONTROL_TRANSFER_SIZE_MASK;
+ tsize = max_bytes_per_lli;
+ llisize = bd->remainder % tsize;
+ if (llisize == 0)
+ llisize = bd->remainder / tsize;
+ else
+ llisize = (bd->remainder / tsize) + 1;
+
+ if (llisize > MAX_NUM_TSFR_LLIS)
+ return -1;
+
+ llisize += 1;
+
+ bd->llis = kmalloc(llisize * sizeof(struct jh7110_pl080_lli), GFP_KERNEL);
+ if (bd->llis == NULL)
+ return -1;
+ memset(bd->llis, 0, llisize * sizeof(struct jh7110_pl080_lli));
+ bd->llis_phy_addr = (u32)bd->llis;
+
+ while (1) {
+ if (!bd->remainder)
+ break;
+
+ if (bd->remainder > tsize)
+ bd->tsize = tsize;
+ else
+ bd->tsize = bd->remainder;
+
+ jh7110_dmac_fill_llis(bd, num_llis++, config);
+
+ if (config->si)
+ bd->src_addr += bd->tsize;
+ if (config->di)
+ bd->dst_addr += bd->tsize;
+ }
+ config->src_addr = bd->src_addr;
+ config->dst_addr = bd->dst_addr;
+
+ return 0;
+}
+
+/*dmac_setup_xfer support dma link item*/
+int jh7110_dmac_setup_xfer(struct jh7110_sec_dev *sdev, u8 chan, struct jh7110_pl080_chan_config *config)
+{
+ u32 val;
+ struct jh7110_pl080_lli *lli = NULL;
+
+ bd = kmalloc(sizeof(struct jh7110_pl080_lli_build_data), GFP_KERNEL);
+ if (bd == NULL)
+ return -1;
+
+ memset(bd, 0, sizeof(struct jh7110_pl080_lli_build_data));
+ jh7110_dmac_llis_prep(bd, config);
+ lli = &bd->llis[0];
+
+ writel_relaxed(lli->src_addr, sdev->pl080->phy_chans[chan].base + PL080_CH_SRC_ADDR);
+ writel_relaxed(lli->dst_addr, sdev->pl080->phy_chans[chan].base + PL080_CH_DST_ADDR);
+ writel_relaxed(lli->next_lli, sdev->pl080->phy_chans[chan].base + PL080_CH_LLI);
+ writel_relaxed(lli->control0, sdev->pl080->phy_chans[chan].base + PL080_CH_CONTROL);
+
+ val = (config->flow << 11)|(config->src_peri<<1)|(config->dst_peri<<6);
+
+ writel_relaxed(val, sdev->pl080->phy_chans[chan].base + PL080_CH_CONFIG);
+
+ jh7110_dmac_interrupt_enable(sdev, chan);
+
+ g_dmac_done &= ~BIT(chan);
+ g_dmac_err &= ~BIT(chan);
+
+
+ if (chan)
+ mutex_lock(&sdev->pl080_doing);
+
+ return 0;
+}
+
+void jh7110_dmac_free_llis(void)
+{
+ if (bd) {
+ kfree(bd->llis);
+ kfree(bd);
+ bd = NULL;
+ }
+}
+
+int jh7110_dmac_wait_done(struct jh7110_sec_dev *sdev, u8 chan)
+{
+ if (jh7110_dma_wait_done(sdev, chan)) {
+ pr_debug("this is debug for lophyel status = %x err = %x control0 = %x control1 = %x %s %s %d\n",
+ readl_relaxed(sdev->dma_base + PL080_TC_STATUS), readl_relaxed(sdev->dma_base + PL080_ERR_STATUS),
+ readl_relaxed(sdev->dma_base + 0x10c), readl_relaxed(sdev->dma_base + 0x12c),
+ __FILE__, __func__, __LINE__);
+
+ return -1;
+ }
+
+ g_dmac_done &= ~BIT(chan);
+ jh7110_dmac_free_llis();
+
+ return 0;
+}
+
+int jh7110_dmac_secdata_in(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size)
+{
+ struct jh7110_pl080_chan_config config;
+ int ret;
+
+ config.si = PL08X_INCREMENT;
+ config.di = PL08X_INCREMENT_FIX;
+ config.src_ahb = PL08X_AHB1;
+ config.dst_ahb = PL08X_AHB2;
+ config.src_width = PL08X_BUS_WIDTH_32_BITS;
+ config.dst_width = PL08X_BUS_WIDTH_32_BITS;
+ config.src_bsize = PL08X_BURST_SZ_32;
+ config.dst_bsize = PL08X_BURST_SZ_32;
+ config.src_peri = 1;
+ config.dst_peri = 1;
+ config.src_addr = src;
+ config.dst_addr = dst;
+ config.xfer_size = size;
+ config.flow = PL080_FLOW_MEM2PER;
+
+ if (jh7110_dmac_channel_wait_busy(sdev, chan))
+ return -ETIMEDOUT;
+
+ ret = jh7110_dmac_setup_xfer(sdev, chan, &config);
+ if (ret != 0)
+ return ret;
+
+ jh7110_dmac_channel_enable(sdev, chan);
+
+ return 0;
+}
+
+int jh7110_dmac_secdata_out(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size)
+{
+ struct jh7110_pl080_chan_config config;
+ int ret;
+
+ config.si = PL08X_INCREMENT_FIX;
+ config.di = PL08X_INCREMENT;
+ config.src_ahb = PL08X_AHB2;
+ config.dst_ahb = PL08X_AHB1;
+ config.src_width = PL08X_BUS_WIDTH_32_BITS;
+ config.dst_width = PL08X_BUS_WIDTH_32_BITS;
+ config.src_bsize = PL08X_BURST_SZ_4;//follow hardware limit
+ config.dst_bsize = PL08X_BURST_SZ_4;
+ config.src_peri = 0;
+ config.dst_peri = 0;
+ config.src_addr = src;
+ config.dst_addr = dst;
+ config.xfer_size = size;
+ config.flow = PL080_FLOW_PER2MEM;
+
+ if (jh7110_dmac_channel_wait_busy(sdev, chan))
+ return -ETIMEDOUT;
+
+ ret = jh7110_dmac_setup_xfer(sdev, chan, &config);
+ if (ret != 0)
+ return ret;
+
+ jh7110_dmac_channel_enable(sdev, chan);
+
+ return 0;
+}
+
+int jh7110_dmac_init(struct jh7110_sec_dev *sdev, int irq)
+{
+ int chan;
+ int ret;
+
+ ret = devm_request_threaded_irq(sdev->dev, irq, jh7110_pl080_irq,
+ jh7110_pl080_irq_thread, 0,
+ dev_name(sdev->dev), sdev);
+ if (ret) {
+ dev_err(sdev->dev, "Can't get interrupt working.\n");
+ return ret;
+ }
+
+ sdev->pl080 = kmalloc(sizeof(struct jh7110_pl08x_device), GFP_KERNEL);
+ if (!sdev->pl080)
+ return -ENOMEM;
+
+ memset(sdev->pl080, 0, sizeof(struct jh7110_pl08x_device));
+
+ for (chan = 0; chan < PL080_CHANNELS_NUM; chan++) {
+ struct jh7110_pl08x_phy_chan *ch = &sdev->pl080->phy_chans[chan];
+
+ ch->id = chan;
+ ch->base = sdev->dma_base + PL080_Cx_BASE(chan);
+ ch->reg_config = ch->base + PL080_CH_CONFIG;
+ ch->reg_control = ch->base + PL080_CH_CONTROL;
+ ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
+ ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
+ ch->reg_lli = ch->base + PL080_CH_LLI;
+ }
+
+ jh7110_dmac_enable(sdev);
+ g_dmac_done = 0;
+ g_dmac_err = 0;
+
+ return 0;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ */
+
+
+#ifndef __JH7110_PL080_H__
+#define __JH7110_PL080_H__
+
+#define PL080_INT_STATUS (0x00)
+#define PL080_TC_STATUS (0x04)
+#define PL080_TC_CLEAR (0x08)
+#define PL080_ERR_STATUS (0x0C)
+#define PL080_ERR_CLEAR (0x10)
+#define PL080_RAW_TC_STATUS (0x14)
+#define PL080_RAW_ERR_STATUS (0x18)
+#define PL080_EN_CHAN (0x1c)
+#define PL080_SOFT_BREQ (0x20)
+#define PL080_SOFT_SREQ (0x24)
+#define PL080_SOFT_LBREQ (0x28)
+#define PL080_SOFT_LSREQ (0x2C)
+#define PL080_CONFIG (0x30)
+#define PL080_CONFIG_M2_BE BIT(2)
+#define PL080_CONFIG_M1_BE BIT(1)
+#define PL080_CONFIG_ENABLE BIT(0)
+#define PL080_SYNC (0x34)
+
+/* Per channel configuration registers */
+#define PL080_Cx_BASE(x) ((0x100 + (x * 0x20)))
+#define PL080_CH_SRC_ADDR (0x00)
+#define PL080_CH_DST_ADDR (0x04)
+#define PL080_CH_LLI (0x08)
+#define PL080_CH_CONTROL (0x0C)
+#define PL080_CH_CONFIG (0x10)
+#define PL080S_CH_CONTROL2 (0x10)
+#define PL080S_CH_CONFIG (0x14)
+
+#define PL080_LLI_ADDR_SHIFT (2)
+#define PL080_CONTROL_PROT_SHIFT (28)
+#define PL080_CONTROL_DWIDTH_SHIFT (21)
+#define PL080_CONTROL_SWIDTH_SHIFT (18)
+#define PL080_CONTROL_DB_SIZE_SHIFT (15)
+#define PL080_CONTROL_SB_SIZE_SHIFT (12)
+#define PL080_CONTROL_TRANSFER_SIZE_SHIFT (0)
+
+#define PL080_LLI_LM_AHB2 BIT(0)
+#define PL080_CONTROL_TC_IRQ_EN BIT(31)
+#define PL080_CONTROL_PROT_CACHE BIT(30)
+#define PL080_CONTROL_PROT_BUFF BIT(29)
+#define PL080_CONTROL_PROT_SYS BIT(28)
+#define PL080_CONTROL_DST_INCR BIT(27)
+#define PL080_CONTROL_SRC_INCR BIT(26)
+#define PL080_CONTROL_DST_AHB2 BIT(25)
+#define PL080_CONTROL_SRC_AHB2 BIT(24)
+
+#define PL080_CONTROL_TRANSFER_SIZE_MASK (0xfff<<0)
+#define PL080N_CONFIG_ITPROT BIT(20)
+#define PL080N_CONFIG_SECPROT BIT(19)
+#define PL080_CONFIG_HALT BIT(18)
+#define PL080_CONFIG_ACTIVE BIT(17) /* RO */
+#define PL080_CONFIG_LOCK BIT(16)
+#define PL080_CONFIG_TC_IRQ_MASK BIT(15)
+#define PL080_CONFIG_ERR_IRQ_MASK BIT(14)
+#define PL080_CONFIG_ENABLE BIT(0)
+
+#define PL080_CONFIG_FLOW_CONTROL_SHIFT (11)
+#define PL080_CONFIG_DST_SEL_SHIFT (6)
+#define PL080_CONFIG_SRC_SEL_SHIFT (1)
+
+#define PL080_CONFIG_FLOW_CONTROL_MASK GENMASK(13, 11)
+#define PL080_CONFIG_DST_SEL_MASK GENMASK(9, 6)
+#define PL080_CONFIG_SRC_SEL_MASK GENMASK(4, 1)
+
+
+#define PL080_CHANNELS_NUM (8)
+#define PL080_SIGNAL_NUM (16)
+
+/* Bitmasks for selecting AHB ports for DMA transfers */
+enum jh7110_pl08x_ahb_master {
+ PL08X_AHB1 = (0 << 0),
+ PL08X_AHB2 = (1 << 0)
+};
+
+enum jh7110_pl08x_burst_size {
+ PL08X_BURST_SZ_1,
+ PL08X_BURST_SZ_4,
+ PL08X_BURST_SZ_8,
+ PL08X_BURST_SZ_16,
+ PL08X_BURST_SZ_32,
+ PL08X_BURST_SZ_64,
+ PL08X_BURST_SZ_128,
+ PL08X_BURST_SZ_256,
+};
+
+enum jh7110_pl08x_bus_width {
+ PL08X_BUS_WIDTH_8_BITS,
+ PL08X_BUS_WIDTH_16_BITS,
+ PL08X_BUS_WIDTH_32_BITS,
+};
+
+enum jh7110_pl08x_flow_control {
+ PL080_FLOW_MEM2MEM,
+ PL080_FLOW_MEM2PER,
+ PL080_FLOW_PER2MEM,
+ PL080_FLOW_SRC2DST,
+ PL080_FLOW_SRC2DST_DST,
+ PL080_FLOW_MEM2PER_PER,
+ PL080_FLOW_PER2MEM_PER,
+ PL080_FLOW_SRC2DST_SRC
+};
+
+enum jh7110_pl08x_increment {
+ PL08X_INCREMENT_FIX,
+ PL08X_INCREMENT,
+};
+
+struct jh7110_pl080_chan_config {
+ u8 src_peri;
+ u8 dst_peri;
+ u32 src_addr;
+ u32 dst_addr;
+ u32 xfer_size;
+ u32 data_scattered;
+ u32 src_distance;
+ u32 dst_distance;
+ enum jh7110_pl08x_increment si;
+ enum jh7110_pl08x_increment di;
+ enum jh7110_pl08x_ahb_master src_ahb;
+ enum jh7110_pl08x_ahb_master dst_ahb;
+ enum jh7110_pl08x_bus_width src_width;
+ enum jh7110_pl08x_bus_width dst_width;
+ enum jh7110_pl08x_burst_size src_bsize;
+ enum jh7110_pl08x_burst_size dst_bsize;
+ enum jh7110_pl08x_flow_control flow;
+};
+
+/* DMA linked list chain structure */
+struct jh7110_pl080_lli {
+ u32 src_addr;
+ u32 dst_addr;
+ u32 next_lli;
+ u32 control0;
+};
+
+struct jh7110_pl080_lli_build_data {
+ u32 src_addr;
+ u32 dst_addr;
+ struct jh7110_pl080_lli *llis;
+ u32 llis_phy_addr;
+ u32 tsize;
+ u32 remainder;
+};
+
+struct jh7110_pl08x_phy_chan {
+ unsigned int id;
+ void *base;
+ void *reg_config;
+ void *reg_control;
+ void *reg_src;
+ void *reg_dst;
+ void *reg_lli;
+};
+
+struct jh7110_pl08x_device {
+ struct jh7110_pl08x_phy_chan phy_chans[PL080_CHANNELS_NUM];
+};
+#endif
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ */
+#ifndef __JH7110_REGS_H__
+#define __JH7110_REGS_H__
+
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+
+#define JH7110_ALG_CR_OFFSET 0x0
+#define JH7110_ALG_FIFO_OFFSET 0x4
+#define JH7110_IE_MASK_OFFSET 0x8
+#define JH7110_IE_FLAG_OFFSET 0xc
+#define JH7110_DMA_IN_LEN_OFFSET 0x10
+#define JH7110_DMA_OUT_LEN_OFFSET 0x14
+
+#define JH7110_AES_REGS_OFFSET 0x100
+#define JH7110_DES_REGS_OFFSET 0x200
+#define JH7110_SHA_REGS_OFFSET 0x300
+#define JH7110_CRYPTO_REGS_OFFSET 0x400
+
+union jh7110_alg_cr {
+ u32 v;
+ struct {
+ u32 start :1;
+ u32 aes_dma_en :1;
+ u32 des_dma_en :1;
+ u32 sha_dma_en :1;
+ u32 alg_done :1;
+ u32 rsvd_0 :3;
+ u32 clear :1;
+ u32 rsvd_1 :23;
+ };
+};
+
+union jh7110_ie_mask {
+ u32 v;
+ struct {
+ u32 aes_ie_mask :1;
+ u32 des_ie_mask :1;
+ u32 sha_ie_mask :1;
+ u32 crypto_ie_mask :1;
+ u32 rsvd_0 :28;
+ };
+};
+
+union jh7110_ie_flag {
+ u32 v;
+ struct {
+ u32 aes_ie_done :1;
+ u32 des_ie_done :1;
+ u32 sha_ie_done :1;
+ u32 crypto_ie_done :1;
+ u32 rsvd_0 :28;
+ };
+};
+
+#define JH7110_CRYPTO_CACR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x0)
+#define JH7110_CRYPTO_CASR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x4)
+#define JH7110_CRYPTO_CAAR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x8)
+#define JH7110_CRYPTO_CAER_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x108)
+#define JH7110_CRYPTO_CANR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x208)
+#define JH7110_CRYPTO_CAAFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x308)
+#define JH7110_CRYPTO_CAEFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x30c)
+#define JH7110_CRYPTO_CANFR_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x310)
+#define JH7110_FIFO_COUNTER_OFFSET (JH7110_CRYPTO_REGS_OFFSET + 0x314)
+
+// R^2 mod N and N0'
+#define CRYPTO_CMD_PRE 0x0
+// (A + A) mod N, ==> A
+#define CRYPTO_CMD_AAN 0x1
+// A ^ E mod N ==> A
+#define CRYPTO_CMD_AMEN 0x2
+// A + E mod N ==> A
+#define CRYPTO_CMD_AAEN 0x3
+// A - E mod N ==> A
+#define CRYPTO_CMD_ADEN 0x4
+// A * R mod N ==> A
+#define CRYPTO_CMD_ARN 0x5
+// A * E * R mod N ==> A
+#define CRYPTO_CMD_AERN 0x6
+// A * A * R mod N ==> A
+#define CRYPTO_CMD_AARN 0x7
+// ECC2P ==> A
+#define CRYPTO_CMD_ECC2P 0x8
+// ECCPQ ==> A
+#define CRYPTO_CMD_ECCPQ 0x9
+
+union jh7110_crypto_cacr {
+ u32 v;
+ struct {
+ u32 start :1;
+ u32 reset :1;
+ u32 ie :1;
+ u32 rsvd_0 :1;
+ u32 fifo_mode :1;
+ u32 not_r2 :1;
+ u32 ecc_sub :1;
+ u32 pre_expf :1;
+
+ u32 cmd :4;
+ u32 rsvd_1 :1;
+ u32 ctrl_dummy :1;
+ u32 ctrl_false :1;
+ u32 cln_done :1;
+
+ u32 opsize :6;
+ u32 rsvd_2 :2;
+
+ u32 exposize :6;
+ u32 rsvd_3 :1;
+ u32 bigendian :1;
+ };
+};
+
+union jh7110_crypto_casr {
+ u32 v;
+ struct {
+#define JH7110_PKA_DONE_FLAGS BIT(0)
+ u32 done :1;
+ u32 rsvd_0 :31;
+ };
+};
+
+#define JH7110_DES_DAEDINR_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x0)
+#define JH7110_DES_DAEDINR_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x4)
+#define JH7110_DES_DAEKIN1R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x8)
+#define JH7110_DES_DAEKIN1R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0xC)
+#define JH7110_DES_DAEKIN2R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x10)
+#define JH7110_DES_DAEKIN2R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x14)
+#define JH7110_DES_DAEKIN3R_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x18)
+#define JH7110_DES_DAEKIN3R_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x1C)
+#define JH7110_DES_DAEIVINR_HI_OFFSET (JH7110_DES_REGS_OFFSET + 0x20)
+#define JH7110_DES_DAEIVINR_LO_OFFSET (JH7110_DES_REGS_OFFSET + 0x24)
+#define JH7110_DES_DAECSR_OFFSET (JH7110_DES_REGS_OFFSET + 0x28)
+#define JH7110_DES_KEY0 (JH7110_DES_REGS_OFFSET + 0X2C)
+#define JH7110_DES_KEY1 (JH7110_DES_REGS_OFFSET + 0X30)
+#define JH7110_DES_IV0 (JH7110_DES_REGS_OFFSET + 0X34)
+#define JH7110_DES_IV1 (JH7110_DES_REGS_OFFSET + 0X38)
+
+union jh7110_des_daecsr {
+ u32 v;
+ struct {
+#define JH7110_DES_BUSY BIT(0)
+ u32 busy :1;
+ u32 en :1;
+ u32 encryt :1;
+ u32 reset :1;
+ u32 disturb :1;
+#define JH7110_DES_KS_64 0x0
+#define JH7110_DES_KS_128 0x2
+#define JH7110_DES_KS_192 0x3
+ u32 ks :2;
+ u32 vdes_en :1;
+
+#define JH7110_DES_MODE_ECB 0x0
+#define JH7110_DES_MODE_CBC 0x1
+#define JH7110_DES_MODE_CFB 0x2
+#define JH7110_DES_MODE_OFB 0x3
+ u32 mode :2;
+ u32 ie :1;
+#define JH7110_DES_CSR_DONE BIT(11)
+ u32 done :1;
+#define JH7110_DES_BITMODE_1 0x0
+#define JH7110_DES_BITMODE_64 0x4
+ u32 bitmode :3;
+ u32 rsvd_0 :17;
+ };
+};
+
+#define JH7110_AES_AESDIO0R (JH7110_AES_REGS_OFFSET + 0x0)
+#define JH7110_AES_KEY0 (JH7110_AES_REGS_OFFSET + 0x4)
+#define JH7110_AES_KEY1 (JH7110_AES_REGS_OFFSET + 0x8)
+#define JH7110_AES_KEY2 (JH7110_AES_REGS_OFFSET + 0xC)
+#define JH7110_AES_KEY3 (JH7110_AES_REGS_OFFSET + 0x10)
+#define JH7110_AES_KEY4 (JH7110_AES_REGS_OFFSET + 0x14)
+#define JH7110_AES_KEY5 (JH7110_AES_REGS_OFFSET + 0x18)
+#define JH7110_AES_KEY6 (JH7110_AES_REGS_OFFSET + 0x1C)
+#define JH7110_AES_KEY7 (JH7110_AES_REGS_OFFSET + 0x20)
+#define JH7110_AES_CSR (JH7110_AES_REGS_OFFSET + 0x24)
+#define JH7110_AES_IV0 (JH7110_AES_REGS_OFFSET + 0x28)
+#define JH7110_AES_IV1 (JH7110_AES_REGS_OFFSET + 0x2C)
+#define JH7110_AES_IV2 (JH7110_AES_REGS_OFFSET + 0x30)
+#define JH7110_AES_IV3 (JH7110_AES_REGS_OFFSET + 0x34)
+#define JH7110_AES_NONCE0 (JH7110_AES_REGS_OFFSET + 0x3C)
+#define JH7110_AES_NONCE1 (JH7110_AES_REGS_OFFSET + 0x40)
+#define JH7110_AES_NONCE2 (JH7110_AES_REGS_OFFSET + 0x44)
+#define JH7110_AES_NONCE3 (JH7110_AES_REGS_OFFSET + 0x48)
+#define JH7110_AES_ALEN0 (JH7110_AES_REGS_OFFSET + 0x4C)
+#define JH7110_AES_ALEN1 (JH7110_AES_REGS_OFFSET + 0x50)
+#define JH7110_AES_MLEN0 (JH7110_AES_REGS_OFFSET + 0x54)
+#define JH7110_AES_MLEN1 (JH7110_AES_REGS_OFFSET + 0x58)
+#define JH7110_AES_IVLEN (JH7110_AES_REGS_OFFSET + 0x5C)
+
+union jh7110_aes_csr {
+ u32 v;
+ struct {
+ u32 cmode :1;
+#define JH7110_AES_KEYMODE_128 0x0
+#define JH7110_AES_KEYMODE_192 0x1
+#define JH7110_AES_KEYMODE_256 0x2
+ u32 keymode :2;
+#define JH7110_AES_BUSY BIT(3)
+ u32 busy :1;
+ u32 done :1;
+#define JH7110_AES_KEY_DONE BIT(5)
+ u32 krdy :1;
+ u32 aesrst :1;
+ u32 aesie :1;
+
+#define JH7110_AES_CCM_START BIT(8)
+ u32 ccm_start :1;
+#define JH7110_AES_MODE_ECB 0x0
+#define JH7110_AES_MODE_CBC 0x1
+#define JH7110_AES_MODE_CFB 0x2
+#define JH7110_AES_MODE_OFB 0x3
+#define JH7110_AES_MODE_CTR 0x4
+#define JH7110_AES_MODE_CCM 0x5
+#define JH7110_AES_MODE_GCM 0x6
+ u32 mode :3;
+#define JH7110_AES_GCM_START BIT(12)
+ u32 gcm_start :1;
+#define JH7110_AES_GCM_DONE BIT(13)
+ u32 gcm_done :1;
+ u32 delay_aes :1;
+ u32 vaes_start :1;
+
+ u32 rsvd_0 :8;
+
+#define JH7110_AES_MODE_XFB_1 0x0
+#define JH7110_AES_MODE_XFB_128 0x5
+ u32 stream_mode :3;
+ u32 rsvd_1 :5;
+ };
+};
+
+#define JH7110_SHA_SHACSR (JH7110_SHA_REGS_OFFSET + 0x0)
+#define JH7110_SHA_SHAWDR (JH7110_SHA_REGS_OFFSET + 0x4)
+#define JH7110_SHA_SHARDR (JH7110_SHA_REGS_OFFSET + 0x8)
+#define JH7110_SHA_SHAWSR (JH7110_SHA_REGS_OFFSET + 0xC)
+#define JH7110_SHA_SHAWLEN3 (JH7110_SHA_REGS_OFFSET + 0x10)
+#define JH7110_SHA_SHAWLEN2 (JH7110_SHA_REGS_OFFSET + 0x14)
+#define JH7110_SHA_SHAWLEN1 (JH7110_SHA_REGS_OFFSET + 0x18)
+#define JH7110_SHA_SHAWLEN0 (JH7110_SHA_REGS_OFFSET + 0x1C)
+#define JH7110_SHA_SHAWKR (JH7110_SHA_REGS_OFFSET + 0x20)
+#define JH7110_SHA_SHAWKLEN (JH7110_SHA_REGS_OFFSET + 0x24)
+
+union jh7110_sha_shacsr {
+ u32 v;
+ struct {
+ u32 start :1;
+ u32 reset :1;
+ u32 ie :1;
+ u32 firstb :1;
+#define JH7110_SHA_SM3 0x0
+#define JH7110_SHA_SHA0 0x1
+#define JH7110_SHA_SHA1 0x2
+#define JH7110_SHA_SHA224 0x3
+#define JH7110_SHA_SHA256 0x4
+#define JH7110_SHA_SHA384 0x5
+#define JH7110_SHA_SHA512 0x6
+#define JH7110_SHA_MODE_MASK 0x7
+ u32 mode :3;
+ u32 rsvd_0 :1;
+
+ u32 final :1;
+ u32 rsvd_1 :2;
+#define JH7110_SHA_HMAC_FLAGS 0x800
+ u32 hmac :1;
+ u32 rsvd_2 :1;
+ u32 key_done :1;
+ u32 key_flag :1;
+ u32 hmac_done :1;
+
+#define JH7110_SHA_BUSY BIT(16)
+ u32 busy :1;
+ u32 shadone :1;
+ u32 rsvd_3 :14;
+ };
+};
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <william.qiu@starfivetech.com>
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+
+#include "jh7110-str.h"
+
+#define DRIVER_NAME "jh7110-sec"
+
+struct jh7110_dev_list {
+ struct list_head dev_list;
+ spinlock_t lock; /* protect dev_list */
+};
+
+static struct jh7110_dev_list dev_list = {
+ .dev_list = LIST_HEAD_INIT(dev_list.dev_list),
+ .lock = __SPIN_LOCK_UNLOCKED(dev_list.lock),
+};
+
+struct jh7110_sec_dev *jh7110_sec_find_dev(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = NULL, *tmp;
+
+ spin_lock_bh(&dev_list.lock);
+ if (!ctx->sdev) {
+ list_for_each_entry(tmp, &dev_list.dev_list, list) {
+ sdev = tmp;
+ break;
+ }
+ ctx->sdev = sdev;
+ } else {
+ sdev = ctx->sdev;
+ }
+
+ spin_unlock_bh(&dev_list.lock);
+
+ return sdev;
+}
+
+static irqreturn_t jh7110_cryp_irq_thread(int irq, void *arg)
+{
+ struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg;
+
+ if (sdev->use_dma)
+ if (sdev->cry_type != JH7110_PKA_TYPE)
+ return IRQ_HANDLED;
+
+ mutex_unlock(&sdev->doing);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t jh7110_cryp_irq(int irq, void *arg)
+{
+ struct jh7110_sec_dev *sdev = (struct jh7110_sec_dev *) arg;
+ union jh7110_sha_shacsr sha_csr;
+ union jh7110_aes_csr aes_csr;
+ union jh7110_des_daecsr des_csr;
+ union jh7110_crypto_cacr cry_cacr;
+ union jh7110_crypto_casr cry_casr;
+ irqreturn_t ret = IRQ_WAKE_THREAD;
+
+ switch (sdev->cry_type) {
+ case JH7110_SHA_TYPE:
+ sha_csr.v = jh7110_sec_read(sdev, JH7110_SHA_SHACSR);
+ if (sha_csr.key_done)
+ sdev->done_flags |= JH7110_SHA_KEY_DONE;
+ if (sha_csr.hmac_done)
+ sdev->done_flags |= JH7110_SHA_HMAC_DONE;
+ if (sha_csr.shadone)
+ sdev->done_flags |= JH7110_SHA_SHA_DONE;
+
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, sha_csr.v | BIT(17));
+ break;
+ case JH7110_AES_TYPE:
+ aes_csr.v = jh7110_sec_read(sdev, JH7110_AES_CSR);
+ if (aes_csr.done) {
+ sdev->done_flags |= JH7110_AES_DONE;
+ jh7110_sec_write(sdev, JH7110_AES_CSR, aes_csr.v);
+ }
+
+ break;
+ case JH7110_DES_TYPE:
+ des_csr.v = jh7110_sec_read(sdev, JH7110_DES_DAECSR_OFFSET);
+ if (des_csr.done) {
+ sdev->done_flags |= JH7110_DES_DONE;
+ jh7110_sec_write(sdev, JH7110_DES_DAECSR_OFFSET, des_csr.v);
+ }
+
+ break;
+ case JH7110_PKA_TYPE:
+ cry_casr.v = jh7110_sec_read(sdev, JH7110_CRYPTO_CASR_OFFSET);
+ if (cry_casr.done)
+ sdev->done_flags |= JH7110_PKA_DONE_FLAGS;
+ cry_cacr.v = jh7110_sec_read(sdev, JH7110_CRYPTO_CACR_OFFSET);
+ cry_cacr.cln_done = 1;
+ jh7110_sec_write(sdev, JH7110_CRYPTO_CACR_OFFSET, cry_cacr.v);
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static const struct of_device_id jh7110_dt_ids[] = {
+ { .compatible = "starfive,jh7110-sec", .data = NULL},
+ {},
+};
+MODULE_DEVICE_TABLE(of, jh7110_dt_ids);
+
+static int jh7110_dma_init(struct jh7110_sec_dev *sdev)
+{
+ dma_cap_mask_t mask;
+ int err;
+
+ sdev->sec_xm_m = NULL;
+ sdev->sec_xm_p = NULL;
+
+ dma_cap_zero(mask);
+ dma_cap_set(DMA_SLAVE, mask);
+
+ sdev->sec_xm_m = dma_request_chan(sdev->dev, "sec_m");
+ if (IS_ERR(sdev->sec_xm_m)) {
+ dev_err(sdev->dev, "Unable to request sec_m dma channel in DMA channel\n");
+ return PTR_ERR(sdev->sec_xm_m);
+ }
+
+ sdev->sec_xm_p = dma_request_chan(sdev->dev, "sec_p");
+ if (IS_ERR(sdev->sec_xm_p)) {
+ dev_err(sdev->dev, "Unable to request sec_p dma channel in DMA channel\n");
+ goto err_sha_out;
+ }
+
+ init_completion(&sdev->sec_comp_m);
+ init_completion(&sdev->sec_comp_p);
+
+ return 0;
+
+err_sha_out:
+ dma_release_channel(sdev->sec_xm_m);
+
+ return err;
+}
+
+static void jh7110_dma_cleanup(struct jh7110_sec_dev *sdev)
+{
+ dma_release_channel(sdev->sec_xm_p);
+ dma_release_channel(sdev->sec_xm_m);
+}
+struct gpio_desc *gpio;
+
+static int jh7110_cryp_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct jh7110_sec_dev *sdev;
+ struct resource *res;
+ int pages = 0;
+ int ret;
+
+ sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL);
+ if (!sdev)
+ return -ENOMEM;
+
+ sdev->dev = dev;
+
+ mutex_init(&sdev->lock);
+ mutex_init(&sdev->doing);
+ mutex_init(&sdev->pl080_doing);
+ mutex_init(&sdev->sha_lock);
+ mutex_init(&sdev->aes_lock);
+ mutex_init(&sdev->des_lock);
+ mutex_init(&sdev->rsa_lock);
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "secreg");
+ sdev->io_base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(sdev->io_base))
+ return PTR_ERR(sdev->io_base);
+
+ sdev->io_phys_base = res->start;
+
+ sdev->dma_base = ioremap(0x16008000, 0x4000);
+ if (IS_ERR(sdev->dma_base))
+ return PTR_ERR(sdev->dma_base);
+
+ sdev->use_dma = device_property_read_bool(dev, "enable-dma");
+ sdev->dma_maxburst = 32;
+
+ sdev->secirq = platform_get_irq_byname(pdev, "secirq");
+ sdev->secirq = platform_get_irq(pdev, 0);
+ if (sdev->secirq < 0) {
+ dev_err(dev, "Cannot get IRQ resource\n");
+ return sdev->secirq;
+ }
+
+ ret = devm_request_threaded_irq(dev, sdev->secirq, jh7110_cryp_irq,
+ jh7110_cryp_irq_thread, IRQF_SHARED,
+ dev_name(dev), sdev);
+ if (ret) {
+ dev_err(&pdev->dev, "Can't get interrupt working.\n");
+ return ret;
+ }
+
+ sdev->sec_hclk = devm_clk_get(dev, "sec_hclk");
+ if (IS_ERR(sdev->sec_hclk)) {
+ dev_err(dev, "failed to get sec clock\n");
+ return PTR_ERR(sdev->sec_hclk);
+ }
+
+ sdev->sec_ahb = devm_clk_get(dev, "sec_ahb");
+ if (IS_ERR(sdev->sec_ahb)) {
+ dev_err(dev, "failed to get sec clock\n");
+ return PTR_ERR(sdev->sec_ahb);
+ }
+
+ sdev->rst_hresetn = devm_reset_control_get_exclusive(sdev->dev, "sec_hre");
+ if (IS_ERR(sdev->rst_hresetn)) {
+ dev_err(sdev->dev, "failed to get sec reset\n");
+ return PTR_ERR(sdev->rst_hresetn);
+ }
+
+ clk_prepare_enable(sdev->sec_hclk);
+ clk_prepare_enable(sdev->sec_ahb);
+ reset_control_deassert(sdev->rst_hresetn);
+
+ platform_set_drvdata(pdev, sdev);
+
+ spin_lock(&dev_list.lock);
+ list_add(&sdev->list, &dev_list.dev_list);
+ spin_unlock(&dev_list.lock);
+
+ if (sdev->use_dma) {
+ ret = jh7110_dma_init(sdev);
+ if (ret) {
+ dev_err(dev, "Cannot initial dma chan\n");
+ goto err_dma_init;
+ }
+ }
+
+ pages = get_order(JH7110_MSG_BUFFER_SIZE);
+
+ sdev->sha_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages);
+ if (!sdev->sha_data) {
+ dev_err(sdev->dev, "Can't allocate aes buffer pages when unaligned\n");
+ goto err_sha_data;
+ }
+
+ sdev->aes_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages);
+ if (!sdev->aes_data) {
+ dev_err(sdev->dev, "Can't allocate aes buffer pages when unaligned\n");
+ goto err_aes_data;
+ }
+
+ sdev->des_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages);
+ if (!sdev->des_data) {
+ dev_err(sdev->dev, "Can't allocate des buffer pages when unaligned\n");
+ goto err_des_data;
+ }
+
+ sdev->pka_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages);
+ if (!sdev->pka_data) {
+ dev_err(sdev->dev, "Can't allocate pka buffer pages when unaligned\n");
+ goto err_pka_data;
+ }
+
+ sdev->pages_count = pages >> 1;
+ sdev->data_buf_len = JH7110_MSG_BUFFER_SIZE >> 1;
+
+ /* Initialize crypto engine */
+ sdev->engine = crypto_engine_alloc_init(dev, 1);
+ if (!sdev->engine) {
+ ret = -ENOMEM;
+ goto err_engine;
+ }
+
+ ret = crypto_engine_start(sdev->engine);
+ if (ret)
+ goto err_engine_start;
+
+ ret = jh7110_hash_register_algs();
+ if (ret)
+ goto err_algs_sha;
+
+ ret = jh7110_aes_register_algs();
+ if (ret)
+ goto err_algs_aes;
+
+ ret = jh7110_des_register_algs();
+ if (ret)
+ goto err_algs_des;
+
+ ret = jh7110_pka_register_algs();
+ if (ret)
+ goto err_algs_pka;
+
+ dev_info(dev, "Initialized\n");
+
+ return 0;
+ err_algs_pka:
+ jh7110_des_unregister_algs();
+ err_algs_des:
+ jh7110_aes_unregister_algs();
+ err_algs_aes:
+ jh7110_hash_unregister_algs();
+ err_algs_sha:
+ crypto_engine_stop(sdev->engine);
+ err_engine_start:
+ crypto_engine_exit(sdev->engine);
+ err_engine:
+ free_pages((unsigned long)sdev->pka_data, pages);
+ err_pka_data:
+ free_pages((unsigned long)sdev->des_data, pages);
+ err_des_data:
+ free_pages((unsigned long)sdev->aes_data, pages);
+ err_aes_data:
+ free_pages((unsigned long)sdev->sha_data, pages);
+ err_sha_data:
+ jh7110_dma_cleanup(sdev);
+ err_dma_init:
+ spin_lock(&dev_list.lock);
+ list_del(&sdev->list);
+ spin_unlock(&dev_list.lock);
+
+ clk_disable_unprepare(sdev->sec_hclk);
+ clk_disable_unprepare(sdev->sec_ahb);
+
+ return ret;
+}
+
+static int jh7110_cryp_remove(struct platform_device *pdev)
+{
+ struct jh7110_sec_dev *sdev = platform_get_drvdata(pdev);
+
+ if (!sdev)
+ return -ENODEV;
+
+ jh7110_pka_unregister_algs();
+ jh7110_des_unregister_algs();
+ jh7110_aes_unregister_algs();
+ jh7110_hash_unregister_algs();
+
+ crypto_engine_stop(sdev->engine);
+ crypto_engine_exit(sdev->engine);
+
+ jh7110_dma_cleanup(sdev);
+
+ free_pages((unsigned long)sdev->pka_data, sdev->pages_count);
+ free_pages((unsigned long)sdev->des_data, sdev->pages_count);
+ free_pages((unsigned long)sdev->aes_data, sdev->pages_count);
+ free_pages((unsigned long)sdev->sha_data, sdev->pages_count);
+ sdev->pka_data = NULL;
+ sdev->des_data = NULL;
+ sdev->aes_data = NULL;
+ sdev->sha_data = NULL;
+
+ spin_lock(&dev_list.lock);
+ list_del(&sdev->list);
+ spin_unlock(&dev_list.lock);
+
+ clk_disable_unprepare(sdev->sec_hclk);
+ clk_disable_unprepare(sdev->sec_ahb);
+
+ return 0;
+}
+
+#ifdef CONFIG_PM
+static int jh7110_cryp_runtime_suspend(struct device *dev)
+{
+ struct jh7110_sec_dev *sdev = dev_get_drvdata(dev);
+
+ clk_disable_unprepare(sdev->clk);
+
+ return 0;
+}
+
+static int jh7110_cryp_runtime_resume(struct device *dev)
+{
+ struct jh7110_sec_dev *sdev = dev_get_drvdata(dev);
+ int ret;
+
+ ret = clk_prepare_enable(sdev->clk);
+ if (ret) {
+ dev_err(sdev->dev, "Failed to prepare_enable clock\n");
+ return ret;
+ }
+
+ return 0;
+}
+#endif
+
+static const struct dev_pm_ops jh7110_cryp_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+ pm_runtime_force_resume)
+ SET_RUNTIME_PM_OPS(jh7110_cryp_runtime_suspend,
+ jh7110_cryp_runtime_resume, NULL)
+};
+
+static struct platform_driver jh7110_cryp_driver = {
+ .probe = jh7110_cryp_probe,
+ .remove = jh7110_cryp_remove,
+ .driver = {
+ .name = DRIVER_NAME,
+ .pm = &jh7110_cryp_pm_ops,
+ .of_match_table = jh7110_dt_ids,
+ },
+};
+
+module_platform_driver(jh7110_cryp_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Huan Feng <huan.feng@starfivetech.com>");
+MODULE_DESCRIPTION("Starfive JH7110 CRYP DES SHA and AES driver");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ *
+ * THE PRESENT SOFTWARE WHICH IS FOR GUIDANCE ONLY AIMS AT PROVIDING
+ * CUSTOMERS WITH CODING INFORMATION REGARDING THEIR PRODUCTS IN ORDER
+ * FOR THEM TO SAVE TIME. AS A RESULT, STARFIVE SHALL NOT BE HELD LIABLE
+ * FOR ANY DIRECT, INDIRECT OR CONSEQUENTIAL DAMAGES WITH RESPECT TO ANY
+ * CLAIMS ARISING FROM THE CONTENT OF SUCH SOFTWARE AND/OR THE USE MADE
+ * BY CUSTOMERS OF THE CODING INFORMATION CONTAINED HEREIN IN CONNECTION
+ * WITH THEIR PRODUCTS.
+ */
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+
+#include <linux/dma-direct.h>
+#include <crypto/hash.h>
+#include <crypto/sm3.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/hash.h>
+
+#include "jh7110-pl080.h"
+#include "jh7110-str.h"
+
+#define HASH_OP_UPDATE 1
+#define HASH_OP_FINAL 2
+
+#define HASH_FLAGS_INIT BIT(0)
+#define HASH_FLAGS_FINAL BIT(1)
+#define HASH_FLAGS_FINUP BIT(2)
+
+#define JH7110_MAX_ALIGN_SIZE SHA512_BLOCK_SIZE
+
+#define JH7110_HASH_BUFLEN 8192
+
+static inline int jh7110_hash_wait_done(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int ret = -1;
+
+ if (sdev->done_flags & (JH7110_SHA_KEY_DONE | JH7110_SHA_HMAC_DONE | JH7110_SHA_SHA_DONE))
+ ret = 0;
+ return ret;
+}
+
+static inline int jh7110_hash_wait_busy(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + JH7110_SHA_SHACSR, status,
+ !(status & JH7110_SHA_BUSY), 10, 100000);
+}
+
+static unsigned int jh7110_hash_reverse(unsigned int data)
+{
+ unsigned int ret;
+
+ ret = (((data & 0x000000ff) << 24) | ((data & 0x0000ff00) << 8) |
+ ((data & 0x00ff0000) >> 8) | ((data & 0xff000000) >> 24));
+
+ return ret;
+}
+
+static int jh7110_get_hash_size(struct jh7110_sec_ctx *ctx)
+{
+ unsigned int hashsize;
+
+ switch (ctx->sha_mode & JH7110_SHA_MODE_MASK) {
+ case JH7110_SHA_SHA1:
+ hashsize = SHA1_DIGEST_SIZE;
+ break;
+ case JH7110_SHA_SHA224:
+ hashsize = SHA224_DIGEST_SIZE;
+ break;
+ case JH7110_SHA_SHA256:
+ hashsize = SHA256_DIGEST_SIZE;
+ break;
+ case JH7110_SHA_SHA384:
+ hashsize = SHA384_DIGEST_SIZE;
+ break;
+ case JH7110_SHA_SHA512:
+ hashsize = SHA512_DIGEST_SIZE;
+ break;
+ case JH7110_SHA_SM3:
+ hashsize = SM3_DIGEST_SIZE;
+ break;
+ default:
+ return 0;
+ }
+ return hashsize;
+}
+
+static void jh7110_hash_start(struct jh7110_sec_ctx *ctx, int flags)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ rctx->csr.sha_csr.v = jh7110_sec_read(sdev, JH7110_SHA_SHACSR);
+ rctx->csr.sha_csr.firstb = 0;
+
+ if (flags)
+ rctx->csr.sha_csr.final = 1;
+
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v);
+}
+
+static void jh7110_sha_hmac_key(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int klen = ctx->keylen, loop;
+ unsigned int *key_tmp;
+
+ sdev->done_flags = 0;
+ sdev->cry_type = JH7110_SHA_TYPE;
+
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWKLEN, ctx->keylen);
+ rctx->csr.sha_csr.key_flag = 1;
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v);
+
+ key_tmp = (unsigned int *)ctx->key;
+
+ for (loop = 0; loop < klen / sizeof(unsigned int); loop++)
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWKR, key_tmp[loop]);
+
+ jh7110_hash_wait_done(ctx);
+}
+
+static void jh7110_hash_write_back(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ unsigned int *data = (unsigned int *)rctx->sha_digest_mid;
+ int loop;
+
+ for (loop = 0; loop < jh7110_get_hash_size(ctx) / sizeof(unsigned int); loop++)
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWSR, data[loop]);
+}
+
+static void jh7110_sha_dma_callback(void *param)
+{
+ struct jh7110_sec_dev *sdev = param;
+
+ complete(&sdev->sec_comp_m);
+}
+
+static int jh7110_hash_xmit_dma(struct jh7110_sec_ctx *ctx, int flags)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct dma_async_tx_descriptor *in_desc;
+ dma_cookie_t cookie;
+ union jh7110_alg_cr alg_cr;
+ int total_len;
+ int ret;
+
+ if (!rctx->bufcnt)
+ return 0;
+
+ ctx->sha_len_total += rctx->bufcnt;
+
+ total_len = rctx->bufcnt;
+
+ jh7110_sec_write(sdev, JH7110_DMA_IN_LEN_OFFSET, rctx->bufcnt);
+
+ total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
+
+ memset(sdev->sha_data + rctx->bufcnt, 0, total_len - rctx->bufcnt);
+
+ alg_cr.v = 0;
+ alg_cr.start = 1;
+ alg_cr.sha_dma_en = 1;
+ jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v);
+
+ sg_init_table(&ctx->sg[0], 1);
+ sg_set_buf(&ctx->sg[0], sdev->sha_data, total_len);
+ sg_dma_address(&ctx->sg[0]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->sha_data));
+ sg_dma_len(&ctx->sg[0]) = total_len;
+
+ ret = dma_map_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+ if (!ret) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ sdev->cfg_in.direction = DMA_MEM_TO_DEV;
+ sdev->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.src_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_addr = sdev->io_phys_base + JH7110_ALG_FIFO_OFFSET;
+
+ dmaengine_slave_config(sdev->sec_xm_m, &sdev->cfg_in);
+
+ in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &ctx->sg[0],
+ 1, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!in_desc)
+ return -EINVAL;
+
+ reinit_completion(&sdev->sec_comp_m);
+
+ in_desc->callback = jh7110_sha_dma_callback;
+ in_desc->callback_param = sdev;
+
+ cookie = dmaengine_submit(in_desc);
+ dma_async_issue_pending(sdev->sec_xm_m);
+
+ if (!wait_for_completion_timeout(&sdev->sec_comp_m,
+ msecs_to_jiffies(10000))) {
+ dev_dbg(sdev->dev, "this is debug for lophyel status = %x err = %x control0 = %x control1 = %x %s %s %d\n",
+ readl_relaxed(sdev->dma_base + PL080_TC_STATUS), readl_relaxed(sdev->dma_base + PL080_ERR_STATUS),
+ readl_relaxed(sdev->dma_base + 0x10c), readl_relaxed(sdev->dma_base + 0x12c),
+ __FILE__, __func__, __LINE__);
+ dev_err(sdev->dev, "wait_for_completion_timeout out error cookie = %x\n",
+ dma_async_is_tx_complete(sdev->sec_xm_p, cookie,
+ NULL, NULL));
+ }
+
+ dma_unmap_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+
+ alg_cr.v = 0;
+ alg_cr.clear = 1;
+ jh7110_sec_write(sdev, JH7110_ALG_CR_OFFSET, alg_cr.v);
+
+ return 0;
+}
+
+static int jh7110_hash_xmit_cpu(struct jh7110_sec_ctx *ctx, int flags)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int total_len, mlen, loop;
+ unsigned int *buffer;
+ unsigned char *cl;
+
+ if (!rctx->bufcnt)
+ return 0;
+
+ ctx->sha_len_total += rctx->bufcnt;
+
+ total_len = rctx->bufcnt;
+ mlen = total_len / sizeof(u32);// DIV_ROUND_UP(total_len, sizeof(u32));
+ buffer = (unsigned int *)ctx->buffer;
+ for (loop = 0; loop < mlen; loop++, buffer++)
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWDR, *buffer);
+
+ if (total_len & 0x3) {
+ cl = (unsigned char *)buffer;
+ for (loop = 0; loop < (total_len & 0x3); loop++, cl++)
+ jh7110_sec_writeb(sdev, JH7110_SHA_SHAWDR, *cl);
+ }
+
+ return 0;
+}
+
+static void jh7110_hash_append_sg(struct jh7110_sec_request_ctx *rctx)
+{
+ struct jh7110_sec_ctx *ctx = rctx->ctx;
+ size_t count;
+
+ while ((rctx->bufcnt < rctx->buflen) && rctx->total) {
+ count = min(rctx->in_sg->length - rctx->offset, rctx->total);
+ count = min(count, rctx->buflen - rctx->bufcnt);
+
+ if (count <= 0) {
+ if ((rctx->in_sg->length == 0) && !sg_is_last(rctx->in_sg)) {
+ rctx->in_sg = sg_next(rctx->in_sg);
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ scatterwalk_map_and_copy(ctx->buffer + rctx->bufcnt, rctx->in_sg,
+ rctx->offset, count, 0);
+
+ rctx->bufcnt += count;
+ rctx->offset += count;
+ rctx->total -= count;
+
+ if (rctx->offset == rctx->in_sg->length) {
+ rctx->in_sg = sg_next(rctx->in_sg);
+ if (rctx->in_sg)
+ rctx->offset = 0;
+ else
+ rctx->total = 0;
+ }
+ }
+}
+
+static int jh7110_hash_xmit(struct jh7110_sec_ctx *ctx, int flags)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ int count, *data, mlen;
+ int ret;
+
+ mlen = jh7110_get_hash_size(ctx) / sizeof(unsigned int);
+
+ sdev->cry_type = JH7110_SHA_TYPE;
+
+ rctx->csr.sha_csr.v = 0;
+ rctx->csr.sha_csr.reset = 1;
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v);
+
+ if (jh7110_hash_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ if (ctx->sha_mode & JH7110_SHA_HMAC_FLAGS)
+ jh7110_sha_hmac_key(ctx);
+
+ rctx->csr.sha_csr.v = 0;
+ rctx->csr.sha_csr.start = 1;
+ rctx->csr.sha_csr.mode = ctx->sha_mode & JH7110_SHA_MODE_MASK;
+
+ if (ctx->sec_init) {
+ rctx->csr.sha_csr.firstb = 1;
+ ctx->sec_init = 0;
+ } else {
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN3, 0X00000000);
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN2, 0X00000000);
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN1,
+ jh7110_hash_reverse(ctx->sha_len_total >> 32));
+ jh7110_sec_write(sdev, JH7110_SHA_SHAWLEN0,
+ jh7110_hash_reverse(ctx->sha_len_total & 0xffffffff));
+ jh7110_hash_write_back(ctx);
+ }
+
+ rctx->csr.sha_csr.hmac = !!(ctx->sha_mode & JH7110_SHA_HMAC_FLAGS);
+ if (!ctx->sdev->use_dma)
+ rctx->csr.sha_csr.ie = 1;
+
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v);
+
+ if (jh7110_hash_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ if (ctx->sdev->use_dma) {
+ ret = jh7110_hash_xmit_dma(ctx, flags);
+ if (flags)
+ rctx->flags |= HASH_FLAGS_FINAL;
+ } else {
+ ret = jh7110_hash_xmit_cpu(ctx, flags);
+ if (flags)
+ rctx->flags |= HASH_FLAGS_FINAL;
+ }
+
+ if (ret)
+ return ret;
+
+ if (jh7110_hash_wait_busy(ctx)) {
+ dev_err(sdev->dev, "jh7110_hash_wait_busy error\n");
+ return -ETIMEDOUT;
+ }
+
+ jh7110_hash_start(ctx, flags);
+
+ if (ctx->sha_mode & JH7110_SHA_HMAC_FLAGS) {
+ rctx->csr.sha_csr.final = 1;
+ jh7110_sec_write(sdev, JH7110_SHA_SHACSR, rctx->csr.sha_csr.v);
+ }
+
+ if (jh7110_hash_wait_busy(ctx))
+ dev_dbg(sdev->dev, "this is debug %s %s %d\n", __FILE__, __func__, __LINE__);
+
+ if (!flags) {
+ data = (unsigned int *)rctx->sha_digest_mid;
+ for (count = 0; count < mlen; count++)
+ data[count] = jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR);
+ }
+
+ return 0;
+}
+
+static int jh7110_hash_update_req(struct jh7110_sec_ctx *ctx)
+{
+ struct jh7110_sec_request_ctx *rctx = ctx->rctx;
+ int err = 0, final;
+
+ final = (rctx->flags & HASH_FLAGS_FINUP);
+
+ while ((rctx->total >= rctx->buflen) ||
+ (rctx->bufcnt + rctx->total >= rctx->buflen)) {
+ jh7110_hash_append_sg(rctx);
+ err = jh7110_hash_xmit(ctx, 0);
+ rctx->bufcnt = 0;
+ }
+
+ jh7110_hash_append_sg(rctx);
+
+ if (final) {
+ err = jh7110_hash_xmit(ctx,
+ (rctx->flags & HASH_FLAGS_FINUP));
+ rctx->bufcnt = 0;
+ }
+
+ return err;
+}
+
+static int jh7110_hash_final_req(struct jh7110_sec_ctx *ctx)
+{
+ struct ahash_request *req = ctx->rctx->req.hreq;
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ int err = 0;
+
+ err = jh7110_hash_xmit(ctx, 1);
+ rctx->bufcnt = 0;
+
+ return err;
+}
+
+
+static int jh7110_hash_out_cpu(struct ahash_request *req)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct jh7110_sec_ctx *ctx = rctx->ctx;
+ int count, *data;
+ int mlen;
+
+ if (!req->result)
+ return 0;
+
+ mlen = jh7110_get_hash_size(ctx) / sizeof(u32);
+
+ if (mlen == 28)
+ jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR);
+
+ if (mlen == 48) {
+ for (count = 0; count < 4; count++)
+ jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR);
+ }
+ data = (u32 *)req->result;
+ for (count = 0; count < mlen; count++)
+ data[count] = jh7110_sec_read(ctx->sdev, JH7110_SHA_SHARDR);
+
+ return 0;
+}
+
+static int jh7110_hash_copy_hash(struct ahash_request *req)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct jh7110_sec_ctx *ctx = rctx->ctx;
+ int hashsize;
+ int ret;
+
+ hashsize = jh7110_get_hash_size(ctx);
+
+ ret = jh7110_hash_out_cpu(req);
+
+ if (ret)
+ return ret;
+
+ memcpy(rctx->sha_digest_mid, req->result, hashsize);
+ rctx->sha_digest_len = hashsize;
+
+ return ret;
+}
+
+static void jh7110_hash_finish_req(struct ahash_request *req, int err)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct jh7110_sec_dev *sdev = rctx->sdev;
+
+ if (!err && (HASH_FLAGS_FINAL & rctx->flags)) {
+ err = jh7110_hash_copy_hash(req);
+ rctx->flags &= ~(HASH_FLAGS_FINAL |
+ HASH_FLAGS_INIT);
+ }
+
+ crypto_finalize_hash_request(sdev->engine, req, err);
+}
+
+static int jh7110_hash_prepare_req(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx;
+
+ if (!sdev)
+ return -ENODEV;
+
+ mutex_lock(&ctx->sdev->lock);
+
+ rctx = ahash_request_ctx(req);
+
+ rctx->req.hreq = req;
+
+ return 0;
+}
+
+static int jh7110_hash_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+ struct jh7110_sec_request_ctx *rctx;
+ int err = 0;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx = ahash_request_ctx(req);
+
+ if (rctx->op == HASH_OP_UPDATE)
+ err = jh7110_hash_update_req(ctx);
+ else if (rctx->op == HASH_OP_FINAL)
+ err = jh7110_hash_final_req(ctx);
+
+ if (err != -EINPROGRESS)
+ /* done task will not finish it, so do it here */
+ jh7110_hash_finish_req(req, err);
+
+ mutex_unlock(&ctx->sdev->lock);
+
+ return 0;
+}
+
+static int jh7110_hash_handle_queue(struct jh7110_sec_dev *sdev,
+ struct ahash_request *req)
+{
+ return crypto_transfer_hash_request_to_engine(sdev->engine, req);
+}
+
+static int jh7110_hash_enqueue(struct ahash_request *req, unsigned int op)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ rctx->op = op;
+
+ return jh7110_hash_handle_queue(sdev, req);
+}
+
+static int jh7110_hash_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct jh7110_sec_dev *sdev = ctx->sdev;
+
+ memset(rctx, 0, sizeof(struct jh7110_sec_request_ctx));
+
+ rctx->sdev = sdev;
+ rctx->ctx = ctx;
+ rctx->req.hreq = req;
+ rctx->bufcnt = 0;
+
+ rctx->total = 0;
+ rctx->offset = 0;
+ rctx->bufcnt = 0;
+ rctx->buflen = JH7110_HASH_BUFLEN;
+
+ memset(ctx->buffer, 0, JH7110_HASH_BUFLEN);
+
+ ctx->rctx = rctx;
+
+ dev_dbg(sdev->dev, "%s Flags %lx\n", __func__, rctx->flags);
+
+ return 0;
+}
+
+static int jh7110_hash_update(struct ahash_request *req)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ if (!req->nbytes)
+ return 0;
+
+ rctx->total = req->nbytes;
+ rctx->in_sg = req->src;
+ rctx->offset = 0;
+
+ if ((rctx->bufcnt + rctx->total < rctx->buflen)) {
+ jh7110_hash_append_sg(rctx);
+ return 0;
+ }
+
+ return jh7110_hash_enqueue(req, HASH_OP_UPDATE);
+}
+
+static int jh7110_hash_final(struct ahash_request *req)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ rctx->flags |= HASH_FLAGS_FINUP;
+
+ return jh7110_hash_enqueue(req, HASH_OP_FINAL);
+}
+
+static int jh7110_hash_finup(struct ahash_request *req)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+ int err1, err2;
+ int nents;
+
+ nents = sg_nents_for_len(req->src, req->nbytes);
+
+ rctx->flags |= HASH_FLAGS_FINUP;
+
+ err1 = jh7110_hash_update(req);
+
+ if (err1 == -EINPROGRESS || err1 == -EBUSY)
+ return err1;
+
+ /*
+ * final() has to be always called to cleanup resources
+ * even if update() failed, except EINPROGRESS
+ */
+ err2 = jh7110_hash_final(req);
+
+ return err1 ?: err2;
+}
+
+static int jh7110_hash_digest(struct ahash_request *req)
+{
+ return jh7110_hash_init(req) ?: jh7110_hash_finup(req);
+}
+
+static int jh7110_hash_export(struct ahash_request *req, void *out)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ memcpy(out, rctx, sizeof(*rctx));
+
+ return 0;
+}
+
+static int jh7110_hash_import(struct ahash_request *req, const void *in)
+{
+ struct jh7110_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ memcpy(rctx, in, sizeof(*rctx));
+
+ return 0;
+}
+
+static int jh7110_hash_cra_init_algs(struct crypto_tfm *tfm,
+ const char *algs_hmac_name,
+ unsigned int mode)
+{
+ struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->sdev = jh7110_sec_find_dev(ctx);
+
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct jh7110_sec_request_ctx));
+
+ ctx->sec_init = 1;
+ ctx->keylen = 0;
+ ctx->sha_mode = mode;
+ ctx->sha_len_total = 0;
+ ctx->buffer = ctx->sdev->sha_data;
+
+ if (algs_hmac_name)
+ ctx->sha_mode |= JH7110_SHA_HMAC_FLAGS;
+
+ ctx->enginectx.op.do_one_request = jh7110_hash_one_request;
+ ctx->enginectx.op.prepare_request = jh7110_hash_prepare_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void jh7110_hash_cra_exit(struct crypto_tfm *tfm)
+{
+ struct jh7110_sec_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int jh7110_hash_long_setkey(struct jh7110_sec_ctx *ctx,
+ const u8 *key, unsigned int keylen,
+ const char *alg_name)
+{
+ struct crypto_wait wait;
+ struct ahash_request *req;
+ struct scatterlist sg;
+ struct crypto_ahash *ahash_tfm;
+ u8 *buf;
+ int ret;
+
+ ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
+ if (IS_ERR(ahash_tfm))
+ return PTR_ERR(ahash_tfm);
+
+ req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
+ if (!req) {
+ ret = -ENOMEM;
+ goto err_free_ahash;
+ }
+
+ crypto_init_wait(&wait);
+ ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+ crypto_ahash_clear_flags(ahash_tfm, ~0);
+
+ buf = kzalloc(keylen + JH7110_MAX_ALIGN_SIZE, GFP_KERNEL);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto err_free_req;
+ }
+
+ memcpy(buf, key, keylen);
+ sg_init_one(&sg, buf, keylen);
+ ahash_request_set_crypt(req, &sg, ctx->key, keylen);
+
+ ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
+
+err_free_req:
+ ahash_request_free(req);
+err_free_ahash:
+ crypto_free_ahash(ahash_tfm);
+ return ret;
+}
+
+static int jh7110_hash1_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha1");
+ }
+
+ return ret;
+}
+
+static int jh7110_hash224_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha224");
+ }
+
+ return ret;
+}
+
+static int jh7110_hash256_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha256");
+ }
+
+ return ret;
+}
+
+static int jh7110_hash384_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha384");
+ }
+
+ return ret;
+}
+
+static int jh7110_hash512_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sha512");
+ }
+
+ return ret;
+}
+
+static int jh7110_sm3_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct jh7110_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = jh7110_hash_long_setkey(ctx, key, keylen, "jh7110-sm3");
+ }
+
+ return ret;
+}
+
+static int jh7110_hash_cra_sha1_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA1);
+}
+
+static int jh7110_hash_cra_sha224_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA224);
+}
+
+static int jh7110_hash_cra_sha256_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA256);
+}
+
+static int jh7110_hash_cra_sha384_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA384);
+}
+
+static int jh7110_hash_cra_sha512_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SHA512);
+}
+
+static int jh7110_hash_cra_sm3_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, NULL, JH7110_SHA_SM3);
+}
+
+static int jh7110_hash_cra_hmac_sha1_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sha1", JH7110_SHA_SHA1);
+}
+
+static int jh7110_hash_cra_hmac_sha224_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sha224", JH7110_SHA_SHA224);
+}
+
+static int jh7110_hash_cra_hmac_sha256_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sha256", JH7110_SHA_SHA256);
+}
+
+static int jh7110_hash_cra_hmac_sha384_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sha384", JH7110_SHA_SHA384);
+}
+
+static int jh7110_hash_cra_hmac_sha512_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sha512", JH7110_SHA_SHA512);
+}
+
+static int jh7110_hash_cra_hmac_sm3_init(struct crypto_tfm *tfm)
+{
+ return jh7110_hash_cra_init_algs(tfm, "sm3", JH7110_SHA_SM3);
+}
+
+static struct ahash_alg algs_sha0_sha512_sm3[] = {
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "jh7110-sha1",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sha1_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .setkey = jh7110_hash1_setkey,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "jh7110-hmac-sha1",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sha1_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "jh7110-sha224",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sha224_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .setkey = jh7110_hash224_setkey,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "jh7110-hmac-sha224",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sha224_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "jh7110-sha256",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sha256_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .setkey = jh7110_hash256_setkey,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "jh7110-hmac-sha256",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sha256_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "jh7110-sha384",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sha384_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .setkey = jh7110_hash384_setkey,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "jh7110-hmac-sha384",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sha384_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "jh7110-sha512",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sha512_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .setkey = jh7110_hash512_setkey,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "jh7110-hmac-sha512",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sha512_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "sm3",
+ .cra_driver_name = "jh7110-sm3",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_sm3_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+ {
+ .init = jh7110_hash_init,
+ .update = jh7110_hash_update,
+ .final = jh7110_hash_final,
+ .finup = jh7110_hash_finup,
+ .digest = jh7110_hash_digest,
+ .setkey = jh7110_sm3_setkey,
+ .export = jh7110_hash_export,
+ .import = jh7110_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct jh7110_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sm3)",
+ .cra_driver_name = "jh7110-hmac-sm3",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct jh7110_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = jh7110_hash_cra_hmac_sm3_init,
+ .cra_exit = jh7110_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+ },
+};
+
+int jh7110_hash_register_algs(void)
+{
+ int ret = 0;
+
+ ret = crypto_register_ahashes(algs_sha0_sha512_sm3, ARRAY_SIZE(algs_sha0_sha512_sm3));
+
+ return ret;
+}
+
+void jh7110_hash_unregister_algs(void)
+{
+ crypto_unregister_ahashes(algs_sha0_sha512_sm3, ARRAY_SIZE(algs_sha0_sha512_sm3));
+}
+
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2021 StarFive, Inc <huan.feng@starfivetech.com>
+ */
+#ifndef __JH7110_STR_H__
+#define __JH7110_STR_H__
+
+#include <crypto/internal/akcipher.h>
+#include <crypto/internal/rsa.h>
+#include <crypto/engine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+
+#include "jh7110-regs.h"
+
+#define JH7110_MSG_BUFFER_SIZE (16 * 1024)
+#define MAX_KEY_SIZE SHA512_BLOCK_SIZE
+
+#define JH7110_AES_IV_LEN AES_BLOCK_SIZE
+#define JH7110_AES_CTR_LEN AES_BLOCK_SIZE
+
+
+struct jh7110_rsa_key {
+ u8 *n;
+ u8 *e;
+ u8 *d;
+ u8 *p;
+ u8 *q;
+ u8 *dp;
+ u8 *dq;
+ u8 *qinv;
+ u8 *rinv;
+ u8 *rinv_p;
+ u8 *rinv_q;
+ u8 *mp;
+ u8 *rsqr;
+ u8 *rsqr_p;
+ u8 *rsqr_q;
+ u8 *pmp;
+ u8 *qmp;
+ int e_bitlen;
+ int d_bitlen;
+ int bitlen;
+ size_t key_sz;
+ bool crt_mode;
+};
+
+struct jh7110_sec_ctx {
+ struct crypto_engine_ctx enginectx;
+
+ struct jh7110_sec_request_ctx *rctx;
+ struct jh7110_sec_dev *sdev;
+
+ unsigned int sha_mode;
+
+ u8 key[MAX_KEY_SIZE];
+ int keylen;
+ int sec_init;
+ struct scatterlist sg[2];
+ struct jh7110_rsa_key rsa_key;
+ size_t sha_len_total;
+ u8 *buffer;
+};
+
+struct jh7110_sec_dev {
+ struct list_head list;
+ struct device *dev;
+
+ struct clk *sec_hclk;
+ struct clk *sec_ahb;
+ struct reset_control *rst_hresetn;
+
+ struct jh7110_pl08x_device *pl080;
+
+ void __iomem *io_base;
+ void __iomem *dma_base;
+ phys_addr_t io_phys_base;
+ void *sha_data;
+ void *aes_data;
+ void *des_data;
+ void *pka_data;
+ unsigned int secirq;
+ unsigned int irq;
+
+ size_t data_buf_len;
+ int pages_count;
+ u32 use_dma;
+ u32 dma_maxburst;
+ struct dma_chan *sec_xm_m;
+ struct dma_chan *sec_xm_p;
+ struct dma_slave_config cfg_in;
+ struct dma_slave_config cfg_out;
+ struct completion sec_comp_m;
+ struct completion sec_comp_p;
+ struct scatterlist in_sg;
+ struct scatterlist out_sg;
+ unsigned long in_sg_len;
+ unsigned long out_sg_len;
+
+
+ struct mutex doing;
+ struct mutex pl080_doing;
+ struct mutex lock; /* protects req / areq */
+ struct mutex sha_lock;
+ struct mutex des_lock;
+ struct mutex aes_lock;
+ struct mutex rsa_lock;
+
+#define JH7110_SHA_KEY_DONE BIT(0)
+#define JH7110_SHA_HMAC_DONE BIT(1)
+#define JH7110_SHA_SHA_DONE BIT(2)
+#define JH7110_AES_DONE BIT(3)
+#define JH7110_DES_DONE BIT(4)
+#define JH7110_PKA_DONE BIT(5)
+ u32 done_flags;
+#define JH7110_SHA_TYPE 0x1
+#define JH7110_AES_TYPE 0x2
+#define JH7110_DES_TYPE 0x3
+#define JH7110_PKA_TYPE 0x4
+ u32 cry_type;
+
+ struct crypto_engine *engine;
+
+ union jh7110_alg_cr alg_cr;
+ union jh7110_ie_mask ie_mask;
+ union jh7110_ie_flag ie_flag;
+};
+
+struct jh7110_sec_request_ctx {
+ struct jh7110_sec_ctx *ctx;
+ struct jh7110_sec_dev *sdev;
+
+ union {
+ struct ahash_request *hreq;
+ struct skcipher_request *sreq;
+ struct aead_request *areq;
+ } req;
+
+#define JH7110_AHASH_REQ 0
+#define JH7110_ABLK_REQ 1
+#define JH7110_AEAD_REQ 2
+ unsigned int req_type;
+
+ union {
+ union jh7110_crypto_cacr pka_csr;
+ union jh7110_des_daecsr des_csr;
+ union jh7110_aes_csr aes_csr;
+ union jh7110_sha_shacsr sha_csr;
+ } csr;
+
+ struct scatterlist *sg;
+ struct scatterlist *in_sg;
+ struct scatterlist *out_sg;
+ struct scatterlist in_sgl;
+ struct scatterlist out_sgl;
+
+ unsigned long sg_len;
+ unsigned long in_sg_len;
+ unsigned long out_sg_len;
+
+ unsigned long flags;
+ unsigned long op;
+ unsigned long stmode;
+ unsigned long long jiffies_hw;
+ unsigned long long jiffies_cp;
+
+ size_t bufcnt;
+ size_t buflen;
+ size_t total;
+ size_t offset;
+ size_t data_offset;
+ size_t authsize;
+ size_t total_in;
+ size_t total_out;
+ size_t assoclen;
+ size_t ctr_over_count;
+
+ u32 msg_end[4];
+ u32 dec_end[4];
+ u32 last_ctr[4];
+ u32 aes_nonce[4];
+ u32 aes_iv[4];
+ u8 sha_digest_mid[SHA512_DIGEST_SIZE]__aligned(sizeof(u32));
+ unsigned int sha_digest_len;
+};
+
+struct jh7110_sec_dma {
+ struct dma_slave_config cfg;
+ union jh7110_alg_cr alg_cr;
+ struct dma_chan *chan;
+ struct completion *dma_comp;
+ struct scatterlist *sg;
+ struct jh7110_sec_ctx *ctx;
+ void *data;
+ size_t total;
+};
+
+static inline u64 jh7110_sec_readq(struct jh7110_sec_dev *sdev, u32 offset)
+{
+#ifdef CONFIG_64BIT
+ return __raw_readq(sdev->io_base + offset);
+#else
+ return ((u64)__raw_readl(sdev->io_base + offset) << 32) | (u64)__raw_readl(sdev->io_base + offset + 4);
+#endif
+}
+
+static inline u32 jh7110_sec_read(struct jh7110_sec_dev *sdev, u32 offset)
+{
+ return __raw_readl(sdev->io_base + offset);
+}
+
+static inline u16 jh7110_sec_readw(struct jh7110_sec_dev *sdev, u32 offset)
+{
+ return __raw_readw(sdev->io_base + offset);
+}
+
+static inline u8 jh7110_sec_readb(struct jh7110_sec_dev *sdev, u32 offset)
+{
+ return __raw_readb(sdev->io_base + offset);
+}
+
+static inline void jh7110_sec_writeq(struct jh7110_sec_dev *sdev,
+ u32 offset, u64 value)
+{
+#ifdef CONFIG_64BIT
+ __raw_writeq(value, sdev->io_base + offset);
+#else
+ __raw_writel((value >> 32), sdev->io_base + offset);
+ __raw_writel(value & 0xffffffff, sdev->io_base + offset + 4);
+#endif
+}
+
+static inline void jh7110_sec_write(struct jh7110_sec_dev *sdev,
+ u32 offset, u32 value)
+{
+ __raw_writel(value, sdev->io_base + offset);
+}
+
+static inline void jh7110_sec_writew(struct jh7110_sec_dev *sdev,
+ u32 offset, u16 value)
+{
+ __raw_writew(value, sdev->io_base + offset);
+}
+
+static inline void jh7110_sec_writeb(struct jh7110_sec_dev *sdev,
+ u32 offset, u8 value)
+{
+ __raw_writeb(value, sdev->io_base + offset);
+}
+
+extern struct jh7110_sec_dev *jh7110_sec_find_dev(struct jh7110_sec_ctx *ctx);
+
+extern int jh7110_hash_register_algs(void);
+extern void jh7110_hash_unregister_algs(void);
+
+extern int jh7110_aes_register_algs(void);
+extern void jh7110_aes_unregister_algs(void);
+
+extern int jh7110_des_register_algs(void);
+extern void jh7110_des_unregister_algs(void);
+
+extern int jh7110_pka_register_algs(void);
+extern void jh7110_pka_unregister_algs(void);
+
+extern int jh7110_dma_sg_to_device(struct jh7110_sec_dma *sdma);
+extern int jh7110_dma_mem_to_device(struct jh7110_sec_dma *sdma);
+extern int jh7110_dma_sg_from_device(struct jh7110_sec_dma *sdma);
+extern int jh7110_dma_mem_from_device(struct jh7110_sec_dma *sdma);
+extern int jh7110_mem_to_mem_test(struct jh7110_sec_ctx *ctx);
+
+extern int jh7110_dmac_init(struct jh7110_sec_dev *sdev, int irq);
+extern int jh7110_dmac_secdata_out(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size);
+extern int jh7110_dmac_secdata_in(struct jh7110_sec_dev *sdev, u8 chan, u32 src, u32 dst, u32 size);
+extern int jh7110_dmac_wait_done(struct jh7110_sec_dev *sdev, u8 chan);
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2006 ARM Ltd.
+ * Copyright (c) 2010 ST-Ericsson SA
+ * Copyirght (c) 2017 Linaro Ltd.
+ *
+ * Author: Peter Pearse <peter.pearse@arm.com>
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ *
+ * Documentation: ARM DDI 0196G == PL080
+ * Documentation: ARM DDI 0218E == PL081
+ * Documentation: S3C6410 User's Manual == PL080S
+ *
+ * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
+ * channel.
+ *
+ * The PL080 has 8 channels available for simultaneous use, and the PL081
+ * has only two channels. So on these DMA controllers the number of channels
+ * and the number of incoming DMA signals are two totally different things.
+ * It is usually not possible to theoretically handle all physical signals,
+ * so a multiplexing scheme with possible denial of use is necessary.
+ *
+ * The PL080 has a dual bus master, PL081 has a single master.
+ *
+ * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
+ * It differs in following aspects:
+ * - CH_CONFIG register at different offset,
+ * - separate CH_CONTROL2 register for transfer size,
+ * - bigger maximum transfer size,
+ * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
+ * - no support for peripheral flow control.
+ *
+ * Memory to peripheral transfer may be visualized as
+ * Get data from memory to DMAC
+ * Until no data left
+ * On burst request from peripheral
+ * Destination burst from DMAC to peripheral
+ * Clear burst request
+ * Raise terminal count interrupt
+ *
+ * For peripherals with a FIFO:
+ * Source burst size == half the depth of the peripheral FIFO
+ * Destination burst size == the depth of the peripheral FIFO
+ *
+ * (Bursts are irrelevant for mem to mem transfers - there are no burst
+ * signals, the DMA controller will simply facilitate its AHB master.)
+ *
+ * ASSUMES default (little) endianness for DMA transfers
+ *
+ * The PL08x has two flow control settings:
+ * - DMAC flow control: the transfer size defines the number of transfers
+ * which occur for the current LLI entry, and the DMAC raises TC at the
+ * end of every LLI entry. Observed behaviour shows the DMAC listening
+ * to both the BREQ and SREQ signals (contrary to documented),
+ * transferring data if either is active. The LBREQ and LSREQ signals
+ * are ignored.
+ *
+ * - Peripheral flow control: the transfer size is ignored (and should be
+ * zero). The data is transferred from the current LLI entry, until
+ * after the final transfer signalled by LBREQ or LSREQ. The DMAC
+ * will then move to the next LLI entry. Unsupported by PL080S.
+ */
+//#include <linux/amba/bus.h>
+#include <linux/amba/pl08x.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dmaengine.h>
+#include <linux/dmapool.h>
+#include <linux/dma-mapping.h>
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/amba/pl080.h>
+
+#include "dmaengine.h"
+#include "virt-dma.h"
+
+#define DRIVER_NAME "pl08xdmac"
+
+#define PL80X_DMA_BUSWIDTHS \
+ BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
+ BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
+
+//static struct amba_driver pl08x_amba_driver;
+struct pl08x_driver_data;
+
+/**
+ * struct vendor_data - vendor-specific config parameters for PL08x derivatives
+ * @config_offset: offset to the configuration register
+ * @channels: the number of channels available in this variant
+ * @signals: the number of request signals available from the hardware
+ * @dualmaster: whether this version supports dual AHB masters or not.
+ * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the
+ * channels have Nomadik security extension bits that need to be checked
+ * for permission before use and some registers are missing
+ * @pl080s: whether this variant is a Samsung PL080S, which has separate
+ * register and LLI word for transfer size.
+ * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020
+ * @max_transfer_size: the maximum single element transfer size for this
+ * PL08x variant.
+ */
+struct vendor_data {
+ u8 config_offset;
+ u8 channels;
+ u8 signals;
+ bool dualmaster;
+ bool nomadik;
+ bool pl080s;
+ bool ftdmac020;
+ u32 max_transfer_size;
+};
+
+/**
+ * struct pl08x_bus_data - information of source or destination
+ * busses for a transfer
+ * @addr: current address
+ * @maxwidth: the maximum width of a transfer on this bus
+ * @buswidth: the width of this bus in bytes: 1, 2 or 4
+ */
+struct pl08x_bus_data {
+ dma_addr_t addr;
+ u8 maxwidth;
+ u8 buswidth;
+};
+
+#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
+
+/**
+ * struct pl08x_phy_chan - holder for the physical channels
+ * @id: physical index to this channel
+ * @base: memory base address for this physical channel
+ * @reg_config: configuration address for this physical channel
+ * @reg_control: control address for this physical channel
+ * @reg_src: transfer source address register
+ * @reg_dst: transfer destination address register
+ * @reg_lli: transfer LLI address register
+ * @reg_busy: if the variant has a special per-channel busy register,
+ * this contains a pointer to it
+ * @lock: a lock to use when altering an instance of this struct
+ * @serving: the virtual channel currently being served by this physical
+ * channel
+ * @locked: channel unavailable for the system, e.g. dedicated to secure
+ * world
+ * @ftdmac020: channel is on a FTDMAC020
+ * @pl080s: channel is on a PL08s
+ */
+struct pl08x_phy_chan {
+ unsigned int id;
+ void __iomem *base;
+ void __iomem *reg_config;
+ void __iomem *reg_control;
+ void __iomem *reg_src;
+ void __iomem *reg_dst;
+ void __iomem *reg_lli;
+ void __iomem *reg_busy;
+ spinlock_t lock;
+ struct pl08x_dma_chan *serving;
+ bool locked;
+ bool ftdmac020;
+ bool pl080s;
+};
+
+/**
+ * struct pl08x_sg - structure containing data per sg
+ * @src_addr: src address of sg
+ * @dst_addr: dst address of sg
+ * @len: transfer len in bytes
+ * @node: node for txd's dsg_list
+ */
+struct pl08x_sg {
+ dma_addr_t src_addr;
+ dma_addr_t dst_addr;
+ size_t len;
+ struct list_head node;
+};
+
+/**
+ * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
+ * @vd: virtual DMA descriptor
+ * @dsg_list: list of children sg's
+ * @llis_bus: DMA memory address (physical) start for the LLIs
+ * @llis_va: virtual memory address start for the LLIs
+ * @cctl: control reg values for current txd
+ * @ccfg: config reg values for current txd
+ * @done: this marks completed descriptors, which should not have their
+ * mux released.
+ * @cyclic: indicate cyclic transfers
+ */
+struct pl08x_txd {
+ struct virt_dma_desc vd;
+ struct list_head dsg_list;
+ dma_addr_t llis_bus;
+ u32 *llis_va;
+ /* Default cctl value for LLIs */
+ u32 cctl;
+ /*
+ * Settings to be put into the physical channel when we
+ * trigger this txd. Other registers are in llis_va[0].
+ */
+ u32 ccfg;
+ bool done;
+ bool cyclic;
+};
+
+/**
+ * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
+ * states
+ * @PL08X_CHAN_IDLE: the channel is idle
+ * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
+ * channel and is running a transfer on it
+ * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
+ * channel, but the transfer is currently paused
+ * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
+ * channel to become available (only pertains to memcpy channels)
+ */
+enum pl08x_dma_chan_state {
+ PL08X_CHAN_IDLE,
+ PL08X_CHAN_RUNNING,
+ PL08X_CHAN_PAUSED,
+ PL08X_CHAN_WAITING,
+};
+
+/**
+ * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
+ * @vc: wrappped virtual channel
+ * @phychan: the physical channel utilized by this channel, if there is one
+ * @name: name of channel
+ * @cd: channel platform data
+ * @cfg: slave configuration
+ * @at: active transaction on this channel
+ * @host: a pointer to the host (internal use)
+ * @state: whether the channel is idle, paused, running etc
+ * @slave: whether this channel is a device (slave) or for memcpy
+ * @signal: the physical DMA request signal which this channel is using
+ * @mux_use: count of descriptors using this DMA request signal setting
+ * @waiting_at: time in jiffies when this channel moved to waiting state
+ */
+struct pl08x_dma_chan {
+ struct virt_dma_chan vc;
+ struct pl08x_phy_chan *phychan;
+ const char *name;
+ struct pl08x_channel_data *cd;
+ struct dma_slave_config cfg;
+ struct pl08x_txd *at;
+ struct pl08x_driver_data *host;
+ enum pl08x_dma_chan_state state;
+ int chan_id;
+ bool slave;
+ int signal;
+ unsigned mux_use;
+ unsigned long waiting_at;
+};
+
+/**
+ * struct pl08x_driver_data - the local state holder for the PL08x
+ * @slave: optional slave engine for this instance
+ * @memcpy: memcpy engine for this instance
+ * @has_slave: the PL08x has a slave engine (routed signals)
+ * @base: virtual memory base (remapped) for the PL08x
+ * @adev: the corresponding AMBA (PrimeCell) bus entry
+ * @vd: vendor data for this PL08x variant
+ * @pd: platform data passed in from the platform/machine
+ * @phy_chans: array of data for the physical channels
+ * @pool: a pool for the LLI descriptors
+ * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
+ * fetches
+ * @mem_buses: set to indicate memory transfers on AHB2.
+ * @lli_words: how many words are used in each LLI item for this variant
+ */
+struct pl08x_driver_data {
+ struct dma_device slave;
+ struct dma_device memcpy;
+ bool has_slave;
+ void __iomem *base;
+ struct platform_device *adev;
+ const struct vendor_data *vd;
+ struct pl08x_platform_data *pd;
+ struct pl08x_phy_chan *phy_chans;
+ struct dma_pool *pool;
+ u8 lli_buses;
+ u8 mem_buses;
+ u8 lli_words;
+};
+
+/*
+ * PL08X specific defines
+ */
+
+/* The order of words in an LLI. */
+#define PL080_LLI_SRC 0
+#define PL080_LLI_DST 1
+#define PL080_LLI_LLI 2
+#define PL080_LLI_CCTL 3
+#define PL080S_LLI_CCTL2 4
+
+/* Total words in an LLI. */
+#define PL080_LLI_WORDS 4
+#define PL080S_LLI_WORDS 8
+
+/*
+ * Number of LLIs in each LLI buffer allocated for one transfer
+ * (maximum times we call dma_pool_alloc on this pool without freeing)
+ */
+#define MAX_NUM_TSFR_LLIS 512
+#define PL08X_ALIGN 8
+
+static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
+{
+ return container_of(chan, struct pl08x_dma_chan, vc.chan);
+}
+
+static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
+{
+ return container_of(tx, struct pl08x_txd, vd.tx);
+}
+
+/*
+ * Mux handling.
+ *
+ * This gives us the DMA request input to the PL08x primecell which the
+ * peripheral described by the channel data will be routed to, possibly
+ * via a board/SoC specific external MUX. One important point to note
+ * here is that this does not depend on the physical channel.
+ */
+static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
+{
+ const struct pl08x_platform_data *pd = plchan->host->pd;
+ int ret;
+
+ if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
+ ret = pd->get_xfer_signal(plchan->cd);
+ if (ret < 0) {
+ plchan->mux_use = 0;
+ return ret;
+ }
+
+ plchan->signal = ret;
+ }
+ return 0;
+}
+
+static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
+{
+ const struct pl08x_platform_data *pd = plchan->host->pd;
+
+ if (plchan->signal >= 0) {
+ WARN_ON(plchan->mux_use == 0);
+
+ if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
+ pd->put_xfer_signal(plchan->cd, plchan->signal);
+ plchan->signal = -1;
+ }
+ }
+}
+
+/*
+ * Physical channel handling
+ */
+
+/* Whether a certain channel is busy or not */
+static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
+{
+ unsigned int val;
+
+ /* If we have a special busy register, take a shortcut */
+ if (ch->reg_busy) {
+ val = readl(ch->reg_busy);
+ return !!(val & BIT(ch->id));
+ }
+ val = readl(ch->reg_config);
+ return val & PL080_CONFIG_ACTIVE;
+}
+
+/*
+ * pl08x_write_lli() - Write an LLI into the DMA controller.
+ *
+ * The PL08x derivatives support linked lists, but the first item of the
+ * list containing the source, destination, control word and next LLI is
+ * ignored. Instead the driver has to write those values directly into the
+ * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
+ * register need to be set up for the first transfer.
+ */
+static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
+ struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
+{
+ if (pl08x->vd->pl080s)
+ dev_vdbg(&pl08x->adev->dev,
+ "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
+ "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
+ phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
+ lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
+ lli[PL080S_LLI_CCTL2], ccfg);
+ else
+ //dev_vdbg(&pl08x->adev->dev,
+ dev_info(&pl08x->adev->dev,
+ "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
+ "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
+ phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
+ lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
+
+ writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);
+ writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);
+ writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);
+
+ /*
+ * The FTMAC020 has a different layout in the CCTL word of the LLI
+ * and the CCTL register which is split in CSR and SIZE registers.
+ * Convert the LLI item CCTL into the proper values to write into
+ * the CSR and SIZE registers.
+ */
+ if (phychan->ftdmac020) {
+ u32 llictl = lli[PL080_LLI_CCTL];
+ u32 val = 0;
+
+ /* Write the transfer size (12 bits) to the size register */
+ writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,
+ phychan->base + FTDMAC020_CH_SIZE);
+ /*
+ * Then write the control bits 28..16 to the control register
+ * by shuffleing the bits around to where they are in the
+ * main register. The mapping is as follows:
+ * Bit 28: TC_MSK - mask on all except last LLI
+ * Bit 27..25: SRC_WIDTH
+ * Bit 24..22: DST_WIDTH
+ * Bit 21..20: SRCAD_CTRL
+ * Bit 19..17: DSTAD_CTRL
+ * Bit 17: SRC_SEL
+ * Bit 16: DST_SEL
+ */
+ if (llictl & FTDMAC020_LLI_TC_MSK)
+ val |= FTDMAC020_CH_CSR_TC_MSK;
+ val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
+ (FTDMAC020_LLI_SRC_WIDTH_SHIFT -
+ FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));
+ val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
+ (FTDMAC020_LLI_DST_WIDTH_SHIFT -
+ FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));
+ val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >>
+ (FTDMAC020_LLI_SRCAD_CTL_SHIFT -
+ FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));
+ val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >>
+ (FTDMAC020_LLI_DSTAD_CTL_SHIFT -
+ FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));
+ if (llictl & FTDMAC020_LLI_SRC_SEL)
+ val |= FTDMAC020_CH_CSR_SRC_SEL;
+ if (llictl & FTDMAC020_LLI_DST_SEL)
+ val |= FTDMAC020_CH_CSR_DST_SEL;
+
+ /*
+ * Set up the bits that exist in the CSR but are not
+ * part the LLI, i.e. only gets written to the control
+ * register right here.
+ *
+ * FIXME: do not just handle memcpy, also handle slave DMA.
+ */
+ switch (pl08x->pd->memcpy_burst_size) {
+ default:
+ case PL08X_BURST_SZ_1:
+ val |= PL080_BSIZE_1 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_4:
+ val |= PL080_BSIZE_4 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_8:
+ val |= PL080_BSIZE_8 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_16:
+ val |= PL080_BSIZE_16 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_32:
+ val |= PL080_BSIZE_32 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_64:
+ val |= PL080_BSIZE_64 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_128:
+ val |= PL080_BSIZE_128 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_256:
+ val |= PL080_BSIZE_256 <<
+ FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
+ break;
+ }
+
+ /* Protection flags */
+ if (pl08x->pd->memcpy_prot_buff)
+ val |= FTDMAC020_CH_CSR_PROT2;
+ if (pl08x->pd->memcpy_prot_cache)
+ val |= FTDMAC020_CH_CSR_PROT3;
+ /* We are the kernel, so we are in privileged mode */
+ val |= FTDMAC020_CH_CSR_PROT1;
+
+ writel_relaxed(val, phychan->reg_control);
+ } else {
+ /* printk("this is debug lli[PL080_LLI_CCTL] = %x reg_control = %x %s %s %d\n",
+ lli[PL080_LLI_CCTL],phychan->reg_control,__FILE__,__func__,__LINE__);
+ */
+ /* Bits are just identical */
+ writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);
+ }
+
+ /* Second control word on the PL080s */
+ if (pl08x->vd->pl080s)
+ writel_relaxed(lli[PL080S_LLI_CCTL2],
+ phychan->base + PL080S_CH_CONTROL2);
+
+ /*
+ printk("this is debug ccfg = %x reg_config = %x %s %s %d\n",
+ ccfg,phychan->reg_config,__FILE__,__func__,__LINE__);
+ */
+ writel(ccfg, phychan->reg_config);
+}
+
+/*
+ * Set the initial DMA register values i.e. those for the first LLI
+ * The next LLI pointer and the configuration interrupt bit have
+ * been set when the LLIs were constructed. Poke them into the hardware
+ * and start the transfer.
+ */
+static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_phy_chan *phychan = plchan->phychan;
+ struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
+ struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
+ u32 val;
+
+ list_del(&txd->vd.node);
+
+ plchan->at = txd;
+
+ /* Wait for channel inactive */
+ while (pl08x_phy_channel_busy(phychan))
+ cpu_relax();
+ //printk("this is debug txd->ccfg = %x %s %s %d\n",txd->ccfg,__FILE__,__func__,__LINE__);
+ pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
+
+ /* Enable the DMA channel */
+ /* Do not access config register until channel shows as disabled */
+ //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
+ while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))
+ cpu_relax();
+
+ //printk("this is debug en_chan = %x id = %d %s %s %d\n",readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
+ /* Do not access config register until channel shows as inactive */
+ if (phychan->ftdmac020) {
+ val = readl(phychan->reg_config);
+ while (val & FTDMAC020_CH_CFG_BUSY)
+ val = readl(phychan->reg_config);
+
+ val = readl(phychan->reg_control);
+ while (val & FTDMAC020_CH_CSR_EN)
+ val = readl(phychan->reg_control);
+
+ writel(val | FTDMAC020_CH_CSR_EN,
+ phychan->reg_control);
+ } else {
+ val = readl(phychan->reg_config);
+ while ((val & PL080_CONFIG_ACTIVE) ||
+ (val & PL080_CONFIG_ENABLE))
+ val = readl(phychan->reg_config);
+ //printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__);
+ writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
+#if 0
+ while(!(readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))){
+ printk("this is debug val = %x phychan->reg_config = %x %s %s %d\n",val, phychan->reg_config,__FILE__,__func__,__LINE__);
+ writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
+ }
+#endif
+ }
+ //printk("this is debug reg_config = %x en_chan = %x id = %d %s %s %d\n",readl(phychan->reg_config),
+ // readl(pl08x->base + PL080_EN_CHAN),phychan->id,__FILE__,__func__,__LINE__);
+}
+
+/*
+ * Pause the channel by setting the HALT bit.
+ *
+ * For M->P transfers, pause the DMAC first and then stop the peripheral -
+ * the FIFO can only drain if the peripheral is still requesting data.
+ * (note: this can still timeout if the DMAC FIFO never drains of data.)
+ *
+ * For P->M transfers, disable the peripheral first to stop it filling
+ * the DMAC FIFO, and then pause the DMAC.
+ */
+static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
+{
+ u32 val;
+ int timeout;
+
+ if (ch->ftdmac020) {
+ /* Use the enable bit on the FTDMAC020 */
+ val = readl(ch->reg_control);
+ val &= ~FTDMAC020_CH_CSR_EN;
+ writel(val, ch->reg_control);
+ return;
+ }
+
+ /* Set the HALT bit and wait for the FIFO to drain */
+ val = readl(ch->reg_config);
+ val |= PL080_CONFIG_HALT;
+ writel(val, ch->reg_config);
+
+ /* Wait for channel inactive */
+ for (timeout = 1000; timeout; timeout--) {
+ if (!pl08x_phy_channel_busy(ch))
+ break;
+ udelay(1);
+ }
+ if (pl08x_phy_channel_busy(ch))
+ pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
+}
+
+static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
+{
+ u32 val;
+
+ /* Use the enable bit on the FTDMAC020 */
+ if (ch->ftdmac020) {
+ val = readl(ch->reg_control);
+ val |= FTDMAC020_CH_CSR_EN;
+ writel(val, ch->reg_control);
+ return;
+ }
+
+ /* Clear the HALT bit */
+ val = readl(ch->reg_config);
+ val &= ~PL080_CONFIG_HALT;
+ writel(val, ch->reg_config);
+}
+
+/*
+ * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
+ * clears any pending interrupt status. This should not be used for
+ * an on-going transfer, but as a method of shutting down a channel
+ * (eg, when it's no longer used) or terminating a transfer.
+ */
+static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
+ struct pl08x_phy_chan *ch)
+{
+ u32 val;
+
+ /* The layout for the FTDMAC020 is different */
+ if (ch->ftdmac020) {
+ /* Disable all interrupts */
+ val = readl(ch->reg_config);
+ val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |
+ FTDMAC020_CH_CFG_INT_ERR_MASK |
+ FTDMAC020_CH_CFG_INT_TC_MASK);
+ writel(val, ch->reg_config);
+
+ /* Abort and disable channel */
+ val = readl(ch->reg_control);
+ val &= ~FTDMAC020_CH_CSR_EN;
+ val |= FTDMAC020_CH_CSR_ABT;
+ writel(val, ch->reg_control);
+
+ /* Clear ABT and ERR interrupt flags */
+ writel(BIT(ch->id) | BIT(ch->id + 16),
+ pl08x->base + PL080_ERR_CLEAR);
+ writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
+
+ return;
+ }
+
+ val = readl(ch->reg_config);
+ val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
+ PL080_CONFIG_TC_IRQ_MASK);
+ writel(val, ch->reg_config);
+
+ writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR);
+ writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
+}
+
+static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)
+{
+ u32 val;
+ u32 bytes;
+
+ if (ch->ftdmac020) {
+ bytes = readl(ch->base + FTDMAC020_CH_SIZE);
+
+ val = readl(ch->reg_control);
+ val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;
+ val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;
+ } else if (ch->pl080s) {
+ val = readl(ch->base + PL080S_CH_CONTROL2);
+ bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
+
+ val = readl(ch->reg_control);
+ val &= PL080_CONTROL_SWIDTH_MASK;
+ val >>= PL080_CONTROL_SWIDTH_SHIFT;
+ } else {
+ /* Plain PL08x */
+ val = readl(ch->reg_control);
+ bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
+
+ val &= PL080_CONTROL_SWIDTH_MASK;
+ val >>= PL080_CONTROL_SWIDTH_SHIFT;
+ }
+
+ switch (val) {
+ case PL080_WIDTH_8BIT:
+ break;
+ case PL080_WIDTH_16BIT:
+ bytes *= 2;
+ break;
+ case PL080_WIDTH_32BIT:
+ bytes *= 4;
+ break;
+ }
+ return bytes;
+}
+
+static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)
+{
+ u32 val;
+ u32 bytes;
+
+ if (ch->ftdmac020) {
+ val = llis_va[PL080_LLI_CCTL];
+ bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;
+
+ val = llis_va[PL080_LLI_CCTL];
+ val &= FTDMAC020_LLI_SRC_WIDTH_MSK;
+ val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;
+ } else if (ch->pl080s) {
+ val = llis_va[PL080S_LLI_CCTL2];
+ bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
+
+ val = llis_va[PL080_LLI_CCTL];
+ val &= PL080_CONTROL_SWIDTH_MASK;
+ val >>= PL080_CONTROL_SWIDTH_SHIFT;
+ } else {
+ /* Plain PL08x */
+ val = llis_va[PL080_LLI_CCTL];
+ bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
+
+ val &= PL080_CONTROL_SWIDTH_MASK;
+ val >>= PL080_CONTROL_SWIDTH_SHIFT;
+ }
+
+ switch (val) {
+ case PL080_WIDTH_8BIT:
+ break;
+ case PL080_WIDTH_16BIT:
+ bytes *= 2;
+ break;
+ case PL080_WIDTH_32BIT:
+ bytes *= 4;
+ break;
+ }
+ return bytes;
+}
+
+/* The channel should be paused when calling this */
+static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_driver_data *pl08x = plchan->host;
+ const u32 *llis_va, *llis_va_limit;
+ struct pl08x_phy_chan *ch;
+ dma_addr_t llis_bus;
+ struct pl08x_txd *txd;
+ u32 llis_max_words;
+ size_t bytes;
+ u32 clli;
+
+ ch = plchan->phychan;
+ txd = plchan->at;
+
+ if (!ch || !txd)
+ return 0;
+
+ /*
+ * Follow the LLIs to get the number of remaining
+ * bytes in the currently active transaction.
+ */
+ clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2;
+
+ /* First get the remaining bytes in the active transfer */
+ bytes = get_bytes_in_phy_channel(ch);
+
+ if (!clli)
+ return bytes;
+
+ llis_va = txd->llis_va;
+ llis_bus = txd->llis_bus;
+
+ llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
+ BUG_ON(clli < llis_bus || clli >= llis_bus +
+ sizeof(u32) * llis_max_words);
+
+ /*
+ * Locate the next LLI - as this is an array,
+ * it's simple maths to find.
+ */
+ llis_va += (clli - llis_bus) / sizeof(u32);
+
+ llis_va_limit = llis_va + llis_max_words;
+
+ for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
+ bytes += get_bytes_in_lli(ch, llis_va);
+
+ /*
+ * A LLI pointer going backward terminates the LLI list
+ */
+ if (llis_va[PL080_LLI_LLI] <= clli)
+ break;
+ }
+
+ return bytes;
+}
+
+/*
+ * Allocate a physical channel for a virtual channel
+ *
+ * Try to locate a physical channel to be used for this transfer. If all
+ * are taken return NULL and the requester will have to cope by using
+ * some fallback PIO mode or retrying later.
+ */
+static struct pl08x_phy_chan *
+pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
+ struct pl08x_dma_chan *virt_chan)
+{
+ struct pl08x_phy_chan *ch = NULL;
+ unsigned long flags;
+ int i;
+
+ //printk("this is debug virt_chan->id = %d %s %s %d\n",virt_chan->chan_id,__FILE__,__func__,__LINE__);
+#if 1
+ ch = &pl08x->phy_chans[virt_chan->chan_id];
+
+ spin_lock_irqsave(&ch->lock, flags);
+
+ if (!ch->locked && !ch->serving) {
+ ch->serving = virt_chan;
+ spin_unlock_irqrestore(&ch->lock, flags);
+ return ch;
+ }
+
+ spin_unlock_irqrestore(&ch->lock, flags);
+#endif
+ for (i = 0; i < pl08x->vd->channels; i++) {
+ ch = &pl08x->phy_chans[i];
+
+ spin_lock_irqsave(&ch->lock, flags);
+
+ if (!ch->locked && !ch->serving) {
+ ch->serving = virt_chan;
+ spin_unlock_irqrestore(&ch->lock, flags);
+ break;
+ }
+
+ spin_unlock_irqrestore(&ch->lock, flags);
+ }
+
+ if (i == pl08x->vd->channels) {
+ /* No physical channel available, cope with it */
+ return NULL;
+ }
+
+ return ch;
+}
+
+/* Mark the physical channel as free. Note, this write is atomic. */
+static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
+ struct pl08x_phy_chan *ch)
+{
+ ch->serving = NULL;
+}
+
+/*
+ * Try to allocate a physical channel. When successful, assign it to
+ * this virtual channel, and initiate the next descriptor. The
+ * virtual channel lock must be held at this point.
+ */
+static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_phy_chan *ch;
+
+ ch = pl08x_get_phy_channel(pl08x, plchan);
+ if (!ch) {
+ dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
+ plchan->state = PL08X_CHAN_WAITING;
+ plchan->waiting_at = jiffies;
+ return;
+ }
+
+ dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
+ ch->id, plchan->name);
+
+ plchan->phychan = ch;
+ plchan->state = PL08X_CHAN_RUNNING;
+ pl08x_start_next_txd(plchan);
+}
+
+static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
+ struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_driver_data *pl08x = plchan->host;
+
+ dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
+ ch->id, plchan->name);
+
+ /*
+ * We do this without taking the lock; we're really only concerned
+ * about whether this pointer is NULL or not, and we're guaranteed
+ * that this will only be called when it _already_ is non-NULL.
+ */
+ ch->serving = plchan;
+ plchan->phychan = ch;
+ plchan->state = PL08X_CHAN_RUNNING;
+ pl08x_start_next_txd(plchan);
+}
+
+/*
+ * Free a physical DMA channel, potentially reallocating it to another
+ * virtual channel if we have any pending.
+ */
+static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_dma_chan *p, *next;
+ unsigned long waiting_at;
+ retry:
+ next = NULL;
+ waiting_at = jiffies;
+
+ /*
+ * Find a waiting virtual channel for the next transfer.
+ * To be fair, time when each channel reached waiting state is compared
+ * to select channel that is waiting for the longest time.
+ */
+ list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
+ if (p->state == PL08X_CHAN_WAITING &&
+ p->waiting_at <= waiting_at) {
+ next = p;
+ waiting_at = p->waiting_at;
+ }
+
+ if (!next && pl08x->has_slave) {
+ list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
+ if (p->state == PL08X_CHAN_WAITING &&
+ p->waiting_at <= waiting_at) {
+ next = p;
+ waiting_at = p->waiting_at;
+ }
+ }
+
+ /* Ensure that the physical channel is stopped */
+ pl08x_terminate_phy_chan(pl08x, plchan->phychan);
+
+ if (next) {
+ bool success;
+
+ /*
+ * Eww. We know this isn't going to deadlock
+ * but lockdep probably doesn't.
+ */
+ spin_lock(&next->vc.lock);
+ /* Re-check the state now that we have the lock */
+ success = next->state == PL08X_CHAN_WAITING;
+ if (success)
+ pl08x_phy_reassign_start(plchan->phychan, next);
+ spin_unlock(&next->vc.lock);
+
+ /* If the state changed, try to find another channel */
+ if (!success)
+ goto retry;
+ } else {
+ /* No more jobs, so free up the physical channel */
+ pl08x_put_phy_channel(pl08x, plchan->phychan);
+ }
+
+ plchan->phychan = NULL;
+ plchan->state = PL08X_CHAN_IDLE;
+}
+
+/*
+ * LLI handling
+ */
+
+static inline unsigned int
+pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,
+ u32 cctl,
+ bool source)
+{
+ u32 val;
+
+ if (pl08x->vd->ftdmac020) {
+ if (source)
+ val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
+ FTDMAC020_LLI_SRC_WIDTH_SHIFT;
+ else
+ val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
+ FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ } else {
+ if (source)
+ val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>
+ PL080_CONTROL_SWIDTH_SHIFT;
+ else
+ val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>
+ PL080_CONTROL_DWIDTH_SHIFT;
+ }
+
+ switch (val) {
+ case PL080_WIDTH_8BIT:
+ return 1;
+ case PL080_WIDTH_16BIT:
+ return 2;
+ case PL080_WIDTH_32BIT:
+ return 4;
+ default:
+ break;
+ }
+ BUG();
+ return 0;
+}
+
+static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
+ u32 cctl,
+ u8 srcwidth, u8 dstwidth,
+ size_t tsize)
+{
+ u32 retbits = cctl;
+
+ /*
+ * Remove all src, dst and transfer size bits, then set the
+ * width and size according to the parameters. The bit offsets
+ * are different in the FTDMAC020 so we need to accound for this.
+ */
+ if (pl08x->vd->ftdmac020) {
+ retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
+ retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;
+ retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;
+
+ switch (srcwidth) {
+ case 1:
+ retbits |= PL080_WIDTH_8BIT <<
+ FTDMAC020_LLI_SRC_WIDTH_SHIFT;
+ break;
+ case 2:
+ retbits |= PL080_WIDTH_16BIT <<
+ FTDMAC020_LLI_SRC_WIDTH_SHIFT;
+ break;
+ case 4:
+ retbits |= PL080_WIDTH_32BIT <<
+ FTDMAC020_LLI_SRC_WIDTH_SHIFT;
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ switch (dstwidth) {
+ case 1:
+ retbits |= PL080_WIDTH_8BIT <<
+ FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ case 2:
+ retbits |= PL080_WIDTH_16BIT <<
+ FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ case 4:
+ retbits |= PL080_WIDTH_32BIT <<
+ FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;
+ retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;
+ } else {
+ retbits &= ~PL080_CONTROL_DWIDTH_MASK;
+ retbits &= ~PL080_CONTROL_SWIDTH_MASK;
+ retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
+
+ switch (srcwidth) {
+ case 1:
+ retbits |= PL080_WIDTH_8BIT <<
+ PL080_CONTROL_SWIDTH_SHIFT;
+ break;
+ case 2:
+ retbits |= PL080_WIDTH_16BIT <<
+ PL080_CONTROL_SWIDTH_SHIFT;
+ break;
+ case 4:
+ retbits |= PL080_WIDTH_32BIT <<
+ PL080_CONTROL_SWIDTH_SHIFT;
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ switch (dstwidth) {
+ case 1:
+ retbits |= PL080_WIDTH_8BIT <<
+ PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ case 2:
+ retbits |= PL080_WIDTH_16BIT <<
+ PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ case 4:
+ retbits |= PL080_WIDTH_32BIT <<
+ PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
+ retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
+ }
+
+ return retbits;
+}
+
+struct pl08x_lli_build_data {
+ struct pl08x_txd *txd;
+ struct pl08x_bus_data srcbus;
+ struct pl08x_bus_data dstbus;
+ size_t remainder;
+ u32 lli_bus;
+};
+
+/*
+ * Autoselect a master bus to use for the transfer. Slave will be the chosen as
+ * victim in case src & dest are not similarly aligned. i.e. If after aligning
+ * masters address with width requirements of transfer (by sending few byte by
+ * byte data), slave is still not aligned, then its width will be reduced to
+ * BYTE.
+ * - prefers the destination bus if both available
+ * - prefers bus with fixed address (i.e. peripheral)
+ */
+static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,
+ struct pl08x_lli_build_data *bd,
+ struct pl08x_bus_data **mbus,
+ struct pl08x_bus_data **sbus,
+ u32 cctl)
+{
+ bool dst_incr;
+ bool src_incr;
+
+ /*
+ * The FTDMAC020 only supports memory-to-memory transfer, so
+ * source and destination always increase.
+ */
+ if (pl08x->vd->ftdmac020) {
+ dst_incr = true;
+ src_incr = true;
+ } else {
+ dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);
+ src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);
+ }
+
+ /*
+ * If either bus is not advancing, i.e. it is a peripheral, that
+ * one becomes master
+ */
+ if (!dst_incr) {
+ *mbus = &bd->dstbus;
+ *sbus = &bd->srcbus;
+ } else if (!src_incr) {
+ *mbus = &bd->srcbus;
+ *sbus = &bd->dstbus;
+ } else {
+ if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
+ *mbus = &bd->dstbus;
+ *sbus = &bd->srcbus;
+ } else {
+ *mbus = &bd->srcbus;
+ *sbus = &bd->dstbus;
+ }
+ }
+}
+
+/*
+ * Fills in one LLI for a certain transfer descriptor and advance the counter
+ */
+static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
+ struct pl08x_lli_build_data *bd,
+ int num_llis, int len, u32 cctl, u32 cctl2)
+{
+ u32 offset = num_llis * pl08x->lli_words;
+ u32 *llis_va = bd->txd->llis_va + offset;
+ dma_addr_t llis_bus = bd->txd->llis_bus;
+
+ BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
+
+ /*
+ printk("this is debug num_llis = %x lli_words = %x %s %s %d\n",
+ num_llis,pl08x->lli_words,__FILE__,__func__,__LINE__);
+ */
+ /* Advance the offset to next LLI. */
+ offset += pl08x->lli_words;
+
+ llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
+ llis_va[PL080_LLI_DST] = bd->dstbus.addr;
+ llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
+ llis_va[PL080_LLI_LLI] |= bd->lli_bus;
+ llis_va[PL080_LLI_CCTL] = cctl;
+ if (pl08x->vd->pl080s)
+ llis_va[PL080S_LLI_CCTL2] = cctl2;
+
+ if (pl08x->vd->ftdmac020) {
+ /* FIXME: only memcpy so far so both increase */
+ bd->srcbus.addr += len;
+ bd->dstbus.addr += len;
+ } else {
+ if (cctl & PL080_CONTROL_SRC_INCR)
+ bd->srcbus.addr += len;
+ if (cctl & PL080_CONTROL_DST_INCR)
+ bd->dstbus.addr += len;
+ }
+
+ BUG_ON(bd->remainder < len);
+
+ bd->remainder -= len;
+}
+
+static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
+ struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
+ int num_llis, size_t *total_bytes)
+{
+ *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len);
+ pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
+ (*total_bytes) += len;
+}
+
+#if 1
+static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
+ const u32 *llis_va, int num_llis)
+{
+ int i;
+
+ if (pl08x->vd->pl080s) {
+ dev_vdbg(&pl08x->adev->dev,
+ "%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
+ "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
+ for (i = 0; i < num_llis; i++) {
+ dev_vdbg(&pl08x->adev->dev,
+ "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
+ i, llis_va, llis_va[PL080_LLI_SRC],
+ llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
+ llis_va[PL080_LLI_CCTL],
+ llis_va[PL080S_LLI_CCTL2]);
+ llis_va += pl08x->lli_words;
+ }
+ } else {
+ //dev_vdbg(&pl08x->adev->dev,
+ dev_info(&pl08x->adev->dev,
+ "%-3s %-9s %-10s %-10s %-10s %s\n",
+ "lli", "", "csrc", "cdst", "clli", "cctl");
+ for (i = 0; i < num_llis; i++) {
+ //dev_vdbg(&pl08x->adev->dev,
+ dev_info(&pl08x->adev->dev,
+ "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+ i, llis_va, llis_va[PL080_LLI_SRC],
+ llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
+ llis_va[PL080_LLI_CCTL]);
+ llis_va += pl08x->lli_words;
+ }
+ }
+}
+#else
+static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
+ const u32 *llis_va, int num_llis) {}
+#endif
+
+extern u64 dw_virt_to_phys(void *vaddr);
+/*
+ * This fills in the table of LLIs for the transfer descriptor
+ * Note that we assume we never have to change the burst sizes
+ * Return 0 for error
+ */
+static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
+ struct pl08x_txd *txd)
+{
+ struct pl08x_bus_data *mbus, *sbus;
+ struct pl08x_lli_build_data bd;
+ int num_llis = 0;
+ u32 cctl, early_bytes = 0;
+ size_t max_bytes_per_lli, total_bytes;
+ u32 *llis_va, *last_lli;
+ struct pl08x_sg *dsg;
+
+ txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
+ if (!txd->llis_va) {
+ dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
+ return 0;
+ }
+
+ /*
+ printk("this is debug txd->llis_bus = %llx pl08x->lli_buses = %x llis_va = %llx %s %s %d\n",
+ txd->llis_bus,pl08x->lli_buses,dw_virt_to_phys(txd->llis_va),__FILE__,__func__,__LINE__);
+ */
+ bd.txd = txd;
+ bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
+ cctl = txd->cctl;
+
+ /* Find maximum width of the source bus */
+ bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true);
+
+ /* Find maximum width of the destination bus */
+ bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false);
+
+ list_for_each_entry(dsg, &txd->dsg_list, node) {
+ total_bytes = 0;
+ cctl = txd->cctl;
+
+ bd.srcbus.addr = dsg->src_addr;
+ bd.dstbus.addr = dsg->dst_addr;
+ bd.remainder = dsg->len;
+ bd.srcbus.buswidth = bd.srcbus.maxwidth;
+ bd.dstbus.buswidth = bd.dstbus.maxwidth;
+
+ pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl);
+
+ dev_vdbg(&pl08x->adev->dev,
+ "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
+ (u64)bd.srcbus.addr,
+ cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
+ bd.srcbus.buswidth,
+ (u64)bd.dstbus.addr,
+ cctl & PL080_CONTROL_DST_INCR ? "+" : "",
+ bd.dstbus.buswidth,
+ bd.remainder);
+ dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
+ mbus == &bd.srcbus ? "src" : "dst",
+ sbus == &bd.srcbus ? "src" : "dst");
+
+ /*
+ * Zero length is only allowed if all these requirements are
+ * met:
+ * - flow controller is peripheral.
+ * - src.addr is aligned to src.width
+ * - dst.addr is aligned to dst.width
+ *
+ * sg_len == 1 should be true, as there can be two cases here:
+ *
+ * - Memory addresses are contiguous and are not scattered.
+ * Here, Only one sg will be passed by user driver, with
+ * memory address and zero length. We pass this to controller
+ * and after the transfer it will receive the last burst
+ * request from peripheral and so transfer finishes.
+ *
+ * - Memory addresses are scattered and are not contiguous.
+ * Here, Obviously as DMA controller doesn't know when a lli's
+ * transfer gets over, it can't load next lli. So in this
+ * case, there has to be an assumption that only one lli is
+ * supported. Thus, we can't have scattered addresses.
+ */
+ if (!bd.remainder) {
+ u32 fc;
+
+ /* FTDMAC020 only does memory-to-memory */
+ if (pl08x->vd->ftdmac020)
+ fc = PL080_FLOW_MEM2MEM;
+ else
+ fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
+ PL080_CONFIG_FLOW_CONTROL_SHIFT;
+ if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
+ (fc <= PL080_FLOW_SRC2DST_SRC))) {
+ dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
+ __func__);
+ return 0;
+ }
+
+ if (!IS_BUS_ALIGNED(&bd.srcbus) ||
+ !IS_BUS_ALIGNED(&bd.dstbus)) {
+ dev_err(&pl08x->adev->dev,
+ "%s src & dst address must be aligned to src"
+ " & dst width if peripheral is flow controller",
+ __func__);
+ return 0;
+ }
+
+ cctl = pl08x_lli_control_bits(pl08x, cctl,
+ bd.srcbus.buswidth, bd.dstbus.buswidth,
+ 0);
+ pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
+ 0, cctl, 0);
+ break;
+ }
+
+ /*
+ * Send byte by byte for following cases
+ * - Less than a bus width available
+ * - until master bus is aligned
+ */
+ if (bd.remainder < mbus->buswidth)
+ early_bytes = bd.remainder;
+ else if (!IS_BUS_ALIGNED(mbus)) {
+ early_bytes = mbus->buswidth -
+ (mbus->addr & (mbus->buswidth - 1));
+ if ((bd.remainder - early_bytes) < mbus->buswidth)
+ early_bytes = bd.remainder;
+ }
+
+ if (early_bytes) {
+ dev_vdbg(&pl08x->adev->dev,
+ "%s byte width LLIs (remain 0x%08zx)\n",
+ __func__, bd.remainder);
+ prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
+ num_llis++, &total_bytes);
+ }
+
+ if (bd.remainder) {
+ /*
+ * Master now aligned
+ * - if slave is not then we must set its width down
+ */
+ if (!IS_BUS_ALIGNED(sbus)) {
+ dev_dbg(&pl08x->adev->dev,
+ "%s set down bus width to one byte\n",
+ __func__);
+
+ sbus->buswidth = 1;
+ }
+
+ /*
+ * Bytes transferred = tsize * src width, not
+ * MIN(buswidths)
+ */
+ max_bytes_per_lli = bd.srcbus.buswidth *
+ pl08x->vd->max_transfer_size;
+ dev_vdbg(&pl08x->adev->dev,
+ "%s max bytes per lli = %zu\n",
+ __func__, max_bytes_per_lli);
+
+ /*
+ * Make largest possible LLIs until less than one bus
+ * width left
+ */
+ while (bd.remainder > (mbus->buswidth - 1)) {
+ size_t lli_len, tsize, width;
+
+ /*
+ * If enough left try to send max possible,
+ * otherwise try to send the remainder
+ */
+ lli_len = min(bd.remainder, max_bytes_per_lli);
+
+ /*
+ * Check against maximum bus alignment:
+ * Calculate actual transfer size in relation to
+ * bus width an get a maximum remainder of the
+ * highest bus width - 1
+ */
+ width = max(mbus->buswidth, sbus->buswidth);
+ lli_len = (lli_len / width) * width;
+ tsize = lli_len / bd.srcbus.buswidth;
+
+ dev_vdbg(&pl08x->adev->dev,
+ "%s fill lli with single lli chunk of "
+ "size 0x%08zx (remainder 0x%08zx)\n",
+ __func__, lli_len, bd.remainder);
+
+ cctl = pl08x_lli_control_bits(pl08x, cctl,
+ bd.srcbus.buswidth, bd.dstbus.buswidth,
+ tsize);
+ pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
+ lli_len, cctl, tsize);
+ total_bytes += lli_len;
+ }
+
+ /*
+ * Send any odd bytes
+ */
+ if (bd.remainder) {
+ dev_vdbg(&pl08x->adev->dev,
+ "%s align with boundary, send odd bytes (remain %zu)\n",
+ __func__, bd.remainder);
+ prep_byte_width_lli(pl08x, &bd, &cctl,
+ bd.remainder, num_llis++, &total_bytes);
+ }
+ }
+
+ if (total_bytes != dsg->len) {
+ dev_err(&pl08x->adev->dev,
+ "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
+ __func__, total_bytes, dsg->len);
+ return 0;
+ }
+
+ if (num_llis >= MAX_NUM_TSFR_LLIS) {
+ dev_err(&pl08x->adev->dev,
+ "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
+ __func__, MAX_NUM_TSFR_LLIS);
+ return 0;
+ }
+ }
+
+ llis_va = txd->llis_va;
+ last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
+
+ if (txd->cyclic) {
+ /* Link back to the first LLI. */
+ last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
+ } else {
+ /* The final LLI terminates the LLI. */
+ last_lli[PL080_LLI_LLI] = 0;
+ /* The final LLI element shall also fire an interrupt. */
+ if (pl08x->vd->ftdmac020)
+ last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;
+ else
+ last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
+ }
+
+ pl08x_dump_lli(pl08x, llis_va, num_llis);
+
+ return num_llis;
+}
+
+static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
+ struct pl08x_txd *txd)
+{
+ struct pl08x_sg *dsg, *_dsg;
+
+ if (txd->llis_va)
+ dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
+
+ list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
+ list_del(&dsg->node);
+ kfree(dsg);
+ }
+
+ kfree(txd);
+}
+
+static void pl08x_desc_free(struct virt_dma_desc *vd)
+{
+ struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
+
+ dma_descriptor_unmap(&vd->tx);
+ if (!txd->done)
+ pl08x_release_mux(plchan);
+
+ pl08x_free_txd(plchan->host, txd);
+}
+
+static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
+ struct pl08x_dma_chan *plchan)
+{
+ LIST_HEAD(head);
+
+ vchan_get_all_descriptors(&plchan->vc, &head);
+ vchan_dma_desc_free_list(&plchan->vc, &head);
+}
+
+/*
+ * The DMA ENGINE API
+ */
+static void pl08x_free_chan_resources(struct dma_chan *chan)
+{
+ /* Ensure all queued descriptors are freed */
+ vchan_free_chan_resources(to_virt_chan(chan));
+}
+
+static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
+ struct dma_chan *chan, unsigned long flags)
+{
+ struct dma_async_tx_descriptor *retval = NULL;
+
+ return retval;
+}
+
+/*
+ * Code accessing dma_async_is_complete() in a tight loop may give problems.
+ * If slaves are relying on interrupts to signal completion this function
+ * must not be called with interrupts disabled.
+ */
+static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
+ dma_cookie_t cookie, struct dma_tx_state *txstate)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+ enum dma_status ret;
+ size_t bytes = 0;
+
+ ret = dma_cookie_status(chan, cookie, txstate);
+ if (ret == DMA_COMPLETE)
+ return ret;
+
+ /*
+ * There's no point calculating the residue if there's
+ * no txstate to store the value.
+ */
+ if (!txstate) {
+ if (plchan->state == PL08X_CHAN_PAUSED)
+ ret = DMA_PAUSED;
+ return ret;
+ }
+
+ spin_lock_irqsave(&plchan->vc.lock, flags);
+ ret = dma_cookie_status(chan, cookie, txstate);
+ if (ret != DMA_COMPLETE) {
+ vd = vchan_find_desc(&plchan->vc, cookie);
+ if (vd) {
+ /* On the issued list, so hasn't been processed yet */
+ struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
+ struct pl08x_sg *dsg;
+
+ list_for_each_entry(dsg, &txd->dsg_list, node)
+ bytes += dsg->len;
+ } else {
+ bytes = pl08x_getbytes_chan(plchan);
+ }
+ }
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+
+ /*
+ * This cookie not complete yet
+ * Get number of bytes left in the active transactions and queue
+ */
+ dma_set_residue(txstate, bytes);
+
+ if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
+ ret = DMA_PAUSED;
+
+ /* Whether waiting or running, we're in progress */
+ return ret;
+}
+
+/* PrimeCell DMA extension */
+struct burst_table {
+ u32 burstwords;
+ u32 reg;
+};
+
+static const struct burst_table burst_sizes[] = {
+ {
+ .burstwords = 256,
+ .reg = PL080_BSIZE_256,
+ },
+ {
+ .burstwords = 128,
+ .reg = PL080_BSIZE_128,
+ },
+ {
+ .burstwords = 64,
+ .reg = PL080_BSIZE_64,
+ },
+ {
+ .burstwords = 32,
+ .reg = PL080_BSIZE_32,
+ },
+ {
+ .burstwords = 16,
+ .reg = PL080_BSIZE_16,
+ },
+ {
+ .burstwords = 8,
+ .reg = PL080_BSIZE_8,
+ },
+ {
+ .burstwords = 4,
+ .reg = PL080_BSIZE_4,
+ },
+ {
+ .burstwords = 0,
+ .reg = PL080_BSIZE_1,
+ },
+};
+
+/*
+ * Given the source and destination available bus masks, select which
+ * will be routed to each port. We try to have source and destination
+ * on separate ports, but always respect the allowable settings.
+ */
+static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)
+{
+ u32 cctl = 0;
+ u32 dst_ahb2;
+ u32 src_ahb2;
+
+ /* The FTDMAC020 use different bits to indicate src/dst bus */
+ if (ftdmac020) {
+ dst_ahb2 = FTDMAC020_LLI_DST_SEL;
+ src_ahb2 = FTDMAC020_LLI_SRC_SEL;
+ } else {
+ dst_ahb2 = PL080_CONTROL_DST_AHB2;
+ src_ahb2 = PL080_CONTROL_SRC_AHB2;
+ }
+
+ if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
+ cctl |= dst_ahb2;
+ if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
+ cctl |= src_ahb2;
+
+ //printk("this is debug dst = %x src = %x cctl = %x %s %s %d\n",dst,src,cctl,__FILE__,__func__,__LINE__);
+
+ return cctl;
+}
+
+static u32 pl08x_cctl(u32 cctl)
+{
+ cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
+ PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
+ PL080_CONTROL_PROT_MASK);
+
+ /* Access the cell in privileged mode, non-bufferable, non-cacheable */
+ return cctl | PL080_CONTROL_PROT_SYS;
+}
+
+static u32 pl08x_width(enum dma_slave_buswidth width)
+{
+ switch (width) {
+ case DMA_SLAVE_BUSWIDTH_1_BYTE:
+ return PL080_WIDTH_8BIT;
+ case DMA_SLAVE_BUSWIDTH_2_BYTES:
+ return PL080_WIDTH_16BIT;
+ case DMA_SLAVE_BUSWIDTH_4_BYTES:
+ return PL080_WIDTH_32BIT;
+ default:
+ return ~0;
+ }
+}
+
+static u32 pl08x_burst(u32 maxburst)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
+ if (burst_sizes[i].burstwords <= maxburst)
+ break;
+
+ return burst_sizes[i].reg;
+}
+
+static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
+ enum dma_slave_buswidth addr_width, u32 maxburst)
+{
+ u32 width, burst, cctl = 0;
+
+ width = pl08x_width(addr_width);
+ if (width == ~0)
+ return ~0;
+
+ cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
+ cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
+
+ /*
+ * If this channel will only request single transfers, set this
+ * down to ONE element. Also select one element if no maxburst
+ * is specified.
+ */
+ if (plchan->cd->single)
+ maxburst = 1;
+
+ burst = pl08x_burst(maxburst);
+ cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
+ cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
+
+ return pl08x_cctl(cctl);
+}
+
+/*
+ * Slave transactions callback to the slave device to allow
+ * synchronization of slave DMA signals with the DMAC enable
+ */
+static void pl08x_issue_pending(struct dma_chan *chan)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ unsigned long flags;
+
+ plchan->chan_id = chan->chan_id;
+
+ spin_lock_irqsave(&plchan->vc.lock, flags);
+ if (vchan_issue_pending(&plchan->vc)) {
+ if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
+ pl08x_phy_alloc_and_start(plchan);
+ }
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+}
+
+static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
+{
+ struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
+
+ if (txd)
+ INIT_LIST_HEAD(&txd->dsg_list);
+ return txd;
+}
+
+static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)
+{
+ u32 cctl = 0;
+
+ /* Conjure cctl */
+ switch (pl08x->pd->memcpy_burst_size) {
+ default:
+ dev_err(&pl08x->adev->dev,
+ "illegal burst size for memcpy, set to 1\n");
+ fallthrough;
+ case PL08X_BURST_SZ_1:
+ cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_4:
+ cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_8:
+ cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_16:
+ cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_32:
+ cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_64:
+ cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_128:
+ cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ case PL08X_BURST_SZ_256:
+ cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
+ PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
+ break;
+ }
+
+ switch (pl08x->pd->memcpy_bus_width) {
+ default:
+ dev_err(&pl08x->adev->dev,
+ "illegal bus width for memcpy, set to 8 bits\n");
+ fallthrough;
+ case PL08X_BUS_WIDTH_8_BITS:
+ cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
+ PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ case PL08X_BUS_WIDTH_16_BITS:
+ cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
+ PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ case PL08X_BUS_WIDTH_32_BITS:
+ cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
+ PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
+ break;
+ }
+
+ /* Protection flags */
+ if (pl08x->pd->memcpy_prot_buff)
+ cctl |= PL080_CONTROL_PROT_BUFF;
+ if (pl08x->pd->memcpy_prot_cache)
+ cctl |= PL080_CONTROL_PROT_CACHE;
+
+ /* We are the kernel, so we are in privileged mode */
+ cctl |= PL080_CONTROL_PROT_SYS;
+
+ /* Both to be incremented or the code will break */
+ cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
+
+ if (pl08x->vd->dualmaster)
+ cctl |= pl08x_select_bus(false,
+ pl08x->mem_buses,
+ pl08x->mem_buses);
+
+ return cctl;
+}
+
+static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)
+{
+ u32 cctl = 0;
+
+ /* Conjure cctl */
+ switch (pl08x->pd->memcpy_bus_width) {
+ default:
+ dev_err(&pl08x->adev->dev,
+ "illegal bus width for memcpy, set to 8 bits\n");
+ fallthrough;
+ case PL08X_BUS_WIDTH_8_BITS:
+ cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
+ PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ case PL08X_BUS_WIDTH_16_BITS:
+ cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
+ PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ case PL08X_BUS_WIDTH_32_BITS:
+ cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
+ PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
+ break;
+ }
+
+ /*
+ * By default mask the TC IRQ on all LLIs, it will be unmasked on
+ * the last LLI item by other code.
+ */
+ cctl |= FTDMAC020_LLI_TC_MSK;
+
+ /*
+ * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL
+ * and FTDMAC020_LLI_DSTAD_CTL as zero
+ */
+ if (pl08x->vd->dualmaster)
+ cctl |= pl08x_select_bus(true,
+ pl08x->mem_buses,
+ pl08x->mem_buses);
+
+ return cctl;
+}
+
+/*
+ * Initialize a descriptor to be used by memcpy submit
+ */
+static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
+ struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+ size_t len, unsigned long flags)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_txd *txd;
+ struct pl08x_sg *dsg;
+ int ret;
+ //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
+ txd = pl08x_get_txd(plchan);
+ if (!txd) {
+ dev_err(&pl08x->adev->dev,
+ "%s no memory for descriptor\n", __func__);
+ return NULL;
+ }
+ //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
+ dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
+ if (!dsg) {
+ pl08x_free_txd(pl08x, txd);
+ return NULL;
+ }
+ list_add_tail(&dsg->node, &txd->dsg_list);
+ //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
+ dsg->src_addr = src;
+ dsg->dst_addr = dest;
+ dsg->len = len;
+ if (pl08x->vd->ftdmac020) {
+ /* Writing CCFG zero ENABLES all interrupts */
+ txd->ccfg = 0;
+ txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);
+ } else {
+ txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
+ PL080_CONFIG_TC_IRQ_MASK |
+ PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+ txd->cctl = pl08x_memcpy_cctl(pl08x);
+ }
+ //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
+ ret = pl08x_fill_llis_for_desc(plchan->host, txd);
+ if (!ret) {
+ pl08x_free_txd(pl08x, txd);
+ return NULL;
+ }
+ //printk("this is debug for lophyel %s %s %d\n",__FILE__,__func__,__LINE__);
+ return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
+}
+
+static struct pl08x_txd *pl08x_init_txd(
+ struct dma_chan *chan,
+ enum dma_transfer_direction direction,
+ dma_addr_t *slave_addr)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_txd *txd;
+ enum dma_slave_buswidth addr_width;
+ int ret, tmp;
+ u8 src_buses, dst_buses;
+ u32 maxburst, cctl;
+
+ txd = pl08x_get_txd(plchan);
+ if (!txd) {
+ dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
+ return NULL;
+ }
+
+ /*
+ * Set up addresses, the PrimeCell configured address
+ * will take precedence since this may configure the
+ * channel target address dynamically at runtime.
+ */
+ if (direction == DMA_MEM_TO_DEV) {
+ //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ cctl = PL080_CONTROL_SRC_INCR;
+ *slave_addr = plchan->cfg.dst_addr;
+ addr_width = plchan->cfg.dst_addr_width;
+ maxburst = plchan->cfg.dst_maxburst;
+ src_buses = pl08x->mem_buses;
+ dst_buses = plchan->cd->periph_buses;
+ } else if (direction == DMA_DEV_TO_MEM) {
+ //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ cctl = PL080_CONTROL_DST_INCR;
+ *slave_addr = plchan->cfg.src_addr;
+ addr_width = plchan->cfg.src_addr_width;
+ maxburst = plchan->cfg.src_maxburst;
+ src_buses = plchan->cd->periph_buses;
+ dst_buses = pl08x->mem_buses;
+ } else {
+ pl08x_free_txd(pl08x, txd);
+ dev_err(&pl08x->adev->dev,
+ "%s direction unsupported\n", __func__);
+ return NULL;
+ }
+
+ cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
+ if (cctl == ~0) {
+ pl08x_free_txd(pl08x, txd);
+ dev_err(&pl08x->adev->dev,
+ "DMA slave configuration botched?\n");
+ return NULL;
+ }
+
+ txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses);
+
+ if (plchan->cfg.device_fc)
+ tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
+ PL080_FLOW_PER2MEM_PER;
+ else
+ tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
+ PL080_FLOW_PER2MEM;
+
+ txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
+ PL080_CONFIG_TC_IRQ_MASK |
+ tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
+ //printk("this is debug cctl = %x ccfg = %x %s %s %d\n",txd->cctl,txd->ccfg,__FILE__,__func__,__LINE__);
+
+ ret = pl08x_request_mux(plchan);
+ if (ret < 0) {
+ pl08x_free_txd(pl08x, txd);
+ dev_dbg(&pl08x->adev->dev,
+ "unable to mux for transfer on %s due to platform restrictions\n",
+ plchan->name);
+ return NULL;
+ }
+
+ dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
+ plchan->signal, plchan->name);
+
+ /* Assign the flow control signal to this channel */
+ if (direction == DMA_MEM_TO_DEV)
+ txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
+ else
+ txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
+
+ return txd;
+}
+
+static int pl08x_tx_add_sg(struct pl08x_txd *txd,
+ enum dma_transfer_direction direction,
+ dma_addr_t slave_addr,
+ dma_addr_t buf_addr,
+ unsigned int len)
+{
+ struct pl08x_sg *dsg;
+
+ dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
+ if (!dsg)
+ return -ENOMEM;
+
+ list_add_tail(&dsg->node, &txd->dsg_list);
+
+ dsg->len = len;
+ if (direction == DMA_MEM_TO_DEV) {
+ dsg->src_addr = buf_addr;
+ dsg->dst_addr = slave_addr;
+ } else {
+ dsg->src_addr = slave_addr;
+ dsg->dst_addr = buf_addr;
+ }
+
+ return 0;
+}
+
+static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
+ struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sg_len, enum dma_transfer_direction direction,
+ unsigned long flags, void *context)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_txd *txd;
+ struct scatterlist *sg;
+ int ret, tmp;
+ dma_addr_t slave_addr;
+
+ dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
+ __func__, sg_dma_len(sgl), plchan->name);
+
+ txd = pl08x_init_txd(chan, direction, &slave_addr);
+ if (!txd)
+ return NULL;
+
+ for_each_sg(sgl, sg, sg_len, tmp) {
+ ret = pl08x_tx_add_sg(txd, direction, slave_addr,
+ sg_dma_address(sg),
+ sg_dma_len(sg));
+ /*
+ printk("this is debug direction = %x slave_addr = %x addr = %x len = %x %s %s %d\n",
+ direction,slave_addr,sg_dma_address(sg),sg_dma_len(sg),
+ __FILE__,__func__,__LINE__);
+ */
+ if (ret) {
+ pl08x_release_mux(plchan);
+ pl08x_free_txd(pl08x, txd);
+ dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
+ __func__);
+ return NULL;
+ }
+ }
+
+ ret = pl08x_fill_llis_for_desc(plchan->host, txd);
+ if (!ret) {
+ pl08x_release_mux(plchan);
+ pl08x_free_txd(pl08x, txd);
+ return NULL;
+ }
+
+ return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
+ struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+ size_t period_len, enum dma_transfer_direction direction,
+ unsigned long flags)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+ struct pl08x_txd *txd;
+ int ret, tmp;
+ dma_addr_t slave_addr;
+
+ dev_dbg(&pl08x->adev->dev,
+ "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
+ __func__, period_len, buf_len,
+ direction == DMA_MEM_TO_DEV ? "to" : "from",
+ plchan->name);
+
+ txd = pl08x_init_txd(chan, direction, &slave_addr);
+ if (!txd)
+ return NULL;
+
+ txd->cyclic = true;
+ txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
+ for (tmp = 0; tmp < buf_len; tmp += period_len) {
+ ret = pl08x_tx_add_sg(txd, direction, slave_addr,
+ buf_addr + tmp, period_len);
+ if (ret) {
+ pl08x_release_mux(plchan);
+ pl08x_free_txd(pl08x, txd);
+ return NULL;
+ }
+ }
+
+ ret = pl08x_fill_llis_for_desc(plchan->host, txd);
+ if (!ret) {
+ pl08x_release_mux(plchan);
+ pl08x_free_txd(pl08x, txd);
+ return NULL;
+ }
+
+ return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
+}
+
+static int pl08x_config(struct dma_chan *chan,
+ struct dma_slave_config *config)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+
+ if (!plchan->slave)
+ return -EINVAL;
+
+ /* Reject definitely invalid configurations */
+ if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
+ config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
+ return -EINVAL;
+
+ if (config->device_fc && pl08x->vd->pl080s) {
+ dev_err(&pl08x->adev->dev,
+ "%s: PL080S does not support peripheral flow control\n",
+ __func__);
+ return -EINVAL;
+ }
+
+ /*
+ printk("this is debug chan = %x chan-id = %d plchan = %x plchan->signal = %d %s %s %d\n",
+ chan,chan->chan_id,plchan,plchan->signal,__FILE__,__func__,__LINE__);
+ */
+ plchan->cfg = *config;
+
+ return 0;
+}
+
+static int pl08x_terminate_all(struct dma_chan *chan)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ struct pl08x_driver_data *pl08x = plchan->host;
+ unsigned long flags;
+
+ spin_lock_irqsave(&plchan->vc.lock, flags);
+ if (!plchan->phychan && !plchan->at) {
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+ return 0;
+ }
+
+ plchan->state = PL08X_CHAN_IDLE;
+
+ if (plchan->phychan) {
+ /*
+ * Mark physical channel as free and free any slave
+ * signal
+ */
+ pl08x_phy_free(plchan);
+ }
+ /* Dequeue jobs and free LLIs */
+ if (plchan->at) {
+ vchan_terminate_vdesc(&plchan->at->vd);
+ plchan->at = NULL;
+ }
+ /* Dequeue jobs not yet fired as well */
+ pl08x_free_txd_list(pl08x, plchan);
+
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+
+ return 0;
+}
+
+static void pl08x_synchronize(struct dma_chan *chan)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+
+ printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
+ vchan_synchronize(&plchan->vc);
+}
+
+static int pl08x_pause(struct dma_chan *chan)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ unsigned long flags;
+
+ /*
+ * Anything succeeds on channels with no physical allocation and
+ * no queued transfers.
+ */
+ printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
+ spin_lock_irqsave(&plchan->vc.lock, flags);
+ if (!plchan->phychan && !plchan->at) {
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+ return 0;
+ }
+
+ pl08x_pause_phy_chan(plchan->phychan);
+ plchan->state = PL08X_CHAN_PAUSED;
+
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+
+ return 0;
+}
+
+static int pl08x_resume(struct dma_chan *chan)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+ unsigned long flags;
+
+ /*
+ * Anything succeeds on channels with no physical allocation and
+ * no queued transfers.
+ */
+ printk("this is debug %s %s %d \n",__FILE__,__func__,__LINE__);
+ spin_lock_irqsave(&plchan->vc.lock, flags);
+ if (!plchan->phychan && !plchan->at) {
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+ return 0;
+ }
+
+ pl08x_resume_phy_chan(plchan->phychan);
+ plchan->state = PL08X_CHAN_RUNNING;
+
+ spin_unlock_irqrestore(&plchan->vc.lock, flags);
+
+ return 0;
+}
+#if 0
+bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
+{
+ struct pl08x_dma_chan *plchan;
+ char *name = chan_id;
+
+ /* Reject channels for devices not bound to this driver */
+ if (chan->device->dev->driver != &pl08x_amba_driver.drv)
+ return false;
+
+ plchan = to_pl08x_chan(chan);
+
+ /* Check that the channel is not taken! */
+ if (!strcmp(plchan->name, name))
+ return true;
+
+ return false;
+}
+EXPORT_SYMBOL_GPL(pl08x_filter_id);
+#endif
+static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
+{
+ struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+
+ return plchan->cd == chan_id;
+}
+
+/*
+ * Just check that the device is there and active
+ * TODO: turn this bit on/off depending on the number of physical channels
+ * actually used, if it is zero... well shut it off. That will save some
+ * power. Cut the clock at the same time.
+ */
+static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
+{
+ /* The Nomadik variant does not have the config register */
+ if (pl08x->vd->nomadik)
+ return;
+ /* The FTDMAC020 variant does this in another register */
+ if (pl08x->vd->ftdmac020) {
+ writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR);
+ return;
+ }
+ writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
+}
+
+static irqreturn_t pl08x_irq(int irq, void *dev)
+{
+ struct pl08x_driver_data *pl08x = dev;
+ u32 mask = 0, err, tc, i;
+
+ /* check & clear - ERR & TC interrupts */
+ err = readl(pl08x->base + PL080_ERR_STATUS);
+ if (err) {
+ dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
+ __func__, err);
+ writel(err, pl08x->base + PL080_ERR_CLEAR);
+ }
+ tc = readl(pl08x->base + PL080_TC_STATUS);
+ if (tc) {
+ writel(tc, pl08x->base + PL080_TC_CLEAR);
+ }
+
+ if (!err && !tc) {
+ return IRQ_NONE;
+ }
+
+ for (i = 0; i < pl08x->vd->channels; i++) {
+ if ((BIT(i) & err) || (BIT(i) & tc)) {
+ /* Locate physical channel */
+ struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
+ struct pl08x_dma_chan *plchan = phychan->serving;
+ struct pl08x_txd *tx;
+
+ if (!plchan) {
+ dev_err(&pl08x->adev->dev,
+ "%s Error TC interrupt on unused channel: 0x%08x\n",
+ __func__, i);
+ continue;
+ }
+
+ spin_lock(&plchan->vc.lock);
+ tx = plchan->at;
+ if (tx && tx->cyclic) {
+ vchan_cyclic_callback(&tx->vd);
+ } else if (tx) {
+ plchan->at = NULL;
+ /*
+ * This descriptor is done, release its mux
+ * reservation.
+ */
+ pl08x_release_mux(plchan);
+ tx->done = true;
+ vchan_cookie_complete(&tx->vd);
+
+ /*
+ * And start the next descriptor (if any),
+ * otherwise free this channel.
+ */
+ if (vchan_next_desc(&plchan->vc))
+ pl08x_start_next_txd(plchan);
+ else
+ pl08x_phy_free(plchan);
+ }
+ spin_unlock(&plchan->vc.lock);
+
+ mask |= BIT(i);
+ }
+ }
+
+ return mask ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
+{
+ chan->slave = true;
+ chan->name = chan->cd->bus_id;
+ chan->cfg.src_addr = chan->cd->addr;
+ chan->cfg.dst_addr = chan->cd->addr;
+}
+
+/*
+ * Initialise the DMAC memcpy/slave channels.
+ * Make a local wrapper to hold required data
+ */
+static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
+ struct dma_device *dmadev, unsigned int channels, bool slave)
+{
+ struct pl08x_dma_chan *chan;
+ int i;
+
+ INIT_LIST_HEAD(&dmadev->channels);
+
+ /*
+ * Register as many many memcpy as we have physical channels,
+ * we won't always be able to use all but the code will have
+ * to cope with that situation.
+ */
+ for (i = 0; i < channels; i++) {
+ chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+ if (!chan)
+ return -ENOMEM;
+
+ chan->host = pl08x;
+ chan->state = PL08X_CHAN_IDLE;
+ chan->signal = -1;
+
+ if (slave) {
+ chan->cd = &pl08x->pd->slave_channels[i];
+ /*
+ * Some implementations have muxed signals, whereas some
+ * use a mux in front of the signals and need dynamic
+ * assignment of signals.
+ */
+ chan->signal = i;
+ pl08x_dma_slave_init(chan);
+ } else {
+ chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);
+ if (!chan->cd) {
+ kfree(chan);
+ return -ENOMEM;
+ }
+ chan->cd->bus_id = "memcpy";
+ chan->cd->periph_buses = pl08x->pd->mem_buses;
+ chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
+ if (!chan->name) {
+ kfree(chan->cd);
+ kfree(chan);
+ return -ENOMEM;
+ }
+ }
+ dev_dbg(&pl08x->adev->dev,
+ "initialize virtual channel \"%s\"\n",
+ chan->name);
+
+ chan->vc.desc_free = pl08x_desc_free;
+ vchan_init(&chan->vc, dmadev);
+ }
+ dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
+ i, slave ? "slave" : "memcpy");
+ return i;
+}
+
+static void pl08x_free_virtual_channels(struct dma_device *dmadev)
+{
+ struct pl08x_dma_chan *chan = NULL;
+ struct pl08x_dma_chan *next;
+
+ list_for_each_entry_safe(chan,
+ next, &dmadev->channels, vc.chan.device_node) {
+ list_del(&chan->vc.chan.device_node);
+ kfree(chan);
+ }
+}
+
+#ifdef CONFIG_DEBUG_FS
+static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
+{
+ switch (state) {
+ case PL08X_CHAN_IDLE:
+ return "idle";
+ case PL08X_CHAN_RUNNING:
+ return "running";
+ case PL08X_CHAN_PAUSED:
+ return "paused";
+ case PL08X_CHAN_WAITING:
+ return "waiting";
+ default:
+ break;
+ }
+ return "UNKNOWN STATE";
+}
+
+static int pl08x_debugfs_show(struct seq_file *s, void *data)
+{
+ struct pl08x_driver_data *pl08x = s->private;
+ struct pl08x_dma_chan *chan;
+ struct pl08x_phy_chan *ch;
+ unsigned long flags;
+ int i;
+
+ seq_printf(s, "PL08x physical channels:\n");
+ seq_printf(s, "CHANNEL:\tUSER:\n");
+ seq_printf(s, "--------\t-----\n");
+ for (i = 0; i < pl08x->vd->channels; i++) {
+ struct pl08x_dma_chan *virt_chan;
+
+ ch = &pl08x->phy_chans[i];
+
+ spin_lock_irqsave(&ch->lock, flags);
+ virt_chan = ch->serving;
+
+ seq_printf(s, "%d\t\t%s%s\n",
+ ch->id,
+ virt_chan ? virt_chan->name : "(none)",
+ ch->locked ? " LOCKED" : "");
+
+ spin_unlock_irqrestore(&ch->lock, flags);
+ }
+
+ seq_printf(s, "\nPL08x virtual memcpy channels:\n");
+ seq_printf(s, "CHANNEL:\tSTATE:\n");
+ seq_printf(s, "--------\t------\n");
+ list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
+ seq_printf(s, "%s\t\t%s\n", chan->name,
+ pl08x_state_str(chan->state));
+ }
+
+ if (pl08x->has_slave) {
+ seq_printf(s, "\nPL08x virtual slave channels:\n");
+ seq_printf(s, "CHANNEL:\tSTATE:\n");
+ seq_printf(s, "--------\t------\n");
+ list_for_each_entry(chan, &pl08x->slave.channels,
+ vc.chan.device_node) {
+ seq_printf(s, "%s\t\t%s\n", chan->name,
+ pl08x_state_str(chan->state));
+ }
+ }
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
+
+static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
+{
+ /* Expose a simple debugfs interface to view all clocks */
+ debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
+ NULL, pl08x, &pl08x_debugfs_fops);
+}
+
+#else
+static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
+{
+}
+#endif
+
+#ifdef CONFIG_OF
+static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
+ u32 id)
+{
+ struct pl08x_dma_chan *chan;
+
+ /* Trying to get a slave channel from something with no slave support */
+ if (!pl08x->has_slave)
+ return NULL;
+
+ list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
+ if (chan->signal == id)
+ return &chan->vc.chan;
+ }
+
+ return NULL;
+}
+
+static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
+ struct of_dma *ofdma)
+{
+ struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
+ struct dma_chan *dma_chan;
+ struct pl08x_dma_chan *plchan;
+
+ if (!pl08x)
+ return NULL;
+
+ if (dma_spec->args_count != 2) {
+ dev_err(&pl08x->adev->dev,
+ "DMA channel translation requires two cells\n");
+ return NULL;
+ }
+
+ dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
+ if (!dma_chan) {
+ dev_err(&pl08x->adev->dev,
+ "DMA slave channel not found\n");
+ return NULL;
+ }
+
+ plchan = to_pl08x_chan(dma_chan);
+ dev_dbg(&pl08x->adev->dev,
+ "translated channel for signal %d\n",
+ dma_spec->args[0]);
+
+ /* Augment channel data for applicable AHB buses */
+ plchan->cd->periph_buses = dma_spec->args[1];
+ return dma_get_slave_channel(dma_chan);
+}
+
+static int pl08x_of_probe(struct platform_device *adev,
+ struct pl08x_driver_data *pl08x,
+ struct device_node *np)
+{
+ struct pl08x_platform_data *pd;
+ struct pl08x_channel_data *chanp = NULL;
+ u32 val;
+ int ret;
+ int i;
+
+ pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
+ if (!pd)
+ return -ENOMEM;
+
+ /* Eligible bus masters for fetching LLIs */
+ if (of_property_read_bool(np, "lli-bus-interface-ahb1"))
+ pd->lli_buses |= PL08X_AHB1;
+ if (of_property_read_bool(np, "lli-bus-interface-ahb2"))
+ pd->lli_buses |= PL08X_AHB2;
+ if (!pd->lli_buses) {
+ dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
+ pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
+ }
+
+ /* Eligible bus masters for memory access */
+ if (of_property_read_bool(np, "mem-bus-interface-ahb1"))
+ pd->mem_buses |= PL08X_AHB1;
+ if (of_property_read_bool(np, "mem-bus-interface-ahb2"))
+ pd->mem_buses |= PL08X_AHB2;
+ if (!pd->mem_buses) {
+ dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
+ pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
+ }
+
+ /* Parse the memcpy channel properties */
+ ret = of_property_read_u32(np, "memcpy-burst-size", &val);
+ if (ret) {
+ dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
+ val = 1;
+ }
+ switch (val) {
+ default:
+ dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
+ fallthrough;
+ case 1:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_1;
+ break;
+ case 4:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_4;
+ break;
+ case 8:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_8;
+ break;
+ case 16:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_16;
+ break;
+ case 32:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_32;
+ break;
+ case 64:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_64;
+ break;
+ case 128:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_128;
+ break;
+ case 256:
+ pd->memcpy_burst_size = PL08X_BURST_SZ_256;
+ break;
+ }
+
+ ret = of_property_read_u32(np, "memcpy-bus-width", &val);
+ if (ret) {
+ dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
+ val = 8;
+ }
+ switch (val) {
+ default:
+ dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
+ fallthrough;
+ case 8:
+ pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;
+ break;
+ case 16:
+ pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;
+ break;
+ case 32:
+ pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;
+ break;
+ }
+
+ /*
+ * Allocate channel data for all possible slave channels (one
+ * for each possible signal), channels will then be allocated
+ * for a device and have it's AHB interfaces set up at
+ * translation time.
+ */
+ if (pl08x->vd->signals) {
+ chanp = devm_kcalloc(&adev->dev,
+ pl08x->vd->signals,
+ sizeof(struct pl08x_channel_data),
+ GFP_KERNEL);
+ if (!chanp)
+ return -ENOMEM;
+
+ pd->slave_channels = chanp;
+ for (i = 0; i < pl08x->vd->signals; i++) {
+ /*
+ * chanp->periph_buses will be assigned at translation
+ */
+ chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);
+ chanp++;
+ }
+ pd->num_slave_channels = pl08x->vd->signals;
+ }
+
+ pl08x->pd = pd;
+
+ return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
+ pl08x);
+}
+#else
+static inline int pl08x_of_probe(struct platform_device *adev,
+ struct pl08x_driver_data *pl08x,
+ struct device_node *np)
+{
+ return -EINVAL;
+}
+#endif
+
+static int pl08x_probe(struct platform_device *adev) //, const struct amba_id *id)
+{
+ struct pl08x_driver_data *pl08x;
+ struct vendor_data *vd;
+ struct device_node *np = adev->dev.of_node;
+ struct resource *res;
+ u32 tsfr_size;
+ int irq, ret = 0;
+ int i;
+
+ //printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+#if 0
+ ret = amba_request_regions(adev, NULL);
+ if (ret){
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ return ret;
+ }
+#endif
+ /* Ensure that we can do DMA */
+ ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
+ if (ret){
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ return ret;
+ }
+
+ /* Create the driver state holder */
+ pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
+ if (!pl08x) {
+ ret = -ENOMEM;
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ return ret;
+ }
+
+ /* Assign useful pointers to the driver state */
+ pl08x->adev = adev;
+ vd = of_device_get_match_data(&adev->dev);
+ if(!vd)
+ return -ENODEV;
+ pl08x->vd = vd;
+
+ res = platform_get_resource_byname(adev, IORESOURCE_MEM, "sec_dma");
+ pl08x->base = devm_ioremap_resource(&adev->dev, res);
+ if (!pl08x->base) {
+ ret = -ENOMEM;
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ goto out_no_ioremap;
+ }
+
+ if (vd->ftdmac020) {
+ u32 val;
+
+ val = readl(pl08x->base + FTDMAC020_REVISION);
+ dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",
+ (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
+ val = readl(pl08x->base + FTDMAC020_FEATURE);
+ dev_dbg(&pl08x->adev->dev, "FTDMAC020 %d channels, "
+ "%s built-in bridge, %s, %s linked lists\n",
+ (val >> 12) & 0x0f,
+ (val & BIT(10)) ? "no" : "has",
+ (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",
+ (val & BIT(8)) ? "supports" : "does not support");
+
+ /* Vendor data from feature register */
+ if (!(val & BIT(8)))
+ dev_warn(&pl08x->adev->dev,
+ "linked lists not supported, required\n");
+ vd->channels = (val >> 12) & 0x0f;
+ vd->dualmaster = !!(val & BIT(9));
+ }
+
+ /* Initialize memcpy engine */
+ dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
+ pl08x->memcpy.dev = &adev->dev;
+ pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
+ pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
+ pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
+ pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
+ pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
+ pl08x->memcpy.device_config = pl08x_config;
+ pl08x->memcpy.device_pause = pl08x_pause;
+ pl08x->memcpy.device_resume = pl08x_resume;
+ pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
+ pl08x->memcpy.device_synchronize = pl08x_synchronize;
+ pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
+ pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
+ pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
+ pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ if (vd->ftdmac020)
+ pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
+
+
+ /*
+ * Initialize slave engine, if the block has no signals, that means
+ * we have no slave support.
+ */
+ if (vd->signals) {
+ pl08x->has_slave = true;
+ dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
+ dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
+ pl08x->slave.dev = &adev->dev;
+ pl08x->slave.device_free_chan_resources =
+ pl08x_free_chan_resources;
+ pl08x->slave.device_prep_dma_interrupt =
+ pl08x_prep_dma_interrupt;
+ pl08x->slave.device_tx_status = pl08x_dma_tx_status;
+ pl08x->slave.device_issue_pending = pl08x_issue_pending;
+ pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
+ pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
+ pl08x->slave.device_config = pl08x_config;
+ pl08x->slave.device_pause = pl08x_pause;
+ pl08x->slave.device_resume = pl08x_resume;
+ pl08x->slave.device_terminate_all = pl08x_terminate_all;
+ pl08x->slave.device_synchronize = pl08x_synchronize;
+ pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
+ pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
+ pl08x->slave.directions =
+ BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+ pl08x->slave.residue_granularity =
+ DMA_RESIDUE_GRANULARITY_SEGMENT;
+ }
+
+ /* Get the platform data */
+ pl08x->pd = dev_get_platdata(&adev->dev);
+ if (!pl08x->pd) {
+ if (np) {
+ ret = pl08x_of_probe(adev, pl08x, np);
+ if (ret) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ goto out_no_platdata;
+ }
+ } else {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_err(&adev->dev, "no platform data supplied\n");
+ ret = -EINVAL;
+ goto out_no_platdata;
+ }
+ } else {
+ pl08x->slave.filter.map = pl08x->pd->slave_map;
+ pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
+ pl08x->slave.filter.fn = pl08x_filter_fn;
+ }
+
+ /* By default, AHB1 only. If dualmaster, from platform */
+ pl08x->lli_buses = PL08X_AHB1;
+ pl08x->mem_buses = PL08X_AHB1;
+ if (pl08x->vd->dualmaster) {
+ pl08x->lli_buses = pl08x->pd->lli_buses;
+ pl08x->mem_buses = pl08x->pd->mem_buses;
+ }
+
+ if (vd->pl080s)
+ pl08x->lli_words = PL080S_LLI_WORDS;
+ else
+ pl08x->lli_words = PL080_LLI_WORDS;
+ tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
+
+ /* A DMA memory pool for LLIs, align on 1-byte boundary */
+ pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
+ tsfr_size, PL08X_ALIGN, 0);
+ if (!pl08x->pool) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ ret = -ENOMEM;
+ goto out_no_lli_pool;
+ }
+
+ /* Turn on the PL08x */
+ pl08x_ensure_on(pl08x);
+
+ /* Clear any pending interrupts */
+ if (vd->ftdmac020)
+ /* This variant has error IRQs in bits 16-19 */
+ writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR);
+ else
+ writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
+ writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
+
+ /* Attach the interrupt handler */
+ irq = platform_get_irq(adev, 0);
+ if (irq < 0) {
+ dev_err(&adev->dev, "Cannot get IRQ resource\n");
+ return irq;
+ }
+
+ ret = request_irq(irq, pl08x_irq, 0, DRIVER_NAME, pl08x);
+ if (ret) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_err(&adev->dev, "%s failed to request interrupt %d\n",
+ __func__, irq);
+ goto out_no_irq;
+ }
+
+ /* Initialize physical channels */
+ pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
+ GFP_KERNEL);
+ if (!pl08x->phy_chans) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ ret = -ENOMEM;
+ goto out_no_phychans;
+ }
+
+ for (i = 0; i < vd->channels; i++) {
+ struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
+
+ ch->id = i;
+ ch->base = pl08x->base + PL080_Cx_BASE(i);
+ if (vd->ftdmac020) {
+ /* FTDMA020 has a special channel busy register */
+ ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;
+ ch->reg_config = ch->base + FTDMAC020_CH_CFG;
+ ch->reg_control = ch->base + FTDMAC020_CH_CSR;
+ ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;
+ ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;
+ ch->reg_lli = ch->base + FTDMAC020_CH_LLP;
+ ch->ftdmac020 = true;
+ } else {
+ printk("this is debug i = %d ch->base = %x %s %s %d\n",i,ch->base,__FILE__,__func__,__LINE__);
+ ch->reg_config = ch->base + vd->config_offset;
+ ch->reg_control = ch->base + PL080_CH_CONTROL;
+ ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
+ ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
+ ch->reg_lli = ch->base + PL080_CH_LLI;
+ }
+ if (vd->pl080s)
+ ch->pl080s = true;
+
+ spin_lock_init(&ch->lock);
+
+ /*
+ * Nomadik variants can have channels that are locked
+ * down for the secure world only. Lock up these channels
+ * by perpetually serving a dummy virtual channel.
+ */
+ if (vd->nomadik) {
+ u32 val;
+
+ val = readl(ch->reg_config);
+ if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
+ dev_dbg(&adev->dev, "physical channel %d reserved for secure access only\n", i);
+ ch->locked = true;
+ }
+ }
+
+ //dev_dbg(&adev->dev, "physical channel %d is %s\n",
+ dev_dbg(&adev->dev, "physical channel %d is %s\n",
+ i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
+ }
+
+ /* Register as many memcpy channels as there are physical channels */
+ ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
+ pl08x->vd->channels, false);
+ if (ret <= 0) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_warn(&pl08x->adev->dev,
+ "%s failed to enumerate memcpy channels - %d\n",
+ __func__, ret);
+ goto out_no_memcpy;
+ }
+
+ /* Register slave channels */
+ if (pl08x->has_slave) {
+ ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
+ pl08x->pd->num_slave_channels, true);
+ if (ret < 0) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_warn(&pl08x->adev->dev,
+ "%s failed to enumerate slave channels - %d\n",
+ __func__, ret);
+ goto out_no_slave;
+ }
+ }
+
+ ret = dma_async_device_register(&pl08x->memcpy);
+ if (ret) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_warn(&pl08x->adev->dev,
+ "%s failed to register memcpy as an async device - %d\n",
+ __func__, ret);
+ goto out_no_memcpy_reg;
+ }
+
+ if (pl08x->has_slave) {
+ ret = dma_async_device_register(&pl08x->slave);
+ if (ret) {
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_warn(&pl08x->adev->dev,
+ "%s failed to register slave as an async device - %d\n",
+ __func__, ret);
+ goto out_no_slave_reg;
+ }
+ }
+
+ platform_set_drvdata(adev, pl08x);
+ init_pl08x_debugfs(pl08x);
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ {
+ int loop;
+
+ for (loop = 0xfe0;loop <= 0xffc; loop += 4) {
+ dev_dbg(&pl08x->adev->dev, "periphid[0x%x] = %x ",
+ loop,readl(pl08x->base + loop));
+ }
+ }
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ dev_dbg(&pl08x->adev->dev, "DMA: PL080 at 0x%08llx irq %d\n",
+ (unsigned long long)res->start, irq);
+
+ return 0;
+
+out_no_slave_reg:
+ dma_async_device_unregister(&pl08x->memcpy);
+out_no_memcpy_reg:
+ if (pl08x->has_slave)
+ pl08x_free_virtual_channels(&pl08x->slave);
+out_no_slave:
+ pl08x_free_virtual_channels(&pl08x->memcpy);
+out_no_memcpy:
+ kfree(pl08x->phy_chans);
+out_no_phychans:
+ free_irq(irq, pl08x);
+out_no_irq:
+ dma_pool_destroy(pl08x->pool);
+out_no_lli_pool:
+out_no_platdata:
+ iounmap(pl08x->base);
+out_no_ioremap:
+ kfree(pl08x);
+ //out_no_pl08x:
+ //amba_release_regions(adev);
+ printk("this is debug %s %s %d\n",__FILE__,__func__,__LINE__);
+ return ret;
+}
+
+/* PL080 has 8 channels and the PL080 have just 2 */
+static struct vendor_data vendor_pl080 = {
+ .config_offset = PL080_CH_CONFIG,
+ .channels = 8,
+ .signals = 16,
+ .dualmaster = true,
+ .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
+};
+#if 0
+static struct vendor_data vendor_nomadik = {
+ .config_offset = PL080_CH_CONFIG,
+ .channels = 8,
+ .signals = 32,
+ .dualmaster = true,
+ .nomadik = true,
+ .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
+};
+
+static struct vendor_data vendor_pl080s = {
+ .config_offset = PL080S_CH_CONFIG,
+ .channels = 8,
+ .signals = 32,
+ .pl080s = true,
+ .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
+};
+
+static struct vendor_data vendor_pl081 = {
+ .config_offset = PL080_CH_CONFIG,
+ .channels = 2,
+ .signals = 16,
+ .dualmaster = false,
+ .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
+};
+
+static struct vendor_data vendor_ftdmac020 = {
+ .config_offset = PL080_CH_CONFIG,
+ .ftdmac020 = true,
+ .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
+};
+
+static const struct amba_id pl08x_ids[] = {
+ /* Samsung PL080S variant */
+ {
+ .id = 0x0a141080,
+ .mask = 0xffffffff,
+ .data = &vendor_pl080s,
+ },
+ /* PL080 */
+ {
+ .id = 0x00041080,
+ .mask = 0x000fffff,
+ .data = &vendor_pl080,
+ },
+ /* PL081 */
+ {
+ .id = 0x00041081,
+ .mask = 0x000fffff,
+ .data = &vendor_pl081,
+ },
+ /* Nomadik 8815 PL080 variant */
+ {
+ .id = 0x00280080,
+ .mask = 0x00ffffff,
+ .data = &vendor_nomadik,
+ },
+ /* Faraday Technology FTDMAC020 */
+ {
+ .id = 0x0003b080,
+ .mask = 0x000fffff,
+ .data = &vendor_ftdmac020,
+ },
+ { 0, 0 },
+};
+
+MODULE_DEVICE_TABLE(amba, pl08x_ids);
+
+static struct amba_driver pl08x_amba_driver = {
+ .drv.name = DRIVER_NAME,
+ .id_table = pl08x_ids,
+ .probe = pl08x_probe,
+};
+#endif
+static const struct of_device_id vic7110_dma_ids[] = {
+ { .compatible = "starfive,pl080", .data = &vendor_pl080},
+ {},
+};
+MODULE_DEVICE_TABLE(of, vic7110_dma_ids);
+
+static struct platform_driver vic7110_pl08x_driver = {
+ .probe = pl08x_probe,
+ .driver = {
+ .name = DRIVER_NAME,
+ .of_match_table = vic7110_dma_ids,
+ },
+};
+
+module_platform_driver(vic7110_pl08x_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Huan Feng <huan.feng@starfivetech.com>");
+MODULE_DESCRIPTION("Starfive VIC7110 CRYP DMA driver");
projects / platform / kernel / linux-starfive.git / commitdiff