drivers/crypto/omap-aes-gcm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Cryptographic API.
   4  *
   5  * Support for OMAP AES GCM HW acceleration.
   6  *
   7  * Copyright (c) 2016 Texas Instruments Incorporated
   8  */
   9
  10 #include <crypto/aes.h>
  11 #include <crypto/engine.h>
  12 #include <crypto/gcm.h>
  13 #include <crypto/internal/aead.h>
  14 #include <crypto/scatterwalk.h>
  15 #include <crypto/skcipher.h>
  16 #include <linux/errno.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmaengine.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/kernel.h>
  21 #include <linux/omap-dma.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/string.h>
  25
  26 #include "omap-crypto.h"
  27 #include "omap-aes.h"
  28
  29 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
  30                                      struct aead_request *req);
  31
  32 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
  33 {
  34         struct aead_request *req = dd->aead_req;
  35
  36         dd->in_sg = NULL;
  37         dd->out_sg = NULL;
  38
  39         crypto_finalize_aead_request(dd->engine, req, ret);
  40
  41         pm_runtime_mark_last_busy(dd->dev);
  42         pm_runtime_put_autosuspend(dd->dev);
  43 }
  44
  45 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
  46 {
  47         u8 *tag;
  48         int alen, clen, i, ret = 0, nsg;
  49         struct omap_aes_reqctx *rctx;
  50
  51         alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
  52         clen = ALIGN(dd->total, AES_BLOCK_SIZE);
  53         rctx = aead_request_ctx(dd->aead_req);
  54
  55         nsg = !!(dd->assoc_len && dd->total);
  56
  57         dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  58                                DMA_FROM_DEVICE);
  59         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  60         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
  61         omap_aes_crypt_dma_stop(dd);
  62
  63         omap_crypto_cleanup(dd->out_sg, dd->orig_out,
  64                             dd->aead_req->assoclen, dd->total,
  65                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  66
  67         if (dd->flags & FLAGS_ENCRYPT)
  68                 scatterwalk_map_and_copy(rctx->auth_tag,
  69                                          dd->aead_req->dst,
  70                                          dd->total + dd->aead_req->assoclen,
  71                                          dd->authsize, 1);
  72
  73         omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
  74                             FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
  75
  76         omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
  77                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  78
  79         if (!(dd->flags & FLAGS_ENCRYPT)) {
  80                 tag = (u8 *)rctx->auth_tag;
  81                 for (i = 0; i < dd->authsize; i++) {
  82                         if (tag[i]) {
  83                                 ret = -EBADMSG;
  84                         }
  85                 }
  86         }
  87
  88         omap_aes_gcm_finish_req(dd, ret);
  89 }
  90
  91 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
  92                                      struct aead_request *req)
  93 {
  94         int alen, clen, cryptlen, assoclen, ret;
  95         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  96         unsigned int authlen = crypto_aead_authsize(aead);
  97         struct scatterlist *tmp, sg_arr[2];
  98         int nsg;
  99         u16 flags;
 100
 101         assoclen = req->assoclen;
 102         cryptlen = req->cryptlen;
 103
 104         if (dd->flags & FLAGS_RFC4106_GCM)
 105                 assoclen -= 8;
 106
 107         if (!(dd->flags & FLAGS_ENCRYPT))
 108                 cryptlen -= authlen;
 109
 110         alen = ALIGN(assoclen, AES_BLOCK_SIZE);
 111         clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
 112
 113         nsg = !!(assoclen && cryptlen);
 114
 115         omap_aes_clear_copy_flags(dd);
 116
 117         sg_init_table(dd->in_sgl, nsg + 1);
 118         if (assoclen) {
 119                 tmp = req->src;
 120                 ret = omap_crypto_align_sg(&tmp, assoclen,
 121                                            AES_BLOCK_SIZE, dd->in_sgl,
 122                                            OMAP_CRYPTO_COPY_DATA |
 123                                            OMAP_CRYPTO_ZERO_BUF |
 124                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 125                                            FLAGS_ASSOC_DATA_ST_SHIFT,
 126                                            &dd->flags);
 127                 if (ret)
 128                         return ret;
 129         }
 130
 131         if (cryptlen) {
 132                 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
 133
 134                 if (nsg)
 135                         sg_unmark_end(dd->in_sgl);
 136
 137                 ret = omap_crypto_align_sg(&tmp, cryptlen,
 138                                            AES_BLOCK_SIZE, &dd->in_sgl[nsg],
 139                                            OMAP_CRYPTO_COPY_DATA |
 140                                            OMAP_CRYPTO_ZERO_BUF |
 141                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 142                                            FLAGS_IN_DATA_ST_SHIFT,
 143                                            &dd->flags);
 144                 if (ret)
 145                         return ret;
 146         }
 147
 148         dd->in_sg = dd->in_sgl;
 149         dd->total = cryptlen;
 150         dd->assoc_len = assoclen;
 151         dd->authsize = authlen;
 152
 153         dd->out_sg = req->dst;
 154         dd->orig_out = req->dst;
 155
 156         dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
 157
 158         flags = 0;
 159         if (req->src == req->dst || dd->out_sg == sg_arr)
 160                 flags |= OMAP_CRYPTO_FORCE_COPY;
 161
 162         if (cryptlen) {
 163                 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
 164                                            AES_BLOCK_SIZE, &dd->out_sgl,
 165                                            flags,
 166                                            FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 167                 if (ret)
 168                         return ret;
 169         }
 170
 171         dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
 172         dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
 173
 174         return 0;
 175 }
 176
 177 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
 178 {
 179         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 180
 181         aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
 182         return 0;
 183 }
 184
 185 void omap_aes_gcm_dma_out_callback(void *data)
 186 {
 187         struct omap_aes_dev *dd = data;
 188         struct omap_aes_reqctx *rctx;
 189         int i, val;
 190         u32 *auth_tag, tag[4];
 191
 192         if (!(dd->flags & FLAGS_ENCRYPT))
 193                 scatterwalk_map_and_copy(tag, dd->aead_req->src,
 194                                          dd->total + dd->aead_req->assoclen,
 195                                          dd->authsize, 0);
 196
 197         rctx = aead_request_ctx(dd->aead_req);
 198         auth_tag = (u32 *)rctx->auth_tag;
 199         for (i = 0; i < 4; i++) {
 200                 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
 201                 auth_tag[i] = val ^ auth_tag[i];
 202                 if (!(dd->flags & FLAGS_ENCRYPT))
 203                         auth_tag[i] = auth_tag[i] ^ tag[i];
 204         }
 205
 206         omap_aes_gcm_done_task(dd);
 207 }
 208
 209 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
 210                                      struct aead_request *req)
 211 {
 212         if (req)
 213                 return crypto_transfer_aead_request_to_engine(dd->engine, req);
 214
 215         return 0;
 216 }
 217
 218 static int omap_aes_gcm_prepare_req(struct aead_request *req,
 219                                     struct omap_aes_dev *dd)
 220 {
 221         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 222         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 223         int err;
 224
 225         dd->aead_req = req;
 226
 227         rctx->mode &= FLAGS_MODE_MASK;
 228         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 229
 230         err = omap_aes_gcm_copy_buffers(dd, req);
 231         if (err)
 232                 return err;
 233
 234         dd->ctx = &ctx->octx;
 235
 236         return omap_aes_write_ctrl(dd);
 237 }
 238
 239 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
 240 {
 241         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 242         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 243         unsigned int authlen = crypto_aead_authsize(aead);
 244         struct omap_aes_dev *dd;
 245         __be32 counter = cpu_to_be32(1);
 246         int err, assoclen;
 247
 248         memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
 249         memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
 250
 251         err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
 252         if (err)
 253                 return err;
 254
 255         if (mode & FLAGS_RFC4106_GCM)
 256                 assoclen = req->assoclen - 8;
 257         else
 258                 assoclen = req->assoclen;
 259         if (assoclen + req->cryptlen == 0) {
 260                 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
 261                                          1);
 262                 return 0;
 263         }
 264
 265         dd = omap_aes_find_dev(rctx);
 266         if (!dd)
 267                 return -ENODEV;
 268         rctx->mode = mode;
 269
 270         return omap_aes_gcm_handle_queue(dd, req);
 271 }
 272
 273 int omap_aes_gcm_encrypt(struct aead_request *req)
 274 {
 275         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 276
 277         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 278         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
 279 }
 280
 281 int omap_aes_gcm_decrypt(struct aead_request *req)
 282 {
 283         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 284
 285         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 286         return omap_aes_gcm_crypt(req, FLAGS_GCM);
 287 }
 288
 289 int omap_aes_4106gcm_encrypt(struct aead_request *req)
 290 {
 291         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 292         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 293
 294         memcpy(rctx->iv, ctx->octx.nonce, 4);
 295         memcpy(rctx->iv + 4, req->iv, 8);
 296         return crypto_ipsec_check_assoclen(req->assoclen) ?:
 297                omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
 298                                   FLAGS_RFC4106_GCM);
 299 }
 300
 301 int omap_aes_4106gcm_decrypt(struct aead_request *req)
 302 {
 303         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 304         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 305
 306         memcpy(rctx->iv, ctx->octx.nonce, 4);
 307         memcpy(rctx->iv + 4, req->iv, 8);
 308         return crypto_ipsec_check_assoclen(req->assoclen) ?:
 309                omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
 310 }
 311
 312 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 313                         unsigned int keylen)
 314 {
 315         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 316         int ret;
 317
 318         ret = aes_expandkey(&ctx->actx, key, keylen);
 319         if (ret)
 320                 return ret;
 321
 322         memcpy(ctx->octx.key, key, keylen);
 323         ctx->octx.keylen = keylen;
 324
 325         return 0;
 326 }
 327
 328 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 329                             unsigned int keylen)
 330 {
 331         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 332         int ret;
 333
 334         if (keylen < 4)
 335                 return -EINVAL;
 336         keylen -= 4;
 337
 338         ret = aes_expandkey(&ctx->actx, key, keylen);
 339         if (ret)
 340                 return ret;
 341
 342         memcpy(ctx->octx.key, key, keylen);
 343         memcpy(ctx->octx.nonce, key + keylen, 4);
 344         ctx->octx.keylen = keylen;
 345
 346         return 0;
 347 }
 348
 349 int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
 350 {
 351         return crypto_gcm_check_authsize(authsize);
 352 }
 353
 354 int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
 355                                  unsigned int authsize)
 356 {
 357         return crypto_rfc4106_check_authsize(authsize);
 358 }
 359
 360 int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
 361 {
 362         struct aead_request *req = container_of(areq, struct aead_request,
 363                                                 base);
 364         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 365         struct omap_aes_dev *dd = rctx->dd;
 366         int ret;
 367
 368         if (!dd)
 369                 return -ENODEV;
 370
 371         ret = omap_aes_gcm_prepare_req(req, dd);
 372         if (ret)
 373                 return ret;
 374
 375         if (dd->in_sg_len)
 376                 ret = omap_aes_crypt_dma_start(dd);
 377         else
 378                 omap_aes_gcm_dma_out_callback(dd);
 379
 380         return ret;
 381 }
 382
 383 int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
 384 {
 385         crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
 386
 387         return 0;
 388 }