drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

   1 /*
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <crypto/internal/geniv.h>
  35 #include <crypto/aead.h>
  36 #include <linux/inetdevice.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/module.h>
  39
  40 #include "en.h"
  41 #include "en_accel/ipsec.h"
  42 #include "en_accel/ipsec_rxtx.h"
  43
  44
  45 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)
  46 {
  47         struct mlx5e_ipsec_sa_entry *sa;
  48
  49         if (!x)
  50                 return NULL;
  51
  52         sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
  53         if (!sa)
  54                 return NULL;
  55
  56         WARN_ON(sa->x != x);
  57         return sa;
  58 }
  59
  60 struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
  61                                               unsigned int handle)
  62 {
  63         struct mlx5e_ipsec_sa_entry *sa_entry;
  64         struct xfrm_state *ret = NULL;
  65
  66         rcu_read_lock();
  67         hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
  68                 if (sa_entry->handle == handle) {
  69                         ret = sa_entry->x;
  70                         xfrm_state_hold(ret);
  71                         break;
  72                 }
  73         rcu_read_unlock();
  74
  75         return ret;
  76 }
  77
  78 static int  mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry,
  79                                     unsigned int handle)
  80 {
  81         struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
  82         struct mlx5e_ipsec_sa_entry *_sa_entry;
  83         unsigned long flags;
  84
  85         rcu_read_lock();
  86         hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle)
  87                 if (_sa_entry->handle == handle) {
  88                         rcu_read_unlock();
  89                         return  -EEXIST;
  90                 }
  91         rcu_read_unlock();
  92
  93         spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
  94         sa_entry->handle = handle;
  95         hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
  96         spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
  97
  98         return 0;
  99 }
 100
 101 static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
 102 {
 103         struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
 104         unsigned long flags;
 105
 106         spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
 107         hash_del_rcu(&sa_entry->hlist);
 108         spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
 109 }
 110
 111 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
 112 {
 113         struct xfrm_replay_state_esn *replay_esn;
 114         u32 seq_bottom;
 115         u8 overlap;
 116         u32 *esn;
 117
 118         if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {
 119                 sa_entry->esn_state.trigger = 0;
 120                 return false;
 121         }
 122
 123         replay_esn = sa_entry->x->replay_esn;
 124         seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;
 125         overlap = sa_entry->esn_state.overlap;
 126
 127         sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,
 128                                                     htonl(seq_bottom));
 129         esn = &sa_entry->esn_state.esn;
 130
 131         sa_entry->esn_state.trigger = 1;
 132         if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
 133                 ++(*esn);
 134                 sa_entry->esn_state.overlap = 0;
 135                 return true;
 136         } else if (unlikely(!overlap &&
 137                             (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) {
 138                 sa_entry->esn_state.overlap = 1;
 139                 return true;
 140         }
 141
 142         return false;
 143 }
 144
 145 static void
 146 mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
 147                                    struct mlx5_accel_esp_xfrm_attrs *attrs)
 148 {
 149         struct xfrm_state *x = sa_entry->x;
 150         struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm;
 151         struct aead_geniv_ctx *geniv_ctx;
 152         struct crypto_aead *aead;
 153         unsigned int crypto_data_len, key_len;
 154         int ivsize;
 155
 156         memset(attrs, 0, sizeof(*attrs));
 157
 158         /* key */
 159         crypto_data_len = (x->aead->alg_key_len + 7) / 8;
 160         key_len = crypto_data_len - 4; /* 4 bytes salt at end */
 161
 162         memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len);
 163         aes_gcm->key_len = key_len * 8;
 164
 165         /* salt and seq_iv */
 166         aead = x->data;
 167         geniv_ctx = crypto_aead_ctx(aead);
 168         ivsize = crypto_aead_ivsize(aead);
 169         memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize);
 170         memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,
 171                sizeof(aes_gcm->salt));
 172
 173         /* iv len */
 174         aes_gcm->icv_len = x->aead->alg_icv_len;
 175
 176         /* esn */
 177         if (sa_entry->esn_state.trigger) {
 178                 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
 179                 attrs->esn = sa_entry->esn_state.esn;
 180                 if (sa_entry->esn_state.overlap)
 181                         attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
 182         }
 183
 184         /* rx handle */
 185         attrs->sa_handle = sa_entry->handle;
 186
 187         /* algo type */
 188         attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
 189
 190         /* action */
 191         attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ?
 192                         MLX5_ACCEL_ESP_ACTION_ENCRYPT :
 193                         MLX5_ACCEL_ESP_ACTION_DECRYPT;
 194         /* flags */
 195         attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ?
 196                         MLX5_ACCEL_ESP_FLAGS_TRANSPORT :
 197                         MLX5_ACCEL_ESP_FLAGS_TUNNEL;
 198
 199         /* spi */
 200         attrs->spi = x->id.spi;
 201
 202         /* source , destination ips */
 203         memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr));
 204         memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr));
 205         attrs->is_ipv6 = (x->props.family != AF_INET);
 206 }
 207
 208 static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
 209 {
 210         struct net_device *netdev = x->xso.dev;
 211         struct mlx5e_priv *priv;
 212
 213         if (x->xso.slave_dev)
 214                 netdev = x->xso.slave_dev;
 215
 216         priv = netdev_priv(netdev);
 217
 218         if (x->props.aalgo != SADB_AALG_NONE) {
 219                 netdev_info(netdev, "Cannot offload authenticated xfrm states\n");
 220                 return -EINVAL;
 221         }
 222         if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
 223                 netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n");
 224                 return -EINVAL;
 225         }
 226         if (x->props.calgo != SADB_X_CALG_NONE) {
 227                 netdev_info(netdev, "Cannot offload compressed xfrm states\n");
 228                 return -EINVAL;
 229         }
 230         if (x->props.flags & XFRM_STATE_ESN &&
 231             !(mlx5_accel_ipsec_device_caps(priv->mdev) &
 232             MLX5_ACCEL_IPSEC_CAP_ESN)) {
 233                 netdev_info(netdev, "Cannot offload ESN xfrm states\n");
 234                 return -EINVAL;
 235         }
 236         if (x->props.family != AF_INET &&
 237             x->props.family != AF_INET6) {
 238                 netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n");
 239                 return -EINVAL;
 240         }
 241         if (x->props.mode != XFRM_MODE_TRANSPORT &&
 242             x->props.mode != XFRM_MODE_TUNNEL) {
 243                 dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n");
 244                 return -EINVAL;
 245         }
 246         if (x->id.proto != IPPROTO_ESP) {
 247                 netdev_info(netdev, "Only ESP xfrm state may be offloaded\n");
 248                 return -EINVAL;
 249         }
 250         if (x->encap) {
 251                 netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n");
 252                 return -EINVAL;
 253         }
 254         if (!x->aead) {
 255                 netdev_info(netdev, "Cannot offload xfrm states without aead\n");
 256                 return -EINVAL;
 257         }
 258         if (x->aead->alg_icv_len != 128) {
 259                 netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n");
 260                 return -EINVAL;
 261         }
 262         if ((x->aead->alg_key_len != 128 + 32) &&
 263             (x->aead->alg_key_len != 256 + 32)) {
 264                 netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
 265                 return -EINVAL;
 266         }
 267         if (x->tfcpad) {
 268                 netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n");
 269                 return -EINVAL;
 270         }
 271         if (!x->geniv) {
 272                 netdev_info(netdev, "Cannot offload xfrm states without geniv\n");
 273                 return -EINVAL;
 274         }
 275         if (strcmp(x->geniv, "seqiv")) {
 276                 netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");
 277                 return -EINVAL;
 278         }
 279         if (x->props.family == AF_INET6 &&
 280             !(mlx5_accel_ipsec_device_caps(priv->mdev) &
 281              MLX5_ACCEL_IPSEC_CAP_IPV6)) {
 282                 netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n");
 283                 return -EINVAL;
 284         }
 285         return 0;
 286 }
 287
 288 static int mlx5e_xfrm_add_state(struct xfrm_state *x)
 289 {
 290         struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
 291         struct net_device *netdev = x->xso.dev;
 292         struct mlx5_accel_esp_xfrm_attrs attrs;
 293         struct mlx5e_priv *priv;
 294         unsigned int sa_handle;
 295         int err;
 296
 297         if (x->xso.slave_dev)
 298                 netdev = x->xso.slave_dev;
 299
 300         priv = netdev_priv(netdev);
 301
 302         err = mlx5e_xfrm_validate_state(x);
 303         if (err)
 304                 return err;
 305
 306         sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
 307         if (!sa_entry) {
 308                 err = -ENOMEM;
 309                 goto out;
 310         }
 311
 312         sa_entry->x = x;
 313         sa_entry->ipsec = priv->ipsec;
 314
 315         /* check esn */
 316         mlx5e_ipsec_update_esn_state(sa_entry);
 317
 318         /* create xfrm */
 319         mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
 320         sa_entry->xfrm =
 321                 mlx5_accel_esp_create_xfrm(priv->mdev, &attrs,
 322                                            MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA);
 323         if (IS_ERR(sa_entry->xfrm)) {
 324                 err = PTR_ERR(sa_entry->xfrm);
 325                 goto err_sa_entry;
 326         }
 327
 328         /* create hw context */
 329         sa_entry->hw_context =
 330                         mlx5_accel_esp_create_hw_context(priv->mdev,
 331                                                          sa_entry->xfrm,
 332                                                          &sa_handle);
 333         if (IS_ERR(sa_entry->hw_context)) {
 334                 err = PTR_ERR(sa_entry->hw_context);
 335                 goto err_xfrm;
 336         }
 337
 338         if (x->xso.flags & XFRM_OFFLOAD_INBOUND) {
 339                 err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle);
 340                 if (err)
 341                         goto err_hw_ctx;
 342         } else {
 343                 sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
 344                                 mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
 345         }
 346
 347         x->xso.offload_handle = (unsigned long)sa_entry;
 348         goto out;
 349
 350 err_hw_ctx:
 351         mlx5_accel_esp_free_hw_context(sa_entry->hw_context);
 352 err_xfrm:
 353         mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);
 354 err_sa_entry:
 355         kfree(sa_entry);
 356
 357 out:
 358         return err;
 359 }
 360
 361 static void mlx5e_xfrm_del_state(struct xfrm_state *x)
 362 {
 363         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 364
 365         if (!sa_entry)
 366                 return;
 367
 368         if (x->xso.flags & XFRM_OFFLOAD_INBOUND)
 369                 mlx5e_ipsec_sadb_rx_del(sa_entry);
 370 }
 371
 372 static void mlx5e_xfrm_free_state(struct xfrm_state *x)
 373 {
 374         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 375
 376         if (!sa_entry)
 377                 return;
 378
 379         if (sa_entry->hw_context) {
 380                 flush_workqueue(sa_entry->ipsec->wq);
 381                 mlx5_accel_esp_free_hw_context(sa_entry->hw_context);
 382                 mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);
 383         }
 384
 385         kfree(sa_entry);
 386 }
 387
 388 int mlx5e_ipsec_init(struct mlx5e_priv *priv)
 389 {
 390         struct mlx5e_ipsec *ipsec = NULL;
 391
 392         if (!MLX5_IPSEC_DEV(priv->mdev)) {
 393                 netdev_dbg(priv->netdev, "Not an IPSec offload device\n");
 394                 return 0;
 395         }
 396
 397         ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL);
 398         if (!ipsec)
 399                 return -ENOMEM;
 400
 401         hash_init(ipsec->sadb_rx);
 402         spin_lock_init(&ipsec->sadb_rx_lock);
 403         ida_init(&ipsec->halloc);
 404         ipsec->en_priv = priv;
 405         ipsec->en_priv->ipsec = ipsec;
 406         ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) &
 407                                MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER);
 408         ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
 409                                             priv->netdev->name);
 410         if (!ipsec->wq) {
 411                 kfree(ipsec);
 412                 return -ENOMEM;
 413         }
 414         netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");
 415         return 0;
 416 }
 417
 418 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
 419 {
 420         struct mlx5e_ipsec *ipsec = priv->ipsec;
 421
 422         if (!ipsec)
 423                 return;
 424
 425         destroy_workqueue(ipsec->wq);
 426
 427         ida_destroy(&ipsec->halloc);
 428         kfree(ipsec);
 429         priv->ipsec = NULL;
 430 }
 431
 432 static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
 433 {
 434         if (x->props.family == AF_INET) {
 435                 /* Offload with IPv4 options is not supported yet */
 436                 if (ip_hdr(skb)->ihl > 5)
 437                         return false;
 438         } else {
 439                 /* Offload with IPv6 extension headers is not support yet */
 440                 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
 441                         return false;
 442         }
 443
 444         return true;
 445 }
 446
 447 struct mlx5e_ipsec_modify_state_work {
 448         struct work_struct              work;
 449         struct mlx5_accel_esp_xfrm_attrs attrs;
 450         struct mlx5e_ipsec_sa_entry     *sa_entry;
 451 };
 452
 453 static void _update_xfrm_state(struct work_struct *work)
 454 {
 455         int ret;
 456         struct mlx5e_ipsec_modify_state_work *modify_work =
 457                 container_of(work, struct mlx5e_ipsec_modify_state_work, work);
 458         struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry;
 459
 460         ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm,
 461                                          &modify_work->attrs);
 462         if (ret)
 463                 netdev_warn(sa_entry->ipsec->en_priv->netdev,
 464                             "Not an IPSec offload device\n");
 465
 466         kfree(modify_work);
 467 }
 468
 469 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
 470 {
 471         struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 472         struct mlx5e_ipsec_modify_state_work *modify_work;
 473         bool need_update;
 474
 475         if (!sa_entry)
 476                 return;
 477
 478         need_update = mlx5e_ipsec_update_esn_state(sa_entry);
 479         if (!need_update)
 480                 return;
 481
 482         modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC);
 483         if (!modify_work)
 484                 return;
 485
 486         mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);
 487         modify_work->sa_entry = sa_entry;
 488
 489         INIT_WORK(&modify_work->work, _update_xfrm_state);
 490         WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work));
 491 }
 492
 493 static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
 494         .xdo_dev_state_add      = mlx5e_xfrm_add_state,
 495         .xdo_dev_state_delete   = mlx5e_xfrm_del_state,
 496         .xdo_dev_state_free     = mlx5e_xfrm_free_state,
 497         .xdo_dev_offload_ok     = mlx5e_ipsec_offload_ok,
 498         .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,
 499 };
 500
 501 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
 502 {
 503         struct mlx5_core_dev *mdev = priv->mdev;
 504         struct net_device *netdev = priv->netdev;
 505
 506         if (!priv->ipsec)
 507                 return;
 508
 509         if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) ||
 510             !MLX5_CAP_ETH(mdev, swp)) {
 511                 mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n");
 512                 return;
 513         }
 514
 515         mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");
 516         netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;
 517         netdev->features |= NETIF_F_HW_ESP;
 518         netdev->hw_enc_features |= NETIF_F_HW_ESP;
 519
 520         if (!MLX5_CAP_ETH(mdev, swp_csum)) {
 521                 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n");
 522                 return;
 523         }
 524
 525         netdev->features |= NETIF_F_HW_ESP_TX_CSUM;
 526         netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM;
 527
 528         if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) ||
 529             !MLX5_CAP_ETH(mdev, swp_lso)) {
 530                 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");
 531                 return;
 532         }
 533
 534         mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n");
 535         netdev->features |= NETIF_F_GSO_ESP;
 536         netdev->hw_features |= NETIF_F_GSO_ESP;
 537         netdev->hw_enc_features |= NETIF_F_GSO_ESP;
 538 }