drivers/net/ethernet/sfc/tc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2019 Solarflare Communications Inc.
   5  * Copyright 2020-2022 Xilinx Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation, incorporated herein by reference.
  10  */
  11
  12 #include <net/pkt_cls.h>
  13 #include <net/vxlan.h>
  14 #include <net/geneve.h>
  15 #include "tc.h"
  16 #include "tc_bindings.h"
  17 #include "mae.h"
  18 #include "ef100_rep.h"
  19 #include "efx.h"
  20
  21 static enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
  22 {
  23         if (netif_is_vxlan(net_dev))
  24                 return EFX_ENCAP_TYPE_VXLAN;
  25         if (netif_is_geneve(net_dev))
  26                 return EFX_ENCAP_TYPE_GENEVE;
  27
  28         return EFX_ENCAP_TYPE_NONE;
  29 }
  30
  31 #define EFX_EFV_PF      NULL
  32 /* Look up the representor information (efv) for a device.
  33  * May return NULL for the PF (us), or an error pointer for a device that
  34  * isn't supported as a TC offload endpoint
  35  */
  36 static struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
  37                                                 struct net_device *dev)
  38 {
  39         struct efx_rep *efv;
  40
  41         if (!dev)
  42                 return ERR_PTR(-EOPNOTSUPP);
  43         /* Is it us (the PF)? */
  44         if (dev == efx->net_dev)
  45                 return EFX_EFV_PF;
  46         /* Is it an efx vfrep at all? */
  47         if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
  48                 return ERR_PTR(-EOPNOTSUPP);
  49         /* Is it ours?  We don't support TC rules that include another
  50          * EF100's netdevices (not even on another port of the same NIC).
  51          */
  52         efv = netdev_priv(dev);
  53         if (efv->parent != efx)
  54                 return ERR_PTR(-EOPNOTSUPP);
  55         return efv;
  56 }
  57
  58 /* Convert a driver-internal vport ID into an internal device (PF or VF) */
  59 static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
  60 {
  61         u32 mport;
  62
  63         if (IS_ERR(efv))
  64                 return PTR_ERR(efv);
  65         if (!efv) /* device is PF (us) */
  66                 efx_mae_mport_uplink(efx, &mport);
  67         else /* device is repr */
  68                 efx_mae_mport_mport(efx, efv->mport, &mport);
  69         return mport;
  70 }
  71
  72 /* Convert a driver-internal vport ID into an external device (wire or VF) */
  73 static s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
  74 {
  75         u32 mport;
  76
  77         if (IS_ERR(efv))
  78                 return PTR_ERR(efv);
  79         if (!efv) /* device is PF (us) */
  80                 efx_mae_mport_wire(efx, &mport);
  81         else /* device is repr */
  82                 efx_mae_mport_mport(efx, efv->mport, &mport);
  83         return mport;
  84 }
  85
  86 static const struct rhashtable_params efx_tc_encap_match_ht_params = {
  87         .key_len        = offsetof(struct efx_tc_encap_match, linkage),
  88         .key_offset     = 0,
  89         .head_offset    = offsetof(struct efx_tc_encap_match, linkage),
  90 };
  91
  92 static const struct rhashtable_params efx_tc_match_action_ht_params = {
  93         .key_len        = sizeof(unsigned long),
  94         .key_offset     = offsetof(struct efx_tc_flow_rule, cookie),
  95         .head_offset    = offsetof(struct efx_tc_flow_rule, linkage),
  96 };
  97
  98 static void efx_tc_free_action_set(struct efx_nic *efx,
  99                                    struct efx_tc_action_set *act, bool in_hw)
 100 {
 101         /* Failure paths calling this on the 'cursor' action set in_hw=false,
 102          * because if the alloc had succeeded we'd've put it in acts.list and
 103          * not still have it in act.
 104          */
 105         if (in_hw) {
 106                 efx_mae_free_action_set(efx, act->fw_id);
 107                 /* in_hw is true iff we are on an acts.list; make sure to
 108                  * remove ourselves from that list before we are freed.
 109                  */
 110                 list_del(&act->list);
 111         }
 112         if (act->count)
 113                 efx_tc_flower_put_counter_index(efx, act->count);
 114         kfree(act);
 115 }
 116
 117 static void efx_tc_free_action_set_list(struct efx_nic *efx,
 118                                         struct efx_tc_action_set_list *acts,
 119                                         bool in_hw)
 120 {
 121         struct efx_tc_action_set *act, *next;
 122
 123         /* Failure paths set in_hw=false, because usually the acts didn't get
 124          * to efx_mae_alloc_action_set_list(); if they did, the failure tree
 125          * has a separate efx_mae_free_action_set_list() before calling us.
 126          */
 127         if (in_hw)
 128                 efx_mae_free_action_set_list(efx, acts);
 129         /* Any act that's on the list will be in_hw even if the list isn't */
 130         list_for_each_entry_safe(act, next, &acts->list, list)
 131                 efx_tc_free_action_set(efx, act, true);
 132         /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
 133 }
 134
 135 static void efx_tc_flow_free(void *ptr, void *arg)
 136 {
 137         struct efx_tc_flow_rule *rule = ptr;
 138         struct efx_nic *efx = arg;
 139
 140         netif_err(efx, drv, efx->net_dev,
 141                   "tc rule %lx still present at teardown, removing\n",
 142                   rule->cookie);
 143
 144         efx_mae_delete_rule(efx, rule->fw_id);
 145
 146         /* Release entries in subsidiary tables */
 147         efx_tc_free_action_set_list(efx, &rule->acts, true);
 148
 149         kfree(rule);
 150 }
 151
 152 /* Boilerplate for the simple 'copy a field' cases */
 153 #define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)       \
 154 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) {            \
 155         struct flow_match_##_type fm;                                   \
 156                                                                         \
 157         flow_rule_match_##_tcget(rule, &fm);                            \
 158         match->value._field = fm.key->_tcfield;                         \
 159         match->mask._field = fm.mask->_tcfield;                         \
 160 }
 161 #define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field)        \
 162         _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
 163 #define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)    \
 164         _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
 165
 166 static int efx_tc_flower_parse_match(struct efx_nic *efx,
 167                                      struct flow_rule *rule,
 168                                      struct efx_tc_match *match,
 169                                      struct netlink_ext_ack *extack)
 170 {
 171         struct flow_dissector *dissector = rule->match.dissector;
 172         unsigned char ipv = 0;
 173
 174         /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
 175          * even on IPv4 filters; so rather than relying on dissector->used_keys
 176          * we check the addr_type in the CONTROL key.  If we don't find it (or
 177          * it's masked, which should never happen), we treat both IPV4_ADDRS
 178          * and IPV6_ADDRS as absent.
 179          */
 180         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 181                 struct flow_match_control fm;
 182
 183                 flow_rule_match_control(rule, &fm);
 184                 if (IS_ALL_ONES(fm.mask->addr_type))
 185                         switch (fm.key->addr_type) {
 186                         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 187                                 ipv = 4;
 188                                 break;
 189                         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 190                                 ipv = 6;
 191                                 break;
 192                         default:
 193                                 break;
 194                         }
 195
 196                 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
 197                         match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
 198                         match->mask.ip_frag = true;
 199                 }
 200                 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
 201                         match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
 202                         match->mask.ip_firstfrag = true;
 203                 }
 204                 if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) {
 205                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x",
 206                                                fm.mask->flags);
 207                         return -EOPNOTSUPP;
 208                 }
 209         }
 210         if (dissector->used_keys &
 211             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
 212               BIT(FLOW_DISSECTOR_KEY_BASIC) |
 213               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
 214               BIT(FLOW_DISSECTOR_KEY_VLAN) |
 215               BIT(FLOW_DISSECTOR_KEY_CVLAN) |
 216               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 217               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 218               BIT(FLOW_DISSECTOR_KEY_PORTS) |
 219               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 220               BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
 221               BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 222               BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
 223               BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
 224               BIT(FLOW_DISSECTOR_KEY_TCP) |
 225               BIT(FLOW_DISSECTOR_KEY_IP))) {
 226                 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#x",
 227                                        dissector->used_keys);
 228                 return -EOPNOTSUPP;
 229         }
 230
 231         MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
 232         /* Make sure we're IP if any L3/L4 keys used. */
 233         if (!IS_ALL_ONES(match->mask.eth_proto) ||
 234             !(match->value.eth_proto == htons(ETH_P_IP) ||
 235               match->value.eth_proto == htons(ETH_P_IPV6)))
 236                 if (dissector->used_keys &
 237                     (BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 238                      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 239                      BIT(FLOW_DISSECTOR_KEY_PORTS) |
 240                      BIT(FLOW_DISSECTOR_KEY_IP) |
 241                      BIT(FLOW_DISSECTOR_KEY_TCP))) {
 242                         NL_SET_ERR_MSG_FMT_MOD(extack, "L3/L4 flower keys %#x require protocol ipv[46]",
 243                                                dissector->used_keys);
 244                         return -EINVAL;
 245                 }
 246
 247         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
 248                 struct flow_match_vlan fm;
 249
 250                 flow_rule_match_vlan(rule, &fm);
 251                 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
 252                         match->value.vlan_proto[0] = fm.key->vlan_tpid;
 253                         match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
 254                         match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
 255                                                                fm.key->vlan_id);
 256                         match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
 257                                                               fm.mask->vlan_id);
 258                 }
 259         }
 260
 261         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
 262                 struct flow_match_vlan fm;
 263
 264                 flow_rule_match_cvlan(rule, &fm);
 265                 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
 266                         match->value.vlan_proto[1] = fm.key->vlan_tpid;
 267                         match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
 268                         match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
 269                                                                fm.key->vlan_id);
 270                         match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
 271                                                               fm.mask->vlan_id);
 272                 }
 273         }
 274
 275         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 276                 struct flow_match_eth_addrs fm;
 277
 278                 flow_rule_match_eth_addrs(rule, &fm);
 279                 ether_addr_copy(match->value.eth_saddr, fm.key->src);
 280                 ether_addr_copy(match->value.eth_daddr, fm.key->dst);
 281                 ether_addr_copy(match->mask.eth_saddr, fm.mask->src);
 282                 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);
 283         }
 284
 285         MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
 286         /* Make sure we're TCP/UDP if any L4 keys used. */
 287         if ((match->value.ip_proto != IPPROTO_UDP &&
 288              match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
 289                 if (dissector->used_keys &
 290                     (BIT(FLOW_DISSECTOR_KEY_PORTS) |
 291                      BIT(FLOW_DISSECTOR_KEY_TCP))) {
 292                         NL_SET_ERR_MSG_FMT_MOD(extack, "L4 flower keys %#x require ipproto udp or tcp",
 293                                                dissector->used_keys);
 294                         return -EINVAL;
 295                 }
 296         MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
 297         MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
 298         if (ipv == 4) {
 299                 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
 300                 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
 301         }
 302 #ifdef CONFIG_IPV6
 303         else if (ipv == 6) {
 304                 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
 305                 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
 306         }
 307 #endif
 308         MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
 309         MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
 310         MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
 311         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 312                 struct flow_match_control fm;
 313
 314                 flow_rule_match_enc_control(rule, &fm);
 315                 if (fm.mask->flags) {
 316                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x",
 317                                                fm.mask->flags);
 318                         return -EOPNOTSUPP;
 319                 }
 320                 if (!IS_ALL_ONES(fm.mask->addr_type)) {
 321                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
 322                                                fm.mask->addr_type,
 323                                                fm.key->addr_type);
 324                         return -EOPNOTSUPP;
 325                 }
 326                 switch (fm.key->addr_type) {
 327                 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 328                         MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
 329                                              src, enc_src_ip);
 330                         MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
 331                                              dst, enc_dst_ip);
 332                         break;
 333 #ifdef CONFIG_IPV6
 334                 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 335                         MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
 336                                              src, enc_src_ip6);
 337                         MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
 338                                              dst, enc_dst_ip6);
 339                         break;
 340 #endif
 341                 default:
 342                         NL_SET_ERR_MSG_FMT_MOD(extack,
 343                                                "Unsupported enc addr_type %u (supported are IPv4, IPv6)",
 344                                                fm.key->addr_type);
 345                         return -EOPNOTSUPP;
 346                 }
 347                 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
 348                 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
 349                 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
 350                 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
 351                 MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
 352         } else if (dissector->used_keys &
 353                    (BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 354                     BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
 355                     BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 356                     BIT(FLOW_DISSECTOR_KEY_ENC_IP) |
 357                     BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
 358                 NL_SET_ERR_MSG_FMT_MOD(extack, "Flower enc keys require enc_control (keys: %#x)",
 359                                        dissector->used_keys);
 360                 return -EOPNOTSUPP;
 361         }
 362
 363         return 0;
 364 }
 365
 366 static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
 367                                             struct efx_tc_match *match,
 368                                             enum efx_encap_type type,
 369                                             struct netlink_ext_ack *extack)
 370 {
 371         struct efx_tc_encap_match *encap, *old;
 372         bool ipv6 = false;
 373         int rc;
 374
 375         /* We require that the socket-defining fields (IP addrs and UDP dest
 376          * port) are present and exact-match.  Other fields are currently not
 377          * allowed.  This meets what OVS will ask for, and means that we don't
 378          * need to handle difficult checks for overlapping matches as could
 379          * come up if we allowed masks or varying sets of match fields.
 380          */
 381         if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
 382                 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
 383                         NL_SET_ERR_MSG_MOD(extack,
 384                                            "Egress encap match is not exact on dst IP address");
 385                         return -EOPNOTSUPP;
 386                 }
 387                 if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
 388                         NL_SET_ERR_MSG_MOD(extack,
 389                                            "Egress encap match is not exact on src IP address");
 390                         return -EOPNOTSUPP;
 391                 }
 392 #ifdef CONFIG_IPV6
 393                 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||
 394                     !ipv6_addr_any(&match->mask.enc_src_ip6)) {
 395                         NL_SET_ERR_MSG_MOD(extack,
 396                                            "Egress encap match on both IPv4 and IPv6, don't understand");
 397                         return -EOPNOTSUPP;
 398                 }
 399         } else {
 400                 ipv6 = true;
 401                 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {
 402                         NL_SET_ERR_MSG_MOD(extack,
 403                                            "Egress encap match is not exact on dst IP address");
 404                         return -EOPNOTSUPP;
 405                 }
 406                 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {
 407                         NL_SET_ERR_MSG_MOD(extack,
 408                                            "Egress encap match is not exact on src IP address");
 409                         return -EOPNOTSUPP;
 410                 }
 411 #endif
 412         }
 413         if (!IS_ALL_ONES(match->mask.enc_dport)) {
 414                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
 415                 return -EOPNOTSUPP;
 416         }
 417         if (match->mask.enc_sport) {
 418                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on src UDP port not supported");
 419                 return -EOPNOTSUPP;
 420         }
 421         if (match->mask.enc_ip_tos) {
 422                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP ToS not supported");
 423                 return -EOPNOTSUPP;
 424         }
 425         if (match->mask.enc_ip_ttl) {
 426                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
 427                 return -EOPNOTSUPP;
 428         }
 429
 430         rc = efx_mae_check_encap_match_caps(efx, ipv6, extack);
 431         if (rc) {
 432                 NL_SET_ERR_MSG_FMT_MOD(extack, "MAE hw reports no support for IPv%d encap matches",
 433                                        ipv6 ? 6 : 4);
 434                 return -EOPNOTSUPP;
 435         }
 436
 437         encap = kzalloc(sizeof(*encap), GFP_USER);
 438         if (!encap)
 439                 return -ENOMEM;
 440         encap->src_ip = match->value.enc_src_ip;
 441         encap->dst_ip = match->value.enc_dst_ip;
 442 #ifdef CONFIG_IPV6
 443         encap->src_ip6 = match->value.enc_src_ip6;
 444         encap->dst_ip6 = match->value.enc_dst_ip6;
 445 #endif
 446         encap->udp_dport = match->value.enc_dport;
 447         encap->tun_type = type;
 448         old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,
 449                                                 &encap->linkage,
 450                                                 efx_tc_encap_match_ht_params);
 451         if (old) {
 452                 /* don't need our new entry */
 453                 kfree(encap);
 454                 if (old->tun_type != type) {
 455                         NL_SET_ERR_MSG_FMT_MOD(extack,
 456                                                "Egress encap match with conflicting tun_type %u != %u",
 457                                                old->tun_type, type);
 458                         return -EEXIST;
 459                 }
 460                 if (!refcount_inc_not_zero(&old->ref))
 461                         return -EAGAIN;
 462                 /* existing entry found */
 463                 encap = old;
 464         } else {
 465                 rc = efx_mae_register_encap_match(efx, encap);
 466                 if (rc) {
 467                         NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
 468                         goto fail;
 469                 }
 470                 refcount_set(&encap->ref, 1);
 471         }
 472         match->encap = encap;
 473         return 0;
 474 fail:
 475         rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
 476                                efx_tc_encap_match_ht_params);
 477         kfree(encap);
 478         return rc;
 479 }
 480
 481 static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
 482                                               struct efx_tc_encap_match *encap)
 483 {
 484         int rc;
 485
 486         if (!refcount_dec_and_test(&encap->ref))
 487                 return; /* still in use */
 488
 489         rc = efx_mae_unregister_encap_match(efx, encap);
 490         if (rc)
 491                 /* Display message but carry on and remove entry from our
 492                  * SW tables, because there's not much we can do about it.
 493                  */
 494                 netif_err(efx, drv, efx->net_dev,
 495                           "Failed to release encap match %#x, rc %d\n",
 496                           encap->fw_id, rc);
 497         rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
 498                                efx_tc_encap_match_ht_params);
 499         kfree(encap);
 500 }
 501
 502 static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
 503 {
 504         efx_mae_delete_rule(efx, rule->fw_id);
 505
 506         /* Release entries in subsidiary tables */
 507         efx_tc_free_action_set_list(efx, &rule->acts, true);
 508         if (rule->match.encap)
 509                 efx_tc_flower_release_encap_match(efx, rule->match.encap);
 510         rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
 511 }
 512
 513 static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
 514 {
 515         switch (typ) {
 516         case EFX_ENCAP_TYPE_NONE:
 517                 return "none";
 518         case EFX_ENCAP_TYPE_VXLAN:
 519                 return "vxlan";
 520         case EFX_ENCAP_TYPE_GENEVE:
 521                 return "geneve";
 522         default:
 523                 pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
 524                 return "unknown";
 525         }
 526 }
 527
 528 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
 529 enum efx_tc_action_order {
 530         EFX_TC_AO_DECAP,
 531         EFX_TC_AO_VLAN_POP,
 532         EFX_TC_AO_VLAN_PUSH,
 533         EFX_TC_AO_COUNT,
 534         EFX_TC_AO_DELIVER
 535 };
 536 /* Determine whether we can add @new action without violating order */
 537 static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
 538                                           enum efx_tc_action_order new)
 539 {
 540         switch (new) {
 541         case EFX_TC_AO_DECAP:
 542                 if (act->decap)
 543                         return false;
 544                 fallthrough;
 545         case EFX_TC_AO_VLAN_POP:
 546                 if (act->vlan_pop >= 2)
 547                         return false;
 548                 /* If we've already pushed a VLAN, we can't then pop it;
 549                  * the hardware would instead try to pop an existing VLAN
 550                  * before pushing the new one.
 551                  */
 552                 if (act->vlan_push)
 553                         return false;
 554                 fallthrough;
 555         case EFX_TC_AO_VLAN_PUSH:
 556                 if (act->vlan_push >= 2)
 557                         return false;
 558                 fallthrough;
 559         case EFX_TC_AO_COUNT:
 560                 if (act->count)
 561                         return false;
 562                 fallthrough;
 563         case EFX_TC_AO_DELIVER:
 564                 return !act->deliver;
 565         default:
 566                 /* Bad caller.  Whatever they wanted to do, say they can't. */
 567                 WARN_ON_ONCE(1);
 568                 return false;
 569         }
 570 }
 571
 572 static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
 573                                          struct net_device *net_dev,
 574                                          struct flow_cls_offload *tc)
 575 {
 576         struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
 577         struct netlink_ext_ack *extack = tc->common.extack;
 578         struct efx_tc_flow_rule *rule = NULL, *old = NULL;
 579         struct efx_tc_action_set *act = NULL;
 580         bool found = false, uplinked = false;
 581         const struct flow_action_entry *fa;
 582         struct efx_tc_match match;
 583         struct efx_rep *to_efv;
 584         s64 rc;
 585         int i;
 586
 587         /* Parse match */
 588         memset(&match, 0, sizeof(match));
 589         rc = efx_tc_flower_parse_match(efx, fr, &match, NULL);
 590         if (rc)
 591                 return rc;
 592         /* The rule as given to us doesn't specify a source netdevice.
 593          * But, determining whether packets from a VF should match it is
 594          * complicated, so leave those to the software slowpath: qualify
 595          * the filter with source m-port == wire.
 596          */
 597         rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
 598         if (rc < 0) {
 599                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
 600                 return rc;
 601         }
 602         match.value.ingress_port = rc;
 603         match.mask.ingress_port = ~0;
 604
 605         if (tc->common.chain_index) {
 606                 NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index");
 607                 return -EOPNOTSUPP;
 608         }
 609         match.mask.recirc_id = 0xff;
 610
 611         flow_action_for_each(i, fa, &fr->action) {
 612                 switch (fa->id) {
 613                 case FLOW_ACTION_REDIRECT:
 614                 case FLOW_ACTION_MIRRED: /* mirred means mirror here */
 615                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
 616                         if (IS_ERR(to_efv))
 617                                 continue;
 618                         found = true;
 619                         break;
 620                 default:
 621                         break;
 622                 }
 623         }
 624         if (!found) { /* We don't care. */
 625                 netif_dbg(efx, drv, efx->net_dev,
 626                           "Ignoring foreign filter that doesn't egdev us\n");
 627                 return -EOPNOTSUPP;
 628         }
 629
 630         rc = efx_mae_match_check_caps(efx, &match.mask, NULL);
 631         if (rc)
 632                 return rc;
 633
 634         if (efx_tc_match_is_encap(&match.mask)) {
 635                 enum efx_encap_type type;
 636
 637                 type = efx_tc_indr_netdev_type(net_dev);
 638                 if (type == EFX_ENCAP_TYPE_NONE) {
 639                         NL_SET_ERR_MSG_MOD(extack,
 640                                            "Egress encap match on unsupported tunnel device");
 641                         return -EOPNOTSUPP;
 642                 }
 643
 644                 rc = efx_mae_check_encap_type_supported(efx, type);
 645                 if (rc) {
 646                         NL_SET_ERR_MSG_FMT_MOD(extack,
 647                                                "Firmware reports no support for %s encap match",
 648                                                efx_tc_encap_type_name(type));
 649                         return rc;
 650                 }
 651
 652                 rc = efx_tc_flower_record_encap_match(efx, &match, type,
 653                                                       extack);
 654                 if (rc)
 655                         return rc;
 656         } else {
 657                 /* This is not a tunnel decap rule, ignore it */
 658                 netif_dbg(efx, drv, efx->net_dev,
 659                           "Ignoring foreign filter without encap match\n");
 660                 return -EOPNOTSUPP;
 661         }
 662
 663         rule = kzalloc(sizeof(*rule), GFP_USER);
 664         if (!rule) {
 665                 rc = -ENOMEM;
 666                 goto out_free;
 667         }
 668         INIT_LIST_HEAD(&rule->acts.list);
 669         rule->cookie = tc->cookie;
 670         old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
 671                                                 &rule->linkage,
 672                                                 efx_tc_match_action_ht_params);
 673         if (old) {
 674                 netif_dbg(efx, drv, efx->net_dev,
 675                           "Ignoring already-offloaded rule (cookie %lx)\n",
 676                           tc->cookie);
 677                 rc = -EEXIST;
 678                 goto out_free;
 679         }
 680
 681         act = kzalloc(sizeof(*act), GFP_USER);
 682         if (!act) {
 683                 rc = -ENOMEM;
 684                 goto release;
 685         }
 686
 687         /* Parse actions.  For foreign rules we only support decap & redirect.
 688          * See corresponding code in efx_tc_flower_replace() for theory of
 689          * operation & how 'act' cursor is used.
 690          */
 691         flow_action_for_each(i, fa, &fr->action) {
 692                 struct efx_tc_action_set save;
 693
 694                 switch (fa->id) {
 695                 case FLOW_ACTION_REDIRECT:
 696                 case FLOW_ACTION_MIRRED:
 697                         /* See corresponding code in efx_tc_flower_replace() for
 698                          * long explanations of what's going on here.
 699                          */
 700                         save = *act;
 701                         if (fa->hw_stats) {
 702                                 struct efx_tc_counter_index *ctr;
 703
 704                                 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
 705                                         NL_SET_ERR_MSG_FMT_MOD(extack,
 706                                                                "hw_stats_type %u not supported (only 'delayed')",
 707                                                                fa->hw_stats);
 708                                         rc = -EOPNOTSUPP;
 709                                         goto release;
 710                                 }
 711                                 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
 712                                         rc = -EOPNOTSUPP;
 713                                         goto release;
 714                                 }
 715
 716                                 ctr = efx_tc_flower_get_counter_index(efx,
 717                                                                       tc->cookie,
 718                                                                       EFX_TC_COUNTER_TYPE_AR);
 719                                 if (IS_ERR(ctr)) {
 720                                         rc = PTR_ERR(ctr);
 721                                         NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
 722                                         goto release;
 723                                 }
 724                                 act->count = ctr;
 725                         }
 726
 727                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
 728                                 /* can't happen */
 729                                 rc = -EOPNOTSUPP;
 730                                 NL_SET_ERR_MSG_MOD(extack,
 731                                                    "Deliver action violates action order (can't happen)");
 732                                 goto release;
 733                         }
 734                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
 735                         /* PF implies egdev is us, in which case we really
 736                          * want to deliver to the uplink (because this is an
 737                          * ingress filter).  If we don't recognise the egdev
 738                          * at all, then we'd better trap so SW can handle it.
 739                          */
 740                         if (IS_ERR(to_efv))
 741                                 to_efv = EFX_EFV_PF;
 742                         if (to_efv == EFX_EFV_PF) {
 743                                 if (uplinked)
 744                                         break;
 745                                 uplinked = true;
 746                         }
 747                         rc = efx_tc_flower_internal_mport(efx, to_efv);
 748                         if (rc < 0) {
 749                                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
 750                                 goto release;
 751                         }
 752                         act->dest_mport = rc;
 753                         act->deliver = 1;
 754                         rc = efx_mae_alloc_action_set(efx, act);
 755                         if (rc) {
 756                                 NL_SET_ERR_MSG_MOD(extack,
 757                                                    "Failed to write action set to hw (mirred)");
 758                                 goto release;
 759                         }
 760                         list_add_tail(&act->list, &rule->acts.list);
 761                         act = NULL;
 762                         if (fa->id == FLOW_ACTION_REDIRECT)
 763                                 break; /* end of the line */
 764                         /* Mirror, so continue on with saved act */
 765                         act = kzalloc(sizeof(*act), GFP_USER);
 766                         if (!act) {
 767                                 rc = -ENOMEM;
 768                                 goto release;
 769                         }
 770                         *act = save;
 771                         break;
 772                 case FLOW_ACTION_TUNNEL_DECAP:
 773                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {
 774                                 rc = -EINVAL;
 775                                 NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
 776                                 goto release;
 777                         }
 778                         act->decap = 1;
 779                         /* If we previously delivered/trapped to uplink, now
 780                          * that we've decapped we'll want another copy if we
 781                          * try to deliver/trap to uplink again.
 782                          */
 783                         uplinked = false;
 784                         break;
 785                 default:
 786                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
 787                                                fa->id);
 788                         rc = -EOPNOTSUPP;
 789                         goto release;
 790                 }
 791         }
 792
 793         if (act) {
 794                 if (!uplinked) {
 795                         /* Not shot/redirected, so deliver to default dest (which is
 796                          * the uplink, as this is an ingress filter)
 797                          */
 798                         efx_mae_mport_uplink(efx, &act->dest_mport);
 799                         act->deliver = 1;
 800                 }
 801                 rc = efx_mae_alloc_action_set(efx, act);
 802                 if (rc) {
 803                         NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
 804                         goto release;
 805                 }
 806                 list_add_tail(&act->list, &rule->acts.list);
 807                 act = NULL; /* Prevent double-free in error path */
 808         }
 809
 810         rule->match = match;
 811
 812         netif_dbg(efx, drv, efx->net_dev,
 813                   "Successfully parsed foreign filter (cookie %lx)\n",
 814                   tc->cookie);
 815
 816         rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
 817         if (rc) {
 818                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
 819                 goto release;
 820         }
 821         rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
 822                                  rule->acts.fw_id, &rule->fw_id);
 823         if (rc) {
 824                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
 825                 goto release_acts;
 826         }
 827         return 0;
 828
 829 release_acts:
 830         efx_mae_free_action_set_list(efx, &rule->acts);
 831 release:
 832         /* We failed to insert the rule, so free up any entries we created in
 833          * subsidiary tables.
 834          */
 835         if (act)
 836                 efx_tc_free_action_set(efx, act, false);
 837         if (rule) {
 838                 rhashtable_remove_fast(&efx->tc->match_action_ht,
 839                                        &rule->linkage,
 840                                        efx_tc_match_action_ht_params);
 841                 efx_tc_free_action_set_list(efx, &rule->acts, false);
 842         }
 843 out_free:
 844         kfree(rule);
 845         if (match.encap)
 846                 efx_tc_flower_release_encap_match(efx, match.encap);
 847         return rc;
 848 }
 849
 850 static int efx_tc_flower_replace(struct efx_nic *efx,
 851                                  struct net_device *net_dev,
 852                                  struct flow_cls_offload *tc,
 853                                  struct efx_rep *efv)
 854 {
 855         struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
 856         struct netlink_ext_ack *extack = tc->common.extack;
 857         struct efx_tc_flow_rule *rule = NULL, *old;
 858         struct efx_tc_action_set *act = NULL;
 859         const struct flow_action_entry *fa;
 860         struct efx_rep *from_efv, *to_efv;
 861         struct efx_tc_match match;
 862         s64 rc;
 863         int i;
 864
 865         if (!tc_can_offload_extack(efx->net_dev, extack))
 866                 return -EOPNOTSUPP;
 867         if (WARN_ON(!efx->tc))
 868                 return -ENETDOWN;
 869         if (WARN_ON(!efx->tc->up))
 870                 return -ENETDOWN;
 871
 872         from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
 873         if (IS_ERR(from_efv)) {
 874                 /* Not from our PF or representors, so probably a tunnel dev */
 875                 return efx_tc_flower_replace_foreign(efx, net_dev, tc);
 876         }
 877
 878         if (efv != from_efv) {
 879                 /* can't happen */
 880                 NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
 881                                        netdev_name(net_dev), efv ? "non-" : "",
 882                                        from_efv ? "non-" : "");
 883                 return -EINVAL;
 884         }
 885
 886         /* Parse match */
 887         memset(&match, 0, sizeof(match));
 888         rc = efx_tc_flower_external_mport(efx, from_efv);
 889         if (rc < 0) {
 890                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
 891                 return rc;
 892         }
 893         match.value.ingress_port = rc;
 894         match.mask.ingress_port = ~0;
 895         rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
 896         if (rc)
 897                 return rc;
 898         if (efx_tc_match_is_encap(&match.mask)) {
 899                 NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
 900                 return -EOPNOTSUPP;
 901         }
 902
 903         if (tc->common.chain_index) {
 904                 NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index");
 905                 return -EOPNOTSUPP;
 906         }
 907         match.mask.recirc_id = 0xff;
 908
 909         rc = efx_mae_match_check_caps(efx, &match.mask, extack);
 910         if (rc)
 911                 return rc;
 912
 913         rule = kzalloc(sizeof(*rule), GFP_USER);
 914         if (!rule)
 915                 return -ENOMEM;
 916         INIT_LIST_HEAD(&rule->acts.list);
 917         rule->cookie = tc->cookie;
 918         old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
 919                                                 &rule->linkage,
 920                                                 efx_tc_match_action_ht_params);
 921         if (old) {
 922                 netif_dbg(efx, drv, efx->net_dev,
 923                           "Already offloaded rule (cookie %lx)\n", tc->cookie);
 924                 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
 925                 kfree(rule);
 926                 return -EEXIST;
 927         }
 928
 929         /* Parse actions */
 930         act = kzalloc(sizeof(*act), GFP_USER);
 931         if (!act) {
 932                 rc = -ENOMEM;
 933                 goto release;
 934         }
 935
 936         /**
 937          * DOC: TC action translation
 938          *
 939          * Actions in TC are sequential and cumulative, with delivery actions
 940          * potentially anywhere in the order.  The EF100 MAE, however, takes
 941          * an 'action set list' consisting of 'action sets', each of which is
 942          * applied to the _original_ packet, and consists of a set of optional
 943          * actions in a fixed order with delivery at the end.
 944          * To translate between these two models, we maintain a 'cursor', @act,
 945          * which describes the cumulative effect of all the packet-mutating
 946          * actions encountered so far; on handling a delivery (mirred or drop)
 947          * action, once the action-set has been inserted into hardware, we
 948          * append @act to the action-set list (@rule->acts); if this is a pipe
 949          * action (mirred mirror) we then allocate a new @act with a copy of
 950          * the cursor state _before_ the delivery action, otherwise we set @act
 951          * to %NULL.
 952          * This ensures that every allocated action-set is either attached to
 953          * @rule->acts or pointed to by @act (and never both), and that only
 954          * those action-sets in @rule->acts exist in hardware.  Consequently,
 955          * in the failure path, @act only needs to be freed in memory, whereas
 956          * for @rule->acts we remove each action-set from hardware before
 957          * freeing it (efx_tc_free_action_set_list()), even if the action-set
 958          * list itself is not in hardware.
 959          */
 960         flow_action_for_each(i, fa, &fr->action) {
 961                 struct efx_tc_action_set save;
 962                 u16 tci;
 963
 964                 if (!act) {
 965                         /* more actions after a non-pipe action */
 966                         NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
 967                         rc = -EINVAL;
 968                         goto release;
 969                 }
 970
 971                 if ((fa->id == FLOW_ACTION_REDIRECT ||
 972                      fa->id == FLOW_ACTION_MIRRED ||
 973                      fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
 974                         struct efx_tc_counter_index *ctr;
 975
 976                         /* Currently the only actions that want stats are
 977                          * mirred and gact (ok, shot, trap, goto-chain), which
 978                          * means we want stats just before delivery.  Also,
 979                          * note that tunnel_key set shouldn't change the length
 980                          * — it's only the subsequent mirred that does that,
 981                          * and the stats are taken _before_ the mirred action
 982                          * happens.
 983                          */
 984                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
 985                                 /* All supported actions that count either steal
 986                                  * (gact shot, mirred redirect) or clone act
 987                                  * (mirred mirror), so we should never get two
 988                                  * count actions on one action_set.
 989                                  */
 990                                 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
 991                                 rc = -EOPNOTSUPP;
 992                                 goto release;
 993                         }
 994
 995                         if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
 996                                 NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
 997                                                        fa->hw_stats);
 998                                 rc = -EOPNOTSUPP;
 999                                 goto release;
1000                         }
1001
1002                         ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
1003                                                               EFX_TC_COUNTER_TYPE_AR);
1004                         if (IS_ERR(ctr)) {
1005                                 rc = PTR_ERR(ctr);
1006                                 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1007                                 goto release;
1008                         }
1009                         act->count = ctr;
1010                 }
1011
1012                 switch (fa->id) {
1013                 case FLOW_ACTION_DROP:
1014                         rc = efx_mae_alloc_action_set(efx, act);
1015                         if (rc) {
1016                                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
1017                                 goto release;
1018                         }
1019                         list_add_tail(&act->list, &rule->acts.list);
1020                         act = NULL; /* end of the line */
1021                         break;
1022                 case FLOW_ACTION_REDIRECT:
1023                 case FLOW_ACTION_MIRRED:
1024                         save = *act;
1025
1026                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
1027                                 /* can't happen */
1028                                 rc = -EOPNOTSUPP;
1029                                 NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
1030                                 goto release;
1031                         }
1032
1033                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1034                         if (IS_ERR(to_efv)) {
1035                                 NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
1036                                 rc = PTR_ERR(to_efv);
1037                                 goto release;
1038                         }
1039                         rc = efx_tc_flower_external_mport(efx, to_efv);
1040                         if (rc < 0) {
1041                                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
1042                                 goto release;
1043                         }
1044                         act->dest_mport = rc;
1045                         act->deliver = 1;
1046                         rc = efx_mae_alloc_action_set(efx, act);
1047                         if (rc) {
1048                                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
1049                                 goto release;
1050                         }
1051                         list_add_tail(&act->list, &rule->acts.list);
1052                         act = NULL;
1053                         if (fa->id == FLOW_ACTION_REDIRECT)
1054                                 break; /* end of the line */
1055                         /* Mirror, so continue on with saved act */
1056                         save.count = NULL;
1057                         act = kzalloc(sizeof(*act), GFP_USER);
1058                         if (!act) {
1059                                 rc = -ENOMEM;
1060                                 goto release;
1061                         }
1062                         *act = save;
1063                         break;
1064                 case FLOW_ACTION_VLAN_POP:
1065                         if (act->vlan_push) {
1066                                 act->vlan_push--;
1067                         } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {
1068                                 act->vlan_pop++;
1069                         } else {
1070                                 NL_SET_ERR_MSG_MOD(extack,
1071                                                    "More than two VLAN pops, or action order violated");
1072                                 rc = -EINVAL;
1073                                 goto release;
1074                         }
1075                         break;
1076                 case FLOW_ACTION_VLAN_PUSH:
1077                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {
1078                                 rc = -EINVAL;
1079                                 NL_SET_ERR_MSG_MOD(extack,
1080                                                    "More than two VLAN pushes, or action order violated");
1081                                 goto release;
1082                         }
1083                         tci = fa->vlan.vid & VLAN_VID_MASK;
1084                         tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
1085                         act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
1086                         act->vlan_proto[act->vlan_push] = fa->vlan.proto;
1087                         act->vlan_push++;
1088                         break;
1089                 default:
1090                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
1091                                                fa->id);
1092                         rc = -EOPNOTSUPP;
1093                         goto release;
1094                 }
1095         }
1096
1097         if (act) {
1098                 /* Not shot/redirected, so deliver to default dest */
1099                 if (from_efv == EFX_EFV_PF)
1100                         /* Rule applies to traffic from the wire,
1101                          * and default dest is thus the PF
1102                          */
1103                         efx_mae_mport_uplink(efx, &act->dest_mport);
1104                 else
1105                         /* Representor, so rule applies to traffic from
1106                          * representee, and default dest is thus the rep.
1107                          * All reps use the same mport for delivery
1108                          */
1109                         efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
1110                                             &act->dest_mport);
1111                 act->deliver = 1;
1112                 rc = efx_mae_alloc_action_set(efx, act);
1113                 if (rc) {
1114                         NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
1115                         goto release;
1116                 }
1117                 list_add_tail(&act->list, &rule->acts.list);
1118                 act = NULL; /* Prevent double-free in error path */
1119         }
1120
1121         netif_dbg(efx, drv, efx->net_dev,
1122                   "Successfully parsed filter (cookie %lx)\n",
1123                   tc->cookie);
1124
1125         rule->match = match;
1126
1127         rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
1128         if (rc) {
1129                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
1130                 goto release;
1131         }
1132         rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
1133                                  rule->acts.fw_id, &rule->fw_id);
1134         if (rc) {
1135                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1136                 goto release_acts;
1137         }
1138         return 0;
1139
1140 release_acts:
1141         efx_mae_free_action_set_list(efx, &rule->acts);
1142 release:
1143         /* We failed to insert the rule, so free up any entries we created in
1144          * subsidiary tables.
1145          */
1146         if (act)
1147                 efx_tc_free_action_set(efx, act, false);
1148         if (rule) {
1149                 rhashtable_remove_fast(&efx->tc->match_action_ht,
1150                                        &rule->linkage,
1151                                        efx_tc_match_action_ht_params);
1152                 efx_tc_free_action_set_list(efx, &rule->acts, false);
1153         }
1154         kfree(rule);
1155         return rc;
1156 }
1157
1158 static int efx_tc_flower_destroy(struct efx_nic *efx,
1159                                  struct net_device *net_dev,
1160                                  struct flow_cls_offload *tc)
1161 {
1162         struct netlink_ext_ack *extack = tc->common.extack;
1163         struct efx_tc_flow_rule *rule;
1164
1165         rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,
1166                                       efx_tc_match_action_ht_params);
1167         if (!rule) {
1168                 /* Only log a message if we're the ingress device.  Otherwise
1169                  * it's a foreign filter and we might just not have been
1170                  * interested (e.g. we might not have been the egress device
1171                  * either).
1172                  */
1173                 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
1174                         netif_warn(efx, drv, efx->net_dev,
1175                                    "Filter %lx not found to remove\n", tc->cookie);
1176                 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
1177                 return -ENOENT;
1178         }
1179
1180         /* Remove it from HW */
1181         efx_tc_delete_rule(efx, rule);
1182         /* Delete it from SW */
1183         rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,
1184                                efx_tc_match_action_ht_params);
1185         netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
1186         kfree(rule);
1187         return 0;
1188 }
1189
1190 static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
1191                                struct flow_cls_offload *tc)
1192 {
1193         struct netlink_ext_ack *extack = tc->common.extack;
1194         struct efx_tc_counter_index *ctr;
1195         struct efx_tc_counter *cnt;
1196         u64 packets, bytes;
1197
1198         ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
1199         if (!ctr) {
1200                 /* See comment in efx_tc_flower_destroy() */
1201                 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
1202                         if (net_ratelimit())
1203                                 netif_warn(efx, drv, efx->net_dev,
1204                                            "Filter %lx not found for stats\n",
1205                                            tc->cookie);
1206                 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
1207                 return -ENOENT;
1208         }
1209         if (WARN_ON(!ctr->cnt)) /* can't happen */
1210                 return -EIO;
1211         cnt = ctr->cnt;
1212
1213         spin_lock_bh(&cnt->lock);
1214         /* Report only new pkts/bytes since last time TC asked */
1215         packets = cnt->packets;
1216         bytes = cnt->bytes;
1217         flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
1218                           packets - cnt->old_packets, 0, cnt->touched,
1219                           FLOW_ACTION_HW_STATS_DELAYED);
1220         cnt->old_packets = packets;
1221         cnt->old_bytes = bytes;
1222         spin_unlock_bh(&cnt->lock);
1223         return 0;
1224 }
1225
1226 int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
1227                   struct flow_cls_offload *tc, struct efx_rep *efv)
1228 {
1229         int rc;
1230
1231         if (!efx->tc)
1232                 return -EOPNOTSUPP;
1233
1234         mutex_lock(&efx->tc->mutex);
1235         switch (tc->command) {
1236         case FLOW_CLS_REPLACE:
1237                 rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
1238                 break;
1239         case FLOW_CLS_DESTROY:
1240                 rc = efx_tc_flower_destroy(efx, net_dev, tc);
1241                 break;
1242         case FLOW_CLS_STATS:
1243                 rc = efx_tc_flower_stats(efx, net_dev, tc);
1244                 break;
1245         default:
1246                 rc = -EOPNOTSUPP;
1247                 break;
1248         }
1249         mutex_unlock(&efx->tc->mutex);
1250         return rc;
1251 }
1252
1253 static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
1254                                          u32 eg_port, struct efx_tc_flow_rule *rule)
1255 {
1256         struct efx_tc_action_set_list *acts = &rule->acts;
1257         struct efx_tc_match *match = &rule->match;
1258         struct efx_tc_action_set *act;
1259         int rc;
1260
1261         match->value.ingress_port = ing_port;
1262         match->mask.ingress_port = ~0;
1263         act = kzalloc(sizeof(*act), GFP_KERNEL);
1264         if (!act)
1265                 return -ENOMEM;
1266         act->deliver = 1;
1267         act->dest_mport = eg_port;
1268         rc = efx_mae_alloc_action_set(efx, act);
1269         if (rc)
1270                 goto fail1;
1271         EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
1272         list_add_tail(&act->list, &acts->list);
1273         rc = efx_mae_alloc_action_set_list(efx, acts);
1274         if (rc)
1275                 goto fail2;
1276         rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,
1277                                  acts->fw_id, &rule->fw_id);
1278         if (rc)
1279                 goto fail3;
1280         return 0;
1281 fail3:
1282         efx_mae_free_action_set_list(efx, acts);
1283 fail2:
1284         list_del(&act->list);
1285         efx_mae_free_action_set(efx, act->fw_id);
1286 fail1:
1287         kfree(act);
1288         return rc;
1289 }
1290
1291 static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)
1292 {
1293         struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;
1294         u32 ing_port, eg_port;
1295
1296         efx_mae_mport_uplink(efx, &ing_port);
1297         efx_mae_mport_wire(efx, &eg_port);
1298         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
1299 }
1300
1301 static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)
1302 {
1303         struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;
1304         u32 ing_port, eg_port;
1305
1306         efx_mae_mport_wire(efx, &ing_port);
1307         efx_mae_mport_uplink(efx, &eg_port);
1308         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
1309 }
1310
1311 int efx_tc_configure_default_rule_rep(struct efx_rep *efv)
1312 {
1313         struct efx_tc_flow_rule *rule = &efv->dflt;
1314         struct efx_nic *efx = efv->parent;
1315         u32 ing_port, eg_port;
1316
1317         efx_mae_mport_mport(efx, efv->mport, &ing_port);
1318         efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
1319         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
1320 }
1321
1322 void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
1323                                      struct efx_tc_flow_rule *rule)
1324 {
1325         if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)
1326                 efx_tc_delete_rule(efx, rule);
1327         rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
1328 }
1329
1330 static int efx_tc_configure_rep_mport(struct efx_nic *efx)
1331 {
1332         u32 rep_mport_label;
1333         int rc;
1334
1335         rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);
1336         if (rc)
1337                 return rc;
1338         pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",
1339                 efx->tc->reps_mport_id, rep_mport_label);
1340         /* Use mport *selector* as vport ID */
1341         efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
1342                             &efx->tc->reps_mport_vport_id);
1343         return 0;
1344 }
1345
1346 static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)
1347 {
1348         efx_mae_free_mport(efx, efx->tc->reps_mport_id);
1349         efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;
1350 }
1351
1352 int efx_tc_insert_rep_filters(struct efx_nic *efx)
1353 {
1354         struct efx_filter_spec promisc, allmulti;
1355         int rc;
1356
1357         if (efx->type->is_vf)
1358                 return 0;
1359         if (!efx->tc)
1360                 return 0;
1361         efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0);
1362         efx_filter_set_uc_def(&promisc);
1363         efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);
1364         rc = efx_filter_insert_filter(efx, &promisc, false);
1365         if (rc < 0)
1366                 return rc;
1367         efx->tc->reps_filter_uc = rc;
1368         efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0);
1369         efx_filter_set_mc_def(&allmulti);
1370         efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);
1371         rc = efx_filter_insert_filter(efx, &allmulti, false);
1372         if (rc < 0)
1373                 return rc;
1374         efx->tc->reps_filter_mc = rc;
1375         return 0;
1376 }
1377
1378 void efx_tc_remove_rep_filters(struct efx_nic *efx)
1379 {
1380         if (efx->type->is_vf)
1381                 return;
1382         if (!efx->tc)
1383                 return;
1384         if (efx->tc->reps_filter_mc >= 0)
1385                 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);
1386         efx->tc->reps_filter_mc = -1;
1387         if (efx->tc->reps_filter_uc >= 0)
1388                 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);
1389         efx->tc->reps_filter_uc = -1;
1390 }
1391
1392 int efx_init_tc(struct efx_nic *efx)
1393 {
1394         int rc;
1395
1396         rc = efx_mae_get_caps(efx, efx->tc->caps);
1397         if (rc)
1398                 return rc;
1399         if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
1400                 /* Firmware supports some match fields the driver doesn't know
1401                  * about.  Not fatal, unless any of those fields are required
1402                  * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
1403                  */
1404                 netif_warn(efx, probe, efx->net_dev,
1405                            "FW reports additional match fields %u\n",
1406                            efx->tc->caps->match_field_count);
1407         if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
1408                 netif_err(efx, probe, efx->net_dev,
1409                           "Too few action prios supported (have %u, need %u)\n",
1410                           efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
1411                 return -EIO;
1412         }
1413         rc = efx_tc_configure_default_rule_pf(efx);
1414         if (rc)
1415                 return rc;
1416         rc = efx_tc_configure_default_rule_wire(efx);
1417         if (rc)
1418                 return rc;
1419         rc = efx_tc_configure_rep_mport(efx);
1420         if (rc)
1421                 return rc;
1422         efx->tc->up = true;
1423         rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);
1424         if (rc)
1425                 return rc;
1426         return 0;
1427 }
1428
1429 void efx_fini_tc(struct efx_nic *efx)
1430 {
1431         /* We can get called even if efx_init_struct_tc() failed */
1432         if (!efx->tc)
1433                 return;
1434         if (efx->tc->up)
1435                 flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);
1436         efx_tc_deconfigure_rep_mport(efx);
1437         efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);
1438         efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);
1439         efx->tc->up = false;
1440 }
1441
1442 /* At teardown time, all TC filter rules (and thus all resources they created)
1443  * should already have been removed.  If we find any in our hashtables, make a
1444  * cursory attempt to clean up the software side.
1445  */
1446 static void efx_tc_encap_match_free(void *ptr, void *__unused)
1447 {
1448         struct efx_tc_encap_match *encap = ptr;
1449
1450         WARN_ON(refcount_read(&encap->ref));
1451         kfree(encap);
1452 }
1453
1454 int efx_init_struct_tc(struct efx_nic *efx)
1455 {
1456         int rc;
1457
1458         if (efx->type->is_vf)
1459                 return 0;
1460
1461         efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);
1462         if (!efx->tc)
1463                 return -ENOMEM;
1464         efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);
1465         if (!efx->tc->caps) {
1466                 rc = -ENOMEM;
1467                 goto fail_alloc_caps;
1468         }
1469         INIT_LIST_HEAD(&efx->tc->block_list);
1470
1471         mutex_init(&efx->tc->mutex);
1472         init_waitqueue_head(&efx->tc->flush_wq);
1473         rc = efx_tc_init_counters(efx);
1474         if (rc < 0)
1475                 goto fail_counters;
1476         rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);
1477         if (rc < 0)
1478                 goto fail_encap_match_ht;
1479         rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
1480         if (rc < 0)
1481                 goto fail_match_action_ht;
1482         efx->tc->reps_filter_uc = -1;
1483         efx->tc->reps_filter_mc = -1;
1484         INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);
1485         efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
1486         INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
1487         efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
1488         efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
1489         return 0;
1490 fail_match_action_ht:
1491         rhashtable_destroy(&efx->tc->encap_match_ht);
1492 fail_encap_match_ht:
1493         efx_tc_destroy_counters(efx);
1494 fail_counters:
1495         mutex_destroy(&efx->tc->mutex);
1496         kfree(efx->tc->caps);
1497 fail_alloc_caps:
1498         kfree(efx->tc);
1499         efx->tc = NULL;
1500         return rc;
1501 }
1502
1503 void efx_fini_struct_tc(struct efx_nic *efx)
1504 {
1505         if (!efx->tc)
1506                 return;
1507
1508         mutex_lock(&efx->tc->mutex);
1509         EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
1510                              MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
1511         EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
1512                              MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
1513         rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
1514                                     efx);
1515         rhashtable_free_and_destroy(&efx->tc->encap_match_ht,
1516                                     efx_tc_encap_match_free, NULL);
1517         efx_tc_fini_counters(efx);
1518         mutex_unlock(&efx->tc->mutex);
1519         mutex_destroy(&efx->tc->mutex);
1520         kfree(efx->tc->caps);
1521         kfree(efx->tc);
1522         efx->tc = NULL;
1523 }