drivers/net/ethernet/netronome/nfp/flower/qos_conf.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2019 Netronome Systems, Inc. */
   3
   4 #include <linux/hash.h>
   5 #include <linux/hashtable.h>
   6 #include <linux/jhash.h>
   7 #include <linux/math64.h>
   8 #include <linux/vmalloc.h>
   9 #include <net/pkt_cls.h>
  10 #include <net/pkt_sched.h>
  11
  12 #include "cmsg.h"
  13 #include "main.h"
  14 #include "../nfp_port.h"
  15
  16 #define NFP_FL_QOS_UPDATE               msecs_to_jiffies(1000)
  17 #define NFP_FL_QOS_PPS  BIT(15)
  18 #define NFP_FL_QOS_METER  BIT(10)
  19
  20 struct nfp_police_cfg_head {
  21         __be32 flags_opts;
  22         union {
  23                 __be32 meter_id;
  24                 __be32 port;
  25         };
  26 };
  27
  28 enum NFP_FL_QOS_TYPES {
  29         NFP_FL_QOS_TYPE_BPS,
  30         NFP_FL_QOS_TYPE_PPS,
  31         NFP_FL_QOS_TYPE_MAX,
  32 };
  33
  34 /* Police cmsg for configuring a trTCM traffic conditioner (8W/32B)
  35  * See RFC 2698 for more details.
  36  * ----------------------------------------------------------------
  37  *    3                   2                   1
  38  *  1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
  39  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  40  * |             Reserved          |p|         Reserved            |
  41  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  42  * |                          Port Ingress                         |
  43  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  44  * |                        Token Bucket Peak                      |
  45  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  46  * |                     Token Bucket Committed                    |
  47  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  48  * |                         Peak Burst Size                       |
  49  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  50  * |                      Committed Burst Size                     |
  51  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  52  * |                      Peak Information Rate                    |
  53  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  54  * |                    Committed Information Rate                 |
  55  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  56  * Word[0](FLag options):
  57  * [15] p(pps) 1 for pps, 0 for bps
  58  *
  59  * Meter control message
  60  *  1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
  61  * +-------------------------------+-+---+-----+-+---------+-+---+-+
  62  * |            Reserved           |p| Y |TYPE |E|TSHFV    |P| PC|R|
  63  * +-------------------------------+-+---+-----+-+---------+-+---+-+
  64  * |                            meter ID                           |
  65  * +-------------------------------+-------------------------------+
  66  *
  67  */
  68 struct nfp_police_config {
  69         struct nfp_police_cfg_head head;
  70         __be32 bkt_tkn_p;
  71         __be32 bkt_tkn_c;
  72         __be32 pbs;
  73         __be32 cbs;
  74         __be32 pir;
  75         __be32 cir;
  76 };
  77
  78 struct nfp_police_stats_reply {
  79         struct nfp_police_cfg_head head;
  80         __be64 pass_bytes;
  81         __be64 pass_pkts;
  82         __be64 drop_bytes;
  83         __be64 drop_pkts;
  84 };
  85
  86 int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress,
  87                                   bool pps, u32 id, u32 rate, u32 burst)
  88 {
  89         struct nfp_police_config *config;
  90         struct sk_buff *skb;
  91
  92         skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config),
  93                                     NFP_FLOWER_CMSG_TYPE_QOS_MOD, GFP_KERNEL);
  94         if (!skb)
  95                 return -ENOMEM;
  96
  97         config = nfp_flower_cmsg_get_data(skb);
  98         memset(config, 0, sizeof(struct nfp_police_config));
  99         if (pps)
 100                 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS);
 101         if (!ingress)
 102                 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_METER);
 103
 104         if (ingress)
 105                 config->head.port = cpu_to_be32(id);
 106         else
 107                 config->head.meter_id = cpu_to_be32(id);
 108
 109         config->bkt_tkn_p = cpu_to_be32(burst);
 110         config->bkt_tkn_c = cpu_to_be32(burst);
 111         config->pbs = cpu_to_be32(burst);
 112         config->cbs = cpu_to_be32(burst);
 113         config->pir = cpu_to_be32(rate);
 114         config->cir = cpu_to_be32(rate);
 115         nfp_ctrl_tx(app->ctrl, skb);
 116
 117         return 0;
 118 }
 119
 120 static int nfp_policer_validate(const struct flow_action *action,
 121                                 const struct flow_action_entry *act,
 122                                 struct netlink_ext_ack *extack,
 123                                 bool ingress)
 124 {
 125         if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
 126                 NL_SET_ERR_MSG_MOD(extack,
 127                                    "Offload not supported when exceed action is not drop");
 128                 return -EOPNOTSUPP;
 129         }
 130
 131         if (ingress) {
 132                 if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE &&
 133                     act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
 134                         NL_SET_ERR_MSG_MOD(extack,
 135                                            "Offload not supported when conform action is not continue or ok");
 136                         return -EOPNOTSUPP;
 137                 }
 138         } else {
 139                 if (act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
 140                     act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
 141                         NL_SET_ERR_MSG_MOD(extack,
 142                                            "Offload not supported when conform action is not pipe or ok");
 143                         return -EOPNOTSUPP;
 144                 }
 145         }
 146
 147         if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
 148             !flow_action_is_last_entry(action, act)) {
 149                 NL_SET_ERR_MSG_MOD(extack,
 150                                    "Offload not supported when conform action is ok, but action is not last");
 151                 return -EOPNOTSUPP;
 152         }
 153
 154         if (act->police.peakrate_bytes_ps ||
 155             act->police.avrate || act->police.overhead) {
 156                 NL_SET_ERR_MSG_MOD(extack,
 157                                    "Offload not supported when peakrate/avrate/overhead is configured");
 158                 return -EOPNOTSUPP;
 159         }
 160
 161         return 0;
 162 }
 163
 164 static int
 165 nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev,
 166                                 struct tc_cls_matchall_offload *flow,
 167                                 struct netlink_ext_ack *extack)
 168 {
 169         struct flow_action_entry *paction = &flow->rule->action.entries[0];
 170         u32 action_num = flow->rule->action.num_entries;
 171         struct nfp_flower_priv *fl_priv = app->priv;
 172         struct flow_action_entry *action = NULL;
 173         struct nfp_flower_repr_priv *repr_priv;
 174         u32 netdev_port_id, i;
 175         struct nfp_repr *repr;
 176         bool pps_support;
 177         u32 bps_num = 0;
 178         u32 pps_num = 0;
 179         u32 burst;
 180         bool pps;
 181         u64 rate;
 182         int err;
 183
 184         if (!nfp_netdev_is_nfp_repr(netdev)) {
 185                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
 186                 return -EOPNOTSUPP;
 187         }
 188         repr = netdev_priv(netdev);
 189         repr_priv = repr->app_priv;
 190         netdev_port_id = nfp_repr_get_port_id(netdev);
 191         pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
 192
 193         if (repr_priv->block_shared) {
 194                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on shared blocks");
 195                 return -EOPNOTSUPP;
 196         }
 197
 198         if (repr->port->type != NFP_PORT_VF_PORT) {
 199                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on non-VF ports");
 200                 return -EOPNOTSUPP;
 201         }
 202
 203         if (pps_support) {
 204                 if (action_num > 2 || action_num == 0) {
 205                         NL_SET_ERR_MSG_MOD(extack,
 206                                            "unsupported offload: qos rate limit offload only support action number 1 or 2");
 207                         return -EOPNOTSUPP;
 208                 }
 209         } else {
 210                 if (!flow_offload_has_one_action(&flow->rule->action)) {
 211                         NL_SET_ERR_MSG_MOD(extack,
 212                                            "unsupported offload: qos rate limit offload requires a single action");
 213                         return -EOPNOTSUPP;
 214                 }
 215         }
 216
 217         if (flow->common.prio != 1) {
 218                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires highest priority");
 219                 return -EOPNOTSUPP;
 220         }
 221
 222         for (i = 0 ; i < action_num; i++) {
 223                 action = paction + i;
 224                 if (action->id != FLOW_ACTION_POLICE) {
 225                         NL_SET_ERR_MSG_MOD(extack,
 226                                            "unsupported offload: qos rate limit offload requires police action");
 227                         return -EOPNOTSUPP;
 228                 }
 229
 230                 err = nfp_policer_validate(&flow->rule->action, action, extack, true);
 231                 if (err)
 232                         return err;
 233
 234                 if (action->police.rate_bytes_ps > 0) {
 235                         if (bps_num++) {
 236                                 NL_SET_ERR_MSG_MOD(extack,
 237                                                    "unsupported offload: qos rate limit offload only support one BPS action");
 238                                 return -EOPNOTSUPP;
 239                         }
 240                 }
 241                 if (action->police.rate_pkt_ps > 0) {
 242                         if (!pps_support) {
 243                                 NL_SET_ERR_MSG_MOD(extack,
 244                                                    "unsupported offload: FW does not support PPS action");
 245                                 return -EOPNOTSUPP;
 246                         }
 247                         if (pps_num++) {
 248                                 NL_SET_ERR_MSG_MOD(extack,
 249                                                    "unsupported offload: qos rate limit offload only support one PPS action");
 250                                 return -EOPNOTSUPP;
 251                         }
 252                 }
 253         }
 254
 255         for (i = 0 ; i < action_num; i++) {
 256                 /* Set QoS data for this interface */
 257                 action = paction + i;
 258                 if (action->police.rate_bytes_ps > 0) {
 259                         rate = action->police.rate_bytes_ps;
 260                         burst = action->police.burst;
 261                 } else if (action->police.rate_pkt_ps > 0) {
 262                         rate = action->police.rate_pkt_ps;
 263                         burst = action->police.burst_pkt;
 264                 } else {
 265                         NL_SET_ERR_MSG_MOD(extack,
 266                                            "unsupported offload: qos rate limit is not BPS or PPS");
 267                         continue;
 268                 }
 269
 270                 if (rate != 0) {
 271                         pps = false;
 272                         if (action->police.rate_pkt_ps > 0)
 273                                 pps = true;
 274                         nfp_flower_offload_one_police(repr->app, true,
 275                                                       pps, netdev_port_id,
 276                                                       rate, burst);
 277                 }
 278         }
 279         repr_priv->qos_table.netdev_port_id = netdev_port_id;
 280         fl_priv->qos_rate_limiters++;
 281         if (fl_priv->qos_rate_limiters == 1)
 282                 schedule_delayed_work(&fl_priv->qos_stats_work,
 283                                       NFP_FL_QOS_UPDATE);
 284
 285         return 0;
 286 }
 287
 288 static int
 289 nfp_flower_remove_rate_limiter(struct nfp_app *app, struct net_device *netdev,
 290                                struct tc_cls_matchall_offload *flow,
 291                                struct netlink_ext_ack *extack)
 292 {
 293         struct nfp_flower_priv *fl_priv = app->priv;
 294         struct nfp_flower_repr_priv *repr_priv;
 295         struct nfp_police_config *config;
 296         u32 netdev_port_id, i;
 297         struct nfp_repr *repr;
 298         struct sk_buff *skb;
 299         bool pps_support;
 300
 301         if (!nfp_netdev_is_nfp_repr(netdev)) {
 302                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
 303                 return -EOPNOTSUPP;
 304         }
 305         repr = netdev_priv(netdev);
 306
 307         netdev_port_id = nfp_repr_get_port_id(netdev);
 308         repr_priv = repr->app_priv;
 309         pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
 310
 311         if (!repr_priv->qos_table.netdev_port_id) {
 312                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot remove qos entry that does not exist");
 313                 return -EOPNOTSUPP;
 314         }
 315
 316         memset(&repr_priv->qos_table, 0, sizeof(struct nfp_fl_qos));
 317         fl_priv->qos_rate_limiters--;
 318         if (!fl_priv->qos_rate_limiters)
 319                 cancel_delayed_work_sync(&fl_priv->qos_stats_work);
 320         for (i = 0 ; i < NFP_FL_QOS_TYPE_MAX; i++) {
 321                 if (i == NFP_FL_QOS_TYPE_PPS && !pps_support)
 322                         break;
 323                 /* 0:bps 1:pps
 324                  * Clear QoS data for this interface.
 325                  * There is no need to check if a specific QOS_TYPE was
 326                  * configured as the firmware handles clearing a QoS entry
 327                  * safely, even if it wasn't explicitly added.
 328                  */
 329                 skb = nfp_flower_cmsg_alloc(repr->app, sizeof(struct nfp_police_config),
 330                                             NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL);
 331                 if (!skb)
 332                         return -ENOMEM;
 333
 334                 config = nfp_flower_cmsg_get_data(skb);
 335                 memset(config, 0, sizeof(struct nfp_police_config));
 336                 if (i == NFP_FL_QOS_TYPE_PPS)
 337                         config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_PPS);
 338                 config->head.port = cpu_to_be32(netdev_port_id);
 339                 nfp_ctrl_tx(repr->app->ctrl, skb);
 340         }
 341
 342         return 0;
 343 }
 344
 345 void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb)
 346 {
 347         struct nfp_flower_priv *fl_priv = app->priv;
 348         struct nfp_flower_repr_priv *repr_priv;
 349         struct nfp_police_stats_reply *msg;
 350         struct nfp_stat_pair *curr_stats;
 351         struct nfp_stat_pair *prev_stats;
 352         struct net_device *netdev;
 353         struct nfp_repr *repr;
 354         u32 netdev_port_id;
 355
 356         msg = nfp_flower_cmsg_get_data(skb);
 357         if (be32_to_cpu(msg->head.flags_opts) & NFP_FL_QOS_METER)
 358                 return nfp_act_stats_reply(app, msg);
 359
 360         netdev_port_id = be32_to_cpu(msg->head.port);
 361         rcu_read_lock();
 362         netdev = nfp_app_dev_get(app, netdev_port_id, NULL);
 363         if (!netdev)
 364                 goto exit_unlock_rcu;
 365
 366         repr = netdev_priv(netdev);
 367         repr_priv = repr->app_priv;
 368         curr_stats = &repr_priv->qos_table.curr_stats;
 369         prev_stats = &repr_priv->qos_table.prev_stats;
 370
 371         spin_lock_bh(&fl_priv->qos_stats_lock);
 372         curr_stats->pkts = be64_to_cpu(msg->pass_pkts) +
 373                            be64_to_cpu(msg->drop_pkts);
 374         curr_stats->bytes = be64_to_cpu(msg->pass_bytes) +
 375                             be64_to_cpu(msg->drop_bytes);
 376
 377         if (!repr_priv->qos_table.last_update) {
 378                 prev_stats->pkts = curr_stats->pkts;
 379                 prev_stats->bytes = curr_stats->bytes;
 380         }
 381
 382         repr_priv->qos_table.last_update = jiffies;
 383         spin_unlock_bh(&fl_priv->qos_stats_lock);
 384
 385 exit_unlock_rcu:
 386         rcu_read_unlock();
 387 }
 388
 389 static void
 390 nfp_flower_stats_rlim_request(struct nfp_flower_priv *fl_priv,
 391                               u32 id, bool ingress)
 392 {
 393         struct nfp_police_cfg_head *head;
 394         struct sk_buff *skb;
 395
 396         skb = nfp_flower_cmsg_alloc(fl_priv->app,
 397                                     sizeof(struct nfp_police_cfg_head),
 398                                     NFP_FLOWER_CMSG_TYPE_QOS_STATS,
 399                                     GFP_ATOMIC);
 400         if (!skb)
 401                 return;
 402         head = nfp_flower_cmsg_get_data(skb);
 403
 404         memset(head, 0, sizeof(struct nfp_police_cfg_head));
 405         if (ingress) {
 406                 head->port = cpu_to_be32(id);
 407         } else {
 408                 head->flags_opts = cpu_to_be32(NFP_FL_QOS_METER);
 409                 head->meter_id = cpu_to_be32(id);
 410         }
 411
 412         nfp_ctrl_tx(fl_priv->app->ctrl, skb);
 413 }
 414
 415 static void
 416 nfp_flower_stats_rlim_request_all(struct nfp_flower_priv *fl_priv)
 417 {
 418         struct nfp_reprs *repr_set;
 419         int i;
 420
 421         rcu_read_lock();
 422         repr_set = rcu_dereference(fl_priv->app->reprs[NFP_REPR_TYPE_VF]);
 423         if (!repr_set)
 424                 goto exit_unlock_rcu;
 425
 426         for (i = 0; i < repr_set->num_reprs; i++) {
 427                 struct net_device *netdev;
 428
 429                 netdev = rcu_dereference(repr_set->reprs[i]);
 430                 if (netdev) {
 431                         struct nfp_repr *priv = netdev_priv(netdev);
 432                         struct nfp_flower_repr_priv *repr_priv;
 433                         u32 netdev_port_id;
 434
 435                         repr_priv = priv->app_priv;
 436                         netdev_port_id = repr_priv->qos_table.netdev_port_id;
 437                         if (!netdev_port_id)
 438                                 continue;
 439
 440                         nfp_flower_stats_rlim_request(fl_priv,
 441                                                       netdev_port_id, true);
 442                 }
 443         }
 444
 445 exit_unlock_rcu:
 446         rcu_read_unlock();
 447 }
 448
 449 static void update_stats_cache(struct work_struct *work)
 450 {
 451         struct delayed_work *delayed_work;
 452         struct nfp_flower_priv *fl_priv;
 453
 454         delayed_work = to_delayed_work(work);
 455         fl_priv = container_of(delayed_work, struct nfp_flower_priv,
 456                                qos_stats_work);
 457
 458         nfp_flower_stats_rlim_request_all(fl_priv);
 459         nfp_flower_stats_meter_request_all(fl_priv);
 460
 461         schedule_delayed_work(&fl_priv->qos_stats_work, NFP_FL_QOS_UPDATE);
 462 }
 463
 464 static int
 465 nfp_flower_stats_rate_limiter(struct nfp_app *app, struct net_device *netdev,
 466                               struct tc_cls_matchall_offload *flow,
 467                               struct netlink_ext_ack *extack)
 468 {
 469         struct nfp_flower_priv *fl_priv = app->priv;
 470         struct nfp_flower_repr_priv *repr_priv;
 471         struct nfp_stat_pair *curr_stats;
 472         struct nfp_stat_pair *prev_stats;
 473         u64 diff_bytes, diff_pkts;
 474         struct nfp_repr *repr;
 475
 476         if (!nfp_netdev_is_nfp_repr(netdev)) {
 477                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port");
 478                 return -EOPNOTSUPP;
 479         }
 480         repr = netdev_priv(netdev);
 481
 482         repr_priv = repr->app_priv;
 483         if (!repr_priv->qos_table.netdev_port_id) {
 484                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot find qos entry for stats update");
 485                 return -EOPNOTSUPP;
 486         }
 487
 488         spin_lock_bh(&fl_priv->qos_stats_lock);
 489         curr_stats = &repr_priv->qos_table.curr_stats;
 490         prev_stats = &repr_priv->qos_table.prev_stats;
 491         diff_pkts = curr_stats->pkts - prev_stats->pkts;
 492         diff_bytes = curr_stats->bytes - prev_stats->bytes;
 493         prev_stats->pkts = curr_stats->pkts;
 494         prev_stats->bytes = curr_stats->bytes;
 495         spin_unlock_bh(&fl_priv->qos_stats_lock);
 496
 497         flow_stats_update(&flow->stats, diff_bytes, diff_pkts, 0,
 498                           repr_priv->qos_table.last_update,
 499                           FLOW_ACTION_HW_STATS_DELAYED);
 500         return 0;
 501 }
 502
 503 void nfp_flower_qos_init(struct nfp_app *app)
 504 {
 505         struct nfp_flower_priv *fl_priv = app->priv;
 506
 507         spin_lock_init(&fl_priv->qos_stats_lock);
 508         mutex_init(&fl_priv->meter_stats_lock);
 509         nfp_init_meter_table(app);
 510
 511         INIT_DELAYED_WORK(&fl_priv->qos_stats_work, &update_stats_cache);
 512 }
 513
 514 void nfp_flower_qos_cleanup(struct nfp_app *app)
 515 {
 516         struct nfp_flower_priv *fl_priv = app->priv;
 517
 518         cancel_delayed_work_sync(&fl_priv->qos_stats_work);
 519 }
 520
 521 int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev,
 522                                  struct tc_cls_matchall_offload *flow)
 523 {
 524         struct netlink_ext_ack *extack = flow->common.extack;
 525         struct nfp_flower_priv *fl_priv = app->priv;
 526         int ret;
 527
 528         if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)) {
 529                 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: loaded firmware does not support qos rate limit offload");
 530                 return -EOPNOTSUPP;
 531         }
 532
 533         mutex_lock(&fl_priv->nfp_fl_lock);
 534         switch (flow->command) {
 535         case TC_CLSMATCHALL_REPLACE:
 536                 ret = nfp_flower_install_rate_limiter(app, netdev, flow, extack);
 537                 break;
 538         case TC_CLSMATCHALL_DESTROY:
 539                 ret = nfp_flower_remove_rate_limiter(app, netdev, flow, extack);
 540                 break;
 541         case TC_CLSMATCHALL_STATS:
 542                 ret = nfp_flower_stats_rate_limiter(app, netdev, flow, extack);
 543                 break;
 544         default:
 545                 ret = -EOPNOTSUPP;
 546                 break;
 547         }
 548         mutex_unlock(&fl_priv->nfp_fl_lock);
 549
 550         return ret;
 551 }
 552
 553 /* Offload tc action, currently only for tc police */
 554
 555 static const struct rhashtable_params stats_meter_table_params = {
 556         .key_offset     = offsetof(struct nfp_meter_entry, meter_id),
 557         .head_offset    = offsetof(struct nfp_meter_entry, ht_node),
 558         .key_len        = sizeof(u32),
 559 };
 560
 561 struct nfp_meter_entry *
 562 nfp_flower_search_meter_entry(struct nfp_app *app, u32 meter_id)
 563 {
 564         struct nfp_flower_priv *priv = app->priv;
 565
 566         return rhashtable_lookup_fast(&priv->meter_table, &meter_id,
 567                                       stats_meter_table_params);
 568 }
 569
 570 static struct nfp_meter_entry *
 571 nfp_flower_add_meter_entry(struct nfp_app *app, u32 meter_id)
 572 {
 573         struct nfp_meter_entry *meter_entry = NULL;
 574         struct nfp_flower_priv *priv = app->priv;
 575
 576         meter_entry = rhashtable_lookup_fast(&priv->meter_table,
 577                                              &meter_id,
 578                                              stats_meter_table_params);
 579         if (meter_entry)
 580                 return meter_entry;
 581
 582         meter_entry = kzalloc(sizeof(*meter_entry), GFP_KERNEL);
 583         if (!meter_entry)
 584                 return NULL;
 585
 586         meter_entry->meter_id = meter_id;
 587         meter_entry->used = jiffies;
 588         if (rhashtable_insert_fast(&priv->meter_table, &meter_entry->ht_node,
 589                                    stats_meter_table_params)) {
 590                 kfree(meter_entry);
 591                 return NULL;
 592         }
 593
 594         priv->qos_rate_limiters++;
 595         if (priv->qos_rate_limiters == 1)
 596                 schedule_delayed_work(&priv->qos_stats_work,
 597                                       NFP_FL_QOS_UPDATE);
 598
 599         return meter_entry;
 600 }
 601
 602 static void nfp_flower_del_meter_entry(struct nfp_app *app, u32 meter_id)
 603 {
 604         struct nfp_meter_entry *meter_entry = NULL;
 605         struct nfp_flower_priv *priv = app->priv;
 606
 607         meter_entry = rhashtable_lookup_fast(&priv->meter_table, &meter_id,
 608                                              stats_meter_table_params);
 609         if (!meter_entry)
 610                 return;
 611
 612         rhashtable_remove_fast(&priv->meter_table,
 613                                &meter_entry->ht_node,
 614                                stats_meter_table_params);
 615         kfree(meter_entry);
 616         priv->qos_rate_limiters--;
 617         if (!priv->qos_rate_limiters)
 618                 cancel_delayed_work_sync(&priv->qos_stats_work);
 619 }
 620
 621 int nfp_flower_setup_meter_entry(struct nfp_app *app,
 622                                  const struct flow_action_entry *action,
 623                                  enum nfp_meter_op op,
 624                                  u32 meter_id)
 625 {
 626         struct nfp_flower_priv *fl_priv = app->priv;
 627         struct nfp_meter_entry *meter_entry = NULL;
 628         int err = 0;
 629
 630         mutex_lock(&fl_priv->meter_stats_lock);
 631
 632         switch (op) {
 633         case NFP_METER_DEL:
 634                 nfp_flower_del_meter_entry(app, meter_id);
 635                 goto exit_unlock;
 636         case NFP_METER_ADD:
 637                 meter_entry = nfp_flower_add_meter_entry(app, meter_id);
 638                 break;
 639         default:
 640                 err = -EOPNOTSUPP;
 641                 goto exit_unlock;
 642         }
 643
 644         if (!meter_entry) {
 645                 err = -ENOMEM;
 646                 goto exit_unlock;
 647         }
 648
 649         if (action->police.rate_bytes_ps > 0) {
 650                 meter_entry->bps = true;
 651                 meter_entry->rate = action->police.rate_bytes_ps;
 652                 meter_entry->burst = action->police.burst;
 653         } else {
 654                 meter_entry->bps = false;
 655                 meter_entry->rate = action->police.rate_pkt_ps;
 656                 meter_entry->burst = action->police.burst_pkt;
 657         }
 658
 659 exit_unlock:
 660         mutex_unlock(&fl_priv->meter_stats_lock);
 661         return err;
 662 }
 663
 664 int nfp_init_meter_table(struct nfp_app *app)
 665 {
 666         struct nfp_flower_priv *priv = app->priv;
 667
 668         return rhashtable_init(&priv->meter_table, &stats_meter_table_params);
 669 }
 670
 671 void
 672 nfp_flower_stats_meter_request_all(struct nfp_flower_priv *fl_priv)
 673 {
 674         struct nfp_meter_entry *meter_entry = NULL;
 675         struct rhashtable_iter iter;
 676
 677         mutex_lock(&fl_priv->meter_stats_lock);
 678         rhashtable_walk_enter(&fl_priv->meter_table, &iter);
 679         rhashtable_walk_start(&iter);
 680
 681         while ((meter_entry = rhashtable_walk_next(&iter)) != NULL) {
 682                 if (IS_ERR(meter_entry))
 683                         continue;
 684                 nfp_flower_stats_rlim_request(fl_priv,
 685                                               meter_entry->meter_id, false);
 686         }
 687
 688         rhashtable_walk_stop(&iter);
 689         rhashtable_walk_exit(&iter);
 690         mutex_unlock(&fl_priv->meter_stats_lock);
 691 }
 692
 693 static int
 694 nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
 695                         struct netlink_ext_ack *extack)
 696 {
 697         struct flow_action_entry *paction = &fl_act->action.entries[0];
 698         u32 action_num = fl_act->action.num_entries;
 699         struct nfp_flower_priv *fl_priv = app->priv;
 700         struct flow_action_entry *action = NULL;
 701         u32 burst, i, meter_id;
 702         bool pps_support, pps;
 703         bool add = false;
 704         u64 rate;
 705         int err;
 706
 707         pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
 708
 709         for (i = 0 ; i < action_num; i++) {
 710                 /* Set qos associate data for this interface */
 711                 action = paction + i;
 712                 if (action->id != FLOW_ACTION_POLICE) {
 713                         NL_SET_ERR_MSG_MOD(extack,
 714                                            "unsupported offload: qos rate limit offload requires police action");
 715                         continue;
 716                 }
 717
 718                 err = nfp_policer_validate(&fl_act->action, action, extack, false);
 719                 if (err)
 720                         return err;
 721
 722                 if (action->police.rate_bytes_ps > 0) {
 723                         rate = action->police.rate_bytes_ps;
 724                         burst = action->police.burst;
 725                 } else if (action->police.rate_pkt_ps > 0 && pps_support) {
 726                         rate = action->police.rate_pkt_ps;
 727                         burst = action->police.burst_pkt;
 728                 } else {
 729                         NL_SET_ERR_MSG_MOD(extack,
 730                                            "unsupported offload: unsupported qos rate limit");
 731                         continue;
 732                 }
 733
 734                 if (rate != 0) {
 735                         meter_id = action->hw_index;
 736                         if (nfp_flower_setup_meter_entry(app, action, NFP_METER_ADD, meter_id))
 737                                 continue;
 738
 739                         pps = false;
 740                         if (action->police.rate_pkt_ps > 0)
 741                                 pps = true;
 742                         nfp_flower_offload_one_police(app, false, pps, meter_id,
 743                                                       rate, burst);
 744                         add = true;
 745                 }
 746         }
 747
 748         return add ? 0 : -EOPNOTSUPP;
 749 }
 750
 751 static int
 752 nfp_act_remove_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
 753                        struct netlink_ext_ack *extack)
 754 {
 755         struct nfp_meter_entry *meter_entry = NULL;
 756         struct nfp_police_config *config;
 757         struct sk_buff *skb;
 758         u32 meter_id;
 759         bool pps;
 760
 761         /* Delete qos associate data for this interface */
 762         if (fl_act->id != FLOW_ACTION_POLICE) {
 763                 NL_SET_ERR_MSG_MOD(extack,
 764                                    "unsupported offload: qos rate limit offload requires police action");
 765                 return -EOPNOTSUPP;
 766         }
 767
 768         meter_id = fl_act->index;
 769         meter_entry = nfp_flower_search_meter_entry(app, meter_id);
 770         if (!meter_entry) {
 771                 NL_SET_ERR_MSG_MOD(extack,
 772                                    "no meter entry when delete the action index.");
 773                 return -ENOENT;
 774         }
 775         pps = !meter_entry->bps;
 776
 777         skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config),
 778                                     NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL);
 779         if (!skb)
 780                 return -ENOMEM;
 781
 782         config = nfp_flower_cmsg_get_data(skb);
 783         memset(config, 0, sizeof(struct nfp_police_config));
 784         config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_METER);
 785         config->head.meter_id = cpu_to_be32(meter_id);
 786         if (pps)
 787                 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS);
 788
 789         nfp_ctrl_tx(app->ctrl, skb);
 790         nfp_flower_setup_meter_entry(app, NULL, NFP_METER_DEL, meter_id);
 791
 792         return 0;
 793 }
 794
 795 void
 796 nfp_act_stats_reply(struct nfp_app *app, void *pmsg)
 797 {
 798         struct nfp_flower_priv *fl_priv = app->priv;
 799         struct nfp_meter_entry *meter_entry = NULL;
 800         struct nfp_police_stats_reply *msg = pmsg;
 801         u32 meter_id;
 802
 803         meter_id = be32_to_cpu(msg->head.meter_id);
 804         mutex_lock(&fl_priv->meter_stats_lock);
 805
 806         meter_entry = nfp_flower_search_meter_entry(app, meter_id);
 807         if (!meter_entry)
 808                 goto exit_unlock;
 809
 810         meter_entry->stats.curr.pkts = be64_to_cpu(msg->pass_pkts) +
 811                                        be64_to_cpu(msg->drop_pkts);
 812         meter_entry->stats.curr.bytes = be64_to_cpu(msg->pass_bytes) +
 813                                         be64_to_cpu(msg->drop_bytes);
 814         meter_entry->stats.curr.drops = be64_to_cpu(msg->drop_pkts);
 815         if (!meter_entry->stats.update) {
 816                 meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts;
 817                 meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes;
 818                 meter_entry->stats.prev.drops = meter_entry->stats.curr.drops;
 819         }
 820
 821         meter_entry->stats.update = jiffies;
 822
 823 exit_unlock:
 824         mutex_unlock(&fl_priv->meter_stats_lock);
 825 }
 826
 827 static int
 828 nfp_act_stats_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
 829                       struct netlink_ext_ack *extack)
 830 {
 831         struct nfp_flower_priv *fl_priv = app->priv;
 832         struct nfp_meter_entry *meter_entry = NULL;
 833         u64 diff_bytes, diff_pkts, diff_drops;
 834         int err = 0;
 835
 836         if (fl_act->id != FLOW_ACTION_POLICE) {
 837                 NL_SET_ERR_MSG_MOD(extack,
 838                                    "unsupported offload: qos rate limit offload requires police action");
 839                 return -EOPNOTSUPP;
 840         }
 841
 842         mutex_lock(&fl_priv->meter_stats_lock);
 843         meter_entry = nfp_flower_search_meter_entry(app, fl_act->index);
 844         if (!meter_entry) {
 845                 err = -ENOENT;
 846                 goto exit_unlock;
 847         }
 848         diff_pkts = meter_entry->stats.curr.pkts > meter_entry->stats.prev.pkts ?
 849                     meter_entry->stats.curr.pkts - meter_entry->stats.prev.pkts : 0;
 850         diff_bytes = meter_entry->stats.curr.bytes > meter_entry->stats.prev.bytes ?
 851                      meter_entry->stats.curr.bytes - meter_entry->stats.prev.bytes : 0;
 852         diff_drops = meter_entry->stats.curr.drops > meter_entry->stats.prev.drops ?
 853                      meter_entry->stats.curr.drops - meter_entry->stats.prev.drops : 0;
 854
 855         flow_stats_update(&fl_act->stats, diff_bytes, diff_pkts, diff_drops,
 856                           meter_entry->stats.update,
 857                           FLOW_ACTION_HW_STATS_DELAYED);
 858
 859         meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts;
 860         meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes;
 861         meter_entry->stats.prev.drops = meter_entry->stats.curr.drops;
 862
 863 exit_unlock:
 864         mutex_unlock(&fl_priv->meter_stats_lock);
 865         return err;
 866 }
 867
 868 int nfp_setup_tc_act_offload(struct nfp_app *app,
 869                              struct flow_offload_action *fl_act)
 870 {
 871         struct netlink_ext_ack *extack = fl_act->extack;
 872         struct nfp_flower_priv *fl_priv = app->priv;
 873
 874         if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_METER))
 875                 return -EOPNOTSUPP;
 876
 877         switch (fl_act->command) {
 878         case FLOW_ACT_REPLACE:
 879                 return nfp_act_install_actions(app, fl_act, extack);
 880         case FLOW_ACT_DESTROY:
 881                 return nfp_act_remove_actions(app, fl_act, extack);
 882         case FLOW_ACT_STATS:
 883                 return nfp_act_stats_actions(app, fl_act, extack);
 884         default:
 885                 return -EOPNOTSUPP;
 886         }
 887 }