drivers/net/ethernet/aquantia/atlantic/aq_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Atlantic Network Driver
   3  *
   4  * Copyright (C) 2014-2019 aQuantia Corporation
   5  * Copyright (C) 2019-2020 Marvell International Ltd.
   6  */
   7
   8 /* File aq_main.c: Main file for aQuantia Linux driver. */
   9
  10 #include "aq_main.h"
  11 #include "aq_nic.h"
  12 #include "aq_pci_func.h"
  13 #include "aq_ethtool.h"
  14 #include "aq_ptp.h"
  15 #include "aq_filters.h"
  16 #include "aq_hw_utils.h"
  17 #include "aq_vec.h"
  18
  19 #include <linux/netdevice.h>
  20 #include <linux/module.h>
  21 #include <linux/ip.h>
  22 #include <linux/udp.h>
  23 #include <net/pkt_cls.h>
  24 #include <linux/filter.h>
  25
  26 MODULE_LICENSE("GPL v2");
  27 MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
  28 MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
  29
  30 DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);
  31 EXPORT_SYMBOL(aq_xdp_locking_key);
  32
  33 static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;
  34
  35 static const struct net_device_ops aq_ndev_ops;
  36
  37 static struct workqueue_struct *aq_ndev_wq;
  38
  39 void aq_ndev_schedule_work(struct work_struct *work)
  40 {
  41         queue_work(aq_ndev_wq, work);
  42 }
  43
  44 struct net_device *aq_ndev_alloc(void)
  45 {
  46         struct net_device *ndev = NULL;
  47         struct aq_nic_s *aq_nic = NULL;
  48
  49         ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_HW_QUEUES_MAX);
  50         if (!ndev)
  51                 return NULL;
  52
  53         aq_nic = netdev_priv(ndev);
  54         aq_nic->ndev = ndev;
  55         ndev->netdev_ops = &aq_ndev_ops;
  56         ndev->ethtool_ops = &aq_ethtool_ops;
  57
  58         return ndev;
  59 }
  60
  61 static int aq_ndev_open(struct net_device *ndev)
  62 {
  63         struct aq_nic_s *aq_nic = netdev_priv(ndev);
  64         int err = 0;
  65
  66         err = aq_nic_init(aq_nic);
  67         if (err < 0)
  68                 goto err_exit;
  69
  70         err = aq_reapply_rxnfc_all_rules(aq_nic);
  71         if (err < 0)
  72                 goto err_exit;
  73
  74         err = aq_filters_vlans_update(aq_nic);
  75         if (err < 0)
  76                 goto err_exit;
  77
  78         err = aq_nic_start(aq_nic);
  79         if (err < 0) {
  80                 aq_nic_stop(aq_nic);
  81                 goto err_exit;
  82         }
  83
  84 err_exit:
  85         if (err < 0)
  86                 aq_nic_deinit(aq_nic, true);
  87
  88         return err;
  89 }
  90
  91 static int aq_ndev_close(struct net_device *ndev)
  92 {
  93         struct aq_nic_s *aq_nic = netdev_priv(ndev);
  94         int err = 0;
  95
  96         err = aq_nic_stop(aq_nic);
  97         aq_nic_deinit(aq_nic, true);
  98
  99         return err;
 100 }
 101
 102 static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 103 {
 104         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 105
 106 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 107         if (unlikely(aq_utils_obj_test(&aq_nic->flags, AQ_NIC_PTP_DPATH_UP))) {
 108                 /* Hardware adds the Timestamp for PTPv2 802.AS1
 109                  * and PTPv2 IPv4 UDP.
 110                  * We have to push even general 320 port messages to the ptp
 111                  * queue explicitly. This is a limitation of current firmware
 112                  * and hardware PTP design of the chip. Otherwise ptp stream
 113                  * will fail to sync
 114                  */
 115                 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
 116                     unlikely((ip_hdr(skb)->version == 4) &&
 117                              (ip_hdr(skb)->protocol == IPPROTO_UDP) &&
 118                              ((udp_hdr(skb)->dest == htons(319)) ||
 119                               (udp_hdr(skb)->dest == htons(320)))) ||
 120                     unlikely(eth_hdr(skb)->h_proto == htons(ETH_P_1588)))
 121                         return aq_ptp_xmit(aq_nic, skb);
 122         }
 123 #endif
 124
 125         skb_tx_timestamp(skb);
 126         return aq_nic_xmit(aq_nic, skb);
 127 }
 128
 129 static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)
 130 {
 131         int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
 132         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 133         struct bpf_prog *prog;
 134         int err;
 135
 136         prog = READ_ONCE(aq_nic->xdp_prog);
 137         if (prog && !prog->aux->xdp_has_frags &&
 138             new_frame_size > AQ_CFG_RX_FRAME_MAX) {
 139                 netdev_err(ndev, "Illegal MTU %d for XDP prog without frags\n",
 140                            ndev->mtu);
 141                 return -EOPNOTSUPP;
 142         }
 143
 144         err = aq_nic_set_mtu(aq_nic, new_mtu + ETH_HLEN);
 145
 146         if (err < 0)
 147                 goto err_exit;
 148         ndev->mtu = new_mtu;
 149
 150 err_exit:
 151         return err;
 152 }
 153
 154 static int aq_ndev_set_features(struct net_device *ndev,
 155                                 netdev_features_t features)
 156 {
 157         bool is_vlan_tx_insert = !!(features & NETIF_F_HW_VLAN_CTAG_TX);
 158         bool is_vlan_rx_strip = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
 159         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 160         bool need_ndev_restart = false;
 161         struct aq_nic_cfg_s *aq_cfg;
 162         bool is_lro = false;
 163         int err = 0;
 164
 165         aq_cfg = aq_nic_get_cfg(aq_nic);
 166
 167         if (!(features & NETIF_F_NTUPLE)) {
 168                 if (aq_nic->ndev->features & NETIF_F_NTUPLE) {
 169                         err = aq_clear_rxnfc_all_rules(aq_nic);
 170                         if (unlikely(err))
 171                                 goto err_exit;
 172                 }
 173         }
 174         if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER)) {
 175                 if (aq_nic->ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) {
 176                         err = aq_filters_vlan_offload_off(aq_nic);
 177                         if (unlikely(err))
 178                                 goto err_exit;
 179                 }
 180         }
 181
 182         aq_cfg->features = features;
 183
 184         if (aq_cfg->aq_hw_caps->hw_features & NETIF_F_LRO) {
 185                 is_lro = features & NETIF_F_LRO;
 186
 187                 if (aq_cfg->is_lro != is_lro) {
 188                         aq_cfg->is_lro = is_lro;
 189                         need_ndev_restart = true;
 190                 }
 191         }
 192
 193         if ((aq_nic->ndev->features ^ features) & NETIF_F_RXCSUM) {
 194                 err = aq_nic->aq_hw_ops->hw_set_offload(aq_nic->aq_hw,
 195                                                         aq_cfg);
 196
 197                 if (unlikely(err))
 198                         goto err_exit;
 199         }
 200
 201         if (aq_cfg->is_vlan_rx_strip != is_vlan_rx_strip) {
 202                 aq_cfg->is_vlan_rx_strip = is_vlan_rx_strip;
 203                 need_ndev_restart = true;
 204         }
 205         if (aq_cfg->is_vlan_tx_insert != is_vlan_tx_insert) {
 206                 aq_cfg->is_vlan_tx_insert = is_vlan_tx_insert;
 207                 need_ndev_restart = true;
 208         }
 209
 210         if (need_ndev_restart && netif_running(ndev)) {
 211                 aq_ndev_close(ndev);
 212                 aq_ndev_open(ndev);
 213         }
 214
 215 err_exit:
 216         return err;
 217 }
 218
 219 static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,
 220                                               netdev_features_t features)
 221 {
 222         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 223         struct bpf_prog *prog;
 224
 225         if (!(features & NETIF_F_RXCSUM))
 226                 features &= ~NETIF_F_LRO;
 227
 228         prog = READ_ONCE(aq_nic->xdp_prog);
 229         if (prog && !prog->aux->xdp_has_frags &&
 230             aq_nic->xdp_prog && features & NETIF_F_LRO) {
 231                 netdev_err(ndev, "LRO is not supported with single buffer XDP, disabling\n");
 232                 features &= ~NETIF_F_LRO;
 233         }
 234
 235         return features;
 236 }
 237
 238 static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)
 239 {
 240         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 241         int err = 0;
 242
 243         err = eth_mac_addr(ndev, addr);
 244         if (err < 0)
 245                 goto err_exit;
 246         err = aq_nic_set_mac(aq_nic, ndev);
 247         if (err < 0)
 248                 goto err_exit;
 249
 250 err_exit:
 251         return err;
 252 }
 253
 254 static void aq_ndev_set_multicast_settings(struct net_device *ndev)
 255 {
 256         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 257
 258         (void)aq_nic_set_multicast_list(aq_nic, ndev);
 259 }
 260
 261 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 262 static int aq_ndev_config_hwtstamp(struct aq_nic_s *aq_nic,
 263                                    struct hwtstamp_config *config)
 264 {
 265         switch (config->tx_type) {
 266         case HWTSTAMP_TX_OFF:
 267         case HWTSTAMP_TX_ON:
 268                 break;
 269         default:
 270                 return -ERANGE;
 271         }
 272
 273         switch (config->rx_filter) {
 274         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 275         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 276         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 277         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 278         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 279         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 280         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 281         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 282                 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 283                 break;
 284         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 285         case HWTSTAMP_FILTER_NONE:
 286                 break;
 287         default:
 288                 return -ERANGE;
 289         }
 290
 291         return aq_ptp_hwtstamp_config_set(aq_nic->aq_ptp, config);
 292 }
 293 #endif
 294
 295 static int aq_ndev_hwtstamp_set(struct aq_nic_s *aq_nic, struct ifreq *ifr)
 296 {
 297         struct hwtstamp_config config;
 298 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 299         int ret_val;
 300 #endif
 301
 302         if (!aq_nic->aq_ptp)
 303                 return -EOPNOTSUPP;
 304
 305         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 306                 return -EFAULT;
 307 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 308         ret_val = aq_ndev_config_hwtstamp(aq_nic, &config);
 309         if (ret_val)
 310                 return ret_val;
 311 #endif
 312
 313         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 314                -EFAULT : 0;
 315 }
 316
 317 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 318 static int aq_ndev_hwtstamp_get(struct aq_nic_s *aq_nic, struct ifreq *ifr)
 319 {
 320         struct hwtstamp_config config;
 321
 322         if (!aq_nic->aq_ptp)
 323                 return -EOPNOTSUPP;
 324
 325         aq_ptp_hwtstamp_config_get(aq_nic->aq_ptp, &config);
 326         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 327                -EFAULT : 0;
 328 }
 329 #endif
 330
 331 static int aq_ndev_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
 332 {
 333         struct aq_nic_s *aq_nic = netdev_priv(netdev);
 334
 335         switch (cmd) {
 336         case SIOCSHWTSTAMP:
 337                 return aq_ndev_hwtstamp_set(aq_nic, ifr);
 338
 339 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 340         case SIOCGHWTSTAMP:
 341                 return aq_ndev_hwtstamp_get(aq_nic, ifr);
 342 #endif
 343         }
 344
 345         return -EOPNOTSUPP;
 346 }
 347
 348 static int aq_ndo_vlan_rx_add_vid(struct net_device *ndev, __be16 proto,
 349                                   u16 vid)
 350 {
 351         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 352
 353         if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
 354                 return -EOPNOTSUPP;
 355
 356         set_bit(vid, aq_nic->active_vlans);
 357
 358         return aq_filters_vlans_update(aq_nic);
 359 }
 360
 361 static int aq_ndo_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto,
 362                                    u16 vid)
 363 {
 364         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 365
 366         if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
 367                 return -EOPNOTSUPP;
 368
 369         clear_bit(vid, aq_nic->active_vlans);
 370
 371         if (-ENOENT == aq_del_fvlan_by_vlan(aq_nic, vid))
 372                 return aq_filters_vlans_update(aq_nic);
 373
 374         return 0;
 375 }
 376
 377 static int aq_validate_mqprio_opt(struct aq_nic_s *self,
 378                                   struct tc_mqprio_qopt_offload *mqprio,
 379                                   const unsigned int num_tc)
 380 {
 381         const bool has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
 382         struct aq_nic_cfg_s *aq_nic_cfg = aq_nic_get_cfg(self);
 383         const unsigned int tcs_max = min_t(u8, aq_nic_cfg->aq_hw_caps->tcs_max,
 384                                            AQ_CFG_TCS_MAX);
 385
 386         if (num_tc > tcs_max) {
 387                 netdev_err(self->ndev, "Too many TCs requested\n");
 388                 return -EOPNOTSUPP;
 389         }
 390
 391         if (num_tc != 0 && !is_power_of_2(num_tc)) {
 392                 netdev_err(self->ndev, "TC count should be power of 2\n");
 393                 return -EOPNOTSUPP;
 394         }
 395
 396         if (has_min_rate && !ATL_HW_IS_CHIP_FEATURE(self->aq_hw, ANTIGUA)) {
 397                 netdev_err(self->ndev, "Min tx rate is not supported\n");
 398                 return -EOPNOTSUPP;
 399         }
 400
 401         return 0;
 402 }
 403
 404 static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type,
 405                            void *type_data)
 406 {
 407         struct tc_mqprio_qopt_offload *mqprio = type_data;
 408         struct aq_nic_s *aq_nic = netdev_priv(dev);
 409         bool has_min_rate;
 410         bool has_max_rate;
 411         int err;
 412         int i;
 413
 414         if (type != TC_SETUP_QDISC_MQPRIO)
 415                 return -EOPNOTSUPP;
 416
 417         has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
 418         has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE);
 419
 420         err = aq_validate_mqprio_opt(aq_nic, mqprio, mqprio->qopt.num_tc);
 421         if (err)
 422                 return err;
 423
 424         for (i = 0; i < mqprio->qopt.num_tc; i++) {
 425                 if (has_max_rate) {
 426                         u64 max_rate = mqprio->max_rate[i];
 427
 428                         do_div(max_rate, AQ_MBPS_DIVISOR);
 429                         aq_nic_setup_tc_max_rate(aq_nic, i, (u32)max_rate);
 430                 }
 431
 432                 if (has_min_rate) {
 433                         u64 min_rate = mqprio->min_rate[i];
 434
 435                         do_div(min_rate, AQ_MBPS_DIVISOR);
 436                         aq_nic_setup_tc_min_rate(aq_nic, i, (u32)min_rate);
 437                 }
 438         }
 439
 440         return aq_nic_setup_tc_mqprio(aq_nic, mqprio->qopt.num_tc,
 441                                       mqprio->qopt.prio_tc_map);
 442 }
 443
 444 static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,
 445                         struct netlink_ext_ack *extack)
 446 {
 447         bool need_update, running = netif_running(ndev);
 448         struct aq_nic_s *aq_nic = netdev_priv(ndev);
 449         struct bpf_prog *old_prog;
 450
 451         if (prog && !prog->aux->xdp_has_frags) {
 452                 if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {
 453                         NL_SET_ERR_MSG_MOD(extack,
 454                                            "prog does not support XDP frags");
 455                         return -EOPNOTSUPP;
 456                 }
 457
 458                 if (prog && ndev->features & NETIF_F_LRO) {
 459                         netdev_err(ndev,
 460                                    "LRO is not supported with single buffer XDP, disabling\n");
 461                         ndev->features &= ~NETIF_F_LRO;
 462                 }
 463         }
 464
 465         need_update = !!aq_nic->xdp_prog != !!prog;
 466         if (running && need_update)
 467                 aq_ndev_close(ndev);
 468
 469         old_prog = xchg(&aq_nic->xdp_prog, prog);
 470         if (old_prog)
 471                 bpf_prog_put(old_prog);
 472
 473         if (!old_prog && prog)
 474                 static_branch_inc(&aq_xdp_locking_key);
 475         else if (old_prog && !prog)
 476                 static_branch_dec(&aq_xdp_locking_key);
 477
 478         if (running && need_update)
 479                 return aq_ndev_open(ndev);
 480
 481         return 0;
 482 }
 483
 484 static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 485 {
 486         switch (xdp->command) {
 487         case XDP_SETUP_PROG:
 488                 return aq_xdp_setup(dev, xdp->prog, xdp->extack);
 489         default:
 490                 return -EINVAL;
 491         }
 492 }
 493
 494 static const struct net_device_ops aq_ndev_ops = {
 495         .ndo_open = aq_ndev_open,
 496         .ndo_stop = aq_ndev_close,
 497         .ndo_start_xmit = aq_ndev_start_xmit,
 498         .ndo_set_rx_mode = aq_ndev_set_multicast_settings,
 499         .ndo_change_mtu = aq_ndev_change_mtu,
 500         .ndo_set_mac_address = aq_ndev_set_mac_address,
 501         .ndo_set_features = aq_ndev_set_features,
 502         .ndo_fix_features = aq_ndev_fix_features,
 503         .ndo_eth_ioctl = aq_ndev_ioctl,
 504         .ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,
 505         .ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,
 506         .ndo_setup_tc = aq_ndo_setup_tc,
 507         .ndo_bpf = aq_xdp,
 508         .ndo_xdp_xmit = aq_xdp_xmit,
 509 };
 510
 511 static int __init aq_ndev_init_module(void)
 512 {
 513         int ret;
 514
 515         aq_ndev_wq = create_singlethread_workqueue(aq_ndev_driver_name);
 516         if (!aq_ndev_wq) {
 517                 pr_err("Failed to create workqueue\n");
 518                 return -ENOMEM;
 519         }
 520
 521         ret = aq_pci_func_register_driver();
 522         if (ret) {
 523                 destroy_workqueue(aq_ndev_wq);
 524                 return ret;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static void __exit aq_ndev_exit_module(void)
 531 {
 532         aq_pci_func_unregister_driver();
 533
 534         if (aq_ndev_wq) {
 535                 destroy_workqueue(aq_ndev_wq);
 536                 aq_ndev_wq = NULL;
 537         }
 538 }
 539
 540 module_init(aq_ndev_init_module);
 541 module_exit(aq_ndev_exit_module);