drivers/net/thunderbolt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Networking over Thunderbolt/USB4 cables using USB4NET protocol
   4  * (formerly Apple ThunderboltIP).
   5  *
   6  * Copyright (C) 2017, Intel Corporation
   7  * Authors: Amir Levy <amir.jer.levy@intel.com>
   8  *          Michael Jamet <michael.jamet@intel.com>
   9  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  10  */
  11
  12 #include <linux/atomic.h>
  13 #include <linux/highmem.h>
  14 #include <linux/if_vlan.h>
  15 #include <linux/jhash.h>
  16 #include <linux/module.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/rtnetlink.h>
  19 #include <linux/sizes.h>
  20 #include <linux/thunderbolt.h>
  21 #include <linux/uuid.h>
  22 #include <linux/workqueue.h>
  23
  24 #include <net/ip6_checksum.h>
  25
  26 /* Protocol timeouts in ms */
  27 #define TBNET_LOGIN_DELAY       4500
  28 #define TBNET_LOGIN_TIMEOUT     500
  29 #define TBNET_LOGOUT_TIMEOUT    1000
  30
  31 #define TBNET_RING_SIZE         256
  32 #define TBNET_LOGIN_RETRIES     60
  33 #define TBNET_LOGOUT_RETRIES    10
  34 #define TBNET_E2E               BIT(0)
  35 #define TBNET_MATCH_FRAGS_ID    BIT(1)
  36 #define TBNET_64K_FRAMES        BIT(2)
  37 #define TBNET_MAX_MTU           SZ_64K
  38 #define TBNET_FRAME_SIZE        SZ_4K
  39 #define TBNET_MAX_PAYLOAD_SIZE  \
  40         (TBNET_FRAME_SIZE - sizeof(struct thunderbolt_ip_frame_header))
  41 /* Rx packets need to hold space for skb_shared_info */
  42 #define TBNET_RX_MAX_SIZE       \
  43         (TBNET_FRAME_SIZE + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
  44 #define TBNET_RX_PAGE_ORDER     get_order(TBNET_RX_MAX_SIZE)
  45 #define TBNET_RX_PAGE_SIZE      (PAGE_SIZE << TBNET_RX_PAGE_ORDER)
  46
  47 #define TBNET_L0_PORT_NUM(route) ((route) & GENMASK(5, 0))
  48
  49 /**
  50  * struct thunderbolt_ip_frame_header - Header for each Thunderbolt frame
  51  * @frame_size: size of the data with the frame
  52  * @frame_index: running index on the frames
  53  * @frame_id: ID of the frame to match frames to specific packet
  54  * @frame_count: how many frames assembles a full packet
  55  *
  56  * Each data frame passed to the high-speed DMA ring has this header. If
  57  * the XDomain network directory announces that %TBNET_MATCH_FRAGS_ID is
  58  * supported then @frame_id is filled, otherwise it stays %0.
  59  */
  60 struct thunderbolt_ip_frame_header {
  61         u32 frame_size;
  62         u16 frame_index;
  63         u16 frame_id;
  64         u32 frame_count;
  65 };
  66
  67 enum thunderbolt_ip_frame_pdf {
  68         TBIP_PDF_FRAME_START = 1,
  69         TBIP_PDF_FRAME_END,
  70 };
  71
  72 enum thunderbolt_ip_type {
  73         TBIP_LOGIN,
  74         TBIP_LOGIN_RESPONSE,
  75         TBIP_LOGOUT,
  76         TBIP_STATUS,
  77 };
  78
  79 struct thunderbolt_ip_header {
  80         u32 route_hi;
  81         u32 route_lo;
  82         u32 length_sn;
  83         uuid_t uuid;
  84         uuid_t initiator_uuid;
  85         uuid_t target_uuid;
  86         u32 type;
  87         u32 command_id;
  88 };
  89
  90 #define TBIP_HDR_LENGTH_MASK            GENMASK(5, 0)
  91 #define TBIP_HDR_SN_MASK                GENMASK(28, 27)
  92 #define TBIP_HDR_SN_SHIFT               27
  93
  94 struct thunderbolt_ip_login {
  95         struct thunderbolt_ip_header hdr;
  96         u32 proto_version;
  97         u32 transmit_path;
  98         u32 reserved[4];
  99 };
 100
 101 #define TBIP_LOGIN_PROTO_VERSION        1
 102
 103 struct thunderbolt_ip_login_response {
 104         struct thunderbolt_ip_header hdr;
 105         u32 status;
 106         u32 receiver_mac[2];
 107         u32 receiver_mac_len;
 108         u32 reserved[4];
 109 };
 110
 111 struct thunderbolt_ip_logout {
 112         struct thunderbolt_ip_header hdr;
 113 };
 114
 115 struct thunderbolt_ip_status {
 116         struct thunderbolt_ip_header hdr;
 117         u32 status;
 118 };
 119
 120 struct tbnet_stats {
 121         u64 tx_packets;
 122         u64 rx_packets;
 123         u64 tx_bytes;
 124         u64 rx_bytes;
 125         u64 rx_errors;
 126         u64 tx_errors;
 127         u64 rx_length_errors;
 128         u64 rx_over_errors;
 129         u64 rx_crc_errors;
 130         u64 rx_missed_errors;
 131 };
 132
 133 struct tbnet_frame {
 134         struct net_device *dev;
 135         struct page *page;
 136         struct ring_frame frame;
 137 };
 138
 139 struct tbnet_ring {
 140         struct tbnet_frame frames[TBNET_RING_SIZE];
 141         unsigned int cons;
 142         unsigned int prod;
 143         struct tb_ring *ring;
 144 };
 145
 146 /**
 147  * struct tbnet - ThunderboltIP network driver private data
 148  * @svc: XDomain service the driver is bound to
 149  * @xd: XDomain the service blongs to
 150  * @handler: ThunderboltIP configuration protocol handler
 151  * @dev: Networking device
 152  * @napi: NAPI structure for Rx polling
 153  * @stats: Network statistics
 154  * @skb: Network packet that is currently processed on Rx path
 155  * @command_id: ID used for next configuration protocol packet
 156  * @login_sent: ThunderboltIP login message successfully sent
 157  * @login_received: ThunderboltIP login message received from the remote
 158  *                  host
 159  * @local_transmit_path: HopID we are using to send out packets
 160  * @remote_transmit_path: HopID the other end is using to send packets to us
 161  * @connection_lock: Lock serializing access to @login_sent,
 162  *                   @login_received and @transmit_path.
 163  * @login_retries: Number of login retries currently done
 164  * @login_work: Worker to send ThunderboltIP login packets
 165  * @connected_work: Worker that finalizes the ThunderboltIP connection
 166  *                  setup and enables DMA paths for high speed data
 167  *                  transfers
 168  * @disconnect_work: Worker that handles tearing down the ThunderboltIP
 169  *                   connection
 170  * @rx_hdr: Copy of the currently processed Rx frame. Used when a
 171  *          network packet consists of multiple Thunderbolt frames.
 172  *          In host byte order.
 173  * @rx_ring: Software ring holding Rx frames
 174  * @frame_id: Frame ID use for next Tx packet
 175  *            (if %TBNET_MATCH_FRAGS_ID is supported in both ends)
 176  * @tx_ring: Software ring holding Tx frames
 177  */
 178 struct tbnet {
 179         const struct tb_service *svc;
 180         struct tb_xdomain *xd;
 181         struct tb_protocol_handler handler;
 182         struct net_device *dev;
 183         struct napi_struct napi;
 184         struct tbnet_stats stats;
 185         struct sk_buff *skb;
 186         atomic_t command_id;
 187         bool login_sent;
 188         bool login_received;
 189         int local_transmit_path;
 190         int remote_transmit_path;
 191         struct mutex connection_lock;
 192         int login_retries;
 193         struct delayed_work login_work;
 194         struct work_struct connected_work;
 195         struct work_struct disconnect_work;
 196         struct thunderbolt_ip_frame_header rx_hdr;
 197         struct tbnet_ring rx_ring;
 198         atomic_t frame_id;
 199         struct tbnet_ring tx_ring;
 200 };
 201
 202 /* Network property directory UUID: c66189ca-1cce-4195-bdb8-49592e5f5a4f */
 203 static const uuid_t tbnet_dir_uuid =
 204         UUID_INIT(0xc66189ca, 0x1cce, 0x4195,
 205                   0xbd, 0xb8, 0x49, 0x59, 0x2e, 0x5f, 0x5a, 0x4f);
 206
 207 /* ThunderboltIP protocol UUID: 798f589e-3616-8a47-97c6-5664a920c8dd */
 208 static const uuid_t tbnet_svc_uuid =
 209         UUID_INIT(0x798f589e, 0x3616, 0x8a47,
 210                   0x97, 0xc6, 0x56, 0x64, 0xa9, 0x20, 0xc8, 0xdd);
 211
 212 static struct tb_property_dir *tbnet_dir;
 213
 214 static bool tbnet_e2e = true;
 215 module_param_named(e2e, tbnet_e2e, bool, 0444);
 216 MODULE_PARM_DESC(e2e, "USB4NET full end-to-end flow control (default: true)");
 217
 218 static void tbnet_fill_header(struct thunderbolt_ip_header *hdr, u64 route,
 219         u8 sequence, const uuid_t *initiator_uuid, const uuid_t *target_uuid,
 220         enum thunderbolt_ip_type type, size_t size, u32 command_id)
 221 {
 222         u32 length_sn;
 223
 224         /* Length does not include route_hi/lo and length_sn fields */
 225         length_sn = (size - 3 * 4) / 4;
 226         length_sn |= (sequence << TBIP_HDR_SN_SHIFT) & TBIP_HDR_SN_MASK;
 227
 228         hdr->route_hi = upper_32_bits(route);
 229         hdr->route_lo = lower_32_bits(route);
 230         hdr->length_sn = length_sn;
 231         uuid_copy(&hdr->uuid, &tbnet_svc_uuid);
 232         uuid_copy(&hdr->initiator_uuid, initiator_uuid);
 233         uuid_copy(&hdr->target_uuid, target_uuid);
 234         hdr->type = type;
 235         hdr->command_id = command_id;
 236 }
 237
 238 static int tbnet_login_response(struct tbnet *net, u64 route, u8 sequence,
 239                                 u32 command_id)
 240 {
 241         struct thunderbolt_ip_login_response reply;
 242         struct tb_xdomain *xd = net->xd;
 243
 244         memset(&reply, 0, sizeof(reply));
 245         tbnet_fill_header(&reply.hdr, route, sequence, xd->local_uuid,
 246                           xd->remote_uuid, TBIP_LOGIN_RESPONSE, sizeof(reply),
 247                           command_id);
 248         memcpy(reply.receiver_mac, net->dev->dev_addr, ETH_ALEN);
 249         reply.receiver_mac_len = ETH_ALEN;
 250
 251         return tb_xdomain_response(xd, &reply, sizeof(reply),
 252                                    TB_CFG_PKG_XDOMAIN_RESP);
 253 }
 254
 255 static int tbnet_login_request(struct tbnet *net, u8 sequence)
 256 {
 257         struct thunderbolt_ip_login_response reply;
 258         struct thunderbolt_ip_login request;
 259         struct tb_xdomain *xd = net->xd;
 260
 261         memset(&request, 0, sizeof(request));
 262         tbnet_fill_header(&request.hdr, xd->route, sequence, xd->local_uuid,
 263                           xd->remote_uuid, TBIP_LOGIN, sizeof(request),
 264                           atomic_inc_return(&net->command_id));
 265
 266         request.proto_version = TBIP_LOGIN_PROTO_VERSION;
 267         request.transmit_path = net->local_transmit_path;
 268
 269         return tb_xdomain_request(xd, &request, sizeof(request),
 270                                   TB_CFG_PKG_XDOMAIN_RESP, &reply,
 271                                   sizeof(reply), TB_CFG_PKG_XDOMAIN_RESP,
 272                                   TBNET_LOGIN_TIMEOUT);
 273 }
 274
 275 static int tbnet_logout_response(struct tbnet *net, u64 route, u8 sequence,
 276                                  u32 command_id)
 277 {
 278         struct thunderbolt_ip_status reply;
 279         struct tb_xdomain *xd = net->xd;
 280
 281         memset(&reply, 0, sizeof(reply));
 282         tbnet_fill_header(&reply.hdr, route, sequence, xd->local_uuid,
 283                           xd->remote_uuid, TBIP_STATUS, sizeof(reply),
 284                           atomic_inc_return(&net->command_id));
 285         return tb_xdomain_response(xd, &reply, sizeof(reply),
 286                                    TB_CFG_PKG_XDOMAIN_RESP);
 287 }
 288
 289 static int tbnet_logout_request(struct tbnet *net)
 290 {
 291         struct thunderbolt_ip_logout request;
 292         struct thunderbolt_ip_status reply;
 293         struct tb_xdomain *xd = net->xd;
 294
 295         memset(&request, 0, sizeof(request));
 296         tbnet_fill_header(&request.hdr, xd->route, 0, xd->local_uuid,
 297                           xd->remote_uuid, TBIP_LOGOUT, sizeof(request),
 298                           atomic_inc_return(&net->command_id));
 299
 300         return tb_xdomain_request(xd, &request, sizeof(request),
 301                                   TB_CFG_PKG_XDOMAIN_RESP, &reply,
 302                                   sizeof(reply), TB_CFG_PKG_XDOMAIN_RESP,
 303                                   TBNET_LOGOUT_TIMEOUT);
 304 }
 305
 306 static void start_login(struct tbnet *net)
 307 {
 308         mutex_lock(&net->connection_lock);
 309         net->login_sent = false;
 310         net->login_received = false;
 311         mutex_unlock(&net->connection_lock);
 312
 313         queue_delayed_work(system_long_wq, &net->login_work,
 314                            msecs_to_jiffies(1000));
 315 }
 316
 317 static void stop_login(struct tbnet *net)
 318 {
 319         cancel_delayed_work_sync(&net->login_work);
 320         cancel_work_sync(&net->connected_work);
 321 }
 322
 323 static inline unsigned int tbnet_frame_size(const struct tbnet_frame *tf)
 324 {
 325         return tf->frame.size ? : TBNET_FRAME_SIZE;
 326 }
 327
 328 static void tbnet_free_buffers(struct tbnet_ring *ring)
 329 {
 330         unsigned int i;
 331
 332         for (i = 0; i < TBNET_RING_SIZE; i++) {
 333                 struct device *dma_dev = tb_ring_dma_device(ring->ring);
 334                 struct tbnet_frame *tf = &ring->frames[i];
 335                 enum dma_data_direction dir;
 336                 unsigned int order;
 337                 size_t size;
 338
 339                 if (!tf->page)
 340                         continue;
 341
 342                 if (ring->ring->is_tx) {
 343                         dir = DMA_TO_DEVICE;
 344                         order = 0;
 345                         size = TBNET_FRAME_SIZE;
 346                 } else {
 347                         dir = DMA_FROM_DEVICE;
 348                         order = TBNET_RX_PAGE_ORDER;
 349                         size = TBNET_RX_PAGE_SIZE;
 350                 }
 351
 352                 if (tf->frame.buffer_phy)
 353                         dma_unmap_page(dma_dev, tf->frame.buffer_phy, size,
 354                                        dir);
 355
 356                 __free_pages(tf->page, order);
 357                 tf->page = NULL;
 358         }
 359
 360         ring->cons = 0;
 361         ring->prod = 0;
 362 }
 363
 364 static void tbnet_tear_down(struct tbnet *net, bool send_logout)
 365 {
 366         netif_carrier_off(net->dev);
 367         netif_stop_queue(net->dev);
 368
 369         stop_login(net);
 370
 371         mutex_lock(&net->connection_lock);
 372
 373         if (net->login_sent && net->login_received) {
 374                 int ret, retries = TBNET_LOGOUT_RETRIES;
 375
 376                 while (send_logout && retries-- > 0) {
 377                         ret = tbnet_logout_request(net);
 378                         if (ret != -ETIMEDOUT)
 379                                 break;
 380                 }
 381
 382                 tb_ring_stop(net->rx_ring.ring);
 383                 tb_ring_stop(net->tx_ring.ring);
 384                 tbnet_free_buffers(&net->rx_ring);
 385                 tbnet_free_buffers(&net->tx_ring);
 386
 387                 ret = tb_xdomain_disable_paths(net->xd,
 388                                                net->local_transmit_path,
 389                                                net->rx_ring.ring->hop,
 390                                                net->remote_transmit_path,
 391                                                net->tx_ring.ring->hop);
 392                 if (ret)
 393                         netdev_warn(net->dev, "failed to disable DMA paths\n");
 394
 395                 tb_xdomain_release_in_hopid(net->xd, net->remote_transmit_path);
 396                 net->remote_transmit_path = 0;
 397         }
 398
 399         net->login_retries = 0;
 400         net->login_sent = false;
 401         net->login_received = false;
 402
 403         mutex_unlock(&net->connection_lock);
 404 }
 405
 406 static int tbnet_handle_packet(const void *buf, size_t size, void *data)
 407 {
 408         const struct thunderbolt_ip_login *pkg = buf;
 409         struct tbnet *net = data;
 410         u32 command_id;
 411         int ret = 0;
 412         u32 sequence;
 413         u64 route;
 414
 415         /* Make sure the packet is for us */
 416         if (size < sizeof(struct thunderbolt_ip_header))
 417                 return 0;
 418         if (!uuid_equal(&pkg->hdr.initiator_uuid, net->xd->remote_uuid))
 419                 return 0;
 420         if (!uuid_equal(&pkg->hdr.target_uuid, net->xd->local_uuid))
 421                 return 0;
 422
 423         route = ((u64)pkg->hdr.route_hi << 32) | pkg->hdr.route_lo;
 424         route &= ~BIT_ULL(63);
 425         if (route != net->xd->route)
 426                 return 0;
 427
 428         sequence = pkg->hdr.length_sn & TBIP_HDR_SN_MASK;
 429         sequence >>= TBIP_HDR_SN_SHIFT;
 430         command_id = pkg->hdr.command_id;
 431
 432         switch (pkg->hdr.type) {
 433         case TBIP_LOGIN:
 434                 if (!netif_running(net->dev))
 435                         break;
 436
 437                 ret = tbnet_login_response(net, route, sequence,
 438                                            pkg->hdr.command_id);
 439                 if (!ret) {
 440                         mutex_lock(&net->connection_lock);
 441                         net->login_received = true;
 442                         net->remote_transmit_path = pkg->transmit_path;
 443
 444                         /* If we reached the number of max retries or
 445                          * previous logout, schedule another round of
 446                          * login retries
 447                          */
 448                         if (net->login_retries >= TBNET_LOGIN_RETRIES ||
 449                             !net->login_sent) {
 450                                 net->login_retries = 0;
 451                                 queue_delayed_work(system_long_wq,
 452                                                    &net->login_work, 0);
 453                         }
 454                         mutex_unlock(&net->connection_lock);
 455
 456                         queue_work(system_long_wq, &net->connected_work);
 457                 }
 458                 break;
 459
 460         case TBIP_LOGOUT:
 461                 ret = tbnet_logout_response(net, route, sequence, command_id);
 462                 if (!ret)
 463                         queue_work(system_long_wq, &net->disconnect_work);
 464                 break;
 465
 466         default:
 467                 return 0;
 468         }
 469
 470         if (ret)
 471                 netdev_warn(net->dev, "failed to send ThunderboltIP response\n");
 472
 473         return 1;
 474 }
 475
 476 static unsigned int tbnet_available_buffers(const struct tbnet_ring *ring)
 477 {
 478         return ring->prod - ring->cons;
 479 }
 480
 481 static int tbnet_alloc_rx_buffers(struct tbnet *net, unsigned int nbuffers)
 482 {
 483         struct tbnet_ring *ring = &net->rx_ring;
 484         int ret;
 485
 486         while (nbuffers--) {
 487                 struct device *dma_dev = tb_ring_dma_device(ring->ring);
 488                 unsigned int index = ring->prod & (TBNET_RING_SIZE - 1);
 489                 struct tbnet_frame *tf = &ring->frames[index];
 490                 dma_addr_t dma_addr;
 491
 492                 if (tf->page)
 493                         break;
 494
 495                 /* Allocate page (order > 0) so that it can hold maximum
 496                  * ThunderboltIP frame (4kB) and the additional room for
 497                  * SKB shared info required by build_skb().
 498                  */
 499                 tf->page = dev_alloc_pages(TBNET_RX_PAGE_ORDER);
 500                 if (!tf->page) {
 501                         ret = -ENOMEM;
 502                         goto err_free;
 503                 }
 504
 505                 dma_addr = dma_map_page(dma_dev, tf->page, 0,
 506                                         TBNET_RX_PAGE_SIZE, DMA_FROM_DEVICE);
 507                 if (dma_mapping_error(dma_dev, dma_addr)) {
 508                         ret = -ENOMEM;
 509                         goto err_free;
 510                 }
 511
 512                 tf->frame.buffer_phy = dma_addr;
 513                 tf->dev = net->dev;
 514
 515                 tb_ring_rx(ring->ring, &tf->frame);
 516
 517                 ring->prod++;
 518         }
 519
 520         return 0;
 521
 522 err_free:
 523         tbnet_free_buffers(ring);
 524         return ret;
 525 }
 526
 527 static struct tbnet_frame *tbnet_get_tx_buffer(struct tbnet *net)
 528 {
 529         struct tbnet_ring *ring = &net->tx_ring;
 530         struct device *dma_dev = tb_ring_dma_device(ring->ring);
 531         struct tbnet_frame *tf;
 532         unsigned int index;
 533
 534         if (!tbnet_available_buffers(ring))
 535                 return NULL;
 536
 537         index = ring->cons++ & (TBNET_RING_SIZE - 1);
 538
 539         tf = &ring->frames[index];
 540         tf->frame.size = 0;
 541
 542         dma_sync_single_for_cpu(dma_dev, tf->frame.buffer_phy,
 543                                 tbnet_frame_size(tf), DMA_TO_DEVICE);
 544
 545         return tf;
 546 }
 547
 548 static void tbnet_tx_callback(struct tb_ring *ring, struct ring_frame *frame,
 549                               bool canceled)
 550 {
 551         struct tbnet_frame *tf = container_of(frame, typeof(*tf), frame);
 552         struct tbnet *net = netdev_priv(tf->dev);
 553
 554         /* Return buffer to the ring */
 555         net->tx_ring.prod++;
 556
 557         if (tbnet_available_buffers(&net->tx_ring) >= TBNET_RING_SIZE / 2)
 558                 netif_wake_queue(net->dev);
 559 }
 560
 561 static int tbnet_alloc_tx_buffers(struct tbnet *net)
 562 {
 563         struct tbnet_ring *ring = &net->tx_ring;
 564         struct device *dma_dev = tb_ring_dma_device(ring->ring);
 565         unsigned int i;
 566
 567         for (i = 0; i < TBNET_RING_SIZE; i++) {
 568                 struct tbnet_frame *tf = &ring->frames[i];
 569                 dma_addr_t dma_addr;
 570
 571                 tf->page = alloc_page(GFP_KERNEL);
 572                 if (!tf->page) {
 573                         tbnet_free_buffers(ring);
 574                         return -ENOMEM;
 575                 }
 576
 577                 dma_addr = dma_map_page(dma_dev, tf->page, 0, TBNET_FRAME_SIZE,
 578                                         DMA_TO_DEVICE);
 579                 if (dma_mapping_error(dma_dev, dma_addr)) {
 580                         __free_page(tf->page);
 581                         tf->page = NULL;
 582                         tbnet_free_buffers(ring);
 583                         return -ENOMEM;
 584                 }
 585
 586                 tf->dev = net->dev;
 587                 tf->frame.buffer_phy = dma_addr;
 588                 tf->frame.callback = tbnet_tx_callback;
 589                 tf->frame.sof = TBIP_PDF_FRAME_START;
 590                 tf->frame.eof = TBIP_PDF_FRAME_END;
 591         }
 592
 593         ring->cons = 0;
 594         ring->prod = TBNET_RING_SIZE - 1;
 595
 596         return 0;
 597 }
 598
 599 static void tbnet_connected_work(struct work_struct *work)
 600 {
 601         struct tbnet *net = container_of(work, typeof(*net), connected_work);
 602         bool connected;
 603         int ret;
 604
 605         if (netif_carrier_ok(net->dev))
 606                 return;
 607
 608         mutex_lock(&net->connection_lock);
 609         connected = net->login_sent && net->login_received;
 610         mutex_unlock(&net->connection_lock);
 611
 612         if (!connected)
 613                 return;
 614
 615         ret = tb_xdomain_alloc_in_hopid(net->xd, net->remote_transmit_path);
 616         if (ret != net->remote_transmit_path) {
 617                 netdev_err(net->dev, "failed to allocate Rx HopID\n");
 618                 return;
 619         }
 620
 621         /* Both logins successful so enable the rings, high-speed DMA
 622          * paths and start the network device queue.
 623          *
 624          * Note we enable the DMA paths last to make sure we have primed
 625          * the Rx ring before any incoming packets are allowed to
 626          * arrive.
 627          */
 628         tb_ring_start(net->tx_ring.ring);
 629         tb_ring_start(net->rx_ring.ring);
 630
 631         ret = tbnet_alloc_rx_buffers(net, TBNET_RING_SIZE);
 632         if (ret)
 633                 goto err_stop_rings;
 634
 635         ret = tbnet_alloc_tx_buffers(net);
 636         if (ret)
 637                 goto err_free_rx_buffers;
 638
 639         ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path,
 640                                       net->rx_ring.ring->hop,
 641                                       net->remote_transmit_path,
 642                                       net->tx_ring.ring->hop);
 643         if (ret) {
 644                 netdev_err(net->dev, "failed to enable DMA paths\n");
 645                 goto err_free_tx_buffers;
 646         }
 647
 648         netif_carrier_on(net->dev);
 649         netif_start_queue(net->dev);
 650         return;
 651
 652 err_free_tx_buffers:
 653         tbnet_free_buffers(&net->tx_ring);
 654 err_free_rx_buffers:
 655         tbnet_free_buffers(&net->rx_ring);
 656 err_stop_rings:
 657         tb_ring_stop(net->rx_ring.ring);
 658         tb_ring_stop(net->tx_ring.ring);
 659         tb_xdomain_release_in_hopid(net->xd, net->remote_transmit_path);
 660 }
 661
 662 static void tbnet_login_work(struct work_struct *work)
 663 {
 664         struct tbnet *net = container_of(work, typeof(*net), login_work.work);
 665         unsigned long delay = msecs_to_jiffies(TBNET_LOGIN_DELAY);
 666         int ret;
 667
 668         if (netif_carrier_ok(net->dev))
 669                 return;
 670
 671         ret = tbnet_login_request(net, net->login_retries % 4);
 672         if (ret) {
 673                 if (net->login_retries++ < TBNET_LOGIN_RETRIES) {
 674                         queue_delayed_work(system_long_wq, &net->login_work,
 675                                            delay);
 676                 } else {
 677                         netdev_info(net->dev, "ThunderboltIP login timed out\n");
 678                 }
 679         } else {
 680                 net->login_retries = 0;
 681
 682                 mutex_lock(&net->connection_lock);
 683                 net->login_sent = true;
 684                 mutex_unlock(&net->connection_lock);
 685
 686                 queue_work(system_long_wq, &net->connected_work);
 687         }
 688 }
 689
 690 static void tbnet_disconnect_work(struct work_struct *work)
 691 {
 692         struct tbnet *net = container_of(work, typeof(*net), disconnect_work);
 693
 694         tbnet_tear_down(net, false);
 695 }
 696
 697 static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf,
 698                               const struct thunderbolt_ip_frame_header *hdr)
 699 {
 700         u32 frame_id, frame_count, frame_size, frame_index;
 701         unsigned int size;
 702
 703         if (tf->frame.flags & RING_DESC_CRC_ERROR) {
 704                 net->stats.rx_crc_errors++;
 705                 return false;
 706         } else if (tf->frame.flags & RING_DESC_BUFFER_OVERRUN) {
 707                 net->stats.rx_over_errors++;
 708                 return false;
 709         }
 710
 711         /* Should be greater than just header i.e. contains data */
 712         size = tbnet_frame_size(tf);
 713         if (size <= sizeof(*hdr)) {
 714                 net->stats.rx_length_errors++;
 715                 return false;
 716         }
 717
 718         frame_count = le32_to_cpu(hdr->frame_count);
 719         frame_size = le32_to_cpu(hdr->frame_size);
 720         frame_index = le16_to_cpu(hdr->frame_index);
 721         frame_id = le16_to_cpu(hdr->frame_id);
 722
 723         if ((frame_size > size - sizeof(*hdr)) || !frame_size) {
 724                 net->stats.rx_length_errors++;
 725                 return false;
 726         }
 727
 728         /* In case we're in the middle of packet, validate the frame
 729          * header based on first fragment of the packet.
 730          */
 731         if (net->skb && net->rx_hdr.frame_count) {
 732                 /* Check the frame count fits the count field */
 733                 if (frame_count != net->rx_hdr.frame_count) {
 734                         net->stats.rx_length_errors++;
 735                         return false;
 736                 }
 737
 738                 /* Check the frame identifiers are incremented correctly,
 739                  * and id is matching.
 740                  */
 741                 if (frame_index != net->rx_hdr.frame_index + 1 ||
 742                     frame_id != net->rx_hdr.frame_id) {
 743                         net->stats.rx_missed_errors++;
 744                         return false;
 745                 }
 746
 747                 if (net->skb->len + frame_size > TBNET_MAX_MTU) {
 748                         net->stats.rx_length_errors++;
 749                         return false;
 750                 }
 751
 752                 return true;
 753         }
 754
 755         /* Start of packet, validate the frame header */
 756         if (frame_count == 0 || frame_count > TBNET_RING_SIZE / 4) {
 757                 net->stats.rx_length_errors++;
 758                 return false;
 759         }
 760         if (frame_index != 0) {
 761                 net->stats.rx_missed_errors++;
 762                 return false;
 763         }
 764
 765         return true;
 766 }
 767
 768 static int tbnet_poll(struct napi_struct *napi, int budget)
 769 {
 770         struct tbnet *net = container_of(napi, struct tbnet, napi);
 771         unsigned int cleaned_count = tbnet_available_buffers(&net->rx_ring);
 772         struct device *dma_dev = tb_ring_dma_device(net->rx_ring.ring);
 773         unsigned int rx_packets = 0;
 774
 775         while (rx_packets < budget) {
 776                 const struct thunderbolt_ip_frame_header *hdr;
 777                 unsigned int hdr_size = sizeof(*hdr);
 778                 struct sk_buff *skb = NULL;
 779                 struct ring_frame *frame;
 780                 struct tbnet_frame *tf;
 781                 struct page *page;
 782                 bool last = true;
 783                 u32 frame_size;
 784
 785                 /* Return some buffers to hardware, one at a time is too
 786                  * slow so allocate MAX_SKB_FRAGS buffers at the same
 787                  * time.
 788                  */
 789                 if (cleaned_count >= MAX_SKB_FRAGS) {
 790                         tbnet_alloc_rx_buffers(net, cleaned_count);
 791                         cleaned_count = 0;
 792                 }
 793
 794                 frame = tb_ring_poll(net->rx_ring.ring);
 795                 if (!frame)
 796                         break;
 797
 798                 dma_unmap_page(dma_dev, frame->buffer_phy,
 799                                TBNET_RX_PAGE_SIZE, DMA_FROM_DEVICE);
 800
 801                 tf = container_of(frame, typeof(*tf), frame);
 802
 803                 page = tf->page;
 804                 tf->page = NULL;
 805                 net->rx_ring.cons++;
 806                 cleaned_count++;
 807
 808                 hdr = page_address(page);
 809                 if (!tbnet_check_frame(net, tf, hdr)) {
 810                         __free_pages(page, TBNET_RX_PAGE_ORDER);
 811                         dev_kfree_skb_any(net->skb);
 812                         net->skb = NULL;
 813                         continue;
 814                 }
 815
 816                 frame_size = le32_to_cpu(hdr->frame_size);
 817
 818                 skb = net->skb;
 819                 if (!skb) {
 820                         skb = build_skb(page_address(page),
 821                                         TBNET_RX_PAGE_SIZE);
 822                         if (!skb) {
 823                                 __free_pages(page, TBNET_RX_PAGE_ORDER);
 824                                 net->stats.rx_errors++;
 825                                 break;
 826                         }
 827
 828                         skb_reserve(skb, hdr_size);
 829                         skb_put(skb, frame_size);
 830
 831                         net->skb = skb;
 832                 } else {
 833                         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
 834                                         page, hdr_size, frame_size,
 835                                         TBNET_RX_PAGE_SIZE - hdr_size);
 836                 }
 837
 838                 net->rx_hdr.frame_size = frame_size;
 839                 net->rx_hdr.frame_count = le32_to_cpu(hdr->frame_count);
 840                 net->rx_hdr.frame_index = le16_to_cpu(hdr->frame_index);
 841                 net->rx_hdr.frame_id = le16_to_cpu(hdr->frame_id);
 842                 last = net->rx_hdr.frame_index == net->rx_hdr.frame_count - 1;
 843
 844                 rx_packets++;
 845                 net->stats.rx_bytes += frame_size;
 846
 847                 if (last) {
 848                         skb->protocol = eth_type_trans(skb, net->dev);
 849                         napi_gro_receive(&net->napi, skb);
 850                         net->skb = NULL;
 851                 }
 852         }
 853
 854         net->stats.rx_packets += rx_packets;
 855
 856         if (cleaned_count)
 857                 tbnet_alloc_rx_buffers(net, cleaned_count);
 858
 859         if (rx_packets >= budget)
 860                 return budget;
 861
 862         napi_complete_done(napi, rx_packets);
 863         /* Re-enable the ring interrupt */
 864         tb_ring_poll_complete(net->rx_ring.ring);
 865
 866         return rx_packets;
 867 }
 868
 869 static void tbnet_start_poll(void *data)
 870 {
 871         struct tbnet *net = data;
 872
 873         napi_schedule(&net->napi);
 874 }
 875
 876 static int tbnet_open(struct net_device *dev)
 877 {
 878         struct tbnet *net = netdev_priv(dev);
 879         struct tb_xdomain *xd = net->xd;
 880         u16 sof_mask, eof_mask;
 881         struct tb_ring *ring;
 882         unsigned int flags;
 883         int hopid;
 884
 885         netif_carrier_off(dev);
 886
 887         ring = tb_ring_alloc_tx(xd->tb->nhi, -1, TBNET_RING_SIZE,
 888                                 RING_FLAG_FRAME);
 889         if (!ring) {
 890                 netdev_err(dev, "failed to allocate Tx ring\n");
 891                 return -ENOMEM;
 892         }
 893         net->tx_ring.ring = ring;
 894
 895         hopid = tb_xdomain_alloc_out_hopid(xd, -1);
 896         if (hopid < 0) {
 897                 netdev_err(dev, "failed to allocate Tx HopID\n");
 898                 tb_ring_free(net->tx_ring.ring);
 899                 net->tx_ring.ring = NULL;
 900                 return hopid;
 901         }
 902         net->local_transmit_path = hopid;
 903
 904         sof_mask = BIT(TBIP_PDF_FRAME_START);
 905         eof_mask = BIT(TBIP_PDF_FRAME_END);
 906
 907         flags = RING_FLAG_FRAME;
 908         /* Only enable full E2E if the other end supports it too */
 909         if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E)
 910                 flags |= RING_FLAG_E2E;
 911
 912         ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags,
 913                                 net->tx_ring.ring->hop, sof_mask,
 914                                 eof_mask, tbnet_start_poll, net);
 915         if (!ring) {
 916                 netdev_err(dev, "failed to allocate Rx ring\n");
 917                 tb_ring_free(net->tx_ring.ring);
 918                 net->tx_ring.ring = NULL;
 919                 return -ENOMEM;
 920         }
 921         net->rx_ring.ring = ring;
 922
 923         napi_enable(&net->napi);
 924         start_login(net);
 925
 926         return 0;
 927 }
 928
 929 static int tbnet_stop(struct net_device *dev)
 930 {
 931         struct tbnet *net = netdev_priv(dev);
 932
 933         napi_disable(&net->napi);
 934
 935         cancel_work_sync(&net->disconnect_work);
 936         tbnet_tear_down(net, true);
 937
 938         tb_ring_free(net->rx_ring.ring);
 939         net->rx_ring.ring = NULL;
 940
 941         tb_xdomain_release_out_hopid(net->xd, net->local_transmit_path);
 942         tb_ring_free(net->tx_ring.ring);
 943         net->tx_ring.ring = NULL;
 944
 945         return 0;
 946 }
 947
 948 static bool tbnet_xmit_csum_and_map(struct tbnet *net, struct sk_buff *skb,
 949         struct tbnet_frame **frames, u32 frame_count)
 950 {
 951         struct thunderbolt_ip_frame_header *hdr = page_address(frames[0]->page);
 952         struct device *dma_dev = tb_ring_dma_device(net->tx_ring.ring);
 953         __wsum wsum = htonl(skb->len - skb_transport_offset(skb));
 954         unsigned int i, len, offset = skb_transport_offset(skb);
 955         __be16 protocol = skb->protocol;
 956         void *data = skb->data;
 957         void *dest = hdr + 1;
 958         __sum16 *tucso;
 959
 960         if (skb->ip_summed != CHECKSUM_PARTIAL) {
 961                 /* No need to calculate checksum so we just update the
 962                  * total frame count and sync the frames for DMA.
 963                  */
 964                 for (i = 0; i < frame_count; i++) {
 965                         hdr = page_address(frames[i]->page);
 966                         hdr->frame_count = cpu_to_le32(frame_count);
 967                         dma_sync_single_for_device(dma_dev,
 968                                 frames[i]->frame.buffer_phy,
 969                                 tbnet_frame_size(frames[i]), DMA_TO_DEVICE);
 970                 }
 971
 972                 return true;
 973         }
 974
 975         if (protocol == htons(ETH_P_8021Q)) {
 976                 struct vlan_hdr *vhdr, vh;
 977
 978                 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(vh), &vh);
 979                 if (!vhdr)
 980                         return false;
 981
 982                 protocol = vhdr->h_vlan_encapsulated_proto;
 983         }
 984
 985         /* Data points on the beginning of packet.
 986          * Check is the checksum absolute place in the packet.
 987          * ipcso will update IP checksum.
 988          * tucso will update TCP/UPD checksum.
 989          */
 990         if (protocol == htons(ETH_P_IP)) {
 991                 __sum16 *ipcso = dest + ((void *)&(ip_hdr(skb)->check) - data);
 992
 993                 *ipcso = 0;
 994                 *ipcso = ip_fast_csum(dest + skb_network_offset(skb),
 995                                       ip_hdr(skb)->ihl);
 996
 997                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
 998                         tucso = dest + ((void *)&(tcp_hdr(skb)->check) - data);
 999                 else if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1000                         tucso = dest + ((void *)&(udp_hdr(skb)->check) - data);
1001                 else
1002                         return false;
1003
1004                 *tucso = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1005                                             ip_hdr(skb)->daddr, 0,
1006                                             ip_hdr(skb)->protocol, 0);
1007         } else if (skb_is_gso_v6(skb)) {
1008                 tucso = dest + ((void *)&(tcp_hdr(skb)->check) - data);
1009                 *tucso = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1010                                           &ipv6_hdr(skb)->daddr, 0,
1011                                           IPPROTO_TCP, 0);
1012                 return false;
1013         } else if (protocol == htons(ETH_P_IPV6)) {
1014                 tucso = dest + skb_checksum_start_offset(skb) + skb->csum_offset;
1015                 *tucso = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1016                                           &ipv6_hdr(skb)->daddr, 0,
1017                                           ipv6_hdr(skb)->nexthdr, 0);
1018         } else {
1019                 return false;
1020         }
1021
1022         /* First frame was headers, rest of the frames contain data.
1023          * Calculate checksum over each frame.
1024          */
1025         for (i = 0; i < frame_count; i++) {
1026                 hdr = page_address(frames[i]->page);
1027                 dest = (void *)(hdr + 1) + offset;
1028                 len = le32_to_cpu(hdr->frame_size) - offset;
1029                 wsum = csum_partial(dest, len, wsum);
1030                 hdr->frame_count = cpu_to_le32(frame_count);
1031
1032                 offset = 0;
1033         }
1034
1035         *tucso = csum_fold(wsum);
1036
1037         /* Checksum is finally calculated and we don't touch the memory
1038          * anymore, so DMA sync the frames now.
1039          */
1040         for (i = 0; i < frame_count; i++) {
1041                 dma_sync_single_for_device(dma_dev, frames[i]->frame.buffer_phy,
1042                         tbnet_frame_size(frames[i]), DMA_TO_DEVICE);
1043         }
1044
1045         return true;
1046 }
1047
1048 static void *tbnet_kmap_frag(struct sk_buff *skb, unsigned int frag_num,
1049                              unsigned int *len)
1050 {
1051         const skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_num];
1052
1053         *len = skb_frag_size(frag);
1054         return kmap_atomic(skb_frag_page(frag)) + skb_frag_off(frag);
1055 }
1056
1057 static netdev_tx_t tbnet_start_xmit(struct sk_buff *skb,
1058                                     struct net_device *dev)
1059 {
1060         struct tbnet *net = netdev_priv(dev);
1061         struct tbnet_frame *frames[MAX_SKB_FRAGS];
1062         u16 frame_id = atomic_read(&net->frame_id);
1063         struct thunderbolt_ip_frame_header *hdr;
1064         unsigned int len = skb_headlen(skb);
1065         unsigned int data_len = skb->len;
1066         unsigned int nframes, i;
1067         unsigned int frag = 0;
1068         void *src = skb->data;
1069         u32 frame_index = 0;
1070         bool unmap = false;
1071         void *dest;
1072
1073         nframes = DIV_ROUND_UP(data_len, TBNET_MAX_PAYLOAD_SIZE);
1074         if (tbnet_available_buffers(&net->tx_ring) < nframes) {
1075                 netif_stop_queue(net->dev);
1076                 return NETDEV_TX_BUSY;
1077         }
1078
1079         frames[frame_index] = tbnet_get_tx_buffer(net);
1080         if (!frames[frame_index])
1081                 goto err_drop;
1082
1083         hdr = page_address(frames[frame_index]->page);
1084         dest = hdr + 1;
1085
1086         /* If overall packet is bigger than the frame data size */
1087         while (data_len > TBNET_MAX_PAYLOAD_SIZE) {
1088                 unsigned int size_left = TBNET_MAX_PAYLOAD_SIZE;
1089
1090                 hdr->frame_size = cpu_to_le32(TBNET_MAX_PAYLOAD_SIZE);
1091                 hdr->frame_index = cpu_to_le16(frame_index);
1092                 hdr->frame_id = cpu_to_le16(frame_id);
1093
1094                 do {
1095                         if (len > size_left) {
1096                                 /* Copy data onto Tx buffer data with
1097                                  * full frame size then break and go to
1098                                  * next frame
1099                                  */
1100                                 memcpy(dest, src, size_left);
1101                                 len -= size_left;
1102                                 dest += size_left;
1103                                 src += size_left;
1104                                 break;
1105                         }
1106
1107                         memcpy(dest, src, len);
1108                         size_left -= len;
1109                         dest += len;
1110
1111                         if (unmap) {
1112                                 kunmap_atomic(src);
1113                                 unmap = false;
1114                         }
1115
1116                         /* Ensure all fragments have been processed */
1117                         if (frag < skb_shinfo(skb)->nr_frags) {
1118                                 /* Map and then unmap quickly */
1119                                 src = tbnet_kmap_frag(skb, frag++, &len);
1120                                 unmap = true;
1121                         } else if (unlikely(size_left > 0)) {
1122                                 goto err_drop;
1123                         }
1124                 } while (size_left > 0);
1125
1126                 data_len -= TBNET_MAX_PAYLOAD_SIZE;
1127                 frame_index++;
1128
1129                 frames[frame_index] = tbnet_get_tx_buffer(net);
1130                 if (!frames[frame_index])
1131                         goto err_drop;
1132
1133                 hdr = page_address(frames[frame_index]->page);
1134                 dest = hdr + 1;
1135         }
1136
1137         hdr->frame_size = cpu_to_le32(data_len);
1138         hdr->frame_index = cpu_to_le16(frame_index);
1139         hdr->frame_id = cpu_to_le16(frame_id);
1140
1141         frames[frame_index]->frame.size = data_len + sizeof(*hdr);
1142
1143         /* In case the remaining data_len is smaller than a frame */
1144         while (len < data_len) {
1145                 memcpy(dest, src, len);
1146                 data_len -= len;
1147                 dest += len;
1148
1149                 if (unmap) {
1150                         kunmap_atomic(src);
1151                         unmap = false;
1152                 }
1153
1154                 if (frag < skb_shinfo(skb)->nr_frags) {
1155                         src = tbnet_kmap_frag(skb, frag++, &len);
1156                         unmap = true;
1157                 } else if (unlikely(data_len > 0)) {
1158                         goto err_drop;
1159                 }
1160         }
1161
1162         memcpy(dest, src, data_len);
1163
1164         if (unmap)
1165                 kunmap_atomic(src);
1166
1167         if (!tbnet_xmit_csum_and_map(net, skb, frames, frame_index + 1))
1168                 goto err_drop;
1169
1170         for (i = 0; i < frame_index + 1; i++)
1171                 tb_ring_tx(net->tx_ring.ring, &frames[i]->frame);
1172
1173         if (net->svc->prtcstns & TBNET_MATCH_FRAGS_ID)
1174                 atomic_inc(&net->frame_id);
1175
1176         net->stats.tx_packets++;
1177         net->stats.tx_bytes += skb->len;
1178
1179         dev_consume_skb_any(skb);
1180
1181         return NETDEV_TX_OK;
1182
1183 err_drop:
1184         /* We can re-use the buffers */
1185         net->tx_ring.cons -= frame_index;
1186
1187         dev_kfree_skb_any(skb);
1188         net->stats.tx_errors++;
1189
1190         return NETDEV_TX_OK;
1191 }
1192
1193 static void tbnet_get_stats64(struct net_device *dev,
1194                               struct rtnl_link_stats64 *stats)
1195 {
1196         struct tbnet *net = netdev_priv(dev);
1197
1198         stats->tx_packets = net->stats.tx_packets;
1199         stats->rx_packets = net->stats.rx_packets;
1200         stats->tx_bytes = net->stats.tx_bytes;
1201         stats->rx_bytes = net->stats.rx_bytes;
1202         stats->rx_errors = net->stats.rx_errors + net->stats.rx_length_errors +
1203                 net->stats.rx_over_errors + net->stats.rx_crc_errors +
1204                 net->stats.rx_missed_errors;
1205         stats->tx_errors = net->stats.tx_errors;
1206         stats->rx_length_errors = net->stats.rx_length_errors;
1207         stats->rx_over_errors = net->stats.rx_over_errors;
1208         stats->rx_crc_errors = net->stats.rx_crc_errors;
1209         stats->rx_missed_errors = net->stats.rx_missed_errors;
1210 }
1211
1212 static const struct net_device_ops tbnet_netdev_ops = {
1213         .ndo_open = tbnet_open,
1214         .ndo_stop = tbnet_stop,
1215         .ndo_start_xmit = tbnet_start_xmit,
1216         .ndo_get_stats64 = tbnet_get_stats64,
1217 };
1218
1219 static void tbnet_generate_mac(struct net_device *dev)
1220 {
1221         const struct tbnet *net = netdev_priv(dev);
1222         const struct tb_xdomain *xd = net->xd;
1223         u8 addr[ETH_ALEN];
1224         u8 phy_port;
1225         u32 hash;
1226
1227         phy_port = tb_phy_port_from_link(TBNET_L0_PORT_NUM(xd->route));
1228
1229         /* Unicast and locally administered MAC */
1230         addr[0] = phy_port << 4 | 0x02;
1231         hash = jhash2((u32 *)xd->local_uuid, 4, 0);
1232         memcpy(addr + 1, &hash, sizeof(hash));
1233         hash = jhash2((u32 *)xd->local_uuid, 4, hash);
1234         addr[5] = hash & 0xff;
1235         eth_hw_addr_set(dev, addr);
1236 }
1237
1238 static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id)
1239 {
1240         struct tb_xdomain *xd = tb_service_parent(svc);
1241         struct net_device *dev;
1242         struct tbnet *net;
1243         int ret;
1244
1245         dev = alloc_etherdev(sizeof(*net));
1246         if (!dev)
1247                 return -ENOMEM;
1248
1249         SET_NETDEV_DEV(dev, &svc->dev);
1250
1251         net = netdev_priv(dev);
1252         INIT_DELAYED_WORK(&net->login_work, tbnet_login_work);
1253         INIT_WORK(&net->connected_work, tbnet_connected_work);
1254         INIT_WORK(&net->disconnect_work, tbnet_disconnect_work);
1255         mutex_init(&net->connection_lock);
1256         atomic_set(&net->command_id, 0);
1257         atomic_set(&net->frame_id, 0);
1258         net->svc = svc;
1259         net->dev = dev;
1260         net->xd = xd;
1261
1262         tbnet_generate_mac(dev);
1263
1264         strcpy(dev->name, "thunderbolt%d");
1265         dev->netdev_ops = &tbnet_netdev_ops;
1266
1267         /* ThunderboltIP takes advantage of TSO packets but instead of
1268          * segmenting them we just split the packet into Thunderbolt
1269          * frames (maximum payload size of each frame is 4084 bytes) and
1270          * calculate checksum over the whole packet here.
1271          *
1272          * The receiving side does the opposite if the host OS supports
1273          * LRO, otherwise it needs to split the large packet into MTU
1274          * sized smaller packets.
1275          *
1276          * In order to receive large packets from the networking stack,
1277          * we need to announce support for most of the offloading
1278          * features here.
1279          */
1280         dev->hw_features = NETIF_F_SG | NETIF_F_ALL_TSO | NETIF_F_GRO |
1281                            NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
1282         dev->features = dev->hw_features | NETIF_F_HIGHDMA;
1283         dev->hard_header_len += sizeof(struct thunderbolt_ip_frame_header);
1284
1285         netif_napi_add(dev, &net->napi, tbnet_poll);
1286
1287         /* MTU range: 68 - 65522 */
1288         dev->min_mtu = ETH_MIN_MTU;
1289         dev->max_mtu = TBNET_MAX_MTU - ETH_HLEN;
1290
1291         net->handler.uuid = &tbnet_svc_uuid;
1292         net->handler.callback = tbnet_handle_packet;
1293         net->handler.data = net;
1294         tb_register_protocol_handler(&net->handler);
1295
1296         tb_service_set_drvdata(svc, net);
1297
1298         ret = register_netdev(dev);
1299         if (ret) {
1300                 tb_unregister_protocol_handler(&net->handler);
1301                 free_netdev(dev);
1302                 return ret;
1303         }
1304
1305         return 0;
1306 }
1307
1308 static void tbnet_remove(struct tb_service *svc)
1309 {
1310         struct tbnet *net = tb_service_get_drvdata(svc);
1311
1312         unregister_netdev(net->dev);
1313         tb_unregister_protocol_handler(&net->handler);
1314         free_netdev(net->dev);
1315 }
1316
1317 static void tbnet_shutdown(struct tb_service *svc)
1318 {
1319         tbnet_tear_down(tb_service_get_drvdata(svc), true);
1320 }
1321
1322 static int __maybe_unused tbnet_suspend(struct device *dev)
1323 {
1324         struct tb_service *svc = tb_to_service(dev);
1325         struct tbnet *net = tb_service_get_drvdata(svc);
1326
1327         stop_login(net);
1328         if (netif_running(net->dev)) {
1329                 netif_device_detach(net->dev);
1330                 tbnet_tear_down(net, true);
1331         }
1332
1333         tb_unregister_protocol_handler(&net->handler);
1334         return 0;
1335 }
1336
1337 static int __maybe_unused tbnet_resume(struct device *dev)
1338 {
1339         struct tb_service *svc = tb_to_service(dev);
1340         struct tbnet *net = tb_service_get_drvdata(svc);
1341
1342         tb_register_protocol_handler(&net->handler);
1343
1344         netif_carrier_off(net->dev);
1345         if (netif_running(net->dev)) {
1346                 netif_device_attach(net->dev);
1347                 start_login(net);
1348         }
1349
1350         return 0;
1351 }
1352
1353 static const struct dev_pm_ops tbnet_pm_ops = {
1354         SET_SYSTEM_SLEEP_PM_OPS(tbnet_suspend, tbnet_resume)
1355 };
1356
1357 static const struct tb_service_id tbnet_ids[] = {
1358         { TB_SERVICE("network", 1) },
1359         { },
1360 };
1361 MODULE_DEVICE_TABLE(tbsvc, tbnet_ids);
1362
1363 static struct tb_service_driver tbnet_driver = {
1364         .driver = {
1365                 .owner = THIS_MODULE,
1366                 .name = "thunderbolt-net",
1367                 .pm = &tbnet_pm_ops,
1368         },
1369         .probe = tbnet_probe,
1370         .remove = tbnet_remove,
1371         .shutdown = tbnet_shutdown,
1372         .id_table = tbnet_ids,
1373 };
1374
1375 static int __init tbnet_init(void)
1376 {
1377         unsigned int flags;
1378         int ret;
1379
1380         tbnet_dir = tb_property_create_dir(&tbnet_dir_uuid);
1381         if (!tbnet_dir)
1382                 return -ENOMEM;
1383
1384         tb_property_add_immediate(tbnet_dir, "prtcid", 1);
1385         tb_property_add_immediate(tbnet_dir, "prtcvers", 1);
1386         tb_property_add_immediate(tbnet_dir, "prtcrevs", 1);
1387
1388         flags = TBNET_MATCH_FRAGS_ID | TBNET_64K_FRAMES;
1389         if (tbnet_e2e)
1390                 flags |= TBNET_E2E;
1391         tb_property_add_immediate(tbnet_dir, "prtcstns", flags);
1392
1393         ret = tb_register_property_dir("network", tbnet_dir);
1394         if (ret) {
1395                 tb_property_free_dir(tbnet_dir);
1396                 return ret;
1397         }
1398
1399         return tb_register_service_driver(&tbnet_driver);
1400 }
1401 module_init(tbnet_init);
1402
1403 static void __exit tbnet_exit(void)
1404 {
1405         tb_unregister_service_driver(&tbnet_driver);
1406         tb_unregister_property_dir("network", tbnet_dir);
1407         tb_property_free_dir(tbnet_dir);
1408 }
1409 module_exit(tbnet_exit);
1410
1411 MODULE_AUTHOR("Amir Levy <amir.jer.levy@intel.com>");
1412 MODULE_AUTHOR("Michael Jamet <michael.jamet@intel.com>");
1413 MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
1414 MODULE_DESCRIPTION("Thunderbolt/USB4 network driver");
1415 MODULE_LICENSE("GPL v2");