net/ipv4/udp_tunnel_nic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (c) 2020 Facebook Inc.
   3
   4 #include <linux/ethtool_netlink.h>
   5 #include <linux/netdevice.h>
   6 #include <linux/slab.h>
   7 #include <linux/types.h>
   8 #include <linux/workqueue.h>
   9 #include <net/udp_tunnel.h>
  10 #include <net/vxlan.h>
  11
  12 enum udp_tunnel_nic_table_entry_flags {
  13         UDP_TUNNEL_NIC_ENTRY_ADD        = BIT(0),
  14         UDP_TUNNEL_NIC_ENTRY_DEL        = BIT(1),
  15         UDP_TUNNEL_NIC_ENTRY_OP_FAIL    = BIT(2),
  16         UDP_TUNNEL_NIC_ENTRY_FROZEN     = BIT(3),
  17 };
  18
  19 struct udp_tunnel_nic_table_entry {
  20         __be16 port;
  21         u8 type;
  22         u8 flags;
  23         u16 use_cnt;
  24 #define UDP_TUNNEL_NIC_USE_CNT_MAX      U16_MAX
  25         u8 hw_priv;
  26 };
  27
  28 /**
  29  * struct udp_tunnel_nic - UDP tunnel port offload state
  30  * @work:       async work for talking to hardware from process context
  31  * @dev:        netdev pointer
  32  * @need_sync:  at least one port start changed
  33  * @need_replay: space was freed, we need a replay of all ports
  34  * @work_pending: @work is currently scheduled
  35  * @n_tables:   number of tables under @entries
  36  * @missed:     bitmap of tables which overflown
  37  * @entries:    table of tables of ports currently offloaded
  38  */
  39 struct udp_tunnel_nic {
  40         struct work_struct work;
  41
  42         struct net_device *dev;
  43
  44         u8 need_sync:1;
  45         u8 need_replay:1;
  46         u8 work_pending:1;
  47
  48         unsigned int n_tables;
  49         unsigned long missed;
  50         struct udp_tunnel_nic_table_entry **entries;
  51 };
  52
  53 /* We ensure all work structs are done using driver state, but not the code.
  54  * We need a workqueue we can flush before module gets removed.
  55  */
  56 static struct workqueue_struct *udp_tunnel_nic_workqueue;
  57
  58 static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
  59 {
  60         switch (type) {
  61         case UDP_TUNNEL_TYPE_VXLAN:
  62                 return "vxlan";
  63         case UDP_TUNNEL_TYPE_GENEVE:
  64                 return "geneve";
  65         case UDP_TUNNEL_TYPE_VXLAN_GPE:
  66                 return "vxlan-gpe";
  67         default:
  68                 return "unknown";
  69         }
  70 }
  71
  72 static bool
  73 udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
  74 {
  75         return entry->use_cnt == 0 && !entry->flags;
  76 }
  77
  78 static bool
  79 udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry)
  80 {
  81         return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN);
  82 }
  83
  84 static bool
  85 udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
  86 {
  87         return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
  88 }
  89
  90 static void
  91 udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
  92 {
  93         if (!udp_tunnel_nic_entry_is_free(entry))
  94                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
  95 }
  96
  97 static void
  98 udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
  99 {
 100         entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
 101 }
 102
 103 static bool
 104 udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
 105 {
 106         return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
 107                                UDP_TUNNEL_NIC_ENTRY_DEL);
 108 }
 109
 110 static void
 111 udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
 112                            struct udp_tunnel_nic_table_entry *entry,
 113                            unsigned int flag)
 114 {
 115         entry->flags |= flag;
 116         utn->need_sync = 1;
 117 }
 118
 119 static void
 120 udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
 121                              struct udp_tunnel_info *ti)
 122 {
 123         memset(ti, 0, sizeof(*ti));
 124         ti->port = entry->port;
 125         ti->type = entry->type;
 126         ti->hw_priv = entry->hw_priv;
 127 }
 128
 129 static bool
 130 udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
 131 {
 132         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 133         unsigned int i, j;
 134
 135         for (i = 0; i < utn->n_tables; i++)
 136                 for (j = 0; j < info->tables[i].n_entries; j++)
 137                         if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 138                                 return false;
 139         return true;
 140 }
 141
 142 static bool
 143 udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 144 {
 145         const struct udp_tunnel_nic_table_info *table;
 146         unsigned int i, j;
 147
 148         if (!utn->missed)
 149                 return false;
 150
 151         for (i = 0; i < utn->n_tables; i++) {
 152                 table = &dev->udp_tunnel_nic_info->tables[i];
 153                 if (!test_bit(i, &utn->missed))
 154                         continue;
 155
 156                 for (j = 0; j < table->n_entries; j++)
 157                         if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
 158                                 return true;
 159         }
 160
 161         return false;
 162 }
 163
 164 static void
 165 __udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
 166                           unsigned int idx, struct udp_tunnel_info *ti)
 167 {
 168         struct udp_tunnel_nic_table_entry *entry;
 169         struct udp_tunnel_nic *utn;
 170
 171         utn = dev->udp_tunnel_nic;
 172         entry = &utn->entries[table][idx];
 173
 174         if (entry->use_cnt)
 175                 udp_tunnel_nic_ti_from_entry(entry, ti);
 176 }
 177
 178 static void
 179 __udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
 180                                unsigned int idx, u8 priv)
 181 {
 182         dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
 183 }
 184
 185 static void
 186 udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
 187                                  int err)
 188 {
 189         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 190
 191         WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 192                      entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);
 193
 194         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
 195             (!err || (err == -EEXIST && dodgy)))
 196                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;
 197
 198         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
 199             (!err || (err == -ENOENT && dodgy)))
 200                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;
 201
 202         if (!err)
 203                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 204         else
 205                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 206 }
 207
 208 static void
 209 udp_tunnel_nic_device_sync_one(struct net_device *dev,
 210                                struct udp_tunnel_nic *utn,
 211                                unsigned int table, unsigned int idx)
 212 {
 213         struct udp_tunnel_nic_table_entry *entry;
 214         struct udp_tunnel_info ti;
 215         int err;
 216
 217         entry = &utn->entries[table][idx];
 218         if (!udp_tunnel_nic_entry_is_queued(entry))
 219                 return;
 220
 221         udp_tunnel_nic_ti_from_entry(entry, &ti);
 222         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
 223                 err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
 224         else
 225                 err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
 226                                                            &ti);
 227         udp_tunnel_nic_entry_update_done(entry, err);
 228
 229         if (err)
 230                 netdev_warn(dev,
 231                             "UDP tunnel port sync failed port %d type %s: %d\n",
 232                             be16_to_cpu(entry->port),
 233                             udp_tunnel_nic_tunnel_type_name(entry->type),
 234                             err);
 235 }
 236
 237 static void
 238 udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
 239                                    struct udp_tunnel_nic *utn)
 240 {
 241         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 242         unsigned int i, j;
 243
 244         for (i = 0; i < utn->n_tables; i++)
 245                 for (j = 0; j < info->tables[i].n_entries; j++)
 246                         udp_tunnel_nic_device_sync_one(dev, utn, i, j);
 247 }
 248
 249 static void
 250 udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
 251                                     struct udp_tunnel_nic *utn)
 252 {
 253         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 254         unsigned int i, j;
 255         int err;
 256
 257         for (i = 0; i < utn->n_tables; i++) {
 258                 /* Find something that needs sync in this table */
 259                 for (j = 0; j < info->tables[i].n_entries; j++)
 260                         if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
 261                                 break;
 262                 if (j == info->tables[i].n_entries)
 263                         continue;
 264
 265                 err = info->sync_table(dev, i);
 266                 if (err)
 267                         netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
 268                                     i, err);
 269
 270                 for (j = 0; j < info->tables[i].n_entries; j++) {
 271                         struct udp_tunnel_nic_table_entry *entry;
 272
 273                         entry = &utn->entries[i][j];
 274                         if (udp_tunnel_nic_entry_is_queued(entry))
 275                                 udp_tunnel_nic_entry_update_done(entry, err);
 276                 }
 277         }
 278 }
 279
 280 static void
 281 __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 282 {
 283         if (!utn->need_sync)
 284                 return;
 285
 286         if (dev->udp_tunnel_nic_info->sync_table)
 287                 udp_tunnel_nic_device_sync_by_table(dev, utn);
 288         else
 289                 udp_tunnel_nic_device_sync_by_port(dev, utn);
 290
 291         utn->need_sync = 0;
 292         /* Can't replay directly here, in case we come from the tunnel driver's
 293          * notification - trying to replay may deadlock inside tunnel driver.
 294          */
 295         utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
 296 }
 297
 298 static void
 299 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 300 {
 301         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 302         bool may_sleep;
 303
 304         if (!utn->need_sync)
 305                 return;
 306
 307         /* Drivers which sleep in the callback need to update from
 308          * the workqueue, if we come from the tunnel driver's notification.
 309          */
 310         may_sleep = info->flags & UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
 311         if (!may_sleep)
 312                 __udp_tunnel_nic_device_sync(dev, utn);
 313         if (may_sleep || utn->need_replay) {
 314                 queue_work(udp_tunnel_nic_workqueue, &utn->work);
 315                 utn->work_pending = 1;
 316         }
 317 }
 318
 319 static bool
 320 udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
 321                                 struct udp_tunnel_info *ti)
 322 {
 323         return table->tunnel_types & ti->type;
 324 }
 325
 326 static bool
 327 udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
 328                           struct udp_tunnel_info *ti)
 329 {
 330         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 331         unsigned int i;
 332
 333         /* Special case IPv4-only NICs */
 334         if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
 335             ti->sa_family != AF_INET)
 336                 return false;
 337
 338         for (i = 0; i < utn->n_tables; i++)
 339                 if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
 340                         return true;
 341         return false;
 342 }
 343
 344 static int
 345 udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
 346                              struct udp_tunnel_info *ti)
 347 {
 348         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 349         struct udp_tunnel_nic_table_entry *entry;
 350         unsigned int i, j;
 351
 352         for (i = 0; i < utn->n_tables; i++)
 353                 for (j = 0; j < info->tables[i].n_entries; j++) {
 354                         entry = &utn->entries[i][j];
 355
 356                         if (!udp_tunnel_nic_entry_is_free(entry) &&
 357                             entry->port == ti->port &&
 358                             entry->type != ti->type) {
 359                                 __set_bit(i, &utn->missed);
 360                                 return true;
 361                         }
 362                 }
 363         return false;
 364 }
 365
 366 static void
 367 udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
 368                          unsigned int table, unsigned int idx, int use_cnt_adj)
 369 {
 370         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 371         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
 372         unsigned int from, to;
 373
 374         WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX);
 375
 376         /* If not going from used to unused or vice versa - all done.
 377          * For dodgy entries make sure we try to sync again (queue the entry).
 378          */
 379         entry->use_cnt += use_cnt_adj;
 380         if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
 381                 return;
 382
 383         /* Cancel the op before it was sent to the device, if possible,
 384          * otherwise we'd need to take special care to issue commands
 385          * in the same order the ports arrived.
 386          */
 387         if (use_cnt_adj < 0) {
 388                 from = UDP_TUNNEL_NIC_ENTRY_ADD;
 389                 to = UDP_TUNNEL_NIC_ENTRY_DEL;
 390         } else {
 391                 from = UDP_TUNNEL_NIC_ENTRY_DEL;
 392                 to = UDP_TUNNEL_NIC_ENTRY_ADD;
 393         }
 394
 395         if (entry->flags & from) {
 396                 entry->flags &= ~from;
 397                 if (!dodgy)
 398                         return;
 399         }
 400
 401         udp_tunnel_nic_entry_queue(utn, entry, to);
 402 }
 403
 404 static bool
 405 udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
 406                              unsigned int table, unsigned int idx,
 407                              struct udp_tunnel_info *ti, int use_cnt_adj)
 408 {
 409         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
 410
 411         if (udp_tunnel_nic_entry_is_free(entry) ||
 412             entry->port != ti->port ||
 413             entry->type != ti->type)
 414                 return false;
 415
 416         if (udp_tunnel_nic_entry_is_frozen(entry))
 417                 return true;
 418
 419         udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
 420         return true;
 421 }
 422
 423 /* Try to find existing matching entry and adjust its use count, instead of
 424  * adding a new one. Returns true if entry was found. In case of delete the
 425  * entry may have gotten removed in the process, in which case it will be
 426  * queued for removal.
 427  */
 428 static bool
 429 udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 430                             struct udp_tunnel_info *ti, int use_cnt_adj)
 431 {
 432         const struct udp_tunnel_nic_table_info *table;
 433         unsigned int i, j;
 434
 435         for (i = 0; i < utn->n_tables; i++) {
 436                 table = &dev->udp_tunnel_nic_info->tables[i];
 437                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 438                         continue;
 439
 440                 for (j = 0; j < table->n_entries; j++)
 441                         if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
 442                                                          use_cnt_adj))
 443                                 return true;
 444         }
 445
 446         return false;
 447 }
 448
 449 static bool
 450 udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 451                             struct udp_tunnel_info *ti)
 452 {
 453         return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
 454 }
 455
 456 static bool
 457 udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
 458                             struct udp_tunnel_info *ti)
 459 {
 460         return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
 461 }
 462
 463 static bool
 464 udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
 465                        struct udp_tunnel_info *ti)
 466 {
 467         const struct udp_tunnel_nic_table_info *table;
 468         unsigned int i, j;
 469
 470         for (i = 0; i < utn->n_tables; i++) {
 471                 table = &dev->udp_tunnel_nic_info->tables[i];
 472                 if (!udp_tunnel_nic_table_is_capable(table, ti))
 473                         continue;
 474
 475                 for (j = 0; j < table->n_entries; j++) {
 476                         struct udp_tunnel_nic_table_entry *entry;
 477
 478                         entry = &utn->entries[i][j];
 479                         if (!udp_tunnel_nic_entry_is_free(entry))
 480                                 continue;
 481
 482                         entry->port = ti->port;
 483                         entry->type = ti->type;
 484                         entry->use_cnt = 1;
 485                         udp_tunnel_nic_entry_queue(utn, entry,
 486                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 487                         return true;
 488                 }
 489
 490                 /* The different table may still fit this port in, but there
 491                  * are no devices currently which have multiple tables accepting
 492                  * the same tunnel type, and false positives are okay.
 493                  */
 494                 __set_bit(i, &utn->missed);
 495         }
 496
 497         return false;
 498 }
 499
 500 static void
 501 __udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
 502 {
 503         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 504         struct udp_tunnel_nic *utn;
 505
 506         utn = dev->udp_tunnel_nic;
 507         if (!utn)
 508                 return;
 509         if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
 510                 return;
 511         if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN &&
 512             ti->port == htons(IANA_VXLAN_UDP_PORT)) {
 513                 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
 514                         netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n");
 515                 return;
 516         }
 517
 518         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 519                 return;
 520
 521         /* It may happen that a tunnel of one type is removed and different
 522          * tunnel type tries to reuse its port before the device was informed.
 523          * Rely on utn->missed to re-add this port later.
 524          */
 525         if (udp_tunnel_nic_has_collision(dev, utn, ti))
 526                 return;
 527
 528         if (!udp_tunnel_nic_add_existing(dev, utn, ti))
 529                 udp_tunnel_nic_add_new(dev, utn, ti);
 530
 531         udp_tunnel_nic_device_sync(dev, utn);
 532 }
 533
 534 static void
 535 __udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
 536 {
 537         struct udp_tunnel_nic *utn;
 538
 539         utn = dev->udp_tunnel_nic;
 540         if (!utn)
 541                 return;
 542
 543         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
 544                 return;
 545
 546         udp_tunnel_nic_del_existing(dev, utn, ti);
 547
 548         udp_tunnel_nic_device_sync(dev, utn);
 549 }
 550
 551 static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
 552 {
 553         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 554         struct udp_tunnel_nic *utn;
 555         unsigned int i, j;
 556
 557         ASSERT_RTNL();
 558
 559         utn = dev->udp_tunnel_nic;
 560         if (!utn)
 561                 return;
 562
 563         utn->need_sync = false;
 564         for (i = 0; i < utn->n_tables; i++)
 565                 for (j = 0; j < info->tables[i].n_entries; j++) {
 566                         struct udp_tunnel_nic_table_entry *entry;
 567
 568                         entry = &utn->entries[i][j];
 569
 570                         entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
 571                                           UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
 572                         /* We don't release rtnl across ops */
 573                         WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
 574                         if (!entry->use_cnt)
 575                                 continue;
 576
 577                         udp_tunnel_nic_entry_queue(utn, entry,
 578                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
 579                 }
 580
 581         __udp_tunnel_nic_device_sync(dev, utn);
 582 }
 583
 584 static size_t
 585 __udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
 586 {
 587         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 588         struct udp_tunnel_nic *utn;
 589         unsigned int j;
 590         size_t size;
 591
 592         utn = dev->udp_tunnel_nic;
 593         if (!utn)
 594                 return 0;
 595
 596         size = 0;
 597         for (j = 0; j < info->tables[table].n_entries; j++) {
 598                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
 599                         continue;
 600
 601                 size += nla_total_size(0) +              /* _TABLE_ENTRY */
 602                         nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */
 603                         nla_total_size(sizeof(u32));     /* _ENTRY_TYPE */
 604         }
 605
 606         return size;
 607 }
 608
 609 static int
 610 __udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
 611                             struct sk_buff *skb)
 612 {
 613         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 614         struct udp_tunnel_nic *utn;
 615         struct nlattr *nest;
 616         unsigned int j;
 617
 618         utn = dev->udp_tunnel_nic;
 619         if (!utn)
 620                 return 0;
 621
 622         for (j = 0; j < info->tables[table].n_entries; j++) {
 623                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
 624                         continue;
 625
 626                 nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
 627                 if (!nest)
 628                         return -EMSGSIZE;
 629
 630                 if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
 631                                  utn->entries[table][j].port) ||
 632                     nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE,
 633                                 ilog2(utn->entries[table][j].type)))
 634                         goto err_cancel;
 635
 636                 nla_nest_end(skb, nest);
 637         }
 638
 639         return 0;
 640
 641 err_cancel:
 642         nla_nest_cancel(skb, nest);
 643         return -EMSGSIZE;
 644 }
 645
 646 static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
 647         .get_port       = __udp_tunnel_nic_get_port,
 648         .set_port_priv  = __udp_tunnel_nic_set_port_priv,
 649         .add_port       = __udp_tunnel_nic_add_port,
 650         .del_port       = __udp_tunnel_nic_del_port,
 651         .reset_ntf      = __udp_tunnel_nic_reset_ntf,
 652         .dump_size      = __udp_tunnel_nic_dump_size,
 653         .dump_write     = __udp_tunnel_nic_dump_write,
 654 };
 655
 656 static void
 657 udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
 658 {
 659         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 660         unsigned int i, j;
 661
 662         for (i = 0; i < utn->n_tables; i++)
 663                 for (j = 0; j < info->tables[i].n_entries; j++) {
 664                         int adj_cnt = -utn->entries[i][j].use_cnt;
 665
 666                         if (adj_cnt)
 667                                 udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
 668                 }
 669
 670         __udp_tunnel_nic_device_sync(dev, utn);
 671
 672         for (i = 0; i < utn->n_tables; i++)
 673                 memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
 674                                                       sizeof(**utn->entries)));
 675         WARN_ON(utn->need_sync);
 676         utn->need_replay = 0;
 677 }
 678
 679 static void
 680 udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
 681 {
 682         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 683         struct udp_tunnel_nic_shared_node *node;
 684         unsigned int i, j;
 685
 686         /* Freeze all the ports we are already tracking so that the replay
 687          * does not double up the refcount.
 688          */
 689         for (i = 0; i < utn->n_tables; i++)
 690                 for (j = 0; j < info->tables[i].n_entries; j++)
 691                         udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
 692         utn->missed = 0;
 693         utn->need_replay = 0;
 694
 695         if (!info->shared) {
 696                 udp_tunnel_get_rx_info(dev);
 697         } else {
 698                 list_for_each_entry(node, &info->shared->devices, list)
 699                         udp_tunnel_get_rx_info(node->dev);
 700         }
 701
 702         for (i = 0; i < utn->n_tables; i++)
 703                 for (j = 0; j < info->tables[i].n_entries; j++)
 704                         udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
 705 }
 706
 707 static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
 708 {
 709         struct udp_tunnel_nic *utn =
 710                 container_of(work, struct udp_tunnel_nic, work);
 711
 712         rtnl_lock();
 713         utn->work_pending = 0;
 714         __udp_tunnel_nic_device_sync(utn->dev, utn);
 715
 716         if (utn->need_replay)
 717                 udp_tunnel_nic_replay(utn->dev, utn);
 718         rtnl_unlock();
 719 }
 720
 721 static struct udp_tunnel_nic *
 722 udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
 723                      unsigned int n_tables)
 724 {
 725         struct udp_tunnel_nic *utn;
 726         unsigned int i;
 727
 728         utn = kzalloc(sizeof(*utn), GFP_KERNEL);
 729         if (!utn)
 730                 return NULL;
 731         utn->n_tables = n_tables;
 732         INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);
 733
 734         utn->entries = kmalloc_array(n_tables, sizeof(void *), GFP_KERNEL);
 735         if (!utn->entries)
 736                 goto err_free_utn;
 737
 738         for (i = 0; i < n_tables; i++) {
 739                 utn->entries[i] = kcalloc(info->tables[i].n_entries,
 740                                           sizeof(*utn->entries[i]), GFP_KERNEL);
 741                 if (!utn->entries[i])
 742                         goto err_free_prev_entries;
 743         }
 744
 745         return utn;
 746
 747 err_free_prev_entries:
 748         while (i--)
 749                 kfree(utn->entries[i]);
 750         kfree(utn->entries);
 751 err_free_utn:
 752         kfree(utn);
 753         return NULL;
 754 }
 755
 756 static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn)
 757 {
 758         unsigned int i;
 759
 760         for (i = 0; i < utn->n_tables; i++)
 761                 kfree(utn->entries[i]);
 762         kfree(utn->entries);
 763         kfree(utn);
 764 }
 765
 766 static int udp_tunnel_nic_register(struct net_device *dev)
 767 {
 768         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 769         struct udp_tunnel_nic_shared_node *node = NULL;
 770         struct udp_tunnel_nic *utn;
 771         unsigned int n_tables, i;
 772
 773         BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
 774                      UDP_TUNNEL_NIC_MAX_TABLES);
 775         /* Expect use count of at most 2 (IPv4, IPv6) per device */
 776         BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX <
 777                      UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2);
 778
 779         /* Check that the driver info is sane */
 780         if (WARN_ON(!info->set_port != !info->unset_port) ||
 781             WARN_ON(!info->set_port == !info->sync_table) ||
 782             WARN_ON(!info->tables[0].n_entries))
 783                 return -EINVAL;
 784
 785         if (WARN_ON(info->shared &&
 786                     info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 787                 return -EINVAL;
 788
 789         n_tables = 1;
 790         for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
 791                 if (!info->tables[i].n_entries)
 792                         continue;
 793
 794                 n_tables++;
 795                 if (WARN_ON(!info->tables[i - 1].n_entries))
 796                         return -EINVAL;
 797         }
 798
 799         /* Create UDP tunnel state structures */
 800         if (info->shared) {
 801                 node = kzalloc(sizeof(*node), GFP_KERNEL);
 802                 if (!node)
 803                         return -ENOMEM;
 804
 805                 node->dev = dev;
 806         }
 807
 808         if (info->shared && info->shared->udp_tunnel_nic_info) {
 809                 utn = info->shared->udp_tunnel_nic_info;
 810         } else {
 811                 utn = udp_tunnel_nic_alloc(info, n_tables);
 812                 if (!utn) {
 813                         kfree(node);
 814                         return -ENOMEM;
 815                 }
 816         }
 817
 818         if (info->shared) {
 819                 if (!info->shared->udp_tunnel_nic_info) {
 820                         INIT_LIST_HEAD(&info->shared->devices);
 821                         info->shared->udp_tunnel_nic_info = utn;
 822                 }
 823
 824                 list_add_tail(&node->list, &info->shared->devices);
 825         }
 826
 827         utn->dev = dev;
 828         dev_hold(dev);
 829         dev->udp_tunnel_nic = utn;
 830
 831         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 832                 udp_tunnel_get_rx_info(dev);
 833
 834         return 0;
 835 }
 836
 837 static void
 838 udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
 839 {
 840         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
 841
 842         /* For a shared table remove this dev from the list of sharing devices
 843          * and if there are other devices just detach.
 844          */
 845         if (info->shared) {
 846                 struct udp_tunnel_nic_shared_node *node, *first;
 847
 848                 list_for_each_entry(node, &info->shared->devices, list)
 849                         if (node->dev == dev)
 850                                 break;
 851                 if (list_entry_is_head(node, &info->shared->devices, list))
 852                         return;
 853
 854                 list_del(&node->list);
 855                 kfree(node);
 856
 857                 first = list_first_entry_or_null(&info->shared->devices,
 858                                                  typeof(*first), list);
 859                 if (first) {
 860                         udp_tunnel_drop_rx_info(dev);
 861                         utn->dev = first->dev;
 862                         goto release_dev;
 863                 }
 864
 865                 info->shared->udp_tunnel_nic_info = NULL;
 866         }
 867
 868         /* Flush before we check work, so we don't waste time adding entries
 869          * from the work which we will boot immediately.
 870          */
 871         udp_tunnel_nic_flush(dev, utn);
 872
 873         /* Wait for the work to be done using the state, netdev core will
 874          * retry unregister until we give up our reference on this device.
 875          */
 876         if (utn->work_pending)
 877                 return;
 878
 879         udp_tunnel_nic_free(utn);
 880 release_dev:
 881         dev->udp_tunnel_nic = NULL;
 882         dev_put(dev);
 883 }
 884
 885 static int
 886 udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
 887                                unsigned long event, void *ptr)
 888 {
 889         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 890         const struct udp_tunnel_nic_info *info;
 891         struct udp_tunnel_nic *utn;
 892
 893         info = dev->udp_tunnel_nic_info;
 894         if (!info)
 895                 return NOTIFY_DONE;
 896
 897         if (event == NETDEV_REGISTER) {
 898                 int err;
 899
 900                 err = udp_tunnel_nic_register(dev);
 901                 if (err)
 902                         netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
 903                 return notifier_from_errno(err);
 904         }
 905         /* All other events will need the udp_tunnel_nic state */
 906         utn = dev->udp_tunnel_nic;
 907         if (!utn)
 908                 return NOTIFY_DONE;
 909
 910         if (event == NETDEV_UNREGISTER) {
 911                 udp_tunnel_nic_unregister(dev, utn);
 912                 return NOTIFY_OK;
 913         }
 914
 915         /* All other events only matter if NIC has to be programmed open */
 916         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
 917                 return NOTIFY_DONE;
 918
 919         if (event == NETDEV_UP) {
 920                 WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
 921                 udp_tunnel_get_rx_info(dev);
 922                 return NOTIFY_OK;
 923         }
 924         if (event == NETDEV_GOING_DOWN) {
 925                 udp_tunnel_nic_flush(dev, utn);
 926                 return NOTIFY_OK;
 927         }
 928
 929         return NOTIFY_DONE;
 930 }
 931
 932 static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
 933         .notifier_call = udp_tunnel_nic_netdevice_event,
 934 };
 935
 936 static int __init udp_tunnel_nic_init_module(void)
 937 {
 938         int err;
 939
 940         udp_tunnel_nic_workqueue = alloc_ordered_workqueue("udp_tunnel_nic", 0);
 941         if (!udp_tunnel_nic_workqueue)
 942                 return -ENOMEM;
 943
 944         rtnl_lock();
 945         udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
 946         rtnl_unlock();
 947
 948         err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 949         if (err)
 950                 goto err_unset_ops;
 951
 952         return 0;
 953
 954 err_unset_ops:
 955         rtnl_lock();
 956         udp_tunnel_nic_ops = NULL;
 957         rtnl_unlock();
 958         destroy_workqueue(udp_tunnel_nic_workqueue);
 959         return err;
 960 }
 961 late_initcall(udp_tunnel_nic_init_module);
 962
 963 static void __exit udp_tunnel_nic_cleanup_module(void)
 964 {
 965         unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);
 966
 967         rtnl_lock();
 968         udp_tunnel_nic_ops = NULL;
 969         rtnl_unlock();
 970
 971         destroy_workqueue(udp_tunnel_nic_workqueue);
 972 }
 973 module_exit(udp_tunnel_nic_cleanup_module);
 974
 975 MODULE_LICENSE("GPL");