drivers/thunderbolt/tb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Thunderbolt driver - bus logic (NHI independent)
   4  *
   5  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
   6  * Copyright (C) 2019, Intel Corporation
   7  */
   8
   9 #include <linux/slab.h>
  10 #include <linux/errno.h>
  11 #include <linux/delay.h>
  12 #include <linux/pm_runtime.h>
  13 #include <linux/platform_data/x86/apple.h>
  14
  15 #include "tb.h"
  16 #include "tb_regs.h"
  17 #include "tunnel.h"
  18
  19 #define TB_TIMEOUT      100 /* ms */
  20
  21 /**
  22  * struct tb_cm - Simple Thunderbolt connection manager
  23  * @tunnel_list: List of active tunnels
  24  * @dp_resources: List of available DP resources for DP tunneling
  25  * @hotplug_active: tb_handle_hotplug will stop progressing plug
  26  *                  events and exit if this is not set (it needs to
  27  *                  acquire the lock one more time). Used to drain wq
  28  *                  after cfg has been paused.
  29  * @remove_work: Work used to remove any unplugged routers after
  30  *               runtime resume
  31  */
  32 struct tb_cm {
  33         struct list_head tunnel_list;
  34         struct list_head dp_resources;
  35         bool hotplug_active;
  36         struct delayed_work remove_work;
  37 };
  38
  39 static inline struct tb *tcm_to_tb(struct tb_cm *tcm)
  40 {
  41         return ((void *)tcm - sizeof(struct tb));
  42 }
  43
  44 struct tb_hotplug_event {
  45         struct work_struct work;
  46         struct tb *tb;
  47         u64 route;
  48         u8 port;
  49         bool unplug;
  50 };
  51
  52 static void tb_handle_hotplug(struct work_struct *work);
  53
  54 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
  55 {
  56         struct tb_hotplug_event *ev;
  57
  58         ev = kmalloc(sizeof(*ev), GFP_KERNEL);
  59         if (!ev)
  60                 return;
  61
  62         ev->tb = tb;
  63         ev->route = route;
  64         ev->port = port;
  65         ev->unplug = unplug;
  66         INIT_WORK(&ev->work, tb_handle_hotplug);
  67         queue_work(tb->wq, &ev->work);
  68 }
  69
  70 /* enumeration & hot plug handling */
  71
  72 static void tb_add_dp_resources(struct tb_switch *sw)
  73 {
  74         struct tb_cm *tcm = tb_priv(sw->tb);
  75         struct tb_port *port;
  76
  77         tb_switch_for_each_port(sw, port) {
  78                 if (!tb_port_is_dpin(port))
  79                         continue;
  80
  81                 if (!tb_switch_query_dp_resource(sw, port))
  82                         continue;
  83
  84                 list_add_tail(&port->list, &tcm->dp_resources);
  85                 tb_port_dbg(port, "DP IN resource available\n");
  86         }
  87 }
  88
  89 static void tb_remove_dp_resources(struct tb_switch *sw)
  90 {
  91         struct tb_cm *tcm = tb_priv(sw->tb);
  92         struct tb_port *port, *tmp;
  93
  94         /* Clear children resources first */
  95         tb_switch_for_each_port(sw, port) {
  96                 if (tb_port_has_remote(port))
  97                         tb_remove_dp_resources(port->remote->sw);
  98         }
  99
 100         list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {
 101                 if (port->sw == sw) {
 102                         tb_port_dbg(port, "DP OUT resource unavailable\n");
 103                         list_del_init(&port->list);
 104                 }
 105         }
 106 }
 107
 108 static void tb_switch_discover_tunnels(struct tb_switch *sw,
 109                                        struct list_head *list,
 110                                        bool alloc_hopids)
 111 {
 112         struct tb *tb = sw->tb;
 113         struct tb_port *port;
 114
 115         tb_switch_for_each_port(sw, port) {
 116                 struct tb_tunnel *tunnel = NULL;
 117
 118                 switch (port->config.type) {
 119                 case TB_TYPE_DP_HDMI_IN:
 120                         tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids);
 121                         /*
 122                          * In case of DP tunnel exists, change host router's
 123                          * 1st children TMU mode to HiFi for CL0s to work.
 124                          */
 125                         if (tunnel)
 126                                 tb_switch_enable_tmu_1st_child(tb->root_switch,
 127                                                 TB_SWITCH_TMU_RATE_HIFI);
 128                         break;
 129
 130                 case TB_TYPE_PCIE_DOWN:
 131                         tunnel = tb_tunnel_discover_pci(tb, port, alloc_hopids);
 132                         break;
 133
 134                 case TB_TYPE_USB3_DOWN:
 135                         tunnel = tb_tunnel_discover_usb3(tb, port, alloc_hopids);
 136                         break;
 137
 138                 default:
 139                         break;
 140                 }
 141
 142                 if (tunnel)
 143                         list_add_tail(&tunnel->list, list);
 144         }
 145
 146         tb_switch_for_each_port(sw, port) {
 147                 if (tb_port_has_remote(port)) {
 148                         tb_switch_discover_tunnels(port->remote->sw, list,
 149                                                    alloc_hopids);
 150                 }
 151         }
 152 }
 153
 154 static void tb_discover_tunnels(struct tb *tb)
 155 {
 156         struct tb_cm *tcm = tb_priv(tb);
 157         struct tb_tunnel *tunnel;
 158
 159         tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true);
 160
 161         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 162                 if (tb_tunnel_is_pci(tunnel)) {
 163                         struct tb_switch *parent = tunnel->dst_port->sw;
 164
 165                         while (parent != tunnel->src_port->sw) {
 166                                 parent->boot = true;
 167                                 parent = tb_switch_parent(parent);
 168                         }
 169                 } else if (tb_tunnel_is_dp(tunnel)) {
 170                         /* Keep the domain from powering down */
 171                         pm_runtime_get_sync(&tunnel->src_port->sw->dev);
 172                         pm_runtime_get_sync(&tunnel->dst_port->sw->dev);
 173                 }
 174         }
 175 }
 176
 177 static int tb_port_configure_xdomain(struct tb_port *port)
 178 {
 179         if (tb_switch_is_usb4(port->sw))
 180                 return usb4_port_configure_xdomain(port);
 181         return tb_lc_configure_xdomain(port);
 182 }
 183
 184 static void tb_port_unconfigure_xdomain(struct tb_port *port)
 185 {
 186         if (tb_switch_is_usb4(port->sw))
 187                 usb4_port_unconfigure_xdomain(port);
 188         else
 189                 tb_lc_unconfigure_xdomain(port);
 190
 191         tb_port_enable(port->dual_link_port);
 192 }
 193
 194 static void tb_scan_xdomain(struct tb_port *port)
 195 {
 196         struct tb_switch *sw = port->sw;
 197         struct tb *tb = sw->tb;
 198         struct tb_xdomain *xd;
 199         u64 route;
 200
 201         if (!tb_is_xdomain_enabled())
 202                 return;
 203
 204         route = tb_downstream_route(port);
 205         xd = tb_xdomain_find_by_route(tb, route);
 206         if (xd) {
 207                 tb_xdomain_put(xd);
 208                 return;
 209         }
 210
 211         xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,
 212                               NULL);
 213         if (xd) {
 214                 tb_port_at(route, sw)->xdomain = xd;
 215                 tb_port_configure_xdomain(port);
 216                 tb_xdomain_add(xd);
 217         }
 218 }
 219
 220 static int tb_enable_tmu(struct tb_switch *sw)
 221 {
 222         int ret;
 223
 224         /* If it is already enabled in correct mode, don't touch it */
 225         if (tb_switch_tmu_is_enabled(sw, sw->tmu.unidirectional_request))
 226                 return 0;
 227
 228         ret = tb_switch_tmu_disable(sw);
 229         if (ret)
 230                 return ret;
 231
 232         ret = tb_switch_tmu_post_time(sw);
 233         if (ret)
 234                 return ret;
 235
 236         return tb_switch_tmu_enable(sw);
 237 }
 238
 239 /**
 240  * tb_find_unused_port() - return the first inactive port on @sw
 241  * @sw: Switch to find the port on
 242  * @type: Port type to look for
 243  */
 244 static struct tb_port *tb_find_unused_port(struct tb_switch *sw,
 245                                            enum tb_port_type type)
 246 {
 247         struct tb_port *port;
 248
 249         tb_switch_for_each_port(sw, port) {
 250                 if (tb_is_upstream_port(port))
 251                         continue;
 252                 if (port->config.type != type)
 253                         continue;
 254                 if (!port->cap_adap)
 255                         continue;
 256                 if (tb_port_is_enabled(port))
 257                         continue;
 258                 return port;
 259         }
 260         return NULL;
 261 }
 262
 263 static struct tb_port *tb_find_usb3_down(struct tb_switch *sw,
 264                                          const struct tb_port *port)
 265 {
 266         struct tb_port *down;
 267
 268         down = usb4_switch_map_usb3_down(sw, port);
 269         if (down && !tb_usb3_port_is_enabled(down))
 270                 return down;
 271         return NULL;
 272 }
 273
 274 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,
 275                                         struct tb_port *src_port,
 276                                         struct tb_port *dst_port)
 277 {
 278         struct tb_cm *tcm = tb_priv(tb);
 279         struct tb_tunnel *tunnel;
 280
 281         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 282                 if (tunnel->type == type &&
 283                     ((src_port && src_port == tunnel->src_port) ||
 284                      (dst_port && dst_port == tunnel->dst_port))) {
 285                         return tunnel;
 286                 }
 287         }
 288
 289         return NULL;
 290 }
 291
 292 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,
 293                                                    struct tb_port *src_port,
 294                                                    struct tb_port *dst_port)
 295 {
 296         struct tb_port *port, *usb3_down;
 297         struct tb_switch *sw;
 298
 299         /* Pick the router that is deepest in the topology */
 300         if (dst_port->sw->config.depth > src_port->sw->config.depth)
 301                 sw = dst_port->sw;
 302         else
 303                 sw = src_port->sw;
 304
 305         /* Can't be the host router */
 306         if (sw == tb->root_switch)
 307                 return NULL;
 308
 309         /* Find the downstream USB4 port that leads to this router */
 310         port = tb_port_at(tb_route(sw), tb->root_switch);
 311         /* Find the corresponding host router USB3 downstream port */
 312         usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);
 313         if (!usb3_down)
 314                 return NULL;
 315
 316         return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);
 317 }
 318
 319 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,
 320         struct tb_port *dst_port, int *available_up, int *available_down)
 321 {
 322         int usb3_consumed_up, usb3_consumed_down, ret;
 323         struct tb_cm *tcm = tb_priv(tb);
 324         struct tb_tunnel *tunnel;
 325         struct tb_port *port;
 326
 327         tb_port_dbg(dst_port, "calculating available bandwidth\n");
 328
 329         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 330         if (tunnel) {
 331                 ret = tb_tunnel_consumed_bandwidth(tunnel, &usb3_consumed_up,
 332                                                    &usb3_consumed_down);
 333                 if (ret)
 334                         return ret;
 335         } else {
 336                 usb3_consumed_up = 0;
 337                 usb3_consumed_down = 0;
 338         }
 339
 340         *available_up = *available_down = 40000;
 341
 342         /* Find the minimum available bandwidth over all links */
 343         tb_for_each_port_on_path(src_port, dst_port, port) {
 344                 int link_speed, link_width, up_bw, down_bw;
 345
 346                 if (!tb_port_is_null(port))
 347                         continue;
 348
 349                 if (tb_is_upstream_port(port)) {
 350                         link_speed = port->sw->link_speed;
 351                 } else {
 352                         link_speed = tb_port_get_link_speed(port);
 353                         if (link_speed < 0)
 354                                 return link_speed;
 355                 }
 356
 357                 link_width = port->bonded ? 2 : 1;
 358
 359                 up_bw = link_speed * link_width * 1000; /* Mb/s */
 360                 /* Leave 10% guard band */
 361                 up_bw -= up_bw / 10;
 362                 down_bw = up_bw;
 363
 364                 tb_port_dbg(port, "link total bandwidth %d Mb/s\n", up_bw);
 365
 366                 /*
 367                  * Find all DP tunnels that cross the port and reduce
 368                  * their consumed bandwidth from the available.
 369                  */
 370                 list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 371                         int dp_consumed_up, dp_consumed_down;
 372
 373                         if (!tb_tunnel_is_dp(tunnel))
 374                                 continue;
 375
 376                         if (!tb_tunnel_port_on_path(tunnel, port))
 377                                 continue;
 378
 379                         ret = tb_tunnel_consumed_bandwidth(tunnel,
 380                                                            &dp_consumed_up,
 381                                                            &dp_consumed_down);
 382                         if (ret)
 383                                 return ret;
 384
 385                         up_bw -= dp_consumed_up;
 386                         down_bw -= dp_consumed_down;
 387                 }
 388
 389                 /*
 390                  * If USB3 is tunneled from the host router down to the
 391                  * branch leading to port we need to take USB3 consumed
 392                  * bandwidth into account regardless whether it actually
 393                  * crosses the port.
 394                  */
 395                 up_bw -= usb3_consumed_up;
 396                 down_bw -= usb3_consumed_down;
 397
 398                 if (up_bw < *available_up)
 399                         *available_up = up_bw;
 400                 if (down_bw < *available_down)
 401                         *available_down = down_bw;
 402         }
 403
 404         if (*available_up < 0)
 405                 *available_up = 0;
 406         if (*available_down < 0)
 407                 *available_down = 0;
 408
 409         return 0;
 410 }
 411
 412 static int tb_release_unused_usb3_bandwidth(struct tb *tb,
 413                                             struct tb_port *src_port,
 414                                             struct tb_port *dst_port)
 415 {
 416         struct tb_tunnel *tunnel;
 417
 418         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 419         return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;
 420 }
 421
 422 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,
 423                                       struct tb_port *dst_port)
 424 {
 425         int ret, available_up, available_down;
 426         struct tb_tunnel *tunnel;
 427
 428         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 429         if (!tunnel)
 430                 return;
 431
 432         tb_dbg(tb, "reclaiming unused bandwidth for USB3\n");
 433
 434         /*
 435          * Calculate available bandwidth for the first hop USB3 tunnel.
 436          * That determines the whole USB3 bandwidth for this branch.
 437          */
 438         ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,
 439                                      &available_up, &available_down);
 440         if (ret) {
 441                 tb_warn(tb, "failed to calculate available bandwidth\n");
 442                 return;
 443         }
 444
 445         tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n",
 446                available_up, available_down);
 447
 448         tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);
 449 }
 450
 451 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)
 452 {
 453         struct tb_switch *parent = tb_switch_parent(sw);
 454         int ret, available_up, available_down;
 455         struct tb_port *up, *down, *port;
 456         struct tb_cm *tcm = tb_priv(tb);
 457         struct tb_tunnel *tunnel;
 458
 459         if (!tb_acpi_may_tunnel_usb3()) {
 460                 tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");
 461                 return 0;
 462         }
 463
 464         up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);
 465         if (!up)
 466                 return 0;
 467
 468         if (!sw->link_usb4)
 469                 return 0;
 470
 471         /*
 472          * Look up available down port. Since we are chaining it should
 473          * be found right above this switch.
 474          */
 475         port = tb_port_at(tb_route(sw), parent);
 476         down = tb_find_usb3_down(parent, port);
 477         if (!down)
 478                 return 0;
 479
 480         if (tb_route(parent)) {
 481                 struct tb_port *parent_up;
 482                 /*
 483                  * Check first that the parent switch has its upstream USB3
 484                  * port enabled. Otherwise the chain is not complete and
 485                  * there is no point setting up a new tunnel.
 486                  */
 487                 parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);
 488                 if (!parent_up || !tb_port_is_enabled(parent_up))
 489                         return 0;
 490
 491                 /* Make all unused bandwidth available for the new tunnel */
 492                 ret = tb_release_unused_usb3_bandwidth(tb, down, up);
 493                 if (ret)
 494                         return ret;
 495         }
 496
 497         ret = tb_available_bandwidth(tb, down, up, &available_up,
 498                                      &available_down);
 499         if (ret)
 500                 goto err_reclaim;
 501
 502         tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",
 503                     available_up, available_down);
 504
 505         tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,
 506                                       available_down);
 507         if (!tunnel) {
 508                 ret = -ENOMEM;
 509                 goto err_reclaim;
 510         }
 511
 512         if (tb_tunnel_activate(tunnel)) {
 513                 tb_port_info(up,
 514                              "USB3 tunnel activation failed, aborting\n");
 515                 ret = -EIO;
 516                 goto err_free;
 517         }
 518
 519         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 520         if (tb_route(parent))
 521                 tb_reclaim_usb3_bandwidth(tb, down, up);
 522
 523         return 0;
 524
 525 err_free:
 526         tb_tunnel_free(tunnel);
 527 err_reclaim:
 528         if (tb_route(parent))
 529                 tb_reclaim_usb3_bandwidth(tb, down, up);
 530
 531         return ret;
 532 }
 533
 534 static int tb_create_usb3_tunnels(struct tb_switch *sw)
 535 {
 536         struct tb_port *port;
 537         int ret;
 538
 539         if (!tb_acpi_may_tunnel_usb3())
 540                 return 0;
 541
 542         if (tb_route(sw)) {
 543                 ret = tb_tunnel_usb3(sw->tb, sw);
 544                 if (ret)
 545                         return ret;
 546         }
 547
 548         tb_switch_for_each_port(sw, port) {
 549                 if (!tb_port_has_remote(port))
 550                         continue;
 551                 ret = tb_create_usb3_tunnels(port->remote->sw);
 552                 if (ret)
 553                         return ret;
 554         }
 555
 556         return 0;
 557 }
 558
 559 static void tb_scan_port(struct tb_port *port);
 560
 561 /*
 562  * tb_scan_switch() - scan for and initialize downstream switches
 563  */
 564 static void tb_scan_switch(struct tb_switch *sw)
 565 {
 566         struct tb_port *port;
 567
 568         pm_runtime_get_sync(&sw->dev);
 569
 570         tb_switch_for_each_port(sw, port)
 571                 tb_scan_port(port);
 572
 573         pm_runtime_mark_last_busy(&sw->dev);
 574         pm_runtime_put_autosuspend(&sw->dev);
 575 }
 576
 577 /*
 578  * tb_scan_port() - check for and initialize switches below port
 579  */
 580 static void tb_scan_port(struct tb_port *port)
 581 {
 582         struct tb_cm *tcm = tb_priv(port->sw->tb);
 583         struct tb_port *upstream_port;
 584         struct tb_switch *sw;
 585         int ret;
 586
 587         if (tb_is_upstream_port(port))
 588                 return;
 589
 590         if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&
 591             !tb_dp_port_is_enabled(port)) {
 592                 tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");
 593                 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,
 594                                  false);
 595                 return;
 596         }
 597
 598         if (port->config.type != TB_TYPE_PORT)
 599                 return;
 600         if (port->dual_link_port && port->link_nr)
 601                 return; /*
 602                          * Downstream switch is reachable through two ports.
 603                          * Only scan on the primary port (link_nr == 0).
 604                          */
 605         if (tb_wait_for_port(port, false) <= 0)
 606                 return;
 607         if (port->remote) {
 608                 tb_port_dbg(port, "port already has a remote\n");
 609                 return;
 610         }
 611
 612         tb_retimer_scan(port, true);
 613
 614         sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,
 615                              tb_downstream_route(port));
 616         if (IS_ERR(sw)) {
 617                 /*
 618                  * If there is an error accessing the connected switch
 619                  * it may be connected to another domain. Also we allow
 620                  * the other domain to be connected to a max depth switch.
 621                  */
 622                 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL)
 623                         tb_scan_xdomain(port);
 624                 return;
 625         }
 626
 627         if (tb_switch_configure(sw)) {
 628                 tb_switch_put(sw);
 629                 return;
 630         }
 631
 632         /*
 633          * If there was previously another domain connected remove it
 634          * first.
 635          */
 636         if (port->xdomain) {
 637                 tb_xdomain_remove(port->xdomain);
 638                 tb_port_unconfigure_xdomain(port);
 639                 port->xdomain = NULL;
 640         }
 641
 642         /*
 643          * Do not send uevents until we have discovered all existing
 644          * tunnels and know which switches were authorized already by
 645          * the boot firmware.
 646          */
 647         if (!tcm->hotplug_active)
 648                 dev_set_uevent_suppress(&sw->dev, true);
 649
 650         /*
 651          * At the moment Thunderbolt 2 and beyond (devices with LC) we
 652          * can support runtime PM.
 653          */
 654         sw->rpm = sw->generation > 1;
 655
 656         if (tb_switch_add(sw)) {
 657                 tb_switch_put(sw);
 658                 return;
 659         }
 660
 661         /* Link the switches using both links if available */
 662         upstream_port = tb_upstream_port(sw);
 663         port->remote = upstream_port;
 664         upstream_port->remote = port;
 665         if (port->dual_link_port && upstream_port->dual_link_port) {
 666                 port->dual_link_port->remote = upstream_port->dual_link_port;
 667                 upstream_port->dual_link_port->remote = port->dual_link_port;
 668         }
 669
 670         /* Enable lane bonding if supported */
 671         tb_switch_lane_bonding_enable(sw);
 672         /* Set the link configured */
 673         tb_switch_configure_link(sw);
 674         /*
 675          * CL0s and CL1 are enabled and supported together.
 676          * Silently ignore CLx enabling in case CLx is not supported.
 677          */
 678         ret = tb_switch_enable_clx(sw, TB_CL1);
 679         if (ret && ret != -EOPNOTSUPP)
 680                 tb_sw_warn(sw, "failed to enable %s on upstream port\n",
 681                            tb_switch_clx_name(TB_CL1));
 682
 683         if (tb_switch_is_clx_enabled(sw, TB_CL1))
 684                 /*
 685                  * To support highest CLx state, we set router's TMU to
 686                  * Normal-Uni mode.
 687                  */
 688                 tb_switch_tmu_configure(sw, TB_SWITCH_TMU_RATE_NORMAL, true);
 689         else
 690                 /* If CLx disabled, configure router's TMU to HiFi-Bidir mode*/
 691                 tb_switch_tmu_configure(sw, TB_SWITCH_TMU_RATE_HIFI, false);
 692
 693         if (tb_enable_tmu(sw))
 694                 tb_sw_warn(sw, "failed to enable TMU\n");
 695
 696         /* Scan upstream retimers */
 697         tb_retimer_scan(upstream_port, true);
 698
 699         /*
 700          * Create USB 3.x tunnels only when the switch is plugged to the
 701          * domain. This is because we scan the domain also during discovery
 702          * and want to discover existing USB 3.x tunnels before we create
 703          * any new.
 704          */
 705         if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))
 706                 tb_sw_warn(sw, "USB3 tunnel creation failed\n");
 707
 708         tb_add_dp_resources(sw);
 709         tb_scan_switch(sw);
 710 }
 711
 712 static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)
 713 {
 714         struct tb_port *src_port, *dst_port;
 715         struct tb *tb;
 716
 717         if (!tunnel)
 718                 return;
 719
 720         tb_tunnel_deactivate(tunnel);
 721         list_del(&tunnel->list);
 722
 723         tb = tunnel->tb;
 724         src_port = tunnel->src_port;
 725         dst_port = tunnel->dst_port;
 726
 727         switch (tunnel->type) {
 728         case TB_TUNNEL_DP:
 729                 /*
 730                  * In case of DP tunnel make sure the DP IN resource is
 731                  * deallocated properly.
 732                  */
 733                 tb_switch_dealloc_dp_resource(src_port->sw, src_port);
 734                 /* Now we can allow the domain to runtime suspend again */
 735                 pm_runtime_mark_last_busy(&dst_port->sw->dev);
 736                 pm_runtime_put_autosuspend(&dst_port->sw->dev);
 737                 pm_runtime_mark_last_busy(&src_port->sw->dev);
 738                 pm_runtime_put_autosuspend(&src_port->sw->dev);
 739                 fallthrough;
 740
 741         case TB_TUNNEL_USB3:
 742                 tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);
 743                 break;
 744
 745         default:
 746                 /*
 747                  * PCIe and DMA tunnels do not consume guaranteed
 748                  * bandwidth.
 749                  */
 750                 break;
 751         }
 752
 753         tb_tunnel_free(tunnel);
 754 }
 755
 756 /*
 757  * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
 758  */
 759 static void tb_free_invalid_tunnels(struct tb *tb)
 760 {
 761         struct tb_cm *tcm = tb_priv(tb);
 762         struct tb_tunnel *tunnel;
 763         struct tb_tunnel *n;
 764
 765         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
 766                 if (tb_tunnel_is_invalid(tunnel))
 767                         tb_deactivate_and_free_tunnel(tunnel);
 768         }
 769 }
 770
 771 /*
 772  * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
 773  */
 774 static void tb_free_unplugged_children(struct tb_switch *sw)
 775 {
 776         struct tb_port *port;
 777
 778         tb_switch_for_each_port(sw, port) {
 779                 if (!tb_port_has_remote(port))
 780                         continue;
 781
 782                 if (port->remote->sw->is_unplugged) {
 783                         tb_retimer_remove_all(port);
 784                         tb_remove_dp_resources(port->remote->sw);
 785                         tb_switch_unconfigure_link(port->remote->sw);
 786                         tb_switch_lane_bonding_disable(port->remote->sw);
 787                         tb_switch_remove(port->remote->sw);
 788                         port->remote = NULL;
 789                         if (port->dual_link_port)
 790                                 port->dual_link_port->remote = NULL;
 791                 } else {
 792                         tb_free_unplugged_children(port->remote->sw);
 793                 }
 794         }
 795 }
 796
 797 static struct tb_port *tb_find_pcie_down(struct tb_switch *sw,
 798                                          const struct tb_port *port)
 799 {
 800         struct tb_port *down = NULL;
 801
 802         /*
 803          * To keep plugging devices consistently in the same PCIe
 804          * hierarchy, do mapping here for switch downstream PCIe ports.
 805          */
 806         if (tb_switch_is_usb4(sw)) {
 807                 down = usb4_switch_map_pcie_down(sw, port);
 808         } else if (!tb_route(sw)) {
 809                 int phy_port = tb_phy_port_from_link(port->port);
 810                 int index;
 811
 812                 /*
 813                  * Hard-coded Thunderbolt port to PCIe down port mapping
 814                  * per controller.
 815                  */
 816                 if (tb_switch_is_cactus_ridge(sw) ||
 817                     tb_switch_is_alpine_ridge(sw))
 818                         index = !phy_port ? 6 : 7;
 819                 else if (tb_switch_is_falcon_ridge(sw))
 820                         index = !phy_port ? 6 : 8;
 821                 else if (tb_switch_is_titan_ridge(sw))
 822                         index = !phy_port ? 8 : 9;
 823                 else
 824                         goto out;
 825
 826                 /* Validate the hard-coding */
 827                 if (WARN_ON(index > sw->config.max_port_number))
 828                         goto out;
 829
 830                 down = &sw->ports[index];
 831         }
 832
 833         if (down) {
 834                 if (WARN_ON(!tb_port_is_pcie_down(down)))
 835                         goto out;
 836                 if (tb_pci_port_is_enabled(down))
 837                         goto out;
 838
 839                 return down;
 840         }
 841
 842 out:
 843         return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);
 844 }
 845
 846 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)
 847 {
 848         struct tb_port *host_port, *port;
 849         struct tb_cm *tcm = tb_priv(tb);
 850
 851         host_port = tb_route(in->sw) ?
 852                 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;
 853
 854         list_for_each_entry(port, &tcm->dp_resources, list) {
 855                 if (!tb_port_is_dpout(port))
 856                         continue;
 857
 858                 if (tb_port_is_enabled(port)) {
 859                         tb_port_dbg(port, "in use\n");
 860                         continue;
 861                 }
 862
 863                 tb_port_dbg(port, "DP OUT available\n");
 864
 865                 /*
 866                  * Keep the DP tunnel under the topology starting from
 867                  * the same host router downstream port.
 868                  */
 869                 if (host_port && tb_route(port->sw)) {
 870                         struct tb_port *p;
 871
 872                         p = tb_port_at(tb_route(port->sw), tb->root_switch);
 873                         if (p != host_port)
 874                                 continue;
 875                 }
 876
 877                 return port;
 878         }
 879
 880         return NULL;
 881 }
 882
 883 static void tb_tunnel_dp(struct tb *tb)
 884 {
 885         int available_up, available_down, ret, link_nr;
 886         struct tb_cm *tcm = tb_priv(tb);
 887         struct tb_port *port, *in, *out;
 888         struct tb_tunnel *tunnel;
 889
 890         if (!tb_acpi_may_tunnel_dp()) {
 891                 tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");
 892                 return;
 893         }
 894
 895         /*
 896          * Find pair of inactive DP IN and DP OUT adapters and then
 897          * establish a DP tunnel between them.
 898          */
 899         tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");
 900
 901         in = NULL;
 902         out = NULL;
 903         list_for_each_entry(port, &tcm->dp_resources, list) {
 904                 if (!tb_port_is_dpin(port))
 905                         continue;
 906
 907                 if (tb_port_is_enabled(port)) {
 908                         tb_port_dbg(port, "in use\n");
 909                         continue;
 910                 }
 911
 912                 tb_port_dbg(port, "DP IN available\n");
 913
 914                 out = tb_find_dp_out(tb, port);
 915                 if (out) {
 916                         in = port;
 917                         break;
 918                 }
 919         }
 920
 921         if (!in) {
 922                 tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");
 923                 return;
 924         }
 925         if (!out) {
 926                 tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n");
 927                 return;
 928         }
 929
 930         /*
 931          * This is only applicable to links that are not bonded (so
 932          * when Thunderbolt 1 hardware is involved somewhere in the
 933          * topology). For these try to share the DP bandwidth between
 934          * the two lanes.
 935          */
 936         link_nr = 1;
 937         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 938                 if (tb_tunnel_is_dp(tunnel)) {
 939                         link_nr = 0;
 940                         break;
 941                 }
 942         }
 943
 944         /*
 945          * DP stream needs the domain to be active so runtime resume
 946          * both ends of the tunnel.
 947          *
 948          * This should bring the routers in the middle active as well
 949          * and keeps the domain from runtime suspending while the DP
 950          * tunnel is active.
 951          */
 952         pm_runtime_get_sync(&in->sw->dev);
 953         pm_runtime_get_sync(&out->sw->dev);
 954
 955         if (tb_switch_alloc_dp_resource(in->sw, in)) {
 956                 tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");
 957                 goto err_rpm_put;
 958         }
 959
 960         /* Make all unused USB3 bandwidth available for the new DP tunnel */
 961         ret = tb_release_unused_usb3_bandwidth(tb, in, out);
 962         if (ret) {
 963                 tb_warn(tb, "failed to release unused bandwidth\n");
 964                 goto err_dealloc_dp;
 965         }
 966
 967         ret = tb_available_bandwidth(tb, in, out, &available_up,
 968                                      &available_down);
 969         if (ret)
 970                 goto err_reclaim;
 971
 972         tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",
 973                available_up, available_down);
 974
 975         tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,
 976                                     available_down);
 977         if (!tunnel) {
 978                 tb_port_dbg(out, "could not allocate DP tunnel\n");
 979                 goto err_reclaim;
 980         }
 981
 982         if (tb_tunnel_activate(tunnel)) {
 983                 tb_port_info(out, "DP tunnel activation failed, aborting\n");
 984                 goto err_free;
 985         }
 986
 987         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 988         tb_reclaim_usb3_bandwidth(tb, in, out);
 989         /*
 990          * In case of DP tunnel exists, change host router's 1st children
 991          * TMU mode to HiFi for CL0s to work.
 992          */
 993         tb_switch_enable_tmu_1st_child(tb->root_switch, TB_SWITCH_TMU_RATE_HIFI);
 994
 995         return;
 996
 997 err_free:
 998         tb_tunnel_free(tunnel);
 999 err_reclaim:
1000         tb_reclaim_usb3_bandwidth(tb, in, out);
1001 err_dealloc_dp:
1002         tb_switch_dealloc_dp_resource(in->sw, in);
1003 err_rpm_put:
1004         pm_runtime_mark_last_busy(&out->sw->dev);
1005         pm_runtime_put_autosuspend(&out->sw->dev);
1006         pm_runtime_mark_last_busy(&in->sw->dev);
1007         pm_runtime_put_autosuspend(&in->sw->dev);
1008 }
1009
1010 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)
1011 {
1012         struct tb_port *in, *out;
1013         struct tb_tunnel *tunnel;
1014
1015         if (tb_port_is_dpin(port)) {
1016                 tb_port_dbg(port, "DP IN resource unavailable\n");
1017                 in = port;
1018                 out = NULL;
1019         } else {
1020                 tb_port_dbg(port, "DP OUT resource unavailable\n");
1021                 in = NULL;
1022                 out = port;
1023         }
1024
1025         tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);
1026         tb_deactivate_and_free_tunnel(tunnel);
1027         list_del_init(&port->list);
1028
1029         /*
1030          * See if there is another DP OUT port that can be used for
1031          * to create another tunnel.
1032          */
1033         tb_tunnel_dp(tb);
1034 }
1035
1036 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)
1037 {
1038         struct tb_cm *tcm = tb_priv(tb);
1039         struct tb_port *p;
1040
1041         if (tb_port_is_enabled(port))
1042                 return;
1043
1044         list_for_each_entry(p, &tcm->dp_resources, list) {
1045                 if (p == port)
1046                         return;
1047         }
1048
1049         tb_port_dbg(port, "DP %s resource available\n",
1050                     tb_port_is_dpin(port) ? "IN" : "OUT");
1051         list_add_tail(&port->list, &tcm->dp_resources);
1052
1053         /* Look for suitable DP IN <-> DP OUT pairs now */
1054         tb_tunnel_dp(tb);
1055 }
1056
1057 static void tb_disconnect_and_release_dp(struct tb *tb)
1058 {
1059         struct tb_cm *tcm = tb_priv(tb);
1060         struct tb_tunnel *tunnel, *n;
1061
1062         /*
1063          * Tear down all DP tunnels and release their resources. They
1064          * will be re-established after resume based on plug events.
1065          */
1066         list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {
1067                 if (tb_tunnel_is_dp(tunnel))
1068                         tb_deactivate_and_free_tunnel(tunnel);
1069         }
1070
1071         while (!list_empty(&tcm->dp_resources)) {
1072                 struct tb_port *port;
1073
1074                 port = list_first_entry(&tcm->dp_resources,
1075                                         struct tb_port, list);
1076                 list_del_init(&port->list);
1077         }
1078 }
1079
1080 static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)
1081 {
1082         struct tb_tunnel *tunnel;
1083         struct tb_port *up;
1084
1085         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1086         if (WARN_ON(!up))
1087                 return -ENODEV;
1088
1089         tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);
1090         if (WARN_ON(!tunnel))
1091                 return -ENODEV;
1092
1093         tb_switch_xhci_disconnect(sw);
1094
1095         tb_tunnel_deactivate(tunnel);
1096         list_del(&tunnel->list);
1097         tb_tunnel_free(tunnel);
1098         return 0;
1099 }
1100
1101 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)
1102 {
1103         struct tb_port *up, *down, *port;
1104         struct tb_cm *tcm = tb_priv(tb);
1105         struct tb_switch *parent_sw;
1106         struct tb_tunnel *tunnel;
1107
1108         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1109         if (!up)
1110                 return 0;
1111
1112         /*
1113          * Look up available down port. Since we are chaining it should
1114          * be found right above this switch.
1115          */
1116         parent_sw = tb_to_switch(sw->dev.parent);
1117         port = tb_port_at(tb_route(sw), parent_sw);
1118         down = tb_find_pcie_down(parent_sw, port);
1119         if (!down)
1120                 return 0;
1121
1122         tunnel = tb_tunnel_alloc_pci(tb, up, down);
1123         if (!tunnel)
1124                 return -ENOMEM;
1125
1126         if (tb_tunnel_activate(tunnel)) {
1127                 tb_port_info(up,
1128                              "PCIe tunnel activation failed, aborting\n");
1129                 tb_tunnel_free(tunnel);
1130                 return -EIO;
1131         }
1132
1133         /*
1134          * PCIe L1 is needed to enable CL0s for Titan Ridge so enable it
1135          * here.
1136          */
1137         if (tb_switch_pcie_l1_enable(sw))
1138                 tb_sw_warn(sw, "failed to enable PCIe L1 for Titan Ridge\n");
1139
1140         if (tb_switch_xhci_connect(sw))
1141                 tb_sw_warn(sw, "failed to connect xHCI\n");
1142
1143         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1144         return 0;
1145 }
1146
1147 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1148                                     int transmit_path, int transmit_ring,
1149                                     int receive_path, int receive_ring)
1150 {
1151         struct tb_cm *tcm = tb_priv(tb);
1152         struct tb_port *nhi_port, *dst_port;
1153         struct tb_tunnel *tunnel;
1154         struct tb_switch *sw;
1155
1156         sw = tb_to_switch(xd->dev.parent);
1157         dst_port = tb_port_at(xd->route, sw);
1158         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1159
1160         mutex_lock(&tb->lock);
1161         tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,
1162                                      transmit_ring, receive_path, receive_ring);
1163         if (!tunnel) {
1164                 mutex_unlock(&tb->lock);
1165                 return -ENOMEM;
1166         }
1167
1168         if (tb_tunnel_activate(tunnel)) {
1169                 tb_port_info(nhi_port,
1170                              "DMA tunnel activation failed, aborting\n");
1171                 tb_tunnel_free(tunnel);
1172                 mutex_unlock(&tb->lock);
1173                 return -EIO;
1174         }
1175
1176         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1177         mutex_unlock(&tb->lock);
1178         return 0;
1179 }
1180
1181 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1182                                           int transmit_path, int transmit_ring,
1183                                           int receive_path, int receive_ring)
1184 {
1185         struct tb_cm *tcm = tb_priv(tb);
1186         struct tb_port *nhi_port, *dst_port;
1187         struct tb_tunnel *tunnel, *n;
1188         struct tb_switch *sw;
1189
1190         sw = tb_to_switch(xd->dev.parent);
1191         dst_port = tb_port_at(xd->route, sw);
1192         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1193
1194         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1195                 if (!tb_tunnel_is_dma(tunnel))
1196                         continue;
1197                 if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port)
1198                         continue;
1199
1200                 if (tb_tunnel_match_dma(tunnel, transmit_path, transmit_ring,
1201                                         receive_path, receive_ring))
1202                         tb_deactivate_and_free_tunnel(tunnel);
1203         }
1204 }
1205
1206 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1207                                        int transmit_path, int transmit_ring,
1208                                        int receive_path, int receive_ring)
1209 {
1210         if (!xd->is_unplugged) {
1211                 mutex_lock(&tb->lock);
1212                 __tb_disconnect_xdomain_paths(tb, xd, transmit_path,
1213                                               transmit_ring, receive_path,
1214                                               receive_ring);
1215                 mutex_unlock(&tb->lock);
1216         }
1217         return 0;
1218 }
1219
1220 /* hotplug handling */
1221
1222 /*
1223  * tb_handle_hotplug() - handle hotplug event
1224  *
1225  * Executes on tb->wq.
1226  */
1227 static void tb_handle_hotplug(struct work_struct *work)
1228 {
1229         struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);
1230         struct tb *tb = ev->tb;
1231         struct tb_cm *tcm = tb_priv(tb);
1232         struct tb_switch *sw;
1233         struct tb_port *port;
1234
1235         /* Bring the domain back from sleep if it was suspended */
1236         pm_runtime_get_sync(&tb->dev);
1237
1238         mutex_lock(&tb->lock);
1239         if (!tcm->hotplug_active)
1240                 goto out; /* during init, suspend or shutdown */
1241
1242         sw = tb_switch_find_by_route(tb, ev->route);
1243         if (!sw) {
1244                 tb_warn(tb,
1245                         "hotplug event from non existent switch %llx:%x (unplug: %d)\n",
1246                         ev->route, ev->port, ev->unplug);
1247                 goto out;
1248         }
1249         if (ev->port > sw->config.max_port_number) {
1250                 tb_warn(tb,
1251                         "hotplug event from non existent port %llx:%x (unplug: %d)\n",
1252                         ev->route, ev->port, ev->unplug);
1253                 goto put_sw;
1254         }
1255         port = &sw->ports[ev->port];
1256         if (tb_is_upstream_port(port)) {
1257                 tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",
1258                        ev->route, ev->port, ev->unplug);
1259                 goto put_sw;
1260         }
1261
1262         pm_runtime_get_sync(&sw->dev);
1263
1264         if (ev->unplug) {
1265                 tb_retimer_remove_all(port);
1266
1267                 if (tb_port_has_remote(port)) {
1268                         tb_port_dbg(port, "switch unplugged\n");
1269                         tb_sw_set_unplugged(port->remote->sw);
1270                         tb_free_invalid_tunnels(tb);
1271                         tb_remove_dp_resources(port->remote->sw);
1272                         tb_switch_tmu_disable(port->remote->sw);
1273                         tb_switch_unconfigure_link(port->remote->sw);
1274                         tb_switch_lane_bonding_disable(port->remote->sw);
1275                         tb_switch_remove(port->remote->sw);
1276                         port->remote = NULL;
1277                         if (port->dual_link_port)
1278                                 port->dual_link_port->remote = NULL;
1279                         /* Maybe we can create another DP tunnel */
1280                         tb_tunnel_dp(tb);
1281                 } else if (port->xdomain) {
1282                         struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);
1283
1284                         tb_port_dbg(port, "xdomain unplugged\n");
1285                         /*
1286                          * Service drivers are unbound during
1287                          * tb_xdomain_remove() so setting XDomain as
1288                          * unplugged here prevents deadlock if they call
1289                          * tb_xdomain_disable_paths(). We will tear down
1290                          * all the tunnels below.
1291                          */
1292                         xd->is_unplugged = true;
1293                         tb_xdomain_remove(xd);
1294                         port->xdomain = NULL;
1295                         __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);
1296                         tb_xdomain_put(xd);
1297                         tb_port_unconfigure_xdomain(port);
1298                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1299                         tb_dp_resource_unavailable(tb, port);
1300                 } else if (!port->port) {
1301                         tb_sw_dbg(sw, "xHCI disconnect request\n");
1302                         tb_switch_xhci_disconnect(sw);
1303                 } else {
1304                         tb_port_dbg(port,
1305                                    "got unplug event for disconnected port, ignoring\n");
1306                 }
1307         } else if (port->remote) {
1308                 tb_port_dbg(port, "got plug event for connected port, ignoring\n");
1309         } else if (!port->port && sw->authorized) {
1310                 tb_sw_dbg(sw, "xHCI connect request\n");
1311                 tb_switch_xhci_connect(sw);
1312         } else {
1313                 if (tb_port_is_null(port)) {
1314                         tb_port_dbg(port, "hotplug: scanning\n");
1315                         tb_scan_port(port);
1316                         if (!port->remote)
1317                                 tb_port_dbg(port, "hotplug: no switch found\n");
1318                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1319                         tb_dp_resource_available(tb, port);
1320                 }
1321         }
1322
1323         pm_runtime_mark_last_busy(&sw->dev);
1324         pm_runtime_put_autosuspend(&sw->dev);
1325
1326 put_sw:
1327         tb_switch_put(sw);
1328 out:
1329         mutex_unlock(&tb->lock);
1330
1331         pm_runtime_mark_last_busy(&tb->dev);
1332         pm_runtime_put_autosuspend(&tb->dev);
1333
1334         kfree(ev);
1335 }
1336
1337 /*
1338  * tb_schedule_hotplug_handler() - callback function for the control channel
1339  *
1340  * Delegates to tb_handle_hotplug.
1341  */
1342 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1343                             const void *buf, size_t size)
1344 {
1345         const struct cfg_event_pkg *pkg = buf;
1346         u64 route;
1347
1348         if (type != TB_CFG_PKG_EVENT) {
1349                 tb_warn(tb, "unexpected event %#x, ignoring\n", type);
1350                 return;
1351         }
1352
1353         route = tb_cfg_get_route(&pkg->header);
1354
1355         if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {
1356                 tb_warn(tb, "could not ack plug event on %llx:%x\n", route,
1357                         pkg->port);
1358         }
1359
1360         tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);
1361 }
1362
1363 static void tb_stop(struct tb *tb)
1364 {
1365         struct tb_cm *tcm = tb_priv(tb);
1366         struct tb_tunnel *tunnel;
1367         struct tb_tunnel *n;
1368
1369         cancel_delayed_work(&tcm->remove_work);
1370         /* tunnels are only present after everything has been initialized */
1371         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1372                 /*
1373                  * DMA tunnels require the driver to be functional so we
1374                  * tear them down. Other protocol tunnels can be left
1375                  * intact.
1376                  */
1377                 if (tb_tunnel_is_dma(tunnel))
1378                         tb_tunnel_deactivate(tunnel);
1379                 tb_tunnel_free(tunnel);
1380         }
1381         tb_switch_remove(tb->root_switch);
1382         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1383 }
1384
1385 static int tb_scan_finalize_switch(struct device *dev, void *data)
1386 {
1387         if (tb_is_switch(dev)) {
1388                 struct tb_switch *sw = tb_to_switch(dev);
1389
1390                 /*
1391                  * If we found that the switch was already setup by the
1392                  * boot firmware, mark it as authorized now before we
1393                  * send uevent to userspace.
1394                  */
1395                 if (sw->boot)
1396                         sw->authorized = 1;
1397
1398                 dev_set_uevent_suppress(dev, false);
1399                 kobject_uevent(&dev->kobj, KOBJ_ADD);
1400                 device_for_each_child(dev, NULL, tb_scan_finalize_switch);
1401         }
1402
1403         return 0;
1404 }
1405
1406 static int tb_start(struct tb *tb)
1407 {
1408         struct tb_cm *tcm = tb_priv(tb);
1409         int ret;
1410
1411         tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
1412         if (IS_ERR(tb->root_switch))
1413                 return PTR_ERR(tb->root_switch);
1414
1415         /*
1416          * ICM firmware upgrade needs running firmware and in native
1417          * mode that is not available so disable firmware upgrade of the
1418          * root switch.
1419          */
1420         tb->root_switch->no_nvm_upgrade = true;
1421         /* All USB4 routers support runtime PM */
1422         tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);
1423
1424         ret = tb_switch_configure(tb->root_switch);
1425         if (ret) {
1426                 tb_switch_put(tb->root_switch);
1427                 return ret;
1428         }
1429
1430         /* Announce the switch to the world */
1431         ret = tb_switch_add(tb->root_switch);
1432         if (ret) {
1433                 tb_switch_put(tb->root_switch);
1434                 return ret;
1435         }
1436
1437         /*
1438          * To support highest CLx state, we set host router's TMU to
1439          * Normal mode.
1440          */
1441         tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_RATE_NORMAL,
1442                                 false);
1443         /* Enable TMU if it is off */
1444         tb_switch_tmu_enable(tb->root_switch);
1445         /* Full scan to discover devices added before the driver was loaded. */
1446         tb_scan_switch(tb->root_switch);
1447         /* Find out tunnels created by the boot firmware */
1448         tb_discover_tunnels(tb);
1449         /*
1450          * If the boot firmware did not create USB 3.x tunnels create them
1451          * now for the whole topology.
1452          */
1453         tb_create_usb3_tunnels(tb->root_switch);
1454         /* Add DP IN resources for the root switch */
1455         tb_add_dp_resources(tb->root_switch);
1456         /* Make the discovered switches available to the userspace */
1457         device_for_each_child(&tb->root_switch->dev, NULL,
1458                               tb_scan_finalize_switch);
1459
1460         /* Allow tb_handle_hotplug to progress events */
1461         tcm->hotplug_active = true;
1462         return 0;
1463 }
1464
1465 static int tb_suspend_noirq(struct tb *tb)
1466 {
1467         struct tb_cm *tcm = tb_priv(tb);
1468
1469         tb_dbg(tb, "suspending...\n");
1470         tb_disconnect_and_release_dp(tb);
1471         tb_switch_suspend(tb->root_switch, false);
1472         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1473         tb_dbg(tb, "suspend finished\n");
1474
1475         return 0;
1476 }
1477
1478 static void tb_restore_children(struct tb_switch *sw)
1479 {
1480         struct tb_port *port;
1481         int ret;
1482
1483         /* No need to restore if the router is already unplugged */
1484         if (sw->is_unplugged)
1485                 return;
1486
1487         /*
1488          * CL0s and CL1 are enabled and supported together.
1489          * Silently ignore CLx re-enabling in case CLx is not supported.
1490          */
1491         ret = tb_switch_enable_clx(sw, TB_CL1);
1492         if (ret && ret != -EOPNOTSUPP)
1493                 tb_sw_warn(sw, "failed to re-enable %s on upstream port\n",
1494                            tb_switch_clx_name(TB_CL1));
1495
1496         if (tb_switch_is_clx_enabled(sw, TB_CL1))
1497                 /*
1498                  * To support highest CLx state, we set router's TMU to
1499                  * Normal-Uni mode.
1500                  */
1501                 tb_switch_tmu_configure(sw, TB_SWITCH_TMU_RATE_NORMAL, true);
1502         else
1503                 /* If CLx disabled, configure router's TMU to HiFi-Bidir mode*/
1504                 tb_switch_tmu_configure(sw, TB_SWITCH_TMU_RATE_HIFI, false);
1505
1506         if (tb_enable_tmu(sw))
1507                 tb_sw_warn(sw, "failed to restore TMU configuration\n");
1508
1509         tb_switch_for_each_port(sw, port) {
1510                 if (!tb_port_has_remote(port) && !port->xdomain)
1511                         continue;
1512
1513                 if (port->remote) {
1514                         tb_switch_lane_bonding_enable(port->remote->sw);
1515                         tb_switch_configure_link(port->remote->sw);
1516
1517                         tb_restore_children(port->remote->sw);
1518                 } else if (port->xdomain) {
1519                         tb_port_configure_xdomain(port);
1520                 }
1521         }
1522 }
1523
1524 static int tb_resume_noirq(struct tb *tb)
1525 {
1526         struct tb_cm *tcm = tb_priv(tb);
1527         struct tb_tunnel *tunnel, *n;
1528         unsigned int usb3_delay = 0;
1529         LIST_HEAD(tunnels);
1530
1531         tb_dbg(tb, "resuming...\n");
1532
1533         /* remove any pci devices the firmware might have setup */
1534         tb_switch_reset(tb->root_switch);
1535
1536         tb_switch_resume(tb->root_switch);
1537         tb_free_invalid_tunnels(tb);
1538         tb_free_unplugged_children(tb->root_switch);
1539         tb_restore_children(tb->root_switch);
1540
1541         /*
1542          * If we get here from suspend to disk the boot firmware or the
1543          * restore kernel might have created tunnels of its own. Since
1544          * we cannot be sure they are usable for us we find and tear
1545          * them down.
1546          */
1547         tb_switch_discover_tunnels(tb->root_switch, &tunnels, false);
1548         list_for_each_entry_safe_reverse(tunnel, n, &tunnels, list) {
1549                 if (tb_tunnel_is_usb3(tunnel))
1550                         usb3_delay = 500;
1551                 tb_tunnel_deactivate(tunnel);
1552                 tb_tunnel_free(tunnel);
1553         }
1554
1555         /* Re-create our tunnels now */
1556         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1557                 /* USB3 requires delay before it can be re-activated */
1558                 if (tb_tunnel_is_usb3(tunnel)) {
1559                         msleep(usb3_delay);
1560                         /* Only need to do it once */
1561                         usb3_delay = 0;
1562                 }
1563                 tb_tunnel_restart(tunnel);
1564         }
1565         if (!list_empty(&tcm->tunnel_list)) {
1566                 /*
1567                  * the pcie links need some time to get going.
1568                  * 100ms works for me...
1569                  */
1570                 tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");
1571                 msleep(100);
1572         }
1573          /* Allow tb_handle_hotplug to progress events */
1574         tcm->hotplug_active = true;
1575         tb_dbg(tb, "resume finished\n");
1576
1577         return 0;
1578 }
1579
1580 static int tb_free_unplugged_xdomains(struct tb_switch *sw)
1581 {
1582         struct tb_port *port;
1583         int ret = 0;
1584
1585         tb_switch_for_each_port(sw, port) {
1586                 if (tb_is_upstream_port(port))
1587                         continue;
1588                 if (port->xdomain && port->xdomain->is_unplugged) {
1589                         tb_retimer_remove_all(port);
1590                         tb_xdomain_remove(port->xdomain);
1591                         tb_port_unconfigure_xdomain(port);
1592                         port->xdomain = NULL;
1593                         ret++;
1594                 } else if (port->remote) {
1595                         ret += tb_free_unplugged_xdomains(port->remote->sw);
1596                 }
1597         }
1598
1599         return ret;
1600 }
1601
1602 static int tb_freeze_noirq(struct tb *tb)
1603 {
1604         struct tb_cm *tcm = tb_priv(tb);
1605
1606         tcm->hotplug_active = false;
1607         return 0;
1608 }
1609
1610 static int tb_thaw_noirq(struct tb *tb)
1611 {
1612         struct tb_cm *tcm = tb_priv(tb);
1613
1614         tcm->hotplug_active = true;
1615         return 0;
1616 }
1617
1618 static void tb_complete(struct tb *tb)
1619 {
1620         /*
1621          * Release any unplugged XDomains and if there is a case where
1622          * another domain is swapped in place of unplugged XDomain we
1623          * need to run another rescan.
1624          */
1625         mutex_lock(&tb->lock);
1626         if (tb_free_unplugged_xdomains(tb->root_switch))
1627                 tb_scan_switch(tb->root_switch);
1628         mutex_unlock(&tb->lock);
1629 }
1630
1631 static int tb_runtime_suspend(struct tb *tb)
1632 {
1633         struct tb_cm *tcm = tb_priv(tb);
1634
1635         mutex_lock(&tb->lock);
1636         tb_switch_suspend(tb->root_switch, true);
1637         tcm->hotplug_active = false;
1638         mutex_unlock(&tb->lock);
1639
1640         return 0;
1641 }
1642
1643 static void tb_remove_work(struct work_struct *work)
1644 {
1645         struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);
1646         struct tb *tb = tcm_to_tb(tcm);
1647
1648         mutex_lock(&tb->lock);
1649         if (tb->root_switch) {
1650                 tb_free_unplugged_children(tb->root_switch);
1651                 tb_free_unplugged_xdomains(tb->root_switch);
1652         }
1653         mutex_unlock(&tb->lock);
1654 }
1655
1656 static int tb_runtime_resume(struct tb *tb)
1657 {
1658         struct tb_cm *tcm = tb_priv(tb);
1659         struct tb_tunnel *tunnel, *n;
1660
1661         mutex_lock(&tb->lock);
1662         tb_switch_resume(tb->root_switch);
1663         tb_free_invalid_tunnels(tb);
1664         tb_restore_children(tb->root_switch);
1665         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1666                 tb_tunnel_restart(tunnel);
1667         tcm->hotplug_active = true;
1668         mutex_unlock(&tb->lock);
1669
1670         /*
1671          * Schedule cleanup of any unplugged devices. Run this in a
1672          * separate thread to avoid possible deadlock if the device
1673          * removal runtime resumes the unplugged device.
1674          */
1675         queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));
1676         return 0;
1677 }
1678
1679 static const struct tb_cm_ops tb_cm_ops = {
1680         .start = tb_start,
1681         .stop = tb_stop,
1682         .suspend_noirq = tb_suspend_noirq,
1683         .resume_noirq = tb_resume_noirq,
1684         .freeze_noirq = tb_freeze_noirq,
1685         .thaw_noirq = tb_thaw_noirq,
1686         .complete = tb_complete,
1687         .runtime_suspend = tb_runtime_suspend,
1688         .runtime_resume = tb_runtime_resume,
1689         .handle_event = tb_handle_event,
1690         .disapprove_switch = tb_disconnect_pci,
1691         .approve_switch = tb_tunnel_pci,
1692         .approve_xdomain_paths = tb_approve_xdomain_paths,
1693         .disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
1694 };
1695
1696 /*
1697  * During suspend the Thunderbolt controller is reset and all PCIe
1698  * tunnels are lost. The NHI driver will try to reestablish all tunnels
1699  * during resume. This adds device links between the tunneled PCIe
1700  * downstream ports and the NHI so that the device core will make sure
1701  * NHI is resumed first before the rest.
1702  */
1703 static void tb_apple_add_links(struct tb_nhi *nhi)
1704 {
1705         struct pci_dev *upstream, *pdev;
1706
1707         if (!x86_apple_machine)
1708                 return;
1709
1710         switch (nhi->pdev->device) {
1711         case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
1712         case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
1713         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
1714         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
1715                 break;
1716         default:
1717                 return;
1718         }
1719
1720         upstream = pci_upstream_bridge(nhi->pdev);
1721         while (upstream) {
1722                 if (!pci_is_pcie(upstream))
1723                         return;
1724                 if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
1725                         break;
1726                 upstream = pci_upstream_bridge(upstream);
1727         }
1728
1729         if (!upstream)
1730                 return;
1731
1732         /*
1733          * For each hotplug downstream port, create add device link
1734          * back to NHI so that PCIe tunnels can be re-established after
1735          * sleep.
1736          */
1737         for_each_pci_bridge(pdev, upstream->subordinate) {
1738                 const struct device_link *link;
1739
1740                 if (!pci_is_pcie(pdev))
1741                         continue;
1742                 if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
1743                     !pdev->is_hotplug_bridge)
1744                         continue;
1745
1746                 link = device_link_add(&pdev->dev, &nhi->pdev->dev,
1747                                        DL_FLAG_AUTOREMOVE_SUPPLIER |
1748                                        DL_FLAG_PM_RUNTIME);
1749                 if (link) {
1750                         dev_dbg(&nhi->pdev->dev, "created link from %s\n",
1751                                 dev_name(&pdev->dev));
1752                 } else {
1753                         dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n",
1754                                  dev_name(&pdev->dev));
1755                 }
1756         }
1757 }
1758
1759 struct tb *tb_probe(struct tb_nhi *nhi)
1760 {
1761         struct tb_cm *tcm;
1762         struct tb *tb;
1763
1764         tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));
1765         if (!tb)
1766                 return NULL;
1767
1768         if (tb_acpi_may_tunnel_pcie())
1769                 tb->security_level = TB_SECURITY_USER;
1770         else
1771                 tb->security_level = TB_SECURITY_NOPCIE;
1772
1773         tb->cm_ops = &tb_cm_ops;
1774
1775         tcm = tb_priv(tb);
1776         INIT_LIST_HEAD(&tcm->tunnel_list);
1777         INIT_LIST_HEAD(&tcm->dp_resources);
1778         INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);
1779
1780         tb_dbg(tb, "using software connection manager\n");
1781
1782         tb_apple_add_links(nhi);
1783         tb_acpi_add_links(nhi);
1784
1785         return tb;
1786 }