spec: Build rt packages
[platform/kernel/linux-rpi.git] / drivers / thunderbolt / tb.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Thunderbolt driver - bus logic (NHI independent)
4  *
5  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
6  * Copyright (C) 2019, Intel Corporation
7  */
8
9 #include <linux/slab.h>
10 #include <linux/errno.h>
11 #include <linux/delay.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/platform_data/x86/apple.h>
14
15 #include "tb.h"
16 #include "tb_regs.h"
17 #include "tunnel.h"
18
19 #define TB_TIMEOUT      100 /* ms */
20
21 /**
22  * struct tb_cm - Simple Thunderbolt connection manager
23  * @tunnel_list: List of active tunnels
24  * @dp_resources: List of available DP resources for DP tunneling
25  * @hotplug_active: tb_handle_hotplug will stop progressing plug
26  *                  events and exit if this is not set (it needs to
27  *                  acquire the lock one more time). Used to drain wq
28  *                  after cfg has been paused.
29  * @remove_work: Work used to remove any unplugged routers after
30  *               runtime resume
31  */
32 struct tb_cm {
33         struct list_head tunnel_list;
34         struct list_head dp_resources;
35         bool hotplug_active;
36         struct delayed_work remove_work;
37 };
38
39 static inline struct tb *tcm_to_tb(struct tb_cm *tcm)
40 {
41         return ((void *)tcm - sizeof(struct tb));
42 }
43
44 struct tb_hotplug_event {
45         struct work_struct work;
46         struct tb *tb;
47         u64 route;
48         u8 port;
49         bool unplug;
50 };
51
52 static void tb_handle_hotplug(struct work_struct *work);
53
54 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
55 {
56         struct tb_hotplug_event *ev;
57
58         ev = kmalloc(sizeof(*ev), GFP_KERNEL);
59         if (!ev)
60                 return;
61
62         ev->tb = tb;
63         ev->route = route;
64         ev->port = port;
65         ev->unplug = unplug;
66         INIT_WORK(&ev->work, tb_handle_hotplug);
67         queue_work(tb->wq, &ev->work);
68 }
69
70 /* enumeration & hot plug handling */
71
72 static void tb_add_dp_resources(struct tb_switch *sw)
73 {
74         struct tb_cm *tcm = tb_priv(sw->tb);
75         struct tb_port *port;
76
77         tb_switch_for_each_port(sw, port) {
78                 if (!tb_port_is_dpin(port))
79                         continue;
80
81                 if (!tb_switch_query_dp_resource(sw, port))
82                         continue;
83
84                 list_add_tail(&port->list, &tcm->dp_resources);
85                 tb_port_dbg(port, "DP IN resource available\n");
86         }
87 }
88
89 static void tb_remove_dp_resources(struct tb_switch *sw)
90 {
91         struct tb_cm *tcm = tb_priv(sw->tb);
92         struct tb_port *port, *tmp;
93
94         /* Clear children resources first */
95         tb_switch_for_each_port(sw, port) {
96                 if (tb_port_has_remote(port))
97                         tb_remove_dp_resources(port->remote->sw);
98         }
99
100         list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {
101                 if (port->sw == sw) {
102                         tb_port_dbg(port, "DP OUT resource unavailable\n");
103                         list_del_init(&port->list);
104                 }
105         }
106 }
107
108 static void tb_discover_dp_resource(struct tb *tb, struct tb_port *port)
109 {
110         struct tb_cm *tcm = tb_priv(tb);
111         struct tb_port *p;
112
113         list_for_each_entry(p, &tcm->dp_resources, list) {
114                 if (p == port)
115                         return;
116         }
117
118         tb_port_dbg(port, "DP %s resource available discovered\n",
119                     tb_port_is_dpin(port) ? "IN" : "OUT");
120         list_add_tail(&port->list, &tcm->dp_resources);
121 }
122
123 static void tb_discover_dp_resources(struct tb *tb)
124 {
125         struct tb_cm *tcm = tb_priv(tb);
126         struct tb_tunnel *tunnel;
127
128         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
129                 if (tb_tunnel_is_dp(tunnel))
130                         tb_discover_dp_resource(tb, tunnel->dst_port);
131         }
132 }
133
134 static void tb_switch_discover_tunnels(struct tb_switch *sw,
135                                        struct list_head *list,
136                                        bool alloc_hopids)
137 {
138         struct tb *tb = sw->tb;
139         struct tb_port *port;
140
141         tb_switch_for_each_port(sw, port) {
142                 struct tb_tunnel *tunnel = NULL;
143
144                 switch (port->config.type) {
145                 case TB_TYPE_DP_HDMI_IN:
146                         tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids);
147                         break;
148
149                 case TB_TYPE_PCIE_DOWN:
150                         tunnel = tb_tunnel_discover_pci(tb, port, alloc_hopids);
151                         break;
152
153                 case TB_TYPE_USB3_DOWN:
154                         tunnel = tb_tunnel_discover_usb3(tb, port, alloc_hopids);
155                         break;
156
157                 default:
158                         break;
159                 }
160
161                 if (tunnel)
162                         list_add_tail(&tunnel->list, list);
163         }
164
165         tb_switch_for_each_port(sw, port) {
166                 if (tb_port_has_remote(port)) {
167                         tb_switch_discover_tunnels(port->remote->sw, list,
168                                                    alloc_hopids);
169                 }
170         }
171 }
172
173 static void tb_discover_tunnels(struct tb *tb)
174 {
175         struct tb_cm *tcm = tb_priv(tb);
176         struct tb_tunnel *tunnel;
177
178         tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true);
179
180         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
181                 if (tb_tunnel_is_pci(tunnel)) {
182                         struct tb_switch *parent = tunnel->dst_port->sw;
183
184                         while (parent != tunnel->src_port->sw) {
185                                 parent->boot = true;
186                                 parent = tb_switch_parent(parent);
187                         }
188                 } else if (tb_tunnel_is_dp(tunnel)) {
189                         /* Keep the domain from powering down */
190                         pm_runtime_get_sync(&tunnel->src_port->sw->dev);
191                         pm_runtime_get_sync(&tunnel->dst_port->sw->dev);
192                 }
193         }
194 }
195
196 static int tb_port_configure_xdomain(struct tb_port *port)
197 {
198         /*
199          * XDomain paths currently only support single lane so we must
200          * disable the other lane according to USB4 spec.
201          */
202         tb_port_disable(port->dual_link_port);
203
204         if (tb_switch_is_usb4(port->sw))
205                 return usb4_port_configure_xdomain(port);
206         return tb_lc_configure_xdomain(port);
207 }
208
209 static void tb_port_unconfigure_xdomain(struct tb_port *port)
210 {
211         if (tb_switch_is_usb4(port->sw))
212                 usb4_port_unconfigure_xdomain(port);
213         else
214                 tb_lc_unconfigure_xdomain(port);
215
216         tb_port_enable(port->dual_link_port);
217 }
218
219 static void tb_scan_xdomain(struct tb_port *port)
220 {
221         struct tb_switch *sw = port->sw;
222         struct tb *tb = sw->tb;
223         struct tb_xdomain *xd;
224         u64 route;
225
226         if (!tb_is_xdomain_enabled())
227                 return;
228
229         route = tb_downstream_route(port);
230         xd = tb_xdomain_find_by_route(tb, route);
231         if (xd) {
232                 tb_xdomain_put(xd);
233                 return;
234         }
235
236         xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,
237                               NULL);
238         if (xd) {
239                 tb_port_at(route, sw)->xdomain = xd;
240                 tb_port_configure_xdomain(port);
241                 tb_xdomain_add(xd);
242         }
243 }
244
245 static int tb_enable_tmu(struct tb_switch *sw)
246 {
247         int ret;
248
249         /* If it is already enabled in correct mode, don't touch it */
250         if (tb_switch_tmu_is_enabled(sw))
251                 return 0;
252
253         ret = tb_switch_tmu_disable(sw);
254         if (ret)
255                 return ret;
256
257         ret = tb_switch_tmu_post_time(sw);
258         if (ret)
259                 return ret;
260
261         return tb_switch_tmu_enable(sw);
262 }
263
264 /**
265  * tb_find_unused_port() - return the first inactive port on @sw
266  * @sw: Switch to find the port on
267  * @type: Port type to look for
268  */
269 static struct tb_port *tb_find_unused_port(struct tb_switch *sw,
270                                            enum tb_port_type type)
271 {
272         struct tb_port *port;
273
274         tb_switch_for_each_port(sw, port) {
275                 if (tb_is_upstream_port(port))
276                         continue;
277                 if (port->config.type != type)
278                         continue;
279                 if (!port->cap_adap)
280                         continue;
281                 if (tb_port_is_enabled(port))
282                         continue;
283                 return port;
284         }
285         return NULL;
286 }
287
288 static struct tb_port *tb_find_usb3_down(struct tb_switch *sw,
289                                          const struct tb_port *port)
290 {
291         struct tb_port *down;
292
293         down = usb4_switch_map_usb3_down(sw, port);
294         if (down && !tb_usb3_port_is_enabled(down))
295                 return down;
296         return NULL;
297 }
298
299 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,
300                                         struct tb_port *src_port,
301                                         struct tb_port *dst_port)
302 {
303         struct tb_cm *tcm = tb_priv(tb);
304         struct tb_tunnel *tunnel;
305
306         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
307                 if (tunnel->type == type &&
308                     ((src_port && src_port == tunnel->src_port) ||
309                      (dst_port && dst_port == tunnel->dst_port))) {
310                         return tunnel;
311                 }
312         }
313
314         return NULL;
315 }
316
317 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,
318                                                    struct tb_port *src_port,
319                                                    struct tb_port *dst_port)
320 {
321         struct tb_port *port, *usb3_down;
322         struct tb_switch *sw;
323
324         /* Pick the router that is deepest in the topology */
325         if (dst_port->sw->config.depth > src_port->sw->config.depth)
326                 sw = dst_port->sw;
327         else
328                 sw = src_port->sw;
329
330         /* Can't be the host router */
331         if (sw == tb->root_switch)
332                 return NULL;
333
334         /* Find the downstream USB4 port that leads to this router */
335         port = tb_port_at(tb_route(sw), tb->root_switch);
336         /* Find the corresponding host router USB3 downstream port */
337         usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);
338         if (!usb3_down)
339                 return NULL;
340
341         return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);
342 }
343
344 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,
345         struct tb_port *dst_port, int *available_up, int *available_down)
346 {
347         int usb3_consumed_up, usb3_consumed_down, ret;
348         struct tb_cm *tcm = tb_priv(tb);
349         struct tb_tunnel *tunnel;
350         struct tb_port *port;
351
352         tb_port_dbg(dst_port, "calculating available bandwidth\n");
353
354         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
355         if (tunnel) {
356                 ret = tb_tunnel_consumed_bandwidth(tunnel, &usb3_consumed_up,
357                                                    &usb3_consumed_down);
358                 if (ret)
359                         return ret;
360         } else {
361                 usb3_consumed_up = 0;
362                 usb3_consumed_down = 0;
363         }
364
365         *available_up = *available_down = 40000;
366
367         /* Find the minimum available bandwidth over all links */
368         tb_for_each_port_on_path(src_port, dst_port, port) {
369                 int link_speed, link_width, up_bw, down_bw;
370
371                 if (!tb_port_is_null(port))
372                         continue;
373
374                 if (tb_is_upstream_port(port)) {
375                         link_speed = port->sw->link_speed;
376                 } else {
377                         link_speed = tb_port_get_link_speed(port);
378                         if (link_speed < 0)
379                                 return link_speed;
380                 }
381
382                 link_width = port->bonded ? 2 : 1;
383
384                 up_bw = link_speed * link_width * 1000; /* Mb/s */
385                 /* Leave 10% guard band */
386                 up_bw -= up_bw / 10;
387                 down_bw = up_bw;
388
389                 tb_port_dbg(port, "link total bandwidth %d Mb/s\n", up_bw);
390
391                 /*
392                  * Find all DP tunnels that cross the port and reduce
393                  * their consumed bandwidth from the available.
394                  */
395                 list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
396                         int dp_consumed_up, dp_consumed_down;
397
398                         if (!tb_tunnel_is_dp(tunnel))
399                                 continue;
400
401                         if (!tb_tunnel_port_on_path(tunnel, port))
402                                 continue;
403
404                         ret = tb_tunnel_consumed_bandwidth(tunnel,
405                                                            &dp_consumed_up,
406                                                            &dp_consumed_down);
407                         if (ret)
408                                 return ret;
409
410                         up_bw -= dp_consumed_up;
411                         down_bw -= dp_consumed_down;
412                 }
413
414                 /*
415                  * If USB3 is tunneled from the host router down to the
416                  * branch leading to port we need to take USB3 consumed
417                  * bandwidth into account regardless whether it actually
418                  * crosses the port.
419                  */
420                 up_bw -= usb3_consumed_up;
421                 down_bw -= usb3_consumed_down;
422
423                 if (up_bw < *available_up)
424                         *available_up = up_bw;
425                 if (down_bw < *available_down)
426                         *available_down = down_bw;
427         }
428
429         if (*available_up < 0)
430                 *available_up = 0;
431         if (*available_down < 0)
432                 *available_down = 0;
433
434         return 0;
435 }
436
437 static int tb_release_unused_usb3_bandwidth(struct tb *tb,
438                                             struct tb_port *src_port,
439                                             struct tb_port *dst_port)
440 {
441         struct tb_tunnel *tunnel;
442
443         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
444         return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;
445 }
446
447 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,
448                                       struct tb_port *dst_port)
449 {
450         int ret, available_up, available_down;
451         struct tb_tunnel *tunnel;
452
453         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
454         if (!tunnel)
455                 return;
456
457         tb_dbg(tb, "reclaiming unused bandwidth for USB3\n");
458
459         /*
460          * Calculate available bandwidth for the first hop USB3 tunnel.
461          * That determines the whole USB3 bandwidth for this branch.
462          */
463         ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,
464                                      &available_up, &available_down);
465         if (ret) {
466                 tb_warn(tb, "failed to calculate available bandwidth\n");
467                 return;
468         }
469
470         tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n",
471                available_up, available_down);
472
473         tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);
474 }
475
476 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)
477 {
478         struct tb_switch *parent = tb_switch_parent(sw);
479         int ret, available_up, available_down;
480         struct tb_port *up, *down, *port;
481         struct tb_cm *tcm = tb_priv(tb);
482         struct tb_tunnel *tunnel;
483
484         if (!tb_acpi_may_tunnel_usb3()) {
485                 tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");
486                 return 0;
487         }
488
489         up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);
490         if (!up)
491                 return 0;
492
493         if (!sw->link_usb4)
494                 return 0;
495
496         /*
497          * Look up available down port. Since we are chaining it should
498          * be found right above this switch.
499          */
500         port = tb_port_at(tb_route(sw), parent);
501         down = tb_find_usb3_down(parent, port);
502         if (!down)
503                 return 0;
504
505         if (tb_route(parent)) {
506                 struct tb_port *parent_up;
507                 /*
508                  * Check first that the parent switch has its upstream USB3
509                  * port enabled. Otherwise the chain is not complete and
510                  * there is no point setting up a new tunnel.
511                  */
512                 parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);
513                 if (!parent_up || !tb_port_is_enabled(parent_up))
514                         return 0;
515
516                 /* Make all unused bandwidth available for the new tunnel */
517                 ret = tb_release_unused_usb3_bandwidth(tb, down, up);
518                 if (ret)
519                         return ret;
520         }
521
522         ret = tb_available_bandwidth(tb, down, up, &available_up,
523                                      &available_down);
524         if (ret)
525                 goto err_reclaim;
526
527         tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",
528                     available_up, available_down);
529
530         tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,
531                                       available_down);
532         if (!tunnel) {
533                 ret = -ENOMEM;
534                 goto err_reclaim;
535         }
536
537         if (tb_tunnel_activate(tunnel)) {
538                 tb_port_info(up,
539                              "USB3 tunnel activation failed, aborting\n");
540                 ret = -EIO;
541                 goto err_free;
542         }
543
544         list_add_tail(&tunnel->list, &tcm->tunnel_list);
545         if (tb_route(parent))
546                 tb_reclaim_usb3_bandwidth(tb, down, up);
547
548         return 0;
549
550 err_free:
551         tb_tunnel_free(tunnel);
552 err_reclaim:
553         if (tb_route(parent))
554                 tb_reclaim_usb3_bandwidth(tb, down, up);
555
556         return ret;
557 }
558
559 static int tb_create_usb3_tunnels(struct tb_switch *sw)
560 {
561         struct tb_port *port;
562         int ret;
563
564         if (!tb_acpi_may_tunnel_usb3())
565                 return 0;
566
567         if (tb_route(sw)) {
568                 ret = tb_tunnel_usb3(sw->tb, sw);
569                 if (ret)
570                         return ret;
571         }
572
573         tb_switch_for_each_port(sw, port) {
574                 if (!tb_port_has_remote(port))
575                         continue;
576                 ret = tb_create_usb3_tunnels(port->remote->sw);
577                 if (ret)
578                         return ret;
579         }
580
581         return 0;
582 }
583
584 static void tb_scan_port(struct tb_port *port);
585
586 /*
587  * tb_scan_switch() - scan for and initialize downstream switches
588  */
589 static void tb_scan_switch(struct tb_switch *sw)
590 {
591         struct tb_port *port;
592
593         pm_runtime_get_sync(&sw->dev);
594
595         tb_switch_for_each_port(sw, port)
596                 tb_scan_port(port);
597
598         pm_runtime_mark_last_busy(&sw->dev);
599         pm_runtime_put_autosuspend(&sw->dev);
600 }
601
602 /*
603  * tb_scan_port() - check for and initialize switches below port
604  */
605 static void tb_scan_port(struct tb_port *port)
606 {
607         struct tb_cm *tcm = tb_priv(port->sw->tb);
608         struct tb_port *upstream_port;
609         struct tb_switch *sw;
610
611         if (tb_is_upstream_port(port))
612                 return;
613
614         if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&
615             !tb_dp_port_is_enabled(port)) {
616                 tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");
617                 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,
618                                  false);
619                 return;
620         }
621
622         if (port->config.type != TB_TYPE_PORT)
623                 return;
624         if (port->dual_link_port && port->link_nr)
625                 return; /*
626                          * Downstream switch is reachable through two ports.
627                          * Only scan on the primary port (link_nr == 0).
628                          */
629         if (tb_wait_for_port(port, false) <= 0)
630                 return;
631         if (port->remote) {
632                 tb_port_dbg(port, "port already has a remote\n");
633                 return;
634         }
635
636         tb_retimer_scan(port, true);
637
638         sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,
639                              tb_downstream_route(port));
640         if (IS_ERR(sw)) {
641                 /*
642                  * If there is an error accessing the connected switch
643                  * it may be connected to another domain. Also we allow
644                  * the other domain to be connected to a max depth switch.
645                  */
646                 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL)
647                         tb_scan_xdomain(port);
648                 return;
649         }
650
651         if (tb_switch_configure(sw)) {
652                 tb_switch_put(sw);
653                 return;
654         }
655
656         /*
657          * If there was previously another domain connected remove it
658          * first.
659          */
660         if (port->xdomain) {
661                 tb_xdomain_remove(port->xdomain);
662                 tb_port_unconfigure_xdomain(port);
663                 port->xdomain = NULL;
664         }
665
666         /*
667          * Do not send uevents until we have discovered all existing
668          * tunnels and know which switches were authorized already by
669          * the boot firmware.
670          */
671         if (!tcm->hotplug_active)
672                 dev_set_uevent_suppress(&sw->dev, true);
673
674         /*
675          * At the moment Thunderbolt 2 and beyond (devices with LC) we
676          * can support runtime PM.
677          */
678         sw->rpm = sw->generation > 1;
679
680         if (tb_switch_add(sw)) {
681                 tb_switch_put(sw);
682                 return;
683         }
684
685         /* Link the switches using both links if available */
686         upstream_port = tb_upstream_port(sw);
687         port->remote = upstream_port;
688         upstream_port->remote = port;
689         if (port->dual_link_port && upstream_port->dual_link_port) {
690                 port->dual_link_port->remote = upstream_port->dual_link_port;
691                 upstream_port->dual_link_port->remote = port->dual_link_port;
692         }
693
694         /* Enable lane bonding if supported */
695         tb_switch_lane_bonding_enable(sw);
696         /* Set the link configured */
697         tb_switch_configure_link(sw);
698
699         if (tb_enable_tmu(sw))
700                 tb_sw_warn(sw, "failed to enable TMU\n");
701
702         /* Scan upstream retimers */
703         tb_retimer_scan(upstream_port, true);
704
705         /*
706          * Create USB 3.x tunnels only when the switch is plugged to the
707          * domain. This is because we scan the domain also during discovery
708          * and want to discover existing USB 3.x tunnels before we create
709          * any new.
710          */
711         if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))
712                 tb_sw_warn(sw, "USB3 tunnel creation failed\n");
713
714         tb_add_dp_resources(sw);
715         tb_scan_switch(sw);
716 }
717
718 static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)
719 {
720         struct tb_port *src_port, *dst_port;
721         struct tb *tb;
722
723         if (!tunnel)
724                 return;
725
726         tb_tunnel_deactivate(tunnel);
727         list_del(&tunnel->list);
728
729         tb = tunnel->tb;
730         src_port = tunnel->src_port;
731         dst_port = tunnel->dst_port;
732
733         switch (tunnel->type) {
734         case TB_TUNNEL_DP:
735                 /*
736                  * In case of DP tunnel make sure the DP IN resource is
737                  * deallocated properly.
738                  */
739                 tb_switch_dealloc_dp_resource(src_port->sw, src_port);
740                 /* Now we can allow the domain to runtime suspend again */
741                 pm_runtime_mark_last_busy(&dst_port->sw->dev);
742                 pm_runtime_put_autosuspend(&dst_port->sw->dev);
743                 pm_runtime_mark_last_busy(&src_port->sw->dev);
744                 pm_runtime_put_autosuspend(&src_port->sw->dev);
745                 fallthrough;
746
747         case TB_TUNNEL_USB3:
748                 tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);
749                 break;
750
751         default:
752                 /*
753                  * PCIe and DMA tunnels do not consume guaranteed
754                  * bandwidth.
755                  */
756                 break;
757         }
758
759         tb_tunnel_free(tunnel);
760 }
761
762 /*
763  * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
764  */
765 static void tb_free_invalid_tunnels(struct tb *tb)
766 {
767         struct tb_cm *tcm = tb_priv(tb);
768         struct tb_tunnel *tunnel;
769         struct tb_tunnel *n;
770
771         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
772                 if (tb_tunnel_is_invalid(tunnel))
773                         tb_deactivate_and_free_tunnel(tunnel);
774         }
775 }
776
777 /*
778  * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
779  */
780 static void tb_free_unplugged_children(struct tb_switch *sw)
781 {
782         struct tb_port *port;
783
784         tb_switch_for_each_port(sw, port) {
785                 if (!tb_port_has_remote(port))
786                         continue;
787
788                 if (port->remote->sw->is_unplugged) {
789                         tb_retimer_remove_all(port);
790                         tb_remove_dp_resources(port->remote->sw);
791                         tb_switch_unconfigure_link(port->remote->sw);
792                         tb_switch_lane_bonding_disable(port->remote->sw);
793                         tb_switch_remove(port->remote->sw);
794                         port->remote = NULL;
795                         if (port->dual_link_port)
796                                 port->dual_link_port->remote = NULL;
797                 } else {
798                         tb_free_unplugged_children(port->remote->sw);
799                 }
800         }
801 }
802
803 static struct tb_port *tb_find_pcie_down(struct tb_switch *sw,
804                                          const struct tb_port *port)
805 {
806         struct tb_port *down = NULL;
807
808         /*
809          * To keep plugging devices consistently in the same PCIe
810          * hierarchy, do mapping here for switch downstream PCIe ports.
811          */
812         if (tb_switch_is_usb4(sw)) {
813                 down = usb4_switch_map_pcie_down(sw, port);
814         } else if (!tb_route(sw)) {
815                 int phy_port = tb_phy_port_from_link(port->port);
816                 int index;
817
818                 /*
819                  * Hard-coded Thunderbolt port to PCIe down port mapping
820                  * per controller.
821                  */
822                 if (tb_switch_is_cactus_ridge(sw) ||
823                     tb_switch_is_alpine_ridge(sw))
824                         index = !phy_port ? 6 : 7;
825                 else if (tb_switch_is_falcon_ridge(sw))
826                         index = !phy_port ? 6 : 8;
827                 else if (tb_switch_is_titan_ridge(sw))
828                         index = !phy_port ? 8 : 9;
829                 else
830                         goto out;
831
832                 /* Validate the hard-coding */
833                 if (WARN_ON(index > sw->config.max_port_number))
834                         goto out;
835
836                 down = &sw->ports[index];
837         }
838
839         if (down) {
840                 if (WARN_ON(!tb_port_is_pcie_down(down)))
841                         goto out;
842                 if (tb_pci_port_is_enabled(down))
843                         goto out;
844
845                 return down;
846         }
847
848 out:
849         return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);
850 }
851
852 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)
853 {
854         struct tb_port *host_port, *port;
855         struct tb_cm *tcm = tb_priv(tb);
856
857         host_port = tb_route(in->sw) ?
858                 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;
859
860         list_for_each_entry(port, &tcm->dp_resources, list) {
861                 if (!tb_port_is_dpout(port))
862                         continue;
863
864                 if (tb_port_is_enabled(port)) {
865                         tb_port_dbg(port, "in use\n");
866                         continue;
867                 }
868
869                 tb_port_dbg(port, "DP OUT available\n");
870
871                 /*
872                  * Keep the DP tunnel under the topology starting from
873                  * the same host router downstream port.
874                  */
875                 if (host_port && tb_route(port->sw)) {
876                         struct tb_port *p;
877
878                         p = tb_port_at(tb_route(port->sw), tb->root_switch);
879                         if (p != host_port)
880                                 continue;
881                 }
882
883                 return port;
884         }
885
886         return NULL;
887 }
888
889 static void tb_tunnel_dp(struct tb *tb)
890 {
891         int available_up, available_down, ret, link_nr;
892         struct tb_cm *tcm = tb_priv(tb);
893         struct tb_port *port, *in, *out;
894         struct tb_tunnel *tunnel;
895
896         if (!tb_acpi_may_tunnel_dp()) {
897                 tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");
898                 return;
899         }
900
901         /*
902          * Find pair of inactive DP IN and DP OUT adapters and then
903          * establish a DP tunnel between them.
904          */
905         tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");
906
907         in = NULL;
908         out = NULL;
909         list_for_each_entry(port, &tcm->dp_resources, list) {
910                 if (!tb_port_is_dpin(port))
911                         continue;
912
913                 if (tb_port_is_enabled(port)) {
914                         tb_port_dbg(port, "in use\n");
915                         continue;
916                 }
917
918                 tb_port_dbg(port, "DP IN available\n");
919
920                 out = tb_find_dp_out(tb, port);
921                 if (out) {
922                         in = port;
923                         break;
924                 }
925         }
926
927         if (!in) {
928                 tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");
929                 return;
930         }
931         if (!out) {
932                 tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n");
933                 return;
934         }
935
936         /*
937          * This is only applicable to links that are not bonded (so
938          * when Thunderbolt 1 hardware is involved somewhere in the
939          * topology). For these try to share the DP bandwidth between
940          * the two lanes.
941          */
942         link_nr = 1;
943         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
944                 if (tb_tunnel_is_dp(tunnel)) {
945                         link_nr = 0;
946                         break;
947                 }
948         }
949
950         /*
951          * DP stream needs the domain to be active so runtime resume
952          * both ends of the tunnel.
953          *
954          * This should bring the routers in the middle active as well
955          * and keeps the domain from runtime suspending while the DP
956          * tunnel is active.
957          */
958         pm_runtime_get_sync(&in->sw->dev);
959         pm_runtime_get_sync(&out->sw->dev);
960
961         if (tb_switch_alloc_dp_resource(in->sw, in)) {
962                 tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");
963                 goto err_rpm_put;
964         }
965
966         /* Make all unused USB3 bandwidth available for the new DP tunnel */
967         ret = tb_release_unused_usb3_bandwidth(tb, in, out);
968         if (ret) {
969                 tb_warn(tb, "failed to release unused bandwidth\n");
970                 goto err_dealloc_dp;
971         }
972
973         ret = tb_available_bandwidth(tb, in, out, &available_up,
974                                      &available_down);
975         if (ret)
976                 goto err_reclaim;
977
978         tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",
979                available_up, available_down);
980
981         tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,
982                                     available_down);
983         if (!tunnel) {
984                 tb_port_dbg(out, "could not allocate DP tunnel\n");
985                 goto err_reclaim;
986         }
987
988         if (tb_tunnel_activate(tunnel)) {
989                 tb_port_info(out, "DP tunnel activation failed, aborting\n");
990                 goto err_free;
991         }
992
993         list_add_tail(&tunnel->list, &tcm->tunnel_list);
994         tb_reclaim_usb3_bandwidth(tb, in, out);
995         return;
996
997 err_free:
998         tb_tunnel_free(tunnel);
999 err_reclaim:
1000         tb_reclaim_usb3_bandwidth(tb, in, out);
1001 err_dealloc_dp:
1002         tb_switch_dealloc_dp_resource(in->sw, in);
1003 err_rpm_put:
1004         pm_runtime_mark_last_busy(&out->sw->dev);
1005         pm_runtime_put_autosuspend(&out->sw->dev);
1006         pm_runtime_mark_last_busy(&in->sw->dev);
1007         pm_runtime_put_autosuspend(&in->sw->dev);
1008 }
1009
1010 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)
1011 {
1012         struct tb_port *in, *out;
1013         struct tb_tunnel *tunnel;
1014
1015         if (tb_port_is_dpin(port)) {
1016                 tb_port_dbg(port, "DP IN resource unavailable\n");
1017                 in = port;
1018                 out = NULL;
1019         } else {
1020                 tb_port_dbg(port, "DP OUT resource unavailable\n");
1021                 in = NULL;
1022                 out = port;
1023         }
1024
1025         tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);
1026         tb_deactivate_and_free_tunnel(tunnel);
1027         list_del_init(&port->list);
1028
1029         /*
1030          * See if there is another DP OUT port that can be used for
1031          * to create another tunnel.
1032          */
1033         tb_tunnel_dp(tb);
1034 }
1035
1036 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)
1037 {
1038         struct tb_cm *tcm = tb_priv(tb);
1039         struct tb_port *p;
1040
1041         if (tb_port_is_enabled(port))
1042                 return;
1043
1044         list_for_each_entry(p, &tcm->dp_resources, list) {
1045                 if (p == port)
1046                         return;
1047         }
1048
1049         tb_port_dbg(port, "DP %s resource available\n",
1050                     tb_port_is_dpin(port) ? "IN" : "OUT");
1051         list_add_tail(&port->list, &tcm->dp_resources);
1052
1053         /* Look for suitable DP IN <-> DP OUT pairs now */
1054         tb_tunnel_dp(tb);
1055 }
1056
1057 static void tb_disconnect_and_release_dp(struct tb *tb)
1058 {
1059         struct tb_cm *tcm = tb_priv(tb);
1060         struct tb_tunnel *tunnel, *n;
1061
1062         /*
1063          * Tear down all DP tunnels and release their resources. They
1064          * will be re-established after resume based on plug events.
1065          */
1066         list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {
1067                 if (tb_tunnel_is_dp(tunnel))
1068                         tb_deactivate_and_free_tunnel(tunnel);
1069         }
1070
1071         while (!list_empty(&tcm->dp_resources)) {
1072                 struct tb_port *port;
1073
1074                 port = list_first_entry(&tcm->dp_resources,
1075                                         struct tb_port, list);
1076                 list_del_init(&port->list);
1077         }
1078 }
1079
1080 static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)
1081 {
1082         struct tb_tunnel *tunnel;
1083         struct tb_port *up;
1084
1085         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1086         if (WARN_ON(!up))
1087                 return -ENODEV;
1088
1089         tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);
1090         if (WARN_ON(!tunnel))
1091                 return -ENODEV;
1092
1093         tb_tunnel_deactivate(tunnel);
1094         list_del(&tunnel->list);
1095         tb_tunnel_free(tunnel);
1096         return 0;
1097 }
1098
1099 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)
1100 {
1101         struct tb_port *up, *down, *port;
1102         struct tb_cm *tcm = tb_priv(tb);
1103         struct tb_switch *parent_sw;
1104         struct tb_tunnel *tunnel;
1105
1106         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1107         if (!up)
1108                 return 0;
1109
1110         /*
1111          * Look up available down port. Since we are chaining it should
1112          * be found right above this switch.
1113          */
1114         parent_sw = tb_to_switch(sw->dev.parent);
1115         port = tb_port_at(tb_route(sw), parent_sw);
1116         down = tb_find_pcie_down(parent_sw, port);
1117         if (!down)
1118                 return 0;
1119
1120         tunnel = tb_tunnel_alloc_pci(tb, up, down);
1121         if (!tunnel)
1122                 return -ENOMEM;
1123
1124         if (tb_tunnel_activate(tunnel)) {
1125                 tb_port_info(up,
1126                              "PCIe tunnel activation failed, aborting\n");
1127                 tb_tunnel_free(tunnel);
1128                 return -EIO;
1129         }
1130
1131         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1132         return 0;
1133 }
1134
1135 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1136                                     int transmit_path, int transmit_ring,
1137                                     int receive_path, int receive_ring)
1138 {
1139         struct tb_cm *tcm = tb_priv(tb);
1140         struct tb_port *nhi_port, *dst_port;
1141         struct tb_tunnel *tunnel;
1142         struct tb_switch *sw;
1143
1144         sw = tb_to_switch(xd->dev.parent);
1145         dst_port = tb_port_at(xd->route, sw);
1146         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1147
1148         mutex_lock(&tb->lock);
1149         tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,
1150                                      transmit_ring, receive_path, receive_ring);
1151         if (!tunnel) {
1152                 mutex_unlock(&tb->lock);
1153                 return -ENOMEM;
1154         }
1155
1156         if (tb_tunnel_activate(tunnel)) {
1157                 tb_port_info(nhi_port,
1158                              "DMA tunnel activation failed, aborting\n");
1159                 tb_tunnel_free(tunnel);
1160                 mutex_unlock(&tb->lock);
1161                 return -EIO;
1162         }
1163
1164         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1165         mutex_unlock(&tb->lock);
1166         return 0;
1167 }
1168
1169 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1170                                           int transmit_path, int transmit_ring,
1171                                           int receive_path, int receive_ring)
1172 {
1173         struct tb_cm *tcm = tb_priv(tb);
1174         struct tb_port *nhi_port, *dst_port;
1175         struct tb_tunnel *tunnel, *n;
1176         struct tb_switch *sw;
1177
1178         sw = tb_to_switch(xd->dev.parent);
1179         dst_port = tb_port_at(xd->route, sw);
1180         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1181
1182         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1183                 if (!tb_tunnel_is_dma(tunnel))
1184                         continue;
1185                 if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port)
1186                         continue;
1187
1188                 if (tb_tunnel_match_dma(tunnel, transmit_path, transmit_ring,
1189                                         receive_path, receive_ring))
1190                         tb_deactivate_and_free_tunnel(tunnel);
1191         }
1192 }
1193
1194 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1195                                        int transmit_path, int transmit_ring,
1196                                        int receive_path, int receive_ring)
1197 {
1198         if (!xd->is_unplugged) {
1199                 mutex_lock(&tb->lock);
1200                 __tb_disconnect_xdomain_paths(tb, xd, transmit_path,
1201                                               transmit_ring, receive_path,
1202                                               receive_ring);
1203                 mutex_unlock(&tb->lock);
1204         }
1205         return 0;
1206 }
1207
1208 /* hotplug handling */
1209
1210 /*
1211  * tb_handle_hotplug() - handle hotplug event
1212  *
1213  * Executes on tb->wq.
1214  */
1215 static void tb_handle_hotplug(struct work_struct *work)
1216 {
1217         struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);
1218         struct tb *tb = ev->tb;
1219         struct tb_cm *tcm = tb_priv(tb);
1220         struct tb_switch *sw;
1221         struct tb_port *port;
1222
1223         /* Bring the domain back from sleep if it was suspended */
1224         pm_runtime_get_sync(&tb->dev);
1225
1226         mutex_lock(&tb->lock);
1227         if (!tcm->hotplug_active)
1228                 goto out; /* during init, suspend or shutdown */
1229
1230         sw = tb_switch_find_by_route(tb, ev->route);
1231         if (!sw) {
1232                 tb_warn(tb,
1233                         "hotplug event from non existent switch %llx:%x (unplug: %d)\n",
1234                         ev->route, ev->port, ev->unplug);
1235                 goto out;
1236         }
1237         if (ev->port > sw->config.max_port_number) {
1238                 tb_warn(tb,
1239                         "hotplug event from non existent port %llx:%x (unplug: %d)\n",
1240                         ev->route, ev->port, ev->unplug);
1241                 goto put_sw;
1242         }
1243         port = &sw->ports[ev->port];
1244         if (tb_is_upstream_port(port)) {
1245                 tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",
1246                        ev->route, ev->port, ev->unplug);
1247                 goto put_sw;
1248         }
1249
1250         pm_runtime_get_sync(&sw->dev);
1251
1252         if (ev->unplug) {
1253                 tb_retimer_remove_all(port);
1254
1255                 if (tb_port_has_remote(port)) {
1256                         tb_port_dbg(port, "switch unplugged\n");
1257                         tb_sw_set_unplugged(port->remote->sw);
1258                         tb_free_invalid_tunnels(tb);
1259                         tb_remove_dp_resources(port->remote->sw);
1260                         tb_switch_tmu_disable(port->remote->sw);
1261                         tb_switch_unconfigure_link(port->remote->sw);
1262                         tb_switch_lane_bonding_disable(port->remote->sw);
1263                         tb_switch_remove(port->remote->sw);
1264                         port->remote = NULL;
1265                         if (port->dual_link_port)
1266                                 port->dual_link_port->remote = NULL;
1267                         /* Maybe we can create another DP tunnel */
1268                         tb_tunnel_dp(tb);
1269                 } else if (port->xdomain) {
1270                         struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);
1271
1272                         tb_port_dbg(port, "xdomain unplugged\n");
1273                         /*
1274                          * Service drivers are unbound during
1275                          * tb_xdomain_remove() so setting XDomain as
1276                          * unplugged here prevents deadlock if they call
1277                          * tb_xdomain_disable_paths(). We will tear down
1278                          * all the tunnels below.
1279                          */
1280                         xd->is_unplugged = true;
1281                         tb_xdomain_remove(xd);
1282                         port->xdomain = NULL;
1283                         __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);
1284                         tb_xdomain_put(xd);
1285                         tb_port_unconfigure_xdomain(port);
1286                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1287                         tb_dp_resource_unavailable(tb, port);
1288                 } else {
1289                         tb_port_dbg(port,
1290                                    "got unplug event for disconnected port, ignoring\n");
1291                 }
1292         } else if (port->remote) {
1293                 tb_port_dbg(port, "got plug event for connected port, ignoring\n");
1294         } else {
1295                 if (tb_port_is_null(port)) {
1296                         tb_port_dbg(port, "hotplug: scanning\n");
1297                         tb_scan_port(port);
1298                         if (!port->remote)
1299                                 tb_port_dbg(port, "hotplug: no switch found\n");
1300                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1301                         tb_dp_resource_available(tb, port);
1302                 }
1303         }
1304
1305         pm_runtime_mark_last_busy(&sw->dev);
1306         pm_runtime_put_autosuspend(&sw->dev);
1307
1308 put_sw:
1309         tb_switch_put(sw);
1310 out:
1311         mutex_unlock(&tb->lock);
1312
1313         pm_runtime_mark_last_busy(&tb->dev);
1314         pm_runtime_put_autosuspend(&tb->dev);
1315
1316         kfree(ev);
1317 }
1318
1319 /*
1320  * tb_schedule_hotplug_handler() - callback function for the control channel
1321  *
1322  * Delegates to tb_handle_hotplug.
1323  */
1324 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1325                             const void *buf, size_t size)
1326 {
1327         const struct cfg_event_pkg *pkg = buf;
1328         u64 route;
1329
1330         if (type != TB_CFG_PKG_EVENT) {
1331                 tb_warn(tb, "unexpected event %#x, ignoring\n", type);
1332                 return;
1333         }
1334
1335         route = tb_cfg_get_route(&pkg->header);
1336
1337         if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {
1338                 tb_warn(tb, "could not ack plug event on %llx:%x\n", route,
1339                         pkg->port);
1340         }
1341
1342         tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);
1343 }
1344
1345 static void tb_stop(struct tb *tb)
1346 {
1347         struct tb_cm *tcm = tb_priv(tb);
1348         struct tb_tunnel *tunnel;
1349         struct tb_tunnel *n;
1350
1351         cancel_delayed_work(&tcm->remove_work);
1352         /* tunnels are only present after everything has been initialized */
1353         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1354                 /*
1355                  * DMA tunnels require the driver to be functional so we
1356                  * tear them down. Other protocol tunnels can be left
1357                  * intact.
1358                  */
1359                 if (tb_tunnel_is_dma(tunnel))
1360                         tb_tunnel_deactivate(tunnel);
1361                 tb_tunnel_free(tunnel);
1362         }
1363         tb_switch_remove(tb->root_switch);
1364         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1365 }
1366
1367 static int tb_scan_finalize_switch(struct device *dev, void *data)
1368 {
1369         if (tb_is_switch(dev)) {
1370                 struct tb_switch *sw = tb_to_switch(dev);
1371
1372                 /*
1373                  * If we found that the switch was already setup by the
1374                  * boot firmware, mark it as authorized now before we
1375                  * send uevent to userspace.
1376                  */
1377                 if (sw->boot)
1378                         sw->authorized = 1;
1379
1380                 dev_set_uevent_suppress(dev, false);
1381                 kobject_uevent(&dev->kobj, KOBJ_ADD);
1382                 device_for_each_child(dev, NULL, tb_scan_finalize_switch);
1383         }
1384
1385         return 0;
1386 }
1387
1388 static int tb_start(struct tb *tb)
1389 {
1390         struct tb_cm *tcm = tb_priv(tb);
1391         int ret;
1392
1393         tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
1394         if (IS_ERR(tb->root_switch))
1395                 return PTR_ERR(tb->root_switch);
1396
1397         /*
1398          * ICM firmware upgrade needs running firmware and in native
1399          * mode that is not available so disable firmware upgrade of the
1400          * root switch.
1401          */
1402         tb->root_switch->no_nvm_upgrade = true;
1403         /* All USB4 routers support runtime PM */
1404         tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);
1405
1406         ret = tb_switch_configure(tb->root_switch);
1407         if (ret) {
1408                 tb_switch_put(tb->root_switch);
1409                 return ret;
1410         }
1411
1412         /* Announce the switch to the world */
1413         ret = tb_switch_add(tb->root_switch);
1414         if (ret) {
1415                 tb_switch_put(tb->root_switch);
1416                 return ret;
1417         }
1418
1419         /* Enable TMU if it is off */
1420         tb_switch_tmu_enable(tb->root_switch);
1421         /* Full scan to discover devices added before the driver was loaded. */
1422         tb_scan_switch(tb->root_switch);
1423         /* Find out tunnels created by the boot firmware */
1424         tb_discover_tunnels(tb);
1425         /* Add DP resources from the DP tunnels created by the boot firmware */
1426         tb_discover_dp_resources(tb);
1427         /*
1428          * If the boot firmware did not create USB 3.x tunnels create them
1429          * now for the whole topology.
1430          */
1431         tb_create_usb3_tunnels(tb->root_switch);
1432         /* Add DP IN resources for the root switch */
1433         tb_add_dp_resources(tb->root_switch);
1434         /* Make the discovered switches available to the userspace */
1435         device_for_each_child(&tb->root_switch->dev, NULL,
1436                               tb_scan_finalize_switch);
1437
1438         /* Allow tb_handle_hotplug to progress events */
1439         tcm->hotplug_active = true;
1440         return 0;
1441 }
1442
1443 static int tb_suspend_noirq(struct tb *tb)
1444 {
1445         struct tb_cm *tcm = tb_priv(tb);
1446
1447         tb_dbg(tb, "suspending...\n");
1448         tb_disconnect_and_release_dp(tb);
1449         tb_switch_suspend(tb->root_switch, false);
1450         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1451         tb_dbg(tb, "suspend finished\n");
1452
1453         return 0;
1454 }
1455
1456 static void tb_restore_children(struct tb_switch *sw)
1457 {
1458         struct tb_port *port;
1459
1460         /* No need to restore if the router is already unplugged */
1461         if (sw->is_unplugged)
1462                 return;
1463
1464         if (tb_enable_tmu(sw))
1465                 tb_sw_warn(sw, "failed to restore TMU configuration\n");
1466
1467         tb_switch_for_each_port(sw, port) {
1468                 if (!tb_port_has_remote(port) && !port->xdomain)
1469                         continue;
1470
1471                 if (port->remote) {
1472                         tb_switch_lane_bonding_enable(port->remote->sw);
1473                         tb_switch_configure_link(port->remote->sw);
1474
1475                         tb_restore_children(port->remote->sw);
1476                 } else if (port->xdomain) {
1477                         tb_port_configure_xdomain(port);
1478                 }
1479         }
1480 }
1481
1482 static int tb_resume_noirq(struct tb *tb)
1483 {
1484         struct tb_cm *tcm = tb_priv(tb);
1485         struct tb_tunnel *tunnel, *n;
1486         unsigned int usb3_delay = 0;
1487         LIST_HEAD(tunnels);
1488
1489         tb_dbg(tb, "resuming...\n");
1490
1491         /* remove any pci devices the firmware might have setup */
1492         tb_switch_reset(tb->root_switch);
1493
1494         tb_switch_resume(tb->root_switch);
1495         tb_free_invalid_tunnels(tb);
1496         tb_free_unplugged_children(tb->root_switch);
1497         tb_restore_children(tb->root_switch);
1498
1499         /*
1500          * If we get here from suspend to disk the boot firmware or the
1501          * restore kernel might have created tunnels of its own. Since
1502          * we cannot be sure they are usable for us we find and tear
1503          * them down.
1504          */
1505         tb_switch_discover_tunnels(tb->root_switch, &tunnels, false);
1506         list_for_each_entry_safe_reverse(tunnel, n, &tunnels, list) {
1507                 if (tb_tunnel_is_usb3(tunnel))
1508                         usb3_delay = 500;
1509                 tb_tunnel_deactivate(tunnel);
1510                 tb_tunnel_free(tunnel);
1511         }
1512
1513         /* Re-create our tunnels now */
1514         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1515                 /* USB3 requires delay before it can be re-activated */
1516                 if (tb_tunnel_is_usb3(tunnel)) {
1517                         msleep(usb3_delay);
1518                         /* Only need to do it once */
1519                         usb3_delay = 0;
1520                 }
1521                 tb_tunnel_restart(tunnel);
1522         }
1523         if (!list_empty(&tcm->tunnel_list)) {
1524                 /*
1525                  * the pcie links need some time to get going.
1526                  * 100ms works for me...
1527                  */
1528                 tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");
1529                 msleep(100);
1530         }
1531          /* Allow tb_handle_hotplug to progress events */
1532         tcm->hotplug_active = true;
1533         tb_dbg(tb, "resume finished\n");
1534
1535         return 0;
1536 }
1537
1538 static int tb_free_unplugged_xdomains(struct tb_switch *sw)
1539 {
1540         struct tb_port *port;
1541         int ret = 0;
1542
1543         tb_switch_for_each_port(sw, port) {
1544                 if (tb_is_upstream_port(port))
1545                         continue;
1546                 if (port->xdomain && port->xdomain->is_unplugged) {
1547                         tb_retimer_remove_all(port);
1548                         tb_xdomain_remove(port->xdomain);
1549                         tb_port_unconfigure_xdomain(port);
1550                         port->xdomain = NULL;
1551                         ret++;
1552                 } else if (port->remote) {
1553                         ret += tb_free_unplugged_xdomains(port->remote->sw);
1554                 }
1555         }
1556
1557         return ret;
1558 }
1559
1560 static int tb_freeze_noirq(struct tb *tb)
1561 {
1562         struct tb_cm *tcm = tb_priv(tb);
1563
1564         tcm->hotplug_active = false;
1565         return 0;
1566 }
1567
1568 static int tb_thaw_noirq(struct tb *tb)
1569 {
1570         struct tb_cm *tcm = tb_priv(tb);
1571
1572         tcm->hotplug_active = true;
1573         return 0;
1574 }
1575
1576 static void tb_complete(struct tb *tb)
1577 {
1578         /*
1579          * Release any unplugged XDomains and if there is a case where
1580          * another domain is swapped in place of unplugged XDomain we
1581          * need to run another rescan.
1582          */
1583         mutex_lock(&tb->lock);
1584         if (tb_free_unplugged_xdomains(tb->root_switch))
1585                 tb_scan_switch(tb->root_switch);
1586         mutex_unlock(&tb->lock);
1587 }
1588
1589 static int tb_runtime_suspend(struct tb *tb)
1590 {
1591         struct tb_cm *tcm = tb_priv(tb);
1592
1593         mutex_lock(&tb->lock);
1594         tb_switch_suspend(tb->root_switch, true);
1595         tcm->hotplug_active = false;
1596         mutex_unlock(&tb->lock);
1597
1598         return 0;
1599 }
1600
1601 static void tb_remove_work(struct work_struct *work)
1602 {
1603         struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);
1604         struct tb *tb = tcm_to_tb(tcm);
1605
1606         mutex_lock(&tb->lock);
1607         if (tb->root_switch) {
1608                 tb_free_unplugged_children(tb->root_switch);
1609                 tb_free_unplugged_xdomains(tb->root_switch);
1610         }
1611         mutex_unlock(&tb->lock);
1612 }
1613
1614 static int tb_runtime_resume(struct tb *tb)
1615 {
1616         struct tb_cm *tcm = tb_priv(tb);
1617         struct tb_tunnel *tunnel, *n;
1618
1619         mutex_lock(&tb->lock);
1620         tb_switch_resume(tb->root_switch);
1621         tb_free_invalid_tunnels(tb);
1622         tb_restore_children(tb->root_switch);
1623         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1624                 tb_tunnel_restart(tunnel);
1625         tcm->hotplug_active = true;
1626         mutex_unlock(&tb->lock);
1627
1628         /*
1629          * Schedule cleanup of any unplugged devices. Run this in a
1630          * separate thread to avoid possible deadlock if the device
1631          * removal runtime resumes the unplugged device.
1632          */
1633         queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));
1634         return 0;
1635 }
1636
1637 static const struct tb_cm_ops tb_cm_ops = {
1638         .start = tb_start,
1639         .stop = tb_stop,
1640         .suspend_noirq = tb_suspend_noirq,
1641         .resume_noirq = tb_resume_noirq,
1642         .freeze_noirq = tb_freeze_noirq,
1643         .thaw_noirq = tb_thaw_noirq,
1644         .complete = tb_complete,
1645         .runtime_suspend = tb_runtime_suspend,
1646         .runtime_resume = tb_runtime_resume,
1647         .handle_event = tb_handle_event,
1648         .disapprove_switch = tb_disconnect_pci,
1649         .approve_switch = tb_tunnel_pci,
1650         .approve_xdomain_paths = tb_approve_xdomain_paths,
1651         .disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
1652 };
1653
1654 /*
1655  * During suspend the Thunderbolt controller is reset and all PCIe
1656  * tunnels are lost. The NHI driver will try to reestablish all tunnels
1657  * during resume. This adds device links between the tunneled PCIe
1658  * downstream ports and the NHI so that the device core will make sure
1659  * NHI is resumed first before the rest.
1660  */
1661 static void tb_apple_add_links(struct tb_nhi *nhi)
1662 {
1663         struct pci_dev *upstream, *pdev;
1664
1665         if (!x86_apple_machine)
1666                 return;
1667
1668         switch (nhi->pdev->device) {
1669         case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
1670         case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
1671         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
1672         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
1673                 break;
1674         default:
1675                 return;
1676         }
1677
1678         upstream = pci_upstream_bridge(nhi->pdev);
1679         while (upstream) {
1680                 if (!pci_is_pcie(upstream))
1681                         return;
1682                 if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
1683                         break;
1684                 upstream = pci_upstream_bridge(upstream);
1685         }
1686
1687         if (!upstream)
1688                 return;
1689
1690         /*
1691          * For each hotplug downstream port, create add device link
1692          * back to NHI so that PCIe tunnels can be re-established after
1693          * sleep.
1694          */
1695         for_each_pci_bridge(pdev, upstream->subordinate) {
1696                 const struct device_link *link;
1697
1698                 if (!pci_is_pcie(pdev))
1699                         continue;
1700                 if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
1701                     !pdev->is_hotplug_bridge)
1702                         continue;
1703
1704                 link = device_link_add(&pdev->dev, &nhi->pdev->dev,
1705                                        DL_FLAG_AUTOREMOVE_SUPPLIER |
1706                                        DL_FLAG_PM_RUNTIME);
1707                 if (link) {
1708                         dev_dbg(&nhi->pdev->dev, "created link from %s\n",
1709                                 dev_name(&pdev->dev));
1710                 } else {
1711                         dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n",
1712                                  dev_name(&pdev->dev));
1713                 }
1714         }
1715 }
1716
1717 struct tb *tb_probe(struct tb_nhi *nhi)
1718 {
1719         struct tb_cm *tcm;
1720         struct tb *tb;
1721
1722         tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));
1723         if (!tb)
1724                 return NULL;
1725
1726         if (tb_acpi_may_tunnel_pcie())
1727                 tb->security_level = TB_SECURITY_USER;
1728         else
1729                 tb->security_level = TB_SECURITY_NOPCIE;
1730
1731         tb->cm_ops = &tb_cm_ops;
1732
1733         tcm = tb_priv(tb);
1734         INIT_LIST_HEAD(&tcm->tunnel_list);
1735         INIT_LIST_HEAD(&tcm->dp_resources);
1736         INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);
1737
1738         tb_dbg(tb, "using software connection manager\n");
1739
1740         tb_apple_add_links(nhi);
1741         tb_acpi_add_links(nhi);
1742
1743         return tb;
1744 }