drivers/thunderbolt/icm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Internal Thunderbolt Connection Manager. This is a firmware running on
   4  * the Thunderbolt host controller performing most of the low-level
   5  * handling.
   6  *
   7  * Copyright (C) 2017, Intel Corporation
   8  * Authors: Michael Jamet <michael.jamet@intel.com>
   9  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  10  */
  11
  12 #include <linux/delay.h>
  13 #include <linux/mutex.h>
  14 #include <linux/moduleparam.h>
  15 #include <linux/pci.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/platform_data/x86/apple.h>
  18 #include <linux/sizes.h>
  19 #include <linux/slab.h>
  20 #include <linux/workqueue.h>
  21
  22 #include "ctl.h"
  23 #include "nhi_regs.h"
  24 #include "tb.h"
  25
  26 #define PCIE2CIO_CMD                    0x30
  27 #define PCIE2CIO_CMD_TIMEOUT            BIT(31)
  28 #define PCIE2CIO_CMD_START              BIT(30)
  29 #define PCIE2CIO_CMD_WRITE              BIT(21)
  30 #define PCIE2CIO_CMD_CS_MASK            GENMASK(20, 19)
  31 #define PCIE2CIO_CMD_CS_SHIFT           19
  32 #define PCIE2CIO_CMD_PORT_MASK          GENMASK(18, 13)
  33 #define PCIE2CIO_CMD_PORT_SHIFT         13
  34
  35 #define PCIE2CIO_WRDATA                 0x34
  36 #define PCIE2CIO_RDDATA                 0x38
  37
  38 #define PHY_PORT_CS1                    0x37
  39 #define PHY_PORT_CS1_LINK_DISABLE       BIT(14)
  40 #define PHY_PORT_CS1_LINK_STATE_MASK    GENMASK(29, 26)
  41 #define PHY_PORT_CS1_LINK_STATE_SHIFT   26
  42
  43 #define ICM_TIMEOUT                     5000    /* ms */
  44 #define ICM_APPROVE_TIMEOUT             10000   /* ms */
  45 #define ICM_MAX_LINK                    4
  46
  47 static bool start_icm;
  48 module_param(start_icm, bool, 0444);
  49 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
  50
  51 /**
  52  * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
  53  * @reply: Reply from ICM firmware is placed here
  54  * @request: Request that is sent to ICM firmware
  55  * @icm: Pointer to ICM private data
  56  */
  57 struct usb4_switch_nvm_auth {
  58         struct icm_usb4_switch_op_response reply;
  59         struct icm_usb4_switch_op request;
  60         struct icm *icm;
  61 };
  62
  63 /**
  64  * struct icm - Internal connection manager private data
  65  * @request_lock: Makes sure only one message is send to ICM at time
  66  * @rescan_work: Work used to rescan the surviving switches after resume
  67  * @upstream_port: Pointer to the PCIe upstream port this host
  68  *                 controller is connected. This is only set for systems
  69  *                 where ICM needs to be started manually
  70  * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
  71  *           (only set when @upstream_port is not %NULL)
  72  * @safe_mode: ICM is in safe mode
  73  * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
  74  * @rpm: Does the controller support runtime PM (RTD3)
  75  * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
  76  * @proto_version: Firmware protocol version
  77  * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
  78  * @veto: Is RTD3 veto in effect
  79  * @is_supported: Checks if we can support ICM on this controller
  80  * @cio_reset: Trigger CIO reset
  81  * @get_mode: Read and return the ICM firmware mode (optional)
  82  * @get_route: Find a route string for given switch
  83  * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
  84  * @driver_ready: Send driver ready message to ICM
  85  * @set_uuid: Set UUID for the root switch (optional)
  86  * @device_connected: Handle device connected ICM message
  87  * @device_disconnected: Handle device disconnected ICM message
  88  * @xdomain_connected: Handle XDomain connected ICM message
  89  * @xdomain_disconnected: Handle XDomain disconnected ICM message
  90  * @rtd3_veto: Handle RTD3 veto notification ICM message
  91  */
  92 struct icm {
  93         struct mutex request_lock;
  94         struct delayed_work rescan_work;
  95         struct pci_dev *upstream_port;
  96         int vnd_cap;
  97         bool safe_mode;
  98         size_t max_boot_acl;
  99         bool rpm;
 100         bool can_upgrade_nvm;
 101         u8 proto_version;
 102         struct usb4_switch_nvm_auth *last_nvm_auth;
 103         bool veto;
 104         bool (*is_supported)(struct tb *tb);
 105         int (*cio_reset)(struct tb *tb);
 106         int (*get_mode)(struct tb *tb);
 107         int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
 108         void (*save_devices)(struct tb *tb);
 109         int (*driver_ready)(struct tb *tb,
 110                             enum tb_security_level *security_level,
 111                             u8 *proto_version, size_t *nboot_acl, bool *rpm);
 112         void (*set_uuid)(struct tb *tb);
 113         void (*device_connected)(struct tb *tb,
 114                                  const struct icm_pkg_header *hdr);
 115         void (*device_disconnected)(struct tb *tb,
 116                                     const struct icm_pkg_header *hdr);
 117         void (*xdomain_connected)(struct tb *tb,
 118                                   const struct icm_pkg_header *hdr);
 119         void (*xdomain_disconnected)(struct tb *tb,
 120                                      const struct icm_pkg_header *hdr);
 121         void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
 122 };
 123
 124 struct icm_notification {
 125         struct work_struct work;
 126         struct icm_pkg_header *pkg;
 127         struct tb *tb;
 128 };
 129
 130 struct ep_name_entry {
 131         u8 len;
 132         u8 type;
 133         u8 data[];
 134 };
 135
 136 #define EP_NAME_INTEL_VSS       0x10
 137
 138 /* Intel Vendor specific structure */
 139 struct intel_vss {
 140         u16 vendor;
 141         u16 model;
 142         u8 mc;
 143         u8 flags;
 144         u16 pci_devid;
 145         u32 nvm_version;
 146 };
 147
 148 #define INTEL_VSS_FLAGS_RTD3    BIT(0)
 149
 150 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
 151 {
 152         const void *end = ep_name + size;
 153
 154         while (ep_name < end) {
 155                 const struct ep_name_entry *ep = ep_name;
 156
 157                 if (!ep->len)
 158                         break;
 159                 if (ep_name + ep->len > end)
 160                         break;
 161
 162                 if (ep->type == EP_NAME_INTEL_VSS)
 163                         return (const struct intel_vss *)ep->data;
 164
 165                 ep_name += ep->len;
 166         }
 167
 168         return NULL;
 169 }
 170
 171 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
 172 {
 173         const struct intel_vss *vss;
 174
 175         vss = parse_intel_vss(ep_name, size);
 176         if (vss)
 177                 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
 178
 179         return false;
 180 }
 181
 182 static inline struct tb *icm_to_tb(struct icm *icm)
 183 {
 184         return ((void *)icm - sizeof(struct tb));
 185 }
 186
 187 static inline u8 phy_port_from_route(u64 route, u8 depth)
 188 {
 189         u8 link;
 190
 191         link = depth ? route >> ((depth - 1) * 8) : route;
 192         return tb_phy_port_from_link(link);
 193 }
 194
 195 static inline u8 dual_link_from_link(u8 link)
 196 {
 197         return link ? ((link - 1) ^ 0x01) + 1 : 0;
 198 }
 199
 200 static inline u64 get_route(u32 route_hi, u32 route_lo)
 201 {
 202         return (u64)route_hi << 32 | route_lo;
 203 }
 204
 205 static inline u64 get_parent_route(u64 route)
 206 {
 207         int depth = tb_route_length(route);
 208         return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
 209 }
 210
 211 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
 212 {
 213         unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
 214         u32 cmd;
 215
 216         do {
 217                 pci_read_config_dword(icm->upstream_port,
 218                                       icm->vnd_cap + PCIE2CIO_CMD, &cmd);
 219                 if (!(cmd & PCIE2CIO_CMD_START)) {
 220                         if (cmd & PCIE2CIO_CMD_TIMEOUT)
 221                                 break;
 222                         return 0;
 223                 }
 224
 225                 msleep(50);
 226         } while (time_before(jiffies, end));
 227
 228         return -ETIMEDOUT;
 229 }
 230
 231 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
 232                          unsigned int port, unsigned int index, u32 *data)
 233 {
 234         struct pci_dev *pdev = icm->upstream_port;
 235         int ret, vnd_cap = icm->vnd_cap;
 236         u32 cmd;
 237
 238         cmd = index;
 239         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 240         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 241         cmd |= PCIE2CIO_CMD_START;
 242         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 243
 244         ret = pci2cio_wait_completion(icm, 5000);
 245         if (ret)
 246                 return ret;
 247
 248         pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
 249         return 0;
 250 }
 251
 252 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
 253                           unsigned int port, unsigned int index, u32 data)
 254 {
 255         struct pci_dev *pdev = icm->upstream_port;
 256         int vnd_cap = icm->vnd_cap;
 257         u32 cmd;
 258
 259         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
 260
 261         cmd = index;
 262         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 263         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 264         cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
 265         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 266
 267         return pci2cio_wait_completion(icm, 5000);
 268 }
 269
 270 static bool icm_match(const struct tb_cfg_request *req,
 271                       const struct ctl_pkg *pkg)
 272 {
 273         const struct icm_pkg_header *res_hdr = pkg->buffer;
 274         const struct icm_pkg_header *req_hdr = req->request;
 275
 276         if (pkg->frame.eof != req->response_type)
 277                 return false;
 278         if (res_hdr->code != req_hdr->code)
 279                 return false;
 280
 281         return true;
 282 }
 283
 284 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
 285 {
 286         const struct icm_pkg_header *hdr = pkg->buffer;
 287
 288         if (hdr->packet_id < req->npackets) {
 289                 size_t offset = hdr->packet_id * req->response_size;
 290
 291                 memcpy(req->response + offset, pkg->buffer, req->response_size);
 292         }
 293
 294         return hdr->packet_id == hdr->total_packets - 1;
 295 }
 296
 297 static int icm_request(struct tb *tb, const void *request, size_t request_size,
 298                        void *response, size_t response_size, size_t npackets,
 299                        unsigned int timeout_msec)
 300 {
 301         struct icm *icm = tb_priv(tb);
 302         int retries = 3;
 303
 304         do {
 305                 struct tb_cfg_request *req;
 306                 struct tb_cfg_result res;
 307
 308                 req = tb_cfg_request_alloc();
 309                 if (!req)
 310                         return -ENOMEM;
 311
 312                 req->match = icm_match;
 313                 req->copy = icm_copy;
 314                 req->request = request;
 315                 req->request_size = request_size;
 316                 req->request_type = TB_CFG_PKG_ICM_CMD;
 317                 req->response = response;
 318                 req->npackets = npackets;
 319                 req->response_size = response_size;
 320                 req->response_type = TB_CFG_PKG_ICM_RESP;
 321
 322                 mutex_lock(&icm->request_lock);
 323                 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
 324                 mutex_unlock(&icm->request_lock);
 325
 326                 tb_cfg_request_put(req);
 327
 328                 if (res.err != -ETIMEDOUT)
 329                         return res.err == 1 ? -EIO : res.err;
 330
 331                 usleep_range(20, 50);
 332         } while (retries--);
 333
 334         return -ETIMEDOUT;
 335 }
 336
 337 /*
 338  * If rescan is queued to run (we are resuming), postpone it to give the
 339  * firmware some more time to send device connected notifications for next
 340  * devices in the chain.
 341  */
 342 static void icm_postpone_rescan(struct tb *tb)
 343 {
 344         struct icm *icm = tb_priv(tb);
 345
 346         if (delayed_work_pending(&icm->rescan_work))
 347                 mod_delayed_work(tb->wq, &icm->rescan_work,
 348                                  msecs_to_jiffies(500));
 349 }
 350
 351 static void icm_veto_begin(struct tb *tb)
 352 {
 353         struct icm *icm = tb_priv(tb);
 354
 355         if (!icm->veto) {
 356                 icm->veto = true;
 357                 /* Keep the domain powered while veto is in effect */
 358                 pm_runtime_get(&tb->dev);
 359         }
 360 }
 361
 362 static void icm_veto_end(struct tb *tb)
 363 {
 364         struct icm *icm = tb_priv(tb);
 365
 366         if (icm->veto) {
 367                 icm->veto = false;
 368                 /* Allow the domain suspend now */
 369                 pm_runtime_mark_last_busy(&tb->dev);
 370                 pm_runtime_put_autosuspend(&tb->dev);
 371         }
 372 }
 373
 374 static bool icm_firmware_running(const struct tb_nhi *nhi)
 375 {
 376         u32 val;
 377
 378         val = ioread32(nhi->iobase + REG_FW_STS);
 379         return !!(val & REG_FW_STS_ICM_EN);
 380 }
 381
 382 static bool icm_fr_is_supported(struct tb *tb)
 383 {
 384         return !x86_apple_machine;
 385 }
 386
 387 static inline int icm_fr_get_switch_index(u32 port)
 388 {
 389         int index;
 390
 391         if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
 392                 return 0;
 393
 394         index = port >> ICM_PORT_INDEX_SHIFT;
 395         return index != 0xff ? index : 0;
 396 }
 397
 398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
 399 {
 400         struct icm_fr_pkg_get_topology_response *switches, *sw;
 401         struct icm_fr_pkg_get_topology request = {
 402                 .hdr = { .code = ICM_GET_TOPOLOGY },
 403         };
 404         size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
 405         int ret, index;
 406         u8 i;
 407
 408         switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
 409         if (!switches)
 410                 return -ENOMEM;
 411
 412         ret = icm_request(tb, &request, sizeof(request), switches,
 413                           sizeof(*switches), npackets, ICM_TIMEOUT);
 414         if (ret)
 415                 goto err_free;
 416
 417         sw = &switches[0];
 418         index = icm_fr_get_switch_index(sw->ports[link]);
 419         if (!index) {
 420                 ret = -ENODEV;
 421                 goto err_free;
 422         }
 423
 424         sw = &switches[index];
 425         for (i = 1; i < depth; i++) {
 426                 unsigned int j;
 427
 428                 if (!(sw->first_data & ICM_SWITCH_USED)) {
 429                         ret = -ENODEV;
 430                         goto err_free;
 431                 }
 432
 433                 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
 434                         index = icm_fr_get_switch_index(sw->ports[j]);
 435                         if (index > sw->switch_index) {
 436                                 sw = &switches[index];
 437                                 break;
 438                         }
 439                 }
 440         }
 441
 442         *route = get_route(sw->route_hi, sw->route_lo);
 443
 444 err_free:
 445         kfree(switches);
 446         return ret;
 447 }
 448
 449 static void icm_fr_save_devices(struct tb *tb)
 450 {
 451         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
 452 }
 453
 454 static int
 455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
 456                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
 457 {
 458         struct icm_fr_pkg_driver_ready_response reply;
 459         struct icm_pkg_driver_ready request = {
 460                 .hdr.code = ICM_DRIVER_READY,
 461         };
 462         int ret;
 463
 464         memset(&reply, 0, sizeof(reply));
 465         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 466                           1, ICM_TIMEOUT);
 467         if (ret)
 468                 return ret;
 469
 470         if (security_level)
 471                 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
 472
 473         return 0;
 474 }
 475
 476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
 477 {
 478         struct icm_fr_pkg_approve_device request;
 479         struct icm_fr_pkg_approve_device reply;
 480         int ret;
 481
 482         memset(&request, 0, sizeof(request));
 483         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 484         request.hdr.code = ICM_APPROVE_DEVICE;
 485         request.connection_id = sw->connection_id;
 486         request.connection_key = sw->connection_key;
 487
 488         memset(&reply, 0, sizeof(reply));
 489         /* Use larger timeout as establishing tunnels can take some time */
 490         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 491                           1, ICM_APPROVE_TIMEOUT);
 492         if (ret)
 493                 return ret;
 494
 495         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 496                 tb_warn(tb, "PCIe tunnel creation failed\n");
 497                 return -EIO;
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
 504 {
 505         struct icm_fr_pkg_add_device_key request;
 506         struct icm_fr_pkg_add_device_key_response reply;
 507         int ret;
 508
 509         memset(&request, 0, sizeof(request));
 510         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 511         request.hdr.code = ICM_ADD_DEVICE_KEY;
 512         request.connection_id = sw->connection_id;
 513         request.connection_key = sw->connection_key;
 514         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
 515
 516         memset(&reply, 0, sizeof(reply));
 517         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 518                           1, ICM_TIMEOUT);
 519         if (ret)
 520                 return ret;
 521
 522         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 523                 tb_warn(tb, "Adding key to switch failed\n");
 524                 return -EIO;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
 531                                        const u8 *challenge, u8 *response)
 532 {
 533         struct icm_fr_pkg_challenge_device request;
 534         struct icm_fr_pkg_challenge_device_response reply;
 535         int ret;
 536
 537         memset(&request, 0, sizeof(request));
 538         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 539         request.hdr.code = ICM_CHALLENGE_DEVICE;
 540         request.connection_id = sw->connection_id;
 541         request.connection_key = sw->connection_key;
 542         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
 543
 544         memset(&reply, 0, sizeof(reply));
 545         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 546                           1, ICM_TIMEOUT);
 547         if (ret)
 548                 return ret;
 549
 550         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 551                 return -EKEYREJECTED;
 552         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
 553                 return -ENOKEY;
 554
 555         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
 556
 557         return 0;
 558 }
 559
 560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 561                                         int transmit_path, int transmit_ring,
 562                                         int receive_path, int receive_ring)
 563 {
 564         struct icm_fr_pkg_approve_xdomain_response reply;
 565         struct icm_fr_pkg_approve_xdomain request;
 566         int ret;
 567
 568         memset(&request, 0, sizeof(request));
 569         request.hdr.code = ICM_APPROVE_XDOMAIN;
 570         request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
 571         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
 572
 573         request.transmit_path = transmit_path;
 574         request.transmit_ring = transmit_ring;
 575         request.receive_path = receive_path;
 576         request.receive_ring = receive_ring;
 577
 578         memset(&reply, 0, sizeof(reply));
 579         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 580                           1, ICM_TIMEOUT);
 581         if (ret)
 582                 return ret;
 583
 584         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 585                 return -EIO;
 586
 587         return 0;
 588 }
 589
 590 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 591                                            int transmit_path, int transmit_ring,
 592                                            int receive_path, int receive_ring)
 593 {
 594         u8 phy_port;
 595         u8 cmd;
 596
 597         phy_port = tb_phy_port_from_link(xd->link);
 598         if (phy_port == 0)
 599                 cmd = NHI_MAILBOX_DISCONNECT_PA;
 600         else
 601                 cmd = NHI_MAILBOX_DISCONNECT_PB;
 602
 603         nhi_mailbox_cmd(tb->nhi, cmd, 1);
 604         usleep_range(10, 50);
 605         nhi_mailbox_cmd(tb->nhi, cmd, 2);
 606         return 0;
 607 }
 608
 609 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
 610                                       const uuid_t *uuid)
 611 {
 612         struct tb *tb = parent_sw->tb;
 613         struct tb_switch *sw;
 614
 615         sw = tb_switch_alloc(tb, &parent_sw->dev, route);
 616         if (IS_ERR(sw)) {
 617                 tb_warn(tb, "failed to allocate switch at %llx\n", route);
 618                 return sw;
 619         }
 620
 621         sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
 622         if (!sw->uuid) {
 623                 tb_switch_put(sw);
 624                 return ERR_PTR(-ENOMEM);
 625         }
 626
 627         init_completion(&sw->rpm_complete);
 628         return sw;
 629 }
 630
 631 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
 632 {
 633         u64 route = tb_route(sw);
 634         int ret;
 635
 636         /* Link the two switches now */
 637         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 638         tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
 639
 640         ret = tb_switch_add(sw);
 641         if (ret)
 642                 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 643
 644         return ret;
 645 }
 646
 647 static void update_switch(struct tb_switch *parent_sw, struct tb_switch *sw,
 648                           u64 route, u8 connection_id, u8 connection_key,
 649                           u8 link, u8 depth, bool boot)
 650 {
 651         /* Disconnect from parent */
 652         tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 653         /* Re-connect via updated port*/
 654         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 655
 656         /* Update with the new addressing information */
 657         sw->config.route_hi = upper_32_bits(route);
 658         sw->config.route_lo = lower_32_bits(route);
 659         sw->connection_id = connection_id;
 660         sw->connection_key = connection_key;
 661         sw->link = link;
 662         sw->depth = depth;
 663         sw->boot = boot;
 664
 665         /* This switch still exists */
 666         sw->is_unplugged = false;
 667
 668         /* Runtime resume is now complete */
 669         complete(&sw->rpm_complete);
 670 }
 671
 672 static void remove_switch(struct tb_switch *sw)
 673 {
 674         struct tb_switch *parent_sw;
 675
 676         parent_sw = tb_to_switch(sw->dev.parent);
 677         tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 678         tb_switch_remove(sw);
 679 }
 680
 681 static void add_xdomain(struct tb_switch *sw, u64 route,
 682                         const uuid_t *local_uuid, const uuid_t *remote_uuid,
 683                         u8 link, u8 depth)
 684 {
 685         struct tb_xdomain *xd;
 686
 687         pm_runtime_get_sync(&sw->dev);
 688
 689         xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
 690         if (!xd)
 691                 goto out;
 692
 693         xd->link = link;
 694         xd->depth = depth;
 695
 696         tb_port_at(route, sw)->xdomain = xd;
 697
 698         tb_xdomain_add(xd);
 699
 700 out:
 701         pm_runtime_mark_last_busy(&sw->dev);
 702         pm_runtime_put_autosuspend(&sw->dev);
 703 }
 704
 705 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
 706 {
 707         xd->link = link;
 708         xd->route = route;
 709         xd->is_unplugged = false;
 710 }
 711
 712 static void remove_xdomain(struct tb_xdomain *xd)
 713 {
 714         struct tb_switch *sw;
 715
 716         sw = tb_to_switch(xd->dev.parent);
 717         tb_port_at(xd->route, sw)->xdomain = NULL;
 718         tb_xdomain_remove(xd);
 719 }
 720
 721 static void
 722 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 723 {
 724         const struct icm_fr_event_device_connected *pkg =
 725                 (const struct icm_fr_event_device_connected *)hdr;
 726         enum tb_security_level security_level;
 727         struct tb_switch *sw, *parent_sw;
 728         bool boot, dual_lane, speed_gen3;
 729         struct icm *icm = tb_priv(tb);
 730         bool authorized = false;
 731         struct tb_xdomain *xd;
 732         u8 link, depth;
 733         u64 route;
 734         int ret;
 735
 736         icm_postpone_rescan(tb);
 737
 738         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 739         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 740                 ICM_LINK_INFO_DEPTH_SHIFT;
 741         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
 742         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
 743                          ICM_FLAGS_SLEVEL_SHIFT;
 744         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
 745         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
 746         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
 747
 748         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
 749                 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
 750                         link, depth);
 751                 return;
 752         }
 753
 754         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
 755         if (sw) {
 756                 u8 phy_port, sw_phy_port;
 757
 758                 parent_sw = tb_to_switch(sw->dev.parent);
 759                 sw_phy_port = tb_phy_port_from_link(sw->link);
 760                 phy_port = tb_phy_port_from_link(link);
 761
 762                 /*
 763                  * On resume ICM will send us connected events for the
 764                  * devices that still are present. However, that
 765                  * information might have changed for example by the
 766                  * fact that a switch on a dual-link connection might
 767                  * have been enumerated using the other link now. Make
 768                  * sure our book keeping matches that.
 769                  */
 770                 if (sw->depth == depth && sw_phy_port == phy_port &&
 771                     !!sw->authorized == authorized) {
 772                         /*
 773                          * It was enumerated through another link so update
 774                          * route string accordingly.
 775                          */
 776                         if (sw->link != link) {
 777                                 ret = icm->get_route(tb, link, depth, &route);
 778                                 if (ret) {
 779                                         tb_err(tb, "failed to update route string for switch at %u.%u\n",
 780                                                link, depth);
 781                                         tb_switch_put(sw);
 782                                         return;
 783                                 }
 784                         } else {
 785                                 route = tb_route(sw);
 786                         }
 787
 788                         update_switch(parent_sw, sw, route, pkg->connection_id,
 789                                       pkg->connection_key, link, depth, boot);
 790                         tb_switch_put(sw);
 791                         return;
 792                 }
 793
 794                 /*
 795                  * User connected the same switch to another physical
 796                  * port or to another part of the topology. Remove the
 797                  * existing switch now before adding the new one.
 798                  */
 799                 remove_switch(sw);
 800                 tb_switch_put(sw);
 801         }
 802
 803         /*
 804          * If the switch was not found by UUID, look for a switch on
 805          * same physical port (taking possible link aggregation into
 806          * account) and depth. If we found one it is definitely a stale
 807          * one so remove it first.
 808          */
 809         sw = tb_switch_find_by_link_depth(tb, link, depth);
 810         if (!sw) {
 811                 u8 dual_link;
 812
 813                 dual_link = dual_link_from_link(link);
 814                 if (dual_link)
 815                         sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
 816         }
 817         if (sw) {
 818                 remove_switch(sw);
 819                 tb_switch_put(sw);
 820         }
 821
 822         /* Remove existing XDomain connection if found */
 823         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 824         if (xd) {
 825                 remove_xdomain(xd);
 826                 tb_xdomain_put(xd);
 827         }
 828
 829         parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
 830         if (!parent_sw) {
 831                 tb_err(tb, "failed to find parent switch for %u.%u\n",
 832                        link, depth);
 833                 return;
 834         }
 835
 836         ret = icm->get_route(tb, link, depth, &route);
 837         if (ret) {
 838                 tb_err(tb, "failed to find route string for switch at %u.%u\n",
 839                        link, depth);
 840                 tb_switch_put(parent_sw);
 841                 return;
 842         }
 843
 844         pm_runtime_get_sync(&parent_sw->dev);
 845
 846         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
 847         if (!IS_ERR(sw)) {
 848                 sw->connection_id = pkg->connection_id;
 849                 sw->connection_key = pkg->connection_key;
 850                 sw->link = link;
 851                 sw->depth = depth;
 852                 sw->authorized = authorized;
 853                 sw->security_level = security_level;
 854                 sw->boot = boot;
 855                 sw->link_speed = speed_gen3 ? 20 : 10;
 856                 sw->link_width = dual_lane ? 2 : 1;
 857                 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
 858
 859                 if (add_switch(parent_sw, sw))
 860                         tb_switch_put(sw);
 861         }
 862
 863         pm_runtime_mark_last_busy(&parent_sw->dev);
 864         pm_runtime_put_autosuspend(&parent_sw->dev);
 865
 866         tb_switch_put(parent_sw);
 867 }
 868
 869 static void
 870 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 871 {
 872         const struct icm_fr_event_device_disconnected *pkg =
 873                 (const struct icm_fr_event_device_disconnected *)hdr;
 874         struct tb_switch *sw;
 875         u8 link, depth;
 876
 877         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 878         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 879                 ICM_LINK_INFO_DEPTH_SHIFT;
 880
 881         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 882                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 883                 return;
 884         }
 885
 886         sw = tb_switch_find_by_link_depth(tb, link, depth);
 887         if (!sw) {
 888                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 889                         depth);
 890                 return;
 891         }
 892
 893         pm_runtime_get_sync(sw->dev.parent);
 894
 895         remove_switch(sw);
 896
 897         pm_runtime_mark_last_busy(sw->dev.parent);
 898         pm_runtime_put_autosuspend(sw->dev.parent);
 899
 900         tb_switch_put(sw);
 901 }
 902
 903 static void
 904 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 905 {
 906         const struct icm_fr_event_xdomain_connected *pkg =
 907                 (const struct icm_fr_event_xdomain_connected *)hdr;
 908         struct tb_xdomain *xd;
 909         struct tb_switch *sw;
 910         u8 link, depth;
 911         u64 route;
 912
 913         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 914         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 915                 ICM_LINK_INFO_DEPTH_SHIFT;
 916
 917         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 918                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 919                 return;
 920         }
 921
 922         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
 923
 924         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
 925         if (xd) {
 926                 u8 xd_phy_port, phy_port;
 927
 928                 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
 929                 phy_port = phy_port_from_route(route, depth);
 930
 931                 if (xd->depth == depth && xd_phy_port == phy_port) {
 932                         update_xdomain(xd, route, link);
 933                         tb_xdomain_put(xd);
 934                         return;
 935                 }
 936
 937                 /*
 938                  * If we find an existing XDomain connection remove it
 939                  * now. We need to go through login handshake and
 940                  * everything anyway to be able to re-establish the
 941                  * connection.
 942                  */
 943                 remove_xdomain(xd);
 944                 tb_xdomain_put(xd);
 945         }
 946
 947         /*
 948          * Look if there already exists an XDomain in the same place
 949          * than the new one and in that case remove it because it is
 950          * most likely another host that got disconnected.
 951          */
 952         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 953         if (!xd) {
 954                 u8 dual_link;
 955
 956                 dual_link = dual_link_from_link(link);
 957                 if (dual_link)
 958                         xd = tb_xdomain_find_by_link_depth(tb, dual_link,
 959                                                            depth);
 960         }
 961         if (xd) {
 962                 remove_xdomain(xd);
 963                 tb_xdomain_put(xd);
 964         }
 965
 966         /*
 967          * If the user disconnected a switch during suspend and
 968          * connected another host to the same port, remove the switch
 969          * first.
 970          */
 971         sw = tb_switch_find_by_route(tb, route);
 972         if (sw) {
 973                 remove_switch(sw);
 974                 tb_switch_put(sw);
 975         }
 976
 977         sw = tb_switch_find_by_link_depth(tb, link, depth);
 978         if (!sw) {
 979                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 980                         depth);
 981                 return;
 982         }
 983
 984         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
 985                     depth);
 986         tb_switch_put(sw);
 987 }
 988
 989 static void
 990 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 991 {
 992         const struct icm_fr_event_xdomain_disconnected *pkg =
 993                 (const struct icm_fr_event_xdomain_disconnected *)hdr;
 994         struct tb_xdomain *xd;
 995
 996         /*
 997          * If the connection is through one or multiple devices, the
 998          * XDomain device is removed along with them so it is fine if we
 999          * cannot find it here.
1000          */
1001         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1002         if (xd) {
1003                 remove_xdomain(xd);
1004                 tb_xdomain_put(xd);
1005         }
1006 }
1007
1008 static int icm_tr_cio_reset(struct tb *tb)
1009 {
1010         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1011 }
1012
1013 static int
1014 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1015                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1016 {
1017         struct icm_tr_pkg_driver_ready_response reply;
1018         struct icm_pkg_driver_ready request = {
1019                 .hdr.code = ICM_DRIVER_READY,
1020         };
1021         int ret;
1022
1023         memset(&reply, 0, sizeof(reply));
1024         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1025                           1, 20000);
1026         if (ret)
1027                 return ret;
1028
1029         if (security_level)
1030                 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1031         if (proto_version)
1032                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1033                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1034         if (nboot_acl)
1035                 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1036                                 ICM_TR_INFO_BOOT_ACL_SHIFT;
1037         if (rpm)
1038                 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1039
1040         return 0;
1041 }
1042
1043 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1044 {
1045         struct icm_tr_pkg_approve_device request;
1046         struct icm_tr_pkg_approve_device reply;
1047         int ret;
1048
1049         memset(&request, 0, sizeof(request));
1050         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1051         request.hdr.code = ICM_APPROVE_DEVICE;
1052         request.route_lo = sw->config.route_lo;
1053         request.route_hi = sw->config.route_hi;
1054         request.connection_id = sw->connection_id;
1055
1056         memset(&reply, 0, sizeof(reply));
1057         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1058                           1, ICM_APPROVE_TIMEOUT);
1059         if (ret)
1060                 return ret;
1061
1062         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1063                 tb_warn(tb, "PCIe tunnel creation failed\n");
1064                 return -EIO;
1065         }
1066
1067         return 0;
1068 }
1069
1070 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1071 {
1072         struct icm_tr_pkg_add_device_key_response reply;
1073         struct icm_tr_pkg_add_device_key request;
1074         int ret;
1075
1076         memset(&request, 0, sizeof(request));
1077         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1078         request.hdr.code = ICM_ADD_DEVICE_KEY;
1079         request.route_lo = sw->config.route_lo;
1080         request.route_hi = sw->config.route_hi;
1081         request.connection_id = sw->connection_id;
1082         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1083
1084         memset(&reply, 0, sizeof(reply));
1085         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1086                           1, ICM_TIMEOUT);
1087         if (ret)
1088                 return ret;
1089
1090         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1091                 tb_warn(tb, "Adding key to switch failed\n");
1092                 return -EIO;
1093         }
1094
1095         return 0;
1096 }
1097
1098 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1099                                        const u8 *challenge, u8 *response)
1100 {
1101         struct icm_tr_pkg_challenge_device_response reply;
1102         struct icm_tr_pkg_challenge_device request;
1103         int ret;
1104
1105         memset(&request, 0, sizeof(request));
1106         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1107         request.hdr.code = ICM_CHALLENGE_DEVICE;
1108         request.route_lo = sw->config.route_lo;
1109         request.route_hi = sw->config.route_hi;
1110         request.connection_id = sw->connection_id;
1111         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1112
1113         memset(&reply, 0, sizeof(reply));
1114         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1115                           1, ICM_TIMEOUT);
1116         if (ret)
1117                 return ret;
1118
1119         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1120                 return -EKEYREJECTED;
1121         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1122                 return -ENOKEY;
1123
1124         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1125
1126         return 0;
1127 }
1128
1129 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1130                                         int transmit_path, int transmit_ring,
1131                                         int receive_path, int receive_ring)
1132 {
1133         struct icm_tr_pkg_approve_xdomain_response reply;
1134         struct icm_tr_pkg_approve_xdomain request;
1135         int ret;
1136
1137         memset(&request, 0, sizeof(request));
1138         request.hdr.code = ICM_APPROVE_XDOMAIN;
1139         request.route_hi = upper_32_bits(xd->route);
1140         request.route_lo = lower_32_bits(xd->route);
1141         request.transmit_path = transmit_path;
1142         request.transmit_ring = transmit_ring;
1143         request.receive_path = receive_path;
1144         request.receive_ring = receive_ring;
1145         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1146
1147         memset(&reply, 0, sizeof(reply));
1148         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1149                           1, ICM_TIMEOUT);
1150         if (ret)
1151                 return ret;
1152
1153         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1154                 return -EIO;
1155
1156         return 0;
1157 }
1158
1159 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1160                                     int stage)
1161 {
1162         struct icm_tr_pkg_disconnect_xdomain_response reply;
1163         struct icm_tr_pkg_disconnect_xdomain request;
1164         int ret;
1165
1166         memset(&request, 0, sizeof(request));
1167         request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1168         request.stage = stage;
1169         request.route_hi = upper_32_bits(xd->route);
1170         request.route_lo = lower_32_bits(xd->route);
1171         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1172
1173         memset(&reply, 0, sizeof(reply));
1174         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1175                           1, ICM_TIMEOUT);
1176         if (ret)
1177                 return ret;
1178
1179         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1180                 return -EIO;
1181
1182         return 0;
1183 }
1184
1185 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1186                                            int transmit_path, int transmit_ring,
1187                                            int receive_path, int receive_ring)
1188 {
1189         int ret;
1190
1191         ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1192         if (ret)
1193                 return ret;
1194
1195         usleep_range(10, 50);
1196         return icm_tr_xdomain_tear_down(tb, xd, 2);
1197 }
1198
1199 static void
1200 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1201                           bool force_rtd3)
1202 {
1203         const struct icm_tr_event_device_connected *pkg =
1204                 (const struct icm_tr_event_device_connected *)hdr;
1205         bool authorized, boot, dual_lane, speed_gen3;
1206         enum tb_security_level security_level;
1207         struct tb_switch *sw, *parent_sw;
1208         struct tb_xdomain *xd;
1209         u64 route;
1210
1211         icm_postpone_rescan(tb);
1212
1213         /*
1214          * Currently we don't use the QoS information coming with the
1215          * device connected message so simply just ignore that extra
1216          * packet for now.
1217          */
1218         if (pkg->hdr.packet_id)
1219                 return;
1220
1221         route = get_route(pkg->route_hi, pkg->route_lo);
1222         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1223         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1224                          ICM_FLAGS_SLEVEL_SHIFT;
1225         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1226         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1227         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1228
1229         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1230                 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1231                         route);
1232                 return;
1233         }
1234
1235         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1236         if (sw) {
1237                 /* Update the switch if it is still in the same place */
1238                 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1239                         parent_sw = tb_to_switch(sw->dev.parent);
1240                         update_switch(parent_sw, sw, route, pkg->connection_id,
1241                                       0, 0, 0, boot);
1242                         tb_switch_put(sw);
1243                         return;
1244                 }
1245
1246                 remove_switch(sw);
1247                 tb_switch_put(sw);
1248         }
1249
1250         /* Another switch with the same address */
1251         sw = tb_switch_find_by_route(tb, route);
1252         if (sw) {
1253                 remove_switch(sw);
1254                 tb_switch_put(sw);
1255         }
1256
1257         /* XDomain connection with the same address */
1258         xd = tb_xdomain_find_by_route(tb, route);
1259         if (xd) {
1260                 remove_xdomain(xd);
1261                 tb_xdomain_put(xd);
1262         }
1263
1264         parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1265         if (!parent_sw) {
1266                 tb_err(tb, "failed to find parent switch for %llx\n", route);
1267                 return;
1268         }
1269
1270         pm_runtime_get_sync(&parent_sw->dev);
1271
1272         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1273         if (!IS_ERR(sw)) {
1274                 sw->connection_id = pkg->connection_id;
1275                 sw->authorized = authorized;
1276                 sw->security_level = security_level;
1277                 sw->boot = boot;
1278                 sw->link_speed = speed_gen3 ? 20 : 10;
1279                 sw->link_width = dual_lane ? 2 : 1;
1280                 sw->rpm = force_rtd3;
1281                 if (!sw->rpm)
1282                         sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1283                                                     sizeof(pkg->ep_name));
1284
1285                 if (add_switch(parent_sw, sw))
1286                         tb_switch_put(sw);
1287         }
1288
1289         pm_runtime_mark_last_busy(&parent_sw->dev);
1290         pm_runtime_put_autosuspend(&parent_sw->dev);
1291
1292         tb_switch_put(parent_sw);
1293 }
1294
1295 static void
1296 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1297 {
1298         __icm_tr_device_connected(tb, hdr, false);
1299 }
1300
1301 static void
1302 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1303 {
1304         const struct icm_tr_event_device_disconnected *pkg =
1305                 (const struct icm_tr_event_device_disconnected *)hdr;
1306         struct tb_switch *sw;
1307         u64 route;
1308
1309         route = get_route(pkg->route_hi, pkg->route_lo);
1310
1311         sw = tb_switch_find_by_route(tb, route);
1312         if (!sw) {
1313                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1314                 return;
1315         }
1316         pm_runtime_get_sync(sw->dev.parent);
1317
1318         remove_switch(sw);
1319
1320         pm_runtime_mark_last_busy(sw->dev.parent);
1321         pm_runtime_put_autosuspend(sw->dev.parent);
1322
1323         tb_switch_put(sw);
1324 }
1325
1326 static void
1327 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1328 {
1329         const struct icm_tr_event_xdomain_connected *pkg =
1330                 (const struct icm_tr_event_xdomain_connected *)hdr;
1331         struct tb_xdomain *xd;
1332         struct tb_switch *sw;
1333         u64 route;
1334
1335         if (!tb->root_switch)
1336                 return;
1337
1338         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1339
1340         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1341         if (xd) {
1342                 if (xd->route == route) {
1343                         update_xdomain(xd, route, 0);
1344                         tb_xdomain_put(xd);
1345                         return;
1346                 }
1347
1348                 remove_xdomain(xd);
1349                 tb_xdomain_put(xd);
1350         }
1351
1352         /* An existing xdomain with the same address */
1353         xd = tb_xdomain_find_by_route(tb, route);
1354         if (xd) {
1355                 remove_xdomain(xd);
1356                 tb_xdomain_put(xd);
1357         }
1358
1359         /*
1360          * If the user disconnected a switch during suspend and
1361          * connected another host to the same port, remove the switch
1362          * first.
1363          */
1364         sw = tb_switch_find_by_route(tb, route);
1365         if (sw) {
1366                 remove_switch(sw);
1367                 tb_switch_put(sw);
1368         }
1369
1370         sw = tb_switch_find_by_route(tb, get_parent_route(route));
1371         if (!sw) {
1372                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1373                 return;
1374         }
1375
1376         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1377         tb_switch_put(sw);
1378 }
1379
1380 static void
1381 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1382 {
1383         const struct icm_tr_event_xdomain_disconnected *pkg =
1384                 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1385         struct tb_xdomain *xd;
1386         u64 route;
1387
1388         route = get_route(pkg->route_hi, pkg->route_lo);
1389
1390         xd = tb_xdomain_find_by_route(tb, route);
1391         if (xd) {
1392                 remove_xdomain(xd);
1393                 tb_xdomain_put(xd);
1394         }
1395 }
1396
1397 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1398 {
1399         struct pci_dev *parent;
1400
1401         parent = pci_upstream_bridge(pdev);
1402         while (parent) {
1403                 if (!pci_is_pcie(parent))
1404                         return NULL;
1405                 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1406                         break;
1407                 parent = pci_upstream_bridge(parent);
1408         }
1409
1410         if (!parent)
1411                 return NULL;
1412
1413         switch (parent->device) {
1414         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1415         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1416         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1417         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1418         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1419         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1420         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1421                 return parent;
1422         }
1423
1424         return NULL;
1425 }
1426
1427 static bool icm_ar_is_supported(struct tb *tb)
1428 {
1429         struct pci_dev *upstream_port;
1430         struct icm *icm = tb_priv(tb);
1431
1432         /*
1433          * Starting from Alpine Ridge we can use ICM on Apple machines
1434          * as well. We just need to reset and re-enable it first.
1435          * However, only start it if explicitly asked by the user.
1436          */
1437         if (icm_firmware_running(tb->nhi))
1438                 return true;
1439         if (!start_icm)
1440                 return false;
1441
1442         /*
1443          * Find the upstream PCIe port in case we need to do reset
1444          * through its vendor specific registers.
1445          */
1446         upstream_port = get_upstream_port(tb->nhi->pdev);
1447         if (upstream_port) {
1448                 int cap;
1449
1450                 cap = pci_find_ext_capability(upstream_port,
1451                                               PCI_EXT_CAP_ID_VNDR);
1452                 if (cap > 0) {
1453                         icm->upstream_port = upstream_port;
1454                         icm->vnd_cap = cap;
1455
1456                         return true;
1457                 }
1458         }
1459
1460         return false;
1461 }
1462
1463 static int icm_ar_cio_reset(struct tb *tb)
1464 {
1465         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1466 }
1467
1468 static int icm_ar_get_mode(struct tb *tb)
1469 {
1470         struct tb_nhi *nhi = tb->nhi;
1471         int retries = 60;
1472         u32 val;
1473
1474         do {
1475                 val = ioread32(nhi->iobase + REG_FW_STS);
1476                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1477                         break;
1478                 msleep(50);
1479         } while (--retries);
1480
1481         if (!retries) {
1482                 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1483                 return -ENODEV;
1484         }
1485
1486         return nhi_mailbox_mode(nhi);
1487 }
1488
1489 static int
1490 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1491                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1492 {
1493         struct icm_ar_pkg_driver_ready_response reply;
1494         struct icm_pkg_driver_ready request = {
1495                 .hdr.code = ICM_DRIVER_READY,
1496         };
1497         int ret;
1498
1499         memset(&reply, 0, sizeof(reply));
1500         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1501                           1, ICM_TIMEOUT);
1502         if (ret)
1503                 return ret;
1504
1505         if (security_level)
1506                 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1507         if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1508                 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1509                                 ICM_AR_INFO_BOOT_ACL_SHIFT;
1510         if (rpm)
1511                 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1512
1513         return 0;
1514 }
1515
1516 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1517 {
1518         struct icm_ar_pkg_get_route_response reply;
1519         struct icm_ar_pkg_get_route request = {
1520                 .hdr = { .code = ICM_GET_ROUTE },
1521                 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1522         };
1523         int ret;
1524
1525         memset(&reply, 0, sizeof(reply));
1526         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1527                           1, ICM_TIMEOUT);
1528         if (ret)
1529                 return ret;
1530
1531         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1532                 return -EIO;
1533
1534         *route = get_route(reply.route_hi, reply.route_lo);
1535         return 0;
1536 }
1537
1538 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1539 {
1540         struct icm_ar_pkg_preboot_acl_response reply;
1541         struct icm_ar_pkg_preboot_acl request = {
1542                 .hdr = { .code = ICM_PREBOOT_ACL },
1543         };
1544         int ret, i;
1545
1546         memset(&reply, 0, sizeof(reply));
1547         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1548                           1, ICM_TIMEOUT);
1549         if (ret)
1550                 return ret;
1551
1552         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1553                 return -EIO;
1554
1555         for (i = 0; i < nuuids; i++) {
1556                 u32 *uuid = (u32 *)&uuids[i];
1557
1558                 uuid[0] = reply.acl[i].uuid_lo;
1559                 uuid[1] = reply.acl[i].uuid_hi;
1560
1561                 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1562                         /* Map empty entries to null UUID */
1563                         uuid[0] = 0;
1564                         uuid[1] = 0;
1565                 } else if (uuid[0] != 0 || uuid[1] != 0) {
1566                         /* Upper two DWs are always one's */
1567                         uuid[2] = 0xffffffff;
1568                         uuid[3] = 0xffffffff;
1569                 }
1570         }
1571
1572         return ret;
1573 }
1574
1575 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1576                                size_t nuuids)
1577 {
1578         struct icm_ar_pkg_preboot_acl_response reply;
1579         struct icm_ar_pkg_preboot_acl request = {
1580                 .hdr = {
1581                         .code = ICM_PREBOOT_ACL,
1582                         .flags = ICM_FLAGS_WRITE,
1583                 },
1584         };
1585         int ret, i;
1586
1587         for (i = 0; i < nuuids; i++) {
1588                 const u32 *uuid = (const u32 *)&uuids[i];
1589
1590                 if (uuid_is_null(&uuids[i])) {
1591                         /*
1592                          * Map null UUID to the empty (all one) entries
1593                          * for ICM.
1594                          */
1595                         request.acl[i].uuid_lo = 0xffffffff;
1596                         request.acl[i].uuid_hi = 0xffffffff;
1597                 } else {
1598                         /* Two high DWs need to be set to all one */
1599                         if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1600                                 return -EINVAL;
1601
1602                         request.acl[i].uuid_lo = uuid[0];
1603                         request.acl[i].uuid_hi = uuid[1];
1604                 }
1605         }
1606
1607         memset(&reply, 0, sizeof(reply));
1608         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1609                           1, ICM_TIMEOUT);
1610         if (ret)
1611                 return ret;
1612
1613         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1614                 return -EIO;
1615
1616         return 0;
1617 }
1618
1619 static int
1620 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1621                      u8 *proto_version, size_t *nboot_acl, bool *rpm)
1622 {
1623         struct icm_tr_pkg_driver_ready_response reply;
1624         struct icm_pkg_driver_ready request = {
1625                 .hdr.code = ICM_DRIVER_READY,
1626         };
1627         int ret;
1628
1629         memset(&reply, 0, sizeof(reply));
1630         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1631                           1, 20000);
1632         if (ret)
1633                 return ret;
1634
1635         if (proto_version)
1636                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1637                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1638
1639         /* Ice Lake always supports RTD3 */
1640         if (rpm)
1641                 *rpm = true;
1642
1643         return 0;
1644 }
1645
1646 static void icm_icl_set_uuid(struct tb *tb)
1647 {
1648         struct tb_nhi *nhi = tb->nhi;
1649         u32 uuid[4];
1650
1651         pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1652         pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1653         uuid[2] = 0xffffffff;
1654         uuid[3] = 0xffffffff;
1655
1656         tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1657 }
1658
1659 static void
1660 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1661 {
1662         __icm_tr_device_connected(tb, hdr, true);
1663 }
1664
1665 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1666 {
1667         const struct icm_icl_event_rtd3_veto *pkg =
1668                 (const struct icm_icl_event_rtd3_veto *)hdr;
1669
1670         tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1671
1672         if (pkg->veto_reason)
1673                 icm_veto_begin(tb);
1674         else
1675                 icm_veto_end(tb);
1676 }
1677
1678 static bool icm_tgl_is_supported(struct tb *tb)
1679 {
1680         unsigned long end = jiffies + msecs_to_jiffies(10);
1681
1682         do {
1683                 u32 val;
1684
1685                 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1686                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1687                         return true;
1688                 usleep_range(100, 500);
1689         } while (time_before(jiffies, end));
1690
1691         return false;
1692 }
1693
1694 static void icm_handle_notification(struct work_struct *work)
1695 {
1696         struct icm_notification *n = container_of(work, typeof(*n), work);
1697         struct tb *tb = n->tb;
1698         struct icm *icm = tb_priv(tb);
1699
1700         mutex_lock(&tb->lock);
1701
1702         /*
1703          * When the domain is stopped we flush its workqueue but before
1704          * that the root switch is removed. In that case we should treat
1705          * the queued events as being canceled.
1706          */
1707         if (tb->root_switch) {
1708                 switch (n->pkg->code) {
1709                 case ICM_EVENT_DEVICE_CONNECTED:
1710                         icm->device_connected(tb, n->pkg);
1711                         break;
1712                 case ICM_EVENT_DEVICE_DISCONNECTED:
1713                         icm->device_disconnected(tb, n->pkg);
1714                         break;
1715                 case ICM_EVENT_XDOMAIN_CONNECTED:
1716                         if (tb_is_xdomain_enabled())
1717                                 icm->xdomain_connected(tb, n->pkg);
1718                         break;
1719                 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1720                         if (tb_is_xdomain_enabled())
1721                                 icm->xdomain_disconnected(tb, n->pkg);
1722                         break;
1723                 case ICM_EVENT_RTD3_VETO:
1724                         icm->rtd3_veto(tb, n->pkg);
1725                         break;
1726                 }
1727         }
1728
1729         mutex_unlock(&tb->lock);
1730
1731         kfree(n->pkg);
1732         kfree(n);
1733 }
1734
1735 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1736                              const void *buf, size_t size)
1737 {
1738         struct icm_notification *n;
1739
1740         n = kmalloc(sizeof(*n), GFP_KERNEL);
1741         if (!n)
1742                 return;
1743
1744         n->pkg = kmemdup(buf, size, GFP_KERNEL);
1745         if (!n->pkg) {
1746                 kfree(n);
1747                 return;
1748         }
1749
1750         INIT_WORK(&n->work, icm_handle_notification);
1751         n->tb = tb;
1752
1753         queue_work(tb->wq, &n->work);
1754 }
1755
1756 static int
1757 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1758                    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1759 {
1760         struct icm *icm = tb_priv(tb);
1761         unsigned int retries = 50;
1762         int ret;
1763
1764         ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1765                                 rpm);
1766         if (ret) {
1767                 tb_err(tb, "failed to send driver ready to ICM\n");
1768                 return ret;
1769         }
1770
1771         /*
1772          * Hold on here until the switch config space is accessible so
1773          * that we can read root switch config successfully.
1774          */
1775         do {
1776                 struct tb_cfg_result res;
1777                 u32 tmp;
1778
1779                 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1780                                       0, 1, 100);
1781                 if (!res.err)
1782                         return 0;
1783
1784                 msleep(50);
1785         } while (--retries);
1786
1787         tb_err(tb, "failed to read root switch config space, giving up\n");
1788         return -ETIMEDOUT;
1789 }
1790
1791 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1792 {
1793         struct icm *icm = tb_priv(tb);
1794         u32 val;
1795
1796         if (!icm->upstream_port)
1797                 return -ENODEV;
1798
1799         /* Put ARC to wait for CIO reset event to happen */
1800         val = ioread32(nhi->iobase + REG_FW_STS);
1801         val |= REG_FW_STS_CIO_RESET_REQ;
1802         iowrite32(val, nhi->iobase + REG_FW_STS);
1803
1804         /* Re-start ARC */
1805         val = ioread32(nhi->iobase + REG_FW_STS);
1806         val |= REG_FW_STS_ICM_EN_INVERT;
1807         val |= REG_FW_STS_ICM_EN_CPU;
1808         iowrite32(val, nhi->iobase + REG_FW_STS);
1809
1810         /* Trigger CIO reset now */
1811         return icm->cio_reset(tb);
1812 }
1813
1814 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1815 {
1816         unsigned int retries = 10;
1817         int ret;
1818         u32 val;
1819
1820         /* Check if the ICM firmware is already running */
1821         if (icm_firmware_running(nhi))
1822                 return 0;
1823
1824         dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1825
1826         ret = icm_firmware_reset(tb, nhi);
1827         if (ret)
1828                 return ret;
1829
1830         /* Wait until the ICM firmware tells us it is up and running */
1831         do {
1832                 /* Check that the ICM firmware is running */
1833                 val = ioread32(nhi->iobase + REG_FW_STS);
1834                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1835                         return 0;
1836
1837                 msleep(300);
1838         } while (--retries);
1839
1840         return -ETIMEDOUT;
1841 }
1842
1843 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1844 {
1845         struct icm *icm = tb_priv(tb);
1846         u32 state0, state1;
1847         int port0, port1;
1848         u32 val0, val1;
1849         int ret;
1850
1851         if (!icm->upstream_port)
1852                 return 0;
1853
1854         if (phy_port) {
1855                 port0 = 3;
1856                 port1 = 4;
1857         } else {
1858                 port0 = 1;
1859                 port1 = 2;
1860         }
1861
1862         /*
1863          * Read link status of both null ports belonging to a single
1864          * physical port.
1865          */
1866         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1867         if (ret)
1868                 return ret;
1869         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1870         if (ret)
1871                 return ret;
1872
1873         state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1874         state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1875         state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1876         state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1877
1878         /* If they are both up we need to reset them now */
1879         if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1880                 return 0;
1881
1882         val0 |= PHY_PORT_CS1_LINK_DISABLE;
1883         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1884         if (ret)
1885                 return ret;
1886
1887         val1 |= PHY_PORT_CS1_LINK_DISABLE;
1888         ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1889         if (ret)
1890                 return ret;
1891
1892         /* Wait a bit and then re-enable both ports */
1893         usleep_range(10, 100);
1894
1895         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1896         if (ret)
1897                 return ret;
1898         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1899         if (ret)
1900                 return ret;
1901
1902         val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1903         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1904         if (ret)
1905                 return ret;
1906
1907         val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1908         return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1909 }
1910
1911 static int icm_firmware_init(struct tb *tb)
1912 {
1913         struct icm *icm = tb_priv(tb);
1914         struct tb_nhi *nhi = tb->nhi;
1915         int ret;
1916
1917         ret = icm_firmware_start(tb, nhi);
1918         if (ret) {
1919                 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1920                 return ret;
1921         }
1922
1923         if (icm->get_mode) {
1924                 ret = icm->get_mode(tb);
1925
1926                 switch (ret) {
1927                 case NHI_FW_SAFE_MODE:
1928                         icm->safe_mode = true;
1929                         break;
1930
1931                 case NHI_FW_CM_MODE:
1932                         /* Ask ICM to accept all Thunderbolt devices */
1933                         nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1934                         break;
1935
1936                 default:
1937                         if (ret < 0)
1938                                 return ret;
1939
1940                         tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1941                         return -ENODEV;
1942                 }
1943         }
1944
1945         /*
1946          * Reset both physical ports if there is anything connected to
1947          * them already.
1948          */
1949         ret = icm_reset_phy_port(tb, 0);
1950         if (ret)
1951                 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1952         ret = icm_reset_phy_port(tb, 1);
1953         if (ret)
1954                 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1955
1956         return 0;
1957 }
1958
1959 static int icm_driver_ready(struct tb *tb)
1960 {
1961         struct icm *icm = tb_priv(tb);
1962         int ret;
1963
1964         ret = icm_firmware_init(tb);
1965         if (ret)
1966                 return ret;
1967
1968         if (icm->safe_mode) {
1969                 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1970                 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1971                 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1972                 return 0;
1973         }
1974
1975         ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1976                                  &tb->nboot_acl, &icm->rpm);
1977         if (ret)
1978                 return ret;
1979
1980         /*
1981          * Make sure the number of supported preboot ACL matches what we
1982          * expect or disable the whole feature.
1983          */
1984         if (tb->nboot_acl > icm->max_boot_acl)
1985                 tb->nboot_acl = 0;
1986
1987         if (icm->proto_version >= 3)
1988                 tb_dbg(tb, "USB4 proxy operations supported\n");
1989
1990         return 0;
1991 }
1992
1993 static int icm_suspend(struct tb *tb)
1994 {
1995         struct icm *icm = tb_priv(tb);
1996
1997         if (icm->save_devices)
1998                 icm->save_devices(tb);
1999
2000         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2001         return 0;
2002 }
2003
2004 /*
2005  * Mark all switches (except root switch) below this one unplugged. ICM
2006  * firmware will send us an updated list of switches after we have send
2007  * it driver ready command. If a switch is not in that list it will be
2008  * removed when we perform rescan.
2009  */
2010 static void icm_unplug_children(struct tb_switch *sw)
2011 {
2012         struct tb_port *port;
2013
2014         if (tb_route(sw))
2015                 sw->is_unplugged = true;
2016
2017         tb_switch_for_each_port(sw, port) {
2018                 if (port->xdomain)
2019                         port->xdomain->is_unplugged = true;
2020                 else if (tb_port_has_remote(port))
2021                         icm_unplug_children(port->remote->sw);
2022         }
2023 }
2024
2025 static int complete_rpm(struct device *dev, void *data)
2026 {
2027         struct tb_switch *sw = tb_to_switch(dev);
2028
2029         if (sw)
2030                 complete(&sw->rpm_complete);
2031         return 0;
2032 }
2033
2034 static void remove_unplugged_switch(struct tb_switch *sw)
2035 {
2036         struct device *parent = get_device(sw->dev.parent);
2037
2038         pm_runtime_get_sync(parent);
2039
2040         /*
2041          * Signal this and switches below for rpm_complete because
2042          * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2043          * for it.
2044          */
2045         complete_rpm(&sw->dev, NULL);
2046         bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2047         tb_switch_remove(sw);
2048
2049         pm_runtime_mark_last_busy(parent);
2050         pm_runtime_put_autosuspend(parent);
2051
2052         put_device(parent);
2053 }
2054
2055 static void icm_free_unplugged_children(struct tb_switch *sw)
2056 {
2057         struct tb_port *port;
2058
2059         tb_switch_for_each_port(sw, port) {
2060                 if (port->xdomain && port->xdomain->is_unplugged) {
2061                         tb_xdomain_remove(port->xdomain);
2062                         port->xdomain = NULL;
2063                 } else if (tb_port_has_remote(port)) {
2064                         if (port->remote->sw->is_unplugged) {
2065                                 remove_unplugged_switch(port->remote->sw);
2066                                 port->remote = NULL;
2067                         } else {
2068                                 icm_free_unplugged_children(port->remote->sw);
2069                         }
2070                 }
2071         }
2072 }
2073
2074 static void icm_rescan_work(struct work_struct *work)
2075 {
2076         struct icm *icm = container_of(work, struct icm, rescan_work.work);
2077         struct tb *tb = icm_to_tb(icm);
2078
2079         mutex_lock(&tb->lock);
2080         if (tb->root_switch)
2081                 icm_free_unplugged_children(tb->root_switch);
2082         mutex_unlock(&tb->lock);
2083 }
2084
2085 static void icm_complete(struct tb *tb)
2086 {
2087         struct icm *icm = tb_priv(tb);
2088
2089         if (tb->nhi->going_away)
2090                 return;
2091
2092         /*
2093          * If RTD3 was vetoed before we entered system suspend allow it
2094          * again now before driver ready is sent. Firmware sends a new RTD3
2095          * veto if it is still the case after we have sent it driver ready
2096          * command.
2097          */
2098         icm_veto_end(tb);
2099         icm_unplug_children(tb->root_switch);
2100
2101         /*
2102          * Now all existing children should be resumed, start events
2103          * from ICM to get updated status.
2104          */
2105         __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2106
2107         /*
2108          * We do not get notifications of devices that have been
2109          * unplugged during suspend so schedule rescan to clean them up
2110          * if any.
2111          */
2112         queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2113 }
2114
2115 static int icm_runtime_suspend(struct tb *tb)
2116 {
2117         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2118         return 0;
2119 }
2120
2121 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2122 {
2123         if (tb_route(sw))
2124                 reinit_completion(&sw->rpm_complete);
2125         return 0;
2126 }
2127
2128 static int icm_runtime_resume_switch(struct tb_switch *sw)
2129 {
2130         if (tb_route(sw)) {
2131                 if (!wait_for_completion_timeout(&sw->rpm_complete,
2132                                                  msecs_to_jiffies(500))) {
2133                         dev_dbg(&sw->dev, "runtime resuming timed out\n");
2134                 }
2135         }
2136         return 0;
2137 }
2138
2139 static int icm_runtime_resume(struct tb *tb)
2140 {
2141         /*
2142          * We can reuse the same resume functionality than with system
2143          * suspend.
2144          */
2145         icm_complete(tb);
2146         return 0;
2147 }
2148
2149 static int icm_start(struct tb *tb)
2150 {
2151         struct icm *icm = tb_priv(tb);
2152         int ret;
2153
2154         if (icm->safe_mode)
2155                 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2156         else
2157                 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2158         if (IS_ERR(tb->root_switch))
2159                 return PTR_ERR(tb->root_switch);
2160
2161         tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2162         tb->root_switch->rpm = icm->rpm;
2163
2164         if (icm->set_uuid)
2165                 icm->set_uuid(tb);
2166
2167         ret = tb_switch_add(tb->root_switch);
2168         if (ret) {
2169                 tb_switch_put(tb->root_switch);
2170                 tb->root_switch = NULL;
2171         }
2172
2173         return ret;
2174 }
2175
2176 static void icm_stop(struct tb *tb)
2177 {
2178         struct icm *icm = tb_priv(tb);
2179
2180         cancel_delayed_work(&icm->rescan_work);
2181         tb_switch_remove(tb->root_switch);
2182         tb->root_switch = NULL;
2183         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2184         kfree(icm->last_nvm_auth);
2185         icm->last_nvm_auth = NULL;
2186 }
2187
2188 static int icm_disconnect_pcie_paths(struct tb *tb)
2189 {
2190         return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2191 }
2192
2193 static void icm_usb4_switch_nvm_auth_complete(void *data)
2194 {
2195         struct usb4_switch_nvm_auth *auth = data;
2196         struct icm *icm = auth->icm;
2197         struct tb *tb = icm_to_tb(icm);
2198
2199         tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2200                get_route(auth->reply.route_hi, auth->reply.route_lo),
2201                auth->reply.hdr.flags, auth->reply.status);
2202
2203         mutex_lock(&tb->lock);
2204         if (WARN_ON(icm->last_nvm_auth))
2205                 kfree(icm->last_nvm_auth);
2206         icm->last_nvm_auth = auth;
2207         mutex_unlock(&tb->lock);
2208 }
2209
2210 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2211 {
2212         struct usb4_switch_nvm_auth *auth;
2213         struct icm *icm = tb_priv(tb);
2214         struct tb_cfg_request *req;
2215         int ret;
2216
2217         auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2218         if (!auth)
2219                 return -ENOMEM;
2220
2221         auth->icm = icm;
2222         auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2223         auth->request.route_hi = upper_32_bits(route);
2224         auth->request.route_lo = lower_32_bits(route);
2225         auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2226
2227         req = tb_cfg_request_alloc();
2228         if (!req) {
2229                 ret = -ENOMEM;
2230                 goto err_free_auth;
2231         }
2232
2233         req->match = icm_match;
2234         req->copy = icm_copy;
2235         req->request = &auth->request;
2236         req->request_size = sizeof(auth->request);
2237         req->request_type = TB_CFG_PKG_ICM_CMD;
2238         req->response = &auth->reply;
2239         req->npackets = 1;
2240         req->response_size = sizeof(auth->reply);
2241         req->response_type = TB_CFG_PKG_ICM_RESP;
2242
2243         tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2244
2245         mutex_lock(&icm->request_lock);
2246         ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2247                              auth);
2248         mutex_unlock(&icm->request_lock);
2249
2250         tb_cfg_request_put(req);
2251         if (ret)
2252                 goto err_free_auth;
2253         return 0;
2254
2255 err_free_auth:
2256         kfree(auth);
2257         return ret;
2258 }
2259
2260 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2261                               u8 *status, const void *tx_data, size_t tx_data_len,
2262                               void *rx_data, size_t rx_data_len)
2263 {
2264         struct icm_usb4_switch_op_response reply;
2265         struct icm_usb4_switch_op request;
2266         struct tb *tb = sw->tb;
2267         struct icm *icm = tb_priv(tb);
2268         u64 route = tb_route(sw);
2269         int ret;
2270
2271         /*
2272          * USB4 router operation proxy is supported in firmware if the
2273          * protocol version is 3 or higher.
2274          */
2275         if (icm->proto_version < 3)
2276                 return -EOPNOTSUPP;
2277
2278         /*
2279          * NVM_AUTH is a special USB4 proxy operation that does not
2280          * return immediately so handle it separately.
2281          */
2282         if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2283                 return icm_usb4_switch_nvm_authenticate(tb, route);
2284
2285         memset(&request, 0, sizeof(request));
2286         request.hdr.code = ICM_USB4_SWITCH_OP;
2287         request.route_hi = upper_32_bits(route);
2288         request.route_lo = lower_32_bits(route);
2289         request.opcode = opcode;
2290         if (metadata)
2291                 request.metadata = *metadata;
2292
2293         if (tx_data_len) {
2294                 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2295                 if (tx_data_len < ARRAY_SIZE(request.data))
2296                         request.data_len_valid =
2297                                 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2298                 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2299         }
2300
2301         memset(&reply, 0, sizeof(reply));
2302         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2303                           1, ICM_TIMEOUT);
2304         if (ret)
2305                 return ret;
2306
2307         if (reply.hdr.flags & ICM_FLAGS_ERROR)
2308                 return -EIO;
2309
2310         if (status)
2311                 *status = reply.status;
2312
2313         if (metadata)
2314                 *metadata = reply.metadata;
2315
2316         if (rx_data_len)
2317                 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2318
2319         return 0;
2320 }
2321
2322 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2323                                                    u32 *status)
2324 {
2325         struct usb4_switch_nvm_auth *auth;
2326         struct tb *tb = sw->tb;
2327         struct icm *icm = tb_priv(tb);
2328         int ret = 0;
2329
2330         if (icm->proto_version < 3)
2331                 return -EOPNOTSUPP;
2332
2333         auth = icm->last_nvm_auth;
2334         icm->last_nvm_auth = NULL;
2335
2336         if (auth && auth->reply.route_hi == sw->config.route_hi &&
2337             auth->reply.route_lo == sw->config.route_lo) {
2338                 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2339                        tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2340                 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2341                         ret = -EIO;
2342                 else
2343                         *status = auth->reply.status;
2344         } else {
2345                 *status = 0;
2346         }
2347
2348         kfree(auth);
2349         return ret;
2350 }
2351
2352 /* Falcon Ridge */
2353 static const struct tb_cm_ops icm_fr_ops = {
2354         .driver_ready = icm_driver_ready,
2355         .start = icm_start,
2356         .stop = icm_stop,
2357         .suspend = icm_suspend,
2358         .complete = icm_complete,
2359         .handle_event = icm_handle_event,
2360         .approve_switch = icm_fr_approve_switch,
2361         .add_switch_key = icm_fr_add_switch_key,
2362         .challenge_switch_key = icm_fr_challenge_switch_key,
2363         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2364         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2365         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2366 };
2367
2368 /* Alpine Ridge */
2369 static const struct tb_cm_ops icm_ar_ops = {
2370         .driver_ready = icm_driver_ready,
2371         .start = icm_start,
2372         .stop = icm_stop,
2373         .suspend = icm_suspend,
2374         .complete = icm_complete,
2375         .runtime_suspend = icm_runtime_suspend,
2376         .runtime_resume = icm_runtime_resume,
2377         .runtime_suspend_switch = icm_runtime_suspend_switch,
2378         .runtime_resume_switch = icm_runtime_resume_switch,
2379         .handle_event = icm_handle_event,
2380         .get_boot_acl = icm_ar_get_boot_acl,
2381         .set_boot_acl = icm_ar_set_boot_acl,
2382         .approve_switch = icm_fr_approve_switch,
2383         .add_switch_key = icm_fr_add_switch_key,
2384         .challenge_switch_key = icm_fr_challenge_switch_key,
2385         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2386         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2387         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2388 };
2389
2390 /* Titan Ridge */
2391 static const struct tb_cm_ops icm_tr_ops = {
2392         .driver_ready = icm_driver_ready,
2393         .start = icm_start,
2394         .stop = icm_stop,
2395         .suspend = icm_suspend,
2396         .complete = icm_complete,
2397         .runtime_suspend = icm_runtime_suspend,
2398         .runtime_resume = icm_runtime_resume,
2399         .runtime_suspend_switch = icm_runtime_suspend_switch,
2400         .runtime_resume_switch = icm_runtime_resume_switch,
2401         .handle_event = icm_handle_event,
2402         .get_boot_acl = icm_ar_get_boot_acl,
2403         .set_boot_acl = icm_ar_set_boot_acl,
2404         .approve_switch = icm_tr_approve_switch,
2405         .add_switch_key = icm_tr_add_switch_key,
2406         .challenge_switch_key = icm_tr_challenge_switch_key,
2407         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2408         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2409         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2410         .usb4_switch_op = icm_usb4_switch_op,
2411         .usb4_switch_nvm_authenticate_status =
2412                 icm_usb4_switch_nvm_authenticate_status,
2413 };
2414
2415 /* Ice Lake */
2416 static const struct tb_cm_ops icm_icl_ops = {
2417         .driver_ready = icm_driver_ready,
2418         .start = icm_start,
2419         .stop = icm_stop,
2420         .complete = icm_complete,
2421         .runtime_suspend = icm_runtime_suspend,
2422         .runtime_resume = icm_runtime_resume,
2423         .handle_event = icm_handle_event,
2424         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2425         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2426         .usb4_switch_op = icm_usb4_switch_op,
2427         .usb4_switch_nvm_authenticate_status =
2428                 icm_usb4_switch_nvm_authenticate_status,
2429 };
2430
2431 struct tb *icm_probe(struct tb_nhi *nhi)
2432 {
2433         struct icm *icm;
2434         struct tb *tb;
2435
2436         tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2437         if (!tb)
2438                 return NULL;
2439
2440         icm = tb_priv(tb);
2441         INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2442         mutex_init(&icm->request_lock);
2443
2444         switch (nhi->pdev->device) {
2445         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2446         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2447                 icm->can_upgrade_nvm = true;
2448                 icm->is_supported = icm_fr_is_supported;
2449                 icm->get_route = icm_fr_get_route;
2450                 icm->save_devices = icm_fr_save_devices;
2451                 icm->driver_ready = icm_fr_driver_ready;
2452                 icm->device_connected = icm_fr_device_connected;
2453                 icm->device_disconnected = icm_fr_device_disconnected;
2454                 icm->xdomain_connected = icm_fr_xdomain_connected;
2455                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2456                 tb->cm_ops = &icm_fr_ops;
2457                 break;
2458
2459         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2460         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2461         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2462         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2463         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2464                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2465                 /*
2466                  * NVM upgrade has not been tested on Apple systems and
2467                  * they don't provide images publicly either. To be on
2468                  * the safe side prevent root switch NVM upgrade on Macs
2469                  * for now.
2470                  */
2471                 icm->can_upgrade_nvm = !x86_apple_machine;
2472                 icm->is_supported = icm_ar_is_supported;
2473                 icm->cio_reset = icm_ar_cio_reset;
2474                 icm->get_mode = icm_ar_get_mode;
2475                 icm->get_route = icm_ar_get_route;
2476                 icm->save_devices = icm_fr_save_devices;
2477                 icm->driver_ready = icm_ar_driver_ready;
2478                 icm->device_connected = icm_fr_device_connected;
2479                 icm->device_disconnected = icm_fr_device_disconnected;
2480                 icm->xdomain_connected = icm_fr_xdomain_connected;
2481                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2482                 tb->cm_ops = &icm_ar_ops;
2483                 break;
2484
2485         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2486         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2487                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2488                 icm->can_upgrade_nvm = !x86_apple_machine;
2489                 icm->is_supported = icm_ar_is_supported;
2490                 icm->cio_reset = icm_tr_cio_reset;
2491                 icm->get_mode = icm_ar_get_mode;
2492                 icm->driver_ready = icm_tr_driver_ready;
2493                 icm->device_connected = icm_tr_device_connected;
2494                 icm->device_disconnected = icm_tr_device_disconnected;
2495                 icm->xdomain_connected = icm_tr_xdomain_connected;
2496                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2497                 tb->cm_ops = &icm_tr_ops;
2498                 break;
2499
2500         case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2501         case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2502                 icm->is_supported = icm_fr_is_supported;
2503                 icm->driver_ready = icm_icl_driver_ready;
2504                 icm->set_uuid = icm_icl_set_uuid;
2505                 icm->device_connected = icm_icl_device_connected;
2506                 icm->device_disconnected = icm_tr_device_disconnected;
2507                 icm->xdomain_connected = icm_tr_xdomain_connected;
2508                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2509                 icm->rtd3_veto = icm_icl_rtd3_veto;
2510                 tb->cm_ops = &icm_icl_ops;
2511                 break;
2512
2513         case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2514         case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2515         case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2516         case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2517         case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2518         case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2519         case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2520         case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2521         case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2522         case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2523         case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2524                 icm->is_supported = icm_tgl_is_supported;
2525                 icm->driver_ready = icm_icl_driver_ready;
2526                 icm->set_uuid = icm_icl_set_uuid;
2527                 icm->device_connected = icm_icl_device_connected;
2528                 icm->device_disconnected = icm_tr_device_disconnected;
2529                 icm->xdomain_connected = icm_tr_xdomain_connected;
2530                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2531                 icm->rtd3_veto = icm_icl_rtd3_veto;
2532                 tb->cm_ops = &icm_icl_ops;
2533                 break;
2534
2535         case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2536         case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2537                 icm->is_supported = icm_tgl_is_supported;
2538                 icm->get_mode = icm_ar_get_mode;
2539                 icm->driver_ready = icm_tr_driver_ready;
2540                 icm->device_connected = icm_tr_device_connected;
2541                 icm->device_disconnected = icm_tr_device_disconnected;
2542                 icm->xdomain_connected = icm_tr_xdomain_connected;
2543                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2544                 tb->cm_ops = &icm_tr_ops;
2545                 break;
2546         }
2547
2548         if (!icm->is_supported || !icm->is_supported(tb)) {
2549                 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2550                 tb_domain_put(tb);
2551                 return NULL;
2552         }
2553
2554         tb_dbg(tb, "using firmware connection manager\n");
2555
2556         return tb;
2557 }